def main(): file_path = os.path.join(DATA_PATH, 'products_tmall.csv') write_head(file_path) db = MysqlHelper() shops = db.get_tmall_shops() for shop in shops: products = db.get_products_by_shop(shop['id']) for product in products: line = format_line() line['1-sn'] = product['sn'] line['2-title'] = product['title'] line['3-price'] = product['price'] line['4-prices'] = list_to_string(format_data(product['prices'])) line['5-colors'] = list_to_string(format_data(product['colors'])) line['6-sizes'] = list_to_string(format_data(product['sizes'])) line['7-url'] = product['url'] line['8-shop_name'] = product['name'] line['9-shop_url'] = product['task_url'] properties = format_data(product['properties']) properties = json.loads(properties) for k, v in properties.items(): line[k] = v.replace("\"", "") print(line) new_line = line.values() print(new_line) write_list_to_csv([new_line], file_path)
def main(): db = MysqlHelper() tasks = db.get_tmall_shops() for task in tasks: crawl_tmall_list(task) driver.close()
def main(): db = MysqlHelper() shops = db.get_tmall_shops() for shop in shops: image_path = mk_image_dir(shop['id']) if shop['id'] == 26: continue products = db.get_products_by_shop(shop['id']) for product in products: images = format_data(product['images']) save_images(product['sn'], images, image_path)