push_data_into_redis(rds, Config, data_mp) else: print('no exist asin') print('push repeat done') store.close() else: print('no repeat asin') def push_data_into_redis(rds, conf, data_mp): data_json = json.dumps(data_mp) rds.rds.lpush(conf.REDIS_DATA_LIST, data_json) if __name__ == '__main__': rds = AmazonRedis() detail_today = rds.get_hash_field('amazon:di:cy:dc01:markdate', 'today') if detail_today: detail_today = detail_today.split()[0] today = time.strftime("%Y-%m-%d") if detail_today == today: print('start handling repeat asin') select_asin(rds) else: print('wait for detail finish')
rds.add_set(Config.REDIS_CATE_ASIN, cate_asin) continue rds.add_set(Config.REDIS_CATE_ASIN, cate_asin) print(row['scgs_id']) page_url = 'https://www.amazon.{}/dp/{}'.format(suffix, asin) mp = {'entry': task_type, 'page_url': page_url, 'category_info': task_category, 'category_entry': category_entry, 'category_url': category_url, 'rank': 101, 'create_time': create_time} rds.rds.rpush('amazon:di:cy:detail', mp) store.close() if __name__ == '__main__': rds = AmazonRedis() today = time.strftime("%Y-%m-%d") asin_today = rds.get_hash_field('amazon:di:cy:asin:markdate', 'today') if asin_today: asin_today = asin_today.split()[0] if asin_today == today: print('toady finish') else: list_today = rds.get_hash_field('amazon:di:cy:lc:markdate', 'today') if list_today: list_today = list_today.split()[0] if list_today == today: rds.set_hash('amazon:di:cy:asin:markdate', {'today': time.strftime("%Y-%m-%d %H:%M:%S")}) print('scan_database') select_asin(rds) rds.delete_key(Config.REDIS_CATE_ASIN) rds.delete_key(Config.REDIS_UNIQUE_ASIN) rds.delete_key('amazon:di:cy:taskcate')