push_data_into_redis(rds, Config, data_mp)
else:
print('no exist asin')
print('push repeat done')
store.close()
else:
print('no repeat asin')
def push_data_into_redis(rds, conf, data_mp):
data_json = json.dumps(data_mp)
rds.rds.lpush(conf.REDIS_DATA_LIST, data_json)
if __name__ == '__main__':
rds = AmazonRedis()
detail_today = rds.get_hash_field('amazon:di:cy:dc01:markdate', 'today')
if detail_today:
detail_today = detail_today.split()[0]
today = time.strftime("%Y-%m-%d")
if detail_today == today:
print('start handling repeat asin')
select_asin(rds)
else:
print('wait for detail finish')
rds.add_set(Config.REDIS_CATE_ASIN, cate_asin)
continue
rds.add_set(Config.REDIS_CATE_ASIN, cate_asin)
print(row['scgs_id'])
page_url = 'https://www.amazon.{}/dp/{}'.format(suffix, asin)
mp = {'entry': task_type, 'page_url': page_url, 'category_info': task_category, 'category_entry': category_entry,
'category_url': category_url, 'rank': 101, 'create_time': create_time}
rds.rds.rpush('amazon:di:cy:detail', mp)
store.close()
if __name__ == '__main__':
rds = AmazonRedis()
today = time.strftime("%Y-%m-%d")
asin_today = rds.get_hash_field('amazon:di:cy:asin:markdate', 'today')
if asin_today:
asin_today = asin_today.split()[0]
if asin_today == today:
print('toady finish')
else:
list_today = rds.get_hash_field('amazon:di:cy:lc:markdate', 'today')
if list_today:
list_today = list_today.split()[0]
if list_today == today:
rds.set_hash('amazon:di:cy:asin:markdate', {'today': time.strftime("%Y-%m-%d %H:%M:%S")})
print('scan_database')
select_asin(rds)
rds.delete_key(Config.REDIS_CATE_ASIN)
rds.delete_key(Config.REDIS_UNIQUE_ASIN)
rds.delete_key('amazon:di:cy:taskcate')