def get_thread_writer(self):
thread_writer = ThreadMongoWriter(redis_key=self.items_redis_key, stop_epoch=12*30, buffer_size=2048,
out_mongo_url="mongodb://192.168.0.13:27017",
db_collection=("jingdong", self.new_retry_collect), bar_name=self.items_redis_key, distinct_field="skuid")
# thread_writer = ThreadFileWriter(redis_key=self.items_redis_key, stop_epoch=12*30, bar_name=self.items_redis_key,
# out_file="jingdong/result/jdskuid.txt",
# table_header=["_seed","_status","phonenumber", "province", "city", "company"])
thread_writer.setDaemon(True)
return thread_writer
def get_thread_writer(self):
thread_writer = ThreadMongoWriter(redis_key=self.items_redis_key, stop_epoch=12*3000,buffer_size=2048,
out_mongo_url="mongodb://192.168.0.13:27017",
db_collection=("jingdong","jdskuid{0}retry0".format(current_date)), bar_name=self.items_redis_key, distinct_field="skuid")
# thread_writer = ThreadFileWriter(redis_key=self.items_redis_key, stop_epoch=12*3000, bar_name=self.items_redis_key,
# out_file="jingdong/result/jdskuid{0}".format(current_date),
# table_header=["_seed","_status","skuid", "cate_id", "brand_id", "shopid","venderid","shop_name","ziying"])
thread_writer.setDaemon(True)
return thread_writer
def get_thread_writer(self):
thread_writer = ThreadMongoWriter(
redis_key=self.items_redis_key,
stop_epoch=12 * 3000,
buffer_size=2048,
out_mongo_url="mongodb://192.168.0.13:27017",
db_collection=("jingdong", self.out_table),
bar_name=self.items_redis_key)
thread_writer.setDaemon(True)
return thread_writer
def get_thread_writer(self):
thread_writer = ThreadMongoWriter(
redis_key=self.items_redis_key,
stop_epoch=12 * 3000,
buffer_size=2048,
out_mongo_url="mongodb://192.168.0.13:27017",
db_collection=("jingdong",
"jdsearch{0}retry0".format(current_date)),
bar_name=self.items_redis_key,
distinct_field="skuid")
thread_writer.setDaemon(False)
return thread_writer
def process_items(self, tablename):
thread_writer = ThreadMongoWriter(redis_key=self.items_redis_key, stop_epoch=12 * 3000, buffer_size=2048,
out_mongo_url="mongodb://192.168.0.13:27017",
db_collection=("jingdong", tablename),
bar_name=self.items_redis_key, distinct_field="skuid")
thread_writer.setDaemon(False)
thread_writer.start()