port=ALIYUN_PORT,
username=ALIYUN_USERNAME,
password=ALIYUN_PASSWORD,
db=ALIYUN_DB)
# 取出HF_前缀的集合名称
db = mongo_conn[mongodb_db]
collection_name_list = COLLECTION_NAME_LIST_XF
# 循环构造spider
for i in range(len(collection_name_list)):
Logger.info(collection_name_list[i])
spider = SpiderCommand()
spider.log_collection_name = collection_name_list[i] + '_comment_log'
spider.python_name = 'tmall_xf_comments.py'
spider.product_index = 0
spider.page_nummber = 1
spider.catalog_collection_name = collection_name_list[i]
spider.target_collexction_name = collection_name_list[i] + '_comments'
spider.maxPage_num_control = 100
collection_name_list[i] = spider
mongo_conn.close()
proc_record = []
for i in range(len(collection_name_list)):
thread = Thread(target=monitoringOneSpider,
args=(
collection_name_list[i],
db,
))
port=ALIYUN_PORT,
username=ALIYUN_USERNAME,
password=ALIYUN_PASSWORD,
db=ALIYUN_DB)
db = mongo_conn[mongodb_db]
# 取出HF_前缀的集合名称
collection_name_list = COLLECTION_NAME_LIST_MY
for i in range(len(collection_name_list)):
Logger.info(collection_name_list[i])
spider = SpiderCommand()
spider.log_collection_name = collection_name_list[i] + '_comment_log'
spider.python_name = 'jd_hf_comment.py'
spider.product_index = 0
spider.page_nummber = 0
spider.catalog_collection_name = collection_name_list[i]
spider.target_collexction_name = collection_name_list[i] + '_comments'
spider.maxPage_num_control = 100
collection_name_list[i] = spider
mongo_conn.close()
proc_record = []
for i in range(len(collection_name_list)):
p = Thread(target=monitoringOneSpider,
args=(
collection_name_list[i],
db,
))
port=ALIYUN_PORT,
username=ALIYUN_USERNAME,
password=ALIYUN_PASSWORD,
db=ALIYUN_DB)
# 取出HF_前缀的集合名称
db = mongo_conn[mongodb_db]
collection_name_list = COLLECTION_NAME_LIST_JYDQ
# 循环构造spider
for i in range(len(collection_name_list)):
Logger.info(collection_name_list[i])
spider = SpiderCommand()
spider.log_collection_name = collection_name_list[i] + '_comment_log'
spider.python_name = 'tmall_jydq_comments.py'
spider.product_index = 0
spider.page_nummber = 1
spider.catalog_collection_name = collection_name_list[i]
spider.target_collexction_name = collection_name_list[i] + '_comments'
spider.maxPage_num_control = 100
collection_name_list[i] = spider
mongo_conn.close()
proc_record = []
for i in range(len(collection_name_list)):
thread = Thread(target=monitoringOneSpider,
args=(
collection_name_list[i],
db,
))
port=ALIYUN_PORT,
username=ALIYUN_USERNAME,
password=ALIYUN_PASSWORD,
db=ALIYUN_DB)
db = mongo_conn[mongodb_db]
# 取出HF_前缀的集合名称
collection_name_list = COLLECTION_NAME_LIST_JYDQ
for i in range(len(collection_name_list)):
Logger.info(collection_name_list[i])
spider = SpiderCommand()
spider.log_collection_name = collection_name_list[i] + '_comment_log'
spider.python_name = 'jd_jydq_comment.py'
spider.product_index = 0
spider.page_nummber = 0
spider.catalog_collection_name = collection_name_list[i]
spider.target_collexction_name = collection_name_list[i] + '_comments'
spider.maxPage_num_control = 100
collection_name_list[i] = spider
mongo_conn.close()
proc_record = []
for i in range(len(collection_name_list)):
p = Thread(target=monitoringOneSpider,
args=(
collection_name_list[i],
db,
))
port=ALIYUN_PORT,
username=ALIYUN_USERNAME,
password=ALIYUN_PASSWORD,
db=ALIYUN_DB)
db = mongo_conn[mongodb_db]
# 取出HF_前缀的集合名称
collection_name_list = COLLECTION_NAME_LIST_MZ
for i in range(len(collection_name_list)):
Logger.info(collection_name_list[i])
spider = SpiderCommand()
spider.log_collection_name = collection_name_list[i] + '_comment_log'
spider.python_name = 'jd_mz_comment.py'
spider.product_index = 0
spider.page_nummber = 0
spider.catalog_collection_name = collection_name_list[i]
spider.target_collexction_name = collection_name_list[i] + '_comments'
spider.maxPage_num_control = 100
collection_name_list[i] = spider
mongo_conn.close()
proc_record = []
for i in range(len(collection_name_list)):
p = Thread(target=monitoringOneSpider,
args=(
collection_name_list[i],
db,
))
port=ALIYUN_PORT,
username=ALIYUN_USERNAME,
password=ALIYUN_PASSWORD,
db=ALIYUN_DB)
# 取出HF_前缀的集合名称
db = mongo_conn[mongodb_db]
collection_name_list = COLLECTION_NAME_LIST_KQHL
# 循环构造spider
for i in range(len(collection_name_list)):
Logger.info(collection_name_list[i])
spider = SpiderCommand()
spider.log_collection_name = collection_name_list[i] + '_comment_log'
spider.python_name = 'tmall_kqhl_comments.py'
spider.product_index = 0
spider.page_nummber = 1
spider.catalog_collection_name = collection_name_list[i]
spider.target_collexction_name = collection_name_list[i] + '_comments'
spider.maxPage_num_control = 100
collection_name_list[i] = spider
mongo_conn.close()
proc_record = []
for i in range(len(collection_name_list)):
thread = Thread(target=monitoringOneSpider,
args=(
collection_name_list[i],
db,
))
port=ALIYUN_PORT,
username=ALIYUN_USERNAME,
password=ALIYUN_PASSWORD,
db=ALIYUN_DB)
db = mongo_conn[mongodb_db]
# 取出HF_前缀的集合名称
collection_name_list = COLLECTION_NAME_LIST_XY
cat_list = CAT_LIST_XY
for i in range(len(collection_name_list)):
Logger.info(collection_name_list[i])
spider = SpiderCommand()
spider.log_collection_name = collection_name_list[i] + '_comment_log'
spider.python_name = 'jd_xy_comment.py'
spider.product_index = 0
spider.page_nummber = 0
spider.catalog_collection_name = cat_list[i]
spider.target_collexction_name = collection_name_list[i] + '_comments'
spider.maxPage_num_control = 100
collection_name_list[i] = spider
mongo_conn.close()
proc_record = []
for i in range(len(collection_name_list)):
p = Thread(target=monitoringOneSpider,
args=(
collection_name_list[i],
db,
))