def crawl_shebao_info(process_count=1):
spider_dict = {
"ShebaoChengdu": ShebaoChengduSpider,
"ShebaoGuangzhou": ShebaoGuangzhouSpider,
}
run_multiple_spider_with_process(spider_dict, process_count)
def crawl_wenshu_info(process_count=1):
spider_dict = {
"WenshuAppSpider": WenshuAppSpider,
"WenshuPcSpider": WenshuPcSpider,
}
run_multiple_spider_with_process(spider_dict, process_count)
def crawl_zhengxin_info(process_count=1):
spider_dict = {
"人行征信": ZhengxinPbcSpider,
# "银行征信": ZhengxinBankSpider,
}
run_multiple_spider_with_process(spider_dict, process_count)
def crawl_ecommerce_info(process_count=1):
spider_dict = {
"京东": JingDongSpider,
"淘宝": TaobaoSpider,
"支付宝": AlipaySpider,
# "一号店": YhdSpider,
}
run_multiple_spider_with_process(spider_dict, process_count)
def crawl_emailbill_info(process_count=1):
spider_dict = {
"163": Email163Spider,
"qq": EmailqqSpider,
"sina": EmailSinaSpider,
'sohu': EmailSohuSpider,
'IMAP': EmailImapSpider,
}
run_multiple_spider_with_process(spider_dict, process_count)
def crawl_user_communication_info(process_count=1):
# p = Process(target=run_scrapy_spider, args=(YournumberSpider,))
# p.start()
spider_dict = {
"中国联通": ChinaUnicomSpider,
"中国移动": ChinaMobileSpider,
#"中国电信": ChinaTelecomAppSpider,
}
run_multiple_spider_with_process(spider_dict, process_count)
def crawl_wenshu_info():
"""
201609到201301
:return:
"""
date_list = [
"201609", "201608", "201607", "201606", "201605", "201604", "201603",
"201602", "201601", "201512", "201511", "201510", "201509", "201508",
"201507", "201506", "201505", "201504", "201503", "201502", "201501",
"201412", "201411", "201410", "201409", "201408", "201407", "201406",
"201405", "201404", "201403", "201402", "201401", "201312", "201311",
"201310", "201309", "201308", "201307", "201306", "201305", "201304",
"201303", "201302", "201301"
]
spider_dict = dict()
for date in date_list:
spider_dict["ConditionSpider_%s" % date] = eval("ConditionSpider_%s" %
date)
run_multiple_spider_with_process(spider_dict, process_count=1)
def crawl_bank_info(process_count=1):
if 'Windows' == get_os():
spider_dict = {
"农业银行": ABCSpider,
"建设银行": CCBSpider,
"邮政银行": PsbcSpider,
"工商银行": IcbcSpider,
"中国银行": BocSpider,
"招商银行": CmbSpider,
"民生银行": CMBCSpider,
"光大银行": CebSpider,
"中信银行": CncbSpider,
"浦发银行": SpdbSpider,
"华夏银行": HxbSpider,
"平安银行": PinganSpider,
}
else:
spider_dict = {
"交通银行": BocomWapSpider,
"兴业银行": CibSpider,
"广发银行": CgbWapSpider,
}
run_multiple_spider_with_process(spider_dict, process_count)
def crawl_other_info(process_count=1):
spider_dict = {
"Gpsspg": GpsspgSpider,
}
run_multiple_spider_with_process(spider_dict, process_count)
def crawl_xuexin_info(process_count=1):
spider_dict = {"学信": XuexinSpider,
}
run_multiple_spider_with_process(spider_dict, process_count)
