def start():
""" 入口函数 """
keys = globals().get('keywords', list())
for key in keys:
raw_companies = tyc_client.search(key)
cost_time = 2 * raw_companies.__len__()
log.info('正在处理爬取[%s]，大概需要%s秒' % (key, cost_time))
# company对象
company = Company()
for raw_company in raw_companies:
company.keyword = key
manager.assembly(company, raw_company)
# company detail
raw_company_detail = tyc_client.search_detail(raw_company.get('id'))
manager.assembly_detail(company, raw_company_detail)
log.info(company)
# save(company.__dict__)
company.clear()
log.info("completed")
def start():
keywords = globals().get('keywords')
for keyword in keywords:
raw_companies = qcc_client.search(keyword)
log.info('正在处理爬取[%s]' % keyword)
# company对象
company = Company()
for raw_company in raw_companies:
company.keyword = keyword
# 组装公司信息
manager.assembly(company, raw_company)
raw_company_detail = qcc_client.search_detail(raw_company.get('KeyNo'))
# 补充公司详细信息
manager.assembly_detail(company, raw_company_detail)
# 保存到数据库
# save(company.__dict__)
log.info(company)
company.clear()
log.info('completed')
def start():
keywords = globals().get('keywords')
if keywords:
for keyword in keywords:
raw_companies = qcc_client.search(keyword)
cost_time = 2 * raw_companies.__len__() + 4
log.info('正在处理爬取[%s]，大概需要%s秒' % (keyword, cost_time))
# company对象
company = Company()
for raw_company in raw_companies:
company.keyword = keyword
# 组装公司信息
manager.assembly(company, raw_company)
raw_company_detail = qcc_client.search_detail(
raw_company.get('KeyNo'))
# 补充公司详细信息
manager.assembly_detail(company, raw_company_detail)
# 保存到数据库
save(company.__dict__)
# 重置当前对象
company.clear()
log.info('completed')