def start(): """ 入口函数 """ keys = globals().get('keywords', list()) for key in keys: raw_companies = tyc_client.search(key) cost_time = 2 * raw_companies.__len__() log.info('正在处理爬取[%s],大概需要%s秒' % (key, cost_time)) # company对象 company = Company() for raw_company in raw_companies: company.keyword = key manager.assembly(company, raw_company) # company detail raw_company_detail = tyc_client.search_detail(raw_company.get('id')) manager.assembly_detail(company, raw_company_detail) log.info(company) # save(company.__dict__) company.clear() log.info("completed")
def start(): keywords = globals().get('keywords') for keyword in keywords: raw_companies = qcc_client.search(keyword) log.info('正在处理爬取[%s]' % keyword) # company对象 company = Company() for raw_company in raw_companies: company.keyword = keyword # 组装公司信息 manager.assembly(company, raw_company) raw_company_detail = qcc_client.search_detail(raw_company.get('KeyNo')) # 补充公司详细信息 manager.assembly_detail(company, raw_company_detail) # 保存到数据库 # save(company.__dict__) log.info(company) company.clear() log.info('completed')
def start(): keywords = globals().get('keywords') if keywords: for keyword in keywords: raw_companies = qcc_client.search(keyword) cost_time = 2 * raw_companies.__len__() + 4 log.info('正在处理爬取[%s],大概需要%s秒' % (keyword, cost_time)) # company对象 company = Company() for raw_company in raw_companies: company.keyword = keyword # 组装公司信息 manager.assembly(company, raw_company) raw_company_detail = qcc_client.search_detail( raw_company.get('KeyNo')) # 补充公司详细信息 manager.assembly_detail(company, raw_company_detail) # 保存到数据库 save(company.__dict__) # 重置当前对象 company.clear() log.info('completed')