def cpca_addr(res): ''' :param res: 列表 :return: ''' if os.path.exists('./addr_2.txt'): os.remove('./addr_2.txt') with open('./addr_2.txt', 'a', encoding='utf-8') as f: for re in res: print(re[0]) l = [re[0].replace(' ', '')] if re[0]: df = transform(l, myumap, cut=False) for addr in df.index: addr_3 = df.loc[addr].values[0] addr_2 = df.loc[addr].values[1] addr_1 = df.loc[addr].values[2] if addr_1: if (addr_3 == '雄县' or addr_3 == '容城县' or addr_3 == '安新县', addr_3 == '容城' or addr_3 == '安新') and addr_1 == '河北省': # 测试 70 update_sql = "update tyc_qybj_jbxx set province ='{sheng}',city='{shi}',county='{xian}' where company_name='{comp_name}'".format( sheng=addr_1, shi=addr_2, xian=addr_3, comp_name=re[1]) # 本地 # update_sql = "update company_11315 set address_1 ='{sheng}',address_2='{shi}',address_3='{xian}' where company_name='{comp_name}'".format(sheng=addr_1, shi=addr_2, xian=addr_3, comp_name=re[1]) single_oracle.execute(update_sql) # print(update_sql) f.write(re[0] + ' 省:' + addr_1 + ',市:' + addr_2 + ',县:' + addr_3 + '\n')
def put_11315_increased(self): ''' 11315新增企业名单 :return: ''' lock_uid = '11315_increased_s' status, sec = self.get_lock(lock_uid) if status: incre_length = self.server.llen('11315_increased') if incre_length > 50000: logger.debug('parses incre_length={}'.format(incre_length)) self.del_lock(lock_uid, sec) return try: incre_sql = 'SELECT max(company_number) FROM company_11315 where {}' # select max(company_number) from company_11315 where company_name != 'NA' tablename_condition = self.get_configs_by_tablename( 'put_11315_increased') print(incre_sql.format(tablename_condition[0])) results = single_oracle.oracle_find_by_param_all( incre_sql.format(tablename_condition[0])) print(results) if results[0][0]: if tablename_condition[1] == 0: sql = 'update config set over = 2 where id = {}'.format( tablename_condition[2]) single_oracle.oracle_update(sql) max_num = single_oracle.oracle_find_by_param_all( 'select max(company_number) from company_11315') if max_num[0][0] > results[0][0]: single_oracle.execute( 'delete from company_11315 where company_number > {}' .format(results[0][0])) # XXX: 需要判段什么时候抓取完毕 for i in range(results[0][0] + 1, results[0][0] + 2000000): company_number = i # print(company_number) self.server.lpush('11315_increased', company_number) else: single_oracle.oracle_update( 'update config set over = 1, over_time = SYSDATE where id = {}' .format(tablename_condition[2])) self.del_lock(lock_uid, sec) except Exception as e: self.del_lock(lock_uid, sec) logger.exception(e)
def business_term(res): for re in res: # print(re[0]) if re[0] and re[0] != '-': term_l = re[0].split('至') # 营业期限自 begin = term_l[0] # 营业期限至 end = term_l[1] if end == '无固定期限': end = '2999-12-31' # print(begin, end) elif re[0] == '-': begin = '-' end = '-' # print(begin,end) sql = "update tyc_qybj_jbxx set business_term_begin ='{begin}', business_term_end = '{end}' where company_name='{name}'".format( begin=begin, end=end, name=re[1]) print(sql) single_oracle.execute(sql)
def main(args): print(u'启动', args) # import sys # # reload(sys) # check_first_start=True flag = single_reids.server.get('11315_increased_flag') if not flag or int(flag.decode()) == 0: single_reids.put_flag() flag = single_reids.server.get('11315_increased_flag') while True: NA_count = single_reids.server.get('11315_NA_count') if NA_count and int(NA_count.decode()) >= int(flag.decode()): time.sleep(60 * 30) continue # i = single_reids.server.rpop('11315_omit') i = single_reids.server.rpop('11315_increased') #i=12872429 print('-' * 20, i) if not i: # time.sleep(60*10) # single_reids.put_11315_omit() single_reids.put_11315_increased() continue Retry = 1 i = i.decode() res = single_oracle.oracle_find_by_param_all( 'select count(*) from company_11315 where company_number = {}'. format(i)) if res[0][0] != 0: print('该id{}已存在'.format(i)) single_oracle.execute( 'delete from company_11315 where company_number = {}'.format( i)) url = "http://" + str(i) + ".11315.com/" print(url) # 一个公司连接最多访问10次 while Retry <= 10: logger.info(url + "------%d", Retry) try: con = web_spider(url) # 对访问状态及页面内容进行初步判断 if con.status_code == 200: html = con.text if "系统检测到您的请求存在异常" not in html and "你可能访问的太快了" not in html and "正在 请求 请 稍候" not in html: try: web_parse(html, i, url) except Exception: logger.info("此number号已存在: %s", i) break else: logger.info("IP 被封或者访问过快") Retry += 1 continue else: logger.info("访问状态码异常%s", str(con.status_code)) Retry += 1 time.sleep(3) continue except Exception as e: logger.info("访问超时或者连接出错") Retry += 1 logger.exception(e) continue except AttributeError as w: logger.exception(w) logger.info("无法获取二次连接url") Retry += 1 continue except Exception as w: logger.exception(w) logger.info("未知访问错误") logger.exception("Exception Logged") Retry += 1 continue