def savedata(data,table): """保存数据到mysql数据库 """ db = mysqlwrap.get_db() if type(data) != list: data = [data] for item in data: fields = [] values = [] for k,v in item.items(): fields.append(k) v=mysqlwrap.addslashes(v) values.append('"%s"' % v) sql = "insert into data_%s(%s)values(%s)" % (table,','.join(fields),','.join(values)) res,desc = db.query(sql,1) #print(sql,res,desc) #数据入库失败处理 if res == -1: #print('insert_error:',desc) if desc.args[0] in [1064,1062]: return (0,()) item['table'] = 'data_%s'%table datamodel.get_tmp_queue().push(item) #插入dat_total总表 #print(item) if 'reg_status' in item and item['reg_status'] in ['存续','在业'] : fields.append('prov_tb') values.append('"%s"' % table) t_table = 'data_total' sql = "insert into %s(%s)values(%s)" % (t_table,','.join(fields),','.join(values)) rs,ds = db.query(sql,1) #print("insert total tb",rs,ds) return (res,desc)
def savedata(data,table): """保存数据到mysql数据库 """ db = mysqlwrap.get_db() fields = [] values = [] for k,v in data.items(): fields.append(k) #v = db.escape_string(v) v=v.replace('"'," ") values.append('"%s"' % v) sql = "insert into data_%s(%s)values(%s)" % (table,','.join(fields),','.join(values)) res,desc = db.query(sql,1) #数据入库失败处理 if res == -1: if desc.args[0] == 1064: print(desc) return data['table'] = 'data_%s'%table datamodel.get_tmp_queue().push(data)
def run(self): db = mysqlwrap.get_db() queue = datamodel.get_tmp_queue() while 1: try: row = queue.pop(3) if row: tb = row.pop('table')[-2:] res,desc = savedata(row,tb) if desc: print(tb,res,desc) except: traceback.print_exc()
def exec_main(mode): """业务线程启动函数 参数mod为业务模块 模块工厂: mod. """ x=0 i=0 queue = datamodel.get_row_queue(mode.biz_flag) while 1: if datamodel.g_exit: break corp = queue.pop(3) if not corp: time.sleep(5) continue try: if exist_corp(corp,mode.biz_flag): print(corp) continue corp,info,status,proxy =mode.get_info(corp) x+=1 if info and status==0: i+=1 res,desc = savedata(info,mode.biz_flag) if res == -1: info['table'] = 'data_%s'%mode.biz_flag datamodel.get_tmp_queue().push(info) elif status ==1: #mode.ille_proxy_ip.add(proxy) queue.push(corp) print(corp,info,status) print("i=%s,x=%s" %(i,x)) except: traceback.print_exc()
def exec_main_proxy(mode): """业务线程启动函数 参数mod为业务模块 模块工厂: mod. """ #socket.setdefaulttimeout(5) queue = datamodel.get_row_queue(mode.biz_flag) while 1: if datamodel.g_exit: break s_time=time.time() print(queue.__len__()) if queue.__len__() >0: proxyinfo = datamodel.get_proxy() #如果没有代理ip则插一个空值,使用本地ip if not proxyinfo: proxyinfo = [''] proxyinfo = set(proxyinfo) proxyinfo.difference_update(mode.ille_proxy_ip) #proxyinfo.add('') if len(proxyinfo) >mode.g_step: proxyinfo=random.sample(proxyinfo,mode.g_step) else: proxyinfo = list(proxyinfo) corps = [] i = 0 if not proxyinfo: print("not proxy ip....") time.sleep(5) print("start...") while i < len(proxyinfo): corp = queue.pop(1) if not corp: break if exist_corp(corp,mode.biz_flag): print(corp) continue corps.append(corp) i+=1 proxyinfo = proxyinfo[:len(corps)] pool = eventlet.GreenPool(len(proxyinfo)) result={} for corp,info,status,proxy in pool.imap(mode.get_info,corps,proxyinfo): if status in result: result[status]+=1 else: result[status]=0 if info and status==0: res,desc = savedata(info,mode.biz_flag) if res == -1: info['table'] = 'data_%s'%mode.biz_flag datamodel.get_tmp_queue().push(info) elif status ==1: mode.ille_proxy_ip.add(proxy) queue.push(corp) print(corp,info,status) print(len(proxyinfo)) print(result) e_time=time.time()-s_time print("kill time:%s"%(e_time)) else: time.sleep(5)