def savedata(data,table):
"""保存数据到mysql数据库
"""
db = mysqlwrap.get_db()
if type(data) != list:
data = [data]
for item in data:
fields = []
values = []
for k,v in item.items():
fields.append(k)
v=mysqlwrap.addslashes(v)
values.append('"%s"' % v)
sql = "insert into data_%s(%s)values(%s)" % (table,','.join(fields),','.join(values))
res,desc = db.query(sql,1)
#print(sql,res,desc)
#数据入库失败处理
if res == -1:
#print('insert_error:',desc)
if desc.args[0] in [1064,1062]:
return (0,())
item['table'] = 'data_%s'%table
datamodel.get_tmp_queue().push(item)
#插入dat_total总表
#print(item)
if 'reg_status' in item and item['reg_status'] in ['存续','在业'] :
fields.append('prov_tb')
values.append('"%s"' % table)
t_table = 'data_total'
sql = "insert into %s(%s)values(%s)" % (t_table,','.join(fields),','.join(values))
rs,ds = db.query(sql,1)
#print("insert total tb",rs,ds)
return (res,desc)
def savedata(data,table):
"""保存数据到mysql数据库
"""
db = mysqlwrap.get_db()
fields = []
values = []
for k,v in data.items():
fields.append(k)
#v = db.escape_string(v)
v=v.replace('"'," ")
values.append('"%s"' % v)
sql = "insert into data_%s(%s)values(%s)" % (table,','.join(fields),','.join(values))
res,desc = db.query(sql,1)
#数据入库失败处理
if res == -1:
if desc.args[0] == 1064:
print(desc)
return
data['table'] = 'data_%s'%table
datamodel.get_tmp_queue().push(data)
def run(self):
db = mysqlwrap.get_db()
queue = datamodel.get_tmp_queue()
while 1:
try:
row = queue.pop(3)
if row:
tb = row.pop('table')[-2:]
res,desc = savedata(row,tb)
if desc:
print(tb,res,desc)
except:
traceback.print_exc()
def exec_main(mode):
"""业务线程启动函数
参数mod为业务模块
模块工厂：
mod.
"""
x=0
i=0
queue = datamodel.get_row_queue(mode.biz_flag)
while 1:
if datamodel.g_exit:
break
corp = queue.pop(3)
if not corp:
time.sleep(5)
continue
try:
if exist_corp(corp,mode.biz_flag):
print(corp)
continue
corp,info,status,proxy =mode.get_info(corp)
x+=1
if info and status==0:
i+=1
res,desc = savedata(info,mode.biz_flag)
if res == -1:
info['table'] = 'data_%s'%mode.biz_flag
datamodel.get_tmp_queue().push(info)
elif status ==1:
#mode.ille_proxy_ip.add(proxy)
queue.push(corp)
print(corp,info,status)
print("i=%s,x=%s" %(i,x))
except:
traceback.print_exc()
def exec_main_proxy(mode):
"""业务线程启动函数
参数mod为业务模块
模块工厂：
mod.
"""
#socket.setdefaulttimeout(5)
queue = datamodel.get_row_queue(mode.biz_flag)
while 1:
if datamodel.g_exit:
break
s_time=time.time()
print(queue.__len__())
if queue.__len__() >0:
proxyinfo = datamodel.get_proxy()
#如果没有代理ip则插一个空值，使用本地ip
if not proxyinfo:
proxyinfo = ['']
proxyinfo = set(proxyinfo)
proxyinfo.difference_update(mode.ille_proxy_ip)
#proxyinfo.add('')
if len(proxyinfo) >mode.g_step:
proxyinfo=random.sample(proxyinfo,mode.g_step)
else:
proxyinfo = list(proxyinfo)
corps = []
i = 0
if not proxyinfo:
print("not proxy ip....")
time.sleep(5)
print("start...")
while i < len(proxyinfo):
corp = queue.pop(1)
if not corp:
break
if exist_corp(corp,mode.biz_flag):
print(corp)
continue
corps.append(corp)
i+=1
proxyinfo = proxyinfo[:len(corps)]
pool = eventlet.GreenPool(len(proxyinfo))
result={}
for corp,info,status,proxy in pool.imap(mode.get_info,corps,proxyinfo):
if status in result:
result[status]+=1
else:
result[status]=0
if info and status==0:
res,desc = savedata(info,mode.biz_flag)
if res == -1:
info['table'] = 'data_%s'%mode.biz_flag
datamodel.get_tmp_queue().push(info)
elif status ==1:
mode.ille_proxy_ip.add(proxy)
queue.push(corp)
print(corp,info,status)
print(len(proxyinfo))
print(result)
e_time=time.time()-s_time
print("kill time:%s"%(e_time))
else:
time.sleep(5)