def get_data(self):
#print(self.data)
r = requests.post(url=self.url,data=self.data,headers=self.post_headers)
xml =str(r.text)
print(xml)
re_e = r'FailedCount>(\d+)'
res = re.compile(re_e)
fail_info =re.findall(res, xml)
try:
if int(fail_info[0]):
log_set(name='fix', msg='fix is fail ,do it youself,id =%s' % self.xc_id)
return False
else:
print('fixied')
log_set(name='fix', msg=' fix is ok!,id =%s,%s,%s,%s,%s,%s'% (self.date,self.DepartPort,self.ArrivePort,self.Flight_No,self.price,self.xc_id))
print(self.date,self.DepartPort,self.ArrivePort,self.Flight_No,self.price,self.xc_id)
update2_xcfly(self.date,self.DepartPort,self.ArrivePort,self.Flight_No,self.price,self.xc_id,self.ivent)
print('fix is successs,%s' % self.id)
return True
except:
#log_set(name='del_info', msg='[Del_XC_Fail2]deal error, please check!,id =%s'% self.id)
#return False
pass
def del_work(xc_id):
print('deling')
del_db_xcinfo(xc_id)
log_set(name='del_info', msg='will to de,id =%s'%xc_id)
do_work = Delete_XieChenData(xc_id)
del2_xcinfo(xc_id)
return do_work.get_data()
def del_db_xcinfo(id):
sql = " DELETE FROM fly_xc where xc_id ='%s'"%id
print(sql)
try:
ret = do_sql(sql)
log_set(name='sql',msg='del_DB,id=%s'%id)
return ret
except Exception as e:
log_set(name='sql', msg='%s,deldb_1 is fail:%s'%(str(e.message),sql))
def update2_xcfly(YearMonthDate1,DepartPort,ArrivePort, Flight_No,price,xc_id,iVent):
all_uni = str(YearMonthDate1)+str(DepartPort)+str(ArrivePort)+str(Flight_No)
try:
sql = " update fly_xc2 set xc_id ='%s',price ='%s',inVent ='%s' WHERE all_uni='%s'"%(xc_id, price, iVent,all_uni)
print(sql)
ret = do_sql(sql)
log_set(name='sql', msg='[update2_save] is ok,%s,%s,%s,%s,%s,%s,%s+%s'%(str(YearMonthDate1),str(DepartPort),str(ArrivePort),str(Flight_No),str(price),str(xc_id),iVent,sql))
return ret
except Exception as e:
log_set(name='sql', msg= '%s,update2 is fail,%s'%(str(e.message),str(sql)))
def save2_xc_info(YearMonthDate1, DepartPort,ArrivePort, Flight_No,inVent,xc_id,price):
all_uni = str(YearMonthDate1)+str(DepartPort)+str(ArrivePort)+str(Flight_No)
sql = "insert into fly_xc2(YearMonthDate1,DepartPort,ArrivePort,Flight_No,inVent,xc_id,all_uni,price) VALUES ('%s','%s','%s','%s','%s','%s','%s','%s')"%(YearMonthDate1,DepartPort,ArrivePort,Flight_No,inVent,xc_id,all_uni,price)
print(sql)
try:
ret = do_sql(sql)
log_set(name='sql',msg='[save2]%s'%all_uni)
return ret
except Exception as e:
time.sleep(2)
log_set(name='sql', msg='%s,save2_fial_to update:%s'%(str(e.message),sql))
update2_xcfly(YearMonthDate1,DepartPort,ArrivePort, Flight_No,price,xc_id,inVent)
def save_xc_info(YearMonthDate1, DepartPort,ArrivePort, Flight_No,Flight_Price,inVent,xc_id):
all_uni = str(YearMonthDate1)+str(DepartPort)+str(ArrivePort)+str(Flight_No)
sql = "insert into fly_xc(YearMonthDate1,DepartPort,ArrivePort,Flight_No,Flight_Price,inVent,xc_id,all_uni) VALUES ('%s','%s','%s','%s','%s','%s','%s','%s')"%(YearMonthDate1,DepartPort,ArrivePort,Flight_No,Flight_Price,inVent,xc_id,all_uni)
print(sql)
try:
ret = do_sql(sql)
log_set(name='sql', msg='[save1_fly_xc],%s'%all_uni)
return ret
except Exception as e:
log_set(name='sql', msg='save1%s,%s,%s,%s,%s,%s,%s'(str(e.message),str(YearMonthDate1),str(DepartPort),str(ArrivePort),str(Flight_No),str(Flight_Price),str(inVent)))
update_xcfly(YearMonthDate1,DepartPort,ArrivePort,Flight_Price,Flight_No,inVent,xc_id)
def update_xcfly(YearMonthDate1,DepartPort, ArrivePort,Flight_Price,Flight_No,inVent,xc_id):
all_uni = str(YearMonthDate1)+str(DepartPort)+str(ArrivePort)+str(Flight_No)
sql = """update fly_xc set xc_id ='{}',Flight_Price ='{}',inVent ='{}'
WHERE all_uni ='{}'""".format(xc_id,Flight_Price,inVent,all_uni)
try:
print(sql)
ret = do_sql(sql)
log_set(name='sql', msg='update_1 is %s'%(sql))
return ret
except Exception as e:
print('b')
log_set(name='sql', msg='%s,update_1 is fail:%s'%(str(e.message),sql))
def work():
# g = multiprocessing.Process(target = lunxun)
# g.start()
with open('hx.txt', 'r') as f:
cont = f.readlines()
for i in range(len(cont)):
print(i)
hangxian = cont[i].replace('\r\n', '')
hangxian = [x for x in hangxian.split("'") if x.isalpha()]
#print(type(hangxian))
p = multiprocessing.Process(target = tongbu, args= (hangxian[0], hangxian[1]))
p.start()
log_set(name='hangxian',msg= (str(i)+str(hangxian[0])+str(hangxian[1])))
def compare(self,feiba_info,day,deptCd,arrCd):
"""
feiba_inf {'KN5851': ('A', '578.0')}
xc[('139.94', 'KN2908'), ('397.09', 'KN2906')
"""
push_info=''
info_list = self.get_data()
#print('xc_info',info_list)
#print('feba_info',feiba_info)
if info_list:
for xc in info_list:
fligh_name_feiba = (list(feiba_info)[0].encode('utf-8')).lower()
xc_flight_no = (xc[2]).encode('utf-8').lower()
#print(xc_flight_no,fligh_name_feiba)
if xc_flight_no == fligh_name_feiba: # flight
# print('有对应的航班:%s'%(xc[1]).upper())
log_set(name='class_xiecheng', msg ='[xiecheng]有对应的航班:%s'%(xc[1]).upper())
feiba_price = feiba_info[fligh_name_feiba.upper()][1].encode('utf-8')
inVent =feiba_info[fligh_name_feiba.upper()][0].encode('utf-8')
xc_id = get2_id_from_info(self.yearDate, self.DepartPort,self.ArrivePort,fligh_name_feiba[2:])
#if 778.0 - float(feiba_price) >6:
#print('----------',float(xc[1]),float(feiba_price))
if float(xc[1]) - float(feiba_price) >6: #推送条件
if self.tag in (0,'0'): # new come
# save_xc_info(self.yearDate, self.DepartPort,self.ArrivePort,xc[2], str(float(feiba_price)),inVent,xc_id=xc[0])
Flight_No=(xc[2][2:]).encode('utf-8')
#print('old',xc_id)
if not xc_id: # never push
XC_send(YearMonthDate1=day, DepartPort=deptCd,ArrivePort=arrCd, Flight_No=Flight_No,Flight_Price=float(xc[1]),feiba_inVent=inVent,feiba_price=feiba_price)
#save_xc_info(YearMonthDate1=day, DepartPort=deptCd,ArrivePort=arrCd, Flight_No=xc[1],Flight_Price=float(xc[0]),inVent=int(feiba_info[(xc[1]).upper()][0]))
push_info = 'yearDate:%s,DepartPort:%s,ArrivePort%s##%s:%s will push !!!!!!'%(self.yearDate,self.DepartPort,self.ArrivePort,xc[0], xc[1])
log_set(name='class_xiecheng',msg=push_info)
else:
#print('update2')
update_xcfly(day,deptCd,arrCd,feiba_price,Flight_No,inVent,xc_id,)
else:
#print('携程数据中没有推送符合条件的航班信息,need delete zhe push order')
log_set(name='class_xiecheng', msg = '[xiecheng]携程数据中没有推送符合条件的航班信息need delete zhe push order,%s,%s,xiecheng:%s,feiba:%s'%(xc[2],self.yearDate,xc[1],feiba_price))
else:
#print('携程没有相应的航班')
else:
#print('can not get data form xiecheng')
log_set(name='class_xiecheng',msg='can not ge date from xiecheng')
return push_info
def XC_send(YearMonthDate1, DepartPort, ArrivePort, Flight_No, Flight_Price, feiba_inVent, feiba_price):
## print (YearMonthDate1, DepartPort,ArrivePort="CIF", Flight_No=2929,Flight_Price=2898,inVent=12)
# 修正数据A投5， 大于6投4， 5个投3 大于3小于5投2 ， 2投1
print("xc_send", YearMonthDate1, DepartPort, ArrivePort, Flight_No, Flight_Price, feiba_inVent)
Flight_No = Flight_No.lower().encode("utf-8")
ret2 = r"(\d+)"
ret2 = re.compile(ret2)
Flight_No = (re.findall(ret2, Flight_No))[0]
Flight_Price = str(float(Flight_Price) - 0.1)
inVent2 = 100
if feiba_inVent in (1, 2, 3, 4, 5, 6, 7, 8, 9, "1", "2", "3", "4", "5", "6", "7", "8", "9"):
if feiba_inVent in (1, "1"):
inVent = 100
if feiba_inVent in (2, "2"):
inVent = 1
if feiba_inVent in (3, 4, "3", "4"):
inVent = 2
if feiba_inVent in ("5", 5):
inVent = 3
if feiba_inVent in ("6", "7", "8", "9", 6, 7, 8, 9):
inVent = 4
else:
inVent = 5
print(inVent)
if inVent != 100:
dowork = Post_XieChenData(
yearDate=YearMonthDate1,
DepartPort=DepartPort,
ArrivePort=ArrivePort,
Flight_No=Flight_No,
Flight_Price=Flight_Price,
inVent=inVent,
feiba_inVent=feiba_inVent,
feiba_price=feiba_price,
)
log_set(
name="send",
msg="[send_fail] %s,%s,%s,%s,%s,%s"
% (YearMonthDate1, DepartPort, ArrivePort, Flight_No, Flight_Price, feiba_inVent),
)
return dowork.push_data()
def feiba_main(deptCd, arrCd, start_day=0, over_day=10):
#q = Queue(connection=Redis())
try:
start_day = int(start_day)
days = int(over_day) #延迟天数
deptCd = deptCd.upper()
arrCd = arrCd.upper()
except:
return {'code':1}
# print(start_day, days, deptCd, arrCd)
now =datetime.now()
for s in range(start_day, days):
a = timedelta(days=s)
YearMonthDate = (str(now+a)[:10])
do_work = spider(deptCd=deptCd, arrCd=arrCd, deptDt=YearMonthDate)
spy_data = do_work.do_work() #获得str数据
if len(spy_data)>2000:
"""有取到有效信息"""
info_list = deal_data(spy_data)
print('all hyl is'+str(info_list))
## log_set(name='lianhe', msg='[lianhe]all hyl is'+str(info_list))
for info in info_list:
one_info=dict()
# print(YearMonthDate+'//飞机航班号：'+ info+'//'+'剩余票数：'+ (info_list[info][0]) + '//欢乐抢单程单价:'+info_list[info][1])
log_set(name='lianhe', msg='lianhe:'+YearMonthDate+',fight_no：'+ info+':'+'invent：'+ (info_list[info][0]) + ':feiba_price:'+info_list[info][1])
# print('#############去做对比工作和投放#############')
one_info[info] = info_list[info]
## log_set(name='duibi', msg='[lianhe]'+YearMonthDate+'//飞机航班号：'+ info+'//'+'剩余票数：'+ (info_list[info][0]) + '//欢乐抢单程单价:'+info_list[info][1]+deptCd+arrCd)
#print('YYYYYYYYYYYYYYY')
#print(one_info)
p = multiprocessing.Process(target=home, args=(YearMonthDate, deptCd, arrCd, one_info, 0))
p.start()
if not len(spy_data):
print('已经没有欢乐抢机票了')
log_set(name='lianhe', msg='[lianhe]it have not invent')
def do_work(self):
# print(self.deptDt)
timeout = 10
socket.setdefaulttimeout(timeout)
randoms = random.uniform(0, 1)
url = self.url + '%.17f' % randoms
############# proxies##########
#prox_list = [{'http': 'http://*****:*****@103.27.125.250:6666'}]
proxies = random.choice(IP_list)
# print(proxies)
###################
try:
# log_set(name='lianhe',msg='#################################################')
log_set(name='class_xiecheng',msg='验证网站'+str(self.url_page))
# print(self.data)
# print(url)
# print(self.headers)
time.sleep(3)
r = requests.post(url=url, data=self.data, headers=self.headers, proxies=proxies, timeout=10)
time.sleep(2)
cont = r.content
# print(cont)
if len(cont) > 2500:
#print('yes')
return cont
else:
# print('sessionId获取失败，参数错误，you send error parmars')
# log_set(name='lianhe', msg='[lianhe]sessionId获取失败，参数错误，%s,%s,%s'%(self.url_page, self.deptDt,proxies))
return ''
except Exception as e:
# print('网络请求问题')
# log_set(name='lianhe', msg='[lianhe]网络请求问题,%s,%s,%s'%(self.url_page, self.deptDt,proxies))
return ''
def get_data(self):
r = requests.post(url=self.url,data=self.data,headers=self.post_headers)
xml =str(r.text)
#print(xml)
re_e = r'FailedCount>(\d+)'
res = re.compile(re_e)
fail_info =re.findall(res, xml)
try:
if int(fail_info[0]):
log_set(name='del_info', msg='[Del_XC_Fail]del is fail ,do it youself,id =%s' % self.id)
return False
else:
log_set(name='del_info', msg='[Del_XC_Ok] del_XC is ok!,id =%s'% self.id)
del2_xcinfo(self.id)
print('xiaohui is successs,%s'%self.id)
return True
except:
log_set(name='del_info', msg='[Del_XC_Fail2]deal error, please internater!,id =%s'% self.id)
def push_data(self):
r = requests.post(url=self.url, data=self.data, headers=self.post_headers)
xml_str = (r.text).lower()
# print(xml_str)
ret = r"<successcount>(\d)</successcount>"
ret = re.compile(ret)
info_list = re.findall(ret, xml_str)
if len(info_list[0]):
print("投放成功")
ret_id = r"<successlist><id>(\d+)"
ret_id = re.compile(ret_id)
id_list = re.findall(ret_id, xml_str)
# print(info_list)
xc_id_new = str(id_list[0])
old_xc_id = get2_id_from_info(self.yearDate, self.DepartPort, self.ArrivePort, self.Flight_No)
print(old_xc_id)
if not old_xc_id:
save_xc_info(
self.yearDate,
self.DepartPort,
self.ArrivePort,
self.Flight_No,
self.feiba_price,
self.feiba_inVent,
xc_id=xc_id_new,
)
save2_xc_info(
self.yearDate,
self.DepartPort,
self.ArrivePort,
self.Flight_No,
self.inVent,
xc_id_new,
self.Flight_Price,
)
else:
update2_xcfly(
self.yearDate,
self.DepartPort,
self.ArrivePort,
self.Flight_No,
self.Flight_Price,
xc_id_new,
self.inVent,
)
log_set(
name="send",
msg="[send] ok!,%s,%s,%s,%s,%s,%s"
% (self.Flight_No, xc_id_new, self.inVent, self.DepartPort, self.ArrivePort, self.yearDate),
)
return True
else:
# print('失败')
log_set(
name="send",
msg="[send_fail] %s,%s,%s,%s,%s"
% (self.Flight_No, self.inVent, self.DepartPort, self.ArrivePort, self.yearDate),
)
return False
def home(day, deptCd, arrCd,feiba_info,tag):
do_work = DoXC (yearDate=day ,DepartPort=deptCd,ArrivePort=arrCd ,tag=tag)
log_set(name='class_xiecheng', msg ='###################################.%s,%s,%s,%s,%s'%(day, deptCd, arrCd,feiba_info,tag))
info = do_work.compare(feiba_info,day, deptCd,arrCd)
def compare(self,feiba_info,day,deptCd,arrCd):
"""
feiba_inf {'KN5851': ('A', '578.0')}
xc[('139.94', 'KN2908'), ('397.09', 'KN2906')
"""
push_info=''
info_list = self.get_data()
#print('xc_info',info_list)
#print('feba_info',feiba_info)
if info_list:
for xc in info_list:
fligh_name_feiba = (list(feiba_info)[0].encode('utf-8')).lower()
xc_flight_no = (xc[2]).encode('utf-8').lower()
#print(xc_flight_no,fligh_name_feiba)
if xc_flight_no == fligh_name_feiba: # flight
# print('有对应的航班:%s'%(xc[1]).upper())
#log_set(name='class_xiecheng', msg ='[xiecheng]有对应的航班:%s'%(xc[1]).upper())
feiba_price = feiba_info[fligh_name_feiba.upper()][1].encode('utf-8')
inVent =feiba_info[fligh_name_feiba.upper()][0].encode('utf-8')
#print(inVent)
xc_id = get2_id_from_info(self.yearDate, self.DepartPort,self.ArrivePort,fligh_name_feiba[2:])
if inVent in (1,'1'):
del_work(xc_id)
#if 778.0 - float(feiba_price) >6:
#print('----------',float(xc[1]),float(feiba_price))
if float(xc[1]) - float(feiba_price) >6: #推送条件
if self.tag in (1,'1'): # yi tou
old_xc_price = get2_pri_from_info(self.yearDate, self.DepartPort,self.ArrivePort,fligh_name_feiba[2:])
if not (str(xc[0])).encode('utf-8') == xc_id: #bie ren gai
#print('old_price'+str(old_xc_price)+'NEW PRICE'+str(float(xc[1])))
xx =float(xc[1])-float(old_xc_price)
if (not (str(xx) == '0.1')): # bu shi wo
price = str(float(xc[1])-0.1)
#print(xc_id,self.yearDate,price,self.DepartPort,self.ArrivePort,fligh_name_feiba[2:])
log_set(name='fix',msg='%s,me%s,will fix price,last time I push:%s,now xcprice:%s,i will push:%s,feibaflight:%s'%(xc[0],xc_id,old_xc_price,xc[1],price,feiba_price))
inVent= feiba_info[fligh_name_feiba.upper()][0].encode('utf-8')
fix_work(xc_id,self.yearDate,price,self.DepartPort,self.ArrivePort,fligh_name_feiba[2:],inVent)
else:
#print('yanshi ,bugaijia')
log_set(name='lunxun_xiecheng', msg='yanshi bu gai jia,old_price'+ str(old_xc_price)+'NEW PRICE'+str(float(xc[1])))
else:
#print('woziji bu gai jia')
log_set(name='lunxun_xiecheng', msg='woziji bu gai jia,old_price'+ str(old_xc_price)+'NEW PRICE'+str(float(xc[1])))
pass #shi wo
else:
#print('need delete zhe push order')
log_set(name='lunxun_xiecheng', msg = '[delete]need delete zhe push order,%s,%s,xiecheng:%s,feiba:%s'%(xc[2],self.yearDate,xc[1],feiba_price))
#print('delete the pushed order')
log_set(name='fix',msg='delete the pushed order:%s,%s,%s,%s,%s'%(self.yearDate, self.DepartPort,self.ArrivePort,fligh_name_feiba[2:],xc_id))
#print('idddddd',xc_id)
del_work(xc_id)
else:
#print('携程没有相应的航班')
else:
#print('can not get data form xiecheng')
log_set(name='lunxun_xiecheng',msg='can not ge date from xiecheng')
return push_info
def compare(self,feiba_info,day,deptCd,arrCd):
"""
feiba_inf {'KN5851': ('A', '578.0')}
xc[('139.94', 'KN2908'), ('397.09', 'KN2906')
"""
push_info=''
info_list = self.get_data()
print('xc_info',info_list)
print('feba_info',feiba_info)
if info_list:
for xc in info_list:
fligh_name_feiba = (list(feiba_info)[0].encode('utf-8')).lower()
xc_flight_no = (xc[2]).encode('utf-8').lower()
print(xc_flight_no,fligh_name_feiba)
if xc_flight_no == fligh_name_feiba: # flight
# print('有对应的航班:%s'%(xc[1]).upper())
log_set(name='class_xiecheng', msg ='[xiecheng]有对应的航班:%s'%(xc[1]).upper())
feiba_price = feiba_info[fligh_name_feiba.upper()][1].encode('utf-8')
inVent =feiba_info[fligh_name_feiba.upper()][0].encode('utf-8')
print(inVent)
xc_id = get2_id_from_info(self.yearDate, self.DepartPort,self.ArrivePort,fligh_name_feiba[2:])
#if 778.0 - float(feiba_price) >6:
print('----------',float(xc[1]),float(feiba_price))
if float(xc[1]) - float(feiba_price) >6: #推送条件
# if self.tag in (1,'1'): # yi tou
# print(str(xc[0]),xc_id)
# old_xc_price = get2_pri_from_info(self.yearDate, self.DepartPort,self.ArrivePort,fligh_name_feiba[2:])
#
# if not (str(xc[0])).encode('utf-8') == xc_id: #bie ren gai
# print('old_price'+str(old_xc_price)+'NEW PRICE'+str(float(xc[1])))
# xx =float(xc[1])-float(old_xc_price)
# if (not (str(xx) == '0.1')): # bu shi wo
# print(xx)
# # log_set(name='fix_byself',msg=('do fix,old_price:'+str(old_xc_price)+'NEW PRICE:'+str(float(xc[1])))+':'+str(xc[0])+':'+str(xc_id))
# print('will do fix')
# price = str(float(xc[1])-0.1)
# print(xc_id,self.yearDate,price,self.DepartPort,self.ArrivePort,fligh_name_feiba[2:])
# log_set(name='fix',msg='will fix price ne:%s,old:%s'%(price,old_xc_price))
# fix_work(xc_id,self.yearDate,price,self.DepartPort,self.ArrivePort,fligh_name_feiba[2:])
# else:
# print('yanshi ,bugaijia')
# log_set(name='class_xiecheng', msg='yanshi bu gai jia,old_price'+ str(old_xc_price)+'NEW PRICE'+str(float(xc[1])))
#
#
# else:
# print('woziji bu gai jia')
# log_set(name='class_xiecheng', msg='woziji bu gai jia,old_price'+ str(old_xc_price)+'NEW PRICE'+str(float(xc[1])))
# pass #shi wo
if self.tag in (0,'0'): # new come
# save_xc_info(self.yearDate, self.DepartPort,self.ArrivePort,xc[2], str(float(feiba_price)),inVent,xc_id=xc[0])
Flight_No=(xc[2][2:]).encode('utf-8')
print('old',xc_id)
if not xc_id: # never push
XC_send(YearMonthDate1=day, DepartPort=deptCd,ArrivePort=arrCd, Flight_No=Flight_No,Flight_Price=float(xc[1]),feiba_inVent=inVent,feiba_price=feiba_price)
#save_xc_info(YearMonthDate1=day, DepartPort=deptCd,ArrivePort=arrCd, Flight_No=xc[1],Flight_Price=float(xc[0]),inVent=int(feiba_info[(xc[1]).upper()][0]))
push_info = 'yearDate:%s,DepartPort:%s,ArrivePort%s##%s:%s will push !!!!!!'%(self.yearDate,self.DepartPort,self.ArrivePort,xc[0], xc[1])
log_set(name='class_xiecheng',msg=push_info)
else:
print('update2')
update2_xcfly(day,deptCd,arrCd, Flight_No,float(xc[1]),xc_id,inVent)
else:
print('携程数据中没有推送符合条件的航班信息,need delete zhe push order')
log_set(name='class_xiecheng', msg = '[xiecheng]携程数据中没有推送符合条件的航班信息need delete zhe push order,%s,%s,xiecheng:%s,feiba:%s'%(xc[2],self.yearDate,xc[1],feiba_price))
if self.tag in (1,'1'):
print('delete the pushed order')
log_set(name='fix_info',msg='delete the pushed order:%s,%s,%s,%s,%s,%s'%(self.yearDate, self.DepartPort,self.ArrivePort,fligh_name_feiba[2:],feiba_price,xc[1]))
xc_id = get2_id_from_info(self.yearDate, self.DepartPort,self.ArrivePort,fligh_name_feiba[2:])
print('idddddd',xc_id)
del_work(xc_id)
else:
print('携程没有相应的航班')
else:
print('can not get data form xiecheng')
log_set(name='class_xiecheng',msg='can not ge date from xiecheng')
return push_info