def getproxy(args = None):
a = NetProcess()
a.Host = 'www.hao123.com'
errmsg = ''
myproxy = SProxy()
ip_list = myproxy.renew_proxy()
newitems = 0
for item in ip_list:
try:
ishave = Proxy.objects.filter(ip = str(item))
if ishave.count() > 0:
continue
a.Proxy = item
res = a.GetResFromRequest('GET', "http://www.hao123.com/", 'utf-8', use_proxy=True)
if isinstance(res, str):
newitems += 1
newproxy = Proxy()
newproxy.ip = item
newproxy.is_online = True
newproxy.is_used = False
newproxy.req_times = 0
newproxy.get_time = datetime.datetime.now()
newproxy.save()
except Exception, e:
except_handle(e)
errmsg = str(e)
continue
def checkproxy(args = None):
a = NetProcess()
useless = 0
off = Proxy.objects.filter(is_online = False).delete()
oters = Proxy.objects.all()
print oters.count()
for item in oters:
a.Proxy = item.ip
res = a.GetResFromRequest('GET', "http://napos.ele.me/login", 'utf-8', use_proxy=True)
if not isinstance(res, str):
print 'del ' + item.ip
item.delete()
useless += 1
else:
# print res
print item.ip + ' connect success'
content = '代理检验完成，删除无效代理' + str(useless) + '个'
print content
newlog = CronLog()
newlog.ltype = 2
newlog.content = content
newlog.status = True
newlog.save()
class SProxy(object):
def __init__(self):
self.HttpR = NetProcess()
self.XICIURL = 'http://www.xici.net.co/nn/'
self.KUAI = 'http://www.kuaidaili.com/free/inha/'
self.CNPROXY = 'http://cn-proxy.com/'
self.ip_list = []
def open_file(self, filename):
with open(filename, 'r') as f1:
html = ''
line = f1.readline()
while line:
html += line
line = f1.readline()
f1.close()
return html
def get_proxy_from_xici(self):
flag = 0
# self.HttpR.Host = 'www.xici.net.co'
for j in range(1, 4):
try:
res = self.HttpR.GetResFromRequest('GET',
self.XICIURL + str(j),
'utf-8')
if res is None:
flag += 1
continue
soup = BeautifulSoup(res)
tr_res = soup.findAll('tr')
# print len(tr_res)
for i in range(1, len(tr_res)):
td_res = tr_res[i].findAll('td')
ip = str(td_res[2].string) + ':' + str(td_res[3].string)
self.ip_list.append(copy.copy(ip))
except Exception, e:
print e
continue
if flag == 3:
return False
return True
def __init__(self, merchantid, use_proxy=False):
self.net = NetProcess()
self.__use_proxy = False
if use_proxy:
self.set_proxy(merchantid)
self.__use_proxy = True
class Ele(object):
def __init__(self, merchantid, use_proxy=False):
self.net = NetProcess()
self.__use_proxy = False
if use_proxy:
self.set_proxy(merchantid)
self.__use_proxy = True
# self.net.Host = 'napos.ele.me'
# def catcheorder(self, merchant):
# thiscount = 0
# # self.net.Referer = 'http://napos.ele.me/'
# print 'catcheele'
# # self.set_proxy(merchant)
# if not self.iflogin(merchant):
# res = self.loginele(merchant)
# if res is None or res is False:
# return None
# # cur_list = Merchant.objects.filter(id = self.merchantid)
# cache_list = CatcheData.objects.filter(merchant = merchant)
# cache = cache_list[0]
# self.net.SetCookie(str(cache.ele_cookie))
# tn = str(time.time())[0:10] + '000'
# requrl = 'http://napos.ele.me/order/list?list=unprocessed_waimai&t=' + tn
# html = self.net.GetResFromRequest('GET', requrl, 'utf-8')
# print html
# # with open('abc.txt', 'r') as f1:
# # line = f1.readline()
# # while line:
# # html += line
# # line = f1.readline()
# # print html
# if not isinstance(html, str):
# newlog = RunTimeLog()
# newlog.content = '爬虫抓单失败'
# newlog.err_message = str(html)
# newlog.status = False
# newlog.ltype = 18
# newlog.merchant = merchant
# newlog.save()
# # check_proxy_times(str(self.net.Proxy))
# return None
# # reset_proxy_times(str(self.net.Proxy))
# soup = BeautifulSoup(html)
# intro = soup.find('ul', attrs={'id': 'list_items'})
# if intro is None:
# notlogin = soup.find('label', attrs={'for': 'tab1'})
# if notlogin is not None:
# self.cookie = None
# print 'no new orders, login failed'
# return False
# res = intro.findAll('li')
# if len(res) == 0:
# print 'no new orders'
# return False
# thiscount = int(DayOrder.objects.filter(merchant=merchant, platform=2).count()) + 1
# totalcount = int(DayOrder.objects.filter(merchant=merchant).count()) + 1
# # print 'start: ' + str(thiscount)
# for item in res:
# print item
# neworder = DayOrder()
# onpay = False
# yu_time = item.find('a', attrs={'rel': 'tipsy'})
# process_num = item.find('span', attrs={'class': 'process_num'})
# detail = item.find('p', attrs={'class': 'list_addr'})
# note = item.find('p', attrs={'class': 'list_description'})
# price = item.find('p', attrs={'class': 'list_price'})
# menu = item.find('div', attrs={'class': 'menu_content'})
# orderid = item['orderid']
# print orderid
# ifhave = DayOrder.objects.filter(order_id_old = str(orderid))
# if ifhave.count() > 0:
# # print 'break'
# # print auto_id + 1
# continue
# timestamp = item['createdat']
# online = item['online-pay']
# if note is not None:
# note = note.string[2:]
# if online != '':
# onpay = True
# phone = item.find('p', attrs={'class': 'list_customer'})
# formattime = time.localtime(float(timestamp))
# datetimee = datetime.datetime(*formattime[:6])
# timee = time.strftime('%Y-%m-%d %H:%M:%S', formattime)
# newid = createAlinOrderNum(2, merchant.id, thiscount)
# thiscount += 1
# qrres = createqr(1, newid)
# time.sleep(1)
# if yu_time.has_attr('explain'):
# res = re.findall(r'([0-9,:, ,-]+)', str(yu_time['explain']))
# if len(res) > 0:
# yutt = time.strptime(str(res[0]), "%Y-%m-%d %H:%M:%S")
# yudatetime = datetime.datetime(*yutt[:6])
# neworder.send_time = yudatetime
# else:
# neworder.send_time = datetimee
# else:
# neworder.send_time = datetimee
# address = detail.string[3:]
# neworder.address = address[1:]
# neworder.order_id_alin = newid
# neworder.order_id_old = orderid
# neworder.origin_price = price.string[1:]
# neworder.real_price = price.string[1:]
# orderphone = phone.string[3:]
# phonelist = orderphone.split(',')
# neworder.phone = phonelist[0]
# neworder.order_time = datetimee
# neworder.status = 1
# neworder.promotion = 'nothing'
# neworder.day_num = str(totalcount)
# neworder.pay = onpay
# if note is None:
# neworder.note = ''
# else:
# neworder.note = note
# if len(phonelist) > 1:
# neworder.note += '应急电话：%s' % phonelist[1]
# neworder.platform = 2
# neworder.qr_path = qrres
# neworder.merchant = merchant
# neworder.plat_num = str(process_num.string)
# neworder.save()
# print timee
# print orderid
# print phone.string[3:]
# print price.string[1:]
# print address
# dishname = menu.findAll(attrs = {'class': 'dishname'})
# dishcount = menu.findAll(attrs = {'class': 'item_quantity'})
# dishprice = menu.findAll(attrs = {'class': 'item_price'})
# for i in range(0, len(dishname)):
# newdish = Dish()
# newdish.dish_name = str(dishname[i].string)
# newdish.dish_count = int(dishcount[i].string)
# newdish.dish_price = float(dishprice[i].string)
# newdish.order = DayOrder.objects.get(order_id_alin = newid)
# newdish.save()
# return True
def catcheorder(self, merchant):
thiscount = 0
# self.net.Referer = 'http://napos.ele.me/'
print 'catcheele'
# self.set_proxy(merchant)
if not self.iflogin(merchant):
res = self.loginele(merchant)
if res is None or res is False:
return None
# cur_list = Merchant.objects.filter(id = self.merchantid)
cache_list = CatcheData.objects.filter(merchant=merchant)
cache = cache_list[0]
self.net.SetCookie(str(cache.ele_cookie))
tn = str(time.time())[0:10] + '000'
requrl = 'http://napos.ele.me/order/list?list=unprocessed_waimai&t=' + tn
if self.__use_proxy:
print 'use proxy catche'
html = self.net.GetResFromRequest('GET', requrl, 'utf-8', use_proxy=True)
else:
html = self.net.GetResFromRequest('GET', requrl, 'utf-8')
print html
# with open('abc.txt', 'r') as f1:
# line = f1.readline()
# while line:
# html += line
# line = f1.readline()
if not isinstance(html, str):
newlog = RunTimeLog()
newlog.content = '爬虫抓单失败'
newlog.err_message = str(html)
newlog.status = False
newlog.ltype = 18
newlog.merchant = merchant
newlog.save()
if self.__use_proxy:
check_proxy_times(str(self.net.Proxy))
return None
if self.__use_proxy:
reset_proxy_times(str(self.net.Proxy))
soup = BeautifulSoup(html)
intro = soup.find('ul', attrs={'id': 'list_items'})
if intro is None:
notlogin = soup.find('label', attrs={'for': 'tab1'})
if notlogin is not None:
self.cookie = None
print 'no new orders, login failed'
return False
res = soup.findAll('div', attrs={'id': 'order_json_data'})
if len(res) == 0:
print 'no new orders'
return False
thiscount = int(DayOrder.objects.filter(merchant=merchant, platform=2).count()) + 1
totalcount = int(DayOrder.objects.filter(merchant=merchant).count()) + 1
# print 'start: ' + str(thiscount)
print len(res)
for item in res:
# print item
onpay = False
jsonres = simplejson.loads(item.string)
if_book = jsonres['isBook']
delivertime = jsonres['deliverTime']
orderid = jsonres['id']
price = jsonres['total']
ordertime = jsonres['createdAt']
phone = str(jsonres['phone']).split(',')
plat_num = jsonres['restaurantNumber']
address = unicode(jsonres['address']).encode('unicode_escape').decode('unicode_escape')
note = jsonres['description']
onpay = jsonres['isOnlinePayment']
print orderid
ifhave = DayOrder.objects.filter(order_id_old = str(orderid))
if ifhave.count() > 0:
continue
if len(phone) > 1:
note += '应急电话：%s' % phone[1]
phone = phone[0]
else:
phone = phone[0]
newid = createAlinOrderNum(2, merchant.id, thiscount)
thiscount += 1
qrres = createqr(1, newid)
ordertime_t = time.strptime(str(ordertime), "%Y-%m-%d %H:%M:%S")
ordertime_datetime = datetime.datetime(*ordertime_t[:6])
neworder = DayOrder(order_id_alin=newid, order_id_old=orderid, phone=phone, pay=onpay, address=address, platform=2, plat_num=plat_num, note=note, origin_price=price, real_price=price, status=1, order_time=ordertime_datetime, day_num=totalcount, qr_path=qrres)
if if_book:
# print len(delivertime)
deliver_u = str(delivertime).replace('分', '').replace('点', '')
deliver_s = deliver_u.encode('unicode_escape').decode('unicode_escape')
# print deliver_s
date = datetime.datetime.now().strftime("%Y-%m-%d")
date += ' ' + deliver_s
deliver_t = time.strptime(str(date), "%Y-%m-%d %H%M")
deliver_datetime = datetime.datetime(*deliver_t[:6])
neworder.send_time = deliver_datetime
else:
neworder.send_time = ordertime_datetime
neworder.merchant = merchant
neworder.save()
dishs = jsonres['items']
norder = DayOrder.objects.get(order_id_alin=newid)
for itm in dishs:
dishname = unicode(itm['name']).encode('unicode_escape').decode('unicode_escape')
dishcount = itm['quantity']
dishprice = float(itm['price'])
newdish = Dish(dish_name=dishname, dish_price=dishprice, dish_count=dishcount)
newdish.order = norder
newdish.save()
return True
def iflogin(self, merchant):
cache_list = CatcheData.objects.filter(merchant=merchant)
if cache_list.count() > 0:
cache = cache_list[0]
if cache.ele_cookie != '':
return True
return False
else:
return False
def loginele(self, curmet):
# self.set_proxy(curmet)
# self.net.Referer = 'http://napos.ele.me/login'
print 'loginele'
user = str(curmet.ele_account)
passwd = str(curmet.ele_passwd)
de_pass = Decrypt(passwd)
postdic = {'username': user, 'password': de_pass}
# print 'login ele'
if self.__use_proxy:
print 'use proxy login'
html = self.net.GetResFromRequest('POST', 'http://napos.ele.me/auth/doLogin', 'utf-8', postdic, use_proxy=True)
else:
html = self.net.GetResFromRequest('POST', 'http://napos.ele.me/auth/doLogin', 'utf-8', postdic)
# print 'login end'
if not isinstance(html, str):
newlog = RunTimeLog()
newlog.content = '饿了么登陆失败'
newlog.merchant = curmet
newlog.err_message = str(html)
newlog.ltype = 15
newlog.status = False
newlog.save()
if self.__use_proxy:
check_proxy_times(str(self.net.Proxy))
else:
if self.__use_proxy:
reset_proxy_times(str(self.net.Proxy))
res_json = simplejson.loads(html)
if res_json['success'] is False:
errmsg = unicode(res_json['message'])
curmet.ele_message = errmsg.encode('unicode_escape').decode('unicode_escape')
curmet.ele_status = False
curmet.save()
newlog = RunTimeLog()
newlog.content = '饿了么登陆失败'
newlog.merchant = curmet
newlog.err_message = errmsg
newlog.ltype = 15
newlog.save()
return False
elif res_json['success'] is True:
# if self.net.Cookies['SSID'] == 'nothing find':
# res = self.loginele()
# return res
curmet.ele_status = True
curmet.save()
cache_list = CatcheData.objects.filter(merchant = curmet)
print str(self.net.OutPutCookie())
if cache_list.count() > 0:
cache = cache_list[0]
cache.ele_cookie = str(self.net.OutPutCookie())
cache.save()
else:
cache = CatcheData()
cache.ele_cookie = str(self.net.OutPutCookie())
cache.merchant = curmet
cache.save()
return True
def ensureorder(self, curmet, order):
# curmet_list = Merchant.objects.filter(id = self.merchantid)
# if curmet_list.count() == 0:
# return None
# self.set_proxy(curmet)
if not self.iflogin(curmet):
res = self.loginele(curmet)
if res is None or res is False:
return None
cache_list = CatcheData.objects.filter(merchant = curmet)
cache = cache_list[0]
self.net.SetCookie(str(cache.ele_cookie))
requrl = 'http://napos.ele.me/order/processOrder/id/' + str(order) + '/category/1'
if self.__use_proxy:
print 'use proxy ensure'
html = self.net.GetResFromRequest('GET', requrl, 'utf-8', use_proxy=True)
else:
html = self.net.GetResFromRequest('GET', requrl, 'utf-8')
if not isinstance(html, str):
newlog = RunTimeLog()
newlog.content = '确认饿了没订单失败'
newlog.status = False
newlog.err_message = str(html)
newlog.ltype = 16
newlog.merchant = curmet
newlog.save()
return False
if str(html) == '0':
newlog = RunTimeLog()
newlog.content = '确认饿了没订单失败'
newlog.status = False
newlog.err_message = str(order) + html
newlog.ltype = 16
newlog.merchant = curmet
newlog.save()
return False
return True
def refuseorder(self, curmet, order):
# self.set_proxy(curmet)
if not self.iflogin(curmet):
res = self.loginele(curmet)
if res is None or res is False:
return None
cache_list = CatcheData.objects.filter(merchant = curmet)
cache = cache_list[0]
self.net.SetCookie(str(cache.ele_cookie))
requrl = 'http://napos.ele.me/order/setInvalid/id/' + str(order) + '/category/1?type=6&remark='
if self.__use_proxy:
print 'use proxy refuse'
html = self.net.GetResFromRequest('GET', requrl, 'utf-8', use_proxy=True)
else:
html = self.net.GetResFromRequest('GET', requrl, 'utf-8')
if not isinstance(html, str):
newlog = RunTimeLog()
newlog.content = '拒绝饿了么订单失败'
newlog.status = False
newlog.err_message = str(html)
newlog.ltype = 17
newlog.merchant = curmet
newlog.save()
return False
if str(html) == '0':
newlog = RunTimeLog()
newlog.content = '拒绝饿了么订单失败'
newlog.ltype = 17
newlog.err_message = str(order) + html
newlog.status = False
newlog.merchant = curmet
newlog.save()
return False
return True
def set_proxy(self, curmet):
ip = distriproxy(str(curmet))
if isinstance(ip, str):
self.net.Proxy = str(ip)
else:
print 'no proxy, use real ip'
self.__use_proxy = False
def __init__(self):
self.HttpR = NetProcess()
self.XICIURL = 'http://www.xici.net.co/nn/'
self.KUAI = 'http://www.kuaidaili.com/free/inha/'
self.CNPROXY = 'http://cn-proxy.com/'
self.ip_list = []
