class IECom():
"""open IE and use IE to auto lock bill"""
url = 'http://ifeidai.com/EDaiWeb/AgentLoan/Login'
fast_ur = 'http://ifeidai.com/EDaiWeb/AgentLogon/MySaleApplyBill?Length=10'
result_file = 'result.txt'
usr_file = 'login.txt'
city_name = [u'青岛', u'济南', u'唐山', u'无锡', u'上海', u'扬州',
u'深圳', u'武汉', u'石家庄', u'北京', u'南昌', u'海口',
u'惠州', u'成都', u'南京', u'广州', u'中山', u'郑州',
u'南宁', u'佛山', u'长沙', u'重庆', u'西安', u'贵阳',
u'厦门', u'青岛市', u'济南市', u'唐山市', u'无锡市',
u'上海市', u'扬州市', u'深圳市', u'武汉市', u'石家庄市',
u'北京市', u'南昌市', u'海口市', u'惠州市', u'成都市',
u'南京市', u'广州市', u'中山市', u'郑州市', u'南宁市',
u'佛山市', u'长沙市', u'重庆市', u'西安市', u'贵阳市',
u'厦门市']
def __init__(self):
'''
init open IE
'''
code = getnode()
self.code_license = code % 3721 + code * 9997 % 997
self.root = cur_file_dir()
try:
lic = open(self.root + os.sep + 'lic')
line = lic.readline()
lic.close()
except:
line = u'请输入激活码'
self.main_window = tk.Tk()
self.main_window.minsize(300, 300)
self.main_window.title(u'锁单神器')
self.main_window.protocol('WM_DELETE_WINDOW', self.quit)
win = tk.Frame()
self.entry_var = tk.StringVar()
self.entry_var.set(line)
self.result_var = tk.StringVar()
self.result_var.set(u'信息文件路径： ' + self.result_file)
self.mac_var = tk.StringVar()
self.mac_var.set(u'机器码: ' + str(code))
self.usr_var = tk.StringVar()
self.usr_var.set(u'账户文件路径： ' + self.usr_file)
self.info_str = u'''
锁单神器v7：自动将信息文件中的数据锁到账户文件中的账户中。
信息文件格式：姓名,身份证号,手机号,城市名
例如：张三,100000000000000000,11100000000,北京
账户文件格式：账户名,密码
例如：100000000000000000,a111111
Copyright (C) 2015 Yeoman Zhuang
'''
self.info_var = tk.StringVar()
self.info_var.set(self.info_str)
win.pack()
label_mac = tk.Entry(win, textvariable=self.mac_var)
label_mac.pack(side=tk.TOP)
self.entry_en = tk.Entry(win, textvariable=self.entry_var)
self.entry_en.pack(side=tk.TOP)
label_result = tk.Label(win, textvariable=self.result_var)
label_result.pack(side=tk.TOP)
label_usr = tk.Label(win, textvariable=self.usr_var)
label_usr.pack(side=tk.TOP)
label_info = tk.Label(win, textvariable=self.info_var)
label_info.pack(side=tk.BOTTOM)
button_act = tk.Button(win, text=u'激活', command=self.act)
button_act.pack(side=tk.LEFT)
self.button_set_result = tk.Button(
win, text=u'打开信息文件', command=self.set_result, state=tk.DISABLED)
self.button_set_result.pack(side=tk.LEFT)
self.button_set_usr = tk.Button(
win, text=u'打开账户文件', command=self.set_info, state=tk.DISABLED)
self.button_set_usr.pack(side=tk.LEFT)
# self.button_open = tk.Button(
# win, text=u'打开IE', command=self.open, state=tk.DISABLED)
# self.button_open.pack(side=tk.LEFT)
self.button_start = tk.Button(
win, text=u'开始锁单', command=self.open_auto, state=tk.DISABLED)
self.button_start.pack(side=tk.LEFT)
win.mainloop()
def open_auto(self):
self.open()
self.auto()
def act(self):
if str(self.code_license) == self.entry_en.get():
self.button_start['state'] = tk.NORMAL
self.button_set_usr['state'] = tk.NORMAL
self.button_set_result['state'] = tk.NORMAL
# self.button_open['state'] = tk.NORMAL
lic = open(self.root + os.sep + 'lic', 'w')
lic.write(str(self.code_license))
lic.close()
self.entry_var.set(u'激活成功')
else:
self.entry_var.set(u'激活失败，请重新输入')
def set_result(self):
self.result_file = tkFileDialog.askopenfilename(initialdir='D:/')
self.result_var.set(u'信息文件路径： ' + self.result_file)
def set_info(self):
self.usr_file = tkFileDialog.askopenfilename(initialdir='D:/')
self.usr_var.set(u'账户文件路径： ' + self.usr_file)
def open(self):
'''
open IE
'''
print(u'打开登陆账号文件和信息文件')
self.result = open(self.result_file)
self.usr = open(self.usr_file)
print(u'绑定IE')
self.__ie = DispatchEx('InternetExplorer.Application')
self.__ie.visible = 1
self.__ie.navigate(self.url)
self.wait()
self.document = self.__ie.Document
def login(self, usr_name, usr_pwd):
'''
login success return 1
'''
print('Login:'******'输入账户密码')
while True:
try:
self.document.Body.getElementsByTagName(
'input')[0].value = usr_name
self.document.getElementsByTagName('input')[1].value = usr_pwd
while(self.identify() == ''):
pass
self.document.getElementsByTagName(
'input')[2].value = self.identify()
self.document.getElementById('btnSave').click()
self.wait()
break
except:
self.__ie.refresh()
sleep(4)
self.wait()
flag = 't'
if flag == 't':
print(u'登陆成功！')
sleep(2)
self.__ie.navigate(self.fast_ur)
self.wait()
self.document.getElementById('FastRecommend').click()
print(u'开始快速推荐')
self.wait()
return 1
else:
print('Error: Can not log in ' + flag)
print('usr_name = ' + usr_name)
print('usr_pwd = ' + usr_pwd)
return 0
def identify(self):
print(u'获得验证码')
return self.document.getElementById('ValidateValue').value
# def reset_flag(self):
# flag = self.document.getElementById(
# 'DataNotFound').getElementsByTagName('p')
# if flag.length > 0:
# flag[0].innerHTML = 't'
def get_flag(self):
flag = self.document.getElementById(
'DataNotFound')
if flag:
flag = flag.getElementsByTagName('p')
if flag.length > 0:
return flag[0].innerHTML
return 't'
def visible(self):
self.__ie.Visible = 1 - self.__ie.Visible
def wait(self):
print(u'等待中')
while self.__ie.Busy or self.__ie.ReadyState != 4:
sleep(1)
def nwait(self):
while not self.__ie.Busy or self.__ie.ReadyState == 4:
sleep(1)
def quit(self):
print(u'退出')
self.main_window.destroy()
self.__ie.quit()
def auto(self):
result_info = self.result.readlines()
usr_info = self.usr.readlines()
while usr_info:
usr_name, usr_pwd = usr_info.pop().split(',')
if self.login(usr_name, usr_pwd) == 0:
continue
while result_info:
line = result_info.pop()
print(u'录入' + line[:-1].decode('gbk'))
info_name, info_id, info_phone, info_city = line.split(',')
info_name = info_name.decode('gbk')
info_city = info_city[:-1].decode('gbk')
if info_city in self.city_name:
# if True:
if self.input(info_name, info_id, info_phone):
result_info.append(line)
break
else:
break
print(u'录入完成')
self.result.close()
self.usr.close()
if(len(result_info) > 0):
print(u'账号已用完，未录入信息在result.txt中')
result = open(self.result_file, 'w')
for line in result_info:
result.write(line)
result.close()
def input(self, info_name, info_id, info_phone):
print(u'输入信息')
tmp_doc = self.document.getElementById(
'BusinessShowDiv').contentWindow.document
tmp_doc.getElementById('txtIDCardNO').value = info_id
tmp_doc.getElementById('txtMobile').value = info_phone
tmp_doc.getElementById('txtCustomerName').value = info_name
tmp_doc.getElementById('btnAdd').click()
sleep(2)
self.wait()
tmp = tmp_doc.getElementById(
'DataNotFound').getElementsByTagName('p')
while tmp.length == 0:
tmp = tmp_doc.getElementById(
'DataNotFound').getElementsByTagName('p')
if tmp.length > 0:
tmp = tmp[0].innerHTML
if u'该客户可提交贷款' in tmp:
print(u'可提交贷款')
tmp_button = tmp_doc.getElementsByTagName('button')
tmp_button[0].click()
sleep(1)
tmp_button = tmp_doc.getElementsByTagName('button')
tmp_button[0].click()
sleep(2)
self.wait()
tmp_doc.getElementById('btnContinuance').click()
self.wait()
return 0
if u'上限' in tmp:
print(u'推荐数量达到上限')
return 1
tmp_button = tmp_doc.getElementsByTagName('button')
# while tmp_button.length == 0:
# tmp_button = tmp_doc.getElementsByTagName('button')
tmp_button[0].click()
return 0
def get_nodes(self, parent_node, tag):
"""
>>> coldiv=GetNodes(body,"div")
"""
child_nodes = []
for childNode in parent_node.getElementsByTagName(tag):
child_nodes.append(childNode)
return child_nodes
def node_by_attr(self, nodes, nodeattr, nodeval):
"""
>>> div_id_editor=node_by_attr(coldiv,"id","editor_ifr")
"""
for node in nodes:
if str(node.getAttribute(nodeattr)) == nodeval:
return node
return None
def set_node(self, node, val):
node.innerHTML = val
class Wcontent:
def __init__(self):
self.ie = DispatchEx('InternetExplorer.Application')
self.ie.Visible = 1
time.sleep(0.5)
def openFullPage(self,url):
self.ie.Navigate(url)
while self.ie.Busy:
time.sleep(0.5)
print self.ie.LocationName, self.ie.LocationURL
def openXHRFull(self,url):
'''
get complete page for html
for XHR content page
'''
self.ie.Navigate(url)
while self.ie.Busy:
time.sleep(0.1)
print self.ie.LocationName, self.ie.LocationURL
t0 = time.time()
while self.currentHtml().find('page=')==-1:
self.ie.Document.parentWindow.execScript('window.scrollTo(0,document.body.scrollHeight-500)')
time.sleep(0.1)
if time.time() - t0 > 5:
self.ie.refresh()
def initSet(self):
'''init target/done id set from file'''
self.preUrl = 'http://weibo.com/u/'
self.data_dir = 'F:\\IS10\\grad\\data\\'
self.uid_set = set()
self.done_content_set = set()
self.loadIdFilesToSet(['collected.ids'],'done.content.ids')
def backupDoneSet(self):
'''save done id set to file'''
print 'saving done set to file'
self.bf = open(self.data_dir+'done.content.ids','a')
self.bf.write(' '+' '.join(str(ele) for ele in self.done_content_set))
self.bf.close()
def currentHtml(self):
return self.ie.Document.body.outerHTML
def getPageText(self,url):
'''
return plain Weibo text from given url
using div=WB_text to filter
'''
self.openXHRFull(url)
html = self.currentHtml()
txt = re.compile(r'WB_text(.*?)div').findall(html)
if txt:
for i,line in enumerate(txt):
txt[i] = ''.join(re.compile(r'>(.*?)<').findall(line)).split('//')[0]
return '\n'.join(txt)
else:
return ''
def getIdContentUrl(self,uid,page=1):
return ''.join([self.preUrl,str(uid),'?page=',str(page)])
def pageCnts(self,html):
'''return Weibo owner nick name and counts'''
fname = re.compile(r'fname=(.*?)&').search(html)
if fname != None:
wb_cnt = re.compile(r'tagweibo[^>]*>[^>]*>(\d+)<').search(html).group(1)
fans_cnt = re.compile(r'tagfans[^>]*>[^>]*>(\d+)<').search(html).group(1)
follow_cnt = re.compile(r'tagfollow[^>]*>[^>]*>(\d+)<').search(html).group(1)
return fname.group(1),follow_cnt,fans_cnt,wb_cnt
'''
To e.weibo & media.weibo page cnts
fname = re.compile(r'title_big..[\n](.*?)[\n]').search(html)
if fname != None:
cseg = re.compile(r'strong>(\d+)<').findall(html)
self.preUrl = 'http://e.weibo.com/'
return '#e#'+fname.group(1),cseg[0],cseg[2],cseg[4]
'''
#retry
return '##',0,0,0
def getIdContent(self,uid):
'''
Main portal to content crawling over uid
allow controlling by wb_cnt of given uid
'''
url = self.getIdContentUrl(uid)
self.openFullPage(url)
if self.ie.LocationURL.find('media.weibo')==7 or self.ie.LocationURL.find('e.weibo')==7:
return (self.ie.LocationName,0,0,0),''
html = self.currentHtml()
#id_cnts = (fname,follow_cnt,fans_cnt,wb_cnt)
id_cnts = self.pageCnts(html)
print id_cnts
wb_cnt = id_cnts[3]
if int(wb_cnt) < 45:
return (id_cnts,'')
if int(wb_cnt) > 900:
#self.getIdOriContent(uid,id_cnts)
wb_cnt = 2000
content = ''
for page in range(1,int(wb_cnt)/45+2):
url = self.getIdContentUrl(uid,page)
#page_content = self.browser.get_text('class=WB_feed')
page_content = self.getPageText(url)
content = content + page_content
return (id_cnts,content)
def getIdOriContentUrl(self,uid,page=1):
return ''.join([self.preUrl,str(uid),'?ori=1','&page=',str(page)])
def getIdOriContent(self,uid,id_cnts):
'''
When the user is too noisy
crawler her original content pages
'''
url = self.getIdOriContentUrl(uid)
self.openXHRFull(url)
time.sleep(1)
html = self.currentHtml()
page = self.getMaxPage(html)
print page
content = self.getPageText(url)
for page in range(2,int(page)+1):
url = self.getIdOriContentUrl(uid,page)
page_content = self.getPageText(url)
content = content + page_content
return (id_cnts,content)
def getMaxPage(self,html):
maxi = re.compile(r'page=(/d+)').search(html)
if maxi:
page = maxi.group(0)
else:
page = 1
return page
def loadIdFilesToSet(self,uid_files,done_file):
uid_file_paths = [self.data_dir+f for f in uid_files]
done_file_hdl = open(d+done_file,'r')
print 'Time before load to set:',str(time.ctime())
for file_path in uid_file_paths:
file_handle = open(file_path,'r')
t_set = set()
for ele in file_handle.read().split(' '):
t_set.add(ele)
self.uid_set = self.uid_set.union(t_set)
file_handle.close()
for ele in done_file_hdl.read().split(' '):
self.done_content_set.add(ele)
done_file_hdl.close()
print 'Time after loaded to set:',str(time.ctime())
print len(self.uid_set)
def writeIdContent(self,uid'):
fh = open(self.data_dir+str(uid),'w')
cnt,text = self.getIdContent(uid)
fh.writelines(cnt[0].encode('utf-8')+' '+' '.join(str(c) for c in cnt[1:]))
fh.write(text.encode('utf-8'))
fh.close()
self.done_content_set.add(uid)#''.join([str(uid),'||',cnt[0].encode('utf-8')]))
class IECom():
"""open IE and use IE to auto lock bill"""
url = 'http://ifeidai.com/EDaiWeb/AgentLoan/Login'
fast_ur = 'http://ifeidai.com/EDaiWeb/AgentLogon/MySaleApplyBill?Length=10'
result_file = 'result.txt'
usr_file = 'login.txt'
city_name = [
u'青岛', u'济南', u'唐山', u'无锡', u'上海', u'扬州', u'深圳', u'武汉', u'石家庄', u'北京',
u'南昌', u'海口', u'惠州', u'成都', u'南京', u'广州', u'中山', u'郑州', u'南宁', u'佛山',
u'长沙', u'重庆', u'西安', u'贵阳', u'厦门', u'青岛市', u'济南市', u'唐山市', u'无锡市',
u'上海市', u'扬州市', u'深圳市', u'武汉市', u'石家庄市', u'北京市', u'南昌市', u'海口市',
u'惠州市', u'成都市', u'南京市', u'广州市', u'中山市', u'郑州市', u'南宁市', u'佛山市', u'长沙市',
u'重庆市', u'西安市', u'贵阳市', u'厦门市'
]
def __init__(self):
'''
init open IE
'''
code = getnode()
self.code_license = code % 3721 + code * 9997 % 997
self.root = cur_file_dir()
try:
lic = open(self.root + os.sep + 'lic')
line = lic.readline()
lic.close()
except:
line = u'请输入激活码'
self.main_window = tk.Tk()
self.main_window.minsize(300, 300)
self.main_window.title(u'锁单神器')
self.main_window.protocol('WM_DELETE_WINDOW', self.quit)
win = tk.Frame()
self.entry_var = tk.StringVar()
self.entry_var.set(line)
self.result_var = tk.StringVar()
self.result_var.set(u'信息文件路径： ' + self.result_file)
self.mac_var = tk.StringVar()
self.mac_var.set(u'机器码: ' + str(code))
self.usr_var = tk.StringVar()
self.usr_var.set(u'账户文件路径： ' + self.usr_file)
self.info_str = u'''
锁单神器v7：自动将信息文件中的数据锁到账户文件中的账户中。
信息文件格式：姓名,身份证号,手机号,城市名
例如：张三,100000000000000000,11100000000,北京
账户文件格式：账户名,密码
例如：100000000000000000,a111111
Copyright (C) 2015 Yeoman Zhuang
'''
self.info_var = tk.StringVar()
self.info_var.set(self.info_str)
win.pack()
label_mac = tk.Entry(win, textvariable=self.mac_var)
label_mac.pack(side=tk.TOP)
self.entry_en = tk.Entry(win, textvariable=self.entry_var)
self.entry_en.pack(side=tk.TOP)
label_result = tk.Label(win, textvariable=self.result_var)
label_result.pack(side=tk.TOP)
label_usr = tk.Label(win, textvariable=self.usr_var)
label_usr.pack(side=tk.TOP)
label_info = tk.Label(win, textvariable=self.info_var)
label_info.pack(side=tk.BOTTOM)
button_act = tk.Button(win, text=u'激活', command=self.act)
button_act.pack(side=tk.LEFT)
self.button_set_result = tk.Button(win,
text=u'打开信息文件',
command=self.set_result,
state=tk.DISABLED)
self.button_set_result.pack(side=tk.LEFT)
self.button_set_usr = tk.Button(win,
text=u'打开账户文件',
command=self.set_info,
state=tk.DISABLED)
self.button_set_usr.pack(side=tk.LEFT)
# self.button_open = tk.Button(
# win, text=u'打开IE', command=self.open, state=tk.DISABLED)
# self.button_open.pack(side=tk.LEFT)
self.button_start = tk.Button(win,
text=u'开始锁单',
command=self.open_auto,
state=tk.DISABLED)
self.button_start.pack(side=tk.LEFT)
win.mainloop()
def open_auto(self):
self.open()
self.auto()
def act(self):
if str(self.code_license) == self.entry_en.get():
self.button_start['state'] = tk.NORMAL
self.button_set_usr['state'] = tk.NORMAL
self.button_set_result['state'] = tk.NORMAL
# self.button_open['state'] = tk.NORMAL
lic = open(self.root + os.sep + 'lic', 'w')
lic.write(str(self.code_license))
lic.close()
self.entry_var.set(u'激活成功')
else:
self.entry_var.set(u'激活失败，请重新输入')
def set_result(self):
self.result_file = tkFileDialog.askopenfilename(initialdir='D:/')
self.result_var.set(u'信息文件路径： ' + self.result_file)
def set_info(self):
self.usr_file = tkFileDialog.askopenfilename(initialdir='D:/')
self.usr_var.set(u'账户文件路径： ' + self.usr_file)
def open(self):
'''
open IE
'''
print(u'打开登陆账号文件和信息文件')
self.result = open(self.result_file)
self.usr = open(self.usr_file)
print(u'绑定IE')
self.__ie = DispatchEx('InternetExplorer.Application')
self.__ie.visible = 1
self.__ie.navigate(self.url)
self.wait()
self.document = self.__ie.Document
def login(self, usr_name, usr_pwd):
'''
login success return 1
'''
print('Login:'******'输入账户密码')
while True:
try:
self.document.Body.getElementsByTagName(
'input')[0].value = usr_name
self.document.getElementsByTagName('input')[1].value = usr_pwd
while (self.identify() == ''):
pass
self.document.getElementsByTagName(
'input')[2].value = self.identify()
self.document.getElementById('btnSave').click()
self.wait()
break
except:
self.__ie.refresh()
sleep(4)
self.wait()
flag = 't'
if flag == 't':
print(u'登陆成功！')
sleep(2)
self.__ie.navigate(self.fast_ur)
self.wait()
self.document.getElementById('FastRecommend').click()
print(u'开始快速推荐')
self.wait()
return 1
else:
print('Error: Can not log in ' + flag)
print('usr_name = ' + usr_name)
print('usr_pwd = ' + usr_pwd)
return 0
def identify(self):
print(u'获得验证码')
return self.document.getElementById('ValidateValue').value
# def reset_flag(self):
# flag = self.document.getElementById(
# 'DataNotFound').getElementsByTagName('p')
# if flag.length > 0:
# flag[0].innerHTML = 't'
def get_flag(self):
flag = self.document.getElementById('DataNotFound')
if flag:
flag = flag.getElementsByTagName('p')
if flag.length > 0:
return flag[0].innerHTML
return 't'
def visible(self):
self.__ie.Visible = 1 - self.__ie.Visible
def wait(self):
print(u'等待中')
while self.__ie.Busy or self.__ie.ReadyState != 4:
sleep(1)
def nwait(self):
while not self.__ie.Busy or self.__ie.ReadyState == 4:
sleep(1)
def quit(self):
print(u'退出')
self.main_window.destroy()
self.__ie.quit()
def auto(self):
result_info = self.result.readlines()
usr_info = self.usr.readlines()
while usr_info:
usr_name, usr_pwd = usr_info.pop().split(',')
if self.login(usr_name, usr_pwd) == 0:
continue
while result_info:
line = result_info.pop()
print(u'录入' + line[:-1].decode('gbk'))
info_name, info_id, info_phone, info_city = line.split(',')
info_name = info_name.decode('gbk')
info_city = info_city[:-1].decode('gbk')
if info_city in self.city_name:
# if True:
if self.input(info_name, info_id, info_phone):
result_info.append(line)
break
else:
break
print(u'录入完成')
self.result.close()
self.usr.close()
if (len(result_info) > 0):
print(u'账号已用完，未录入信息在result.txt中')
result = open(self.result_file, 'w')
for line in result_info:
result.write(line)
result.close()
def input(self, info_name, info_id, info_phone):
print(u'输入信息')
tmp_doc = self.document.getElementById(
'BusinessShowDiv').contentWindow.document
tmp_doc.getElementById('txtIDCardNO').value = info_id
tmp_doc.getElementById('txtMobile').value = info_phone
tmp_doc.getElementById('txtCustomerName').value = info_name
tmp_doc.getElementById('btnAdd').click()
sleep(2)
self.wait()
tmp = tmp_doc.getElementById('DataNotFound').getElementsByTagName('p')
while tmp.length == 0:
tmp = tmp_doc.getElementById('DataNotFound').getElementsByTagName(
'p')
if tmp.length > 0:
tmp = tmp[0].innerHTML
if u'该客户可提交贷款' in tmp:
print(u'可提交贷款')
tmp_button = tmp_doc.getElementsByTagName('button')
tmp_button[0].click()
sleep(1)
tmp_button = tmp_doc.getElementsByTagName('button')
tmp_button[0].click()
sleep(2)
self.wait()
tmp_doc.getElementById('btnContinuance').click()
self.wait()
return 0
if u'上限' in tmp:
print(u'推荐数量达到上限')
return 1
tmp_button = tmp_doc.getElementsByTagName('button')
# while tmp_button.length == 0:
# tmp_button = tmp_doc.getElementsByTagName('button')
tmp_button[0].click()
return 0
def get_nodes(self, parent_node, tag):
"""
>>> coldiv=GetNodes(body,"div")
"""
child_nodes = []
for childNode in parent_node.getElementsByTagName(tag):
child_nodes.append(childNode)
return child_nodes
def node_by_attr(self, nodes, nodeattr, nodeval):
"""
>>> div_id_editor=node_by_attr(coldiv,"id","editor_ifr")
"""
for node in nodes:
if str(node.getAttribute(nodeattr)) == nodeval:
return node
return None
def set_node(self, node, val):
node.innerHTML = val
