def get(self):
try:
arguments = self.request.arguments
for key in arguments:
arguments[key] = arguments[key][-1].decode()
if WebDriverPool().queueSize() > 0:
msg = WebDriverPool().getOneDriverNoWait().deal(arguments)
else:
# 现在没有可用driver,所以暂时不下发任务
msg = {
"state": 701,
"errMsg": "当前无可用Webdriver实例,根据其他参数查看当前任务的工作状态。"
}
jsonStr = json.dumps(msg, ensure_ascii=False, cls=CJsonEncoder)
self.write(jsonStr)
return
except:
if 'Empty' in traceback.format_exc():
self.write({'state': 701, 'errMsg': "系统繁忙！无空闲实例,请稍后再试"})
traceback.print_exc()
return
self.write({
'state': 799,
'errMsg': "异常\n" + traceback.format_exc()
})
traceback.print_exc()
return
def changeModel(request):
timeStart = time.time()
try:
if request.method == 'GET':
arguments = dict(request.GET)
for arg in arguments:
if type(arguments[arg]) == type([]):
arguments[arg] = arguments[arg][0]
elif request.method == 'POST':
arguments = json.loads(request.body.decode())
else:
return "不支持除了GET和POST之外的请求方式"
# 将每个参数的队列转换成单个结果
if WebDriverPool().queueSize() > 0:
print("输入参数", arguments)
msg = WebDriverPool().getOneDriverNoWait().deal(arguments)
else:
# 现在没有可用driver,所以暂时不下发任务
msg = {"state": 701,
"errMsg": "当前无可用Webdriver实例,通过参数stateMsg当前任务的工作状态。",
"stateMsg": WebDriverPool().getDriverState(),
"missionStateMsg": "最近任务的完成情况{}/{}".format(misFinishNum, misTotalNum),
}
msg['timeTotal'] = time.time() - timeStart
jsonStr = json.dumps(msg, ensure_ascii=False, cls=CJsonEncoder)
return HttpResponse(jsonStr)
except:
if 'Empty' in traceback.format_exc():
jsonStr = json.dumps({'state': 701, 'errMsg': "系统繁忙！无空闲实例,请稍后再试"}, ensure_ascii=False, cls=CJsonEncoder)
traceback.print_exc()
return HttpResponse(jsonStr)
jsonStr = json.dumps({'state': 799, 'errMsg': "异常\n" + traceback.format_exc()}, ensure_ascii=False,
cls=CJsonEncoder)
traceback.print_exc()
return HttpResponse(jsonStr)
def getMission(redis_key, dBean, num):
try:
mission = r15.lpop(redis_key)
if mission is not None:
data = json.loads(mission.decode())
WebDriverPool(dBean=dBean, num=num,
headless=False).getOneDriver().deal(data)
else:
print("队列为空，间隔一段时间再尝试")
time.sleep(1)
except DuplicateKeyError:
mylog.warning("存储主键重复,跳过")
except:
traceback.print_exc()
def changeModelBatch(request):
global misFinishNum, misTotalNum
timeStart = time.time()
try:
if request.method == 'GET':
arguments = dict(request.GET)
for arg in arguments:
if type(arguments[arg]) == type([]):
arguments[arg] = arguments[arg][0]
elif request.method == 'POST':
arguments = json.loads(request.body.decode())
else:
return "不支持除了GET和POST之外的请求方式"
# 将每个参数的队列转换成单个结果
if WebDriverPool().queueSize() > 0:
threads = []
for itemData in arguments['data']:
if 'result' in itemData:
misTotalNum = len(itemData['result'])
for item in itemData['result']:
# msg = WebDriverPool().getOneDriver().deal(item)
driver = WebDriverPool().getOneDriver()
print("输入参数", item)
t = MyThread(driver, item)
threads.append(t)
t.start()
else:
misTotalNum = len(arguments['data'])
driver = WebDriverPool().getOneDriver()
print("输入参数", itemData)
t = MyThread(driver, itemData)
threads.append(t)
t.start()
# 阻塞全部完成
for threadNow in threads:
threadNow.join()
# 组装最终结果
msg = {'result': [], 'errMsg': "成功！结果请遍历result查看", 'state': 200}
for threadNow in threads:
msg['result'].append(threadNow.get_result())
msg['timeTotal'] = time.time() - timeStart
jsonStr = json.dumps(msg, ensure_ascii=False, cls=CJsonEncoder)
return HttpResponse(jsonStr)
else:
# 现在没有可用driver,所以暂时不下发任务
msg = {"state": 701,
"errMsg": "当前无可用Webdriver实例,根据其他参数查看当前任务的工作状态。",
"stateMsg": WebDriverPool().getDriverState(),
"missionStateMsg": "最近任务的完成情况{}/{}".format(misFinishNum, misTotalNum)
}
jsonStr = json.dumps(msg, ensure_ascii=False, cls=CJsonEncoder)
return HttpResponse(jsonStr)
except:
if 'Empty' in traceback.format_exc():
jsonStr = json.dumps({'state': 701,
'errMsg': "系统繁忙！无空闲实例,请稍后再试",
"missionStateMsg": "最近任务的完成情况{}/{}".format(misFinishNum, misTotalNum)
}, ensure_ascii=False, cls=CJsonEncoder)
traceback.print_exc()
return HttpResponse(jsonStr)
jsonStr = json.dumps({'state': 799, 'errMsg': "异常\n" + traceback.format_exc()}, ensure_ascii=False,
cls=CJsonEncoder)
traceback.print_exc()
return HttpResponse(jsonStr)
finally:
misFinishNum = 0
misTotalNum = 0
if __name__ == '__main__':
# # 1
# fpdm = "4403181130"
# fphm = "27671246"
# kprq = "20180920"
# kjje = "351.69"
# #2
# fpdm = "4403181130"
# fphm = "27671246"
# kprq = "20180920"
# kjje = "351.69"
# 3
fpdm = "1100182130"
fphm = "15024752"
kprq = "20180614"
kjje = "18679.25"
jym = "123"
# # 4
# fpdm = "011001800211"
# fphm = "28519832"
# kprq = "20190109"
# kjje = "1020"
# jym = "07842591941327323876"
data = {'fpdm': fpdm, 'fphm': fphm, 'kprq': kprq, 'kjje': kjje, 'jym': jym}
print(
WebDriverPool(dBean=fapiaoImpl, num=1,
headless=False).getOneDriver().deal(data))
'account': 'ytbl0011',
'keyword': 'ytbl0011aDmin'
}, {
'account': 'ytbl0012',
'keyword': 'ytbl0012aDmin'
}, {
'account': 'ytbl0013',
'keyword': 'ytbl0013aDmin'
}, {
'account': 'ytbl0014',
'keyword': 'ytbl0014aDmin'
}]
testAccount = [{'account': 'ytbl0011', 'keyword': 'ytbl0011aDmin'}]
if 'inux' in platform.system():
WebDriverPool(dBean=zhongDengDengJiImpl,
num=onlineAccount,
headless=True)
else:
WebDriverPool(dBean=zhongDengDengJiImpl,
num=testAccount,
headless=True)
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'django_start.settings')
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?") from exc
args = [sys.argv[0], "runserver", "0.0.0.0:9090", "--noreload"]
# http://39.108.188.34:9088/spider/fapiao.go?fpdm=4403181130&fphm=27671246&kprq=20180920&kjje=351.69
# http://39.108.188.34:9088/spider/fapiao.go?fpdm=011001800211&fphm=28519832&kprq=20190109&kjje=1020&jym=07842591941327323876
# http://39.108.188.34:9088/spider/fapiao.go?fpdm=044031800211&fphm=28519832&kprq=20190215&kjje=42.93&jym=7006849628077367328
# http://localhost:9088/spider/fapiao.go?fpdm=011001800104&fphm=55518592&kprq=20190304&kjje=1020&jym=67547104442692182289
from django.conf.urls import url
from webdriver_service.django_start.django_start.urls import urlpatterns
from webdriver_service.django_start.django_start.view import changeModel
if __name__ == '__main__':
# 演示时使用下面的代码
# WebDriverPool(dBean=fapiaoImpl, num=1, headless=False)
print("启动时间startTime", datetime.datetime.now())
if 'inux' in platform.system():
WebDriverPool(dBean=fapiaoImpl, num=10, headless=True)
else:
WebDriverPool(dBean=fapiaoImpl, num=1, headless=False)
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'django_start.settings')
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?") from exc
args = [sys.argv[0], "runserver", "0.0.0.0:9088", "--noreload"]
print(args)
execute_from_command_line(args)
# 更换图片再来一次
self.click100_by_tag(self.driver.find_element_by_id('img'))
def setAttribute(self, elementObj, attributeName, value):
# 封装设置页面对象的属性值的方法
# 调用JavaScript代码修改页面元素的属性值，arguments[0]－［2］分别会用后面的
# element、attributeName和value参数值进行替换，并执行该JavaScript代码
self.driver.execute_script(
"arguments[0].setAttribute\
(arguments[1],arguments[2])", elementObj, attributeName, value)
def click100_by_tag(self, tag):
# 保证可以点击到的方法
time.sleep(0.01)
action_chains = ActionChains(self.driver)
action_chains.move_to_element_with_offset(tag, 10, 10)
action_chains.click()
action_chains.perform()
print("完成100TAG按键操作")
if __name__ == '__main__':
pool = WebDriverPool(dBean=zhongDengImpl, num=1, headless=False)
impl = WebDriverPool.getOneDriver(pool)
dictNow = impl.deal({
'companyName': "上海中建航建筑工业发展有限",
"afterDate": "20170501"
})
print(json.dumps(dictNow, cls=CJsonEncoder, ensure_ascii=False))
# impl.deal({'companyName': "深圳银泰保理有限公司"})