def mdr_profile():
isLogin = False
while not isLogin:
isLogin = login2.login()
for qrow in utils.MDR_Report_Query(July_cal()):
mdr_parse.mdr_import_report(qrow)
def gather_report_total(_timedict):
t = _timedict
t_start = t['beginTime']
t_end = t['endTime']
logincounter = 10
while logincounter:
print logincounter
if login():
print "ok"
break
else:
print "clear"
cookieManager.clear()
logincounter = logincounter -1
#首先获取记录条数总记录
totalparamsx = {"funcID":"QUERY_DATA","userID":78919,"operations":[{"actionName":"query","operationDatas":[{"ENTRY_START_DATE_2401":t_start,"ENTRY_END_DATE_2401":t_end,"ACTIVE_CONSTITUENT_2430":"","TRADENAME_CN_2431":"","DRUGNAME_CN_FORM_2431":"","DOSEFORM_2431":"","DRUG_STATUS_2431":"","APPROVAL_NO_2431":"","REPORT_ID_2430":"","PICKER_UNIT_NAME_2430":"","START_APPLY_DATE_2430":"","END_APPLY_DATE_2430":"","listid":"2430_1","start":0,"limit":10}]}]}
_idcdata = send_post_json(Psur_url, totalparamsx)
#print _idcdata
usa = 0
try:
_data = json.loads(_idcdata)
usa = _data['ResponseMessage']['operations'][0]['pageTotal']
#TypeError: expected string or buffer
except (IndexError, KeyError, ValueError,TypeError), exception:
print "gather_report_total:",exception
print "IN json Data:", _idcdata
def mdr_profile():
isLogin = False
while not isLogin:
isLogin = login2.login()
for qrow in utils.MDR_Report_Query(July_cal()):
mdr_parse.mdr_import_report(qrow)
def main3(somed1, somed2, s3):
#主逻辑部分
print u"正在login..."
logincounter = 6
while logincounter:
print logincounter
loginTag = login2.login()
print u"login Tag(zoe3):",loginTag
if loginTag:
break
elif logincounter == 1:
#
print u"6次login(main3)失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter -1
time.sleep(8)
#
print u"已经登录成功"
#获得所有日期参数列表
querydays = right2(int(somed1), int(somed2), int(s3))
#读取B平台参数数据下记录报告总数
days_local = harvestdaystomonthlocal(querydays)
#辅助输出信息，供使用者阅读
#{
print u"本平台数据:",
for il in days_local:
print il
print u"正在从国家站点读取数据........:",
#}
#读取A平台参数数据下记录报告总数
days_remote = harvestdaystomonthremote(querydays)
#辅助输出信息，供使用者阅读
#{
print u"国家站点数据:",
for ir in days_remote:
print ir
#}
#取得参数长度
dl_len = len(days_local)
for i in range(dl_len):
#以A数据参数为基准参数
someday = days_remote[i][0]
date_remote = days_remote[i][1]
date_local = days_local[i][1]
date_check = int(date_remote)-int(date_local)
if date_check ==0:
print u"%s days Matched" % someday
else:
#
print "Date_Check NOT MATCH:",date_check
print u"%s days Not Match" % someday
print u"匹配开始......."
main_adr.crawler_today2(someday)
print u"核对结束"
def main3(somed1, somed2, s3):
#主逻辑部分
print u"正在login..."
logincounter = 6
while logincounter:
print logincounter
loginTag = login2.login()
print u"login Tag(zoe3):", loginTag
if loginTag:
break
elif logincounter == 1:
#
print u"6次login(main3)失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter - 1
time.sleep(8)
#
print u"已经登录成功"
#获得所有日期参数列表
querydays = right2(int(somed1), int(somed2), int(s3))
#读取B平台参数数据下记录报告总数
days_local = harvestdaystomonthlocal(querydays)
#辅助输出信息，供使用者阅读
#{
print u"本平台数据:",
for il in days_local:
print il
print u"正在从国家站点读取数据........:",
#}
#读取A平台参数数据下记录报告总数
days_remote = harvestdaystomonthremote(querydays)
#辅助输出信息，供使用者阅读
#{
print u"国家站点数据:",
for ir in days_remote:
print ir
#}
#取得参数长度
dl_len = len(days_local)
for i in range(dl_len):
#以A数据参数为基准参数
someday = days_remote[i][0]
date_remote = days_remote[i][1]
date_local = days_local[i][1]
date_check = int(date_remote) - int(date_local)
if date_check == 0:
print u"%s days Matched" % someday
else:
#
print "Date_Check NOT MATCH:", date_check
print u"%s days Not Match" % someday
print u"匹配开始......."
main_adr.crawler_today2(someday)
print u"核对结束"
def gather_report_total(_timedict):
t = _timedict
t_start = t['beginTime']
t_end = t['endTime']
logincounter = 10
while logincounter:
print logincounter
if login():
print "ok"
break
else:
print "clear"
cookieManager.clear()
logincounter = logincounter - 1
#首先获取记录条数总记录
totalparamsx = {
"funcID":
"QUERY_DATA",
"userID":
78919,
"operations": [{
"actionName":
"query",
"operationDatas": [{
"ENTRY_START_DATE_2401": t_start,
"ENTRY_END_DATE_2401": t_end,
"ACTIVE_CONSTITUENT_2430": "",
"TRADENAME_CN_2431": "",
"DRUGNAME_CN_FORM_2431": "",
"DOSEFORM_2431": "",
"DRUG_STATUS_2431": "",
"APPROVAL_NO_2431": "",
"REPORT_ID_2430": "",
"PICKER_UNIT_NAME_2430": "",
"START_APPLY_DATE_2430": "",
"END_APPLY_DATE_2430": "",
"listid": "2430_1",
"start": 0,
"limit": 10
}]
}]
}
_idcdata = send_post_json(Psur_url, totalparamsx)
#print _idcdata
usa = 0
try:
_data = json.loads(_idcdata)
usa = _data['ResponseMessage']['operations'][0]['pageTotal']
#TypeError: expected string or buffer
except (IndexError, KeyError, ValueError, TypeError), exception:
print "gather_report_total:", exception
print "IN json Data:", _idcdata
def crawler_last_month(username, password):
#抓取上月的
logging.info(u"开始抓取上月数据！")
logincounter = 6
while logincounter:
print logincounter
loginTag = login2.login()
print u"login Tag(last_month):",loginTag
if loginTag:
break
elif logincounter == 1:
#
print u"6次login(last_month)失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter -1
time.sleep(8)
page_idx = 1
step = 100
cur_time = datetime.datetime.today()
year = cur_time.year
month = cur_time.month - 1
if month == 0 :
month = 12
year -= 1
start_time = datetime.datetime(year, month, 1)
end_time = datetime.datetime( cur_time.year, cur_time.month, 1 ) - datetime.timedelta(days=1)
start_time = start_time.strftime("%a %b %d %Y 00:00:00 GMT+0800")
end_time = end_time.strftime("%a %b %d %Y 00:00:00 GMT+0800")
filters = {
"beginTime": start_time,
"endTime": end_time
}
logging.info(u"start time:%s \t end time:%s" % (start_time, end_time))
start_pos = 0
for res_page in login_new_adr.get_page_list(filters,start_pos, step):# 一次一页，一页10条
print u"page: ",page_idx
page_idx += 1
business_idx = 1
for qrow in res_page: # 一次一个案例
print u"business idx: %s of %s " % (business_idx , step)
business_idx += 1
adr_adr.import_from_html(qrow)
def crawler_appraise(year,beginMonth, endMonth):
logincounter = 0
loginOk = False
while logincounter < 6 and not loginOk :
loginOk = login2.login()
logincounter = logincounter + 1
time.sleep(8)
beginDay = datetime.date(int(year),int(beginMonth),1)
while beginDay.month <= int(endMonth) and beginDay.year == int(year) :
mdr_update_appraise(beginDay.strftime("%Y-%m-%d"))
beginDay = beginDay + datetime.timedelta(days=1)
def extradown():
logincounter = 10
while logincounter:
print logincounter
if login():
print "ok"
break
else:
print "clear"
cookieManager.clear()
logincounter = logincounter - 1
psur.myautodown2()
def crawler_appraise(year, beginMonth, endMonth):
logincounter = 0
loginOk = False
while logincounter < 6 and not loginOk:
loginOk = login2.login()
logincounter = logincounter + 1
time.sleep(8)
beginDay = datetime.date(int(year), int(beginMonth), 1)
while beginDay.month <= int(endMonth) and beginDay.year == int(year):
mdr_update_appraise(beginDay.strftime("%Y-%m-%d"))
beginDay = beginDay + datetime.timedelta(days=1)
def extradown():
logincounter = 10
while logincounter:
print logincounter
if login():
print "ok"
break
else:
print "clear"
cookieManager.clear()
logincounter = logincounter -1
psur.myautodown2()
def crawler_last_month(username, password):
#抓取上月的
logging.info(u"开始抓取上月数据！")
logincounter = 6
while logincounter:
print logincounter
loginTag = login2.login()
print u"login Tag(last_month):", loginTag
if loginTag:
break
elif logincounter == 1:
#
print u"6次login(last_month)失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter - 1
time.sleep(8)
page_idx = 1
step = 100
cur_time = datetime.datetime.today()
year = cur_time.year
month = cur_time.month - 1
if month == 0:
month = 12
year -= 1
start_time = datetime.datetime(year, month, 1)
end_time = datetime.datetime(cur_time.year, cur_time.month,
1) - datetime.timedelta(days=1)
start_time = start_time.strftime("%a %b %d %Y 00:00:00 GMT+0800")
end_time = end_time.strftime("%a %b %d %Y 00:00:00 GMT+0800")
filters = {"beginTime": start_time, "endTime": end_time}
logging.info(u"start time:%s \t end time:%s" % (start_time, end_time))
start_pos = 0
for res_page in login_new_adr.get_page_list(filters, start_pos,
step): # 一次一页，一页10条
print u"page: ", page_idx
page_idx += 1
business_idx = 1
for qrow in res_page: # 一次一个案例
print u"business idx: %s of %s " % (business_idx, step)
business_idx += 1
adr_adr.import_from_html(qrow)
def crawler_by_time(username, password,start_date,end_date):
#开始抓取指定日期区间数据
logging.info("开始抓取指定日期区间数据！")
print u"正在login..."
logincounter = 6
while logincounter:
print logincounter
loginTag = login2.login()
print u"login Tag(zoe3):",loginTag
if loginTag:
break
elif logincounter == 1:
#
print u"6次login(main3)失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter -1
time.sleep(8)
page_idx = 1
step = 100
end_time = datetime.datetime.strptime(end_date,"%Y-%m-%d")
start_time = datetime.datetime.strptime(start_date,"%Y-%m-%d")
start_time = start_time.strftime("%a %b %d %Y 00:00:00 GMT+0800")
end_time = end_time.strftime("%a %b %d %Y 00:00:00 GMT+0800")
filters = {
"beginTime": start_time,
"endTime": end_time
}
logging.info("start time:%s \t end time:%s" % (start_time, end_time))
start_pos = 0
for res_page in login_new_adr.get_page_list(filters,start_pos, step):
print "page: ",page_idx
page_idx += 1
business_idx = 1
for qrow in res_page: # 一次一个案例
print "business idx: %s of %s " % (business_idx , step)
business_idx += 1
try:
delete_by_bianma(qrow["report_id2"]) #删除成功则添加
adr_adr.import_from_html(qrow)
except Exception as err :
print err
def crawler_by_time(username, password, start_date, end_date):
#开始抓取指定日期区间数据
logging.info("开始抓取指定日期区间数据！")
print u"正在login..."
logincounter = 6
while logincounter:
print logincounter
loginTag = login2.login()
print u"login Tag(zoe3):", loginTag
if loginTag:
break
elif logincounter == 1:
#
print u"6次login(main3)失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter - 1
time.sleep(8)
page_idx = 1
step = 100
end_time = datetime.datetime.strptime(end_date, "%Y-%m-%d")
start_time = datetime.datetime.strptime(start_date, "%Y-%m-%d")
start_time = start_time.strftime("%a %b %d %Y 00:00:00 GMT+0800")
end_time = end_time.strftime("%a %b %d %Y 00:00:00 GMT+0800")
filters = {"beginTime": start_time, "endTime": end_time}
logging.info("start time:%s \t end time:%s" % (start_time, end_time))
start_pos = 0
for res_page in login_new_adr.get_page_list(filters, start_pos, step):
print "page: ", page_idx
page_idx += 1
business_idx = 1
for qrow in res_page: # 一次一个案例
print "business idx: %s of %s " % (business_idx, step)
business_idx += 1
try:
delete_by_bianma(qrow["report_id2"]) #删除成功则添加
adr_adr.import_from_html(qrow)
except Exception as err:
print err
def crawler_first_half_year(username, password):
#抓取上半年度的
logging.info(u"开始抓取上半年度数据！")
logincounter = 6
while logincounter:
print logincounter
loginTag = login2.login()
print u"login Tag(first_half_year):",loginTag
if loginTag:
break
elif logincounter == 1:
#
print u"6次login(first_half_year)失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter -1
time.sleep(8)
page_idx = 1
step = 100
year = datetime.datetime.today().year
last_year_start = datetime.date(year,1,1)
last_year_end = datetime.date(year,6,30)
start_time = last_year_start.strftime("%a %b %d %Y 00:00:00 GMT+0800")
end_time = last_year_end.strftime("%a %b %d %Y 00:00:00 GMT+0800")
filters = {
"beginTime": start_time,
"endTime": end_time
}
logging.info(u"start time:%s \t end time:%s" % (start_time, end_time))
start_pos = 0
for res_page in login_new_adr.get_page_list(filters,start_pos, step):
print u"page: ",page_idx
page_idx += 1
business_idx = 1
for qrow in res_page:
print u"business idx: %s of %s " % (business_idx , step)
business_idx += 1
adr_adr.import_from_html(qrow)
def crawler_last_year(username, password):
#抓取上一年度的
logging.info(u"开始抓取上一年度数据！")
logincounter = 6
while logincounter:
print logincounter
loginTag = login2.login()
print u"login Tag(last_year):", loginTag
if loginTag:
break
elif logincounter == 1:
#
print u"6次login(last_year)失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter - 1
time.sleep(8)
page_idx = 1
step = 100
year = datetime.datetime.today().year - 1
last_year_start = datetime.date(year, 1, 1)
last_year_end = datetime.date(year, 12, 31)
start_time = last_year_start.strftime("%a %b %d %Y 00:00:00 GMT+0800 ")
end_time = last_year_end.strftime("%a %b %d %Y 00:00:00 GMT+0800 ")
filters = {"beginTime": start_time, "endTime": end_time}
logging.info(u"start time:%s \t end time:%s" % (start_time, end_time))
start_pos = 0
for res_page in login_new_adr.get_page_list(filters, start_pos,
step): # 一次一页，一页10条
print "page: ", page_idx
page_idx += 1
business_idx = 1
for qrow in res_page: # 一次一个案例
print u"business idx: %s of %s " % (business_idx, step)
business_idx += 1
adr_adr.import_from_html(qrow)
def crawler_first_half_year():
#抓取上半年度
logincounter = 6
while logincounter:
print logincounter
loginTag = login()
print u"login Tag(first_half_year):",loginTag
if loginTag:
break
elif logincounter == 1:
#
print u"6次login(first_half_year)失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter -1
time.sleep(8)
deletedance.dodelete(halfyear_before_cal())
report_query.MDR_Report_Query(halfyear_before_cal())
def adr_regulate(year, beginMonth, endMonth):
'''
校正adr数据
@year : 指定校正的年份
@beginMonth : 指定开始校正的月分
@endMonth : 指定结束校正的月份
'''
print u"正在login..."
logincounter = 0
loginOk = False
while logincounter < 6 and not loginOk :
loginOk = login2.login()
logincounter = logincounter + 1
time.sleep(8)
beginDay = datetime.date(int(year),int(beginMonth),1)
while beginDay.month <= int(endMonth) and beginDay.year == int(year) :
adr_regulate_by_day(beginDay.strftime("%Y-%m-%d"))
beginDay = beginDay + datetime.timedelta(days=1)
def crawler_anytime2(start, end):
#
logincounter = 6
while logincounter:
print logincounter
loginTag = login()
print u"login Tag(anytime2):",loginTag
if loginTag:
break
elif logincounter == 1:
#
print u"6次login(anytime2)失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter -1
time.sleep(1)
deletedance.dodelete(anytime2(start, end))
report_query.MDR_Report_Query(anytime2(start, end))
def crawler_today():
#抓取今天
logincounter = 6
while logincounter:
print logincounter
loginTag = login()
print u"login Tag(today):",loginTag
if loginTag:
break
elif logincounter == 1:
#
print u"6次login(today)失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter -1
time.sleep(8)
deletedance.dodelete(toady_cal())
report_query.MDR_Report_Query(toady_cal())
def crawler_today():
#抓取今天
logincounter = 6
while logincounter:
print logincounter
loginTag = login()
print u"login Tag(auto today):",loginTag
if loginTag:
break
elif logincounter == 1:
#
print u"6次login(auto today)失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter -1
time.sleep(8)
deletedance.dodelete(toady_cal())
report_query.MDR_Report_Query(toady_cal())
def crawler_current_week():
#抓取本周
logincounter = 6
while logincounter:
print logincounter
loginTag = login()
print u"login Tag(current_week):",loginTag
if loginTag:
break
elif logincounter == 1:
#
print u"6次login(current_week)失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter -1
time.sleep(8)
deletedance.dodelete(current_week_cal())
report_query.MDR_Report_Query(current_week_cal())
def crawler_today():
#
logging.info(u"开始抓取今天数据！")
logincounter = 6
while logincounter:
print logincounter
loginTag = login2.login()
print u"login Tag(ADR Today):",loginTag
if loginTag:
break
elif logincounter == 1:
#
print u"6次login(auto today)失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter -1
time.sleep(8)
page_idx = 1
step = 100
start_time = datetime.datetime.today()
end_time = start_time + datetime.timedelta(days=1)
start_time = start_time.strftime("%a %b %d %Y 00:00:00 GMT+0800")
end_time = end_time.strftime("%a %b %d %Y 00:00:00 GMT+0800")
filters = {
"beginTime": start_time,
"endTime": end_time
}
logging.info(u"start time:%s \t end time:%s" % (start_time, end_time))
start_pos = 0
for res_page in login_new_adr.get_page_list(filters,start_pos, step):
page_idx += 1
business_idx = 1
for qrow in res_page:
print "business idx: %s of %s " % (business_idx , step)
business_idx += 1
adr_adr.import_from_html(qrow)
def get_report():
#
from login2 import login
logincounter = 10
while logincounter:
print logincounter
if login():
print "ok"
break
else:
print "clear"
cookieManager.clear()
logincounter = logincounter -1
report_url = "http://www.adrs.org.cn/ADR/page/adrpage/psur/viewPsurReport.jsp?fdObjectId=243014104899856270079204100000&type=2"
somedata = send_url2(report_url)
print "test:",somedata
n =11
extrainfo = extradata3(somedata)
dataout = [extrainfo[i:i+n] for i in range(0, len(extrainfo), n)]
print "temp:", dataout
erd = data2clean2(extrainfo)
print erd
def get_report():
#
from login2 import login
logincounter = 10
while logincounter:
print logincounter
if login():
print "ok"
break
else:
print "clear"
cookieManager.clear()
logincounter = logincounter - 1
report_url = "http://www.adrs.org.cn/ADR/page/adrpage/psur/viewPsurReport.jsp?fdObjectId=243014104899856270079204100000&type=2"
somedata = send_url2(report_url)
print "test:", somedata
n = 11
extrainfo = extradata3(somedata)
dataout = [extrainfo[i:i + n] for i in range(0, len(extrainfo), n)]
print "temp:", dataout
erd = data2clean2(extrainfo)
print erd
def crawler_today():
#
logging.info(u"开始抓取今天数据！")
logincounter = 6
while logincounter:
print logincounter
loginTag = login2.login()
print u"login Tag(ADR Today):", loginTag
if loginTag:
break
elif logincounter == 1:
#
print u"6次login(auto today)失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter - 1
time.sleep(8)
page_idx = 1
step = 100
start_time = datetime.datetime.today()
end_time = start_time + datetime.timedelta(days=1)
start_time = start_time.strftime("%a %b %d %Y 00:00:00 GMT+0800")
end_time = end_time.strftime("%a %b %d %Y 00:00:00 GMT+0800")
filters = {"beginTime": start_time, "endTime": end_time}
logging.info(u"start time:%s \t end time:%s" % (start_time, end_time))
start_pos = 0
for res_page in login_new_adr.get_page_list(filters, start_pos, step):
page_idx += 1
business_idx = 1
for qrow in res_page:
print "business idx: %s of %s " % (business_idx, step)
business_idx += 1
adr_adr.import_from_html(qrow)
utils.get_smd_report(smdrReportViewHtml, ReportID, viewId,
ReportUnitName, mdr_data["reporter"],
mdr_data["reporter_class"], ReportDate)
print u"%s的不良事件评价更新完成！" % (date)
def crawler_appraise(year, beginMonth, endMonth):
logincounter = 0
loginOk = False
while logincounter < 6 and not loginOk:
loginOk = login2.login()
logincounter = logincounter + 1
time.sleep(8)
beginDay = datetime.date(int(year), int(beginMonth), 1)
while beginDay.month <= int(endMonth) and beginDay.year == int(year):
mdr_update_appraise(beginDay.strftime("%Y-%m-%d"))
beginDay = beginDay + datetime.timedelta(days=1)
if __name__ == "__main__":
logincounter = 0
loginOk = False
while logincounter < 6 and not loginOk:
loginOk = login2.login()
logincounter = logincounter + 1
time.sleep(8)
mdr_update_appraise('2016-01-14')
def harvest(sometime):
#
print u"正在login..."
logincounter = 6
while logincounter:
print logincounter
loginTag = login()
print u"login Tag (MDR list):", loginTag
if loginTag:
break
elif logincounter == 1:
print u"6次login失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter - 1
time.sleep(8)
#
send_url(secondHome)
print u"已经登录成功"
#mdr
send_url(thirdHome)
#获取记录条数总记录
stack = sometime
_stack = stack[0]
stack_l = len(_stack)
reportlog = codecs.open('item.txt', 'a', 'utf-8')
myconn = myconnect()
mycu = myconn.cursor()
for i in range(stack_l):
#
print u"正在读取记录条数..."
out = _stack.pop()
out1 = stack[1].pop()
const_time_o = out1.split("-")
const_time_o_ = out.split("-")
yaer = const_time_o_[0]
month = const_time_o_[1]
#print yaer, month
mycu.execute(
"delete from mdr_checklist where CheckTag='Q' and Year=%s and Month=%s"
% (yaer, month))
myconn.commit()
totalparamsx = {
"funcID":
"QUERY_DATA",
"userID":
78919,
"operations": [{
"actionName":
"query",
"operationDatas": [{
"PROD_NAME_1540": "",
"REG_NO_1540": "",
"DEVICE_CLASS_ID_1540": "",
"DEVICE_CLASS_NAME_1540": "",
"REPORT_DATE_START": out,
"REPORT_DATE_END": out1,
"REPORT_NUMBER_1540": "",
"CREATE_DATE_START": "",
"CREATE_DATE_END": "",
"SUPERVISE_ORG_ID_1540": "",
"SUPERVISE_ORG_NAME_1540": "",
"MAN_NAME_1540": "",
"REPORT_UNIT_NAME_1540": "",
"PATIENT_NAME_1540": "",
"EVALUATE_DATE_START": "",
"EVALUATE_DATE_END": "",
"MANAGE_CATEGORY_1540": "",
"listid": "1540",
"start": 0,
"limit": 100
}]
}]
}
_idcdata = send_post_json(totalHome, totalparamsx)
#print totalparamsx
_data = json.loads(_idcdata)
usa = _data['ResponseMessage']['operations'][0]['pageTotal']
total_usa = int(usa)
print out, out1
print u"报告条数:", str(total_usa)
if int(const_time_o[1]) - int(const_time_o_[1]) > 10:
mycu.execute(
"insert into mdr_checklist (BeginDate,EndDate,Item,Tag) "
"value(%s,%s,%s,%s)", (out, out1, str(total_usa), "Y"))
myconn.commit()
else:
mycu.execute(
"insert into mdr_checklist (BeginDate,EndDate,Item,Tag,CheckTag,Year,Month) "
"value(%s,%s,%s,%s,%s,%s,%s)",
(out, out1, str(total_usa), "M", "Q", yaer, month))
myconn.commit()
reportlog.close()
print u"本次任务已经结束"
else:
if data["patient_birthday"]:
data["patient_age"] = utils.str_to_unicode(str(int(_year) - int(data["patient_birthday"][0:4])))
else:
data["patient_age"] = ''
if data["patient_birthday"]:
pass
else:
data["patient_birthday"] = '0000-00-00'
if not data["EffectiveDate"]:
data["EffectiveDate"] = '0000-00-00'
if not data["production_date"]:
data["production_date"] = '0000-00-00'
if not data["StopDate"]:
data["StopDate"] = '0000-00-00'
if not data["ImplantationDate"]:
data["ImplantationDate"] = '0000-00-00'
data["age_class"] = utils.age_check(data["patient_age"])
return data
login()
mdr_data = mdr_get_report_data(obj_id, bianma, sned_date)
print mdr_data
def checklistauto(t1, t2, t3):
#
logincounter = 6
while logincounter:
print logincounter
loginTag = login2.login()
print u"login Tag (checklist):", loginTag
if loginTag:
break
elif logincounter == 1:
print u"6次login失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter - 1
time.sleep(8)
some = timetomonth3(int(t1), int(t2), int(t3))
stack = some
_stack = stack[0]
stack_l = len(_stack)
conn = database_adr.getConnection()
cur = conn.cursor()
for i in range(stack_l):
#
print u"正在读取记录条数..."
out = _stack.pop()
out1 = stack[1].pop()
print out, out1
const_time_o = out1.split("-")
const_time_o_ = out.split("-")
yaer = const_time_o_[0]
month = const_time_o_[1]
cur.execute(
"delete from mdr_checklist where CheckTag='Y' and Year=%s and Month=%s"
% (yaer, month))
conn.commit()
end_time = datetime.datetime.strptime(out1, "%Y-%m-%d")
start_time = datetime.datetime.strptime(out, "%Y-%m-%d")
start_time = start_time.strftime("%a %b %d %Y 00:00:00 GMT+0800")
end_time = end_time.strftime("%a %b %d %Y 00:00:00 GMT+0800")
filters = {"beginTime": start_time, "endTime": end_time}
step = 10
start_pos = 0
batchId = 0
query_data = build_query(filters, start_pos, batchId, step)
#print query_data
adr_url = 'http://www.adrs.org.cn/ADR/dwr/call/plaincall/reportSearchService.dwrReportSearchCg.dwr'
opt = {
"c0-e1":
"number:" + str(start_pos),
"c0-e2":
"number:" + str(step),
"c0-e3":
"string:123_w_w_w_321_colon",
"c0-e4":
"string:123_w_w_w_321_comma",
"c0-e5":
"string:",
"c0-param0":
"Object_Object:{start:reference:c0-e1, limit:reference:c0-e2, analyticsRuleColon:reference:c0-e3, analyticsRuleComma:reference:c0-e4, queryString:reference:c0-e5}"
}
out_data = send_post3(adr_url, query_data, opt_headers=opt)
if out_data is None:
#
print u'返回空数据，重新操作本月份数据'
else:
#print len(out_data)
total = out_data.split(";")
if len(total) != 683:
print u'返回数据长度不够，重新操作本月份数据'
else:
total_no = total.pop()
temp = total[681].split(",")[3].split(":")[1].strip(")")
temp2 = temp.split("}")[0]
print u"本月条数：", temp2
cur.execute(
"insert into mdr_checklist (BeginDate,EndDate,Item,Tag,CheckTag,Year,Month) "
"value(%s,%s,%s,%s,%s,%s,%s)",
(out, out1, str(temp2), "M", "Y", yaer, month))
conn.commit()
print u"本次任务已经结束"
print err
conn.commit()
conn.close()
def crawler_appraise(year,beginMonth, endMonth):
import login2
logincounter = 0
loginOk = False
while logincounter < 6 and not loginOk :
loginOk = login2.login()
logincounter = logincounter + 1
beginDay = datetime.date(int(year),int(beginMonth),1)
while beginDay.month <= int(endMonth) and beginDay.year == int(year) :
adr_update_appraise(beginDay.strftime("%Y-%m-%d"))
beginDay = beginDay + datetime.timedelta(days=1)
if __name__ == "__main__":
logincounter = 0
loginOk = False
import login2
while logincounter < 6 and not loginOk :
loginOk = login2.login()
logincounter = logincounter + 1
adr_update_appraise('2016-01-14')
data["patient_age"] = utils.str_to_unicode(
str(int(_year) - int(data["patient_birthday"][0:4])))
else:
data["patient_age"] = ''
if data["patient_birthday"]:
pass
else:
data["patient_birthday"] = '0000-00-00'
if not data["EffectiveDate"]:
data["EffectiveDate"] = '0000-00-00'
if not data["production_date"]:
data["production_date"] = '0000-00-00'
if not data["StopDate"]:
data["StopDate"] = '0000-00-00'
if not data["ImplantationDate"]:
data["ImplantationDate"] = '0000-00-00'
data["age_class"] = utils.age_check(data["patient_age"])
return data
login()
mdr_data = mdr_get_report_data(obj_id, bianma, sned_date)
print mdr_data
def harvest(sometime):
#
print u"正在login..."
logincounter = 6
while logincounter:
print logincounter
loginTag = login()
print u"login Tag (MDR list):",loginTag
if loginTag:
break
elif logincounter == 1:
print u"6次login失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter -1
time.sleep(8)
#
send_url(secondHome)
print u"已经登录成功"
#mdr
send_url(thirdHome)
#获取记录条数总记录
stack = sometime
_stack = stack[0]
stack_l = len(_stack)
reportlog = codecs.open('item.txt', 'a', 'utf-8')
myconn = myconnect()
mycu = myconn.cursor()
for i in range(stack_l):
#
print u"正在读取记录条数..."
out = _stack.pop()
out1 = stack[1].pop()
const_time_o = out1.split("-")
const_time_o_ = out.split("-")
yaer = const_time_o_[0]
month = const_time_o_[1]
#print yaer, month
mycu.execute("delete from mdr_checklist where CheckTag='Q' and Year=%s and Month=%s" % (yaer, month))
myconn.commit()
totalparamsx = {"funcID":"QUERY_DATA","userID":78919,"operations":[{"actionName":"query","operationDatas":[{"PROD_NAME_1540":"","REG_NO_1540":"","DEVICE_CLASS_ID_1540":"","DEVICE_CLASS_NAME_1540":"","REPORT_DATE_START":out,"REPORT_DATE_END":out1,"REPORT_NUMBER_1540":"","CREATE_DATE_START":"","CREATE_DATE_END":"","SUPERVISE_ORG_ID_1540":"","SUPERVISE_ORG_NAME_1540":"","MAN_NAME_1540":"","REPORT_UNIT_NAME_1540":"","PATIENT_NAME_1540":"","EVALUATE_DATE_START":"","EVALUATE_DATE_END":"","MANAGE_CATEGORY_1540":"","listid":"1540","start":0,"limit":100}]}]}
_idcdata = send_post_json(totalHome, totalparamsx)
#print totalparamsx
_data = json.loads(_idcdata)
usa = _data['ResponseMessage']['operations'][0]['pageTotal']
total_usa = int(usa)
print out, out1
print u"报告条数:",str(total_usa)
if int(const_time_o[1])-int(const_time_o_[1])> 10:
mycu.execute(
"insert into mdr_checklist (BeginDate,EndDate,Item,Tag) "
"value(%s,%s,%s,%s)", (out, out1, str(total_usa), "Y"))
myconn.commit()
else:
mycu.execute(
"insert into mdr_checklist (BeginDate,EndDate,Item,Tag,CheckTag,Year,Month) "
"value(%s,%s,%s,%s,%s,%s,%s)", (out, out1, str(total_usa), "M", "Q", yaer, month))
myconn.commit()
reportlog.close()
print u"本次任务已经结束"
def checklistauto(t1, t2, t3):
#
logincounter = 6
while logincounter:
print logincounter
loginTag = login2.login()
print u"login Tag (checklist):",loginTag
if loginTag:
break
elif logincounter == 1:
print u"6次login失败，重新尝试此操作"
print u"本次任务已经结束"
return
else:
cookieManager.clear()
logincounter = logincounter -1
time.sleep(8)
some = timetomonth3(int(t1), int(t2), int(t3))
stack = some
_stack = stack[0]
stack_l = len(_stack)
conn = database_adr.getConnection()
cur = conn.cursor()
for i in range(stack_l):
#
print u"正在读取记录条数..."
out = _stack.pop()
out1 = stack[1].pop()
print out, out1
const_time_o = out1.split("-")
const_time_o_ = out.split("-")
yaer = const_time_o_[0]
month = const_time_o_[1]
cur.execute("delete from mdr_checklist where CheckTag='Y' and Year=%s and Month=%s" % (yaer, month))
conn.commit()
end_time = datetime.datetime.strptime(out1, "%Y-%m-%d")
start_time = datetime.datetime.strptime(out, "%Y-%m-%d")
start_time = start_time.strftime("%a %b %d %Y 00:00:00 GMT+0800")
end_time = end_time.strftime("%a %b %d %Y 00:00:00 GMT+0800")
filters = {
"beginTime": start_time,
"endTime": end_time
}
step = 10
start_pos = 0
batchId = 0
query_data = build_query(filters, start_pos, batchId, step)
#print query_data
adr_url = 'http://www.adrs.org.cn/ADR/dwr/call/plaincall/reportSearchService.dwrReportSearchCg.dwr'
opt = {
"c0-e1" : "number:" + str(start_pos),
"c0-e2" : "number:" + str(step),
"c0-e3" : "string:123_w_w_w_321_colon",
"c0-e4": "string:123_w_w_w_321_comma",
"c0-e5" : "string:",
"c0-param0": "Object_Object:{start:reference:c0-e1, limit:reference:c0-e2, analyticsRuleColon:reference:c0-e3, analyticsRuleComma:reference:c0-e4, queryString:reference:c0-e5}"
}
out_data = send_post3(adr_url,query_data,opt_headers=opt)
if out_data is None:
#
print u'返回空数据，重新操作本月份数据'
else:
#print len(out_data)
total = out_data.split(";")
if len(total)!= 683:
print u'返回数据长度不够，重新操作本月份数据'
else:
total_no = total.pop()
temp =total[681].split(",")[3].split(":")[1].strip(")")
temp2 = temp.split("}")[0]
print u"本月条数：",temp2
cur.execute(
"insert into mdr_checklist (BeginDate,EndDate,Item,Tag,CheckTag,Year,Month) "
"value(%s,%s,%s,%s,%s,%s,%s)", (out, out1, str(temp2), "M", "Y", yaer, month)
)
conn.commit()
print u"本次任务已经结束"
