def mdr_profile(): isLogin = False while not isLogin: isLogin = login2.login() for qrow in utils.MDR_Report_Query(July_cal()): mdr_parse.mdr_import_report(qrow)
def gather_report_total(_timedict): t = _timedict t_start = t['beginTime'] t_end = t['endTime'] logincounter = 10 while logincounter: print logincounter if login(): print "ok" break else: print "clear" cookieManager.clear() logincounter = logincounter -1 #首先获取记录条数总记录 totalparamsx = {"funcID":"QUERY_DATA","userID":78919,"operations":[{"actionName":"query","operationDatas":[{"ENTRY_START_DATE_2401":t_start,"ENTRY_END_DATE_2401":t_end,"ACTIVE_CONSTITUENT_2430":"","TRADENAME_CN_2431":"","DRUGNAME_CN_FORM_2431":"","DOSEFORM_2431":"","DRUG_STATUS_2431":"","APPROVAL_NO_2431":"","REPORT_ID_2430":"","PICKER_UNIT_NAME_2430":"","START_APPLY_DATE_2430":"","END_APPLY_DATE_2430":"","listid":"2430_1","start":0,"limit":10}]}]} _idcdata = send_post_json(Psur_url, totalparamsx) #print _idcdata usa = 0 try: _data = json.loads(_idcdata) usa = _data['ResponseMessage']['operations'][0]['pageTotal'] #TypeError: expected string or buffer except (IndexError, KeyError, ValueError,TypeError), exception: print "gather_report_total:",exception print "IN json Data:", _idcdata
def main3(somed1, somed2, s3): #主逻辑部分 print u"正在login..." logincounter = 6 while logincounter: print logincounter loginTag = login2.login() print u"login Tag(zoe3):",loginTag if loginTag: break elif logincounter == 1: # print u"6次login(main3)失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter -1 time.sleep(8) # print u"已经登录成功" #获得所有日期参数列表 querydays = right2(int(somed1), int(somed2), int(s3)) #读取B平台参数数据下记录报告总数 days_local = harvestdaystomonthlocal(querydays) #辅助输出信息,供使用者阅读 #{ print u"本平台数据:", for il in days_local: print il print u"正在从国家站点读取数据........:", #} #读取A平台参数数据下记录报告总数 days_remote = harvestdaystomonthremote(querydays) #辅助输出信息,供使用者阅读 #{ print u"国家站点数据:", for ir in days_remote: print ir #} #取得参数长度 dl_len = len(days_local) for i in range(dl_len): #以A数据参数为基准参数 someday = days_remote[i][0] date_remote = days_remote[i][1] date_local = days_local[i][1] date_check = int(date_remote)-int(date_local) if date_check ==0: print u"%s days Matched" % someday else: # print "Date_Check NOT MATCH:",date_check print u"%s days Not Match" % someday print u"匹配开始......." main_adr.crawler_today2(someday) print u"核对结束"
def main3(somed1, somed2, s3): #主逻辑部分 print u"正在login..." logincounter = 6 while logincounter: print logincounter loginTag = login2.login() print u"login Tag(zoe3):", loginTag if loginTag: break elif logincounter == 1: # print u"6次login(main3)失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter - 1 time.sleep(8) # print u"已经登录成功" #获得所有日期参数列表 querydays = right2(int(somed1), int(somed2), int(s3)) #读取B平台参数数据下记录报告总数 days_local = harvestdaystomonthlocal(querydays) #辅助输出信息,供使用者阅读 #{ print u"本平台数据:", for il in days_local: print il print u"正在从国家站点读取数据........:", #} #读取A平台参数数据下记录报告总数 days_remote = harvestdaystomonthremote(querydays) #辅助输出信息,供使用者阅读 #{ print u"国家站点数据:", for ir in days_remote: print ir #} #取得参数长度 dl_len = len(days_local) for i in range(dl_len): #以A数据参数为基准参数 someday = days_remote[i][0] date_remote = days_remote[i][1] date_local = days_local[i][1] date_check = int(date_remote) - int(date_local) if date_check == 0: print u"%s days Matched" % someday else: # print "Date_Check NOT MATCH:", date_check print u"%s days Not Match" % someday print u"匹配开始......." main_adr.crawler_today2(someday) print u"核对结束"
def gather_report_total(_timedict): t = _timedict t_start = t['beginTime'] t_end = t['endTime'] logincounter = 10 while logincounter: print logincounter if login(): print "ok" break else: print "clear" cookieManager.clear() logincounter = logincounter - 1 #首先获取记录条数总记录 totalparamsx = { "funcID": "QUERY_DATA", "userID": 78919, "operations": [{ "actionName": "query", "operationDatas": [{ "ENTRY_START_DATE_2401": t_start, "ENTRY_END_DATE_2401": t_end, "ACTIVE_CONSTITUENT_2430": "", "TRADENAME_CN_2431": "", "DRUGNAME_CN_FORM_2431": "", "DOSEFORM_2431": "", "DRUG_STATUS_2431": "", "APPROVAL_NO_2431": "", "REPORT_ID_2430": "", "PICKER_UNIT_NAME_2430": "", "START_APPLY_DATE_2430": "", "END_APPLY_DATE_2430": "", "listid": "2430_1", "start": 0, "limit": 10 }] }] } _idcdata = send_post_json(Psur_url, totalparamsx) #print _idcdata usa = 0 try: _data = json.loads(_idcdata) usa = _data['ResponseMessage']['operations'][0]['pageTotal'] #TypeError: expected string or buffer except (IndexError, KeyError, ValueError, TypeError), exception: print "gather_report_total:", exception print "IN json Data:", _idcdata
def crawler_last_month(username, password): #抓取上月的 logging.info(u"开始抓取上月数据!") logincounter = 6 while logincounter: print logincounter loginTag = login2.login() print u"login Tag(last_month):",loginTag if loginTag: break elif logincounter == 1: # print u"6次login(last_month)失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter -1 time.sleep(8) page_idx = 1 step = 100 cur_time = datetime.datetime.today() year = cur_time.year month = cur_time.month - 1 if month == 0 : month = 12 year -= 1 start_time = datetime.datetime(year, month, 1) end_time = datetime.datetime( cur_time.year, cur_time.month, 1 ) - datetime.timedelta(days=1) start_time = start_time.strftime("%a %b %d %Y 00:00:00 GMT+0800") end_time = end_time.strftime("%a %b %d %Y 00:00:00 GMT+0800") filters = { "beginTime": start_time, "endTime": end_time } logging.info(u"start time:%s \t end time:%s" % (start_time, end_time)) start_pos = 0 for res_page in login_new_adr.get_page_list(filters,start_pos, step):# 一次一页,一页10条 print u"page: ",page_idx page_idx += 1 business_idx = 1 for qrow in res_page: # 一次一个案例 print u"business idx: %s of %s " % (business_idx , step) business_idx += 1 adr_adr.import_from_html(qrow)
def crawler_appraise(year,beginMonth, endMonth): logincounter = 0 loginOk = False while logincounter < 6 and not loginOk : loginOk = login2.login() logincounter = logincounter + 1 time.sleep(8) beginDay = datetime.date(int(year),int(beginMonth),1) while beginDay.month <= int(endMonth) and beginDay.year == int(year) : mdr_update_appraise(beginDay.strftime("%Y-%m-%d")) beginDay = beginDay + datetime.timedelta(days=1)
def extradown(): logincounter = 10 while logincounter: print logincounter if login(): print "ok" break else: print "clear" cookieManager.clear() logincounter = logincounter - 1 psur.myautodown2()
def crawler_appraise(year, beginMonth, endMonth): logincounter = 0 loginOk = False while logincounter < 6 and not loginOk: loginOk = login2.login() logincounter = logincounter + 1 time.sleep(8) beginDay = datetime.date(int(year), int(beginMonth), 1) while beginDay.month <= int(endMonth) and beginDay.year == int(year): mdr_update_appraise(beginDay.strftime("%Y-%m-%d")) beginDay = beginDay + datetime.timedelta(days=1)
def extradown(): logincounter = 10 while logincounter: print logincounter if login(): print "ok" break else: print "clear" cookieManager.clear() logincounter = logincounter -1 psur.myautodown2()
def crawler_last_month(username, password): #抓取上月的 logging.info(u"开始抓取上月数据!") logincounter = 6 while logincounter: print logincounter loginTag = login2.login() print u"login Tag(last_month):", loginTag if loginTag: break elif logincounter == 1: # print u"6次login(last_month)失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter - 1 time.sleep(8) page_idx = 1 step = 100 cur_time = datetime.datetime.today() year = cur_time.year month = cur_time.month - 1 if month == 0: month = 12 year -= 1 start_time = datetime.datetime(year, month, 1) end_time = datetime.datetime(cur_time.year, cur_time.month, 1) - datetime.timedelta(days=1) start_time = start_time.strftime("%a %b %d %Y 00:00:00 GMT+0800") end_time = end_time.strftime("%a %b %d %Y 00:00:00 GMT+0800") filters = {"beginTime": start_time, "endTime": end_time} logging.info(u"start time:%s \t end time:%s" % (start_time, end_time)) start_pos = 0 for res_page in login_new_adr.get_page_list(filters, start_pos, step): # 一次一页,一页10条 print u"page: ", page_idx page_idx += 1 business_idx = 1 for qrow in res_page: # 一次一个案例 print u"business idx: %s of %s " % (business_idx, step) business_idx += 1 adr_adr.import_from_html(qrow)
def crawler_by_time(username, password,start_date,end_date): #开始抓取指定日期区间数据 logging.info("开始抓取指定日期区间数据!") print u"正在login..." logincounter = 6 while logincounter: print logincounter loginTag = login2.login() print u"login Tag(zoe3):",loginTag if loginTag: break elif logincounter == 1: # print u"6次login(main3)失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter -1 time.sleep(8) page_idx = 1 step = 100 end_time = datetime.datetime.strptime(end_date,"%Y-%m-%d") start_time = datetime.datetime.strptime(start_date,"%Y-%m-%d") start_time = start_time.strftime("%a %b %d %Y 00:00:00 GMT+0800") end_time = end_time.strftime("%a %b %d %Y 00:00:00 GMT+0800") filters = { "beginTime": start_time, "endTime": end_time } logging.info("start time:%s \t end time:%s" % (start_time, end_time)) start_pos = 0 for res_page in login_new_adr.get_page_list(filters,start_pos, step): print "page: ",page_idx page_idx += 1 business_idx = 1 for qrow in res_page: # 一次一个案例 print "business idx: %s of %s " % (business_idx , step) business_idx += 1 try: delete_by_bianma(qrow["report_id2"]) #删除成功则添加 adr_adr.import_from_html(qrow) except Exception as err : print err
def crawler_by_time(username, password, start_date, end_date): #开始抓取指定日期区间数据 logging.info("开始抓取指定日期区间数据!") print u"正在login..." logincounter = 6 while logincounter: print logincounter loginTag = login2.login() print u"login Tag(zoe3):", loginTag if loginTag: break elif logincounter == 1: # print u"6次login(main3)失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter - 1 time.sleep(8) page_idx = 1 step = 100 end_time = datetime.datetime.strptime(end_date, "%Y-%m-%d") start_time = datetime.datetime.strptime(start_date, "%Y-%m-%d") start_time = start_time.strftime("%a %b %d %Y 00:00:00 GMT+0800") end_time = end_time.strftime("%a %b %d %Y 00:00:00 GMT+0800") filters = {"beginTime": start_time, "endTime": end_time} logging.info("start time:%s \t end time:%s" % (start_time, end_time)) start_pos = 0 for res_page in login_new_adr.get_page_list(filters, start_pos, step): print "page: ", page_idx page_idx += 1 business_idx = 1 for qrow in res_page: # 一次一个案例 print "business idx: %s of %s " % (business_idx, step) business_idx += 1 try: delete_by_bianma(qrow["report_id2"]) #删除成功则添加 adr_adr.import_from_html(qrow) except Exception as err: print err
def crawler_first_half_year(username, password): #抓取上半年度的 logging.info(u"开始抓取上半年度数据!") logincounter = 6 while logincounter: print logincounter loginTag = login2.login() print u"login Tag(first_half_year):",loginTag if loginTag: break elif logincounter == 1: # print u"6次login(first_half_year)失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter -1 time.sleep(8) page_idx = 1 step = 100 year = datetime.datetime.today().year last_year_start = datetime.date(year,1,1) last_year_end = datetime.date(year,6,30) start_time = last_year_start.strftime("%a %b %d %Y 00:00:00 GMT+0800") end_time = last_year_end.strftime("%a %b %d %Y 00:00:00 GMT+0800") filters = { "beginTime": start_time, "endTime": end_time } logging.info(u"start time:%s \t end time:%s" % (start_time, end_time)) start_pos = 0 for res_page in login_new_adr.get_page_list(filters,start_pos, step): print u"page: ",page_idx page_idx += 1 business_idx = 1 for qrow in res_page: print u"business idx: %s of %s " % (business_idx , step) business_idx += 1 adr_adr.import_from_html(qrow)
def crawler_last_year(username, password): #抓取上一年度的 logging.info(u"开始抓取上一年度数据!") logincounter = 6 while logincounter: print logincounter loginTag = login2.login() print u"login Tag(last_year):", loginTag if loginTag: break elif logincounter == 1: # print u"6次login(last_year)失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter - 1 time.sleep(8) page_idx = 1 step = 100 year = datetime.datetime.today().year - 1 last_year_start = datetime.date(year, 1, 1) last_year_end = datetime.date(year, 12, 31) start_time = last_year_start.strftime("%a %b %d %Y 00:00:00 GMT+0800 ") end_time = last_year_end.strftime("%a %b %d %Y 00:00:00 GMT+0800 ") filters = {"beginTime": start_time, "endTime": end_time} logging.info(u"start time:%s \t end time:%s" % (start_time, end_time)) start_pos = 0 for res_page in login_new_adr.get_page_list(filters, start_pos, step): # 一次一页,一页10条 print "page: ", page_idx page_idx += 1 business_idx = 1 for qrow in res_page: # 一次一个案例 print u"business idx: %s of %s " % (business_idx, step) business_idx += 1 adr_adr.import_from_html(qrow)
def crawler_first_half_year(): #抓取上半年度 logincounter = 6 while logincounter: print logincounter loginTag = login() print u"login Tag(first_half_year):",loginTag if loginTag: break elif logincounter == 1: # print u"6次login(first_half_year)失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter -1 time.sleep(8) deletedance.dodelete(halfyear_before_cal()) report_query.MDR_Report_Query(halfyear_before_cal())
def adr_regulate(year, beginMonth, endMonth): ''' 校正adr数据 @year : 指定校正的年份 @beginMonth : 指定开始校正的月分 @endMonth : 指定结束校正的月份 ''' print u"正在login..." logincounter = 0 loginOk = False while logincounter < 6 and not loginOk : loginOk = login2.login() logincounter = logincounter + 1 time.sleep(8) beginDay = datetime.date(int(year),int(beginMonth),1) while beginDay.month <= int(endMonth) and beginDay.year == int(year) : adr_regulate_by_day(beginDay.strftime("%Y-%m-%d")) beginDay = beginDay + datetime.timedelta(days=1)
def crawler_anytime2(start, end): # logincounter = 6 while logincounter: print logincounter loginTag = login() print u"login Tag(anytime2):",loginTag if loginTag: break elif logincounter == 1: # print u"6次login(anytime2)失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter -1 time.sleep(1) deletedance.dodelete(anytime2(start, end)) report_query.MDR_Report_Query(anytime2(start, end))
def crawler_today(): #抓取今天 logincounter = 6 while logincounter: print logincounter loginTag = login() print u"login Tag(today):",loginTag if loginTag: break elif logincounter == 1: # print u"6次login(today)失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter -1 time.sleep(8) deletedance.dodelete(toady_cal()) report_query.MDR_Report_Query(toady_cal())
def crawler_today(): #抓取今天 logincounter = 6 while logincounter: print logincounter loginTag = login() print u"login Tag(auto today):",loginTag if loginTag: break elif logincounter == 1: # print u"6次login(auto today)失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter -1 time.sleep(8) deletedance.dodelete(toady_cal()) report_query.MDR_Report_Query(toady_cal())
def crawler_current_week(): #抓取本周 logincounter = 6 while logincounter: print logincounter loginTag = login() print u"login Tag(current_week):",loginTag if loginTag: break elif logincounter == 1: # print u"6次login(current_week)失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter -1 time.sleep(8) deletedance.dodelete(current_week_cal()) report_query.MDR_Report_Query(current_week_cal())
def crawler_today(): # logging.info(u"开始抓取今天数据!") logincounter = 6 while logincounter: print logincounter loginTag = login2.login() print u"login Tag(ADR Today):",loginTag if loginTag: break elif logincounter == 1: # print u"6次login(auto today)失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter -1 time.sleep(8) page_idx = 1 step = 100 start_time = datetime.datetime.today() end_time = start_time + datetime.timedelta(days=1) start_time = start_time.strftime("%a %b %d %Y 00:00:00 GMT+0800") end_time = end_time.strftime("%a %b %d %Y 00:00:00 GMT+0800") filters = { "beginTime": start_time, "endTime": end_time } logging.info(u"start time:%s \t end time:%s" % (start_time, end_time)) start_pos = 0 for res_page in login_new_adr.get_page_list(filters,start_pos, step): page_idx += 1 business_idx = 1 for qrow in res_page: print "business idx: %s of %s " % (business_idx , step) business_idx += 1 adr_adr.import_from_html(qrow)
def get_report(): # from login2 import login logincounter = 10 while logincounter: print logincounter if login(): print "ok" break else: print "clear" cookieManager.clear() logincounter = logincounter -1 report_url = "http://www.adrs.org.cn/ADR/page/adrpage/psur/viewPsurReport.jsp?fdObjectId=243014104899856270079204100000&type=2" somedata = send_url2(report_url) print "test:",somedata n =11 extrainfo = extradata3(somedata) dataout = [extrainfo[i:i+n] for i in range(0, len(extrainfo), n)] print "temp:", dataout erd = data2clean2(extrainfo) print erd
def get_report(): # from login2 import login logincounter = 10 while logincounter: print logincounter if login(): print "ok" break else: print "clear" cookieManager.clear() logincounter = logincounter - 1 report_url = "http://www.adrs.org.cn/ADR/page/adrpage/psur/viewPsurReport.jsp?fdObjectId=243014104899856270079204100000&type=2" somedata = send_url2(report_url) print "test:", somedata n = 11 extrainfo = extradata3(somedata) dataout = [extrainfo[i:i + n] for i in range(0, len(extrainfo), n)] print "temp:", dataout erd = data2clean2(extrainfo) print erd
def crawler_today(): # logging.info(u"开始抓取今天数据!") logincounter = 6 while logincounter: print logincounter loginTag = login2.login() print u"login Tag(ADR Today):", loginTag if loginTag: break elif logincounter == 1: # print u"6次login(auto today)失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter - 1 time.sleep(8) page_idx = 1 step = 100 start_time = datetime.datetime.today() end_time = start_time + datetime.timedelta(days=1) start_time = start_time.strftime("%a %b %d %Y 00:00:00 GMT+0800") end_time = end_time.strftime("%a %b %d %Y 00:00:00 GMT+0800") filters = {"beginTime": start_time, "endTime": end_time} logging.info(u"start time:%s \t end time:%s" % (start_time, end_time)) start_pos = 0 for res_page in login_new_adr.get_page_list(filters, start_pos, step): page_idx += 1 business_idx = 1 for qrow in res_page: print "business idx: %s of %s " % (business_idx, step) business_idx += 1 adr_adr.import_from_html(qrow)
utils.get_smd_report(smdrReportViewHtml, ReportID, viewId, ReportUnitName, mdr_data["reporter"], mdr_data["reporter_class"], ReportDate) print u"%s的不良事件评价更新完成!" % (date) def crawler_appraise(year, beginMonth, endMonth): logincounter = 0 loginOk = False while logincounter < 6 and not loginOk: loginOk = login2.login() logincounter = logincounter + 1 time.sleep(8) beginDay = datetime.date(int(year), int(beginMonth), 1) while beginDay.month <= int(endMonth) and beginDay.year == int(year): mdr_update_appraise(beginDay.strftime("%Y-%m-%d")) beginDay = beginDay + datetime.timedelta(days=1) if __name__ == "__main__": logincounter = 0 loginOk = False while logincounter < 6 and not loginOk: loginOk = login2.login() logincounter = logincounter + 1 time.sleep(8) mdr_update_appraise('2016-01-14')
def harvest(sometime): # print u"正在login..." logincounter = 6 while logincounter: print logincounter loginTag = login() print u"login Tag (MDR list):", loginTag if loginTag: break elif logincounter == 1: print u"6次login失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter - 1 time.sleep(8) # send_url(secondHome) print u"已经登录成功" #mdr send_url(thirdHome) #获取记录条数总记录 stack = sometime _stack = stack[0] stack_l = len(_stack) reportlog = codecs.open('item.txt', 'a', 'utf-8') myconn = myconnect() mycu = myconn.cursor() for i in range(stack_l): # print u"正在读取记录条数..." out = _stack.pop() out1 = stack[1].pop() const_time_o = out1.split("-") const_time_o_ = out.split("-") yaer = const_time_o_[0] month = const_time_o_[1] #print yaer, month mycu.execute( "delete from mdr_checklist where CheckTag='Q' and Year=%s and Month=%s" % (yaer, month)) myconn.commit() totalparamsx = { "funcID": "QUERY_DATA", "userID": 78919, "operations": [{ "actionName": "query", "operationDatas": [{ "PROD_NAME_1540": "", "REG_NO_1540": "", "DEVICE_CLASS_ID_1540": "", "DEVICE_CLASS_NAME_1540": "", "REPORT_DATE_START": out, "REPORT_DATE_END": out1, "REPORT_NUMBER_1540": "", "CREATE_DATE_START": "", "CREATE_DATE_END": "", "SUPERVISE_ORG_ID_1540": "", "SUPERVISE_ORG_NAME_1540": "", "MAN_NAME_1540": "", "REPORT_UNIT_NAME_1540": "", "PATIENT_NAME_1540": "", "EVALUATE_DATE_START": "", "EVALUATE_DATE_END": "", "MANAGE_CATEGORY_1540": "", "listid": "1540", "start": 0, "limit": 100 }] }] } _idcdata = send_post_json(totalHome, totalparamsx) #print totalparamsx _data = json.loads(_idcdata) usa = _data['ResponseMessage']['operations'][0]['pageTotal'] total_usa = int(usa) print out, out1 print u"报告条数:", str(total_usa) if int(const_time_o[1]) - int(const_time_o_[1]) > 10: mycu.execute( "insert into mdr_checklist (BeginDate,EndDate,Item,Tag) " "value(%s,%s,%s,%s)", (out, out1, str(total_usa), "Y")) myconn.commit() else: mycu.execute( "insert into mdr_checklist (BeginDate,EndDate,Item,Tag,CheckTag,Year,Month) " "value(%s,%s,%s,%s,%s,%s,%s)", (out, out1, str(total_usa), "M", "Q", yaer, month)) myconn.commit() reportlog.close() print u"本次任务已经结束"
else: if data["patient_birthday"]: data["patient_age"] = utils.str_to_unicode(str(int(_year) - int(data["patient_birthday"][0:4]))) else: data["patient_age"] = '' if data["patient_birthday"]: pass else: data["patient_birthday"] = '0000-00-00' if not data["EffectiveDate"]: data["EffectiveDate"] = '0000-00-00' if not data["production_date"]: data["production_date"] = '0000-00-00' if not data["StopDate"]: data["StopDate"] = '0000-00-00' if not data["ImplantationDate"]: data["ImplantationDate"] = '0000-00-00' data["age_class"] = utils.age_check(data["patient_age"]) return data login() mdr_data = mdr_get_report_data(obj_id, bianma, sned_date) print mdr_data
def checklistauto(t1, t2, t3): # logincounter = 6 while logincounter: print logincounter loginTag = login2.login() print u"login Tag (checklist):", loginTag if loginTag: break elif logincounter == 1: print u"6次login失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter - 1 time.sleep(8) some = timetomonth3(int(t1), int(t2), int(t3)) stack = some _stack = stack[0] stack_l = len(_stack) conn = database_adr.getConnection() cur = conn.cursor() for i in range(stack_l): # print u"正在读取记录条数..." out = _stack.pop() out1 = stack[1].pop() print out, out1 const_time_o = out1.split("-") const_time_o_ = out.split("-") yaer = const_time_o_[0] month = const_time_o_[1] cur.execute( "delete from mdr_checklist where CheckTag='Y' and Year=%s and Month=%s" % (yaer, month)) conn.commit() end_time = datetime.datetime.strptime(out1, "%Y-%m-%d") start_time = datetime.datetime.strptime(out, "%Y-%m-%d") start_time = start_time.strftime("%a %b %d %Y 00:00:00 GMT+0800") end_time = end_time.strftime("%a %b %d %Y 00:00:00 GMT+0800") filters = {"beginTime": start_time, "endTime": end_time} step = 10 start_pos = 0 batchId = 0 query_data = build_query(filters, start_pos, batchId, step) #print query_data adr_url = 'http://www.adrs.org.cn/ADR/dwr/call/plaincall/reportSearchService.dwrReportSearchCg.dwr' opt = { "c0-e1": "number:" + str(start_pos), "c0-e2": "number:" + str(step), "c0-e3": "string:123_w_w_w_321_colon", "c0-e4": "string:123_w_w_w_321_comma", "c0-e5": "string:", "c0-param0": "Object_Object:{start:reference:c0-e1, limit:reference:c0-e2, analyticsRuleColon:reference:c0-e3, analyticsRuleComma:reference:c0-e4, queryString:reference:c0-e5}" } out_data = send_post3(adr_url, query_data, opt_headers=opt) if out_data is None: # print u'返回空数据,重新操作本月份数据' else: #print len(out_data) total = out_data.split(";") if len(total) != 683: print u'返回数据长度不够,重新操作本月份数据' else: total_no = total.pop() temp = total[681].split(",")[3].split(":")[1].strip(")") temp2 = temp.split("}")[0] print u"本月条数:", temp2 cur.execute( "insert into mdr_checklist (BeginDate,EndDate,Item,Tag,CheckTag,Year,Month) " "value(%s,%s,%s,%s,%s,%s,%s)", (out, out1, str(temp2), "M", "Y", yaer, month)) conn.commit() print u"本次任务已经结束"
print err conn.commit() conn.close() def crawler_appraise(year,beginMonth, endMonth): import login2 logincounter = 0 loginOk = False while logincounter < 6 and not loginOk : loginOk = login2.login() logincounter = logincounter + 1 beginDay = datetime.date(int(year),int(beginMonth),1) while beginDay.month <= int(endMonth) and beginDay.year == int(year) : adr_update_appraise(beginDay.strftime("%Y-%m-%d")) beginDay = beginDay + datetime.timedelta(days=1) if __name__ == "__main__": logincounter = 0 loginOk = False import login2 while logincounter < 6 and not loginOk : loginOk = login2.login() logincounter = logincounter + 1 adr_update_appraise('2016-01-14')
data["patient_age"] = utils.str_to_unicode( str(int(_year) - int(data["patient_birthday"][0:4]))) else: data["patient_age"] = '' if data["patient_birthday"]: pass else: data["patient_birthday"] = '0000-00-00' if not data["EffectiveDate"]: data["EffectiveDate"] = '0000-00-00' if not data["production_date"]: data["production_date"] = '0000-00-00' if not data["StopDate"]: data["StopDate"] = '0000-00-00' if not data["ImplantationDate"]: data["ImplantationDate"] = '0000-00-00' data["age_class"] = utils.age_check(data["patient_age"]) return data login() mdr_data = mdr_get_report_data(obj_id, bianma, sned_date) print mdr_data
def harvest(sometime): # print u"正在login..." logincounter = 6 while logincounter: print logincounter loginTag = login() print u"login Tag (MDR list):",loginTag if loginTag: break elif logincounter == 1: print u"6次login失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter -1 time.sleep(8) # send_url(secondHome) print u"已经登录成功" #mdr send_url(thirdHome) #获取记录条数总记录 stack = sometime _stack = stack[0] stack_l = len(_stack) reportlog = codecs.open('item.txt', 'a', 'utf-8') myconn = myconnect() mycu = myconn.cursor() for i in range(stack_l): # print u"正在读取记录条数..." out = _stack.pop() out1 = stack[1].pop() const_time_o = out1.split("-") const_time_o_ = out.split("-") yaer = const_time_o_[0] month = const_time_o_[1] #print yaer, month mycu.execute("delete from mdr_checklist where CheckTag='Q' and Year=%s and Month=%s" % (yaer, month)) myconn.commit() totalparamsx = {"funcID":"QUERY_DATA","userID":78919,"operations":[{"actionName":"query","operationDatas":[{"PROD_NAME_1540":"","REG_NO_1540":"","DEVICE_CLASS_ID_1540":"","DEVICE_CLASS_NAME_1540":"","REPORT_DATE_START":out,"REPORT_DATE_END":out1,"REPORT_NUMBER_1540":"","CREATE_DATE_START":"","CREATE_DATE_END":"","SUPERVISE_ORG_ID_1540":"","SUPERVISE_ORG_NAME_1540":"","MAN_NAME_1540":"","REPORT_UNIT_NAME_1540":"","PATIENT_NAME_1540":"","EVALUATE_DATE_START":"","EVALUATE_DATE_END":"","MANAGE_CATEGORY_1540":"","listid":"1540","start":0,"limit":100}]}]} _idcdata = send_post_json(totalHome, totalparamsx) #print totalparamsx _data = json.loads(_idcdata) usa = _data['ResponseMessage']['operations'][0]['pageTotal'] total_usa = int(usa) print out, out1 print u"报告条数:",str(total_usa) if int(const_time_o[1])-int(const_time_o_[1])> 10: mycu.execute( "insert into mdr_checklist (BeginDate,EndDate,Item,Tag) " "value(%s,%s,%s,%s)", (out, out1, str(total_usa), "Y")) myconn.commit() else: mycu.execute( "insert into mdr_checklist (BeginDate,EndDate,Item,Tag,CheckTag,Year,Month) " "value(%s,%s,%s,%s,%s,%s,%s)", (out, out1, str(total_usa), "M", "Q", yaer, month)) myconn.commit() reportlog.close() print u"本次任务已经结束"
def checklistauto(t1, t2, t3): # logincounter = 6 while logincounter: print logincounter loginTag = login2.login() print u"login Tag (checklist):",loginTag if loginTag: break elif logincounter == 1: print u"6次login失败,重新尝试此操作" print u"本次任务已经结束" return else: cookieManager.clear() logincounter = logincounter -1 time.sleep(8) some = timetomonth3(int(t1), int(t2), int(t3)) stack = some _stack = stack[0] stack_l = len(_stack) conn = database_adr.getConnection() cur = conn.cursor() for i in range(stack_l): # print u"正在读取记录条数..." out = _stack.pop() out1 = stack[1].pop() print out, out1 const_time_o = out1.split("-") const_time_o_ = out.split("-") yaer = const_time_o_[0] month = const_time_o_[1] cur.execute("delete from mdr_checklist where CheckTag='Y' and Year=%s and Month=%s" % (yaer, month)) conn.commit() end_time = datetime.datetime.strptime(out1, "%Y-%m-%d") start_time = datetime.datetime.strptime(out, "%Y-%m-%d") start_time = start_time.strftime("%a %b %d %Y 00:00:00 GMT+0800") end_time = end_time.strftime("%a %b %d %Y 00:00:00 GMT+0800") filters = { "beginTime": start_time, "endTime": end_time } step = 10 start_pos = 0 batchId = 0 query_data = build_query(filters, start_pos, batchId, step) #print query_data adr_url = 'http://www.adrs.org.cn/ADR/dwr/call/plaincall/reportSearchService.dwrReportSearchCg.dwr' opt = { "c0-e1" : "number:" + str(start_pos), "c0-e2" : "number:" + str(step), "c0-e3" : "string:123_w_w_w_321_colon", "c0-e4": "string:123_w_w_w_321_comma", "c0-e5" : "string:", "c0-param0": "Object_Object:{start:reference:c0-e1, limit:reference:c0-e2, analyticsRuleColon:reference:c0-e3, analyticsRuleComma:reference:c0-e4, queryString:reference:c0-e5}" } out_data = send_post3(adr_url,query_data,opt_headers=opt) if out_data is None: # print u'返回空数据,重新操作本月份数据' else: #print len(out_data) total = out_data.split(";") if len(total)!= 683: print u'返回数据长度不够,重新操作本月份数据' else: total_no = total.pop() temp =total[681].split(",")[3].split(":")[1].strip(")") temp2 = temp.split("}")[0] print u"本月条数:",temp2 cur.execute( "insert into mdr_checklist (BeginDate,EndDate,Item,Tag,CheckTag,Year,Month) " "value(%s,%s,%s,%s,%s,%s,%s)", (out, out1, str(temp2), "M", "Y", yaer, month) ) conn.commit() print u"本次任务已经结束"