def main(self, data: dict) -> bool: c = zhengxin91_crawler_log() try: self.verify_query(data, c) if data[self.FLAG] == self.BSDFLAG and self.not_expire( zhengxin91_content, data): try: self.update_existed(data) self.update_succLog(c) logger.info("the data has been in database,update ctime") return True except: self.update_failLog(c) raise Exception(self.Error.EXISTED_FAIL_HINT) resp = self.request(data, c) logger.info("this is receive response") logger.info(resp) parsed = self.parse(resp, c) logger.info("this is parsed") logger.info(self.json_marshal(parsed[1])) self.storage(parsed, data, c) self.save_log(self.RIGHT, c) return True except: self.save_log(self.WRONG, c) logger.error("query_storage error", exc_info=True) return False
def post(self, request) -> JsonResponse: logger.info("helpme accept post info") post_info = self.parse_ReceivedData(request) # handle post data is empty if len(post_info.err) > 0: logger.info(post_info.err) return JsonResponse(post_info.err) # instantiation hd_interface/Hd_EMW025.... logger.info("helpme instantiation hd_interface and hd saving") hdsave = HD.hd_interface() # multiple equals is used to storage hdsave.name = post_info.name hdsave.idcard = post_info.idcard hdsave.token = post_info.token hdsave.phone = post_info.phone hdsave.flag = post_info.flag hdsave.seqId = post_info.seqId logger.info( f"helpme accepted arguments:{hdsave.name},{hdsave.idcard},{hdsave.token},{hdsave.phone},{hdsave.flag}" ) # 2017/10/24/10:20 => add storage in helpMe's database res = hdsave.main() # store emd008 details in database boshidun. hdsave.emd008_details_storage(flag="HM") logger.info(res.get("retmessage")) return JsonResponse(res)
def hd_blackList(self)->dict: # EMW025 黑名单模糊汇总查询 try: # crawler token = self._get_newtoken("hd_token") data = dict( idcard=self.idcard, ACCESS_TOKEN=token, ) # storage if self.flag == self.BSDFLAG: ## storage in bsd_hy_blacklist_hit_summary parseRes = self._hd_blackList_judge_before_crawl(data) bsd = bsd_hy_blacklist_hit_summary() bsd.p_person_id = self.idcard bsd.p_person_name = self.name bsd.p_phone = self.phone bsd.query_id = self.queryId bsd.status = "1000" bsd.black_risk_type = "网贷黑名单" bsd.black_hit_count = parseRes.get("WDHMDCOUNT","") bsd.save(using=self.HMDB) # storage in bsd_blacklisthitnum db = bsd_blacklisthitnum() db.seq_id = self.seqId db.p_person_id = self.idcard db.black_risk_type = "" db.black_hit_count = parseRes.get("WDHMDCOUNT",0) db.CODE = "200" db.data_source = "hd" db.save(using=self.HMDB) elif self.flag == self.SSDFLAG: parseRes = self.req_hdInterface(self.hdBlackListUrl, data) ssd = cfd_hd_huifamohufact() ssd.p_idcode = self.idcard ssd.status = parseRes.get("CODE","") ssd.totalnumber = parseRes.get("totalCount","") ssd.casenumber = parseRes.get("ajlcCount","") ssd.zhixinnumber = parseRes.get("zxggCount","") ssd.taxnumber = "" ssd.cuiqiannumber = "" ssd.kaitinnumber = parseRes.get("ktggCount","") ssd.othernumber = "" ssd.shenpannumber = parseRes.get("cpwsCount","") ssd.wangdainumber = parseRes.get("wdhmdCount","") ssd.success = "200" ssd.save(using=self.SSDDB) # storage in crawler ori = Hd_EMW025() ori.flag = self.flag ori.idcard = self.idcard ori.phone = self.phone ori.name = self.name ori.ori_data = json.dumps(parseRes) ori.save() except: logger.error("hd_blackList failed",exc_info=True) return self.FAILED logger.info("EMW025 黑名单模糊汇总查询 success") return self.SUCCESS
def main(self, rcv: str) -> (str, bool): log_db = ZhengXin91ShareLog() try: self.verify(rcv, log_db) parsed = self.parse(rcv, log_db) self.verify_meanful_rcv(parsed, log_db) if self.not_expire(ZhengXin91ShareOriData, parsed): try: extracted = self.extract_not_expire(parsed) log_db.note = "response from cache, succeed!" self.save_log(self.RIGHT, log_db) logger.info(log_db.note) return extracted, True except: log_db.note = "response from cache, failed!" self.save_log(self.WRONG, log_db) logger.info(log_db.note) raise Exception(self.Error.EXISTED_FAIL_HINT) loan_data = self.query(parsed, log_db) transed = self.trans(loan_data, log_db) self.save_cache(parsed, transed, log_db) self.save_log(self.RIGHT, log_db) return transed.val, True except: self.save_log(self.WRONG, log_db) logger.error("shareLoanInfo91 failed", exc_info=True) return "", False
def save_log(self, flag, table): status = "successfully" if flag == self.RIGHT: table.status = self.RIGHT else: table.status = self.WRONG status = "un" + status table.save() logger.info(f"{table.__class__.__name__} {status} saved")
def verify(self, rcv: str, log_db): try: assert len(rcv) > 0 log_db.verify_rcv = self.RIGHT logger.info("verify receive success") except: logger.error(self.Error.ILLEAGL_NOTE, exc_info=True) log_db.verify_rcv = self.WRONG log_db.note = self.Error.ILLEAGL_NOTE raise Exception("verify receive => False")
def hd_webloan(self)->dict: # EMR002 信贷平台注册详情,统计这个接口的数量 try: for cycle in ("3","6","9","12","24"): self._hd_webloan(cycle) except: logger.error("EMR002 =>", exc_info=True) return self.FAILED logger.info("EMR002 信贷平台注册详情/webloan success") return self.SUCCESS
def _store_bsd(self, parsed: tuple, data: dict): code, concrete = parsed zx = zhengxin91_content() zx.status_code = code zx.name = data[self.UserInfo.REC_NAME] zx.idcard = data[self.UserInfo.REC_IDCARD] zx.ctime_stamp = self.now_stamp if isinstance(concrete, str): zx.err_msg = concrete zx.save(using=self.HMDB) raise Exception("Interface return symolic error of code") elif isinstance(concrete, dict): loanInfos = concrete["loanInfos"] zx_list = [] core = self.ZhengXin91LoanCoreInfo for loan in loanInfos: zx_list.append( zhengxin91_content( status_code=0 if code == 0 else code, name=data[self.UserInfo.REC_NAME], idcard=data[self.UserInfo.REC_IDCARD], ctime_stamp=self.now_stamp, trx_no=concrete["trxNo"], borrow_type=loan.get(core.BORROWSTATE, 0), borrow_status=loan.get(core.BORROWSTATE, 0), borrow_amount=loan.get(core.BORROWAMOUNT, 0), contract_data=loan.get(core.CONTRACTDATE, ""), loan_period=loan.get(core.LOANPERIOD, 0), repay_state=loan.get(core.REPAYSTATE, 0), arrears_amount=loan.get(core.ARREARSAMOUNT, 0), company_code=loan.get(core.COMPANYCODE, ""), )) if len(loanInfos) == 0: logger.info("the message of zhengxin91 responsed is empty") zx_list.append( zhengxin91_content( status_code=0 if code == 0 else code, name=data[self.UserInfo.REC_NAME], idcard=data[self.UserInfo.REC_IDCARD], ctime_stamp=self.now_stamp, trx_no=concrete["trxNo"], borrow_type=0, borrow_status=0, borrow_amount=0, contract_data="", loan_period=0, repay_state=0, arrears_amount=0, company_code="", )) zhengxin91_content.objects.using(self.HMDB).bulk_create(zx_list) else: print(self.now, concrete) print(self.now, type(concrete)) raise Exception("Unexpected concrete struct")
def main(self): if len(self.data) != 5: logger.info("hd emd008 response is {},empty!") self.store_emd008() return self.parse_data() emd_id = self.store_emd008() if len(self.result) == 0: logger.info("hd emd008 loan details are empty") else: self.store_detail(emd_id)
def trans(self, loan_data: dict, log_db) -> namedtuple: S = self.step_return() try: encoded = self.zhengxin_encode(loan_data) log_db.trans_res = self.RIGHT logger.info("trans success") return S(val=self.make_resp_template(encoded), res=False) except: log_db.trans_res = self.WRONG logger.error("trans failed", exc_info=True) raise Exception("trans failed")
def parse_data(self): try: self.code = self.data["CODE"] self.province = self.data["PROVINCE"] self.city = self.data["CITY"] self.result = self.data["RESULTS"] logger.info(f"emd008 parse succeed,{self.code},{self.phone},{self.province},{self.city}") except: err = "emd008 parse failed" logger.error(err,exc_info = True) raise Exception(err)
def request(self, data, c) -> str: try: user_data = self._encode_query(data) resp = requests.post(self.Authentication.QUERY_URL, data=user_data) c.request_res = self.RIGHT logger.info("request success") return resp.text except: c.request_res = self.WRONG tag = "request error" logger.error(tag, exc_info=True) raise Exception(tag)
def post(self,request)->JsonResponse: try: py_obj = json.loads(request.body) zx = ZhengXin91() if zx.main(py_obj): logger.info("Accepted_Query: zhengxin91storage") return JsonResponse(self.success) else: raise Exception("zhengxin91 query or storatrge error") except: logger.error("zhengxin91 error traceback",exc_info=True) return JsonResponse(self.classNum_empty)
def insert_Token(self, table: str, ret: str, effective: str): if table == self.t: t = models.hd_token() t.token = ret t.effctive = effective t.save(using="token") elif table == self.t2: t = models.hd_tokenwebloan() t.token = ret t.effctive = effective t.save(using="token") logger.info(f"{table} insert a new token {ret},effective {effective}")
def parse(self, rcv: bytes, log_db) -> dict: try: to_list = rcv.decode().split("|") assert len(to_list) == self.ZX_LENGTH assert to_list[3] == self.Code.SHARE_REQ_CODE user_data = self.zhengxin_decode(to_list[6]) log_db.parse_res = self.RIGHT logger.info("parse success") return user_data except: logger.error("parse failed", exc_info=True) log_db.parse_res = self.WRONG raise Exception("parse failed")
def store_emd008(self)->int: hd = HDEMD008Manager() hd.name = self.name hd.phone = self.phone hd.idcard = self.idcard hd.code = self.code hd.province = self.province hd.city = self.city hd.result = "0" if len(self.result) == 0 else "1" hd.save(using=self.flag) saved_id = hd.id logger.info(f"store in HDEMD008Manager id is {saved_id}") return saved_id
def _hd_webloandetail_oneCycle(self,cycle)->None: # EMR002 信贷平台注册详情,统计这个接口的数量 if self.flag == self.BSDFLAG: logger.info(f"{self.BSDFLAG} _hd_webloandetail started") times = self._hd_webloandetail_oneCycle_judge_before_crawler(cycle) if times > 0: bsd = bsd_hy_webloan_detail() bsd.query_id = self.queryId bsd.period_name = f"webloan_hit_detail_m{cycle}" bsd.period_months = cycle bsd.class_name = "P2P网贷" bsd.class_count = times bsd.save(using=self.HMDB) self._save_bsd_loanapplicationnum(cycle,str(times)) logger.info(f"{self.BSDFLAG} _hd_webloandetail ended") elif self.flag == self.SSDFLAG: logger.info(f"{self.SSDFLAG} _hd_webloandetail started") res = self._EMD008_integrate_interface(cycle) EMR002 = res.get("RESULTS",[{}])[0] EMR002_data = EMR002.get("DATA","") EMR002_cycle = EMR002.get("CYCLE","") times = len(EMR002_data) ssd = cfd_hy_webloan_detail() ssd.query_id = self.idcard ssd.period_name = EMR002_cycle ssd.period_months = int(cycle) ssd.class_name = "all" ssd.class_count = times ssd.save(using=self.SSDDB) logger.info(f"{self.SSDDB} _hd_webloandetail ended")
def deltoken(self, table: str) -> str: msg = "" try: if table == self.t: hd_token.objects.using("token").all().delete() elif table == self.t2: # t = hd_tokenwebloan() # t.delete(using="token") hd_tokenwebloan.objects.using("token").all().delete() logger.info(f"{table} delete old token") except: logger.error("del error", exc_info=True) msg = traceback.format_exc() return msg
def verify_query(self, data: dict, c): try: assert type(data) == dict assert data[self.UserInfo.REC_NAME] != "" assert data[self.UserInfo.REC_IDCARD] != "" assert data[self.FLAG] != "" c.verify_res = self.RIGHT c.idcard = data.get(self.UserInfo.REC_IDCARD, "") c.name = data.get(self.UserInfo.REC_NAME, "") logger.info("verify query success") except: c.verify_res = self.WRONG c.note = self.Error.ILLEAGL_NOTE logger.error("verify error", exc_info=True) raise Exception(self.Error.ILLEAGL_NOTE)
def _save_bsd_loanapplicationnum(self,cycle,times): try: logger.info(f"bsd_loanapplicationnum saved;user phone number=>{self.phone}") db = bsd_loanapplicationnum() db.seq_id = self.seqId db.p_Phone = self.phone db.p_cycle = cycle db.p_Platform = "0" db.CODE = "200" db.CYCLE = "".join((str(int(time.time())), "000")) db.LOANNUM = times db.data_source = "hd" db.save(using=self.HMDB) except: logger.error("error",exc_info=True)
def EMR013(self,data:dict): emr = [] for d in data: emr.append( HDEMD008EMR013( emd_id=self.emd_id, name = self.name, mobile = self.phone, idcard = self.idcard, cycle = self.cycle, platform = d.get("PLATFORM", "empty"), money = d.get("MONEY", "empty"), ) ) HDEMD008EMR013.objects.using(self.flag).bulk_create(emr) logger.info("emd008:emr013 save succeed")
def save_cache(self, parsed: dict, transed: namedtuple, log_db): try: name, idcard, _ = self._user_info(parsed) zx = ZhengXin91ShareOriData() zx.name = name zx.idcard = idcard zx.status_code = self.Code.SHARE_EXISTED zx.ori_data = transed.val zx.ctime_stamp = self.now_stamp zx.save() log_db.save_cache = self.RIGHT logger.info("save cache success") except: log_db.save_cache = self.WRONG logger.error("save_cache", exc_info=True) raise Exception("save cache failed")
def post(self,request)->HttpResponse: default_data = "" try: zx = ZhengXin91Shared() resp,state = zx.main(request.body) if state: logger.info("query table success") return HttpResponse(resp) else: logger.info("query failed,using default data") e = UnexpectedHandler() default_data = e.except_resp(request.body) return HttpResponse(default_data) except: logger.error("Exception error",exc_info = True) return HttpResponse(default_data)
def EMR002(self,data:dict): emr = [] for d in data: emr.append( HDEMD008EMR002( emd_id = self.emd_id, name=self.name, mobile=self.phone, idcard=self.idcard, cycle=self.cycle, platform_code=d.get("PLATFORMCODE", "empty"), register_time=d.get("REGISTERTIME", "empty"), p_type = d.get("P_TYPE", "empty"), ) ) HDEMD008EMR002.objects.using(self.flag).bulk_create(emr) logger.info("emd008:emr002 save succeed")
def EMR007(self,data:dict): emr = [] for d in data: emr.append( HDEMD008EMR007( emd_id=self.emd_id, name = self.name, mobile = self.phone, idcard = self.idcard, cycle = self.cycle, platform_code = d.get("PLATFORMCODE", "empty"), loan_lenders_time = d.get("LOANLENDERSTIME", "empty"), loan_lenders_amount = d.get("LOANLENDERSAMOUNT", "empty"), p_type = d.get("P_TYPE", "empty"), ) ) HDEMD008EMR007.objects.using(self.flag).bulk_create(emr) logger.info("emd008:emr007 save succeed")
def query(self, parsed: dict, log_db) -> dict: try: zx = self.ZhengXin91LoanCoreInfo loan_info = self.make_module(parsed) loan_infos = {"loan_infos": [loan_info]} state = loan_info[zx.BORROWSTATE] if state == self._borrowState().loaned: uid, _ = self._has_applied(parsed) JkUserLoanRecord, _ = self._has_loaned(uid) loan_info[zx.BORROWAMOUNT] = JkUserLoanRecord.amount loan_info[zx.CONTRACTDATE] = JkUserLoanRecord.pay_time log_db.query_table = self.RIGHT logger.info("query success") return loan_infos except: log_db.query_table = self.WRONG logger.error("query failed", exc_info=True) raise Exception("query")
def storage(self, parsed: tuple, data: dict, c): try: self._store_ori(parsed, data) code, concrete = parsed assert code == self.Code.SYNCHRONOUS_INTERFACE_CODE flag = data[self.FLAG] if flag == self.BSDFLAG: self._store_bsd(parsed, data) elif flag == self.SSDFLAG: self._store_ssd(parsed, data) else: raise Exception(f"Unexpected flag {flag}") c.insert_res = self.RIGHT logger.info("storage success") except: logger.error("storage error", exc_info=True) c.insert_res = self.WRONG raise Exception("storage error")
def _get_newtoken(self,table)->str: token = "" try: if table == hd_token.__name__: t = hd_token.objects.using("ssd").values("token").order_by("-id") assert len(t) > 0 token = t[0].get("token","") elif table == hd_tokenwebloan.__name__: t = hd_tokenwebloan.objects.using("ssd").values("token").order_by("-id") assert len(t) > 0 token = t[0].get("token","") else: Exception("_get_newtoken unexpect!") logger.error("ERROR",exc_info=True) except: logger.error("get newtoken error",exc_info=True) raise Exception("get newtoken error") logger.info(f"get newtoken {token} from {table}") return token
def store_detail(self,emd_id:int): msg = "hd emd008 loan detail" try: for r in self.result: if len(r) == 0: logger.info("one of hd emd008 details is empty") else: setattr(EMD008Detail,"emd_id",emd_id) setattr(EMD008Detail,"name",self.name) setattr(EMD008Detail,"idcard",self.idcard) setattr(EMD008Detail,"phone",self.phone) setattr(EMD008Detail,"flag",self.flag) setattr(EMD008Detail,"cycle",r["CYCLE"]) emd008 = EMD008Detail() func = getattr(emd008,r["TYPE"]) func(r["DATA"]) logger.info(f"{msg} successfully stored") except: logger.error(f"{msg} failed",exc_info=True)
def hd_illegal_info(self)->dict: # EMW018不良信息查询, # http://opensdk.emay.cn:9099/SF_YZ_API/SFService.asmx?op=Get_EMW_IllegalInfo_BLXXCX try: # crawler token = self._get_newtoken("hd_token") data = dict( name=self.name, idcard=self.idcard, ACCESS_TOKEN=token, ) if self.flag == self.BSDFLAG: parseRes = self._hd_illegal_info_judge_before_crawl(data) bsd = bsd_hy_illegalinfo_summary() bsd.seq_id = self.seqId bsd.p_person_id = self.idcard bsd.p_person_name = self.name bsd.CODE = "200" bsd.result = parseRes.get("RESULTDESC","") bsd.resultdesc = parseRes.get("MESSAGE","") bsd.save(using=self.HMDB) elif self.flag == self.SSDFLAG: parseRes = self.req_hdInterface(self.hdIllegalInfoUrl, data) ssd = cfd_hd_IllegalInfo_BLXXCX() ssd.p_person_id = self.idcard ssd.p_person_name = self.name ssd.CODE = "200" ssd.result = parseRes.get("RESULTDESC","") ssd.resultdesc = parseRes.get("MESSAGE","") ssd.save(using=self.SSDDB) # storage Hd_EMW018 ori = Hd_EMW018() ori.flag = self.flag ori.idcard = self.idcard ori.phone = self.phone ori.name = self.name ori.ori_data = json.dumps(parseRes) ori.save() except: logger.error("EMW018不良信息查询 =>",exc_info=True) return self.FAILED logger.info(f"{self.flag} {self.name},{self.idcard} insert into success") return self.SUCCESS