def _hd_blackList_detail_judge_before_crawl(self)->dict: try: bsd_n = bsd_hy_blacklist_hit_detail queryRes = bsd_n.objects.using(self.HMDB).filter(p_person_id=self.idcard).order_by("-request_id") if len(queryRes) == 0: res = self._EMD008_integrate_interface("24") else: cln = queryRes[:1][0] if self.detect_expire(cln.ctime): res = self._EMD008_integrate_interface("24") else: res = { "RESULTS": [ {}, {}, {}, {}, {"DATA": [ { "MONEY": cln.overdue_amount_most, "COUNTS": cln.overdue_count, "D_TIME": cln.overdue_time_latest, } ]} ] } except: logger.error("_hd_blackList_detail_judge_before_crawl",exc_info=True) res = self._EMD008_integrate_interface("24") return res
def _decode(self, extracted: str) -> "dict/str": de_res = {} try: de_res = self.zhengxin_decode(extracted) except: logger.error("_decode", exc_info=True) return de_res
def main(self, rcv: str) -> (str, bool): log_db = ZhengXin91ShareLog() try: self.verify(rcv, log_db) parsed = self.parse(rcv, log_db) self.verify_meanful_rcv(parsed, log_db) if self.not_expire(ZhengXin91ShareOriData, parsed): try: extracted = self.extract_not_expire(parsed) log_db.note = "response from cache, succeed!" self.save_log(self.RIGHT, log_db) logger.info(log_db.note) return extracted, True except: log_db.note = "response from cache, failed!" self.save_log(self.WRONG, log_db) logger.info(log_db.note) raise Exception(self.Error.EXISTED_FAIL_HINT) loan_data = self.query(parsed, log_db) transed = self.trans(loan_data, log_db) self.save_cache(parsed, transed, log_db) self.save_log(self.RIGHT, log_db) return transed.val, True except: self.save_log(self.WRONG, log_db) logger.error("shareLoanInfo91 failed", exc_info=True) return "", False
def main(self, data: dict) -> bool: c = zhengxin91_crawler_log() try: self.verify_query(data, c) if data[self.FLAG] == self.BSDFLAG and self.not_expire( zhengxin91_content, data): try: self.update_existed(data) self.update_succLog(c) logger.info("the data has been in database,update ctime") return True except: self.update_failLog(c) raise Exception(self.Error.EXISTED_FAIL_HINT) resp = self.request(data, c) logger.info("this is receive response") logger.info(resp) parsed = self.parse(resp, c) logger.info("this is parsed") logger.info(self.json_marshal(parsed[1])) self.storage(parsed, data, c) self.save_log(self.RIGHT, c) return True except: self.save_log(self.WRONG, c) logger.error("query_storage error", exc_info=True) return False
def exe_sql(self, sqlString, values): db = pymysql.connect( host=self.host, port=self.port, user=self.usr, passwd=self.pwd, db=self.dbName, charset='utf8' ) cursor = db.cursor() try: cursor.execute(sqlString, values) if sqlString.split()[0] == "select": returnData = cursor.fetchall() return returnData else: db.commit() return True except: logger.error("exe_sql error",exc_info = True) print("ERROR SQL & VAL",sqlString, values) db.rollback() traceback.print_exc() return False finally: cursor.close()
def hd_blackList(self)->dict: # EMW025 黑名单模糊汇总查询 try: # crawler token = self._get_newtoken("hd_token") data = dict( idcard=self.idcard, ACCESS_TOKEN=token, ) # storage if self.flag == self.BSDFLAG: ## storage in bsd_hy_blacklist_hit_summary parseRes = self._hd_blackList_judge_before_crawl(data) bsd = bsd_hy_blacklist_hit_summary() bsd.p_person_id = self.idcard bsd.p_person_name = self.name bsd.p_phone = self.phone bsd.query_id = self.queryId bsd.status = "1000" bsd.black_risk_type = "网贷黑名单" bsd.black_hit_count = parseRes.get("WDHMDCOUNT","") bsd.save(using=self.HMDB) # storage in bsd_blacklisthitnum db = bsd_blacklisthitnum() db.seq_id = self.seqId db.p_person_id = self.idcard db.black_risk_type = "" db.black_hit_count = parseRes.get("WDHMDCOUNT",0) db.CODE = "200" db.data_source = "hd" db.save(using=self.HMDB) elif self.flag == self.SSDFLAG: parseRes = self.req_hdInterface(self.hdBlackListUrl, data) ssd = cfd_hd_huifamohufact() ssd.p_idcode = self.idcard ssd.status = parseRes.get("CODE","") ssd.totalnumber = parseRes.get("totalCount","") ssd.casenumber = parseRes.get("ajlcCount","") ssd.zhixinnumber = parseRes.get("zxggCount","") ssd.taxnumber = "" ssd.cuiqiannumber = "" ssd.kaitinnumber = parseRes.get("ktggCount","") ssd.othernumber = "" ssd.shenpannumber = parseRes.get("cpwsCount","") ssd.wangdainumber = parseRes.get("wdhmdCount","") ssd.success = "200" ssd.save(using=self.SSDDB) # storage in crawler ori = Hd_EMW025() ori.flag = self.flag ori.idcard = self.idcard ori.phone = self.phone ori.name = self.name ori.ori_data = json.dumps(parseRes) ori.save() except: logger.error("hd_blackList failed",exc_info=True) return self.FAILED logger.info("EMW025 黑名单模糊汇总查询 success") return self.SUCCESS
def hd_webloan(self)->dict: # EMR002 信贷平台注册详情,统计这个接口的数量 try: for cycle in ("3","6","9","12","24"): self._hd_webloan(cycle) except: logger.error("EMR002 =>", exc_info=True) return self.FAILED logger.info("EMR002 信贷平台注册详情/webloan success") return self.SUCCESS
def parse(self, resp: str, c) -> tuple: try: status_code, extracted = self._extract(resp) py_obj = self._decode(extracted) c.parse_res = self.RIGHT return status_code, py_obj except: logger.error("parse error", exc_info=True) c.parse_res = self.WRONG raise Exception("parse error")
def notEmptyDict(self, adict: dict) -> bool: try: assert isinstance(adict, dict) for k in adict: assert adict[k] != "" return True except: print("assert dict", adict) logger.error("not Empty dict", exc_info=True) return False
def verify(self, rcv: str, log_db): try: assert len(rcv) > 0 log_db.verify_rcv = self.RIGHT logger.info("verify receive success") except: logger.error(self.Error.ILLEAGL_NOTE, exc_info=True) log_db.verify_rcv = self.WRONG log_db.note = self.Error.ILLEAGL_NOTE raise Exception("verify receive => False")
def trans(self, loan_data: dict, log_db) -> namedtuple: S = self.step_return() try: encoded = self.zhengxin_encode(loan_data) log_db.trans_res = self.RIGHT logger.info("trans success") return S(val=self.make_resp_template(encoded), res=False) except: log_db.trans_res = self.WRONG logger.error("trans failed", exc_info=True) raise Exception("trans failed")
def parse_data(self): try: self.code = self.data["CODE"] self.province = self.data["PROVINCE"] self.city = self.data["CITY"] self.result = self.data["RESULTS"] logger.info(f"emd008 parse succeed,{self.code},{self.phone},{self.province},{self.city}") except: err = "emd008 parse failed" logger.error(err,exc_info = True) raise Exception(err)
def hd_webloandetail(self)->dict: try: logger.info("hd_webloandetail started") cycle = ["3","6","9","12","24"] for c in cycle: self._hd_webloandetail_oneCycle(c) logger.info("hd_webloandetail end") except: logger.error("hd_webloandetail=>",exc_info=True) return self.FAILED logger.info("webloandetail success") return self.SUCCESS
def request(self, data, c) -> str: try: user_data = self._encode_query(data) resp = requests.post(self.Authentication.QUERY_URL, data=user_data) c.request_res = self.RIGHT logger.info("request success") return resp.text except: c.request_res = self.WRONG tag = "request error" logger.error(tag, exc_info=True) raise Exception(tag)
def post(self,request)->JsonResponse: try: py_obj = json.loads(request.body) zx = ZhengXin91() if zx.main(py_obj): logger.info("Accepted_Query: zhengxin91storage") return JsonResponse(self.success) else: raise Exception("zhengxin91 query or storatrge error") except: logger.error("zhengxin91 error traceback",exc_info=True) return JsonResponse(self.classNum_empty)
def parse(self, rcv: bytes, log_db) -> dict: try: to_list = rcv.decode().split("|") assert len(to_list) == self.ZX_LENGTH assert to_list[3] == self.Code.SHARE_REQ_CODE user_data = self.zhengxin_decode(to_list[6]) log_db.parse_res = self.RIGHT logger.info("parse success") return user_data except: logger.error("parse failed", exc_info=True) log_db.parse_res = self.WRONG raise Exception("parse failed")
def deltoken(self, table: str) -> str: msg = "" try: if table == self.t: hd_token.objects.using("token").all().delete() elif table == self.t2: # t = hd_tokenwebloan() # t.delete(using="token") hd_tokenwebloan.objects.using("token").all().delete() logger.info(f"{table} delete old token") except: logger.error("del error", exc_info=True) msg = traceback.format_exc() return msg
def parse_ReceivedData(self, request) -> DataStruct: d = DataStruct() try: parse_res = json.loads(request.body) d.idcard = parse_res.get(self.post_idcard, "empty") d.name = parse_res.get(self.post_name, "empty") d.phone = parse_res.get(self.post_phone, "empty") d.token = parse_res.get(self.post_token, "empty") d.seqId = parse_res.get(self.post_seqId, "empty") d.flag = parse_res.get(self.flag, "empty") except: print("accept request.body ==>", request.body) logger.error("", exc_info=True) return d
def _save_bsd_loanapplicationnum(self,cycle,times): try: logger.info(f"bsd_loanapplicationnum saved;user phone number=>{self.phone}") db = bsd_loanapplicationnum() db.seq_id = self.seqId db.p_Phone = self.phone db.p_cycle = cycle db.p_Platform = "0" db.CODE = "200" db.CYCLE = "".join((str(int(time.time())), "000")) db.LOANNUM = times db.data_source = "hd" db.save(using=self.HMDB) except: logger.error("error",exc_info=True)
def verify_query(self, data: dict, c): try: assert type(data) == dict assert data[self.UserInfo.REC_NAME] != "" assert data[self.UserInfo.REC_IDCARD] != "" assert data[self.FLAG] != "" c.verify_res = self.RIGHT c.idcard = data.get(self.UserInfo.REC_IDCARD, "") c.name = data.get(self.UserInfo.REC_NAME, "") logger.info("verify query success") except: c.verify_res = self.WRONG c.note = self.Error.ILLEAGL_NOTE logger.error("verify error", exc_info=True) raise Exception(self.Error.ILLEAGL_NOTE)
def post(self,request)->HttpResponse: default_data = "" try: zx = ZhengXin91Shared() resp,state = zx.main(request.body) if state: logger.info("query table success") return HttpResponse(resp) else: logger.info("query failed,using default data") e = UnexpectedHandler() default_data = e.except_resp(request.body) return HttpResponse(default_data) except: logger.error("Exception error",exc_info = True) return HttpResponse(default_data)
def save_cache(self, parsed: dict, transed: namedtuple, log_db): try: name, idcard, _ = self._user_info(parsed) zx = ZhengXin91ShareOriData() zx.name = name zx.idcard = idcard zx.status_code = self.Code.SHARE_EXISTED zx.ori_data = transed.val zx.ctime_stamp = self.now_stamp zx.save() log_db.save_cache = self.RIGHT logger.info("save cache success") except: log_db.save_cache = self.WRONG logger.error("save_cache", exc_info=True) raise Exception("save cache failed")
def query(self, parsed: dict, log_db) -> dict: try: zx = self.ZhengXin91LoanCoreInfo loan_info = self.make_module(parsed) loan_infos = {"loan_infos": [loan_info]} state = loan_info[zx.BORROWSTATE] if state == self._borrowState().loaned: uid, _ = self._has_applied(parsed) JkUserLoanRecord, _ = self._has_loaned(uid) loan_info[zx.BORROWAMOUNT] = JkUserLoanRecord.amount loan_info[zx.CONTRACTDATE] = JkUserLoanRecord.pay_time log_db.query_table = self.RIGHT logger.info("query success") return loan_infos except: log_db.query_table = self.WRONG logger.error("query failed", exc_info=True) raise Exception("query")
def _hd_blackList_judge_before_crawl(self,data:dict)->dict: try: bsd_n = bsd_hy_blacklist_hit_summary queryRes = bsd_n.objects.using(self.HMDB).filter(p_person_id=self.idcard).order_by("-request_id") if len(queryRes) == 0: parseRes = self.req_hdInterface(self.hdBlackListUrl, data) else: cln = queryRes[:1][0] if self.detect_expire(cln.ctime): parseRes = self.req_hdInterface(self.hdBlackListUrl, data) else: parseRes = { "WDHMDCOUNT": cln.black_hit_count, } except: logger.error("_hd_blackList_judge_before_crawl",exc_info = True) parseRes = self.req_hdInterface(self.hdBlackListUrl, data) return parseRes
def storage(self, parsed: tuple, data: dict, c): try: self._store_ori(parsed, data) code, concrete = parsed assert code == self.Code.SYNCHRONOUS_INTERFACE_CODE flag = data[self.FLAG] if flag == self.BSDFLAG: self._store_bsd(parsed, data) elif flag == self.SSDFLAG: self._store_ssd(parsed, data) else: raise Exception(f"Unexpected flag {flag}") c.insert_res = self.RIGHT logger.info("storage success") except: logger.error("storage error", exc_info=True) c.insert_res = self.WRONG raise Exception("storage error")
def _hd_illegal_info_judge_before_crawl(self,data:dict)->dict: try: bsd_n = bsd_hy_illegalinfo_summary queryRes = bsd_n.objects.using(self.HMDB).filter(p_person_id=self.idcard).order_by("-request_id") if len(queryRes) == 0: parseRes = self.req_hdInterface(self.hdIllegalInfoUrl, data) else: cln = queryRes[:1][0] if self.detect_expire(cln.ctime): parseRes = self.req_hdInterface(self.hdIllegalInfoUrl, data) else: parseRes = { "RESULTDESC": cln.result, "MESSAGE": cln.resultdesc, } except: logger.error("_hd_illegal_info_judge_before_crawl",exc_info = True) parseRes = self.req_hdInterface(self.hdIllegalInfoUrl, data) return parseRes
def hd_illegal_info(self)->dict: # EMW018不良信息查询, # http://opensdk.emay.cn:9099/SF_YZ_API/SFService.asmx?op=Get_EMW_IllegalInfo_BLXXCX try: # crawler token = self._get_newtoken("hd_token") data = dict( name=self.name, idcard=self.idcard, ACCESS_TOKEN=token, ) if self.flag == self.BSDFLAG: parseRes = self._hd_illegal_info_judge_before_crawl(data) bsd = bsd_hy_illegalinfo_summary() bsd.seq_id = self.seqId bsd.p_person_id = self.idcard bsd.p_person_name = self.name bsd.CODE = "200" bsd.result = parseRes.get("RESULTDESC","") bsd.resultdesc = parseRes.get("MESSAGE","") bsd.save(using=self.HMDB) elif self.flag == self.SSDFLAG: parseRes = self.req_hdInterface(self.hdIllegalInfoUrl, data) ssd = cfd_hd_IllegalInfo_BLXXCX() ssd.p_person_id = self.idcard ssd.p_person_name = self.name ssd.CODE = "200" ssd.result = parseRes.get("RESULTDESC","") ssd.resultdesc = parseRes.get("MESSAGE","") ssd.save(using=self.SSDDB) # storage Hd_EMW018 ori = Hd_EMW018() ori.flag = self.flag ori.idcard = self.idcard ori.phone = self.phone ori.name = self.name ori.ori_data = json.dumps(parseRes) ori.save() except: logger.error("EMW018不良信息查询 =>",exc_info=True) return self.FAILED logger.info(f"{self.flag} {self.name},{self.idcard} insert into success") return self.SUCCESS
def _get_newtoken(self,table)->str: token = "" try: if table == hd_token.__name__: t = hd_token.objects.using("ssd").values("token").order_by("-id") assert len(t) > 0 token = t[0].get("token","") elif table == hd_tokenwebloan.__name__: t = hd_tokenwebloan.objects.using("ssd").values("token").order_by("-id") assert len(t) > 0 token = t[0].get("token","") else: Exception("_get_newtoken unexpect!") logger.error("ERROR",exc_info=True) except: logger.error("get newtoken error",exc_info=True) raise Exception("get newtoken error") logger.info(f"get newtoken {token} from {table}") return token
def get_token(self, table: str, appid: str, appsecret: str, key: str) -> tuple: err = "" token = "" effective = "" res = "" try: client = Client(self.url) res = client.service.GetACCESS_TOKEN(AppID=appid, AppSecret=appsecret, Key=key) parseRes = json.loads(res) access_token = parseRes["access_token"] effective = parseRes["Effective"] token = access_token except: logger.error("update token", exc_info=True) err = f"get {table} data error" return (token, effective, err)
def _hd_webloandetail_oneCycle_judge_before_crawler(self,cycle:str)->int: try: bsd_n = bsd_hy_webloan queryRes = bsd_n.objects.using(self.HMDB).filter(p_id_card=self.idcard).order_by("-request_id") if len(queryRes) == 0: res = self._EMD008_integrate_interface(cycle) time = len(res.get("RESULTS",[{}])[0].get("DATA",{})) else: cln = queryRes[:1][0] if self.detect_expire(cln.ctime): res = self._EMD008_integrate_interface(cycle) time = len(res.get("RESULTS", [{}])[0].get("DATA", {})) else: time = cln.ttl_hit_count except: logger.error("_hd_webloandetail_oneCycle_judge_before_crawler",exc_info = True) res = self._EMD008_integrate_interface(cycle) time = len(res.get("RESULTS", [{}])[0].get("DATA", {})) return time