def _hd_blackList_detail_judge_before_crawl(self)->dict:
try:
bsd_n = bsd_hy_blacklist_hit_detail
queryRes = bsd_n.objects.using(self.HMDB).filter(p_person_id=self.idcard).order_by("-request_id")
if len(queryRes) == 0:
res = self._EMD008_integrate_interface("24")
else:
cln = queryRes[:1][0]
if self.detect_expire(cln.ctime):
res = self._EMD008_integrate_interface("24")
else:
res = {
"RESULTS": [
{}, {}, {}, {},
{"DATA": [
{
"MONEY": cln.overdue_amount_most,
"COUNTS": cln.overdue_count,
"D_TIME": cln.overdue_time_latest,
}
]}
]
}
except:
logger.error("_hd_blackList_detail_judge_before_crawl",exc_info=True)
res = self._EMD008_integrate_interface("24")
return res
def _decode(self, extracted: str) -> "dict/str":
de_res = {}
try:
de_res = self.zhengxin_decode(extracted)
except:
logger.error("_decode", exc_info=True)
return de_res
def main(self, rcv: str) -> (str, bool):
log_db = ZhengXin91ShareLog()
try:
self.verify(rcv, log_db)
parsed = self.parse(rcv, log_db)
self.verify_meanful_rcv(parsed, log_db)
if self.not_expire(ZhengXin91ShareOriData, parsed):
try:
extracted = self.extract_not_expire(parsed)
log_db.note = "response from cache, succeed!"
self.save_log(self.RIGHT, log_db)
logger.info(log_db.note)
return extracted, True
except:
log_db.note = "response from cache, failed!"
self.save_log(self.WRONG, log_db)
logger.info(log_db.note)
raise Exception(self.Error.EXISTED_FAIL_HINT)
loan_data = self.query(parsed, log_db)
transed = self.trans(loan_data, log_db)
self.save_cache(parsed, transed, log_db)
self.save_log(self.RIGHT, log_db)
return transed.val, True
except:
self.save_log(self.WRONG, log_db)
logger.error("shareLoanInfo91 failed", exc_info=True)
return "", False
def main(self, data: dict) -> bool:
c = zhengxin91_crawler_log()
try:
self.verify_query(data, c)
if data[self.FLAG] == self.BSDFLAG and self.not_expire(
zhengxin91_content, data):
try:
self.update_existed(data)
self.update_succLog(c)
logger.info("the data has been in database,update ctime")
return True
except:
self.update_failLog(c)
raise Exception(self.Error.EXISTED_FAIL_HINT)
resp = self.request(data, c)
logger.info("this is receive response")
logger.info(resp)
parsed = self.parse(resp, c)
logger.info("this is parsed")
logger.info(self.json_marshal(parsed[1]))
self.storage(parsed, data, c)
self.save_log(self.RIGHT, c)
return True
except:
self.save_log(self.WRONG, c)
logger.error("query_storage error", exc_info=True)
return False
def exe_sql(self, sqlString, values):
db = pymysql.connect(
host=self.host,
port=self.port,
user=self.usr,
passwd=self.pwd,
db=self.dbName,
charset='utf8'
)
cursor = db.cursor()
try:
cursor.execute(sqlString, values)
if sqlString.split()[0] == "select":
returnData = cursor.fetchall()
return returnData
else:
db.commit()
return True
except:
logger.error("exe_sql error",exc_info = True)
print("ERROR SQL & VAL",sqlString, values)
db.rollback()
traceback.print_exc()
return False
finally:
cursor.close()
def hd_blackList(self)->dict:
# EMW025 黑名单模糊汇总查询
try:
# crawler
token = self._get_newtoken("hd_token")
data = dict(
idcard=self.idcard,
ACCESS_TOKEN=token,
)
# storage
if self.flag == self.BSDFLAG:
## storage in bsd_hy_blacklist_hit_summary
parseRes = self._hd_blackList_judge_before_crawl(data)
bsd = bsd_hy_blacklist_hit_summary()
bsd.p_person_id = self.idcard
bsd.p_person_name = self.name
bsd.p_phone = self.phone
bsd.query_id = self.queryId
bsd.status = "1000"
bsd.black_risk_type = "网贷黑名单"
bsd.black_hit_count = parseRes.get("WDHMDCOUNT","")
bsd.save(using=self.HMDB)
# storage in bsd_blacklisthitnum
db = bsd_blacklisthitnum()
db.seq_id = self.seqId
db.p_person_id = self.idcard
db.black_risk_type = ""
db.black_hit_count = parseRes.get("WDHMDCOUNT",0)
db.CODE = "200"
db.data_source = "hd"
db.save(using=self.HMDB)
elif self.flag == self.SSDFLAG:
parseRes = self.req_hdInterface(self.hdBlackListUrl, data)
ssd = cfd_hd_huifamohufact()
ssd.p_idcode = self.idcard
ssd.status = parseRes.get("CODE","")
ssd.totalnumber = parseRes.get("totalCount","")
ssd.casenumber = parseRes.get("ajlcCount","")
ssd.zhixinnumber = parseRes.get("zxggCount","")
ssd.taxnumber = ""
ssd.cuiqiannumber = ""
ssd.kaitinnumber = parseRes.get("ktggCount","")
ssd.othernumber = ""
ssd.shenpannumber = parseRes.get("cpwsCount","")
ssd.wangdainumber = parseRes.get("wdhmdCount","")
ssd.success = "200"
ssd.save(using=self.SSDDB)
# storage in crawler
ori = Hd_EMW025()
ori.flag = self.flag
ori.idcard = self.idcard
ori.phone = self.phone
ori.name = self.name
ori.ori_data = json.dumps(parseRes)
ori.save()
except:
logger.error("hd_blackList failed",exc_info=True)
return self.FAILED
logger.info("EMW025 黑名单模糊汇总查询 success")
return self.SUCCESS
def hd_webloan(self)->dict:
# EMR002 信贷平台注册详情，统计这个接口的数量
try:
for cycle in ("3","6","9","12","24"):
self._hd_webloan(cycle)
except:
logger.error("EMR002 =>", exc_info=True)
return self.FAILED
logger.info("EMR002 信贷平台注册详情/webloan success")
return self.SUCCESS
def parse(self, resp: str, c) -> tuple:
try:
status_code, extracted = self._extract(resp)
py_obj = self._decode(extracted)
c.parse_res = self.RIGHT
return status_code, py_obj
except:
logger.error("parse error", exc_info=True)
c.parse_res = self.WRONG
raise Exception("parse error")
def notEmptyDict(self, adict: dict) -> bool:
try:
assert isinstance(adict, dict)
for k in adict:
assert adict[k] != ""
return True
except:
print("assert dict", adict)
logger.error("not Empty dict", exc_info=True)
return False
def verify(self, rcv: str, log_db):
try:
assert len(rcv) > 0
log_db.verify_rcv = self.RIGHT
logger.info("verify receive success")
except:
logger.error(self.Error.ILLEAGL_NOTE, exc_info=True)
log_db.verify_rcv = self.WRONG
log_db.note = self.Error.ILLEAGL_NOTE
raise Exception("verify receive => False")
def trans(self, loan_data: dict, log_db) -> namedtuple:
S = self.step_return()
try:
encoded = self.zhengxin_encode(loan_data)
log_db.trans_res = self.RIGHT
logger.info("trans success")
return S(val=self.make_resp_template(encoded), res=False)
except:
log_db.trans_res = self.WRONG
logger.error("trans failed", exc_info=True)
raise Exception("trans failed")
def parse_data(self):
try:
self.code = self.data["CODE"]
self.province = self.data["PROVINCE"]
self.city = self.data["CITY"]
self.result = self.data["RESULTS"]
logger.info(f"emd008 parse succeed,{self.code},{self.phone},{self.province},{self.city}")
except:
err = "emd008 parse failed"
logger.error(err,exc_info = True)
raise Exception(err)
def hd_webloandetail(self)->dict:
try:
logger.info("hd_webloandetail started")
cycle = ["3","6","9","12","24"]
for c in cycle:
self._hd_webloandetail_oneCycle(c)
logger.info("hd_webloandetail end")
except:
logger.error("hd_webloandetail=>",exc_info=True)
return self.FAILED
logger.info("webloandetail success")
return self.SUCCESS
def request(self, data, c) -> str:
try:
user_data = self._encode_query(data)
resp = requests.post(self.Authentication.QUERY_URL, data=user_data)
c.request_res = self.RIGHT
logger.info("request success")
return resp.text
except:
c.request_res = self.WRONG
tag = "request error"
logger.error(tag, exc_info=True)
raise Exception(tag)
def post(self,request)->JsonResponse:
try:
py_obj = json.loads(request.body)
zx = ZhengXin91()
if zx.main(py_obj):
logger.info("Accepted_Query: zhengxin91storage")
return JsonResponse(self.success)
else:
raise Exception("zhengxin91 query or storatrge error")
except:
logger.error("zhengxin91 error traceback",exc_info=True)
return JsonResponse(self.classNum_empty)
def parse(self, rcv: bytes, log_db) -> dict:
try:
to_list = rcv.decode().split("|")
assert len(to_list) == self.ZX_LENGTH
assert to_list[3] == self.Code.SHARE_REQ_CODE
user_data = self.zhengxin_decode(to_list[6])
log_db.parse_res = self.RIGHT
logger.info("parse success")
return user_data
except:
logger.error("parse failed", exc_info=True)
log_db.parse_res = self.WRONG
raise Exception("parse failed")
def deltoken(self, table: str) -> str:
msg = ""
try:
if table == self.t:
hd_token.objects.using("token").all().delete()
elif table == self.t2:
# t = hd_tokenwebloan()
# t.delete(using="token")
hd_tokenwebloan.objects.using("token").all().delete()
logger.info(f"{table} delete old token")
except:
logger.error("del error", exc_info=True)
msg = traceback.format_exc()
return msg
def parse_ReceivedData(self, request) -> DataStruct:
d = DataStruct()
try:
parse_res = json.loads(request.body)
d.idcard = parse_res.get(self.post_idcard, "empty")
d.name = parse_res.get(self.post_name, "empty")
d.phone = parse_res.get(self.post_phone, "empty")
d.token = parse_res.get(self.post_token, "empty")
d.seqId = parse_res.get(self.post_seqId, "empty")
d.flag = parse_res.get(self.flag, "empty")
except:
print("accept request.body ==>", request.body)
logger.error("", exc_info=True)
return d
def _save_bsd_loanapplicationnum(self,cycle,times):
try:
logger.info(f"bsd_loanapplicationnum saved;user phone number=>{self.phone}")
db = bsd_loanapplicationnum()
db.seq_id = self.seqId
db.p_Phone = self.phone
db.p_cycle = cycle
db.p_Platform = "0"
db.CODE = "200"
db.CYCLE = "".join((str(int(time.time())), "000"))
db.LOANNUM = times
db.data_source = "hd"
db.save(using=self.HMDB)
except:
logger.error("error",exc_info=True)
def verify_query(self, data: dict, c):
try:
assert type(data) == dict
assert data[self.UserInfo.REC_NAME] != ""
assert data[self.UserInfo.REC_IDCARD] != ""
assert data[self.FLAG] != ""
c.verify_res = self.RIGHT
c.idcard = data.get(self.UserInfo.REC_IDCARD, "")
c.name = data.get(self.UserInfo.REC_NAME, "")
logger.info("verify query success")
except:
c.verify_res = self.WRONG
c.note = self.Error.ILLEAGL_NOTE
logger.error("verify error", exc_info=True)
raise Exception(self.Error.ILLEAGL_NOTE)
def post(self,request)->HttpResponse:
default_data = ""
try:
zx = ZhengXin91Shared()
resp,state = zx.main(request.body)
if state:
logger.info("query table success")
return HttpResponse(resp)
else:
logger.info("query failed,using default data")
e = UnexpectedHandler()
default_data = e.except_resp(request.body)
return HttpResponse(default_data)
except:
logger.error("Exception error",exc_info = True)
return HttpResponse(default_data)
def save_cache(self, parsed: dict, transed: namedtuple, log_db):
try:
name, idcard, _ = self._user_info(parsed)
zx = ZhengXin91ShareOriData()
zx.name = name
zx.idcard = idcard
zx.status_code = self.Code.SHARE_EXISTED
zx.ori_data = transed.val
zx.ctime_stamp = self.now_stamp
zx.save()
log_db.save_cache = self.RIGHT
logger.info("save cache success")
except:
log_db.save_cache = self.WRONG
logger.error("save_cache", exc_info=True)
raise Exception("save cache failed")
def query(self, parsed: dict, log_db) -> dict:
try:
zx = self.ZhengXin91LoanCoreInfo
loan_info = self.make_module(parsed)
loan_infos = {"loan_infos": [loan_info]}
state = loan_info[zx.BORROWSTATE]
if state == self._borrowState().loaned:
uid, _ = self._has_applied(parsed)
JkUserLoanRecord, _ = self._has_loaned(uid)
loan_info[zx.BORROWAMOUNT] = JkUserLoanRecord.amount
loan_info[zx.CONTRACTDATE] = JkUserLoanRecord.pay_time
log_db.query_table = self.RIGHT
logger.info("query success")
return loan_infos
except:
log_db.query_table = self.WRONG
logger.error("query failed", exc_info=True)
raise Exception("query")
def _hd_blackList_judge_before_crawl(self,data:dict)->dict:
try:
bsd_n = bsd_hy_blacklist_hit_summary
queryRes = bsd_n.objects.using(self.HMDB).filter(p_person_id=self.idcard).order_by("-request_id")
if len(queryRes) == 0:
parseRes = self.req_hdInterface(self.hdBlackListUrl, data)
else:
cln = queryRes[:1][0]
if self.detect_expire(cln.ctime):
parseRes = self.req_hdInterface(self.hdBlackListUrl, data)
else:
parseRes = {
"WDHMDCOUNT": cln.black_hit_count,
}
except:
logger.error("_hd_blackList_judge_before_crawl",exc_info = True)
parseRes = self.req_hdInterface(self.hdBlackListUrl, data)
return parseRes
def storage(self, parsed: tuple, data: dict, c):
try:
self._store_ori(parsed, data)
code, concrete = parsed
assert code == self.Code.SYNCHRONOUS_INTERFACE_CODE
flag = data[self.FLAG]
if flag == self.BSDFLAG:
self._store_bsd(parsed, data)
elif flag == self.SSDFLAG:
self._store_ssd(parsed, data)
else:
raise Exception(f"Unexpected flag {flag}")
c.insert_res = self.RIGHT
logger.info("storage success")
except:
logger.error("storage error", exc_info=True)
c.insert_res = self.WRONG
raise Exception("storage error")
def _hd_illegal_info_judge_before_crawl(self,data:dict)->dict:
try:
bsd_n = bsd_hy_illegalinfo_summary
queryRes = bsd_n.objects.using(self.HMDB).filter(p_person_id=self.idcard).order_by("-request_id")
if len(queryRes) == 0:
parseRes = self.req_hdInterface(self.hdIllegalInfoUrl, data)
else:
cln = queryRes[:1][0]
if self.detect_expire(cln.ctime):
parseRes = self.req_hdInterface(self.hdIllegalInfoUrl, data)
else:
parseRes = {
"RESULTDESC": cln.result,
"MESSAGE": cln.resultdesc,
}
except:
logger.error("_hd_illegal_info_judge_before_crawl",exc_info = True)
parseRes = self.req_hdInterface(self.hdIllegalInfoUrl, data)
return parseRes
def hd_illegal_info(self)->dict:
# EMW018不良信息查询,
# http://opensdk.emay.cn:9099/SF_YZ_API/SFService.asmx?op=Get_EMW_IllegalInfo_BLXXCX
try:
# crawler
token = self._get_newtoken("hd_token")
data = dict(
name=self.name,
idcard=self.idcard,
ACCESS_TOKEN=token,
)
if self.flag == self.BSDFLAG:
parseRes = self._hd_illegal_info_judge_before_crawl(data)
bsd = bsd_hy_illegalinfo_summary()
bsd.seq_id = self.seqId
bsd.p_person_id = self.idcard
bsd.p_person_name = self.name
bsd.CODE = "200"
bsd.result = parseRes.get("RESULTDESC","")
bsd.resultdesc = parseRes.get("MESSAGE","")
bsd.save(using=self.HMDB)
elif self.flag == self.SSDFLAG:
parseRes = self.req_hdInterface(self.hdIllegalInfoUrl, data)
ssd = cfd_hd_IllegalInfo_BLXXCX()
ssd.p_person_id = self.idcard
ssd.p_person_name = self.name
ssd.CODE = "200"
ssd.result = parseRes.get("RESULTDESC","")
ssd.resultdesc = parseRes.get("MESSAGE","")
ssd.save(using=self.SSDDB)
# storage Hd_EMW018
ori = Hd_EMW018()
ori.flag = self.flag
ori.idcard = self.idcard
ori.phone = self.phone
ori.name = self.name
ori.ori_data = json.dumps(parseRes)
ori.save()
except:
logger.error("EMW018不良信息查询 =>",exc_info=True)
return self.FAILED
logger.info(f"{self.flag} {self.name},{self.idcard} insert into success")
return self.SUCCESS
def _get_newtoken(self,table)->str:
token = ""
try:
if table == hd_token.__name__:
t = hd_token.objects.using("ssd").values("token").order_by("-id")
assert len(t) > 0
token = t[0].get("token","")
elif table == hd_tokenwebloan.__name__:
t = hd_tokenwebloan.objects.using("ssd").values("token").order_by("-id")
assert len(t) > 0
token = t[0].get("token","")
else:
Exception("_get_newtoken unexpect!")
logger.error("ERROR",exc_info=True)
except:
logger.error("get newtoken error",exc_info=True)
raise Exception("get newtoken error")
logger.info(f"get newtoken {token} from {table}")
return token
def get_token(self, table: str, appid: str, appsecret: str,
key: str) -> tuple:
err = ""
token = ""
effective = ""
res = ""
try:
client = Client(self.url)
res = client.service.GetACCESS_TOKEN(AppID=appid,
AppSecret=appsecret,
Key=key)
parseRes = json.loads(res)
access_token = parseRes["access_token"]
effective = parseRes["Effective"]
token = access_token
except:
logger.error("update token", exc_info=True)
err = f"get {table} data error"
return (token, effective, err)
def _hd_webloandetail_oneCycle_judge_before_crawler(self,cycle:str)->int:
try:
bsd_n = bsd_hy_webloan
queryRes = bsd_n.objects.using(self.HMDB).filter(p_id_card=self.idcard).order_by("-request_id")
if len(queryRes) == 0:
res = self._EMD008_integrate_interface(cycle)
time = len(res.get("RESULTS",[{}])[0].get("DATA",{}))
else:
cln = queryRes[:1][0]
if self.detect_expire(cln.ctime):
res = self._EMD008_integrate_interface(cycle)
time = len(res.get("RESULTS", [{}])[0].get("DATA", {}))
else:
time = cln.ttl_hit_count
except:
logger.error("_hd_webloandetail_oneCycle_judge_before_crawler",exc_info = True)
res = self._EMD008_integrate_interface(cycle)
time = len(res.get("RESULTS", [{}])[0].get("DATA", {}))
return time
