def main(self, data: dict) -> bool:
c = zhengxin91_crawler_log()
try:
self.verify_query(data, c)
if data[self.FLAG] == self.BSDFLAG and self.not_expire(
zhengxin91_content, data):
try:
self.update_existed(data)
self.update_succLog(c)
logger.info("the data has been in database,update ctime")
return True
except:
self.update_failLog(c)
raise Exception(self.Error.EXISTED_FAIL_HINT)
resp = self.request(data, c)
logger.info("this is receive response")
logger.info(resp)
parsed = self.parse(resp, c)
logger.info("this is parsed")
logger.info(self.json_marshal(parsed[1]))
self.storage(parsed, data, c)
self.save_log(self.RIGHT, c)
return True
except:
self.save_log(self.WRONG, c)
logger.error("query_storage error", exc_info=True)
return False
def post(self, request) -> JsonResponse:
logger.info("helpme accept post info")
post_info = self.parse_ReceivedData(request)
# handle post data is empty
if len(post_info.err) > 0:
logger.info(post_info.err)
return JsonResponse(post_info.err)
# instantiation hd_interface/Hd_EMW025....
logger.info("helpme instantiation hd_interface and hd saving")
hdsave = HD.hd_interface()
# multiple equals is used to storage
hdsave.name = post_info.name
hdsave.idcard = post_info.idcard
hdsave.token = post_info.token
hdsave.phone = post_info.phone
hdsave.flag = post_info.flag
hdsave.seqId = post_info.seqId
logger.info(
f"helpme accepted arguments:{hdsave.name},{hdsave.idcard},{hdsave.token},{hdsave.phone},{hdsave.flag}"
)
# 2017/10/24/10:20 => add storage in helpMe's database
res = hdsave.main()
# store emd008 details in database boshidun.
hdsave.emd008_details_storage(flag="HM")
logger.info(res.get("retmessage"))
return JsonResponse(res)
def hd_blackList(self)->dict:
# EMW025 黑名单模糊汇总查询
try:
# crawler
token = self._get_newtoken("hd_token")
data = dict(
idcard=self.idcard,
ACCESS_TOKEN=token,
)
# storage
if self.flag == self.BSDFLAG:
## storage in bsd_hy_blacklist_hit_summary
parseRes = self._hd_blackList_judge_before_crawl(data)
bsd = bsd_hy_blacklist_hit_summary()
bsd.p_person_id = self.idcard
bsd.p_person_name = self.name
bsd.p_phone = self.phone
bsd.query_id = self.queryId
bsd.status = "1000"
bsd.black_risk_type = "网贷黑名单"
bsd.black_hit_count = parseRes.get("WDHMDCOUNT","")
bsd.save(using=self.HMDB)
# storage in bsd_blacklisthitnum
db = bsd_blacklisthitnum()
db.seq_id = self.seqId
db.p_person_id = self.idcard
db.black_risk_type = ""
db.black_hit_count = parseRes.get("WDHMDCOUNT",0)
db.CODE = "200"
db.data_source = "hd"
db.save(using=self.HMDB)
elif self.flag == self.SSDFLAG:
parseRes = self.req_hdInterface(self.hdBlackListUrl, data)
ssd = cfd_hd_huifamohufact()
ssd.p_idcode = self.idcard
ssd.status = parseRes.get("CODE","")
ssd.totalnumber = parseRes.get("totalCount","")
ssd.casenumber = parseRes.get("ajlcCount","")
ssd.zhixinnumber = parseRes.get("zxggCount","")
ssd.taxnumber = ""
ssd.cuiqiannumber = ""
ssd.kaitinnumber = parseRes.get("ktggCount","")
ssd.othernumber = ""
ssd.shenpannumber = parseRes.get("cpwsCount","")
ssd.wangdainumber = parseRes.get("wdhmdCount","")
ssd.success = "200"
ssd.save(using=self.SSDDB)
# storage in crawler
ori = Hd_EMW025()
ori.flag = self.flag
ori.idcard = self.idcard
ori.phone = self.phone
ori.name = self.name
ori.ori_data = json.dumps(parseRes)
ori.save()
except:
logger.error("hd_blackList failed",exc_info=True)
return self.FAILED
logger.info("EMW025 黑名单模糊汇总查询 success")
return self.SUCCESS
def main(self, rcv: str) -> (str, bool):
log_db = ZhengXin91ShareLog()
try:
self.verify(rcv, log_db)
parsed = self.parse(rcv, log_db)
self.verify_meanful_rcv(parsed, log_db)
if self.not_expire(ZhengXin91ShareOriData, parsed):
try:
extracted = self.extract_not_expire(parsed)
log_db.note = "response from cache, succeed!"
self.save_log(self.RIGHT, log_db)
logger.info(log_db.note)
return extracted, True
except:
log_db.note = "response from cache, failed!"
self.save_log(self.WRONG, log_db)
logger.info(log_db.note)
raise Exception(self.Error.EXISTED_FAIL_HINT)
loan_data = self.query(parsed, log_db)
transed = self.trans(loan_data, log_db)
self.save_cache(parsed, transed, log_db)
self.save_log(self.RIGHT, log_db)
return transed.val, True
except:
self.save_log(self.WRONG, log_db)
logger.error("shareLoanInfo91 failed", exc_info=True)
return "", False
def save_log(self, flag, table):
status = "successfully"
if flag == self.RIGHT:
table.status = self.RIGHT
else:
table.status = self.WRONG
status = "un" + status
table.save()
logger.info(f"{table.__class__.__name__} {status} saved")
def verify(self, rcv: str, log_db):
try:
assert len(rcv) > 0
log_db.verify_rcv = self.RIGHT
logger.info("verify receive success")
except:
logger.error(self.Error.ILLEAGL_NOTE, exc_info=True)
log_db.verify_rcv = self.WRONG
log_db.note = self.Error.ILLEAGL_NOTE
raise Exception("verify receive => False")
def hd_webloan(self)->dict:
# EMR002 信贷平台注册详情，统计这个接口的数量
try:
for cycle in ("3","6","9","12","24"):
self._hd_webloan(cycle)
except:
logger.error("EMR002 =>", exc_info=True)
return self.FAILED
logger.info("EMR002 信贷平台注册详情/webloan success")
return self.SUCCESS
def _store_bsd(self, parsed: tuple, data: dict):
code, concrete = parsed
zx = zhengxin91_content()
zx.status_code = code
zx.name = data[self.UserInfo.REC_NAME]
zx.idcard = data[self.UserInfo.REC_IDCARD]
zx.ctime_stamp = self.now_stamp
if isinstance(concrete, str):
zx.err_msg = concrete
zx.save(using=self.HMDB)
raise Exception("Interface return symolic error of code")
elif isinstance(concrete, dict):
loanInfos = concrete["loanInfos"]
zx_list = []
core = self.ZhengXin91LoanCoreInfo
for loan in loanInfos:
zx_list.append(
zhengxin91_content(
status_code=0 if code == 0 else code,
name=data[self.UserInfo.REC_NAME],
idcard=data[self.UserInfo.REC_IDCARD],
ctime_stamp=self.now_stamp,
trx_no=concrete["trxNo"],
borrow_type=loan.get(core.BORROWSTATE, 0),
borrow_status=loan.get(core.BORROWSTATE, 0),
borrow_amount=loan.get(core.BORROWAMOUNT, 0),
contract_data=loan.get(core.CONTRACTDATE, ""),
loan_period=loan.get(core.LOANPERIOD, 0),
repay_state=loan.get(core.REPAYSTATE, 0),
arrears_amount=loan.get(core.ARREARSAMOUNT, 0),
company_code=loan.get(core.COMPANYCODE, ""),
))
if len(loanInfos) == 0:
logger.info("the message of zhengxin91 responsed is empty")
zx_list.append(
zhengxin91_content(
status_code=0 if code == 0 else code,
name=data[self.UserInfo.REC_NAME],
idcard=data[self.UserInfo.REC_IDCARD],
ctime_stamp=self.now_stamp,
trx_no=concrete["trxNo"],
borrow_type=0,
borrow_status=0,
borrow_amount=0,
contract_data="",
loan_period=0,
repay_state=0,
arrears_amount=0,
company_code="",
))
zhengxin91_content.objects.using(self.HMDB).bulk_create(zx_list)
else:
print(self.now, concrete)
print(self.now, type(concrete))
raise Exception("Unexpected concrete struct")
def main(self):
if len(self.data) != 5:
logger.info("hd emd008 response is {},empty!")
self.store_emd008()
return
self.parse_data()
emd_id = self.store_emd008()
if len(self.result) == 0:
logger.info("hd emd008 loan details are empty")
else:
self.store_detail(emd_id)
def trans(self, loan_data: dict, log_db) -> namedtuple:
S = self.step_return()
try:
encoded = self.zhengxin_encode(loan_data)
log_db.trans_res = self.RIGHT
logger.info("trans success")
return S(val=self.make_resp_template(encoded), res=False)
except:
log_db.trans_res = self.WRONG
logger.error("trans failed", exc_info=True)
raise Exception("trans failed")
def parse_data(self):
try:
self.code = self.data["CODE"]
self.province = self.data["PROVINCE"]
self.city = self.data["CITY"]
self.result = self.data["RESULTS"]
logger.info(f"emd008 parse succeed,{self.code},{self.phone},{self.province},{self.city}")
except:
err = "emd008 parse failed"
logger.error(err,exc_info = True)
raise Exception(err)
def request(self, data, c) -> str:
try:
user_data = self._encode_query(data)
resp = requests.post(self.Authentication.QUERY_URL, data=user_data)
c.request_res = self.RIGHT
logger.info("request success")
return resp.text
except:
c.request_res = self.WRONG
tag = "request error"
logger.error(tag, exc_info=True)
raise Exception(tag)
def post(self,request)->JsonResponse:
try:
py_obj = json.loads(request.body)
zx = ZhengXin91()
if zx.main(py_obj):
logger.info("Accepted_Query: zhengxin91storage")
return JsonResponse(self.success)
else:
raise Exception("zhengxin91 query or storatrge error")
except:
logger.error("zhengxin91 error traceback",exc_info=True)
return JsonResponse(self.classNum_empty)
def insert_Token(self, table: str, ret: str, effective: str):
if table == self.t:
t = models.hd_token()
t.token = ret
t.effctive = effective
t.save(using="token")
elif table == self.t2:
t = models.hd_tokenwebloan()
t.token = ret
t.effctive = effective
t.save(using="token")
logger.info(f"{table} insert a new token {ret},effective {effective}")
def parse(self, rcv: bytes, log_db) -> dict:
try:
to_list = rcv.decode().split("|")
assert len(to_list) == self.ZX_LENGTH
assert to_list[3] == self.Code.SHARE_REQ_CODE
user_data = self.zhengxin_decode(to_list[6])
log_db.parse_res = self.RIGHT
logger.info("parse success")
return user_data
except:
logger.error("parse failed", exc_info=True)
log_db.parse_res = self.WRONG
raise Exception("parse failed")
def store_emd008(self)->int:
hd = HDEMD008Manager()
hd.name = self.name
hd.phone = self.phone
hd.idcard = self.idcard
hd.code = self.code
hd.province = self.province
hd.city = self.city
hd.result = "0" if len(self.result) == 0 else "1"
hd.save(using=self.flag)
saved_id = hd.id
logger.info(f"store in HDEMD008Manager id is {saved_id}")
return saved_id
def _hd_webloandetail_oneCycle(self,cycle)->None:
# EMR002 信贷平台注册详情，统计这个接口的数量
if self.flag == self.BSDFLAG:
logger.info(f"{self.BSDFLAG} _hd_webloandetail started")
times = self._hd_webloandetail_oneCycle_judge_before_crawler(cycle)
if times > 0:
bsd = bsd_hy_webloan_detail()
bsd.query_id = self.queryId
bsd.period_name = f"webloan_hit_detail_m{cycle}"
bsd.period_months = cycle
bsd.class_name = "P2P网贷"
bsd.class_count = times
bsd.save(using=self.HMDB)
self._save_bsd_loanapplicationnum(cycle,str(times))
logger.info(f"{self.BSDFLAG} _hd_webloandetail ended")
elif self.flag == self.SSDFLAG:
logger.info(f"{self.SSDFLAG} _hd_webloandetail started")
res = self._EMD008_integrate_interface(cycle)
EMR002 = res.get("RESULTS",[{}])[0]
EMR002_data = EMR002.get("DATA","")
EMR002_cycle = EMR002.get("CYCLE","")
times = len(EMR002_data)
ssd = cfd_hy_webloan_detail()
ssd.query_id = self.idcard
ssd.period_name = EMR002_cycle
ssd.period_months = int(cycle)
ssd.class_name = "all"
ssd.class_count = times
ssd.save(using=self.SSDDB)
logger.info(f"{self.SSDDB} _hd_webloandetail ended")
def deltoken(self, table: str) -> str:
msg = ""
try:
if table == self.t:
hd_token.objects.using("token").all().delete()
elif table == self.t2:
# t = hd_tokenwebloan()
# t.delete(using="token")
hd_tokenwebloan.objects.using("token").all().delete()
logger.info(f"{table} delete old token")
except:
logger.error("del error", exc_info=True)
msg = traceback.format_exc()
return msg
def verify_query(self, data: dict, c):
try:
assert type(data) == dict
assert data[self.UserInfo.REC_NAME] != ""
assert data[self.UserInfo.REC_IDCARD] != ""
assert data[self.FLAG] != ""
c.verify_res = self.RIGHT
c.idcard = data.get(self.UserInfo.REC_IDCARD, "")
c.name = data.get(self.UserInfo.REC_NAME, "")
logger.info("verify query success")
except:
c.verify_res = self.WRONG
c.note = self.Error.ILLEAGL_NOTE
logger.error("verify error", exc_info=True)
raise Exception(self.Error.ILLEAGL_NOTE)
def _save_bsd_loanapplicationnum(self,cycle,times):
try:
logger.info(f"bsd_loanapplicationnum saved;user phone number=>{self.phone}")
db = bsd_loanapplicationnum()
db.seq_id = self.seqId
db.p_Phone = self.phone
db.p_cycle = cycle
db.p_Platform = "0"
db.CODE = "200"
db.CYCLE = "".join((str(int(time.time())), "000"))
db.LOANNUM = times
db.data_source = "hd"
db.save(using=self.HMDB)
except:
logger.error("error",exc_info=True)
def EMR013(self,data:dict):
emr = []
for d in data:
emr.append(
HDEMD008EMR013(
emd_id=self.emd_id,
name = self.name,
mobile = self.phone,
idcard = self.idcard,
cycle = self.cycle,
platform = d.get("PLATFORM", "empty"),
money = d.get("MONEY", "empty"),
)
)
HDEMD008EMR013.objects.using(self.flag).bulk_create(emr)
logger.info("emd008:emr013 save succeed")
def save_cache(self, parsed: dict, transed: namedtuple, log_db):
try:
name, idcard, _ = self._user_info(parsed)
zx = ZhengXin91ShareOriData()
zx.name = name
zx.idcard = idcard
zx.status_code = self.Code.SHARE_EXISTED
zx.ori_data = transed.val
zx.ctime_stamp = self.now_stamp
zx.save()
log_db.save_cache = self.RIGHT
logger.info("save cache success")
except:
log_db.save_cache = self.WRONG
logger.error("save_cache", exc_info=True)
raise Exception("save cache failed")
def post(self,request)->HttpResponse:
default_data = ""
try:
zx = ZhengXin91Shared()
resp,state = zx.main(request.body)
if state:
logger.info("query table success")
return HttpResponse(resp)
else:
logger.info("query failed,using default data")
e = UnexpectedHandler()
default_data = e.except_resp(request.body)
return HttpResponse(default_data)
except:
logger.error("Exception error",exc_info = True)
return HttpResponse(default_data)
def EMR002(self,data:dict):
emr = []
for d in data:
emr.append(
HDEMD008EMR002(
emd_id = self.emd_id,
name=self.name,
mobile=self.phone,
idcard=self.idcard,
cycle=self.cycle,
platform_code=d.get("PLATFORMCODE", "empty"),
register_time=d.get("REGISTERTIME", "empty"),
p_type = d.get("P_TYPE", "empty"),
)
)
HDEMD008EMR002.objects.using(self.flag).bulk_create(emr)
logger.info("emd008:emr002 save succeed")
def EMR007(self,data:dict):
emr = []
for d in data:
emr.append(
HDEMD008EMR007(
emd_id=self.emd_id,
name = self.name,
mobile = self.phone,
idcard = self.idcard,
cycle = self.cycle,
platform_code = d.get("PLATFORMCODE", "empty"),
loan_lenders_time = d.get("LOANLENDERSTIME", "empty"),
loan_lenders_amount = d.get("LOANLENDERSAMOUNT", "empty"),
p_type = d.get("P_TYPE", "empty"),
)
)
HDEMD008EMR007.objects.using(self.flag).bulk_create(emr)
logger.info("emd008:emr007 save succeed")
def query(self, parsed: dict, log_db) -> dict:
try:
zx = self.ZhengXin91LoanCoreInfo
loan_info = self.make_module(parsed)
loan_infos = {"loan_infos": [loan_info]}
state = loan_info[zx.BORROWSTATE]
if state == self._borrowState().loaned:
uid, _ = self._has_applied(parsed)
JkUserLoanRecord, _ = self._has_loaned(uid)
loan_info[zx.BORROWAMOUNT] = JkUserLoanRecord.amount
loan_info[zx.CONTRACTDATE] = JkUserLoanRecord.pay_time
log_db.query_table = self.RIGHT
logger.info("query success")
return loan_infos
except:
log_db.query_table = self.WRONG
logger.error("query failed", exc_info=True)
raise Exception("query")
def storage(self, parsed: tuple, data: dict, c):
try:
self._store_ori(parsed, data)
code, concrete = parsed
assert code == self.Code.SYNCHRONOUS_INTERFACE_CODE
flag = data[self.FLAG]
if flag == self.BSDFLAG:
self._store_bsd(parsed, data)
elif flag == self.SSDFLAG:
self._store_ssd(parsed, data)
else:
raise Exception(f"Unexpected flag {flag}")
c.insert_res = self.RIGHT
logger.info("storage success")
except:
logger.error("storage error", exc_info=True)
c.insert_res = self.WRONG
raise Exception("storage error")
def _get_newtoken(self,table)->str:
token = ""
try:
if table == hd_token.__name__:
t = hd_token.objects.using("ssd").values("token").order_by("-id")
assert len(t) > 0
token = t[0].get("token","")
elif table == hd_tokenwebloan.__name__:
t = hd_tokenwebloan.objects.using("ssd").values("token").order_by("-id")
assert len(t) > 0
token = t[0].get("token","")
else:
Exception("_get_newtoken unexpect!")
logger.error("ERROR",exc_info=True)
except:
logger.error("get newtoken error",exc_info=True)
raise Exception("get newtoken error")
logger.info(f"get newtoken {token} from {table}")
return token
def store_detail(self,emd_id:int):
msg = "hd emd008 loan detail"
try:
for r in self.result:
if len(r) == 0:
logger.info("one of hd emd008 details is empty")
else:
setattr(EMD008Detail,"emd_id",emd_id)
setattr(EMD008Detail,"name",self.name)
setattr(EMD008Detail,"idcard",self.idcard)
setattr(EMD008Detail,"phone",self.phone)
setattr(EMD008Detail,"flag",self.flag)
setattr(EMD008Detail,"cycle",r["CYCLE"])
emd008 = EMD008Detail()
func = getattr(emd008,r["TYPE"])
func(r["DATA"])
logger.info(f"{msg} successfully stored")
except:
logger.error(f"{msg} failed",exc_info=True)
def hd_illegal_info(self)->dict:
# EMW018不良信息查询,
# http://opensdk.emay.cn:9099/SF_YZ_API/SFService.asmx?op=Get_EMW_IllegalInfo_BLXXCX
try:
# crawler
token = self._get_newtoken("hd_token")
data = dict(
name=self.name,
idcard=self.idcard,
ACCESS_TOKEN=token,
)
if self.flag == self.BSDFLAG:
parseRes = self._hd_illegal_info_judge_before_crawl(data)
bsd = bsd_hy_illegalinfo_summary()
bsd.seq_id = self.seqId
bsd.p_person_id = self.idcard
bsd.p_person_name = self.name
bsd.CODE = "200"
bsd.result = parseRes.get("RESULTDESC","")
bsd.resultdesc = parseRes.get("MESSAGE","")
bsd.save(using=self.HMDB)
elif self.flag == self.SSDFLAG:
parseRes = self.req_hdInterface(self.hdIllegalInfoUrl, data)
ssd = cfd_hd_IllegalInfo_BLXXCX()
ssd.p_person_id = self.idcard
ssd.p_person_name = self.name
ssd.CODE = "200"
ssd.result = parseRes.get("RESULTDESC","")
ssd.resultdesc = parseRes.get("MESSAGE","")
ssd.save(using=self.SSDDB)
# storage Hd_EMW018
ori = Hd_EMW018()
ori.flag = self.flag
ori.idcard = self.idcard
ori.phone = self.phone
ori.name = self.name
ori.ori_data = json.dumps(parseRes)
ori.save()
except:
logger.error("EMW018不良信息查询 =>",exc_info=True)
return self.FAILED
logger.info(f"{self.flag} {self.name},{self.idcard} insert into success")
return self.SUCCESS
