def parse_arg():
parser = argparse.ArgumentParser(description="train crnn")
parser.add_argument('--cfg',
help='experiment configuration filename',
required=True,
type=str)
args = parser.parse_args()
config = load_yml(args.cfg)
return config
class RuleEngine:
rule_data = utils.load_yml("rule_engine_data.yaml")
@staticmethod
def rule(req, params):
utils.log('call api rule engine [rule] for [{}]'.format(
config.COUNTRY))
url = RuleEngine.rule_data['Host'][
config.COUNTRY] + RuleEngine.rule_data['RuleEnginePort'][
config.COUNTRY] + RuleEngine.rule_data['rule']['path']
header = RuleEngine.rule_data['rule']['header']
result = json.loads(req.get(url, headers=header, params=params).text)
return result
class Pipeline:
model_data = utils.load_yml("rule_model_data.yaml")
@staticmethod
def borrow_apply(req, data):
utils.log('call api [borrow-apply] for [{}]'.format(config.COUNTRY))
url = Pipeline.model_data['Host'][
config.COUNTRY] + Pipeline.model_data['CeleryPort'][
config.COUNTRY] + Pipeline.model_data['BorrowApply']['path']
header = Pipeline.model_data['BorrowApply']['header']
result = json.loads(req.post(url, headers=header, json=data).text)
return result
@staticmethod
def kyc_switch(req, data):
utils.log('call api [kyc-switch] for [{}]'.format(config.COUNTRY))
pass
@staticmethod
def process_post_kyc(req, data):
utils.log('call api [process-post-kyc] for [{}]'.format(
config.COUNTRY))
pass
class TestRuleModel(requestbase.RequestBase):
model_data = utils.load_yml("rule_model_data.yaml")
resource = utils.load_yml("resources.yaml")
query_count = model_data['QueryCount']
@classmethod
def setUpClass(cls):
utils.log(
"=================================================================="
)
utils.log(
"= ="
)
utils.log(
"=============== TEST CLASS SETUP ========="
)
utils.log(
"=================================================================="
)
utils.log("Set up test class...")
cls.req = requestlib.RequestLib()
cls.trade_conn = mysqlhelper.MysqlConnector(
cls.resource['TradeDB'][config.COUNTRY]['host'],
cls.resource['TradeDB']['user'],
cls.resource['TradeDB']['password'], cls.resource['TradeDB']['db'])
cls.risk_conn = mysqlhelper.MysqlConnector(
cls.resource['StagingRiskDB'][config.COUNTRY]['host'],
cls.resource['StagingRiskDB']['user'],
cls.resource['StagingRiskDB']['password'],
cls.resource['StagingRiskDB']['db'],
cls.resource['StagingRiskDB'][config.COUNTRY]['port'])
cls.prefix = cls.resource['StagingRiskDB'][config.COUNTRY]['tbPrefix']
cls.time_now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
utils.log(
"==================================================================\n\n"
)
@classmethod
def tearDownClass(cls):
utils.log(
"=================================================================="
)
utils.log(
"= ="
)
utils.log(
"================ TEST CLASS CLEANUP =========="
)
utils.log(
"=================================================================="
)
if cls.req: cls.req.close_session()
if cls.trade_conn: cls.trade_conn.close()
if cls.risk_conn: cls.risk_conn.close()
utils.log(
"==================================================================\n\n"
)
def call_borrow_apply(self, data):
utils.log('call api [borrow-apply] for [{}]'.format(config.COUNTRY))
url = self.model_data['Host'][
config.COUNTRY] + self.model_data['CeleryPort'][
config.COUNTRY] + self.model_data['BorrowApply']['path']
header = self.model_data['BorrowApply']['header']
model_result = json.loads(
self.req.post(url, headers=header, json=data).text)
utils.log('1.verify call RiskMQ-app API borrow-apply successfully.')
self.assertEqual(utils.query_json(model_result, 'success'), 'true',
'call api borrow-apply failed!')
def get_borrow_order_from_trade(self):
utils.log('pick latest borrow orders from trade db.')
sql = 'select * from BorrowOrders order by id desc limit 30'
return self.trade_conn.fetchall(sql)
def update_trade_person(self, home_own_code, stay_length, person_property,
occupation, occupation_code, id_account):
utils.log(
"update [trade][Persons], set homeOwnershipCode={}, stayLength='{}' , property='{}', occupation='{}'"
", occupationCode={} for accountId={}".format(
home_own_code, stay_length, person_property, occupation,
occupation_code, id_account))
sql = """
update Persons
set homeOwnershipCode='{}', stayLength='{}', property='{}', occupation='{}', occupationCode='{}'
where accountId={}
""".format(home_own_code, stay_length, person_property, occupation,
occupation_code, id_account)
result = self.trade_conn.update(sql)
time.sleep(1)
# self.assertTrue(result == 1, 'update Persons failed!')
def get_vn_cash_model_test_data(self):
"""#1 model"""
utils.log(
'get records from user_apply match cash model: loan_success=0 and loan_type in (1,2)'
)
sql = 'select * from vnrisk_user_apply where loan_success=0 and loan_type in (1,2) order by id desc limit ' + str(
self.query_count)
picked_apply_orders = self.risk_conn.fetchall(sql)
self.assertTrue(
len(picked_apply_orders) > 0,
'there is no qualify records in table user_apply!')
return picked_apply_orders
def pre_check_order_exist_in_trade(self, id_borrow, id_account):
utils.log(
'verify record with id={} and accountId={} exist in origin [trade] db [BorrowOrders] table.'
.format(id_borrow, id_account))
trade_sql = 'select * from BorrowOrders where id={} and accountId={}'.format(
id_borrow, id_account)
origin_record = self.trade_conn.fetchone(trade_sql)
self.assertIsNotNone(
origin_record,
'Not find any record with given id_borrow and id_account! please pick another one'
)
def clean_borrow_case_if_exist(self, id_account, id_borrow):
utils.log(
'check whether borrow case with id_account:{}, id_borrow:{} already exist, delete it if exist.'
.format(id_account, id_borrow))
query_sql = "select * from {}risk_borrow_case where id_account={} and id_borrow={}".format(
self.prefix, id_account, id_borrow)
query_result = self.risk_conn.fetchone(query_sql)
if query_result:
utils.log('delete current borrow case record')
delete_sql = "delete from {}risk_borrow_case where id_account={} and id_borrow={}".format(
self.prefix, id_account, id_borrow)
update_result = self.risk_conn.update(delete_sql)
self.assertTrue(update_result == 1,
'delete current borrow case record failed!')
def get_test_id_borrow_id_account(self, picked_apply_orders, index):
borrow_order = picked_apply_orders[index]
id_borrow, id_account = borrow_order['id_borrow'], borrow_order[
'id_account']
return id_borrow, id_account
def verify_borrow_case_status(self, id_account, id_borrow):
"""验证borrow_case表case_status字段更新正确"""
utils.log(
'verify filed [case_status] in [borrow_case] updated with correct status(success, postkyc_success)'
)
sql = "select * from {}risk_borrow_case where id_account={} and id_borrow={}".format(
self.prefix, id_account, id_borrow)
time.sleep(15) # need time to process
result = self.risk_conn.fetchone(sql)
utils.log('result in borrow case:\n{}'.format(result))
case_status, update_at = result['case_status'], result['updated_at']
total_seconds = (datetime.now() - update_at).total_seconds()
minutes = abs(int(total_seconds /
60)) # 数据库时间比local快几分钟，取绝对值查看相差分钟数，小于10分钟
self.assertTrue(
case_status in ('success', 'postkyc_success') and minutes < 10,
'borrow case status Not In correct status')
def prepare_model_data(self, picked_apply_orders, random=False):
id_borrow, id_account = self.get_test_id_borrow_id_account(
picked_apply_orders, 0) # get last one
if random:
id_borrow, id_account = self.get_test_id_borrow_id_account(
picked_apply_orders,
utils.get_random_numb(0,
len(picked_apply_orders) - 1))
self.pre_check_order_exist_in_trade(id_borrow, id_account)
self.clean_borrow_case_if_exist(id_account, id_borrow)
data = self.model_data['BorrowApply']['payload']['General']
data.update({'id_account': id_account, 'id_borrow': id_borrow})
return id_borrow, id_account, data
def verify_model_workflow_pass(self, picked_apply_orders, random=False):
id_borrow, id_account, data = self.prepare_model_data(
picked_apply_orders, random)
self.call_borrow_apply(data)
self.verify_borrow_case_status(id_account, id_borrow)
def verify_model_feature(self, id_borrow, id_account, expect_feature_json):
utils.log('verify filed [feature] in [model_feature] updated correct')
sql = "select * from {}risk_model_feature where id_account={} and id_borrow={}".format(
self.prefix, id_account, id_borrow)
result = self.risk_conn.fetchone(sql)
feature_json = json.loads(result['feature'])
# TestRuleModel.update_None_str_2_None(expect_feature_json)
utils.update_none_str_2_none(expect_feature_json)
utils.log('verify expected feature generated in table model_feature')
utils.log('expected feature:{}\nactual feature:{}'.format(
expect_feature_json, feature_json))
self.assertTrue(
set(expect_feature_json.items()).issubset(set(
feature_json.items())),
'expected feature value Not write to db model_feature!')
def verify_model_total_score(self, id_borrow, id_account):
utils.log(
'verify filed [total_score] in [model_total_score] updated correct'
)
sql = "select * from {}risk_rule_model_total_score where id_account={} and id_borrow={}".format(
self.prefix, id_account, id_borrow)
result = self.risk_conn.fetchone(sql)
self.assertIsNotNone(
result,
'there is No total_score found in [model_total_score] for this id_borrow.'
)
total_score = result['total_score']
utils.log('got total_score={}'.format(str(total_score)))
def verify_riskcontrol_resfin(self, id_borrow, id_account):
utils.log(
'verify filed [res],[status] in [riskcontrol_resfin] updated correct'
)
sql = "select * from {}risk_riskcontrol_resfin where id_account={} and id_borrow={} and updated_at>'{}'".format(
self.prefix, id_account, id_borrow, self.time_now_str)
result_list = self.risk_conn.fetchall(sql)
self.assertTrue(
len(result_list) > 1,
'there should more than 1 resfin record for one (id_account, id_borrow) pair!'
)
res_set, status_set = set(), set()
for result in result_list:
res_set.add(result['res'])
status_set.add(result['status'])
expected_res = ('approve', 'success', 'reject')
expected_status = ('new', 'new-score', 'post-kyc')
@unittest.skipUnless(config.COUNTRY == constants.CN, 'only for CN')
def test_cash_model_workflow_pass(self):
"""验证VN #1 model流程是通的"""
self.verify_model_workflow_pass(self.get_vn_cash_model_test_data())
@unittest.skipUnless(config.COUNTRY == constants.CN, 'only for CN')
def test_CN_cash_model(self):
utils.log(
'test borrow orders with condition: loan_type in (1,2) and loan_success=0 to hit CN cash model.'
)
id_borrow, id_account, data = self.prepare_model_data(
self.get_vn_cash_model_test_data(), True) # random data
# id_borrow, id_account, data = self.prepare_model_data(self.get_vn_cash_model_test_data()) # last one
test_scenario = 2 # rule_model_data VNCashModel
self.update_trade_person(
self.model_data['VNCashModel']['SetField'][test_scenario]
['homeOwnershipCode'], self.model_data['VNCashModel']['SetField']
[test_scenario]['stayLength'], self.model_data['VNCashModel']
['SetField'][test_scenario]['property'],
self.model_data['VNCashModel']['SetField'][test_scenario]
['occupation'], self.model_data['VNCashModel']['SetField']
[test_scenario]['occupationCode'], id_account)
self.call_borrow_apply(data)
self.verify_borrow_case_status(id_account, id_borrow)
self.verify_model_feature(
id_borrow, id_account,
self.model_data['VNCashModel']['expectFeature'][test_scenario])
self.verify_model_total_score(id_borrow, id_account)
class TestProdTags(requestbase.RequestBase):
crm_data = utils.load_yml("crm_people_library_prod.yaml")
@classmethod
def setUpClass(cls):
utils.log("==================================================================")
utils.log("= =")
utils.log("=============== TEST CLASS SETUP =========")
utils.log("==================================================================")
utils.log("Set up test class...")
cls.req = requestlib.RequestLib(silence=True)
cls.indices_size = cls.get_indices_count(config.COUNTRY)
cls.reds = redashlib.Redash()
utils.log("==================================================================\n\n")
@classmethod
def get_kibana_url(cls, region, query_type='_search'):
"""
get kibana url via region and type.
:param region: PH, US, CN
:param query_type: eg: _search, _stats
:return: kibana url
"""
utils.log('get [{}] kibana [{}] url.'.format(region, query_type))
if region == constants.US:
url = cls.crm_data['ElasticSearch']['prod_host'] + cls.crm_data['ElasticSearch']['prod_ph'+query_type]
elif region == constants.JP:
url = cls.crm_data['ElasticSearch']['prod_host'] + cls.crm_data['ElasticSearch']['prod_id'+query_type]
else:
url = cls.crm_data['ElasticSearch']['prod_host'] + cls.crm_data['ElasticSearch']['prod_vn'+query_type]
utils.log('kibana search url:{}'.format(url))
return url
@classmethod
def kibana_search(cls, es_query):
utils.log('kibana query:\n{}'.format(es_query))
return cls.req.post(cls.get_kibana_url(config.COUNTRY), headers=cls.crm_data['ElasticSearch']['prod_headers'], json=es_query)
@classmethod
def get_indices_count(cls, region):
resp = cls.req.post(cls.get_kibana_url(region, query_type='_stats'), headers=cls.crm_data['ElasticSearch']['prod_headers'])
count = json.loads(resp.text)['_all']['primaries']['docs']['count']
utils.log("kibana indices total count is: {}".format(count))
return count
def verify_prod_wide_es_equal_for_specify_value(self, field_name, check_value):
wide_sql = self.crm_data['TagValuesUpdate']['wide'][constants.US][field_name].format(check_value)
es_query = self.crm_data['TagValuesUpdate']['kibana'][constants.US][field_name] % check_value
wide_count = self.reds.get_query_results_count(wide_sql)
resp = self.kibana_search(json.loads(es_query))
kibana_count = json.loads(resp.text)['hits']['total']
utils.log("verify field [{}]=[{}] count equals between wide and Kibana for product env.".format(field_name, check_value))
utils.log("wide count: {}, kibana count: {}".format(wide_count, kibana_count))
self.assertEqual(wide_count, kibana_count, "field [{}] value [{}] count in wide not consistance with kibana!".format(field_name, check_value))
def verify_tag_value_updates(self, field):
values_list = self.crm_data['TagValuesUpdate']['checkValues'][constants.US][field]
for value in values_list:
with self.subTest(check_tag_value=value):
self.verify_prod_wide_es_equal_for_specify_value(field, value)
def test_wide_kibana_total_equals(self):
"""验证宽表和Kibana Indices数据总量相同"""
utils.log('verify user_tag_wide and kibana indices total count equal.')
prod_wide_total = self.reds.get_query_results_count(self.crm_data['Wide'][config.COUNTRY]['totalSize'])
utils.log("Total count for user_tag_wide:{}, kibana:{}".format(prod_wide_total, self.indices_size))
self.assertEqual(prod_wide_total, self.indices_size, 'user_tag_wide and kibana total count Not equal!')
@unittest.skipUnless(config.COUNTRY == constants.US, 'only for US')
def test_first_application_loantype_update(self):
"""验证标签first_application_loantype值更新正确,wide及Kibana数据一致"""
self.verify_tag_value_updates('first_application_loantype')
@unittest.skipUnless(config.COUNTRY == constants.US, 'only for US')
def test_second_approved_loantype_update(self):
"""验证标签second_approved_loantype值更新正确,wide及Kibana数据一致"""
self.verify_tag_value_updates('second_approved_loantype')
@unittest.skipUnless(config.COUNTRY == constants.US, 'only for US')
def test_first_approved_loantype_update(self):
"""验证标签first_approved_loantype值更新正确,wide及Kibana数据一致"""
self.verify_tag_value_updates('first_approved_loantype')
class TestPipeline(requestbase.RequestBase):
model_data = utils.load_yml("rule_model_data.yaml")
resource = utils.load_yml("resources.yaml")
query_count = model_data['QueryCount']
@classmethod
def setUpClass(cls):
utils.log("==================================================================")
utils.log("= =")
utils.log("=============== TEST CLASS SETUP =========")
utils.log("==================================================================")
utils.log("Set up test class...")
cls.req = requestlib.RequestLib()
cls.trade_conn = mysqlhelper.MysqlConnector(cls.resource['TradeDB'][config.COUNTRY]['host'],
cls.resource['TradeDB']['user'],
cls.resource['TradeDB']['password'], cls.resource['TradeDB']['db'])
cls.risk_conn = mysqlhelper.MysqlConnector(cls.resource['StagingRiskDB'][config.COUNTRY]['host'],
cls.resource['StagingRiskDB']['user'],
cls.resource['StagingRiskDB']['password'],
cls.resource['StagingRiskDB']['db'],
cls.resource['StagingRiskDB'][config.COUNTRY]['port'])
cls.prefix = cls.resource['StagingRiskDB'][config.COUNTRY]['tbPrefix']
cls.time_now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
utils.log("==================================================================\n\n")
@classmethod
def tearDownClass(cls):
utils.log("==================================================================")
utils.log("= =")
utils.log("================ TEST CLASS CLEANUP ==========")
utils.log("==================================================================")
if cls.req: cls.req.close_session()
if cls.trade_conn: cls.trade_conn.close()
if cls.risk_conn: cls.risk_conn.close()
utils.log("==================================================================\n\n")
def clean_borrow_case_if_exist(self, id_account, id_borrow):
utils.log('check whether borrow case with id_account:{}, id_borrow:{} already exist, delete it if exist.'.format(id_account, id_borrow))
query_sql = "select * from {}risk_borrow_case where id_account={} and id_borrow={}".format(self.prefix, id_account, id_borrow)
query_result = self.risk_conn.fetchone(query_sql)
if query_result:
utils.log('delete current borrow case record')
delete_sql = "delete from {}risk_borrow_case where id_account={} and id_borrow={}".format(self.prefix, id_account, id_borrow)
update_result = self.risk_conn.update(delete_sql)
self.assertTrue(update_result == 1, 'delete current borrow case record failed!')
def call_borrow_apply_verify_kyc_call_type(self, data, call_type):
self.call_borrow_apply(data)
self.verify_kyc_result(data, call_type)
def call_borrow_apply(self, data, clean_borrow_case=True):
id_account, id_borrow = data['id_account'], data['id_borrow']
if clean_borrow_case:
self.clean_borrow_case_if_exist(id_account, id_borrow)
transaction_id = "RT{}{}{}".format(id_account, id_borrow, utils.get_random_numb(1001, 9999))
data.update({"transaction_id": transaction_id})
result = Pipeline.borrow_apply(self.req, data)
utils.log('2.verify call api borrow-apply successfully.')
self.assertEqual(utils.query_json(result, 'success'), 'true', 'call api borrow-apply failed!')
time.sleep(6)
return result, transaction_id
def verify_kyc_result(self, data, call_type):
id_account, id_borrow = data['id_account'], data['id_borrow']
kyc_result_sql = "select id,id_account,id_borrow,transaction_id,kyc_type_res,updated_at from vnrisk_rule_kyc_type_result where id_account={} order by id desc limit 1;"
kyc_result = self.risk_conn.fetchone(kyc_result_sql.format(id_account))
self.assertIsNotNone(kyc_result, 'Not find kyc type result for id_account:{} id_borrow:{}'.format(id_account, id_borrow))
utils.log("kyc_type_result:\n{}".format(kyc_result))
kyc_result_dict = json.loads(kyc_result['kyc_type_res'])
self_call_type, family_call_type = kyc_result_dict[0]['call_type'], kyc_result_dict[1]['call_type']
utils.log('3.verify kyc call type is [{}] for id_account:{} id_borrow:{}'.format(call_type, id_account, id_borrow))
self.assertTrue(self_call_type == call_type and family_call_type == call_type, 'KYC call type is Not As expected!')
def insert_kyc_score_result(self, id_account, sql):
utils.log('1.insert a kyc approve record in vnrisk_rule_kyc_score_result for id_account:{}'.format(id_account))
query_kyc_score_cnt_sql = "select count(*) as cnt from vnrisk_rule_kyc_score_result where id_account={};"
kyc_score_result_cnt = self.risk_conn.fetchone(query_kyc_score_cnt_sql.format(id_account)).get('cnt')
if kyc_score_result_cnt > 0:
delete_kyc_score_sql = "delete from vnrisk_rule_kyc_score_result where id_account={};".format(id_account)
self.risk_conn.execute(delete_kyc_score_sql)
return self.risk_conn.execute(sql)
def add_account_to_black_list(self, id_account):
utils.log('1.add id_account to black user list.')
query_black_account_cnt_sql = "select count(*) as cnt from vnrisk_black_user_id where black_id_account={};".format(id_account)
black_account_cnt = self.risk_conn.fetchone(query_black_account_cnt_sql.format(id_account)).get('cnt')
if black_account_cnt > 0:
return utils.log('id_account:{} already in vnrisk_black_user_id.'.format(id_account))
sql = self.model_data['SQL']['insertBlackUser'].format(id_account)
return self.risk_conn.execute(sql)
def verify_rule_resfinally_reject_result(self, id_account, id_borrow, result='reject', status='new'):
query_rule_resfin = "select * from vnrisk_rule_resfinally where id_account={} and id_borrow={} order by id desc limit 1;".format(id_account, id_borrow)
time.sleep(2)
resfinally_result = self.risk_conn.fetchone(query_rule_resfin)
utils.log("rule_resfinally result:\n{}".format(resfinally_result))
actual_result, actual_status, blocking_day = resfinally_result['result'], resfinally_result['status'], resfinally_result['blocking_day']
self.assertTrue(actual_result == result and actual_status == status and blocking_day >= 3, 'rule result Not correct! please check.')
def update_mongo_loans_repaid_at(self, id_account, previous_day):
utils.log('update mongo loans')
action = 'update' if mongolib.MongoDAL.get_loans(id_account) else 'insert'
doc_json = self.model_data['Mongo'][action]['loans_repaid_at']
set_repaid_at = {'id_account': id_account, 'repaid_at': utils.get_previous_date(previous_day)}
if action == 'insert': set_repaid_at['loan_id'] = utils.get_random_numb(10001, 99999)
doc_json.update(set_repaid_at)
mongolib.MongoDAL.upsert_user_loans([doc_json])
time.sleep(1)
def test_kyc_manual_no_approve_no_repaid(self):
"""验证KYC3.0 call type manual, no kyc approve in 90 days, no repaid in 30 days"""
data = self.model_data['BorrowApply']['payloadnew'][config.COUNTRY][0]
self.call_borrow_apply_verify_kyc_call_type(data, "manual")
def test_kyc_skip_in_90_approve(self):
"""验证KYC3.0 call type skip, has kyc approve in 90 days"""
data = self.model_data['BorrowApply']['payloadnew'][config.COUNTRY][1]
id_account, id_borrow = data['id_account'], data['id_borrow']
insert_kyc_score_result_sql = self.model_data['SQL']['insertKycScoreNow'].format(id_account)
self.insert_kyc_score_result(id_account, insert_kyc_score_result_sql)
self.call_borrow_apply_verify_kyc_call_type(data, "skip")
def test_kyc_manual_in_180_approve_no_repaid(self):
"""验证KYC3.0 call type manual, has kyc approve 90<date<180 days, no repaid in 30 days"""
data = self.model_data['BorrowApply']['payloadnew'][config.COUNTRY][2]
id_account, id_borrow = data['id_account'], data['id_borrow']
insert_sql = self.model_data['SQL']['insertKycScoreDayGap'].format(id_account, utils.get_previous_date(145)) # set created_at to 145 days ago
self.insert_kyc_score_result(id_account, insert_sql)
self.call_borrow_apply_verify_kyc_call_type(data, "manual")
def test_kyc_skip_in_180_approve_has_repaid(self):
"""验证KYC3.0 call type skip, has kyc approve 90<date<180 days, has repaid in 30 days"""
data = self.model_data['BorrowApply']['payloadnew'][config.COUNTRY][5]
id_account, id_borrow = data['id_account'], data['id_borrow']
insert_sql = self.model_data['SQL']['insertKycScoreDayGap'].format(id_account, utils.get_previous_date(160))
self.insert_kyc_score_result(id_account, insert_sql)
self.update_mongo_loans_repaid_at(id_account, 23) # insert a repaid record in 30 days in loans
self.call_borrow_apply_verify_kyc_call_type(data, "skip")
def test_new_reject_and_block_day(self):
"""验证new轮黑名单->reject, blocking day>=3"""
data = self.model_data['BorrowApply']['payloadnew'][config.COUNTRY][3]
id_account, id_borrow = data['id_account'], data['id_borrow']
self.add_account_to_black_list(id_account)
utils.log('2.fire new borrow-apply')
self.call_borrow_apply(data)
utils.log('3.verify rule result is reject, status is [new] and blocking day>=3, cause blockingDay + 3')
self.verify_rule_resfinally_reject_result(id_account, id_borrow)
def test_adjust_reject_and_block_day(self):
"""验证adjust轮黑名单->reject, blocking day>=3"""
data = self.model_data['BorrowApply']['payloadnew'][config.COUNTRY][4]
id_account, id_borrow = data['id_account'], data['id_borrow']
utils.log('1.fire new borrow-apply')
delete_from_black_list_sql = "delete from vnrisk_black_user_id where black_id_account={}".format(id_account)
self.risk_conn.update(delete_from_black_list_sql)
result, transaction_id = self.call_borrow_apply(data)
utils.log('2.fire adjust borrow-apply')
self.add_account_to_black_list(id_account)
data.update({"is_review": 'true', "transaction_id": transaction_id}) # keep same transaction_id with new round
self.call_borrow_apply(data, False) # adjust round(more info)
utils.log('3.verify rule result is reject, status is [adjust] and blocking day>=3, cause blockingDay + 3')
self.verify_rule_resfinally_reject_result(id_account, id_borrow, status='adjust')
class TestKycReason(requestbase.RequestBase):
rule_data = utils.load_yml("rule_manager_data.yaml")
resource = utils.load_yml("resources.yaml")
@classmethod
def setUpClass(cls):
utils.log(
"=================================================================="
)
utils.log(
"= ="
)
utils.log(
"=============== TEST CLASS SETUP ========="
)
utils.log(
"=================================================================="
)
utils.log("Set up test class...")
cls.req = requestlib.RequestLib()
cls.login_api(
cls.rule_data['BaseUrl'] + cls.rule_data['FrontendPort'] +
cls.rule_data['login']['path'], cls.rule_data['login']['header'],
cls.rule_data['login']['payload'])
cls.risk_conn = utils.get_db_connector(
cls.resource['Mysql']['RiskDB']['db'][
config.COUNTRY]['risk_dev_tmp'])
cls.tb_prefix = cls.resource['Mysql']['RiskDB']['db'][
config.COUNTRY]['tbPrefix']
utils.log(
"==================================================================\n\n"
)
@classmethod
def tearDownClass(cls):
utils.log(
"=================================================================="
)
utils.log(
"= ="
)
utils.log(
"================ TEST CLASS CLEANUP =========="
)
utils.log(
"=================================================================="
)
if cls.req: cls.req.close_session()
if cls.risk_conn: cls.risk_conn.close()
utils.log(
"==================================================================\n\n"
)
def test_search_kyc_reason_exist(self):
"""验证搜索存在的Flipping ids成功"""
search_resp = self.search_kyc_with_params(
self.rule_data['searchKyc']['validParams'][config.COUNTRY])
kyc_reason_total = utils.query_json(json.loads(search_resp),
'data.total')
utils.log(
'verify search kyc reason records success with exist Flipping ids')
self.assertTrue(kyc_reason_total > 0,
'search kyc reason with exist Flipping ids failed!')
def test_search_kyc_reason_not_exist(self):
"""验证搜索不存在的Flipping ids成功"""
search_resp = self.search_kyc_with_params(
self.rule_data['searchKyc']['invalidParams'])
kyc_reason_total = utils.query_json(json.loads(search_resp),
'data.total')
utils.log(
'verify search kyc reason records success with Not exist Flipping ids'
)
self.assertEqual(
kyc_reason_total, 0,
'search kyc reason with Not exist Flipping ids failed! should equal 0!'
)
def test_update_publish_kyc_valid(self):
"""验证更新Kyc reason, snapshotversion等字段更新，设置isValid=1"""
utils.log("Case. update rule and publish it, set rule_valid=1.")
self.verify_kyc_update('payloadValid')
def test_update_publish_kyc_invalid(self):
"""验证更新Kyc reason, snapshotversion等字段更新，设置isValid=0"""
utils.log("Case. update rule and publish it, set rule_valid=0.")
self.verify_kyc_update('payloadInValid')
def test_reset_kyc_reason(self):
"""验证Reset Kyc reason成功"""
update_payload = self.rule_data['update_kyc']['payloadInValid'][
config.COUNTRY]
update_payload.update(
{'topicType': 'autoRej' + utils.get_time_stamp()})
url_host = self.rule_data['BaseUrl'] + self.rule_data['BackendPort'][
config.COUNTRY]
self.verify_update_kyc(url_host + self.rule_data['update_kyc']['path'],
self.rule_data['update_kyc']['header'],
update_payload, update_payload['flippingId'])
self.verify_reset_kyc(
url_host + self.rule_data['reset_kyc']['path'].format(
update_payload['flippingId']),
self.rule_data['reset_kyc']['header'],
update_payload['flippingId'])
def verify_kyc_update(self, test_payload):
"""
1.验证update kyc，publish kyc
2.验证kyc在table rule_kyc_reject_reason更新
3.验证kyc在table rule_kyc_reject_reason_operation更新
4.验证kyc在table rule_kyc_reject_reason_snapshot更新
"""
if config.COUNTRY == constants.US:
table_prefix = "usrisk"
elif config.COUNTRY == constants.JP:
table_prefix = "jprisk"
elif config.COUNTRY == constants.CN:
table_prefix = "cnrisk"
else:
table_prefix = ""
update_payload = self.rule_data['update_kyc'][test_payload][
config.COUNTRY]
new_desc = 'autoRej' + utils.get_time_stamp()
update_payload.update({'topicType': new_desc})
update_flipping_id = update_payload['flippingId']
update_header = self.rule_data['update_kyc']['header']
self.verify_update_kyc(
self.rule_data['BaseUrl'] +
self.rule_data['BackendPort'][config.COUNTRY] +
self.rule_data['update_kyc']['path'], update_header,
update_payload, update_flipping_id)
self.verify_publish_kyc(
self.rule_data['BaseUrl'] + self.rule_data['FrontendPort'] +
self.rule_data['publish_kyc']['path'], update_header,
update_flipping_id)
utils.log(
"1.verify flipping_id={} record [topic_type] updated in table {}_rule_kyc_reject_reason"
.format(update_flipping_id, table_prefix))
query_kyc = "select * from {}_rule_kyc_reject_reason where flipping_id={}".format(
table_prefix, update_flipping_id)
kyc_record = self.risk_conn.fetchone(query_kyc)
utils.log(kyc_record)
self.assertEqual(
new_desc, kyc_record['topic_type'],
"kyc reason record not update value:{}! please check".format(
new_desc))
utils.log(
"2.verify flipping_id={} record,[snapshot_version] updated in table {}_rule_kyc_operation"
.format(update_flipping_id, table_prefix))
query_kyc_operation = "select * from {}_rule_kyc_reject_reason_operation where flipping_id={} order by id desc".format(
table_prefix, update_flipping_id)
kyc_operation_record = self.risk_conn.fetchall(query_kyc_operation)[0]
utils.log(kyc_operation_record)
dt_now = utils.get_datetime_per_timezone()
minutes_pass = utils.get_time_minus(dt_now,
kyc_operation_record['updated_at'])
utils.log('verify record update timestamp is latest, less than 2 min.')
self.assertTrue(
minutes_pass < 2,
"rule not updated for rule_id={} in {}_rule_operation!".format(
update_flipping_id, table_prefix))
utils.log(
"3.verify flipping_id={} record [topic_type],[snapshot_version] updated in table {}_rule_kyc_reject_reason_snapshot"
.format(update_flipping_id, table_prefix))
query_rule_snapshot = "select * from {}_rule_kyc_reject_reason_snapshot where flipping_id={} and snapshot_version='{}'".format(
table_prefix, update_flipping_id,
kyc_operation_record['snapshot_version'])
kyc_snapshot_record = self.risk_conn.fetchall(query_rule_snapshot)
utils.log(kyc_snapshot_record)
self.assertTrue(
len(kyc_snapshot_record) == 1,
"there should only 1 record in table {}_rule_kyc_reject_reason_snapshot for this query!"
.format(table_prefix))
self.assertEqual(
new_desc, kyc_snapshot_record[0]['topic_type'],
"kyc not update in {}_rule_kyc_reject_reason_snapshot!".format(
table_prefix))
self.assertTrue(
kyc_record['is_valid'] == kyc_snapshot_record[0]['is_valid']
and kyc_record['apply_amount_min']
== kyc_snapshot_record[0]['apply_amount_min'],
'field [is_valid] and [apply_amount_min] are same between {}_rule_kyc_reject_reason and {}_rule_kyc_reject_reason_snapshot'
.format(table_prefix, table_prefix))
def verify_update_kyc(self, url, header, payload, update_flipping_id):
header.update({'Region': config.COUNTRY})
update_resp = self.update_kyc_reason_api(url, header, payload)
utils.log("verify update kyc reason successfully.")
self.assertEqual(
utils.query_json(json.loads(update_resp), 'msg'), 'success',
'update flipping_id={} kyc reason failed!'.format(
update_flipping_id))
def verify_publish_kyc(self, url, header, update_flipping_id):
header.update({'Region': config.COUNTRY})
publish_resp = self.publish_kyc_reason_api(url, header)
utils.log("verify publish kyc reason successfully.")
self.assertEqual(
utils.query_json(json.loads(publish_resp), 'msg'), 'success',
'publish flipping_id={} kyc reason failed!'.format(
update_flipping_id))
def verify_reset_kyc(self, url, header, update_flipping_id):
header.update({'Region': config.COUNTRY})
reset_kyc_resp = self.reset_kyc_reason_api(url, header)
utils.log(
'verify reset kyc reason for flipping_id={} successfully.'.format(
update_flipping_id))
self.assertEqual(
json.loads(reset_kyc_resp.text)['msg'], 'success',
'call reset kyc reason api failed!')
def search_kyc_with_params(self, params):
url = self.rule_data['BaseUrl'] + self.rule_data['BackendPort'][
config.COUNTRY] + self.rule_data['searchKyc']['path']
return self.search_kyc_reason_api(
url, self.rule_data['searchKyc']['header'], params)
@classmethod
def login_api(cls, url, header, payload):
utils.log(">>login rule manager...")
login_response = cls.req.post(url, headers=header, json=payload)
utils.log("response is: {}".format(login_response.text))
return login_response
def search_kyc_reason_api(self, url, header, params):
utils.log(">>search kyc reason records...")
kyc_list_resp = self.req.get(url, headers=header, params=params)
self.assertEqual(
json.loads(kyc_list_resp.text)['msg'], 'success',
'call search kyc reason api failed!')
return kyc_list_resp.text
def update_kyc_reason_api(self, url, header, payload):
utils.log(">>update kyc reason records...")
save_resp = self.req.post(url, headers=header, json=payload)
return save_resp.text
def publish_kyc_reason_api(self, url, header):
utils.log(">>publish kyc reason...")
publish_result = self.req.post(url, headers=header)
time.sleep(1)
utils.debug(json.loads(publish_result.text)['data'])
return publish_result.text
def reset_kyc_reason_api(self, url, header):
utils.log(">>reset kyc reason...")
return self.req.get(url, headers=header)
class TestPeople(requestbase.RequestBase):
crm_data = utils.load_yml("crm_people_library.yaml")
resource = utils.load_yml("resources.yaml")
desc_prefix = "autoTest"
desc_complex_prefix = "autoTestComplex"
@classmethod
def setUpClass(cls):
utils.log(
"=================================================================="
)
utils.log(
"= ="
)
utils.log(
"=============== TEST CLASS SETUP ========="
)
utils.log(
"=================================================================="
)
utils.log("Set up test class...")
cls.req = requestlib.RequestLib()
cls.login_api(cls.req)
cls.crm_db_conn = utils.get_db_connector(
cls.resource['Mysql']['RiskDB']['db']['crm_db'])
utils.log(
"==================================================================\n\n"
)
@staticmethod
def login_api(req):
utils.log("call login crm api...")
resp = req.post(config.QA_SERVER +
TestPeople.crm_data['CRM']['login']['path'],
headers=TestPeople.crm_data['CRM']['header'],
json=TestPeople.crm_data['CRM']['login']['payload'])
utils.log(resp.text)
return resp
def add_people_lib_task(self, people_lib_id):
"""
设置人库开始结束时间， 添加定时任务
"""
utils.log("1.提交后通知选人平台人库开始时间，结束时间")
people_lib_query = self.crm_data['CRM']['peopleLibrary'][
'query'].format(
# self.crm_data['CRM']['peopleLibrary']['people_lib_id'],
people_lib_id,
self.crm_data['CRM']['peopleLibrary']['start_time'],
self.crm_data['CRM']['peopleLibrary']['end_time'],
self.crm_data['CRM']['peopleLibrary']['cron'])
people_lib_url = self.crm_data['CRM']['baseUrl'] + self.crm_data[
'CRM']['peopleLibrary']['path'] + people_lib_query
headers = self.crm_data['CRM']['header']
headers["Authorization"] = self.crm_data['CRM']['peopleLibrary'][
'authorization']
people_lib_resp = self.req.post(people_lib_url, headers=headers)
utils.log(people_lib_resp.text)
utils.log("2.人库添加任务")
people_lib_task_query = self.crm_data['CRM']['peopleLibraryTask'][
'query'].format(
# self.crm_data['CRM']['peopleLibrary']['people_lib_id'],
people_lib_id,
self.crm_data['CRM']['peopleLibraryTask']['cron'])
people_lib_task_url = self.crm_data['CRM']['baseUrl'] + self.crm_data[
'CRM']['peopleLibraryTask']['path'] + people_lib_task_query
headers["Authorization"] = self.crm_data['CRM']['peopleLibraryTask'][
'authorization']
people_lib_task_resp = self.req.post(people_lib_task_url,
headers=headers)
utils.log(people_lib_task_resp.text)
def search_people_api(self, payload):
utils.log("call search people api...")
search_url = config.QA_SERVER + self.crm_data['CRM']['search']['path']
headers = self.crm_data['CRM']['search']['header']
label_payload = {'labelListJson': payload}
search_resp = self.req.post(search_url,
headers=headers,
json=label_payload)
self.assertMsg(search_resp)
resp_dict = self.turn2dict(search_resp)
utils.log("dataCount:{}, selectCount:{}".format(
resp_dict['data']['dataCount'], resp_dict['data']['selectCount']))
return search_resp, resp_dict['data']['selectCount']
def save_people_api(self,
payload,
control_group,
select_count,
is_complex=False):
"""
save people api
:param payload:
:param control_group:
:param select_count: the select count in search api
:param is_complex: is select people complex
:return:
"""
utils.log("call create people library api...")
# construct save payload
if not isinstance(select_count, int):
select_count = int(select_count)
grp1_ratio = utils.get_random_numb(
55, 100) + 0.23 * utils.get_random_numb(
1, 10) # The sum of Radio should be 100
grp2_ratio = 100 - grp1_ratio
grp1_user_count = round(select_count * grp1_ratio * 0.01) # 四舍五入取整
grp2_user_count = round(select_count * grp2_ratio * 0.01) # 四舍五入取整
if is_complex:
url = config.QA_SERVER + self.crm_data['CRM']['saveComplex']['path']
header = self.crm_data['CRM']['saveComplex']['header']
response_keyword = 'result'
control_grp = json.loads(
control_group %
(grp1_ratio, grp1_user_count, grp2_ratio, grp2_user_count))
control_grp_json = {"crmControlGroupReqList": control_grp}
desc = self.desc_complex_prefix + utils.get_time_stamp()
else:
url = config.QA_SERVER + self.crm_data['CRM']['save']['path']
header = self.crm_data['CRM']['save']['header']
response_keyword = 'data'
control_grp = control_group % (grp1_ratio, grp1_user_count,
grp2_ratio, grp2_user_count)
control_grp_json = {"controlGroupJson": control_grp}
desc = self.desc_prefix + utils.get_time_stamp()
people_library_desc = {'peopleLibraryDesc': desc}
payload.update(people_library_desc)
payload.update(control_grp_json)
save_resp = self.req.post(url, headers=header, json=payload)
self.assertMsg(save_resp)
created_people_library_id = self.turn2dict(save_resp)[response_keyword]
utils.log(
"Create Successfully, The ID of People Library is: {}".format(
created_people_library_id))
return save_resp, created_people_library_id
def delete_people_library_api(self, people_library_id):
"""
Not really delete in db, just set it's field [state] to 9, means delete
:param people_library_id:
"""
utils.log("delete people library for ID: {}".format(people_library_id))
url = config.QA_SERVER + self.crm_data['CRM']['delete']['path'] + str(
people_library_id)
header = self.crm_data['CRM']['delete']['header']
resp = self.req.put(url=url, headers=header)
self.assertMsg(resp)
@unittest.skip("")
def test_select_save_people(self):
"""
select people and save people library
"""
utils.log("1. Call search people api.")
utils.log(
"Select people: gender= Male, birthday<=2006-10-03, marita status in ('Single','Married'), "
"last application status not in (Failure, Rejected),last login date >current_date-20"
)
# search_save_payload = self.crm_data['CRM']['search']['payload1']
search_save_payload = self.crm_data['CRM']['search']['payload2']
search_resp, select_count = self.search_people_api(search_save_payload)
self.assertGreater(int(select_count), 0,
"Select People Failed!!! The count less than 0")
utils.log("2. Save searched people library.")
save_resp, people_lib_id = self.save_people_api(
{"labelListJson": search_save_payload},
self.crm_data['CRM']['save']['controlGroupJson'],
int(select_count))
self.assertGreater(int(people_lib_id), 1,
"Save People Library Failed!!!")
utils.log("3. Verify new save people library exist in db.")
self.verify_people_lib_id_in_db(people_lib_id, self.desc_prefix)
utils.log("4. Add people library task.")
self.add_people_lib_task(people_lib_id)
def search_people_complex_api(self, payload):
utils.log("call search people complex api...")
search_complex_url = config.QA_SERVER + self.crm_data['CRM'][
'searchComplex']['path']
headers = self.crm_data['CRM']['searchComplex']['header']
search_resp = self.req.post(search_complex_url,
headers=headers,
json=payload)
self.assertMsg(search_resp)
resp_dict = self.turn2dict(search_resp)
utils.log("dataCount:{}, selectCount:{}".format(
resp_dict['result']['dataCount'],
resp_dict['result']['selectCount']))
return search_resp, resp_dict['result']['selectCount']
def verify_people_lib_id_in_db(self, people_lib_id, people_lib_desc):
query_sql = "select * from crm_people_library where id in({})".format(
people_lib_id)
people_lib_record = self.crm_db_conn.fetchone(query_sql)
utils.log("got people library record from db as below:\n{}".format(
people_lib_record))
self.assertTrue(
people_lib_desc in people_lib_record['people_library_desc'],
"Not find new saved people library in crm db!")
def test_search_complex_save_people(self):
utils.log("1. Call search complex people api.")
search_complex_payload = select_people_complex_search.get_select_people_complex_payload(
self.crm_data['CRM']['selectPeopleComplex'][config.COUNTRY], [
'monthlyIncome', 'marryStatus', 'deviceOfFirstApply',
'lastLoginDate'
], ['latestRiskScore', 'installDate'])
search_resp, select_count = self.search_people_complex_api(
search_complex_payload)
utils.log("2. Save searched people library.")
save_resp, people_lib_id = self.save_people_api(
search_complex_payload,
self.crm_data['CRM']['saveComplex']['controlGroupJson'],
int(select_count), True)
self.assertGreater(int(people_lib_id), 1,
"Save People Library Failed!!!")
utils.log("3. Verify new save people library exist in db.")
self.verify_people_lib_id_in_db(people_lib_id,
self.desc_complex_prefix)
utils.log("4. Add people library task.")
self.add_people_lib_task(people_lib_id)
class TestCRMTag(requestbase.RequestBase):
crm_data = utils.load_yml("crm_people_library.yaml")
resource = utils.load_yml("resources.yaml")
partition_0 = utils.get_time_stamp('%Y%m%d')
@classmethod
def setUpClass(cls):
utils.log("==================================================================")
utils.log("= =")
utils.log("=============== TEST CLASS SETUP =========")
utils.log("==================================================================")
utils.log("Set up test class...")
cls.req = requestlib.RequestLib(silence=True)
cls.indices_size = cls.get_indices_count(config.COUNTRY)
cls.reds = redashlib.Redash()
utils.log("==================================================================\n\n")
@classmethod
def get_es_url(cls, region, query_type='_search'):
"""
get ElasticSearch url via region and type.
:param region: US, JP, CN
:param query_type: eg: _search, _stats
:return: ES url
"""
utils.log('get [{}] ElasticSearch [{}] url.'.format(region, query_type))
if region == constants.US:
url = cls.crm_data['ElasticSearch']['host'] + cls.crm_data['ElasticSearch']['ph'+query_type] # Philippine
elif region == constants.JP:
url = cls.crm_data['ElasticSearch']['host'] + cls.crm_data['ElasticSearch']['id'+query_type] # Indonesia
else:
url = cls.crm_data['ElasticSearch']['host'] + cls.crm_data['ElasticSearch']['vn'+query_type] # Vietnam
utils.log('es search url:{}'.format(url))
return url
@classmethod
def es_search(cls, es_query):
utils.log('es query:\n{}'.format(es_query))
return cls.req.post(
cls.get_es_url(config.COUNTRY),
headers=cls.crm_data['ElasticSearch']['headers'],
data=es_query.encode('utf-8'))
@classmethod
def get_indices_count(cls, region):
resp = cls.req.get(cls.get_es_url(region, query_type='_stats'), headers=cls.crm_data['ElasticSearch']['headers'])
count = json.loads(resp.text)['_all']['primaries']['docs']['count']
utils.log("indices total count is: {}".format(count))
return count
def verify_tag_in_es(self, tag_name, query):
"""
快速验证Tag在ES里按条件查询出来的records数量在正常范围，即0<count<整个indices size。
等于0或等于整个indices size，大概率数据异常。
"""
utils.log("Test tag [{}] from ElasticSearch. Make sure there are records matched.".format(tag_name))
resp = self.es_search(query)
hit_total = json.loads(resp.text)['hits']['total']
utils.log("got hits total:{}".format(hit_total))
utils.log("Verify get records count, 0<count<{}(total indices count)".format(self.indices_size))
self.assertTrue(0 < hit_total < self.indices_size,
"Not find any records or equal indices count in ES for current query, please check!")
def verify_wide_es_equal(self, tag, wide_sql, es_query):
"""
compare user_tag_wide and ES data count should equal for tag.
"""
redash_count = self.reds.get_query_results_count(wide_sql)
resp = self.es_search(es_query)
es_count = json.loads(resp.text)['hits']['total']
utils.log('verify Redash and ES query count for tag [{}] should equal.'.format(tag))
utils.log("got redash count:{}, es count:{}".format(redash_count, es_count))
self.assertEqual(redash_count, es_count, 'The Redash and ES query count Not Equal!')
def verify_src_bigger_than_ext(self, src_table, backup_table, partition_0=partition_0, diff_size=2000, extra=None):
sql1 = 'select count(*) from {}'.format(src_table)
if extra:
sql2 = 'select count(*) from {} where partition_0={} {}'.format(backup_table, partition_0, extra)
else:
sql2 = 'select count(*) from {} where partition_0={}'.format(backup_table, partition_0)
src_result = self.reds.query_results(sql1)[0].get('count')
back_result = self.reds.query_results(sql2)[0].get('count')
utils.log("verify src table [{}] record count bigger than backup ext table [{}] and the count is close.".format(
src_table, backup_table))
utils.log("src count:{}, backup ext count:{}".format(src_result, back_result))
self.assertTrue(
0 <= src_result - back_result < diff_size,
"source table count should bigger than backup ext table count! And their count should be close!")
def verify_storage_wide_consistent(self, tag, storage_sql, wide_sql):
if config.IS_PROD_STORAGE:
storage_sql = storage_sql.replace("crmtest_ext", "crm_ext")
storage_result = self.reds.get_query_results_count(storage_sql)
wide_result = self.reds.get_query_results_count(wide_sql)
utils.log('verify for tag:[{}], user_tag_storage and user_tag_wide Not Null count should Equal.'.format(tag))
self.assertEqual(storage_result, wide_result,
'user_tag_storage and user_tag_wide for tag:[{}] not null count Not Equal!'.format(tag))
@unittest.skipUnless(config.COUNTRY == constants.CN, 'case only for Vietnam')
def test_vn_storage_wide_total_consistent(self):
"""验证窄表与宽表数据总量一致"""
utils.log("verify user_tag_storage and user_tag_wide total count equals")
self.verify_storage_wide_consistent(
'',
"select count(1) from vncrmtest_ext.user_tag_storage where tag_id='accountname'",
"select count(1) from vncrmtest_ext.user_tag_wide")
@unittest.skipUnless(config.COUNTRY == constants.CN, 'case only for Vietnam')
def test_vn_storage_wide_consistent(self):
"""验证窄表和宽表各个Tag的Not Null数据一致."""
storage_query_list = self.crm_data['StorageWideQuery']['Storage'][constants.CN]
wide_query_list = self.crm_data['StorageWideQuery']['Wide'][constants.CN]
for sq, wq in zip(storage_query_list, wide_query_list):
with self.subTest(storage_query=sq, wide_query=wq):
self.verify_storage_wide_consistent(next(iter(sq.keys())), next(iter(sq.values())),
next(iter(wq.values())))
@unittest.skipUnless(config.COUNTRY == constants.CN, 'case only for Vietnam')
def test_vn_wide_es_total_count_equal(self):
"""验证宽表和ES Indices:vn_crm_user_profile_alias数据总量相同"""
utils.log('verify vncrmtest_ext.user_tag_wide and es indices:vn_crm_user_profile_alias total count equal.')
wide_total = self.reds.get_query_results_count('select count(1) from vncrmtest_ext.user_tag_wide')
self.assertEqual(wide_total, self.indices_size, 'user_tag_wide and es total count Not equal!')
@unittest.skipUnless(config.COUNTRY == constants.CN, 'case only for Vietnam')
def test_vietnam_wide_es_consistent(self):
"""验证宽表和ES里各个Tag数据一致"""
wide_sql = self.crm_data['UserTagWide'][constants.CN]
es_query = self.crm_data['ESQuery'][constants.CN]
for wq, eq in zip(wide_sql, es_query):
with self.subTest(wide_query=wq, es_query=eq):
self.verify_wide_es_equal(next(iter(wq.keys())), next(iter(wq.values())), next(iter(eq.values())))
def test_tag_in_es(self):
"""验证tag在ES里按条件查询出的数量是正常的(0<count<total_count)"""
tag_query_list, skip_list = self.crm_data['ESQuery'][config.COUNTRY], self.crm_data['ESQuerySkip'][config.COUNTRY]
final_query_list = [item for item in tag_query_list if next(iter(item.keys())) not in skip_list]
for query in final_query_list:
with self.subTest(query=query):
self.verify_tag_in_es(next(iter(query.keys())), next(iter(query.values())))
@unittest.skipUnless(config.COUNTRY == constants.CN, 'case only for Vietnam')
def test_vnods_persons_and_ext(self):
"""验证原始表和它的备份表ext，备份表数据总量应小于原始表，且相差不大(默认相差阀值：2000)"""
extra_condition = "and createdat !='' and updatedat !='' and mobile is not null and residentialdistrictaddress is not null"
self.verify_src_bigger_than_ext('vnods.persons', 'vnods_ext.persons', extra=extra_condition)
def get_es_data_details_list(self, query):
"""get es data details record data list"""
detail_result_list = []
resp = self.es_search(query)
hits_list = json.loads(resp.text)['hits']['hits']
for hit in hits_list:
detail_result_list.append(hit['_source'])
return detail_result_list
def verify_es_details_with_src(self, es_check_list, src_check_list, es_data_list, src_query, partition_0=None):
if len(es_check_list) != len(src_check_list):
return utils.warn("please make sure the src and es compare fields number equal!")
es_data = es_data_list if len(es_data_list) <= 3 else es_data_list[0:3]
for num, es in enumerate(es_data, start=1):
es_actual, src_expect = [], []
utils.log("check record #{}".format(num))
if partition_0:
src = self.reds.query_results(src_query.format(es['accountid'], partition_0))[0]
else:
src = self.reds.query_results(src_query.format(es['accountid']))[0]
for es_col, src_col in zip(es_check_list, src_check_list):
es_actual.append(str(es[es_col]) if es[es_col] is not None else es[es_col])
src_expect.append(str(src[src_col]) if src[src_col] is not None else src[src_col])
utils.log("verify es and src [{}] data matched for below fields.\n{}".format(src_query.split(' ')[3], es_check_list))
self.assertListEqual(src_expect, es_actual, 'Not Match between es and src data!')
@unittest.skipUnless(config.COUNTRY == constants.CN, 'case only for Vietnam')
def test_vn_es_data_consistence_with_account(self):
"""验证ES数据内容与源表account一致"""
utils.log("verify es details data is consistence with source account.")
self.verify_es_details_with_src(self.crm_data['ESSrcDetail']['Check'][constants.CN]['es_account'],
self.crm_data['ESSrcDetail']['Check'][constants.CN]['account'],
self.get_es_data_details_list(self.crm_data['ESSrcDetail']['ES'][constants.CN]['query']),
self.crm_data['ESSrcDetail']['Src'][constants.CN]['account'])
@unittest.skipUnless(config.COUNTRY == constants.CN, 'case only for Vietnam')
def test_vn_es_data_consistence_with_person(self):
"""验证ES数据内容与源表person一致"""
utils.log("verify es details data is consistence with source person.")
self.verify_es_details_with_src(self.crm_data['ESSrcDetail']['Check'][constants.CN]['person'],
self.crm_data['ESSrcDetail']['Check'][constants.CN]['person'],
self.get_es_data_details_list(self.crm_data['ESSrcDetail']['ES'][constants.CN]['query']),
self.crm_data['ESSrcDetail']['Src'][constants.CN]['person'])
@unittest.skipUnless(config.COUNTRY == constants.CN, 'case only for Vietnam')
def test_vn_es_data_consistence_with_work(self):
"""验证ES数据内容与源表work一致"""
utils.log("verify es details data is consistence with source work.")
self.verify_es_details_with_src(self.crm_data['ESSrcDetail']['Check'][constants.CN]['es_work'],
self.crm_data['ESSrcDetail']['Check'][constants.CN]['work'],
self.get_es_data_details_list(self.crm_data['ESSrcDetail']['ES'][constants.CN]['workQuery']),
self.crm_data['ESSrcDetail']['Src'][constants.CN]['work'])
def verify_es_location_with_src(self, es_data_list, src_query):
es_data = es_data_list if len(es_data_list) <= 3 else es_data_list[0:3]
for num, es in enumerate(es_data, start=1):
utils.log("check record #{}".format(num))
src = self.reds.query_results(src_query.format(es['accountid'], self.partition_0))[0]
src_location = ','.join(str(i) for i in [src['residential_latitude'], src['residential_longitude']])
utils.log("verify es [location] data is matched src construct str 'residential_latitude,residential_longitude'.")
self.assertEqual(src_location, es['location'], 'Not Match between es and src data!')
@unittest.skipUnless(config.COUNTRY == constants.US, 'case only for US')
def test_ph_es_geo_consistence_with_user_addr_gps(self):
"""验证ES经纬度与源表crm_ext.user_addr_gps一致"""
utils.log("verify es residential_latitude, residential_longitude data is consistence with source user_addr_gps")
self.verify_es_details_with_src(self.crm_data['ESSrcDetail']['Check'][constants.US]['geo'],
self.crm_data['ESSrcDetail']['Check'][constants.US]['geo'],
self.get_es_data_details_list(
self.crm_data['ESSrcDetail']['ES'][constants.US]['query']),
self.crm_data['ESSrcDetail']['Src'][constants.US]['gps'], self.partition_0)
@unittest.skipUnless(config.COUNTRY == constants.US, 'case only for US')
def test_ph_es_location_consistence_with_user_addr_gps(self):
"""验证ES location与源表crm_ext.user_addr_gps数据一致"""
self.verify_es_location_with_src(
self.get_es_data_details_list(self.crm_data['ESSrcDetail']['ES'][constants.US]['query']),
self.crm_data['ESSrcDetail']['Src'][constants.US]['gps'])
@unittest.skipUnless(config.COUNTRY == constants.US, 'case only for US')
def test_ph_storage_wide_total_consistent(self):
"""验证窄表与宽表数据总量一致"""
utils.log("verify US user_tag_storage and user_tag_wide total count equals")
self.verify_storage_wide_consistent(
'',
"select count(1) from crmtest_ext.user_tag_storage where tag_id='accountname'",
"select count(1) from crmtest_ext.user_tag_wide")
@unittest.skipUnless(config.COUNTRY == constants.US, 'case only for US')
def test_ph_storage_wide_consistent(self):
"""验证窄表和宽表各个Tag的Not Null数据一致."""
storage_query_list = self.crm_data['StorageWideQuery']['Storage'][constants.US]
wide_query_list = self.crm_data['StorageWideQuery']['Wide'][constants.US]
for sq, wq in zip(storage_query_list, wide_query_list):
with self.subTest(storage_query=sq, wide_query=wq):
self.verify_storage_wide_consistent(next(iter(sq.keys())), next(iter(sq.values())),
next(iter(wq.values())))
@unittest.skipUnless(config.COUNTRY == constants.US, 'case only for US')
def test_ph_wide_es_total_count_equal(self):
"""验证宽表和ES Indices:crm_user_profile_alias数据总量相同"""
utils.log('verify crmtest_ext.user_tag_wide and es indices:crm_user_profile_alias total count equal.')
wide_total = self.reds.get_query_results_count('select count(1) from crmtest_ext.user_tag_wide')
self.assertEqual(wide_total, self.indices_size, 'user_tag_wide and es total count Not equal!')
@unittest.skip('')
def test_update_increment_new_tb_consistence_with_src(self):
"""验证tag数据增量更新,new表数据与源表一致"""
utils.log('Test tables use incremental update works.')
origin_query_list = self.crm_data['IncrementalUpdate']['originTb'][config.COUNTRY]
new_query_list = self.crm_data['IncrementalUpdate']['newTb'][config.COUNTRY]
ignore_items = self.crm_data['IncrementalUpdate']['ignoreItems'][0]
for o, n in zip(origin_query_list, new_query_list):
with self.subTest(origin_tb_query=o, new_tb_query=n):
self.verify_incremental_update(next(iter(o.keys())), next(iter(n.keys())), next(iter(o.values())),
next(iter(n.values())), ignore_items)
def verify_incremental_update(self, origin_tb, new_tb, origin_query, new_query, ignore_items):
is_diff = False
added_list, modified_list = [], []
for origin_record, new_record in zip(self.reds.query_results(origin_query), self.reds.query_results(new_query)):
for ignore in ignore_items:
origin_record.pop(ignore)
new_record.pop(ignore)
new_record.pop('updatedat') # new add col in new table, ignore it
added, modified = utils.dict_compare(origin_record, new_record)
if added: added_list.append(added)
if modified: modified_list.append(modified)
if origin_record != new_record: is_diff = True
utils.log('verify {} and {} data details is consistence.'.format(origin_tb, new_tb))
if added_list: utils.log('new columns item found in {}:{}'.format(new_tb, added_list))
if modified_list: utils.log('diff items details in two table:{}'.format(modified_list))
self.assertFalse(is_diff, 'There are records diff between {} and {}!'.format(origin_tb, new_tb))
def test_wide_tag_all_type_sum_equal_total(self):
"""验证宽表总量=tag value(not null+null+'')count之和"""
wide_total = self.reds.get_query_results_count(self.crm_data['WideTotalQuery'][config.COUNTRY]['query'])
not_null_query_list = self.crm_data['StorageWideQuery']['Wide'][config.COUNTRY]
null_query_list = self.crm_data['WideNullEmpty'][config.COUNTRY]
for notnq, nq in zip(not_null_query_list, null_query_list):
with self.subTest(wide_not_null_query=notnq, wide_null_query=nq):
not_null_count = self.reds.get_query_results_count(next(iter(notnq.values())))
null_empty_count = self.reds.get_query_results_count(next(iter(nq.values())))
utils.log("verify tag:[{}], it's value with (not null+null+'') count sum should equal wide total count".
format(next(iter(notnq.keys()))))
utils.log("got tag {}, not null count:[{}], null+'' count:[{}], and wide total:[{}]".format(
next(iter(notnq.keys())), not_null_count, null_empty_count, wide_total))
self.assertEqual(wide_total, (not_null_count+null_empty_count),
"wide total count Not Equal tag value(not null+null+'') sum!")
@unittest.skipUnless(config.COUNTRY == constants.US, 'case only for US')
def test_ph_wide_es_consistent(self):
"""验证宽表和ES里各个Tag数据一致"""
wide_sql = self.crm_data['UserTagWide'][constants.US]
es_query = self.crm_data['ESQuery'][constants.US]
for wq, eq in zip(wide_sql, es_query):
with self.subTest(wide_query=wq, es_query=eq):
self.verify_wide_es_equal(next(iter(wq.keys())), next(iter(wq.values())), next(iter(eq.values())))
@unittest.skipUnless(config.COUNTRY == constants.JP, 'case only for JP')
def test_id_storage_wide_consistent(self):
"""验证窄表和宽表各个Tag的Not Null数据一致."""
storage_query_list = self.crm_data['StorageWideQuery']['Storage'][constants.JP]
wide_query_list = self.crm_data['StorageWideQuery']['Wide'][constants.JP]
for sq, wq in zip(storage_query_list, wide_query_list):
with self.subTest(storage_query=sq, wide_query=wq):
self.verify_storage_wide_consistent(next(iter(sq.keys())), next(iter(sq.values())),
next(iter(wq.values())))
@unittest.skipUnless(config.COUNTRY == constants.JP, 'case only for JP')
def test_id_wide_es_total_count_equal(self):
"""验证宽表和ES Indices:id_crm_user_profile_alias"""
utils.log('verify crmtest_ext.user_tag_wide and es indices:id_crm_user_profile_alias total count equal.')
wide_total = self.reds.get_query_results_count('select count(1) from idcrmtest_ext.user_tag_wide')
self.assertEqual(wide_total, self.indices_size, 'user_tag_wide and es total count Not equal!')
@unittest.skipUnless(config.COUNTRY == constants.JP, 'case only for JP')
def test_id_wide_es_consistent(self):
"""验证宽表和ES里各个Tag数据一致"""
wide_sql = self.crm_data['UserTagWide'][constants.JP]
es_query = self.crm_data['ESQuery'][constants.JP]
for wq, eq in zip(wide_sql, es_query):
with self.subTest(wide_query=wq, es_query=eq):
self.verify_wide_es_equal(next(iter(wq.keys())), next(iter(wq.values())), next(iter(eq.values())))
@unittest.skipUnless(config.COUNTRY == constants.JP, 'case only for JP')
def test_id_es_risk_score_consistence_with_src(self):
"""验证ES数据latest_risk_score与源表account_credits一致"""
utils.log("verify es data latest_risk_score data is consistence with source account_credits.")
self.verify_es_details_with_src(self.crm_data['ESSrcDetail']['Check'][constants.JP]['es_latest_risk_score'],
self.crm_data['ESSrcDetail']['Check'][constants.JP]['score'],
self.get_es_data_details_list(
self.crm_data['ESSrcDetail']['ES'][constants.JP]['query']),
self.crm_data['ESSrcDetail']['Src'][constants.JP]['account_credits'],
self.partition_0)
@unittest.skipUnless(config.COUNTRY == constants.JP, 'case only for JP')
def test_id_es_salary_consistence_with_src(self):
"""验证ES数据monthly_income与源表account_credits一致"""
utils.log("verify es data monthly_income data is consistence with source works.")
self.verify_es_details_with_src(self.crm_data['ESSrcDetail']['Check'][constants.JP]['es_work'],
self.crm_data['ESSrcDetail']['Check'][constants.JP]['work'],
self.get_es_data_details_list(
self.crm_data['ESSrcDetail']['ES'][constants.JP]['query']),
self.crm_data['ESSrcDetail']['Src'][constants.JP]['work'])
@unittest.skipUnless(config.COUNTRY == constants.JP, 'case only for JP')
def test_id_storage_wide_total_consistent(self):
"""验证窄表与宽表数据总量一致"""
utils.log("verify JP user_tag_storage and user_tag_wide total count equals")
self.verify_storage_wide_consistent(
'',
"select count(1) from idcrmtest_ext.user_tag_storage where tag_id='accountname'",
"select count(1) from idcrmtest_ext.user_tag_wide")
def test_wide_union_query_sum_same_with_total(self):
"""验证宽表标签组合查询之和与总数一致"""
wide_total_list = self.crm_data['WideUnionSeparate']['Total']
wide_sep_sql1_list = self.crm_data['WideUnionSeparate']['Separate1']
wide_sep_sql2_list = self.crm_data['WideUnionSeparate']['Separate2']
for wtq, wsq1, wsq2 in zip(wide_total_list, wide_sep_sql1_list, wide_sep_sql2_list):
with self.subTest(wide_total_query=wtq, wide_sep_query1=wsq1, wide_sep_query2=wsq2):
self.verify_wide_union_query_sum_equal_total(next(iter(wtq.keys())), next(iter(wtq.values())),
next(iter(wsq1.values())), next(iter(wsq2.values())))
def verify_wide_union_query_sum_equal_total(self, tag, wide_total, wide_sep_sql1, wide_sep_sql2):
prefix = self.get_prefix()
wide_total, wide_sep_sql1, wide_sep_sql2 = wide_total.format(prefix), wide_sep_sql1.format(prefix), wide_sep_sql2.format(prefix)
wide_total_result = self.reds.get_query_results_count(wide_total)
wide_sep1_result = self.reds.get_query_results_count(wide_sep_sql1)
wide_sep2_result = self.reds.get_query_results_count(wide_sep_sql2)
utils.log('verify for tag:[{}], user_tag_wide union query count sum should Equal total count.'.format(tag))
self.assertEqual(wide_sep1_result+wide_sep2_result, wide_total_result,
'user_tag_wide union query count sum for tag:[{}] Not Equal with total count!'.format(tag))
def test_wide_not_null_with_storage_null_2_zero(self):
"""验证窄表中把null值转为0的标签，宽表Not null=窄表null+not null"""
wide_not_null_list = self.crm_data['WideStorageNullToZero']['WideNotNull']
storage_not_null_list = self.crm_data['WideStorageNullToZero']['StorageNotNull']
storage_null_list = self.crm_data['WideStorageNullToZero']['StorageNull']
for wq, sq1, sq2 in zip(wide_not_null_list, storage_not_null_list, storage_null_list):
with self.subTest(wide_not_null_query=wq, storage_not_null_query=sq1, storage_null_query=sq2):
self.verify_wide_storage_null_to_0_consistent(next(iter(wq.keys())), next(iter(wq.values())),
next(iter(sq1.values())), next(iter(sq2.values())))
def verify_wide_storage_null_to_0_consistent(self, tag, wide_not_null_sql, storage_not_null_sql, storage_null_sql):
"""窄表中把null值转为0， 宽表Not null=窄表null+not null"""
prefix = self.get_prefix()
wide_not_null_sql, storage_not_null_sql, storage_null_sql = wide_not_null_sql.format(prefix), storage_not_null_sql.format(prefix), storage_null_sql.format(prefix)
if config.IS_PROD_STORAGE:
storage_not_null_sql = storage_not_null_sql.replace("crmtest_ext", "crm_ext")
storage_null_sql = storage_null_sql.replace("crmtest_ext", "crm_ext")
wide_not_null_result = self.reds.get_query_results_count(wide_not_null_sql)
storage_not_null_result = self.reds.get_query_results_count(storage_not_null_sql)
storage_null_result = self.reds.get_query_results_count(storage_null_sql)
utils.log('verify for tag:[{}], user_tag_wide not null count should Equal (storage not null + null) count.'.format(tag))
self.assertEqual(wide_not_null_result, storage_not_null_result+storage_null_result,
'user_tag_wide tag[{}] not null count Not Equal (storage not null + null) count!'.format(tag))
def get_prefix(self):
prefix = ''
if config.COUNTRY == constants.JP:
prefix = 'id'
elif config.COUNTRY == constants.CN:
prefix = 'vn'
return prefix
import inspect
from pymongo import MongoClient, ASCENDING, UpdateOne
from pymongo.results import UpdateResult
from utils import utils
from config import config
resource = utils.load_yml("resources.yaml")
MONGO_URL = resource['MongoDB']['MONGO_URL']
MONGO_DB_NAME = resource['MongoDB']['MONGO_DB_NAME'][config.COUNTRY]
client = MongoClient(MONGO_URL, connect=False)
db = client[MONGO_DB_NAME]
coll_users = db.users
coll_loans = db.loans
class MongoDAL:
def __init__(self):
pass
@staticmethod
def create_index():
if not MongoDAL.created_index_called:
coll_loans.create_index([('id_account', ASCENDING)])
MongoDAL.created_index_called = True
@staticmethod
def pretty_print_result(result):
class TestRuleEngine(requestbase.RequestBase):
rule_data = utils.load_yml("rule_engine_data.yaml")
resource = utils.load_yml("resources.yaml")
@classmethod
def setUpClass(cls):
utils.log(
"=================================================================="
)
utils.log(
"= ="
)
utils.log(
"=============== TEST CLASS SETUP ========="
)
utils.log(
"=================================================================="
)
utils.log("Set up test class...")
cls.req = requestlib.RequestLib()
cls.trade_conn = mysqlhelper.MysqlConnector(
cls.resource['TradeDB'][config.COUNTRY]['host'],
cls.resource['TradeDB']['user'],
cls.resource['TradeDB']['password'], cls.resource['TradeDB']['db'])
cls.risk_conn = mysqlhelper.MysqlConnector(
cls.resource['StagingRiskDB'][config.COUNTRY]['host'],
cls.resource['StagingRiskDB']['user'],
cls.resource['StagingRiskDB']['password'],
cls.resource['StagingRiskDB']['db'],
cls.resource['StagingRiskDB'][config.COUNTRY]['port'])
cls.prefix = cls.resource['StagingRiskDB'][config.COUNTRY]['tbPrefix']
cls.time_now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
utils.log(
"==================================================================\n\n"
)
@classmethod
def tearDownClass(cls):
utils.log(
"=================================================================="
)
utils.log(
"= ="
)
utils.log(
"================ TEST CLASS CLEANUP =========="
)
utils.log(
"=================================================================="
)
if cls.req: cls.req.close_session()
if cls.trade_conn: cls.trade_conn.close()
if cls.risk_conn: cls.risk_conn.close()
utils.log(
"==================================================================\n\n"
)
def call_rule(self, test_data):
params = self.rule_data['rule']['params']['general']
params.update(test_data)
result = RuleEngine.rule(self.req, params)
utils.log('Step. verify call api borrow-apply successfully.')
self.assertEqual(utils.query_json(result, 'msg'), 'success',
'call api rule failed!')
time.sleep(3)
utils.log(result)
return result
def verify_rule_result(self, data, rule_id, expected_result):
id_account, id_borrow = data['idAccount'], data['idBorrow']
rule_result_sql = self.rule_data['SQL']['query']['ruleResult'].format(
id_account, rule_id)
fetch_rule_result = self.risk_conn.fetchone(rule_result_sql)
self.assertIsNotNone(
fetch_rule_result,
'Not find rule #{} result for id_account:{} id_borrow:{}'.format(
rule_id, id_account, id_borrow))
utils.log("rule_result:\n{}".format(fetch_rule_result))
rule_result = fetch_rule_result['rule_result']
utils.log(
'Step. verify rule_id={} result is [{}] for id_account:{} id_borrow:{}'
.format(rule_id, expected_result, id_account, id_borrow))
self.assertEqual(expected_result, rule_result,
'rule result Not as expected! please check!')
def insert_mongo_users(self, id_account):
utils.log('insert mongo users if not exist for id_account: {}'.format(
id_account))
if not mongolib.MongoDAL.get_user(int(id_account)):
mongolib.MongoDAL.upsert_user(
int(id_account), self.rule_data['Mongo']['insert']['users'])
time.sleep(1)
def prepare_rule_99_100_data_verify_result(self,
data_name,
rule_id,
expected_rule_result,
set_score=False,
total_score=None):
test_data = self.rule_data['ruleTestData'][data_name]
id_account, id_borrow = test_data['idAccount'], test_data['idBorrow']
transaction_id = "RT{}{}{}".format(id_account, id_borrow,
utils.get_random_numb(100, 200))
test_data['transactionId'] = transaction_id
utils.log(
'Step. make sure there are record in mongo users collection.')
self.insert_mongo_users(id_account)
utils.log(
'Step. make sure no model total score result via delete it in rule_model_total_score'
)
delete_total_score_sql = self.rule_data['SQL']['delete'][
'modelTotalScore'].format(id_account)
self.risk_conn.update(delete_total_score_sql)
if set_score:
utils.log(
'Step. insert a total score={} record in rule_model_total_score'
.format(total_score))
insert_total_score_sql = self.rule_data['SQL']['insert'][
'modelTotalScore'].format(id_account, id_borrow, total_score,
transaction_id)
self.risk_conn.update(insert_total_score_sql)
self.call_rule(test_data)
utils.log('Step. verify rule #{} result is {}'.format(
rule_id, expected_rule_result))
self.verify_rule_result(test_data, rule_id, expected_rule_result)
def test_rule_99_has_apply_no_total_score(self):
"""验证VNKyc3.0.2 has user apply, no model total score,rule 99,loan_type='1'-> -2 not_null_but_null"""
self.prepare_rule_99_100_data_verify_result('rule99', 99, -2)
def test_rule_99_has_apply_total_score_null(self):
"""验证验证VNKyc3.0.2 has user apply, model total score is null,rule 99,loan_type='1'-> -2 not_null_but_null"""
self.prepare_rule_99_100_data_verify_result('rule99', 99, -2, True,
'NULL')
def test_rule_99_has_apply_total_score_less_560(self):
"""验证VNKyc3.0.2 has user apply, model total score<560,rule 99,loan_type='1'-> 1 bad guy"""
self.prepare_rule_99_100_data_verify_result('rule99', 99, 1, True, 550)
def test_rule_99_has_apply_total_score_more_560(self):
"""验证VNKyc3.0.2 has user apply, model total score>=560,rule 99,loan_type='1'-> 0 good guy"""
self.prepare_rule_99_100_data_verify_result('rule99', 99, 0, True, 560)
def test_rule_100_has_apply_no_total_score(self):
"""验证VNKyc3.0.2 has user apply, no model total score,rule 100,loan_type='2'-> -2 not_null_but_null"""
self.prepare_rule_99_100_data_verify_result('rule100', 100, -2)
def test_rule_100_has_apply_total_score_null(self):
"""验证验证VNKyc3.0.2 has user apply, model total score is null,rule 100,loan_type='2'-> -2 not_null_but_null"""
self.prepare_rule_99_100_data_verify_result('rule100', 100, -2, True,
'NULL')
def test_rule_100_has_apply_total_score_less_560(self):
"""验证VNKyc3.0.2 has user apply, model total score<560,rule 100,loan_type='2'-> 1 bad guy"""
self.prepare_rule_99_100_data_verify_result('rule100', 100, 1, True,
550)
def test_rule_100_has_apply_total_score_more_560(self):
"""验证VNKyc3.0.2 has user apply, model total score>=560,rule 100,loan_type='2'-> 0 good guy"""
self.prepare_rule_99_100_data_verify_result('rule100', 100, 0, True,
560)
def test_rule_101_relative_cnt_null_loan1(self):
"""验证VNKyc3.0.2联系人手机号关联账号次数 is null,rule 101,loan_type in (1,2)-> 0 good guy"""
self.prepare_rule_101_data_verify_result('rule99',
101,
0,
mobil_numb='852627326')
def test_rule_101_relative_cnt_less_5_loan1(self):
"""验证VNKyc3.0.2 0<联系人手机号关联账号次数<5,rule 101,loan_type in (1,2)-> 0 good guy"""
self.prepare_rule_101_data_verify_result('rule99', 101, 0, True, 3,
'852627326')
def test_rule_101_relative_cnt_eq_more_5_loan1(self):
"""验证VNKyc3.0.2联系人手机号关联账号次数>=5,rule 101,loan_type in (1,2)-> 1 bad guy"""
self.prepare_rule_101_data_verify_result('rule99', 101, 1, True, 5,
'852627326')
def test_rule_101_relative_cnt_null_loan2(self):
"""验证VNKyc3.0.2联系人手机号关联账号次数 is null,rule 101,loan_type in (1,2)-> 0 good guy"""
self.prepare_rule_101_data_verify_result('rule100',
101,
0,
mobil_numb='852627638')
def test_rule_101_relative_cnt_less_5_loan2(self):
"""验证VNKyc3.0.2 0<联系人手机号关联账号次数<5,rule 101,loan_type in (1,2)-> 0 good guy"""
self.prepare_rule_101_data_verify_result('rule100', 101, 0, True, 3,
'852627638')
def test_rule_101_relative_cnt_eq_more_5_loan2(self):
"""验证VNKyc3.0.2联系人手机号关联账号次数>=5,rule 101,loan_type in (1,2)-> 1 bad guy"""
self.prepare_rule_101_data_verify_result('rule100', 101, 1, True, 5,
'852627638')
def prepare_rule_101_data_verify_result(self,
data_name,
rule_id,
expected_rule_result,
set_contacts=False,
relative_cnt=None,
mobil_numb=None):
test_data = self.rule_data['ruleTestData'][data_name]
id_account, id_borrow = test_data['idAccount'], test_data['idBorrow']
transaction_id = "RT{}{}{}".format(id_account, id_borrow,
utils.get_random_numb(201, 300))
test_data.update({
'transactionId': transaction_id,
'roundId': '1'
}) # roundId 更新为1
utils.log(
'Step. make sure there are record in mongo users collection.')
self.insert_mongo_users(id_account)
utils.log(
'Step. make sure no relative_cnt via delete it in vnrisk_applied_contacts'
)
delete_applied_contacts_sql = self.rule_data['SQL']['delete'][
'appliedContacts'].format(mobil_numb)
self.risk_conn.update(delete_applied_contacts_sql)
if set_contacts:
utils.log(
'Step. insert a record in vnrisk_user_contacts for id_account:{} with mobil number:{}'
.format(id_account, mobil_numb))
query_user_contacts_cnt = self.rule_data['SQL']['query'][
'userContactsCnt'].format(id_account, mobil_numb)
if self.risk_conn.fetchone(query_user_contacts_cnt)['cnt'] == 0:
insert_user_contacts_sql = self.rule_data['SQL']['insert'][
'userContacts'].format(utils.get_random_numb(2999, 5555),
id_account, mobil_numb)
self.risk_conn.update(insert_user_contacts_sql)
utils.log(
'Step. insert a record in vnrisk_applied_contacts for mobile={},set relative_cnt:{}'
.format(mobil_numb, relative_cnt))
query_applied_contacts_cnt = self.rule_data['SQL']['query'][
'applied_contacts_cnt'].format(mobil_numb)
# if self.risk_conn.fetchone(query_applied_contacts_cnt)['cnt'] == 0:
# applied_contacts_sql = self.rule_data['SQL']['insert']['applied_contacts'].format(mobil_numb, id_account, relative_cnt)
# else:
# applied_contacts_sql = self.rule_data['SQL']['update']['applied_contacts'].format(id_account, relative_cnt, mobil_numb)
insert_applied_contacts = self.rule_data['SQL']['insert'][
'applied_contacts'].format(mobil_numb, id_account,
relative_cnt)
update_applied_contacts = self.rule_data['SQL']['update'][
'applied_contacts'].format(id_account, relative_cnt,
mobil_numb)
applied_contacts_sql = update_applied_contacts if self.risk_conn.fetchone(
query_applied_contacts_cnt)['cnt'] else insert_applied_contacts
self.risk_conn.update(applied_contacts_sql)
self.call_rule(test_data)
utils.log('Step. verify rule #{} result is {}'.format(
rule_id, expected_rule_result))
self.verify_rule_result(test_data, rule_id, expected_rule_result)
def test_rule_102_no_history_device(self):
"""验证VNKyc3.0.2 三个月内historyDeviceCount=0,rule 102,loan_type in (1,2)-> -2 not_null_but_null"""
self.prepare_rule_102_data_verify_result('rule99', 102, -2)
def test_rule_102_history_device_more2_has_share_device(self):
"""验证VNKyc3.0.2 三个月内 historyDeviceCount>2且deviceShareCount> 0,rule 102,loan_type in (1,2)-> 1 bad guy"""
self.prepare_rule_102_data_verify_result('rule99', 102, 1, True, True,
True)
def test_rule_102_history_device_less2_has_share_device(self):
"""验证VNKyc3.0.2 三个月内historyDeviceCount<=2但deviceShareCount>0,rule 102,loan_type in (1,2)-> 0 good guy"""
self.prepare_rule_102_data_verify_result('rule99', 102, 0, True, False,
True)
def test_rule_102_history_device_more2_no_share_device(self):
"""验证VNKyc3.0.2 三个月内historyDeviceCount>2但无deviceShareCount,rule 102,loan_type in (1,2)-> 0 good guy"""
self.prepare_rule_102_data_verify_result('rule99', 102, 0, True, True,
False)
def prepare_rule_102_data_verify_result(self,
data_name,
rule_id,
expected_rule_result,
set_device=False,
two_more_device=False,
share_device=False):
test_data = self.rule_data['ruleTestData'][data_name]
id_account, id_borrow = test_data['idAccount'], test_data['idBorrow']
transaction_id = "RT{}{}{}".format(id_account, id_borrow,
utils.get_random_numb(301, 400))
test_data.update({
'transactionId': transaction_id,
'roundId': '1'
}) # roundId 更新为1
utils.log(
'Step. make sure there are record in mongo users collection.')
self.insert_mongo_users(id_account)
utils.log(
'Step. make sure no test device_id info(in yml) via delete them in vnrpt_bhv_user_device_mapping.'
)
delete_user_device_sql = self.rule_data['SQL']['delete'][
'userDeviceMapping']
self.risk_conn.update(delete_user_device_sql)
if set_device:
update_ts = utils.get_previous_date(6) # 6 days ago
if two_more_device:
utils.log(
'Step. insert 3 device info for account:{} within 3 months(historyDeviceCount > 2)'
.format(id_account)) # vnrpt_bhv_user_device_mapping
insert_device_in_3_month_sql = self.rule_data['SQL']['insert'][
'3devices'].format(id_account, update_ts, id_account,
update_ts, id_account, update_ts)
else:
utils.log(
'Step. insert 2 device info for account:{} within 3 months(historyDeviceCount <= 2)'
.format(id_account))
insert_device_in_3_month_sql = self.rule_data['SQL']['insert'][
'2devices'].format(id_account, update_ts, id_account,
update_ts)
self.risk_conn.update(insert_device_in_3_month_sql)
if share_device:
utils.log(
'Step. insert a share device:[300728abda0ec6e7] record with account:10086 for account:{} (deviceShareCount>0)'
.format(id_account)) # vnrpt_bhv_user_device_mapping
insert_1_share_device_with_another_sql = self.rule_data['SQL'][
'insert']['sharedevice']
self.risk_conn.update(insert_1_share_device_with_another_sql)
time.sleep(2)
self.call_rule(test_data)
utils.log('Step. verify rule #{} result is {}'.format(
rule_id, expected_rule_result))
self.verify_rule_result(test_data, rule_id, expected_rule_result)
class TestReportSchedule(loginbase.LoginBase):
data = utils.load_yml(config.TEST_DATA_DIR+"/report_schedule_data.yaml")
new_biz_type_list = []
def test_get_report_simple(self):
"""
quick check there are data in Report View.
"""
utils.log("Case. Test there are schedule reports in view.")
self.reportSchedule.click_report_biz_list()
schedule_reports = self.reportSchedule.get_data_grid_data()
utils.log("verify schedule report is not empty.")
self.assertIsNotNone(schedule_reports, "There are No schedule reports in view!")
def test_add_schedule_excel(self):
utils.log("Case. Test add a new excel report schedule and publish it.")
self.add_publish_report('excel-report-2')
def test_add_schedule_csv(self):
utils.log("Case. Test add a new csv report schedule and publish it.")
self.add_publish_report('csv-report-2')
def test_add_schedule_table(self):
utils.log("Case. Test add a new table report schedule and publish it.")
self.add_publish_table_report('table-report-2')
def add_schedule_report(self, test_data):
self.reportSchedule.click_report_biz_list()
suffix = utils.get_random_numb()
biz_type = self.data[test_data]['BizType'] + str(suffix)
TestReportSchedule.new_biz_type_list.append(biz_type)
self.reportSchedule.click_add()
self.reportSchedule.fill_add_report_config(
biz_type,
self.data[test_data]['EmailTitle'] + str(suffix),
self.data[test_data]['ScheduleTimeCreate'],
self.data[test_data]['Template'],
description=self.data[test_data]['Description'],
email_date_format=self.data[test_data]['EmailDateFormat'],
mail_title_timezone=self.data[test_data]['EmailTitleTimezone'],
backdate=self.data[test_data]['BackDate'])
self.reportSchedule.click_confirm()
utils.log("success create report schedule, biz type is: {}".format(biz_type))
return biz_type
def action_on_report(self, action, biz_type):
self.reportSchedule.click_action_in_row(action, constants.BIZ_TYPE, biz_type)
def view_edit_report(self, biz_type, test_data):
"""
View Report, Add Report List, Email Recipients For excel, csv file type.
"""
self.action_on_report(constants.ReportView, biz_type)
self.reportSchedule.fill_view_report_config(
self.data[test_data]['ExcelName'] + biz_type.split('_')[2],
self.data[test_data]['SheetName'],
self.data[test_data]['SheetOrder'],
self.data[test_data]['AttachmentDateFormat'],
self.data[test_data]['FileType'],
self.data[test_data]['MailHtml'],
self.data[test_data]['ExcelReportSql'],
self.data[test_data]['EmailRecipients'],
self.data[test_data]['AttachBackDate'],
self.data[test_data]['EmailOrder'])
def publish_report(self, biz_type):
self.reportSchedule.click_report_biz_list()
self.action_on_report(constants.ReportPublish, biz_type)
self.reportSchedule.click_publish_in_confirm()
def add_publish_report(self, test_data):
"""
Add Report, View Report, Publish Report.
"""
utils.log("1. Add new schedule report.")
biz_type = self.add_schedule_report(test_data)
target_rows, target_row_id, action_column_index = self.reportSchedule.get_target_row(constants.BIZ_TYPE, biz_type)
self.assertTrue(len(target_rows) > 0, "create new report schedule, biz-type:[] failed!!!".format(biz_type))
utils.log("2. View schedule, add 'Report List', 'Email Recipients' for it.")
self.action_on_report(constants.ReportView, biz_type)
self.view_edit_report(biz_type, test_data)
utils.log("3. Publish report schedule.")
self.publish_report(biz_type)
publish_success = self.reportSchedule.is_action_in_row(constants.BIZ_TYPE, biz_type, constants.ReportPause)
self.assertTrue(publish_success, "publish report failed! Not find 'Pause' in row Action column.")
def add_publish_table_report(self, test_data):
"""
Add Report, View Report, Publish table Report.
"""
utils.log("1. Add new schedule report.")
biz_type = self.add_schedule_report(test_data)
target_rows, target_row_id, action_column_index = self.reportSchedule.get_target_row(constants.BIZ_TYPE, biz_type)
self.assertTrue(len(target_rows) > 0, "create new report schedule, biz-type:[] failed!!!".format(biz_type))
utils.log("2. View schedule, add 'Report List', 'Email HTML', 'Email Recipients' for it.")
self.view_edit_table_report(biz_type, test_data)
utils.log("3. Publish report schedule.")
self.publish_report(biz_type)
publish_success = self.reportSchedule.is_action_in_row(constants.BIZ_TYPE, biz_type, constants.ReportPause)
self.assertTrue(publish_success, "publish report failed! Not find 'Pause' in row Action column.")
def view_edit_table_report(self, biz_type, test_data):
"""
View Report, Add Report List, Email HTML, Email Recipients For table file type.
"""
self.action_on_report(constants.ReportView, biz_type)
self.reportSchedule.fill_view_report_config_table(
self.data[test_data]['Title'] + biz_type.split('_')[2],
self.data[test_data]['Type'],
self.data[test_data]['Order'],
self.data[test_data]['TableSQL'],
self.data[test_data]['EmailHtmlHeader'],
self.data[test_data]['EmailHtmlTail'],
self.data[test_data]['EmailRecipients'])