class connDB():
def __init__(self, host, port, username, passwd, dbname, charset, timeout):
self.host = host
self.port = port
self.username = username
self.passwd = passwd
self.dbname = dbname
self.charset = charset
self.timeout = timeout
self.log = Logger(common.logpath)
def connect(self):
try:
cdb = pymysql.connect(host=self.host,
user=self.username,
passwd=self.passwd,
db=self.dbname,
charset=self.charset,
read_timeout=self.timeout)
return cdb
except Exception as error:
self.log.error(error)
def getcursor(self, cdb):
if cdb is not None:
return cdb.cursor()
def closecursor(self, cur):
if cur is not None:
return cur.close()
def closeconn(self, cdb):
if cdb is not None:
return cdb.close()
def _result_insert(fee_resp, renewal_data_dt):
log = Logger()
insureCarId = renewal_data_dt.get('insureCarId', '')
CPlateNo = renewal_data_dt.get('plateNumber', '')
client = renewal_data_dt.get('client', '')
isPhone = renewal_data_dt['isPhone']
sessionId = renewal_data_dt.get('sessionId', '')
insuranceType = renewal_data_dt.get("insuranceType", {})
if isinstance(insuranceType, list):
insureTypeGroupId = insuranceType[0].get("insuranceTypeGroupId", "")
insuranceTypeGroup = insuranceType[0].get("insuranceTypeGroup", "")
else:
insureTypeGroupId = insuranceType.get("insuranceTypeGroupId", "")
insuranceTypeGroup = insuranceType.get("insuranceTypeGroup", "")
if isinstance(fee_resp, dict):
data = []
data.append(fee_resp['c01beginTime'])
data.append(fee_resp['c01endTime'])
data.append(fee_resp['vehicleSeats'])
data.append(insureTypeGroupId)
data.append(insureCarId)
data.append("1")
# log.info("平安开始入库 %s" % CPlateNo)
soupDb(fee_resp['fee'], data)
log.info("平安入库成功 %s ", CPlateNo)
send_mq(client, CPlateNo, "", "1", "1", sessionId, isPhone,
insureTypeGroupId, insuranceTypeGroup)
else:
if isinstance(fee_resp, list):
fee_resp = json.dumps(fee_resp, encoding=False)
log.error(fee_resp)
else:
log.error(fee_resp)
send_mq(client, CPlateNo, fee_resp, "2", "1", sessionId, isPhone,
insureTypeGroupId, insuranceTypeGroup)
def get_hn_userinfo(session, licenseNo="", vinNo=""):
log = Logger()
try:
rsp1 = is_hn_renewal(session, licenseNo, vinNo)
policyNo = ""
rsp2 = ""
syStart = syEnd = jqStart = jqEnd = ""
if rsp1 != 0:
JQ = jsonpath.jsonpath(rsp1, "$.[?(@.riskCode=='0507')]")
SY = jsonpath.jsonpath(rsp1, "$.[?(@.riskCode=='0506')]")
if JQ != False:
jq_policyNo = JQ[0]['policyNo']
jqStart = JQ[0]['startDate']
jqEnd = JQ[0]['endDate']
JQ_RSP = hn_renewal_userinfo(session, jq_policyNo)
if SY != False:
sy_policyNo = SY[0]['policyNo']
syStart = JQ[0]['startDate']
syEnd = JQ[0]['endDate']
SY_RSP = hn_renewal_userinfo(session, sy_policyNo)
if JQ_RSP != False:
rsp2 = JQ_RSP
else:
rsp2 = SY_RSP
insuranceType = hn_parse_renrwal(JQ_RSP, SY_RSP)
out = {
"licenseNo": licenseNo,
"vinNo": rsp2['car']['vinNo'],
"endDate": rsp2['base']['endDate'],
"CCardDetail": rsp2['car']['vehicleStyle'],
"brandName": rsp2['car']['modelName'],
"insuredName": rsp2['persons'][0]['insuredName'],
"identifyNumber": rsp2['persons'][0]['identifyNumber'],
"CUsageCde": "",
"NNewPurchaseValue": rsp2['car']['purchasePrice'],
"insuredAddress": rsp2['persons'][0]['insuredAddress'],
"mobile": rsp2['persons'][0]['mobile'],
"enrollDate": rsp2['car']['enrollDate'],
"engineNo": rsp2['car']['engineNo'],
"CModelCde": rsp2['car']['modelCode'],
"NSeatNum": "",
"COMPANY_ID": "13",
"insuranceType": insuranceType,
"insuranceTime": {
'syEnd': syEnd,
'syStart': syStart,
'jqStart': jqStart,
'jqEnd': jqEnd
}
}
return out
except Exception as e:
log.error(e)
return 0
def get_hn_premium(session, alldata=None, body_org=None):
# 获取body体
log = Logger()
try:
if body_org == None:
body_org = get_hn_body(alldata)
# print(json.dumps(body_org,ensure_ascii=False))
url = "http://qcar.chinahuanong.com.cn/quotepriceasync/preciseQuote.do"
body = "model=" + urllib.quote(json.dumps(body_org))
headers[
'Content-Type'] = "application/x-www-form-urlencoded; charset=UTF-8"
rsp = session.post(url=url, data=body, headers=headers)
rsp_text = rsp.text
if "终保日期" in rsp_text and "起保日期" in rsp_text:
log.error(u"华农重复投保 正在重复请求")
str1 = "\d{4}-\d{2}-\d{2}"
datelist = re.findall(str1, rsp_text, re.S)
if len(datelist) == 2:
bigdate = compare_date(datelist[0], datelist[1])
bigdate = compare_date(bigdate, getlatedate(0))
endDate = getlatedate(0, bigdate)
# alldata['carInfo']['endDate'] = endDate
alldata['carInfo']['insuranceTime']['jqStart'] = endDate
alldata['carInfo']['insuranceTime']['syStart'] = endDate
# rsp = get_hn_premium(session, alldata)
return get_hn_premium(session, alldata)
# rsp_text = rsp.text
if "重复投保" in rsp_text and "reinsureStr" in rsp_text:
errmsg = rsp.json()
msg = errmsg['base']['reinsureStr']
log.error(msg)
return msg
if "发生异常" in rsp_text:
log.error(rsp_text)
return rsp.text
if "agentComm" in rsp_text:
all = {}
rsp_json = rsp.json()
PriumeInf = hn_getPriumeInf(rsp_json)
all['fee'] = PriumeInf
all['c01beginTime'] = body_org['base']['startDate']
all['c01endTime'] = body_org['base']['endDate']
all['vehicleSeats'] = body_org['vehicleJingyou']['seat']
JQ = jsonpath.jsonpath(body_org, "$.kinds[?(@.kindCode=='BZ')]")
if JQ != False:
r.set_vin(body_org['car']['vinNo'], "13",
json.dumps(body_org, ensure_ascii=False))
return all
except Exception as e:
import traceback
log.error(e)
log.error(traceback.format_exc())
return "未知异常"
def set_cjbx_session(session):
log = Logger()
r = CRedis()
base64_session = codecs.encode(pickle.dumps(session), "base64").decode()
try:
# file_object = open('session.txt', 'w')
# file_object.write(base64_session)
r.set("99_COMPANY", base64_session)
except Exception as e:
log.error("session error")
log.error(traceback.format_exc())
def get_carInof(session, vinNo):
log = Logger()
rsp = _get_carInof(session, vinNo)
count = 1
while u"录入的校验码有误" in rsp and count < 5:
log.error(u"录入的校验码有误")
rsp = _get_carInof(session, vinNo)
count = count + 1
if "licenseNo" in rsp:
return json.loads(rsp)['carArr'][0]
else:
log.error(u"华农未获取到车管所信息 %s ", vinNo)
return 0
class configRD():
def __init__(self, path):
self.path = path
self.log = Logger(common.logpath)
try:
self.cRD = SafeConfigParser()
self.cRD.read(self.path)
except Exception as error:
self.log.error(error)
def get(self, section, key):
try:
return self.cRD.get(section, key)
except Exception as error:
self.log.error(error)
def getint(self, section, key):
try:
return self.cRD.getint(section, key)
except Exception as error:
self.log.error(error)
def getitems(self, field):
try:
self.items = self.cRD.items(field)
values = []
keys = []
for key, value in self.items:
values.append(value)
keys.append(key)
return values, keys
except Exception as error:
self.log.error(error)
def hn_renewal_userinfo(session, policyNo):
# 0507 JQ
# 0506 SY
log = Logger()
try:
url = "http://qcar.chinahuanong.com.cn/quotepriceasync/getPolicyInfo.do?policyNo=" + policyNo
rsp = session.get(url=url)
if 'licenseNo' in rsp.text:
return rsp.json()
else:
return False
except Exception as e:
log.error(e)
return False
class MyDB:
global host, username, password, port, database, config
host = localReadConfig.get_db("host")
username = localReadConfig.get_db("username")
password = localReadConfig.get_db("password")
port = localReadConfig.get_db("port")
database = localReadConfig.get_db("database")
config = {
'host': str(host),
'user': username,
'passwd': password,
'port': int(port),
'db': database
}
def __init__(self):
self.logger = Logger('dbUtil.py')
self.db = None
self.cursor = None
def connectDB(self):
try:
# connect to DB
self.db = pymysql.connect(**config)
# create cursor
self.cursor = self.db.cursor()
print("Connect DB successfully!")
except ConnectionError as ex:
self.logger.error(str(ex))
def executeSQL(self, sql):
self.connectDB()
# executing sql
self.cursor.execute(sql)
# executing by committing to DB
self.db.commit()
return self.cursor
def get_all(self, cursor):
value = cursor.fetchall()
return value
def get_one(self, cursor):
value = cursor.fetchone()
return value
def closeDB(self):
self.db.close()
print("Database closed!")
class ConfigHttp:
def __init__(self):
self.host = localReadConfig.get_http("baseurl")
self.port = localReadConfig.get_http("port")
self.timeout = localReadConfig.get_http("timeout")
self.logger = Logger("httpUtil.py").getlog()
# defined http get method
def get(self, url, param=None):
new_url = self.host + ":" + self.port + "/" + url
header = {'Content-Type': 'application/json'}
try:
if param != None:
textmod = parse.urlencode(param)
req = request.Request(url='%s%s%s' % (new_url, '?', textmod), headers=header)
self.logger.debug('%s%s%s' % (new_url, '?', textmod))
else:
req = request.Request(url='%s' % (new_url), headers=header)
self.logger.debug(new_url)
res = request.urlopen(req)
res = res.read()
hjson = json.loads(res)
self.logger.debug(json.dumps(hjson, indent=4))
return hjson
except TimeoutError:
self.logger.error("Time out!")
return None
# defined http post method
def post(self, url, param):
data = json.dumps(param)
data = bytes(data, "utf-8")
new_url = self.host + ":" + self.port + "/" + url
self.logger.info('post data: \n' + json.dumps(param, indent=4))
try:
headers = {'Content-Type': 'application/json'}
req = request.Request(new_url, data, headers)
response = request.urlopen(req)
res = response.read()
hjson = json.loads(res)
self.logger.debug(json.dumps(hjson, indent=4))
except Exception as err:
print(err, "\n")
return hjson
def is_hn_renewal(session, licenseNo="", vinNo=""):
log = Logger()
try:
url = "http://qcar.chinahuanong.com.cn/quotepriceasync/getOldPolicy.do"
headers[
'Content-Type'] = "application/x-www-form-urlencoded; charset=UTF-8"
data = {"licenseNo": licenseNo, "vinNo": vinNo, "engineNo": ""}
rsp = session.post(url=url, data=data, headers=headers)
if u"发生异常" in rsp.text:
return 0
else:
if len(rsp.json()) != 0:
log.info(u"是华农续保用户")
return rsp.json()
else:
return 0
except Exception as e:
log.error(e)
return 0
def get_cjbx_session():
log = Logger()
try:
r = CRedis()
session = ""
# file_object = open('session.txt')
sessionbase64 = r.get("99_COMPANY")
if sessionbase64 == None:
session = login_cjbx()
else:
session = pickle.loads(
codecs.decode(sessionbase64.encode(), "base64"))
iplist = socket.gethostbyname_ex(socket.gethostname())
if se.proxy not in json.dumps(iplist):
session.proxies = se.proxies
return session
except Exception as e:
log.error("session error")
log.error(traceback.format_exc())
def process(num_of_batches, unix_pid, logfile):
global logger
log_json_file = LOG_CFG_PATH + '/logging.json'
logger = Logger(name='firms_fpb',
config_path=log_json_file,
log_file=logfile)
batch_range = range(num_of_batches)
unix_pid = str(unix_pid)
batch_range = map(lambda x: '{}-{}'.format(x, unix_pid), batch_range)
p = Pool(8, init_worker)
try:
results = []
r = p.map_async(exec_process, batch_range, callback=results.append)
r.wait()
if 'ERROR' in results[0]:
print("Error occurred while processing FiRMS engine, exiting now")
p.terminate()
p.join()
exit(1)
p.close()
p.join()
except KeyboardInterrupt:
print("Process killed by user")
logger.error('Process was interrupted by the user, process exited')
p.terminate()
p.join()
exit(1)
except Exception as e:
logger.error('Error occurred while processing {}'.format(e))
p.terminate()
#raise
return 1
class Session:
def __init__(self):
self.config = Config.Config()
self.log = Logger().get_logger()
def get_session(self, env):
"""
获取session
:param env: 环境变量
:return:
"""
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko)\
Chrome/67.0.3396.99 Safari/537.36",
"Content-Type": "application/x-www-form-urlencoded",
"Connection": "close"
}
if env == "debug":
login_url = 'http://' + self.config.loginHost_debug
parm = self.config.loginInfo_debug
session_debug = requests.session()
response = session_debug.post(login_url, parm, headers=headers)
print(response.cookies)
self.log.debug('cookies: %s' % response.cookies.get_dict())
return response.cookies.get_dict()
elif env == "release":
login_url = 'http://' + self.config.loginHost_release
parm = self.config.loginInfo_release
session_release = requests.session()
response = session_release.post(login_url, parm, headers=headers)
print(response.cookies)
self.log.debug('cookies: %s' % response.cookies.get_dict())
return response.cookies.get_dict()
else:
print("get cookies error")
self.log.error('get cookies error, please checkout!!!')
def _Get_RandomNum(session_req):
log = Logger()
try:
millis = int(round(time.time() * 1000))
url = "http://32.0.192.232:88/sinoiais/pages/login/RandomNumUtil.jsp?d={0}".format(
str(millis))
code_resp = session_req.get(url=url, headers=headers)
# 验证码
codestr = None
if code_resp:
code_base64 = base64.b64encode(code_resp.content)
codestr = pic2Str(base64.b64decode(code_base64))
if not codestr:
log.info("获取验证码异常")
log.info("codestr=%s" % codestr)
return codestr
except Exception as e:
log.error(e)
log.error(traceback.format_exc())
return None
def email_init(report, report_name):
with open(report, 'rb')as f:
mail_body = f.read()
my_log = Logger("error_log.log", logging.WARNING, logging.DEBUG)
# 创建一个带附件的邮件实例
msg = MIMEMultipart('mixed')
# 以测试报告作为邮件正文
msg.attach(MIMEText(mail_body, 'html', 'utf-8'))
report_file = MIMEText(mail_body, 'html', 'utf-8')
# 定义附件名称（附件的名称可以随便定义，你写的是什么邮件里面显示的就是什么）
report_file["Content-Disposition"] = 'attachment; filename=' + report_name
msg.attach(report_file) # 添加附件
msg['Subject'] = '版权存证网站自动化测试报告:' + report_name # 邮件标题
msg['From'] = gl.email_name # 发件人
msg['To'] = ";".join(gl.email_To) # 收件人列表
try:
server = smtplib.SMTP()
server.connect(gl.smtp_sever, 25)
server.login(gl.email_name, gl.email_password)
server.sendmail(gl.email_name, gl.email_To, msg.as_string())
server.quit()
except smtplib.SMTPException:
my_log.error(u'邮件发送测试报告失败 at ' + __file__)
class xlsw():
def __init__(self, bookname):
try:
self.log = Logger(common.logpath)
self.bookname = bookname
self.creatXlsx()
self.createColname()
except Exception as error:
self.log.error(error)
def creatXlsx(self):
try:
self.book = xlsxwriter.Workbook(self.bookname)
self.sheet = self.book.add_worksheet()
self.sheet.set_column('A:H', 20)
except Exception as error:
self.log.error(error)
def createColname(self):
try:
self.sheet.write(0, 0, 'recipe_id/event_no/patient_id')
self.sheet.write(0, 1, 'message_id')
self.sheet.write(0, 2, 'branch_name')
self.sheet.write(0, 3, 'subject_id')
self.sheet.write(0, 4, 'subject_type')
self.sheet.write(0, 5, 'msg_content')
self.sheet.write(0, 6, 'advice')
self.sheet.write(0, 7, 'severity')
except Exception as error:
self.log.error(error)
def writeData(self, count, contents):
try:
for i in range(len(contents)):
self.sheet.write(count, i, str(contents[i]))
# print(count,i,str(contents[i]))
except Exception as error:
self.log.error(error)
def closeXlsx(self):
self.book.close()
def _checkLoginInfo(session_req, codestr):
log = Logger()
try:
url = "http://32.0.192.232:88/sinoiais/checklogin/checkLoginInfo.do"
headers['Content-Type'] = "application/x-www-form-urlencoded"
data_1 = {
"sysUserCode": "CJCXwm01",
"sysPassWord": "******",
"random": codestr
}
login_1_resp = session_req.post(url=url, data=data_1, headers=headers)
if "success" in login_1_resp.text:
log.info(u"登录成功")
return 1
elif "randomError" in login_1_resp.text:
log.error(u"验证码错误")
return 0
else:
return 2
except Exception as e:
log.error(u"登录失败")
log.error(traceback.format_exc())
return 2
dt['checkAnswer'] = dama("3", dt['jsCheckCode'])
log.info(dt['checkAnswer'])
#上报打码结果获取车管所返回信息
post_checkAnswer_res = post_checkAnswer(dt)
post_checkAnswer_res_items = jsonpath(post_checkAnswer_res,
"$.result.items")
count = 0
log.info(u"上报验证码完成")
log.info(dt['vinNo'])
#检验返回信息中是否有校验码错误信息，校验码错误会自动重发5次
while post_checkAnswer_res['err'] is not None:
log.info(post_checkAnswer_res['err'])
if count > 3:
log.error(u"打码失败")
send_mq(client, dt['licenseNo'], "打码失败", "2", "5",
sessionId, isPhone, groupId,
insuranceTypeGroup)
return "打码失败"
#
# if (post_checkAnswer_res['err'] is None) and (post_checkAnswer_res_items[0][0].get('replacementValue') is None):
# log.info(u"大地无法通过车架号获取车辆型号，正在调用中华联合接口重新尝试获取车辆型号")
#
# car_model = get_car_model(dt['vinNo'])
# gCIndustryModelName = car_model['gCIndustryModelName']
# dt['gCIndustryModelName'] = gCIndustryModelName
# post_checkAnswer_res = post_checkAnswer(dt)
# post_checkAnswer_res_items = jsonpath(post_checkAnswer_res, "$.result.items")
# count = count + 1
# break
class Request:
def __init__(self, host):
self.host = host
self.Log = Logger()
def request(self, url, data, type):
url = self.host + url
if type.upper() == "GET":
try:
self.Log.info("正在进GET：【%s】接口" % url)
req = requests.get(url, params=data)
result = req.json()
status_code = req.status_code
except Exception as e:
self.Log.error(e)
else:
if status_code == 200:
self.Log.info("请求完成【status:%s】" % status_code)
else:
self.Log.error("请求出错【status:%s】" % status_code)
return result
elif type.upper() == "POST":
try:
self.Log.info("正在进POST：【%s】接口" % url)
req = requests.post(url, data=data)
result = req.json()
status_code = req(url, data)
except Exception as e:
self.Log.error(e)
else:
if status_code == 200:
self.Log.info("请求完成【status:%s】" % status_code)
else:
self.Log.error("请求出错【status:%s】" % status_code)
return result
else:
self.Log.error("请求类型错误")
self.Log.close()
def requests(self, url, data, type):
url = self.host + url
if type.upper() == "GET":
try:
self.Log.info("正在进GET：【%s】接口" % url)
req = requests.get(url, params=data)
result = req.text
status_code = req.status_code
except Exception as e:
self.Log.error(e)
else:
if status_code == 200:
self.Log.info("请求完成【status:%s】" % status_code)
else:
self.Log.error("请求出错【status:%s】" % status_code)
return result
elif type.upper() == "POST":
try:
self.Log.info("正在进POST：【%s】接口" % url)
result = requests.post(url, data=data).json()
status_code = requests.post(url, data)
except Exception as e:
self.Log.error(e)
else:
if status_code == 200:
self.Log.info("请求完成【status:%s】" % status_code)
else:
self.Log.error("请求出错【status:%s】" % status_code)
return result
else:
self.Log.error("请求类型错误")
self.Log.close()
for i in cookie:
if i.get('name', '') == 'TWFID':
global TWFID
TWFID = i.get('value', '')
print TWFID
break
with open('cookie.txt', 'w') as f:
f.write(TWFID)
while True:
driver.refresh()
time.sleep(50)
except Exception, e:
print e
log.error("VPN登录异常，正在重新登录")
return False
def check_session():
#第二次登陆之后请求修改密码页面保持内部
from request_cjbx.login import get_cjbx_session, login_cjbx
headers = se.headers
session = get_cjbx_session()
url = "http://32.0.192.232:88/sinoiais/modifyPuTongPassword.do"
rsp = session.get(url=url, headers=headers)
if rsp.url == u'http://32.0.192.232:88/sinoiais/modifyPuTongPassword.do' and u"Session超时" not in rsp.text:
print(rsp.status_code)
print("session is alive")
else:
class Login(object):
def __init__(self, username=None, password=None):
self.log = Logger()
self.session_req = requests.session()
self.session_req.headers.update(se.headers)
if username is None or password is None:
self.username = se.login_username
self.password = se.login_password
# 刷新获取验证码
def __refresh_code(self, reqsss):
code_url = "https://icore-pts.pingan.com.cn/ebusiness/auto/rand-code-imgage.do" \
"?random=0.37327206414192915"
code_resp = reqsss.get(url=code_url, headers=se.headers, verify=False)
code_base64 = base64.b64encode(code_resp.content)
codestr = dama("2", code_base64)
self.log.info(u"平安登录验证码为%s", codestr)
return codestr
# 发送验证码
def __validate_code(self, reqsss, codestr):
set_code_url = "https://icore-pts.pingan.com.cn/ebusiness/auto/" \
"VerifyRandCodeController.do?randCode=" + codestr
set_code_resp = reqsss.get(url=set_code_url,
headers=se.headers,
verify=False)
set_code_result = set_code_resp.text
if set_code_result == "true":
self.log.info(u"平安验证码校验正确")
return 1
self.log.error(u"平安登录验证码错误")
return 0
def __use_login(self, reqsss):
# 登录前置请求
login_monitor_url = "https://icore-pts.pingan.com.cn/ebusiness/auto/monitor.do?" \
"type=insertLoginLog&flowid=&userId=STDJNDS-00001&_t=1487214110283"
reqsss.post(url=login_monitor_url, headers=se.headers, verify=False)
def __login(self, reqsss, codestr):
login_url = 'https://icore-pts.pingan.com.cn/ebusiness/j_security_check'
se.headers['Content-Type'] = "application/x-www-form-urlencoded"
headers = se.headers
login_data = "j_username="******"&j_password="******"&SMAUTHREASON=0&randCode=" + codestr
req = reqsss.post(url=login_url,
data=login_data,
headers=headers,
verify=False)
if req.url == u'https://icore-pts.pingan.com.cn/ebusiness/':
self.log.info(u"平安登录成功")
# 往redis中存储session
set_session(self.session_req, "1")
return self.session_req
self.log.error(u"平安登录失败")
self.log.error(req.text)
return 0
def login(self):
try:
# 刷新验证码
codestr = self.__refresh_code(self.session_req)
# 校验验证码
count = 1
validate_code_result = self.__validate_code(
self.session_req, codestr)
while validate_code_result == 0 and count < 4:
codestr = self.__refresh_code(self.session_req)
validate_code_result = self.__validate_code(
self.session_req, codestr)
count += 1
# 发送登录前置请求
self.__use_login(self.session_req)
out = self.__login(self.session_req, codestr)
# self.auto_index_do(out)
if out != 0:
#self.auto_index_do(out)
return out
except Exception as error:
self.log.info(error)
# 获取保持连接的id
def auto_index_do(self, session):
'''
:param session:登录后的session
:return:flow_id
'''
index_url = "https://icore-pts.pingan.com.cn/ebusiness/auto/newness/index.do"
index_rsp = session.get(url=index_url, verify=False)
index_rsp_text = index_rsp.text
flow_id = re.findall(r"<div id=\"main\" flowid=\"(.+?)\">",
index_rsp_text, re.S)[0]
r = CRedis()
r.set("pingan_keepid", flow_id)
'''
def OnlyInterface(**data):
url = data['url']
type = data['type']
header = data['header']
testdata = data['参数']
check = data['校验']
if isinstance(testdata, dict):
pass
else:
try:
testdata = eval(testdata)
except Exception as e:
Logger().error(e)
testdata = {}
up_data = data['继承参数']
if up_data == '':
up_data = {}
else:
try:
up_data = eval(up_data)
except Exception as e:
Logger().error(e)
up_data = {}
# raise e
parameter = data['调用参数']
downdata = data['下传参数']
if header == '':
header = {}
else:
try:
header = eval(header)
except Exception as e:
Logger().error(e)
header = {}
if parameter == '':
testdata = testdata
else:
try:
parameter = eval(parameter)
except Exception as e:
Logger().error(e)
parameter = ''
else:
for each in parameter:
testdata[each] = up_data[each]
respone = Request().requests(url, testdata, type, headers=header)
respone = json.loads(respone)
if check != '':
try:
check = eval(check)
except Exception as e:
Logger().error(e)
check = {}
for each in check.keys():
try:
assert respone[each] == check[each], "接口【%s】响应校验失败" % url
except Exception as e:
Logger().error(e)
else:
if downdata != '':
try:
downdata = eval(downdata)
except Exception as e:
downdata = {}
Logger.error(e)
else:
for key in downdata.keys():
down_value = downdata[key].split("=>")
s = respone
for i in range(0, len(down_value)):
s = s[down_value[i]]
i += 1
downdata[key] = s
up_data.update(downdata)
result = [respone, up_data]
return result
new_dir_json = output_dir + '/data/' + username + '/' + yy + '/' + mm + '/' + dd + '/' + user_dir + '/'
new_dir_txt = output_dir + '/out/' + username + '/' + yy + '/' + mm + '/' + dd + '/' + user_dir + '/'
try:
logger.info('Creating data directory at {}'.format(new_dir_json))
if not os.path.exists(new_dir_json):
os.makedirs(new_dir_json)
logger.info('Creating out directory at {}'.format(new_dir_txt))
if not os.path.exists(new_dir_txt):
os.makedirs(new_dir_txt)
except:
print("Failed to create output directory, exiting..")
logger.error("Failed to create output directory, exiting {}".format(e))
exit(1)
if os.access(new_dir_json, os.W_OK) is not True:
logger.error(
'Output Directory {} is not writable'.format(new_dir_json))
print("Output Directory is not writable", new_dir_json)
exit(1)
if os.access(new_dir_txt, os.W_OK) is not True:
logger.error('Output Directory {} is not writable'.format(new_dir_txt))
print("Output Directory is not writable", new_dir_txt)
exit(1)
(path, input_file_name) = os.path.split(input_file)
driver.execute_script(
'document.getElementById("uname").value = "%s";' % name)
driver.execute_script('document.getElementById("pwd").value = "%s";' %
pwd)
driver.find_element_by_xpath('//*[@id="loginbtn"]').click()
time.sleep(15)
result = checkVPN()
if result == 1:
log.error("vpn连接异常，正在重新连接")
killTask(["IEDriverServer.exe", "iexplore.exe"])
login_vpn()
else:
log.error("vpn连接正常")
return True
except Exception, e:
log.error("VPN登录失败，正在重新登录")
return False
def check_session():
srssion = get_session('2')
if srssion is not None:
sessBase = srssion
req_session = pickle.loads(codecs.decode(sessBase.encode(), "base64"))
req_session.mount('https://', MyAdapter())
try:
if_renewal_url = "http://10.134.136.112:8000/prpall/business/selectRenewalPolicyNo.do"
req_session.post(if_renewal_url, verify=False).json()
return True
except Exception, e:
log.error(e)
for arg in xrange(10):
th = threading.Thread(target=runrealtime)
th.start()
threads.append(th)
try:
for th in threads:
th.join()
except Exception,e:
log.error(e)
log.error(traceback.format_exc())
log.info(u"程序结束运行%s" % datetime.now())
'''
if __name__ == "__main__":
from multiprocessing import Process
# 方法运行
process = []
log.info(u"程序开始运行%s" % datetime.now())
for arg in xrange(1):
th = Process(target=main)
th.start()
process.append(th)
try:
for th in process:
th.join()
except Exception, e:
log.error(e)
log.error(traceback.format_exc())
log.info(u"程序结束运行%s" % datetime.now())
def is_pingan_renwal(session, plateNumber, repet=0):
log = Logger()
try:
if '-' not in plateNumber:
plateNumber = plateNumber[:2] + "-" + plateNumber[2:]
if session == '' or session is None:
login = Login()
session = login.login()
try:
repet = 0
user_data = is_renewal(session, plateNumber)
except Exception as e:
if repet == 0:
user_data = is_renewal(session, plateNumber)
repet += 1
# 续保用户
if isinstance(user_data, dict):
icorepnbs_session = user_data['request']
quick_resp = quickSearchVoucher(icorepnbs_session, user_data)
# 如果商业险的结束时间为空那么取交强险的结束时间作为保险时间
str_date = quick_resp['vehicleTarget']['firstRegisterDate']
insuranceType = quick_resp['insuranceType']
endDate = user_data.get("sy_insuranceEndTime", "").split(" ")[0]
if endDate == "":
endDate = user_data.get("jq_insuranceEndTime",
"").split(" ")[0]
out = {
"licenseNo":
user_data['vehicleLicenceCode'].replace('-', ''),
"vinNo":
user_data['vehicleFrameNo'],
"endDate":
endDate,
"CCardDetail":
quick_resp['extendInfo'].get("ownerVehicleTypeCode", ""),
"brandName":
quick_resp['vehicleTarget']['modifyAutoModelName'],
"insuredName":
user_data['ownerName'],
"identifyNumber":
quick_resp.get("insurantInfo", {}).get("certificateTypeNo",
""),
"CUsageCde":
"",
"NNewPurchaseValue":
"",
"insuredAddress":
"",
"mobile":
"",
"enrollDate":
str_date.split(" ")
[0], # str_date[:4]+"-"+str_date[4:6]+"-"+str_date[6:],
"engineNo":
user_data['engineNo'],
"CModelCde":
user_data['makerModel'],
"NSeatNum":
quick_resp['vehicleTarget']['vehicleSeats'],
"COMPANY_ID":
"1",
"insuranceType":
insuranceType,
"insuranceTime": {
'syEnd': user_data.get("sy_insuranceEndTime", ""),
'syStart': user_data.get("sy_insuranceBeginTime", ""),
'jqStart': user_data.get("jq_insuranceBeginTime", ""),
'jqEnd': user_data.get("jq_insuranceEndTime", "")
}
}
return out
else:
log.info(u"未查询到续信息%s", plateNumber)
return 0
except Exception, e:
log.error(e)
log.error(traceback.format_exc())
return 0
def pingan_request(session, renewal_data_dt):
'''
:param session:
:param dt:
:return:
'''
log = Logger()
try:
alldata = {}
# insureCarId = renewal_data_dt.get('insureCarId', '')
CPlateNo = renewal_data_dt.get('plateNumber', '')
searchVin = renewal_data_dt.get('vinNo', '')
client = renewal_data_dt.get('client', '')
isPhone = renewal_data_dt['isPhone']
sessionId = renewal_data_dt.get('sessionId', '')
#endDate = renewal_data_dt.get('endDate', '')
insuranceType = renewal_data_dt.get("insuranceType", {})
if isinstance(insuranceType, list):
insureTypeGroupId = insuranceType[0].get("insuranceTypeGroupId",
"")
insuranceTypeGroup = insuranceType[0].get("insuranceTypeGroup", "")
else:
insureTypeGroupId = insuranceType.get("insuranceTypeGroupId", "")
insuranceTypeGroup = insuranceType.get("insuranceTypeGroup", "")
'''
if endDate != "":
min_time = compare_time40(endDate)
if min_time > config.days:
log.error("平安保险查询在40天内的 保险结束时间为 %s", endDate)
send_mq(client, CPlateNo, "平安保险查询在40天内的 保险结束时间为 %s" % endDate, "2", "1", sessionId, isPhone,
insureTypeGroupId, insuranceTypeGroup)
return "平安保险查询在40天内的 保险结束时间为 %s" % endDate
if min_time < 0:
endDate = getlatedate(1)
'''
if not isinstance(insuranceType, list):
# 使用redis缓存进行查询
re_out = get_pingan_redis_body(session, renewal_data_dt)
if re_out == 1:
return
if '-' not in CPlateNo:
plateNumber = CPlateNo[:2] + "-" + CPlateNo[2:]
if session == '' or session is None:
login = Login()
session = login.login()
# 判断是否为续保用户 直接从续保信息中取用户信息 然后交换域名跳转的session
user_data = is_renewal(session, plateNumber=plateNumber)
if isinstance(user_data, dict):
sy_insuranceEndTime = user_data.get("sy_insuranceEndTime", "")
jq_insuranceEndTime = user_data.get("jq_insuranceEndTime", "")
if sy_insuranceEndTime == "" and jq_insuranceEndTime != "":
sy_insuranceEndTime = jq_insuranceEndTime
if sy_insuranceEndTime != "" and jq_insuranceEndTime == "":
jq_insuranceEndTime = sy_insuranceEndTime
icorepnbs_session = user_data['request']
quick_resp = quickSearchVoucher(icorepnbs_session, user_data)
# 获取车管所信息
session = quick_resp['request']
vehicleFrameNo = quick_resp['vehicleTarget']['vehicleFrameNo']
queryDMVehicleInfoConfirm_json = queryDMVehicleInfoConfirm(
session, vehicleFrameNo, carMark="")
if not isinstance(queryDMVehicleInfoConfirm_json, dict):
send_mq(client, CPlateNo, queryDMVehicleInfoConfirm_json, "2",
"1", sessionId, isPhone, insureTypeGroupId,
insuranceTypeGroup)
return
# 查询车辆具体型号
model = query_carmodel(session, vehicleFrameNo)
if model == 0:
model = queryAutoModelType(
session, queryDMVehicleInfoConfirm_json['vehicleModel'])
# 型号最后一位小写
if model == 0:
model = queryAutoModelType(
session,
queryDMVehicleInfoConfirm_json['vehicleModel'][0:-1] +
queryDMVehicleInfoConfirm_json['vehicleModel'][-1].lower())
log.info(model)
alldata['sy_insuranceEndTime'] = sy_insuranceEndTime
alldata['jq_insuranceEndTime'] = jq_insuranceEndTime
alldata['insurantInfo'] = quick_resp['insurantInfo']
alldata['vehicleTarget'] = quick_resp['vehicleTarget']
alldata['autoModelType'] = quick_resp['autoModelType']
alldata['DMVehicleInfo'] = queryDMVehicleInfoConfirm_json
alldata['model'] = model
alldata["insuranceType"] = insuranceType
if isinstance(insuranceType, list):
for itype in insuranceType:
alldata['insuranceType'] = itype
fee_resp = get_fee(session, alldata)
_result_insert(fee_resp, renewal_data_dt)
else:
fee_resp = get_fee(session, alldata)
_result_insert(fee_resp, renewal_data_dt)
else:
if searchVin != "":
sy_insuranceEndTime = renewal_data_dt.get("insuranceTime",
{}).get(
"syStart",
getlatedate(1))
jq_insuranceEndTime = renewal_data_dt.get("insuranceTime",
{}).get(
"jqStart",
getlatedate(1))
session = user_data
vehicleFrameNo = searchVin
# 使用redis缓存进行查询
re_out = get_pingan_redis_body(session, renewal_data_dt)
if re_out == 1:
return
# 请求车管所
queryDMVehicleInfoConfirm_json = queryDMVehicleInfoConfirm(
session, vehicleFrameNo, carMark="")
# log.info(json.dumps(queryDMVehicleInfoConfirm_json,ensure_ascii=False))
if not isinstance(queryDMVehicleInfoConfirm_json, dict):
send_mq(client, CPlateNo, queryDMVehicleInfoConfirm_json,
"2", "1", sessionId, isPhone, insureTypeGroupId,
insuranceTypeGroup)
return
# 通过车架号查询车辆类型
model = query_carmodel(session, vehicleFrameNo)
# 通过车辆编码查询车辆类型
if model == 0:
model = queryAutoModelType(
session,
queryDMVehicleInfoConfirm_json['vehicleModel'])
# 型号最后一位小写
if model == 0:
log.info(u"通过最后一位字母小写查询")
model = queryAutoModelType(
session,
queryDMVehicleInfoConfirm_json['vehicleModel'][0:-1] +
queryDMVehicleInfoConfirm_json['vehicleModel']
[-1].lower())
# log.info(model)
# 编码查询不到使用续保信息中的车辆名字进行查询
if model == 0 and renewal_data_dt.get("vehicleBrand",
"") != "":
model = queryAutoModelType(session,
renewal_data_dt['vehicleBrand'])
alldata['DMVehicleInfo'] = queryDMVehicleInfoConfirm_json
alldata['model'] = model
alldata['sy_insuranceEndTime'] = sy_insuranceEndTime
alldata['jq_insuranceEndTime'] = jq_insuranceEndTime
alldata["insuranceType"] = insuranceType
if isinstance(insuranceType, list):
for itype in insuranceType:
alldata['insuranceType'] = itype
fee_resp = get_fee(session, alldata)
_result_insert(fee_resp, renewal_data_dt)
else:
fee_resp = get_fee(session, alldata)
_result_insert(fee_resp, renewal_data_dt)
except Exception, e:
log.error(e)
log.error(traceback.format_exc())
send_mq(client, CPlateNo, "%s 未知错误" % CPlateNo, "2", "1", sessionId,
isPhone, insureTypeGroupId, insuranceTypeGroup)
class getmessage():
def __init__(self, conrd):
self.conrd = conrd
self.log = Logger(logpath)
self.havemids = [] # 存储已经核对的messageid
#标准数据库实例化
self.conn_standard = connDB(
self.conrd.get('sMySQL', 'host'),
self.conrd.getint('sMySQL', 'port'),
self.conrd.get('sMySQL', 'user'),
self.conrd.get('sMySQL', 'passwd'),
self.conrd.get('sMySQL', 'dbname'),
self.conrd.get('sMySQL', 'charset'),
self.conrd.getint('sMySQL', 'read_timeout'))
self.conns = self.conn_standard.connect()
self.curs = self.conn_standard.getcursor(self.conns)
#实际结果数据库实例化
self.conn_reality = connDB(self.conrd.get('rMySQL', 'host'),
self.conrd.getint('rMySQL', 'port'),
self.conrd.get('rMySQL', 'user'),
self.conrd.get('rMySQL', 'passwd'),
self.conrd.get('rMySQL', 'dbname'),
self.conrd.get('rMySQL', 'charset'),
self.conrd.getint('rMySQL', 'read_timeout'))
self.connr = self.conn_reality.connect()
self.curr = self.conn_reality.getcursor(self.connr)
def result(self, sqlRlt):
result = {}
for value in sqlRlt:
result[value[0]] = value[1].split(',')
return result
#获取标准数据库中的警示信息内容
def getMessage_standard(self, sql, dbname, starttime, endtime):
try:
self.curs.execute(sql.format(dbname, starttime, endtime))
rlt = self.curs.fetchall()
return self.result(rlt)
except Exception as error:
self.log.error(error)
# 获取需验证引擎的数据库中的警示信息内容
def getMessage_reality(self, sql, dbname, starttime, endtime):
try:
self.curr.execute(sql.format(dbname, starttime, endtime))
rlt = self.curr.fetchall()
return self.result(rlt)
except Exception as error:
self.log.error(error)
#比较两个数据库中的点评数据
def compareMessage(self, message_s, message_r):
samemessage = {} #存储警示信息相同的数据
standardHave = {} #存储同一个处方或患者或就诊流水号，标准库里面有跑出警示信息，实际对照的库里面没有跑出警示信息
realityHave = {} #存储同一个处方或患者或就诊流水号，标准库里面没有跑出警示信息，实际对照的库里面有跑出警示信息
for key, message in message_s.items():
if message_r:
if key in message_r:
if message_r[key] == message_s[key]:
samemessage[key] = message
elif message_r[key] != message_s[key]:
standardHave[key] = set(message_s[key]) - set(
message_r[key])
realityHave[key] = set(message_r[key]) - set(
message_s[key])
else:
self.log.info('message_r is null')
break
return samemessage, dict(standardHave), dict(realityHave)
#获取门诊的key_date
def getoptKeyDate_s(self, dbname):
self.sql = self.conrd.get('keydate', 'optalert')
try:
self.curs.execute(self.sql.format(dbname))
self.date = self.curs.fetchall()
self.starttime, self.endtime = self.date[4], self.date[11]
return self.starttime[0], self.endtime[0]
except Exception as error:
self.log.error(error)
# 获取住院的key_date
def getiptKeyDate_s(self, dbname):
self.sql = self.conrd.get('keydate', 'iptalert')
try:
self.curs.execute(self.sql.format(dbname))
self.date = self.curs.fetchall()
self.starttime, self.endtime = self.date[4], self.date[11]
return self.starttime[0], self.endtime[0]
except Exception as error:
self.log.error(error)
# 获取未跑出的警示信息的相关内容便于排查问题
def messageContent_s(self, db_knowledge, db_dp, messageid, id, sql):
self.sql = sql
# print(self.sql.format(db_knowledge,db_dp,messageid,id))
self.curs.execute(self.sql.format(db_knowledge, db_dp, messageid, id))
sqlrlt = self.curs.fetchall()
if sqlrlt:
rlt = sqlrlt[0]
return rlt
# 获取未跑出的警示信息的相关内容便于排查问题
def messageContent_r(self, db_knowledge, db_dp, messageid, id, sql):
self.sql = sql
# print(self.sql.format(db_knowledge,db_dp,messageid,id))
self.curr.execute(self.sql.format(db_knowledge, db_dp, messageid, id))
sqlrlt = self.curr.fetchall()
if sqlrlt:
rlt = sqlrlt[0]
return rlt
# 标准库里有跑出来，实际库里未跑出来的规则统计出来
def messageContents_s(self, standardHave, db_knowledge, db_dp, sql, name):
count = 1
self.xlsx = xlsw('./result/{}_{}_s.xlsx'.format(curdate, name))
for id, messageids in standardHave.items():
if set(messageids) & set(self.havemids) == set(
messageids): #如果messageid已经在其它处方或医嘱中验证过便跳出此次循环
# print(messageids)
continue
for messageid in messageids:
rlt = self.messageContent_s(db_knowledge, db_dp, messageid, id,
sql)
if rlt:
if messageid not in self.havemids:
self.xlsx.writeData(count, rlt)
# print(rlt)
self.havemids.append(messageid)
count += 1
# if count==30:break
# if count==30:break
self.xlsx.closeXlsx()
# 实际库里有跑出来，标准库里未跑出来的规则统计出来
def messageContents_r(self, realityHave, db_knowledge, db_dp, sql, name):
print("输出实际数据库跑出来的结果")
count = 1
havemids = [] #存储已经核对的messageid
self.xlsx_r = xlsw('./result/{}_{}_r.xlsx'.format(curdate, name))
for id, messageids in realityHave.items():
if set(messageids) & set(havemids) == set(
messageids): #如果messageid已经在其它处方或医嘱中验证过便跳出此次循环
# print(messageids)
continue
for messageid in messageids:
rlt = self.messageContent_r(db_knowledge, db_dp, messageid, id,
sql)
if rlt:
if messageid not in havemids:
self.xlsx_r.writeData(count, rlt)
havemids.append(messageid)
count += 1
# if count==30:break
# if count==30:break
self.xlsx_r.closeXlsx()
#获取知识建设库与点评库的messageid,公共的方法,代替以上备注的四个方法
def messages_data(self, dbname, sqlname, field):
self.sql = self.conrd.get(field, sqlname)
self.curs.execute(self.sql.format(dbname))
messages = self.curs.fetchall()
return ([str(i[0]).strip() for i in messages])
#获取点评库中跑出警示信息的recipeid/eventno
def getdpID(self, dbname, sqlname, messageid, field):
self.sql = self.conrd.get(field, sqlname)
# print(self.sql.format(dbname,messageid))
self.curs.execute(self.sql.format(dbname, messageid))
id = self.curs.fetchall()
if id:
return id[0][0]
else:
return None
#实际数据库是否有跑出警示信息
def getdbr_messageid(self, dbnamer_dp, dbnamer_k, dpid, longid, sqlname,
field):
self.sql = self.conrd.get(field, sqlname)
# print(self.sql)
# print(self.sql.format(dbnamer_dp,dpid,dbnamer_k,longid))
self.curr.execute(self.sql.format(dbnamer_dp, dpid, dbnamer_k, longid))
id = self.curr.fetchall()
if id:
return str(id[0][0])
else:
return None
def checkmessages(self, **kwargs):
#标准知识库中所有规则的messageid
self.knowledge_messages = self.messages_data(kwargs['dbnames_k'],
kwargs['sqlnameAll'],
kwargs['field1'])
#标准点评库中门诊处方跑出来的所有messageid
self.opt_messages = self.messages_data(kwargs['dbnames_dp'],
kwargs['sqlnameOpt'],
kwargs['field1'])
#标准点评库中住院跑出来的所有messageid
self.ipt_messages = self.messages_data(kwargs['dbnames_dp'],
kwargs['sqlnameIpt'],
kwargs['field1'])
# 标准点评库中门诊患者跑出来的所有messageid
self.optp_messages = self.messages_data(kwargs['dbnames_dp'],
kwargs['sqlnameOpt_p'],
kwargs['field1'])
print(self.knowledge_messages)
passmessages = []
failmessages = []
count_i, count_optr, count_optp = 1, 1, 1
# self.xlsx = xlsw('./result/{}2.xlsx'.format(curdate))
self.xlsx_ipt = xlsw('./result/{}_ipt.xlsx'.format(curdate))
self.xlsx_optr = xlsw('./result/{}_optr.xlsx'.format(curdate))
self.xlsx_optp = xlsw('./result/{}_optp.xlsx'.format(curdate))
for messageid in self.knowledge_messages:
if messageid in self.ipt_messages:
self.id = self.getdpID(kwargs['dbnames_dp'], kwargs['eventno'],
messageid, kwargs['field2'])
self.rlt = self.getdbr_messageid(kwargs['dbnamer_dp'],
kwargs['dbnamer_k'], self.id,
messageid, kwargs['iptrlt'],
kwargs['field3'])
if self.rlt and self.rlt == messageid:
passmessages.append(messageid)
else:
self.sql = self.conrd.get("enginecontent", 'iptalert')
# print(self.id,self.sql)
result = self.messageContent(kwargs['dbnames_k'],
kwargs['dbnames_dp'],
messageid, self.id, self.sql)
# print(result)
self.xlsx_ipt.writeData(count_i, result)
count_i += 1
# if count==30:break
elif messageid in self.opt_messages:
self.id = self.getdpID(kwargs['dbnames_dp'],
kwargs['recipeid'], messageid,
kwargs['field2'])
self.rlt = self.getdbr_messageid(kwargs['dbnamer_dp'],
kwargs['dbnamer_k'], self.id,
messageid, kwargs['optrlt'],
kwargs['field3'])
if self.rlt and self.rlt == messageid:
passmessages.append(messageid)
else:
self.sql = self.conrd.get("enginecontent", 'optalert')
result = self.messageContent(kwargs['dbnames_k'],
kwargs['dbnames_dp'],
messageid, self.id, self.sql)
self.xlsx_optr.writeData(count_optr, result)
count_optr += 1
# if count==23:break
elif messageid in self.optp_messages:
self.id = self.getdpID(kwargs['dbnames_dp'],
kwargs['patinetid'], messageid,
kwargs['field2'])
self.rlt = self.getdbr_messageid(kwargs['dbnamer_dp'],
kwargs['dbnamer_k'], self.id,
messageid, kwargs['optrlt_p'],
kwargs['field3'])
if self.rlt and self.rlt == messageid:
passmessages.append(messageid)
else:
self.sql = self.conrd.get("enginecontent", 'optalert_p')
result = self.messageContent(kwargs['dbnames_k'],
kwargs['dbnames_dp'],
messageid, self.id, self.sql)
self.xlsx_optp.writeData(count_optp, result)
count_optp += 1
# if count == 10: break
else:
self.rlt = None
failmessages.append(messageid)
print(len(passmessages), passmessages)
print(len(failmessages))
self.xlsx_ipt.closeXlsx()
self.xlsx_optr.closeXlsx()
self.xlsx_optp.closeXlsx()
def validate_json(input_file, logfile):
log_json_file = LOG_CFG_PATH + '/logging.json'
global logger
logger = Logger(name='firms_fpb',
config_path=log_json_file,
log_file=logfile)
with open(input_file) as json_data:
jsonData = json.load(json_data)
# print the keys and values
count = 0
all_errors = OrderedDict()
final_errors = ""
for rec in jsonData:
for key, value in rec.iteritems():
line_number = str(rec['input_row_id'])
# Check 1 - Value should not be empty
if key not in 'protocol' and not value:
retVal = " Error : Record number :" + line_number + " Column " + key + " is empty \n"
final_errors += retVal
continue
# Check 2 - Only ASCII characters allowed
try:
temp_val = str(value)
temp_val.encode('ascii')
except:
retVal = "Error : Record number : " + line_number + " Column " + str(
key) + " has non ASCII character \n"
final_errors += retVal
continue
# Check 3 - Valid values
if key == 'source':
retVal = is_valid_format(key, str(value))
elif key == 'dest-port':
retVal = is_valid_port_format(str(value))
elif key == 'destination':
retVal = is_valid_format(key, str(value))
elif key == 'protocol':
retVal = is_valid_protocol(str(value))
elif key == 'input_row_id':
retVal = "Check passed"
if 'Error' in retVal:
final_errors += "Record number: " + line_number + " " + str(
retVal) + "\n"
count += 1
logger.info('Total records {}'.format(count))
print('Total records : ', count)
# IF there are validation errors
if final_errors:
pattern = re.compile('(.*?).json')
match = pattern.match(input_file)
filename = match.group(1)
new_filename = filename.replace("data", "out")
curr_time = time.time()
output_filename = new_filename + '__val__errors__' + str(
curr_time) + '.' + 'txt'
# Write to file
try:
with open(output_filename, 'w') as f:
f.write(final_errors)
print("Input file parsing errors generated at : ", output_filename)
logger.info('Excel validation errors generated at : {}'.format(
output_filename))
print("\nExiting FiRMS Gen, failed to validate input file \n")
exit(1)
except Exception as e:
print("Failed to create validation errors file", e)
logger.error(
'Failed to create validation errors file {}'.format(e))
exit(1)
return (count)
