def query_doc(self,
index_name=None,
type_name=None,
cond_value=None,
params=None,
p_conn=None):
if index_name is None or type_name is None or cond_value is None or (
cond_value is not None and type(cond_value) != dict):
log.log(log.set_loglevel().error(), index_name, type_name,
cond_value, p_conn)
raise err.MyError(log.debug().position(), err.MyError.errcode[4],
err.MyError.errmsg[4])
if p_conn is None:
try:
esdb = dbconn.db_es()
conn = esdb.create_conn()
except Exception as e:
log.log(
log.set_loglevel().error(),
log.debug().position(),
e,
)
raise err.MyError(log.debug().position(),
err.MyError.errcode[1],
err.MyError.errmsg[1])
else:
conn = p_conn
cond_value['version'] = True
ret_val = conn.search(index_name, type_name, cond_value)
if p_conn is None:
if 'esdb' in locals().keys():
esdb.close_conn()
del esdb
return ret_val
def add_memo(self,
ordercode='',
memo='',
operator='',
remark=None,
p_conn=None):
"""
暂时不生效
:param ordercode:
:type ordercode:
:param memo:
:type memo:
:param operator:
:type operator:
:param remark:
:type remark:
:param p_conn:
:type p_conn:
:return:
:rtype:
"""
if ordercode == '' or ordercode is None or memo == '' or memo is None or operator == '' or operator is None:
log.log(log.set_loglevel().error(),
log.debug().position(), err.MyError.errcode[4],
err.MyError.errmsg[4])
raise err.MyError(log.set_loglevel().error(),
log.debug().position(), err.MyError.errcode[4],
err.MyError.errmsg[4])
if p_conn is None:
vipdb = dbconn.vipdb()
conn = vipdb.create_conn()
else:
conn = p_conn
table_name = '%s.t_insurance_memo' % config.env['vipschema']
column_name = ('ordercode', 'memo', 'operator', 'remark', 'cdt', 'udt')
str_curr_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
column_value = (ordercode, memo, operator, remark, str_curr_time,
str_curr_time)
memo_id = self.new_table_record(table_name, column_name, column_value,
conn)
if p_conn is None:
conn.rollback()
del conn
vipdb.close_conn()
return memo_id
def new_table_record(self,
table_name=None,
column_name=None,
column_value=None,
p_conn=None):
if column_name is None or column_value is None or table_name is None:
log.log(log.set_loglevel().error(), table_name, column_name,
column_value, p_conn)
raise err.MyError(log.debug().position(), err.MyError.errcode[4],
err.MyError.errmsg[4])
if p_conn is None:
try:
# conn = mysql.connector.connect(**config.env['db_connection'])
db = dbconn.db()
conn = db.create_conn()
except mysql.connector.Error as e:
log.log(
log.set_loglevel().error(),
log.debug().position(),
e,
)
log.debugfile().printlog2(
log.debug().position(),
e,
)
raise err.MyError(log.debug().position(),
err.MyError.errcode[1],
err.MyError.errmsg[1])
else:
conn = p_conn
insert_sql, column_value = component.c_db_dml.mysql().new_record_sql(
table_name, column_name, column_value)
cursor = conn.cursor()
try:
if config.env['env'] == 'dev':
log.log(log.set_loglevel().debug(),
log.debug().position(), insert_sql, column_value)
cursor.execute(insert_sql, column_value)
new_id = cursor._last_insert_id
except Exception as e:
log.log(log.set_loglevel().error(),
log.debug().position(), e, insert_sql, column_value)
raise err.MyError(
log.debug().position(), err.MyError.errcode[1],
err.MyError.errmsg[1] + insert_sql + ' ' + column_value)
finally:
if cursor:
cursor.close()
if p_conn is None:
if conn:
conn.commit()
conn.close()
db.close_conn()
if 'new_id' in locals():
return new_id
def httpserver(self):
"""
:param ordercode:
:type ordercode:
:return:
:rtype:
"""
# 不再需要消息解析，纯保险公司查询
url = "%s/py/pressure" % (1) #(config.env['domain_address'])
header_dict = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko'}
req = urllib.request.Request(url=url, headers=header_dict)
try:
f = urllib.request.urlopen(req, timeout=15)
return str(f.read().decode('utf-8'))
except Exception as e:
if str(e).__contains__('500'):
pass
else:
log.log(log.set_loglevel().error(), log.debug().position(), req.data, req.full_url, req.headers, e)
return '{"code":"-1","message":"%s","info":%s}' % (e, log.debug().position())
def delete_table_record(self,
table_name=None,
column_name=None,
column_value=None,
p_conn=None):
if column_name is None or column_value is None or table_name is None:
raise err.MyError(log.debug().position(), err.MyError.errcode[4],
err.MyError.errmsg[4])
if p_conn is None:
try:
# conn = mysql.connector.connect(**config.env['db_connection'])
db = dbconn.db()
conn = db.create_conn()
except mysql.connector.Error as e:
log.log(
log.set_loglevel().error(),
log.debug().position(),
e,
)
log.debugfile().printlog2(
log.debug().position(),
e,
)
raise err.MyError(log.debug().position(),
err.MyError.errcode[1],
err.MyError.errmsg[1])
else:
conn = p_conn
delete_sql = component.c_db_dml.mysql().delete_record_sql(
table_name, column_name, column_value)
cursor = conn.cursor()
try:
if config.env['env'] == 'dev':
log.log(log.set_loglevel().debug(),
log.debug().position(), delete_sql)
cursor.execute(delete_sql)
delete_count = cursor._rowcount
except Exception as e:
log.log(log.set_loglevel().error(),
log.debug().position(), e, delete_sql)
raise err.MyError(log.debug().position(), err.MyError.errcode[1],
err.MyError.errmsg[1], delete_sql)
finally:
if cursor:
cursor.close()
if p_conn is None:
if conn:
conn.commit()
conn.close()
db.close_conn()
if 'delete_count' in locals():
return delete_count
def query_table_record(self, sql=None, p_conn=None):
if sql is None:
log.log(log.set_loglevel().error(), sql, p_conn)
raise err.MyError(log.debug().position(), err.MyError.errcode[4],
err.MyError.errmsg[4])
if p_conn is None:
try:
# conn = mysql.connector.connect(**config.env['db_connection'])
db = dbconn.db()
conn = db.create_conn()
except mysql.connector.Error as e:
log.log(
log.set_loglevel().error(),
log.debug().position(),
e,
)
log.debugfile().printlog2(
log.debug().position(),
e,
)
raise err.MyError(log.debug().position(),
err.MyError.errcode[1],
err.MyError.errmsg[1])
else:
conn = p_conn
cursor = conn.cursor()
try:
if config.env['env'] == 'dev':
log.log(log.set_loglevel().debug(),
log.debug().position(), sql)
cursor.execute(sql)
results = cursor.fetchall()
column_names = cursor.column_names
except Exception as e:
log.log(log.set_loglevel().error(), log.debug().position(), e, sql)
raise err.MyError(log.debug().position(), e, sql)
finally:
if cursor:
cursor.close()
if p_conn is None:
if conn:
conn.commit()
conn.close()
db.close_conn()
if 'results' in locals():
return column_names, results
def main(self, path_query):
rel_type = '00' # normal
rel_value = ''
try:
method, params = self.parse_path_query(path_query)
log.log(log.set_loglevel().debug(),
log.debug().position(), method, params)
if method == '/redirect':
pass
# return requests.get('http://127.0.0.1:10008/cross')
elif method == '/cross':
rel_type, rel_value = '00', 'OK'
elif method == '/rgegeg':
if params is None:
rel_type, rel_value = -1, 'paramater error'
else:
rel_type, rel_value = 0, 'this request method is %s, params is %s ' % (
method, params)
elif method == '/es_insert_check':
draft.d_mile_analyze.test().aaa()
rel_type, rel_value = 0, ''
elif method == '/es_select_check':
rel_type, rel_value = 0, draft.d_mile_analyze.test().bbb()
elif method == '/data_masking':
if params is None:
rel_type, rel_value = -1, 'paramater error'
elif params['key'] != '1234':
rel_type, rel_value = -1, 'authorization error'
elif method == '/train_mile_filter_collect':
if type(params.get('trainlist')) == type(
None) or 'trainlist' not in params.keys():
rel_type, rel_value = '100', 'paramater error'
else:
if params.get('trainlist') == 'all':
t_r, n_r, du_r, de_r = draft.d_mile_analyze.deal_data(
).all_carriage_noise_deal()
else:
t_r, n_r, du_r, de_r = draft.d_mile_analyze.deal_data(
).all_carriage_noise_deal(
params.get('trainlist').split(','))
rel_type, rel_value = '00', {
'all_records': t_r,
'noise_records': n_r,
'noise_duplicated_records': du_r,
'delete_records': de_r
}
elif method == '/train_mile_detail':
if type(params.get('train_no')) == type(
None) or 'train_no' not in params.keys():
rel_type, rel_value = '100', 'paramater error'
else:
result = draft.d_mile_analyze.deal_data(
).get_train_mile_detail(params.get('train_no').split(','))
rel_type, rel_value = '00', result
elif method == '/train_mile_statistic':
rel_type, rel_value = route_tools().param_keys_check(
params, 'train_no', 'period_from', 'period_to')
# if type(params.get('train_no')) == type(None) or 'train_no' not in params.keys():
# rel_type, rel_value = '100', 'paramater error'
# else:
if rel_type == '':
if params.get('train_no') == 'all':
result = draft.d_mile_analyze.deal_data(
).get_train_mile_statistic('all')
else:
result = draft.d_mile_analyze.deal_data(
).get_train_mile_statistic(
params.get('train_no').split(','))
rel_type, rel_value = '00', result
elif method == '/nhpp_coefficient':
log.log(log.set_loglevel().debug(),
log.debug().position(), 'this is %s request' % method)
if 'train' not in params.keys() or type(
params.get('train')) != list:
log.log(
log.set_loglevel().error(),
log.debug().position(),
'this is %s request, parameter error. parameter: %s' %
(method, params))
rel_type, rel_value = '100', 'paramater error'
else:
log.log(log.set_loglevel().debug(),
log.debug().position(), 'nhpp_coefficient start')
result = logic.l_nhpp_func.nhpp_func().main(params)
log.log(log.set_loglevel().debug(),
log.debug().position(), 'nhpp_coefficient end')
rel_type, rel_value = '00', result
elif method == '/excel_func':
if 'params' not in params.keys(
) or 'func_name' not in params.keys() or type(
params.get('params')) != list:
rel_type, rel_value = '100', 'paramater error, paramater name are params and func_name, and must be set to a value, and params should be list type.'
else:
result = logic.l_excel_func.excel_func().run(
params.get('func_name'), params.get('params'))
rel_type, rel_value = '00', result
elif method == '/life_analysis_func':
if 'data' not in params.keys() or type(
params.get('data')) != list:
rel_type, rel_value = '100', 'paramater error, paramater name are params and func_name, and must be set to a value, and data should be list type.'
else:
result = logic.l_life_characteristic_analysis_func.life_characteristic_analysis_func(
).evaluate(sorted(params.get('data')),
len(params.get('data')))
rel_type, rel_value = '00', result
else:
rel_type, rel_value = 'EPYC99999', 'Wrong request method'
log.log(log.set_loglevel().debug(),
log.debug().position(),
'logic finished. return migrating......')
except Exception as e:
log.log(log.set_loglevel().error(),
err.err_catch().catch(sys.exc_info()), e, path_query)
rel_type, rel_value = '9x99999', str(
e.args[0]) + ' ' + '<<<<<-----'.join(err.err_catch().catch(
sys.exc_info()))
finally:
try:
return route_tools().return_json(rel_type, rel_value)
except Exception as e:
return json.dumps({
'errcode': '999',
'errmsg': e,
'result': None
})
finally:
log.log(log.set_loglevel().debug(),
log.debug().position(), rel_type, rel_value)
log.log(log.set_loglevel().debug(),
log.debug().position(), 'return migrate over.')
def handle_request(s, sleep):
try:
if (config.env['env'] == 'dev' and config.env['whiteip']
== []) or s.getpeername()[0] in config.env['whiteip']:
pass
else:
s.send(b'HTTP/1.1 400 NG ')
s.send(
bytes('\r\n\r\n%s\n' % s.getpeername()[0] +
'not in white ip list',
encoding='utf8'))
s.shutdown(socket.SHUT_WR)
return
data = b''
MSGLEN = 1024
# solution 1 start
# i=0
# s.shutdown(socket.SHUT_RD)
# while True:
# data_tmp = s.recv(MSGLEN)
# if data_tmp.__len__()==0 and i>0: # for post request, the 1st line is None
# break
# else:
# i=1
# data += data_tmp
# sleep(0.1) # 必须要，否则会快速满足条件退出
# solution 1 end
# solution 2 start
while True:
data_tmp = b'' # 必须要保留，不懂为啥
with gevent.Timeout(0.5, False) as timeout:
data_tmp = s.recv(MSGLEN)
# check=s.recv(MSGLEN,socket.MSG_PEEK)
data += data_tmp
if data_tmp.__len__() == 0:
break
# solution 2 end
# if len(data)<79:
# break
# else:
# if data.decode().endswith('\r\n\r\n'):
# break
# sleep(0.1)
data = data.decode()
log.log(log.set_loglevel().debug(), data)
a = Request(data)
# path,query = a.parse_path()
response = route.route_main().main(a.parse_path())
log.log(log.set_loglevel().debug(),
log.debug().position(), len(response))
# print('header:',a.headers)
# print('body:',a.body)
# print('path:',a.path)
# print('query:',a.parse_path())
# # method=unquote(a.path.split('?', 1)[0][1:])
# # parameter=unquote(a.path.split('?', 1)[1])
# # query_string=parse_qs(parameter)
# # print(method)
# # print(parameter)
# # print(query_string)
# print('method:',a.method)
# print('content:',a.content)
# request_string = "GET %(request)s HTTP/1.1\r\nHost: %(hostip)s\r\n\r\n%(content)s\n" % {'request':'index.html','hostip':hostip,'content':11111}
request_string = '\r\n\r\n%s\n' % response
log.log(log.set_loglevel().debug(),
log.debug().position(), len(request_string))
s.send(b'HTTP/1.1 200 OK')
s.send(b"\r\nAccess-Control-Allow-Origin:*")
s.sendall(bytes(request_string, encoding='utf8'))
log.log(log.set_loglevel().debug(),
log.debug().position(), 'send over.')
s.shutdown(socket.SHUT_WR)
log.log(log.set_loglevel().debug(),
log.debug().position(), 'socket closed')
# print('.','be killed')
except KeyboardInterrupt:
raise
except Exception as ex:
print(time.asctime(), log.debug().position(), 'ERROR:', data, ex)
finally:
s.close()
def update_table_record(self,
table_name=None,
column_name=None,
column_value=None,
p_conn=None):
if column_name is None or column_value is None or table_name is None:
raise err.MyError(log.debug().position(), err.MyError.errcode[4],
err.MyError.errmsg[4])
if p_conn is None:
try:
# conn = mysql.connector.connect(**config.env['db_connection'])
db = dbconn.db()
conn = db.create_conn()
except mysql.connector.Error as e:
log.log(
log.set_loglevel().error(),
log.debug().position(),
e,
)
log.debugfile().printlog2(
log.debug().position(),
e,
)
raise err.MyError(log.debug().position(),
err.MyError.errcode[1],
err.MyError.errmsg[1])
else:
conn = p_conn
update_sql, column_value = component.c_db_dml.mysql(
).update_record_sql(table_name, column_name, column_value)
cursor = conn.cursor()
try:
try_count = 1
while try_count <= 5:
try:
if config.env['env'] == 'dev':
log.log(log.set_loglevel().debug(),
log.debug().position(), update_sql,
column_value)
cursor.execute(update_sql, column_value)
update_count = cursor._rowcount
break
except Exception as e:
# log.log(log.set_loglevel().error(),log.debug().position(),err.err_catch().catch(sys.exc_info()), update_sql, column_name, column_value,e)
log.log(log.set_loglevel().error(),
log.debug().position(), update_sql, column_name,
column_value, e)
process_db = dbconn.db()
process_conn = process_db.create_conn()
process_cursor = process_conn.cursor()
process_cursor.execute("show full processlist;")
process_lists = process_cursor.fetchall()
log.log(log.set_loglevel().error(),
log.debug().position(), process_lists)
process_cursor.close()
process_conn.close()
process_db.close_conn()
cursor.close()
cursor = conn.cursor()
if str(e).count('Lock wait timeout exceeded') > 0:
try_count += 1
time.sleep(5)
if try_count > 5:
raise
else:
raise
finally:
if cursor:
cursor.close()
if p_conn is None:
if conn:
conn.commit()
conn.close()
db.close_conn()
if 'update_count' in locals():
return update_count
def delete_doc(self,
index_name=None,
type_name=None,
id=None,
cond_value=None,
p_conn=None):
"""
ES delete id有值，condvalue没值，删除对象为id；condvalue有值，删除对象为condvalue，和id无关 a=db_es_dml().update_doc('log_test','python_core','AWVqRo0K5SxcWh31QmD5',version=None,cond_value=None,doc_value={'aaa':47})
:param index_name:
:type index_name: string
:param type_name:
:type type_name: string
:param id:
:type id: string
:param p_conn:
:type p_conn: object
:return:
:rtype:
"""
if index_name is None or type_name is None or (
cond_value is not None and type(cond_value) != dict):
log.log(log.set_loglevel().error(), index_name, type_name, id,
cond_value, p_conn)
raise err.MyError(log.debug().position(), err.MyError.errcode[4],
err.MyError.errmsg[4])
if p_conn is None:
try:
esdb = dbconn.db_es()
conn = esdb.create_conn()
except Exception as e:
log.log(
log.set_loglevel().error(),
log.debug().position(),
e,
)
raise err.MyError(log.debug().position(),
err.MyError.errcode[1],
err.MyError.errmsg[1])
else:
conn = p_conn
body_value = {}
continue_flag = True
if id is not None:
body_value['query'] = {'term': {'_id': id}}
body_value['version'] = True
if cond_value is not None:
body_value.pop('query')
elif id is None:
body_value['query'] = {'match_all': {}}
body_value_tmp = component.c_db_dml.es().delete_record_sql(
index_name, type_name, cond_value)
if 'query' in body_value_tmp.keys():
if 'query' in body_value.keys():
body_value['query'].update(body_value_tmp['query'])
else:
body_value['query'] = body_value_tmp['query']
result = conn.delete_by_query(index=index_name,
doc_type=type_name,
body=body_value)
ret_val = result['deleted']
if p_conn is None:
if 'esdb' in locals().keys():
esdb.close_conn()
del esdb
return ret_val
def update_doc(self,
index_name=None,
type_name=None,
id=None,
version=None,
doc_value=None,
cond_value=None,
p_conn=None):
"""
ES update id version同时有值则会校验version；id有值，condvalue没值，更新对象为id；condvalue有值，更新对象为condvalue，和id无关 a=db_es_dml().update_doc('log_test','python_core','AWVqRo0K5SxcWh31QmD5',version=None,cond_value=None,doc_value={'aaa':47})
:param index_name:
:type index_name: string
:param type_name:
:type type_name: string
:param id:
:type id: string
:param version:
:type version: integer
:param doc_value:
:type doc_value: dict
:param p_conn:
:type p_conn: object
:return:
:rtype:
"""
if index_name is None or type_name is None or doc_value is None or (
version is not None and id is None) or (
cond_value is not None and type(cond_value) != dict) or (
len(doc_value) > 0 and type(doc_value) != dict):
log.log(log.set_loglevel().error(), index_name, type_name, id,
version, doc_value, cond_value, p_conn)
raise err.MyError(log.debug().position(), err.MyError.errcode[4],
err.MyError.errmsg[4])
if p_conn is None:
try:
esdb = dbconn.db_es()
conn = esdb.create_conn()
except Exception as e:
log.log(
log.set_loglevel().error(),
log.debug().position(),
e,
)
raise err.MyError(log.debug().position(),
err.MyError.errcode[1],
err.MyError.errmsg[1])
else:
conn = p_conn
body_value = {}
continue_flag = True
if id is not None:
body_value['query'] = {'term': {'_id': id}}
body_value['version'] = True
if version is not None:
results = conn.search(index_name, type_name, body_value)
result_version = results['hits']['hits'][0]['_version']
if result_version != version:
continue_flag = False
ret_val = 0
if cond_value is not None:
body_value.pop('query')
elif id is None:
body_value['query'] = {'match_all': {}}
if continue_flag:
body_value_tmp = component.c_db_dml.es().update_record_sql(
index_name, type_name, doc_value, cond_value)
if 'query' in body_value_tmp.keys():
if 'query' in body_value.keys():
body_value['query'].update(body_value_tmp['query'])
else:
body_value['query'] = body_value_tmp['query']
if 'script' in body_value.keys():
body_value['script'].update(body_value_tmp['script'])
else:
body_value['script'] = body_value_tmp['script']
if 'version' in body_value.keys():
try:
body_value['version'].update(body_value_tmp['version'])
except AttributeError as e:
body_value['version'] = body_value_tmp['version']
else:
body_value['version'] = body_value_tmp['version']
result = conn.update_by_query(index=index_name,
doc_type=type_name,
body=body_value)
ret_val = result['updated']
if p_conn is None:
if 'esdb' in locals().keys():
esdb.close_conn()
del esdb
return ret_val
def new_doc(self,
index_name=None,
type_name=None,
doc_value=None,
p_conn=None):
"""
ES insert
:param index_name:
:type index_name: string
:param type_name:
:type type_name: string
:param doc_value:
:type doc_value: dict or list for bulk
:param p_conn:
:type p_conn: es object
:return: (created True or False, id , version)
:rtype: array
"""
if index_name is None or type_name is None or doc_value is None:
log.log(log.set_loglevel().error(), index_name, type_name,
doc_value, p_conn)
raise err.MyError(log.debug().position(), err.MyError.errcode[4],
err.MyError.errmsg[4])
if p_conn is None:
try:
esdb = dbconn.db_es()
conn = esdb.create_conn()
except Exception as e:
log.log(
log.set_loglevel().error(),
log.debug().position(),
e,
)
raise err.MyError(log.debug().position(),
err.MyError.errcode[1],
err.MyError.errmsg[1])
else:
conn = p_conn
result = {
'created': False,
'result': 'created',
'_id': None,
'_version': None
}
try:
if len(doc_value) == 0 or type(doc_value) == dict:
result = conn.index(index=index_name,
doc_type=type_name,
body=doc_value)
result['created'] = True
# '_op_type':'delete'
# result = elasticsearch.helpers.bulk(conn,[{'_index':index_name,'_type':type_name,'_source':doc_value}],raise_on_error=True)
elif type(doc_value) == list:
docs = []
for doc in doc_value:
one_row = {
'_index': index_name,
'_type': type_name,
'_source': doc
}
docs.append(one_row)
result = elasticsearch.helpers.bulk(conn,
docs,
raise_on_error=True)
result = {
'created': True,
'result': 'created',
'_id': '' if result[0] > 0 else None,
'_version': '' if result[0] > 0 else None
}
else:
log.log(log.set_loglevel().error(), index_name, type_name,
doc_value, p_conn)
raise err.MyError(log.debug().position(),
err.MyError.errcode[4],
err.MyError.errmsg[4])
# {'created': True, 'result': 'created', '_shards': {'successful': 1, 'failed': 0, 'total': 2}, '_version': 1, '_type': 'python_core', '_id': 'AWVqIzR35SxcWh31QmD1', '_index': 'log_test'}
except Exception as e:
log.log(log.set_loglevel().error(),
log.debug().position(), e, index_name, type_name,
doc_value, p_conn)
raise e
finally:
if p_conn is None:
if 'esdb' in locals().keys():
esdb.close_conn()
del esdb
return result['created'], result['_id'], result['_version']