def query_doc(self, index_name=None, type_name=None, cond_value=None, params=None, p_conn=None): if index_name is None or type_name is None or cond_value is None or ( cond_value is not None and type(cond_value) != dict): log.log(log.set_loglevel().error(), index_name, type_name, cond_value, p_conn) raise err.MyError(log.debug().position(), err.MyError.errcode[4], err.MyError.errmsg[4]) if p_conn is None: try: esdb = dbconn.db_es() conn = esdb.create_conn() except Exception as e: log.log( log.set_loglevel().error(), log.debug().position(), e, ) raise err.MyError(log.debug().position(), err.MyError.errcode[1], err.MyError.errmsg[1]) else: conn = p_conn cond_value['version'] = True ret_val = conn.search(index_name, type_name, cond_value) if p_conn is None: if 'esdb' in locals().keys(): esdb.close_conn() del esdb return ret_val
def add_memo(self, ordercode='', memo='', operator='', remark=None, p_conn=None): """ 暂时不生效 :param ordercode: :type ordercode: :param memo: :type memo: :param operator: :type operator: :param remark: :type remark: :param p_conn: :type p_conn: :return: :rtype: """ if ordercode == '' or ordercode is None or memo == '' or memo is None or operator == '' or operator is None: log.log(log.set_loglevel().error(), log.debug().position(), err.MyError.errcode[4], err.MyError.errmsg[4]) raise err.MyError(log.set_loglevel().error(), log.debug().position(), err.MyError.errcode[4], err.MyError.errmsg[4]) if p_conn is None: vipdb = dbconn.vipdb() conn = vipdb.create_conn() else: conn = p_conn table_name = '%s.t_insurance_memo' % config.env['vipschema'] column_name = ('ordercode', 'memo', 'operator', 'remark', 'cdt', 'udt') str_curr_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') column_value = (ordercode, memo, operator, remark, str_curr_time, str_curr_time) memo_id = self.new_table_record(table_name, column_name, column_value, conn) if p_conn is None: conn.rollback() del conn vipdb.close_conn() return memo_id
def new_table_record(self, table_name=None, column_name=None, column_value=None, p_conn=None): if column_name is None or column_value is None or table_name is None: log.log(log.set_loglevel().error(), table_name, column_name, column_value, p_conn) raise err.MyError(log.debug().position(), err.MyError.errcode[4], err.MyError.errmsg[4]) if p_conn is None: try: # conn = mysql.connector.connect(**config.env['db_connection']) db = dbconn.db() conn = db.create_conn() except mysql.connector.Error as e: log.log( log.set_loglevel().error(), log.debug().position(), e, ) log.debugfile().printlog2( log.debug().position(), e, ) raise err.MyError(log.debug().position(), err.MyError.errcode[1], err.MyError.errmsg[1]) else: conn = p_conn insert_sql, column_value = component.c_db_dml.mysql().new_record_sql( table_name, column_name, column_value) cursor = conn.cursor() try: if config.env['env'] == 'dev': log.log(log.set_loglevel().debug(), log.debug().position(), insert_sql, column_value) cursor.execute(insert_sql, column_value) new_id = cursor._last_insert_id except Exception as e: log.log(log.set_loglevel().error(), log.debug().position(), e, insert_sql, column_value) raise err.MyError( log.debug().position(), err.MyError.errcode[1], err.MyError.errmsg[1] + insert_sql + ' ' + column_value) finally: if cursor: cursor.close() if p_conn is None: if conn: conn.commit() conn.close() db.close_conn() if 'new_id' in locals(): return new_id
def httpserver(self): """ :param ordercode: :type ordercode: :return: :rtype: """ # 不再需要消息解析,纯保险公司查询 url = "%s/py/pressure" % (1) #(config.env['domain_address']) header_dict = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko'} req = urllib.request.Request(url=url, headers=header_dict) try: f = urllib.request.urlopen(req, timeout=15) return str(f.read().decode('utf-8')) except Exception as e: if str(e).__contains__('500'): pass else: log.log(log.set_loglevel().error(), log.debug().position(), req.data, req.full_url, req.headers, e) return '{"code":"-1","message":"%s","info":%s}' % (e, log.debug().position())
def query_table_record(self, sql=None, p_conn=None): if sql is None: log.log(log.set_loglevel().error(), sql, p_conn) raise err.MyError(log.debug().position(), err.MyError.errcode[4], err.MyError.errmsg[4]) if p_conn is None: try: # conn = mysql.connector.connect(**config.env['db_connection']) db = dbconn.db() conn = db.create_conn() except mysql.connector.Error as e: log.log( log.set_loglevel().error(), log.debug().position(), e, ) log.debugfile().printlog2( log.debug().position(), e, ) raise err.MyError(log.debug().position(), err.MyError.errcode[1], err.MyError.errmsg[1]) else: conn = p_conn cursor = conn.cursor() try: if config.env['env'] == 'dev': log.log(log.set_loglevel().debug(), log.debug().position(), sql) cursor.execute(sql) results = cursor.fetchall() column_names = cursor.column_names except Exception as e: log.log(log.set_loglevel().error(), log.debug().position(), e, sql) raise err.MyError(log.debug().position(), e, sql) finally: if cursor: cursor.close() if p_conn is None: if conn: conn.commit() conn.close() db.close_conn() if 'results' in locals(): return column_names, results
def delete_table_record(self, table_name=None, column_name=None, column_value=None, p_conn=None): if column_name is None or column_value is None or table_name is None: raise err.MyError(log.debug().position(), err.MyError.errcode[4], err.MyError.errmsg[4]) if p_conn is None: try: # conn = mysql.connector.connect(**config.env['db_connection']) db = dbconn.db() conn = db.create_conn() except mysql.connector.Error as e: log.log( log.set_loglevel().error(), log.debug().position(), e, ) log.debugfile().printlog2( log.debug().position(), e, ) raise err.MyError(log.debug().position(), err.MyError.errcode[1], err.MyError.errmsg[1]) else: conn = p_conn delete_sql = component.c_db_dml.mysql().delete_record_sql( table_name, column_name, column_value) cursor = conn.cursor() try: if config.env['env'] == 'dev': log.log(log.set_loglevel().debug(), log.debug().position(), delete_sql) cursor.execute(delete_sql) delete_count = cursor._rowcount except Exception as e: log.log(log.set_loglevel().error(), log.debug().position(), e, delete_sql) raise err.MyError(log.debug().position(), err.MyError.errcode[1], err.MyError.errmsg[1], delete_sql) finally: if cursor: cursor.close() if p_conn is None: if conn: conn.commit() conn.close() db.close_conn() if 'delete_count' in locals(): return delete_count
def main(self, path_query): rel_type = '00' # normal rel_value = '' try: method, params = self.parse_path_query(path_query) log.log(log.set_loglevel().debug(), log.debug().position(), method, params) if method == '/redirect': pass # return requests.get('http://127.0.0.1:10008/cross') elif method == '/cross': rel_type, rel_value = '00', 'OK' elif method == '/rgegeg': if params is None: rel_type, rel_value = -1, 'paramater error' else: rel_type, rel_value = 0, 'this request method is %s, params is %s ' % ( method, params) elif method == '/es_insert_check': draft.d_mile_analyze.test().aaa() rel_type, rel_value = 0, '' elif method == '/es_select_check': rel_type, rel_value = 0, draft.d_mile_analyze.test().bbb() elif method == '/data_masking': if params is None: rel_type, rel_value = -1, 'paramater error' elif params['key'] != '1234': rel_type, rel_value = -1, 'authorization error' elif method == '/train_mile_filter_collect': if type(params.get('trainlist')) == type( None) or 'trainlist' not in params.keys(): rel_type, rel_value = '100', 'paramater error' else: if params.get('trainlist') == 'all': t_r, n_r, du_r, de_r = draft.d_mile_analyze.deal_data( ).all_carriage_noise_deal() else: t_r, n_r, du_r, de_r = draft.d_mile_analyze.deal_data( ).all_carriage_noise_deal( params.get('trainlist').split(',')) rel_type, rel_value = '00', { 'all_records': t_r, 'noise_records': n_r, 'noise_duplicated_records': du_r, 'delete_records': de_r } elif method == '/train_mile_detail': if type(params.get('train_no')) == type( None) or 'train_no' not in params.keys(): rel_type, rel_value = '100', 'paramater error' else: result = draft.d_mile_analyze.deal_data( ).get_train_mile_detail(params.get('train_no').split(',')) rel_type, rel_value = '00', result elif method == '/train_mile_statistic': rel_type, rel_value = route_tools().param_keys_check( params, 'train_no', 'period_from', 'period_to') # if type(params.get('train_no')) == type(None) or 'train_no' not in params.keys(): # rel_type, rel_value = '100', 'paramater error' # else: if rel_type == '': if params.get('train_no') == 'all': result = draft.d_mile_analyze.deal_data( ).get_train_mile_statistic('all') else: result = draft.d_mile_analyze.deal_data( ).get_train_mile_statistic( params.get('train_no').split(',')) rel_type, rel_value = '00', result elif method == '/nhpp_coefficient': log.log(log.set_loglevel().debug(), log.debug().position(), 'this is %s request' % method) if 'train' not in params.keys() or type( params.get('train')) != list: log.log( log.set_loglevel().error(), log.debug().position(), 'this is %s request, parameter error. parameter: %s' % (method, params)) rel_type, rel_value = '100', 'paramater error' else: log.log(log.set_loglevel().debug(), log.debug().position(), 'nhpp_coefficient start') result = logic.l_nhpp_func.nhpp_func().main(params) log.log(log.set_loglevel().debug(), log.debug().position(), 'nhpp_coefficient end') rel_type, rel_value = '00', result elif method == '/excel_func': if 'params' not in params.keys( ) or 'func_name' not in params.keys() or type( params.get('params')) != list: rel_type, rel_value = '100', 'paramater error, paramater name are params and func_name, and must be set to a value, and params should be list type.' else: result = logic.l_excel_func.excel_func().run( params.get('func_name'), params.get('params')) rel_type, rel_value = '00', result elif method == '/life_analysis_func': if 'data' not in params.keys() or type( params.get('data')) != list: rel_type, rel_value = '100', 'paramater error, paramater name are params and func_name, and must be set to a value, and data should be list type.' else: result = logic.l_life_characteristic_analysis_func.life_characteristic_analysis_func( ).evaluate(sorted(params.get('data')), len(params.get('data'))) rel_type, rel_value = '00', result else: rel_type, rel_value = 'EPYC99999', 'Wrong request method' log.log(log.set_loglevel().debug(), log.debug().position(), 'logic finished. return migrating......') except Exception as e: log.log(log.set_loglevel().error(), err.err_catch().catch(sys.exc_info()), e, path_query) rel_type, rel_value = '9x99999', str( e.args[0]) + ' ' + '<<<<<-----'.join(err.err_catch().catch( sys.exc_info())) finally: try: return route_tools().return_json(rel_type, rel_value) except Exception as e: return json.dumps({ 'errcode': '999', 'errmsg': e, 'result': None }) finally: log.log(log.set_loglevel().debug(), log.debug().position(), rel_type, rel_value) log.log(log.set_loglevel().debug(), log.debug().position(), 'return migrate over.')
def re_expr(self,re_word='',string=''): log.log(log.set_loglevel().debug(),re_word,string) pattern = re.compile(r'%s' % re_word) match_result = re.findall(pattern,string) return match_result
def handle_request(s, sleep): try: if (config.env['env'] == 'dev' and config.env['whiteip'] == []) or s.getpeername()[0] in config.env['whiteip']: pass else: s.send(b'HTTP/1.1 400 NG ') s.send( bytes('\r\n\r\n%s\n' % s.getpeername()[0] + 'not in white ip list', encoding='utf8')) s.shutdown(socket.SHUT_WR) return data = b'' MSGLEN = 1024 # solution 1 start # i=0 # s.shutdown(socket.SHUT_RD) # while True: # data_tmp = s.recv(MSGLEN) # if data_tmp.__len__()==0 and i>0: # for post request, the 1st line is None # break # else: # i=1 # data += data_tmp # sleep(0.1) # 必须要,否则会快速满足条件退出 # solution 1 end # solution 2 start while True: data_tmp = b'' # 必须要保留,不懂为啥 with gevent.Timeout(0.5, False) as timeout: data_tmp = s.recv(MSGLEN) # check=s.recv(MSGLEN,socket.MSG_PEEK) data += data_tmp if data_tmp.__len__() == 0: break # solution 2 end # if len(data)<79: # break # else: # if data.decode().endswith('\r\n\r\n'): # break # sleep(0.1) data = data.decode() log.log(log.set_loglevel().debug(), data) a = Request(data) # path,query = a.parse_path() response = route.route_main().main(a.parse_path()) log.log(log.set_loglevel().debug(), log.debug().position(), len(response)) # print('header:',a.headers) # print('body:',a.body) # print('path:',a.path) # print('query:',a.parse_path()) # # method=unquote(a.path.split('?', 1)[0][1:]) # # parameter=unquote(a.path.split('?', 1)[1]) # # query_string=parse_qs(parameter) # # print(method) # # print(parameter) # # print(query_string) # print('method:',a.method) # print('content:',a.content) # request_string = "GET %(request)s HTTP/1.1\r\nHost: %(hostip)s\r\n\r\n%(content)s\n" % {'request':'index.html','hostip':hostip,'content':11111} request_string = '\r\n\r\n%s\n' % response log.log(log.set_loglevel().debug(), log.debug().position(), len(request_string)) s.send(b'HTTP/1.1 200 OK') s.send(b"\r\nAccess-Control-Allow-Origin:*") s.sendall(bytes(request_string, encoding='utf8')) log.log(log.set_loglevel().debug(), log.debug().position(), 'send over.') s.shutdown(socket.SHUT_WR) log.log(log.set_loglevel().debug(), log.debug().position(), 'socket closed') # print('.','be killed') except KeyboardInterrupt: raise except Exception as ex: print(time.asctime(), log.debug().position(), 'ERROR:', data, ex) finally: s.close()
log.debug().position(), 'socket closed') # print('.','be killed') except KeyboardInterrupt: raise except Exception as ex: print(time.asctime(), log.debug().position(), 'ERROR:', data, ex) finally: s.close() if __name__ == '__main__': starttime = time.time() try: portNum = config.env['port'] hostname = socket.gethostname() hostip = socket.gethostbyname(socket.gethostname()) log.log(log.set_loglevel().top(), time.asctime(), "Server Starts - %s,%s:%s" % (hostname, hostip, portNum)) # print(time.asctime(), "Server Starts - %s,%s:%s" % (hostname,hostip, portNum)) # while ~start_check().port_open(hostip,portNum): # print('waiting for port release...') # time.sleep(1) server(portNum) finally: endtime = time.time() log.log(log.set_loglevel().top(), "kept alive %d seconds" % (endtime - starttime)) log.log(log.set_loglevel().top(), time.asctime(), "Server Stops - %s,%s:%s" % (hostname, hostip, portNum)) # print("kept alive %d seconds" % (endtime - starttime)) # print(time.asctime(), "Server Stops - %s,%s:%s" % (hostname,hostip, portNum))
def update_table_record(self, table_name=None, column_name=None, column_value=None, p_conn=None): if column_name is None or column_value is None or table_name is None: raise err.MyError(log.debug().position(), err.MyError.errcode[4], err.MyError.errmsg[4]) if p_conn is None: try: # conn = mysql.connector.connect(**config.env['db_connection']) db = dbconn.db() conn = db.create_conn() except mysql.connector.Error as e: log.log( log.set_loglevel().error(), log.debug().position(), e, ) log.debugfile().printlog2( log.debug().position(), e, ) raise err.MyError(log.debug().position(), err.MyError.errcode[1], err.MyError.errmsg[1]) else: conn = p_conn update_sql, column_value = component.c_db_dml.mysql( ).update_record_sql(table_name, column_name, column_value) cursor = conn.cursor() try: try_count = 1 while try_count <= 5: try: if config.env['env'] == 'dev': log.log(log.set_loglevel().debug(), log.debug().position(), update_sql, column_value) cursor.execute(update_sql, column_value) update_count = cursor._rowcount break except Exception as e: # log.log(log.set_loglevel().error(),log.debug().position(),err.err_catch().catch(sys.exc_info()), update_sql, column_name, column_value,e) log.log(log.set_loglevel().error(), log.debug().position(), update_sql, column_name, column_value, e) process_db = dbconn.db() process_conn = process_db.create_conn() process_cursor = process_conn.cursor() process_cursor.execute("show full processlist;") process_lists = process_cursor.fetchall() log.log(log.set_loglevel().error(), log.debug().position(), process_lists) process_cursor.close() process_conn.close() process_db.close_conn() cursor.close() cursor = conn.cursor() if str(e).count('Lock wait timeout exceeded') > 0: try_count += 1 time.sleep(5) if try_count > 5: raise else: raise finally: if cursor: cursor.close() if p_conn is None: if conn: conn.commit() conn.close() db.close_conn() if 'update_count' in locals(): return update_count
def delete_doc(self, index_name=None, type_name=None, id=None, cond_value=None, p_conn=None): """ ES delete id有值,condvalue没值,删除对象为id;condvalue有值,删除对象为condvalue,和id无关 a=db_es_dml().update_doc('log_test','python_core','AWVqRo0K5SxcWh31QmD5',version=None,cond_value=None,doc_value={'aaa':47}) :param index_name: :type index_name: string :param type_name: :type type_name: string :param id: :type id: string :param p_conn: :type p_conn: object :return: :rtype: """ if index_name is None or type_name is None or ( cond_value is not None and type(cond_value) != dict): log.log(log.set_loglevel().error(), index_name, type_name, id, cond_value, p_conn) raise err.MyError(log.debug().position(), err.MyError.errcode[4], err.MyError.errmsg[4]) if p_conn is None: try: esdb = dbconn.db_es() conn = esdb.create_conn() except Exception as e: log.log( log.set_loglevel().error(), log.debug().position(), e, ) raise err.MyError(log.debug().position(), err.MyError.errcode[1], err.MyError.errmsg[1]) else: conn = p_conn body_value = {} continue_flag = True if id is not None: body_value['query'] = {'term': {'_id': id}} body_value['version'] = True if cond_value is not None: body_value.pop('query') elif id is None: body_value['query'] = {'match_all': {}} body_value_tmp = component.c_db_dml.es().delete_record_sql( index_name, type_name, cond_value) if 'query' in body_value_tmp.keys(): if 'query' in body_value.keys(): body_value['query'].update(body_value_tmp['query']) else: body_value['query'] = body_value_tmp['query'] result = conn.delete_by_query(index=index_name, doc_type=type_name, body=body_value) ret_val = result['deleted'] if p_conn is None: if 'esdb' in locals().keys(): esdb.close_conn() del esdb return ret_val
def update_doc(self, index_name=None, type_name=None, id=None, version=None, doc_value=None, cond_value=None, p_conn=None): """ ES update id version同时有值则会校验version;id有值,condvalue没值,更新对象为id;condvalue有值,更新对象为condvalue,和id无关 a=db_es_dml().update_doc('log_test','python_core','AWVqRo0K5SxcWh31QmD5',version=None,cond_value=None,doc_value={'aaa':47}) :param index_name: :type index_name: string :param type_name: :type type_name: string :param id: :type id: string :param version: :type version: integer :param doc_value: :type doc_value: dict :param p_conn: :type p_conn: object :return: :rtype: """ if index_name is None or type_name is None or doc_value is None or ( version is not None and id is None) or ( cond_value is not None and type(cond_value) != dict) or ( len(doc_value) > 0 and type(doc_value) != dict): log.log(log.set_loglevel().error(), index_name, type_name, id, version, doc_value, cond_value, p_conn) raise err.MyError(log.debug().position(), err.MyError.errcode[4], err.MyError.errmsg[4]) if p_conn is None: try: esdb = dbconn.db_es() conn = esdb.create_conn() except Exception as e: log.log( log.set_loglevel().error(), log.debug().position(), e, ) raise err.MyError(log.debug().position(), err.MyError.errcode[1], err.MyError.errmsg[1]) else: conn = p_conn body_value = {} continue_flag = True if id is not None: body_value['query'] = {'term': {'_id': id}} body_value['version'] = True if version is not None: results = conn.search(index_name, type_name, body_value) result_version = results['hits']['hits'][0]['_version'] if result_version != version: continue_flag = False ret_val = 0 if cond_value is not None: body_value.pop('query') elif id is None: body_value['query'] = {'match_all': {}} if continue_flag: body_value_tmp = component.c_db_dml.es().update_record_sql( index_name, type_name, doc_value, cond_value) if 'query' in body_value_tmp.keys(): if 'query' in body_value.keys(): body_value['query'].update(body_value_tmp['query']) else: body_value['query'] = body_value_tmp['query'] if 'script' in body_value.keys(): body_value['script'].update(body_value_tmp['script']) else: body_value['script'] = body_value_tmp['script'] if 'version' in body_value.keys(): try: body_value['version'].update(body_value_tmp['version']) except AttributeError as e: body_value['version'] = body_value_tmp['version'] else: body_value['version'] = body_value_tmp['version'] result = conn.update_by_query(index=index_name, doc_type=type_name, body=body_value) ret_val = result['updated'] if p_conn is None: if 'esdb' in locals().keys(): esdb.close_conn() del esdb return ret_val
def new_doc(self, index_name=None, type_name=None, doc_value=None, p_conn=None): """ ES insert :param index_name: :type index_name: string :param type_name: :type type_name: string :param doc_value: :type doc_value: dict or list for bulk :param p_conn: :type p_conn: es object :return: (created True or False, id , version) :rtype: array """ if index_name is None or type_name is None or doc_value is None: log.log(log.set_loglevel().error(), index_name, type_name, doc_value, p_conn) raise err.MyError(log.debug().position(), err.MyError.errcode[4], err.MyError.errmsg[4]) if p_conn is None: try: esdb = dbconn.db_es() conn = esdb.create_conn() except Exception as e: log.log( log.set_loglevel().error(), log.debug().position(), e, ) raise err.MyError(log.debug().position(), err.MyError.errcode[1], err.MyError.errmsg[1]) else: conn = p_conn result = { 'created': False, 'result': 'created', '_id': None, '_version': None } try: if len(doc_value) == 0 or type(doc_value) == dict: result = conn.index(index=index_name, doc_type=type_name, body=doc_value) result['created'] = True # '_op_type':'delete' # result = elasticsearch.helpers.bulk(conn,[{'_index':index_name,'_type':type_name,'_source':doc_value}],raise_on_error=True) elif type(doc_value) == list: docs = [] for doc in doc_value: one_row = { '_index': index_name, '_type': type_name, '_source': doc } docs.append(one_row) result = elasticsearch.helpers.bulk(conn, docs, raise_on_error=True) result = { 'created': True, 'result': 'created', '_id': '' if result[0] > 0 else None, '_version': '' if result[0] > 0 else None } else: log.log(log.set_loglevel().error(), index_name, type_name, doc_value, p_conn) raise err.MyError(log.debug().position(), err.MyError.errcode[4], err.MyError.errmsg[4]) # {'created': True, 'result': 'created', '_shards': {'successful': 1, 'failed': 0, 'total': 2}, '_version': 1, '_type': 'python_core', '_id': 'AWVqIzR35SxcWh31QmD1', '_index': 'log_test'} except Exception as e: log.log(log.set_loglevel().error(), log.debug().position(), e, index_name, type_name, doc_value, p_conn) raise e finally: if p_conn is None: if 'esdb' in locals().keys(): esdb.close_conn() del esdb return result['created'], result['_id'], result['_version']