def sitemap_data(): urlset = [] try: for post in Post.where(status=1): lastmod_tmp = datetime.datetime.strptime(post.modified,"%Y-%m-%d %H:%M:%S") lastmod = lastmod_tmp.strftime("%Y-%m-%dT%H:%M:%SZ") url = dict(loc="http://www.talkincode.org/news/post/view/%s"%post.id, lastmod=lastmod,chgfreq="daily") urlset.append(url) for code in Code.where(): lastmod_tmp = datetime.datetime.strptime(code.create_time,"%Y-%m-%d %H:%M:%S") lastmod = lastmod_tmp.strftime("%Y-%m-%dT%H:%M:%SZ") url = dict(loc="http://www.talkincode.org/code/view/%s"%code.id, lastmod=lastmod,chgfreq="monthly") urlset.append(url) for proj in Project.where(): lastmod_tmp = datetime.datetime.strptime(proj.created,"%Y-%m-%d %H:%M:%S") lastmod = lastmod_tmp.strftime("%Y-%m-%dT%H:%M:%SZ") url = dict(loc="http://www.talkincode.org/open/proj/view/%s"%proj.id, lastmod=lastmod,chgfreq="monthly") urlset.append(url) return urlset except Exception,e: logger.error("login error,%s"%e) return []
def get_tags(limit): tagset = {} try: for code in Code.where(): tag = code.tags if not tag: continue ts = tag.split(",") for t in ts: if tagset.has_key(t): tagset[t] += 1 else: tagset[t] = 1 for post in Post.where(status=1): tag = post.tags if not tag: continue ts = tag.split(",") for t in ts: if tagset.has_key(t): tagset[t] += 1 else: tagset[t] = 1 sort_tags = sorted( tagset.items(),key=lambda d:d[1],reverse=True) print sort_tags return sort_tags[:limit] except Exception,e: logger.error("get tags error,%s"%e) raise e
def open(self): # TO DO: Handle possible errors device_type = 'WEB' token = self.get_argument('token', None) app_token = self.get_argument('fbtoken', None) device_id = self.get_argument('idandroid', None) if device_id is not None: device_type = 'ANDROID' else: device_id = self.get_argument('idios', None) if device_id is not None: device_type = 'IOS' logger.info('WEBSOCKET OPEN: token={}'.format(token)) if token: if token not in TOKENS: self.handle_token(token=token, device_id=device_id, device_type=device_type, app_token=app_token) else: user_id = TOKENS[token] self.add_websocket(user_id) else: logger.error("Websocket attempt without token : IP {} HOST {}".format(self.request.remote_ip, self.request.host))
def findAppsWithAutoscaleLabels(self): list = self.marathon_cli.list_apps(embed_counts=True, embed_task_stats=True) logger.debug('Lista recebida {}'.format(list)) if len(list) == 0: logger.warning('0 apps loaded. Your marathon have apps?') for app in list: if LABEL_FOR_AUTOSCALE_ENABLE in app.labels: new_app = MarathonApp(app.id) new_app.tasksRunning = app.tasks_running new_app.tasksStaged = app.tasks_staged for label in MANDATORY_LABELS_APP: if label in app.labels: value = app.labels[label] if value.isnumeric(): value = int(value) new_app.__setattr__(label, value) else: logger.error( 'App: [{}] :: dont have MANDATORY_LABELS :: {}'. format(app.id, label)) for label in OPTIONAL_LABELS_APP: if label in app.labels: value = app.labels[label] if value.isnumeric(): value = int(value) new_app.__setattr__(label, value) self.dict_apps[app.id] = new_app else: logger.debug( 'App: [{}] :: dont have {} = True. If you want to scale, please add labels.' .format(app.id, LABEL_FOR_AUTOSCALE_ENABLE))
async def update_by_conditions(cls, values: dict, conditions: list = None, return_fields: list or set = None, force: bool = False) -> dict or bool: # result update ru = False if not conditions and force: conditions = [] if conditions: # create query query = update(cls).values(values) for condition in conditions: query = query.where(condition) # list returning-fields if not return_fields: return_fields = ['id'] # execute query try: ru = await DBManager().query_fetch( query.returning( *map(lambda field: cls.__getattribute__(cls, field), return_fields))) except Exception as e: logger.error( 'common.entity.baseEntity.BaseEntity#update: {}'.format(e)) return ru
async def handle_async_response(self, response): if response and response.code == 200: user_id = json.loads(response.body.decode())['id'] TOKENS[self.token] = user_id self.add_websocket(user_id) logger.info('WEBSOCKET VALIDATED TOKEN: user={}'.format(user_id)) if self.device_type != 'WEB': db = self.settings['db'] await db.test.find_one_and_delete({ 'device_id': self.device_id, }) await db.user_credentials.update_one( { 'user_id': user_id, 'device_id': self.device_id, }, { '$set': { 'app_token': self.app_token, 'device_type': self.device_type, } }, upsert=True, ) else: logger.error('Error on token validation: {}'.format(response.error))
def apply_transactions(transactions, auto=False): ''' Apply renaming transactions. apply_transactions(transactions) transactions = [(old_path, new_path),(old_path),(new_path),...] Manual review of transactions is required. ''' if auto: logger.warning('Auto is On. No confirmation required.') print('=' * 30) if not transactions: logger.debug('NO TRANSACTIONS') sys.exit('No Transactions to apply.') return for t in transactions: print('[{}] > [{}]'.format(t[0].name, t[1].name)) print('{} Transactions to apply. Renaming...'.format(len(transactions))) count = 0 if auto or input('EXECUTE ? [y]\n>') == 'y': for src, dst in transactions: try: src.rename(dst) except: logger.error(sys.exc_info()[0].__name__) logger.error('Could not rename: [{}]>[{}]'.format(src, dst)) else: logger.debug('[{}] renamed to [{}]'.format(src, dst)) count += 1 print('{} folders renamed.'.format(count))
async def filter_errors_request(request: web.Request, handler) -> web.Response: # allowed method OPTIONS if request.method == hdrs.METH_OPTIONS: return web.json_response(status=200) try: response = await handler(request) # exception "default params not validate", code = 400 except IncorrectParamsException as e: logger.error(e) response = web.json_response(status=400, data=dict(errors=e.errors)) # exception "access denied" except AccessException as e: response = web.Response(status=e.code, reason=e.msg) # TODO: Write the correct handling of errors # other exceptions except Exception as e: if asyncio.get_event_loop().get_debug(): raise e # else return response with code 500 else: logger.error('Fail request, err: {}'.format(repr(e))) response = web.json_response( status=500, data={ 'errors': { 'reason': 'Error on running API-handlers: {}'.format(repr(e)) } }) return response
def create_net(num_classes, dnn='resnet20', **kwargs): ext = None if dnn in ['resnet20', 'resnet56', 'resnet110']: net = models.__dict__[dnn](num_classes=num_classes) elif dnn == 'resnet50': net = models.__dict__['resnet50'](num_classes=num_classes) elif dnn == 'mnistnet': net = MnistNet() elif dnn == 'mnistflnet': net = MnistFLNet() elif dnn == 'cifar10flnet': net = Cifar10FLNet() elif dnn == 'vgg16': net = models.VGG(dnn.upper()) elif dnn == 'alexnet': net = torchvision.models.alexnet() elif dnn == 'lstman4': net, ext = models.LSTMAN4(datapath=kwargs['datapath']) elif dnn == 'lstm': net = lstmpy.lstm(vocab_size=kwargs['vocab_size'], batch_size=kwargs['batch_size']) else: errstr = 'Unsupport neural network %s' % dnn logger.error(errstr) raise errstr return net, ext
async def insert_data(pg, table, values, res=False): """ Universal method for inserting data into table :param pg: connect to DB engine(PostgreSQL) :param table: table name :param values: list of users :param res: If true - return result id list :param verbose: set logs :return: None or list of table_id """ if verbose_insert_data: logger.debug('INSERT_DATA_INTO: %s' % table) async with pg.acquire() as conn: try: if res: query = table.insert().values(values).\ returning(table.c.id) cursor = await conn.execute(query) resp = await cursor.fetchall() return [r[0] for r in resp] else: query = table.insert().values(values) await conn.execute(query) except psycopg2.ProgrammingError as e: logger.error('INSERT_DATA_INTO: %s' % e)
def create_occurrence(username, body): last_occurrences, code = db.get_all(Occurrence, {'user': [username]}) if code != 200: return "Erro ao carregar ocorrências passadas do usuário" errors = validate_create_occurrence(body, last_occurrences) if errors: return errors, 400 try: occurrence = Occurrence( user=username, occurrence_date_time=datetime.datetime.strptime( body['occurrence_date_time'], '%Y-%m-%d %H:%M:%S'), physical_aggression=body['physical_aggression'], victim=body['victim'], police_report=body['police_report'], gun=body['gun'], location=body['location'], occurrence_type=body['occurrence_type']) result, code = db.insert_one(occurrence) return result, code except Exception as error: logger.error(error) return str(error), 401
def write_new_student(self, telegram: str, email: str): conn = self.conn_to_db() cursor = conn.cursor() if cursor: # email add validation insert_student_command = "INSERT INTO students (telegram, email) VALUES (%s, %s)" try: logger.info( f"Trying execute command {insert_student_command} ") cursor.execute(insert_student_command, (telegram, email)) conn.commit() logger.info( f"Command {insert_student_command} was executed succesfully" ) cursor.execute(insert_student_command, (telegram, email)) cursor.close() logger.info("Cursor closed ") conn.close() logger.info("Connection closed") return True except Exception as e: logger.error(e) raise Exception(e) else: logger.error("Something wrong with Postgres - cursor is None") raise Exception("Something wrong with Postgres - cursor is None")
async def _init_connect(self, retry_delay=REDIS_RECONNECT_DELAY, num_retries=REDIS_RECONNECT_RETRIES): """ This method create Redis client by config params. If Redis server refused connection do retries. :param int retry_delay: delay between retries :param int num_retries: number of retries :return: None """ for _ in range(num_retries): try: self.pool = await aioredis.create_pool( (self.conf['host'], self.conf['port']), db=self.conf['db'], password=self.conf['password'], encoding=self.conf['encoding'], minsize=self.conf['minsize'], maxsize=self.conf['maxsize'], loop=self.loop) break except ConnectionRefusedError: logger.error( 'Cant establish connection to redis. Retry after %s s.', retry_delay) await asyncio.sleep(retry_delay)
def get_tests_groups_from_jenkins(runner_name, build_number, distros): runner_build = Build(runner_name, build_number) res = {} for b in runner_build.build_data['subBuilds']: # Get the test group from the console of the job z = Build(b['jobName'], b['buildNumber']) console = z.get_job_console() groups = [ keyword.split('=')[1] for line in console for keyword in line.split() if 'run_tests.py' in line and '--group=' in keyword ] if not groups: logger.error( "No test group found in console of the job {0}/{1}".format( b['jobName'], b['buildNumber'])) continue # Use the last group (there can be several groups in upgrade jobs) test_group = groups[-1] # Get the job suffix job_name = b['jobName'] for distro in distros: if distro in job_name: sep = '.' + distro + '.' job_suffix = job_name.split(sep)[-1] break else: job_suffix = job_name.split('.')[-1] res[job_suffix] = test_group return res
def discover_functions(self): self.self_cursor.execute("SELECT pro_oid,func_name,id FROM function_name WHERE {0}={1} AND alive".format(self.sub_fk,self.id)) local_funcs=self.self_cursor.fetchall() try: self.prod_cursor.execute("""SELECT p.oid AS pro_oid,p.proname AS funcname,p.proretset,t.typname,l.lanname FROM pg_proc p LEFT JOIN pg_namespace n ON n.oid = p.pronamespace JOIN pg_type t ON p.prorettype=t.oid JOIN pg_language l ON p.prolang=l.oid WHERE (p.prolang <> (12)::oid) AND n.oid=(SELECT oid FROM pg_namespace WHERE nspname='public')""") except Exception as e: logger.error("Cannot execute function discovery query: {0}".format(e.pgerror)) return prod_funcs=self.prod_cursor.fetchall() for l_func in local_funcs: for p_func in prod_funcs: if l_func[0]==p_func[0] and l_func[1]==p_func[1]: break else: logger.info("Retired function {0} in schema {1}".format(l_func[1],self.db_fields['sch_name'])) old_func=FunctionName(l_func[2]) # old_func.populate() old_func.retire() for p_func in prod_funcs: for l_func in local_funcs: if p_func[0]==l_func[0] and p_func[1]==l_func[1]: break else: logger.info("Created new function: {0} in schema {1}".format(p_func[1],self.db_fields['sch_name'])) new_func=FunctionName() new_func.set_fields(sn_id=self.id,pro_oid=p_func[0],func_name=p_func[1],proretset=p_func[2],prorettype=p_func[3],prolang=p_func[4]) new_func.create() new_func.truncate()
def _small_talk(message="Привет"): """ This method connect to Api.ai Small Talk domain :param message: input message :return: output message from Api.ai """ ai = apiai.ApiAI(api_ai_conf['client_access_token']) request = ai.text_request() request.lang = 'ru' # optional, default value equal 'en' request.session_id = api_ai_conf['sessid'][:35] request.query = message response = request.getresponse().read() obj = json.loads(response.decode('utf-8')) try: alternate_result = obj.get('alternateResult') if not alternate_result: # If response with answer from domain(result) - Small Talk answer = obj.get('result').get('fulfillment').get('speech') return answer else: answer_from_domain = obj.get('alternateResult').get('fulfillment').get('speech') if not answer_from_domain: # If response with answer from agent(result) answer = obj.get('result').get('fulfillment').get('speech') return answer else: # If response with answer from domain(alternate result) - Small Talk return answer_from_domain except AttributeError as e: logger.error('Handle ERROR: {0}'.format(e))
async def filter_errors_request(request: web.Request, handler) -> web.Response: # allowed method OPTIONS if request.method == hdrs.METH_OPTIONS: return web.json_response(status=200) try: response = await handler(request) # exception "default params not validate", code = 400 except IncorrectParamsException as e: logger.error(e) response = web.json_response(status=400, data=dict(errors=e.errors)) # TODO: Write the correct handling of errors except Exception as e: # if dev - exception raise e # TODO: enable code by after fix DEV_MOD # if DEV_MOD: # raise e # # else return response with code 500 # else: # logger.error('Fail request, err: ', e) # response = web.json_response( # status=500, # data={'errors': { # 'reason': 'Error on running API-handlers: %s' % e # }} # ) return response
async def set_to_cache(cache: BaseCache, key: str, value: Dict) -> None: """Save result to cache and ignore errors.""" try: await cache.set(key, value, ttl=60 * 60) # cache successful items for an hour except Exception as e: logger.error(e)
def fetch(self, table, cond): try: for item in getattr(self.db, table).find(cond): yield item except errors.ServerSelectionTimeoutError as e: logger.error("timed out to fetch {0}".format(table)) raise exceptions.ConnectionError("connecting timeout")
def txt_2_xlsx(src_file, rows, category, project_id): rm = RecordManager() partitions = smash_merged(src_file, rows) if not is_valid(partitions, rows): logger.error("rows in each partitions are not equal.") raise ConvertingError else: header = settings.XLSX_HEADERS[category] xlsx_path = os.path.join(settings.SAVE_DIRECTORY, str(project_id)) last_end = rm.get_previous_end(project_id, category) retry_time = 0 while not last_end: logger.info( "retriving value for last end failed, waiting for the next lookup" ) time.sleep(60) # waiting for next looking up retry_time += 1 last_end = rm.get_previous_end(project_id, category) # TODO: use an alternative way to recover if retry_time > 10: logger.error("unable to find an avaible value for %d" % project_id) raise ConvertingError start = last_end + 1 generate_xlsxs(partitions, header, xlsx_path, category, project_id, start) rm.update_record(project_id, start, start + len(partitions), rows)
def fetch_result(self, cond={}): try: for item in self.db.Result.find(cond): yield item except errors.ServerSelectionTimeoutError as e: logger.error("timed out to fetch {0}".format(table)) raise exceptions.ConnectionError("connecting timeout")
def apply_transactions(transactions, auto=False): ''' Apply renaming transactions. apply_transactions(transactions) transactions = [(old_path, new_path),(old_path),(new_path),...] Manual review of transactions is required. ''' if auto: logger.warning('Auto is On. No confirmation required.') print('='*30) if not transactions: logger.debug('NO TRANSACTIONS') sys.exit('No Transactions to apply.') return for t in transactions: print('[{}] > [{}]'.format(t[0].name, t[1].name)) print('{} Transactions to apply. Renaming...'.format(len(transactions))) count = 0 if auto or input('EXECUTE ? [y]\n>') == 'y': for src, dst in transactions: try: src.rename(dst) except: logger.error(sys.exc_info()[0].__name__) logger.error('Could not rename: [{}]>[{}]'.format(src, dst)) else: logger.debug('[{}] renamed to [{}]'.format(src, dst)) count += 1 print('{} folders renamed.'.format(count))
def get_qty(self): qty_res = [p.text for p in self.s.select( settings.WELL['qty_css_selector'])] try: # Extract product attributes: |Size|QTY|Weight|Unit_Price|Price qty_regex_result = re.findall( settings.WELL['qty_regex'], qty_res[0]) assert(len(qty_regex_result) == 1) # Strip unnecessary stuff qty_regex_result[0] = re.sub( r"swatch_stock_list\[0\]='.*'; ", "", qty_regex_result[0]) qty_regex_results = qty_regex_result[0].split(" ") # Second element is package quantity self.pkg_qty = qty_regex_results[1] try: # Calculate unit price self.unit_price = "%.2f" % ( (float)((self.current_price).strip('$')) / (float)(self.pkg_qty)) # Reconvert to string by adding the $ sign self.unit_price = "$" + self.unit_price except ValueError as err: logger.error("Unable calculating price: %s" % (err), exc_info=True) except (AssertionError, IndexError) as err: logger.error("Unable to get qty: %s" % (err), exc_info=True)
def get(self): uid = self.current_user type_ = self.get_argument('type', None) if not type_: self.set_status(400) result = dict(code=40011, msg=u'缺少type参数') return self.jsonify(result) keep_info = self.keep_map(type_) key = "uid:{}:keep:{}".format(uid, type_) times = rdb.incr(key) if times == 1: rdb.expire(key, get_to_tomorrow()) else: logger.warning('have try times {}'.format(times)) result = dict(code=40010, msg=u'每天只能{}一次哦!'.format(keep_info['name'])) return self.jsonify(result) try: row = Pet.keep(uid=uid, score=keep_info['score']) logger.info('keep pet {}'.format(row)) except Exception, e: self.set_status(500) logger.error('keep pet error {}'.format(e)) result = dict(code=40012, msg=u'更新服务器错误, 请稍后重试!') return self.jsonify(result)
async def get_from_cache(cache: BaseCache, key: str) -> Optional[Dict]: """Loads result from cache and ignore errors.""" try: return await cache.get(key) except Exception as e: logger.error(e) return None
def rank_source_file_reply(self): """ 获取日排名请求返回 """ reply = self.sender() request_url = reply.request().url().url() # 解析出请求的品种 request_filename = request_url.rsplit("/", 1)[1] request_variety = request_filename.split("_")[0] if reply.error(): reply.deleteLater() self.spider_finished.emit( "获取{}排名数据文件。\n失败:{}".format(request_variety[:2], str(reply.error())), True) logger.error("获取{}排名数据文件失败了!".format(request_url[:2])) return save_path = os.path.join( LOCAL_SPIDER_SRC, 'cffex/rank/{}_{}.csv'.format(request_variety, self.date.strftime("%Y-%m-%d"))) file_data = reply.readAll() file_obj = QFile(save_path) is_open = file_obj.open(QFile.WriteOnly) if is_open: file_obj.write(file_data) file_obj.close() reply.deleteLater() tip = "获取中金所{}_{}日持仓排名数据保存到文件成功!".format( request_variety, self.date.strftime("%Y-%m-%d")) if request_variety == "T": tip = "获取中金所{}日所有品种持仓排名数据保存到文件成功!".format( self.date.strftime("%Y-%m-%d")) self.spider_finished.emit(tip, True) self.event_loop.quit()
def discover_tables(self): self.self_cursor.execute("SELECT obj_oid,tbl_name,id FROM table_name WHERE {0}={1} AND alive".format(self.sub_fk,self.id)) local_tbls=self.self_cursor.fetchall() try: self.prod_cursor.execute("""SELECT r.oid,r.relname, CASE WHEN h.inhrelid IS NULL THEN 'f'::boolean ELSE 't'::boolean END AS has_parent FROM pg_class r LEFT JOIN pg_inherits h ON r.oid=h.inhrelid WHERE r.relkind='r' AND r.relnamespace=(SELECT oid FROM pg_namespace WHERE nspname='public')""") except Exception as e: logger.error("Cannot execute tables discovery query: {0}".format(e.pgerror)) return prod_tbls=self.prod_cursor.fetchall() for l_table in local_tbls: for p_table in prod_tbls: if l_table[0]==p_table[0] and l_table[1]==p_table[1]: break else: logger.info("Retired table {0} in schema {1}".format(l_table[1],self.db_fields['sch_name'])) old_table=TableName(l_table[2]) # old_table.populate() old_table.retire() for p_table in prod_tbls: for l_table in local_tbls: if p_table[0]==l_table[0] and p_table[1]==l_table[1]: break else: logger.info("Created new table: {0} in schema {1}".format(p_table[1],self.db_fields['sch_name'])) new_table=TableName() new_table.set_fields(sn_id=self.id,tbl_name=p_table[1],obj_oid=p_table[0],has_parent=p_table[2]) new_table.create() new_table.truncate()
def discover_indexes(self): self.self_cursor.execute("SELECT obj_oid,idx_name,id FROM index_name WHERE tn_id={0} AND alive".format(self.id)) local_idxs=self.self_cursor.fetchall() try: self.prod_cursor.execute("""SELECT i.indexrelid,c.relname,i.indisunique,i.indisprimary FROM pg_index i JOIN pg_class c ON i.indexrelid=c.oid WHERE i.indrelid={0}""".format(self.db_fields['obj_oid'])) except Exception as e: logger.error("Cannot execute index discovery query: {0}".format(e.pgerror)) return prod_idxs=self.prod_cursor.fetchall() for l_idx in local_idxs: for p_idx in prod_idxs: if l_idx[0]==p_idx[0] and l_idx[1]==p_idx[1]: break else: logger.info("Retired index {0} in table {1}".format(l_idx[1],self.db_fields['tbl_name'])) old_idx=IndexName(l_idx[2]) old_idx.retire() for p_idx in prod_idxs: for l_idx in local_idxs: if l_idx[0]==p_idx[0] and l_idx[1]==p_idx[1]: break else: logger.info("Create new index {0} in table {1}".format(p_idx[1],self.db_fields['tbl_name'])) new_index=IndexName() new_index.set_fields(tn_id=self.id,obj_oid=p_idx[0],idx_name=p_idx[1],is_unique=p_idx[2],is_primary=p_idx[3]) new_index.create() new_index.truncate()
def saveUserInfo(self, appid, username, password): """ 将用户注册信息入库 :param appid: :param username: :param password: :return: 保存成功返回True : False """ sql = """ BEGIN; INSERT INTO userinfo (appid_id, username, createtime, updatetime) VALUES (%s, %s, %s, %s); INSERT INTO localauth (userid_id, password) VALUES (%s, %s); COMMIT; """ insert_userinfo = "INSERT INTO userinfo (appid_id, username, createtime, updatetime) VALUES (%s, %s, %s, %s)" insert_localauth = "INSERT INTO localauth (userid_id, password) VALUES (%s, %s)" now = str(int(time.time())) try: userid_id = self.db.insert(insert_userinfo, appid, username, now, now) except Exception as e: logger.error(str(e)) return False else: try: self.db.insert(insert_localauth, userid_id, utils.encodePassword(password)) except Exception as e: logger.error(str(e)) return False return True
def r_file(filename, mode=None, content="", action=""): is_exists_file = os.path.exists(filename) if action == "create": if is_exists_file: try: os.remove(filename) os.mknod(filename) logger.info("Create File Ok.") with open(filename , 'w+') as f: f.write(content) except OSError, e: logger.error("filename: %s " % (filename) + str(e) ) sys.exit(1) else: try: os.mknod(filename) logger.info("Create File Ok.") with open(filename , 'w+') as f: f.write(content) except OSError, e: logger.error("filename: %s" % (filename) + str(e)) sys.exit(1)
async def send_message(sid, message): """ Custom event handler with event_name and Socket.IO namespace for the event. This handler works like echo-server. :param sid: Session ID of the client :param message: message payload :return: None """ # Added transport mode checker transport_mode = sio.transport(sid) logger.debug('MESSAGE TRANSPORT MODE (%s): %s' % (sid, transport_mode)) logger.debug('EVENT("sendMessage"): %s' % message['data']) try: if isinstance(message, dict): if message.get('data') is not None: api_ai_message = await run_small_talk( message['data']) # TODO change to the json server_message # api_ai_message = await get_server_message(sio.pg, message) await sio.emit('sendMessageResponse', {'data': api_ai_message}, room=sid, namespace='/chat') logger.debug('EVENT("sendMessageResponse"): %s' % api_ai_message) else: raise ValueError('Message should have key("data")') else: raise TypeError('Message should be dict: {"data": "some text"}') except ValueError as e: logger.error('Handle ERROR: %s' % e) except TypeError as e1: logger.error('Handle ERROR: %s' % e1)
async def send_binary_message(sid): """ Custom event handler with event_name and Socket.IO namespace for the event. This handler send image file in base64 gzip. :param sid: Session ID of the client :return: emit file base64 gzip """ content_b64 = '' hash_sum = '' try: async with aiofiles.open('static/test.png', mode='rb') as image_file: content = await image_file.read() gzip_file = gzip.compress(content) content_b64 = base64.b64encode(gzip_file) hash_sum = hashlib.md5(content_b64).hexdigest() except OSError as e: logger.error('Handle ERROR: %s' % e) await sio.emit('file response', { 'data': content_b64.decode('utf-8'), 'hash_sum': hash_sum }, room=sid, namespace='/chat', callback=call_back_from_client) logger.debug('My EVENT(FILE) (%s): %s' % (sid, content_b64[:20])) del content_b64
def get_tests_groups_from_jenkins(runner_name, build_number, distros): runner_build = Build(runner_name, build_number) res = {} for b in runner_build.build_data['subBuilds']: if b['result'] is None: logger.debug("Skipping '{0}' job (build #{1}) because it's still " "running...".format(b['jobName'], b['buildNumber'],)) continue # Get the test group from the console of the job z = Build(b['jobName'], b['buildNumber']) console = z.get_job_console() groups = [keyword.split('=')[1] for line in console for keyword in line.split() if 'run_tests.py' in line and '--group=' in keyword] if not groups: logger.error("No test group found in console of the job {0}/{1}" .format(b['jobName'], b['buildNumber'])) continue # Use the last group (there can be several groups in upgrade jobs) test_group = groups[-1] # Get the job suffix job_name = b['jobName'] for distro in distros: if distro in job_name: sep = '.' + distro + '.' job_suffix = job_name.split(sep)[-1] break else: job_suffix = job_name.split('.')[-1] res[job_suffix] = test_group return res
def get_check_create_test_run(self, plan, cases): plan = self.project.plans.get(plan.id) suite_cases = self.suite.cases() run_name = self.get_run_name() runs = plan.runs.find_all(name=run_name) run = self.check_need_create_run(plan, runs, suite_cases) if run is None: logger.info('Run not found in plan "{}", create: "{}"'.format( plan.name, run_name)) # Create new test run with cases from test suite suite_cases = self.get_suite_cases() if not suite_cases: logger.error('Empty test cases set.') return None # suite_cases = self.suite.cases.find(type_id=type_ids[0]) run = Run(name=run_name, description=self.run_description, suite_id=self.suite.id, milestone_id=self.milestone.id, config_ids=[], case_ids=[x.id for x in suite_cases] ) plan.add_run(run) logger.debug('Run created "{}"'.format(run_name)) return run
def dataframe_to_orm(orm, dataframe): field_name_list = orm.__mapper__.tables[0].columns._data._list tempdict1 = DataFrame(dataframe)[field_name_list] for field_name in field_name_list: try: stype = orm.__dict__.get(field_name).type.__visit_name__ if stype == 'string': tempdict1[field_name] = tempdict1[field_name].astype(str) tempdict1[field_name].replace({ 'nan': None, 'None': None }, inplace=True) elif stype == 'text': tempdict1[field_name] = tempdict1[field_name].astype(str) tempdict1[field_name].replace({ 'nan': None, 'None': None }, inplace=True) elif stype == 'numeric': tempdict1[field_name] = tempdict1[field_name].astype(float) tempdict1[field_name] = tempdict1[field_name].where( tempdict1[field_name].notnull(), None) elif stype == 'datetime': tempdict1[field_name] = pd.to_datetime(tempdict1[field_name]) except: logger.error('数据类型转换出错') df_dict = tempdict1.to_dict(orient='record') recordlist = [] for r in df_dict: record = orm(**r) recordlist.append(record) return recordlist
def get_check_create_test_run(self, plan, cases): plan = self.project.plans.get(plan.id) suite_cases = self.suite.cases() run_name = self.get_run_name() runs = plan.runs.find_all(name=run_name) run = self.check_need_create_run(plan, runs, suite_cases) if run is None: logger.info('Run not found in plan "{}", create: "{}"'.format( plan.name, run_name)) # Create new test run with cases from test suite suite_cases = self.get_suite_cases() if not suite_cases: logger.error('Empty test cases set.') return None # suite_cases = self.suite.cases.find(type_id=type_ids[0]) run = Run(name=run_name, description=self.run_description, suite_id=self.suite.id, milestone_id=self.milestone.id, config_ids=[], case_ids=[x.id for x in suite_cases]) plan.add_run(run) logger.debug('Run created "{}"'.format(run_name)) return run
async def update(cls, values: dict, rec_id: int=None, conditions: list=[], return_fields: list or set=None) -> {dict or bool, str}: # message/error msg = '' # result update ru = False # add condition by id if rec_id: conditions.append(cls.id == rec_id) # list returning-fields if not return_fields: return_fields = {'id'} # create and execute query try: # query to db query = update(cls).values(values).returning(*map(lambda field: cls.__getattribute__(cls, field), return_fields)) for condition in conditions: if isinstance(condition, elements.ColumnElement): query = query.where(condition) ru = await DBManager().query_fetchrow(query) except Exception as e: logger.error('common.entity.baseEntity.BaseEntity#update: {}'.format(e)) msg = 'Error update record' return ru, msg
def create(self, request): if all(value != None for value in request.data.values()): email = request.data.get("email") username = request.data.get("username") password = request.data.get("password") or request.data.get( "password1") logger.info(f"TRYING SERIALIZE NEW USER: {request.data}") serializer = UserSerializer(data={ "username": username, "email": email, "password": password }) if serializer.is_valid(): serializer.save() logger.info(f"NEW USER CREATED: {serializer.data} ") new_user = User.objects.get(email=email, username=username) if UserProfile.objects.create(user=new_user, id=new_user.id, user_courses=[]): logger.info(f"USER PROFILE WAS CREATED - {new_user.id}") return Response(serializer.data, status=status.HTTP_201_CREATED) else: logger.error( f"NEW USER WAS NOT CREATED {serializer.data} BECAUSE OF {serializer.errors}" ) return Response(serializer.errors, status=status.HTTP_412_PRECONDITION_FAILED) else: logger.error( f"NEW USER WAS NOT CREATED - SOME EMPTY INPUT FIELDS: {request.data}" ) return Response(status=status.HTTP_412_PRECONDITION_FAILED)
def parse_data(self): """Parse info for each product""" # Extract price information regular_price = self.s.select( settings.TOYSRUS['regular_price_css_selector']) try: # Sometimes regular price is lower than current assert(len(regular_price) <= 1) current_price = self.s.select( settings.TOYSRUS['current_price_css_selector']) assert(len(current_price) == 1) if(regular_price): self.regular_price = regular_price[0].text.strip() self.current_price = current_price[0].text.strip() except AssertionError as err: logger.error("Unable to get price: %s" % (err), exc_info=True) # Extract upc info upc = self.s.select(settings.TOYSRUS['upc_css_selector']) try: assert(len(upc) == 1) self.upc = upc[0].text.strip() except AssertionError as err: logger.error("Unable to get UPC code: %s" % (err), exc_info=True) # Extract qty and unit price info self.get_qty() # Extract size information self.get_size()
def send_email(from_, to_, subject, body_text, body_html): if not from_: from_ = SMTP["default_sender"] if not re.match(r"[^@]+@[^@]+\.[^@]+", from_) or not re.match( r"[^@]+@[^@]+\.[^@]+", to_): logger.error("Invalid email address from {} -- to {}".format( from_, to_)) return False # Create message container - the correct MIME type is multipart/alternative. msg = MIMEMultipart('alternative') msg['Subject'] = subject msg['From'] = from_ msg['To'] = to_ # Record the MIME types of both parts - text/plain and text/html. parts = [] if body_text: parts.append(MIMEText(body_text, 'plain')) if body_html: parts.append(MIMEText(body_html, 'html')) for part in parts: msg.attach(part) logger.warning("EMAIL: " + msg.as_string()) return True server = smtplib.SMTP_SSL(SMTP["host"], SMTP["port"]) server.ehlo() server.login(SMTP["user"], SMTP["password"]) server.sendmail(from_, to_, msg.as_string()) server.close()
def package(pkg_name, option="-y"): if pkg_name: command_line_str = "yum " + option + " install " + pkg_name if os.system(command_line_str)!=0: logger.error("exec: %s error" % (command_line_str)) sys.exit(1)
def exec_dcfg(self): # first exec support script to install system tables. try: logger.info("============MYSQL DEFAULT CONFIG===========") logger.info("install system db.") os.chdir(mysql_home) exec_command('./scripts/mysql_install_db --user=mysql') logger.info("copy boot script to correct directory.") exec_command('cp ' + mysql_boot_script + ' /etc/init.d/') # sed config exec_command('sed -i -e "46s/basedir=/basedir=\/opt\/magima\/mysql/g" /etc/init.d/mysql.server') exec_command('sed -i -e "47s/datadir=/datadir=\/opt\/magima\/mysql\/data/g" /etc/init.d/mysql.server') exec_command("/etc/init.d/mysql.server start") exec_command("/etc/init.d/mysql.server status") exec_command("/etc/init.d/mysql.server stop") logger.info("==============TOMCAT DEFAULT CONFIG==============") logger.info("copy tomcat bootscript to /etc/init.d/") exec_command("cp " + tomcat_bootstrap + " /etc/init.d/tomcat6") exec_command("sudo /etc/init.d/tomcat6 start") exec_command("sudo /etc/init.d/tomcat6 status") exec_command("sudo /etc/init.d/tomcat6 stop") except OSError , oserr: logger.error("os error: %s " % str(oserr)) sys.exit(1)
def load_data(self, data_path, test_size=0.15): logger.info("Loading data.") try: data = pickle.load(open(data_path, "rb")) logger.debug(f"Loaded {len(data)} samples.") except Exception as err: # TODO Exception logger.error(err) exit() X = [] y = [] for feature, label in data: X.append(feature) y.append(label) X = np.array( X).reshape(-1, app_cfg['img_size'], app_cfg['img_size'], 3) X = X/255.0 y = keras.utils.to_categorical(y, num_classes=None) logger.info('Splitting dataset.') # TODO find another way to split dataset, using too much memory. self.train_data, self.test_data, self.train_label, self.test_label = train_test_split( X, y, test_size=test_size, random_state=1) logger.info( f'Data load complete. (Train_data:{len(self.train_data)}; Test_data:{len(self.test_data)})')
def parse_blocks(self): lineno, interval, intervals = 0, {}, [] def block_ends(interval, intervals): interval['lineno'] = lineno intervals.append(interval) # iterator for MULTILINES_PATTERN mp_iter = CycleIterator(self.__pack(TextgridBlocksParser.PATTERN_KEYS, TextgridBlocksParser.MULTILINES_PATTERN)) # iterator for BLOCK_PATTERN bp_iter = CycleIterator(self.__pack(TextgridBlocksParser.PATTERN_KEYS, TextgridBlocksParser.BLOCK_PATTERN)) line_pattern = bp_iter.next() for line in self.lines: lineno += 1 # always try to match the begining pattern at first to avoid missing a normal block # therefore, reset the block parsing once a line was matched to the begining pattern # but unmatched to the current one. if not bp_iter.begins() and self.__match(bp_iter.head(), line): logger.error('unable to parse line %d, ignored' % (lineno-1)) interval, line_pattern = {}, bp_iter.reset() # to match the pattern one by one until it ends if self.__match(line_pattern, line): self.__update(interval, line_pattern, line) # if the end of block was matched # block ends here for most situation if bp_iter.ends(): block_ends(interval, intervals) interval = {} # when a text existed in multiple lines elif bp_iter.ends(): # match the begining of text in multi-lines if self.__match(mp_iter.head(), line): self.__update(interval, mp_iter.head(), line) continue # should not to call the next block pattern # match the pattern of end line # block also may end here for multiple lines elif self.__match(mp_iter.tail(), line): self.__update(interval, mp_iter.tail(), line, append=True) block_ends(interval, intervals) interval = {} # match the pattern without quotes else: # append the middle part of the text self.__update(interval, mp_iter.index(1), line, append=True) continue else: # does not match anything # logger.error('unable to parse line %d, ignored' % (lineno-1)) continue line_pattern = bp_iter.next() # match the next pattern return intervals
def weater(lat, lng, timezone, unit='celsius', weather_key_=weather_key): res = {} try: owm = pyowm.OWM(weather_key_) observation = owm.three_hours_forecast_at_coords(lat, lng) except Exception, e: logger.error("weater {0}".format(e.message))
def parse_nodes(self): node_list = [] for controller_node in self.topology[self.NODE]: node = {} prefix_array = [] node_dict = html_style(controller_node[self.NODE_ID]) attributes = controller_node[self.NODE_ATTRIBUTES] if self.PREFIX in attributes: for prefix in attributes[self.PREFIX]: prefix_array.append(prefix[self.PREFIX]) if self.ROUTER_ID in attributes: if self.NAME in attributes: node[self.NAME] = attributes[self.NAME] else: success, name = name_check(attributes[self.ROUTER_ID][0]) node[self.NAME] = name if success else node_dict[self.ROUTER] node[self.LOOPBACK] = attributes[self.ROUTER_ID][0] else: node[self.NAME] = node_dict[self.ROUTER] node[self.LOOPBACK] = "0.0.0.0" node[self.PREFIX] = prefix_array node[self.ID] = controller_node[self.NODE_ID] try: node[self.INTERFACE] = self.parse_interfaces(self.fetch_interfaces(node[self.NAME])) node_list.append(node) except HttpClientException as e: logger.error("Fetch interfaces for node '{}' failed.".format(node[self.NAME])) logger.exception(e.message) return node_list
def crypto_in(): if request.method == "POST": auth = request.form.get("ipn_auth") if auth != IPN_AUTH: abort(401) address = request.form.get("address") confirmed = float(request.form.get("confirmed")) unconfirmed = float(request.form.get("unconfirmed")) logger.warning("IPN address {} -- confirmed {} -- unconfirmed {}".format(address, confirmed, unconfirmed)) session = database.Session(autocommit=False) user = CryptoAddress.get_address_user(address=address, session=session) if user: logger.warning("IPN for user {} -- address {}".format(user, address)) dep = UserTransactions.get_user_netdeposits(user, session=session) if confirmed > dep: UserTransactions.add_transaction(user, confirmed-dep, "deposit", reference="IPN", session=session) logger.warning("Deposit confirmed! user {} -- address {} -- amount {}".format(user, address, confirmed-dep)) else: logger.warning("Confirmed is lower than net deposit yet. User {} -- address {} -- conf {} -- dep {} " .format(user, address, confirmed, dep)) else: logger.error("Address {} don't match any user.".format(address)) data = { "address": address, "confirmed": confirmed, "unconfirmed": unconfirmed, "ipn_auth": auth } return jsonify(data) else: logger.error("Invalid request at IPN url!") abort(401)
def process(self, raw_data): data, name = raw_data try: # cipher_text must be a multiple of 16, fill the margin with 0 data = data if len(data)%16==0 else ''.join([data, '0'*(16-len(data)%16)]) return (AESDecipherer.decipherer.decrypt(data), name) except ValueError, e: logger.error(u'unable to decrypt %s by the means of AES' % name)
def retrieve(self, *args, **kwargs): self.queue.join() try: for data in self.storage.retrieve(*args, **kwargs): for t in data: self.queue.put(t) except ArgumentsError, e: logger.error(e.msg)
def getMetadata(self, tag): elems = self._root.findall(tag) if not elems: logger.error("Tag %s is not valid!") return None elif len(elems) >1: logger.warning("Tag %s has more than one element (len = %d)! Returning first!"%(tag,len(elems))) return elems[0].text
def OnRspOrderAction(self, pInputOrderAction, pRspInfo, nRequestID, bIsLast): if not self.strategy: if pRspInfo.ErrorID != 0: logger.error(pRspInfo.ErrorMsg.decode('gb2312')) return self.strategy.on_rsp_order_action(pInputOrderAction, pRspInfo, nRequestID, bIsLast)
def close(self): try: self.conn.close() self.conn = None self.cursor = None except AttributeError, e: logger.error("connection closed already, invalid call") raise AttributeError
def save_to_db(self): """Call sqlalchemy methods to insert data into DB""" if(self.unit_price is not None): product = db_handler.DiaperDealsPipeline() product.process_item(self.map_deal_dict()) else: logger.error("Error processing item to database: no unit_price info!") logger.error("Link to product for debug: %s" % (self.url))
def process(self, raw_data): service_name, path = raw_data file_obj = StringIO() try: self.smb_conn.retrieve_file(service_name, path, file_obj) except OperationFailure, e: logger.error('unable to retrieve files in path %s' % path) raise e
def OnRspOrderInsert(self, pInputOrder, pRspInfo, nRequestID, bIsLast): if not self.strategy: if pRspInfo.ErrorID != 0: logger.error('{}, requestID: {}'.format( pRspInfo.ErrorMsg, nRequestID).decode('gb2312')) return self.strategy.on_rsp_order_insert(pInputOrder, pRspInfo, nRequestID, bIsLast)
def on_connection_closed(self, connection, reply_code, reply_text): self._channel = None if self._closing: self.connection.ioloop.stop() else: logger.error('lost connection...') logger.error('reconnect 5 seconds later...') self.connection.add_timeout(5, self.reconnect)
def retire(self): if self.id: upd_stat="UPDATE {0} SET alive=FALSE WHERE id={1}".format(self.table,self.id) # print upd_stat try: self.self_cursor.execute(upd_stat) except Exception as e: logger.error("Cannot retire table {0}. {1}".format(self.table,e.pgerror)) return
def check(*args, **kargs): uid, gid = os.getuid(), os.getgid() if uid != 0 and gid != 0: logger.error("please use root privilege to exec.") sys.exit(1) return func(*args, **kargs)
def exec_command(command): try: if os.system(command) != 0: logger.error("exec " + command + " error!") sys.exit(1) except OSError, oserr: logger.error(str(oserr)) sys.exit(1)
def get_price(self): current_price_list = [ p.text for p in self.s.select(settings.WELL['current_price_css_selector'])] try: assert(len(current_price_list) == 1) self.current_price = current_price_list[0].strip() except AssertionError as err: logger.error("Unable to get price information: %s" % (err), exc_info=True)