def on_failure(self, exc, task_id, args, kwargs, einfo): super(EmailTask, self).on_failure(exc, task_id, args, kwargs, einfo) logger.info('Task Failed : Task Name - {0}, args - {1}, id - {2}, stack_trace - {3}'.format( self.request.task, args, self.request.id, einfo )) email = (json.loads(args[0])).get('email') user_id = session.query(User.id).filter(User.email == email).first() if not user_id: raise ValueError("Enter-Valid-Email") try: celery_task_obj = CeleryTask( user_id=int(user_id[0]), task_module=self.request.task, payload_data=email, task_status='EXCEPTION', started_on=datetime.datetime.now(), task_id=self.request.id, ) session.add(celery_task_obj) session.commit() except Exception as e: logger.info("Insertion Failed with - {}".format(str(e))) session.rollback() session.remove()
def send_email(self, url, data): sender = self._smtp_settings['user'] message = MIMEMultipart("alternative") message['Subject'] = f'Data from {url}' message['From'] = sender text_list = [f"Hi. There is a parsed result from {url}:\n"] html_list = [ f"<html><body><p>Hi. There is a parsed result from {url}:</p><br/><ul>" ] for d in data: text = d.get_text() href = d.get_href() text_list.append(f"{text} - {href}\n") html_list.append(f'<li><a href="{href}">{text}</a></li>') html_list.append("</ul><br/><body><html>") part1 = MIMEText(''.join(text_list), "plain") part2 = MIMEText(''.join(html_list), "html") message.attach(part1) message.attach(part2) context = ssl.create_default_context() with smtplib.SMTP_SSL(self._smtp_settings['address'], self._smtp_settings['port'], context=context) as server: server.login(self._smtp_settings['user'], self._smtp_settings['password']) for recipient in self._recipients: server.sendmail(sender, recipient, message.as_string()) logger.info(f'Data was sent to {recipient}')
def server_status_check(self): status = self.svc.request_status & self.svc.connect_status & self.svc.listen_status if status is True: logger.info('{} status : OK\n'.format(self.svc.name)) else: warning_request(self.svc.name, 0) logger.warning('{} status : FAIL\n'.format(self.svc.name))
def on_success(self, retval, task_id, args, kwargs): super(EmailTask, self).on_success(retval, task_id, args, kwargs) logger.info('Task Completed Successfully : Task Name - {0}, args - {1}, id - {2}'.format( self.request.task, args, self.request.id )) email = (json.loads(args[0])).get('email') user_id = session.query(User.id).filter(User.email == email).first() if not user_id: raise ValueError("Enter-Valid-Email") try: celery_task_obj = CeleryTask( user_id=int(user_id[0]), task_module=self.request.task, payload_data=email, task_status='COMPLETED', started_on=datetime.datetime.now(), task_id=self.request.id, ) session.add(celery_task_obj) session.commit() except Exception as e: logger.info("Insertion Failed with - {}".format(str(e))) session.rollback() session.remove()
def search_question(self, question=''): """ 在问题库中查询一个问题 :param question: :return: """ try: if not self.db: logger.info("没有主动链接MySQL数据库") return {} try: self.db.close() # 必须要强行关闭数据库,不然查询的数据不是最新的 except pymysql.err.Error: logger.info("数据库已经关闭") self.db.ping() # 检测数据连接是否正常 sql = self.gen_sql(question) # logger.info("sql: {}".format(sql)) with self.db.cursor() as cursor: cursor.execute(sql) result = cursor.fetchone() or {} except Exception as e: logger.info("出错:{}".format(e)) logger.info("错误详情:{}".format(traceback.format_exc())) result = {} logger.info("问题`{}`的查询结果: {}".format(question, result)) return result
def __init__(self, config: Config): self._scheduler = Scheduler() self._email_notification = EmailNotification(config.get('smtp'), config.get('recipients')) for site in config.get('sites'): self._scheduler.every_minutes(Site(site), self.parse) logger.info(f"Will be parsing {len(config.get('sites'))} site(s)")
def start(): if validate_properties(config.get_properties()): logger.info('Starting application') parser = Parser(config) parser.run() else: logger.info('Something went wrong')
def train(self): params = filter(lambda p: p.requires_grad, self.model.parameters()) optimizer = Optimizer(params, args) patient = 0 best_dev_acc, best_test_acc = 0, 0 for ep in range(1, self.args.epoch + 1): train_loss, train_acc = self.train_iter(ep, self.train_set, optimizer) dev_acc = self.eval(self.val_set) if dev_acc > best_dev_acc: best_dev_acc = dev_acc test_acc = self.eval(self.test_set) if test_acc > best_test_acc: best_test_acc = test_acc patient = 0 else: patient += 1 logger.info( '[Epoch %d] train loss: %.4f, lr: %f, Train ACC: %.4f, Dev ACC: %.4f, Best Dev ACC: %.4f, Best Test ACC: %.4f, patient: %d' % (ep, train_loss, optimizer.get_lr(), train_acc, dev_acc, best_dev_acc, best_test_acc, patient)) if patient >= args.patient: break logger.info('Final Test ACC: %.4f' % best_test_acc)
def train_iter(self, ep, train_set, optimizer): t1 = time.time() train_acc, train_loss = 0., 0. train_loader = DataLoader(train_set, batch_size=self.args.batch_size, shuffle=True) self.model.train() for i, batcher in enumerate(train_loader): batch = batch_variable(batcher, self.vocabs) batch.to_device(self.args.device) pred = self.model(batch.x, batch.nx, batch.ew) loss = F.nll_loss(pred, batch.y) loss.backward() nn_utils.clip_grad_norm_(filter(lambda p: p.requires_grad, self.model.parameters()), max_norm=args.grad_clip) optimizer.step() self.model.zero_grad() loss_val = loss.data.item() train_loss += loss_val train_acc += (pred.data.argmax(dim=-1) == batch.y).sum().item() logger.info( '[Epoch %d] Iter%d time cost: %.2fs, lr: %.6f, train acc: %.4f, train loss: %.4f' % (ep, i + 1, (time.time() - t1), optimizer.get_lr(), train_acc / len(train_set), loss_val)) return train_loss / len(train_set), train_acc / len(train_set)
def do_GET(self): uuid = self.clean_path() logger.info("Searching for DR log with ID [ {} ]".format(uuid)) self.send_response(202) self.send_header(Handler.CONTENT_TYPE_HEADER, Handler.CONTENT_TYPE_JSON) self.end_headers() self.wfile.write(json.dumps(self.get_server().get_json_with_id(uuid)).encode('utf-8'))
def save_dataset_df(self, df): df.drop(columns=['id'], inplace=True) df.to_csv(cfg['paths']['data_root'] + self.dataset_info['name'] + "_dataframe.csv", index_label='id') logger.info("Generated dataframe and stored at " + cfg['paths']['data_root'])
def __init__(self, host='', user='', password='', db='', port=3306): logger.info("链接数据库: {}".format((host, port, db, user))) self.__host = host self.__port = int(port) self.__user = user self.__db = db self.__password = password self.reconnect()
def sell(self, tick: Tick): for key, order in enumerate(self.orders): if order["tick"].instrument_id == tick.instrument_id: self.balance += tick.price * order["count"] self.balance -= self.fee self.orders.pop(key) logger.info("Sell: {}. Balance: {} Count orders: {}".format(tick, self.balance, len(self.orders))) return True
def buy(self, tick: Tick, count: int): if tick.price*count > (self.balance - self.fee): return False self.balance -= self.fee self.balance -= tick.price*count self.orders.append({"count": count, "tick": tick}) logger.info("Buy: {}. Balance: {}. Count orders: {}".format(tick, self.balance, len(self.orders))) return True
def main(): logger.info("Run githubImporter") args = checkParams() importer = GithubImporter(args.url, args.dateFormat, args.dest) importer.setBroker(RabbitMq(conf["broker"])) importer.setCallURL() importer.work() logger.info("End githubImporter")
def save_label_text_to_label_id_dict(self, label_text_to_label_id): with open( cfg['paths']['data_root'] + self.dataset_info['name'] + "_label_text_to_label_id.json", "w") as f: json_dict = json.dumps(label_text_to_label_id) f.write(json_dict) logger.info("Generated Label to ID mapping and stored at " + cfg['paths']['data_root'])
def validate_properties(properties): try: main_schema(properties) logger.info('Properties JSON schema is valid') return True except MultipleInvalid as ex: logger.error(ex) return False
def memory_check(): try: mem = Memory() percent_metric = float(config.get("memory", "percent")) if mem.percent > percent_metric: logger.warning(memory_usage_warning + 'memory usage: {:.2f} %\n'.format(mem.percent)) logger.info(mem) except Exception as e: logger.error(traceback.format_exc())
def io_check(): try: io = InputOutput() await_metric = float(config.get("input-output", "await")) util_metric = float(config.get("input-output", "util")) if io.await > await_metric: logger.warning(io_delay_warning + 'time: {} ms\n'.format(io.await)) if io.util > util_metric: logger.warning(io_queue_warning + 'io queue usage : {} %\n'.format(io.util)) logger.info(io)
def getEventStatus(self): logger.info('Collecting event status') slug = 'event-status/' formattedSlug = slug completedUrl = self.getCompleteUrl(formattedSlug) logger.info('Request for event status : {}'.format(completedUrl)) r = requests.get(completedUrl) responseData = r.json() return responseData
def save_graphs(self, path, graphs, labels_dict=None): if labels_dict is not None: labels_dict_tensor = { "glabel": torch.tensor(labels_dict["glabel"]) } else: labels_dict_tensor = None graph_utils.save_dgl_graphs(path, graphs, labels_dict_tensor) logger.info(f"Saving {path}")
def getFixtures(self, gwNum): logger.info('Collecting fixture information') slug = 'fixtures/?event={}' formattedSlug = slug.format(gwNum) completedUrl = self.getCompleteUrl(formattedSlug) logger.info('Request for event status : {}'.format(completedUrl)) r = requests.get(completedUrl) responseData = r.json() return responseData
def __init__(self, name, address='127.0.0.1'): self.name = name self.address = address self.pid = 1 self.port = 0 self.port_check_cmd = "ss -tunlp | grep {}".format(self.name) self.process_status = False self.listen_status = False self.connect_status = False self.request_status = False logger.info('Checking {} status'.format(self.name))
def getGWPlayerPick(self, teamId, gw=1): logger.info('Collecting player picks for {}, {}'.format(id, gw)) slug = 'entry/{}/event/{}/picks/' formattedSlug = slug.format(teamId, gw) completedUrl = self.getCompleteUrl(formattedSlug) logger.info('Request for player picks : {}'.format(completedUrl)) r = requests.get(completedUrl) responseData = r.json() # logger.info('Collected response for classic league standings : {}'.format(responseData)) return responseData
def port_request_check(self): try: s = socket.socket() # logger.info('Attempting to connect to %s on port %s' % (self.name, self.port)) try: s.connect((self.address, self.port)) logger.info("Connected to %s on port %s" % (self.name, self.port)) return True except socket.error as e: logger.error("Connected to %s on port %s failed: %s" % (self.name, self.port, e)) return False except Exception as e: logger.error(traceback.format_exc())
def getStaticDataBootstrap(self): if self._staticFPLBootstrapData is not None: return self._staticFPLBootstrapData logger.info('Collecting bootstrap static data') slug = 'bootstrap-static/' formattedSlug = slug completedUrl = self.getCompleteUrl(formattedSlug) logger.info('Request for bootstrap static data : {}'.format(completedUrl)) r = requests.get(completedUrl) responseData = r.json() # logger.info('Collected response for bootstrap static data : {}'.format(responseData)) self._staticFPLBootstrapData = responseData return self._staticFPLBootstrapData
def port_listen_check(self): try: result = subprocess.Popen(self.port_check_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) line = result.stdout.readline() if line and 'LISTEN' in line: self.port = int(line.split()[4].split(':')[-1]) logger.info('{} is listening at {}'.format( self.name, self.port)) return True else: logger.warning('{} port state is not listen'.format(self.name)) return False except Exception as e: logger.error(traceback.format_exc())
def api_check(self): try: logger.info('sending request to {}'.format(self.name)) cmd = '`which zkServer.sh` status' result = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) f = result.stdout.read() if 'Mode' in f: logger.info('get response from {} successfully'.format( self.name)) return True else: return False except Exception as e: logger.error(traceback.format_exc())
def process_check(self): try: cmd = 'ps -ef | grep {} | grep -v grep '.format(self.name) result = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) line = result.stdout.readline() if len(line) > 0: self.pid = line.split()[1] logger.info('{} is running with pid {}'.format( self.name, self.pid)) return True else: logger.error('{} is not running'.format(self.name)) return False except Exception as e: logger.error(traceback.format_exc())
def api_check(self): try: logger.info('sending request to {}'.format(self.name)) redis_login = config.redis_auth() cmd = "`which redis-cli` -p {} -a {} info".format( self.port, redis_login['password']) result = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) f = result.stdout.read() if 'Server' in f: logger.info('get response from {} successfully'.format( self.name)) return True else: return False except Exception as e: logger.error(traceback.format_exc())
def __init__(self, tree=None, input_tries=None): """参数tree,从字典中导入tries树; 参数input_tries, 可以是文件,但只限于json文件(tries树结构的dict,或者list) 或者tries树的dict数据 或者list数据""" # logger.info("初始化tries树") if tree is None: tree = {} self.tree = tree if not input_tries: pass elif isinstance(input_tries, (unicode, str)) and input_tries.startswith("NER_"): pass elif isinstance(input_tries, (unicode, str)) and input_tries.startswith("RE_PATTERN_"): pass elif isinstance(input_tries, (unicode, str)) and os.path.isfile(input_tries): with open(input_tries, encoding='utf8')as f: logger.info("加载文件:{}".format(input_tries)) tries = json.load(f) if isinstance(tries, dict): tree = tries self.tree = tree elif isinstance(tries, (list, set)): for word in tries: self.add(word) else: logger.info("输入数据文件有误") elif isinstance(input_tries, dict): tree = input_tries self.tree = tree elif isinstance(input_tries, (list, set)): for word in input_tries: self.add(word) else: logger.info("输入参数有误:{}".format(input_tries))
import tornado import tornado.ioloop import tornado.web import url_server.router.router_settings as settings from url_server.router.router import create_application from logger.logger import logger as logger # main.py is the main access point of the tornado app, to run the application, just run "python main.py" # What this will do is listen to port in the settings.py file, and then we can access the app using # http://localhost:settings.port on any browser, or using python requests library if __name__ == "__main__": # Set the application to listen to port 8888 application = create_application() application.listen(settings.port) # Get the current IOLoop currentIOLoop = tornado.ioloop.IOLoop.current() # Log the port that is listened logger.info("Started application on port:" + str(settings.port)) # Start the IOLoop currentIOLoop.start()