def run(self): logger.info("{log_prefix} start to run".format( log_prefix=self.log_prefix, model=self.name)) self.timer.start() for metric in self.metrics: if metric not in Metrics: logger.error( "{log_prefix}[metric:{metric}] is not supported".format( log_prefix=self.log_prefix, metric=metric)) continue val = Metrics[metric] if metric not in self.cfg.metrics: logger.error( "{log_prefix}[metric:{metric}] can't found the config of this metric" .format(log_prefix=self.log_prefix, metric=metric)) continue self.report.metrics_report[metric] = [] t = Thread(target=self.run_action, args=(metric, val, self.cfg.metrics[metric])) t.start() self.threads[metric] = t self.status = MODEL_RUNNING self.save_model()
def parse_config(): options.parse_command_line() if options.config != "": logger.info("parse config from config file: {config}".format( config=options.config)) options.parse_config_file(options.config) if options.slack_token == "": logger.error("slack token is required!!") sys.exit(1) if options.models_path == "": logger.error("models path is required!!") sys.exit(1) if options.db_path == "": logger.info( "path of database is not set, use the default db path: {path}". format(path=DEFAULT_DB_PATH)) options.db_path = DEFAULT_DB_PATH mg.MODELS_PATH = options.models_path alert.SLACK_TOKEN = options.slack_token job.REPORT_ADDRESS = options.address logger.info("config: {config}".format(config=options.items()))
def run_job(self, job): logger.info( "add new job: job-{id} to tasks queue".format(id=job.get('id'))) try: self.new_task(Task(START_JOB, job)) except Exception as e: logger.error("add new job: {job} failed: {err}".format(job=job, err=str(e))) raise NewJobException(job, str(e))
def on_error(self, metric, rule): logger.error( "{log_prefix}[metric:{metric}] not match rule: {rule}".format( log_prefix=self.log_prefix, metric=metric, rule=rule)) send_to_slack( "{log_prefix}[model:{model}][metric:{metric}] not match rule: {rule}" .format(log_prefix=self.log_prefix, model=self.name, metric=metric, rule=rule), self.slack_channel)
def check_is_triggered(left_value, operator, right_value): if operator not in OPERATORS: logger.error("operator {} is invalid".format(operator)) return False return { "==": (lambda: abs(left_value - right_value) < EPS), "!=": (lambda: abs(left_value - right_value) > EPS), "<=": (lambda: left_value <= right_value), ">=": (lambda: left_value >= right_value), "=": (lambda: abs(left_value - right_value) < EPS), "<": (lambda: left_value < right_value), ">": (lambda: left_value > right_value) }.get(operator, lambda: False)()
def start_job_handler(self, job): logger.info("run job: {job}".format(job=job)) if job.get('id') in self.jobs.keys(): logger.warn("the job is running") return job_instance = Job(self.storage, job) if not job_instance.valid(): logger.error("the job is invalid") return self.jobs[job.get('id')] = job_instance t = Thread(target=job_instance.run) t.start()
def get(self, job_id): if job_id == "": logger.error("job id is required") self.finish({ "code": HTTP_MISS_ARGS, "message": "job id is required" }) return try: logger.info("close running job:{job_id}".format(job_id=job_id)) manager.stop_job(int(job_id)) except JobNotExistsException: logger.error( "close job:{job_id} failed: job is not exist".format( job_id=job_id)) logger.exception("Exception Logged") data = { "code": HTTP_FAIL, "message": "close job:{job_id} failed: job is not exist".format( job_id=job_id) } except JobNotRunningException: logger.error( "close job:{job_id} failed: job is not running".format( job_id=job_id)) logger.exception("Exception Logged") data = { "code": HTTP_FAIL, "message": "close job:{job_id} failed: job is not running".format( job_id=job_id) } except Exception as e: logger.error("close job:{job_id} failed:{err}".format( job_id=job_id, err=str(e))) logger.exception("Exception Logged") data = { "code": HTTP_FAIL, "message": "close job:{job_id} failed:{err}".format(job_id=job_id, err=str(e)) } else: data = {"code": HTTP_OK, "message": "OK"} finally: self.finish(data)
def parse_rule(rules): rules_list = [] for rule in rules: match = re.match(FEATURE_RULE, rule) if match: try: value = float(match.group(3)) except Exception as e: logger.error("parse rule: {rule} failed, error: {err}" .format(rule=rule, err=str(e))) raise ParseRuleException(rule=rule) rules_list.append({"expr": rule, "feature": match.group(1), "operator": match.group(2), "value": value}) else: logger.error("parse rule: {rule} failed" .format(rule=rule)) raise ParseRuleException(rule=rule) return rules_list
def post(self): try: data = json.loads(self.request.body) except Exception as e: logger.error("load json: {json_data}\nfailed: {err}".format( json_data=self.request.body, err=str(e))) logger.exception("Exception Logged") self.finish({ "code": HTTP_FAIL, "message": "load json data failed: {err}".format(err=str(e)) }) return slack_channel = data.get("slack_channel", DEFAULT_CHANNEL) timeout = data.get("timeout", DEFAULT_JOB_TIMEOUT) try: job = manager.new_job(data["data_source"], data["models"], slack_channel, timeout) except KeyError as e: logger.error("add new job failed: miss args %s" % e.args[0]) logger.exception("Exception Logged") data = { "code": HTTP_MISS_ARGS, "message": "miss args %s" % e.args[0] } except NewJobException as e: logger.error("add new job failed: {err}".format(err=str(e))) logger.exception("Exception Logged") data = { "code": HTTP_FAIL, "message": "add new job failed: {err}".format(err=str(e)) } except Exception as e: logger.error("add new job failed: {err}".format(err=str(e))) logger.exception("Exception Logged") data = { "code": HTTP_FAIL, "message": "add new job failed: {err}".format(err=str(e)) } else: data = {"code": HTTP_OK, "message": "OK", "data": job} finally: self.finish(data)
def get(self, job_id): if job_id == "": logger.error("job id is required") self.finish({ "code": HTTP_MISS_ARGS, "message": "job id is required" }) return try: logger.info("get job:{job_id} detail".format(job_id=job_id)) job = manager.get_job(int(job_id)) except JobNotExistsException: logger.error( "get job:{job_id} detail failed: job is not exist".format( job_id=job_id)) logger.exception("Exception Logged") data = { "code": HTTP_NOT_FOUND, "message": "get job:{job_id} failed: job is not exist".format( job_id=job_id) } except Exception as e: logger.error("get job:{job_id} detail failed:{err}".format( job_id=job_id, err=str(e))) logger.exception("Exception Logged") data = { "code": HTTP_FAIL, "message": "get job:{job_id} detail failed:{err}".format( job_id=job_id, err=str(e)) } else: data = {"code": HTTP_OK, "message": "OK", "data": job} finally: self.finish(data)