def run(self):
logger.info("{log_prefix} start to run".format(
log_prefix=self.log_prefix, model=self.name))
self.timer.start()
for metric in self.metrics:
if metric not in Metrics:
logger.error(
"{log_prefix}[metric:{metric}] is not supported".format(
log_prefix=self.log_prefix, metric=metric))
continue
val = Metrics[metric]
if metric not in self.cfg.metrics:
logger.error(
"{log_prefix}[metric:{metric}] can't found the config of this metric"
.format(log_prefix=self.log_prefix, metric=metric))
continue
self.report.metrics_report[metric] = []
t = Thread(target=self.run_action,
args=(metric, val, self.cfg.metrics[metric]))
t.start()
self.threads[metric] = t
self.status = MODEL_RUNNING
self.save_model()
def parse_config():
options.parse_command_line()
if options.config != "":
logger.info("parse config from config file: {config}".format(
config=options.config))
options.parse_config_file(options.config)
if options.slack_token == "":
logger.error("slack token is required!!")
sys.exit(1)
if options.models_path == "":
logger.error("models path is required!!")
sys.exit(1)
if options.db_path == "":
logger.info(
"path of database is not set, use the default db path: {path}".
format(path=DEFAULT_DB_PATH))
options.db_path = DEFAULT_DB_PATH
mg.MODELS_PATH = options.models_path
alert.SLACK_TOKEN = options.slack_token
job.REPORT_ADDRESS = options.address
logger.info("config: {config}".format(config=options.items()))
def run_job(self, job):
logger.info(
"add new job: job-{id} to tasks queue".format(id=job.get('id')))
try:
self.new_task(Task(START_JOB, job))
except Exception as e:
logger.error("add new job: {job} failed: {err}".format(job=job,
err=str(e)))
raise NewJobException(job, str(e))
def on_error(self, metric, rule):
logger.error(
"{log_prefix}[metric:{metric}] not match rule: {rule}".format(
log_prefix=self.log_prefix, metric=metric, rule=rule))
send_to_slack(
"{log_prefix}[model:{model}][metric:{metric}] not match rule: {rule}"
.format(log_prefix=self.log_prefix,
model=self.name,
metric=metric,
rule=rule), self.slack_channel)
def check_is_triggered(left_value, operator, right_value):
if operator not in OPERATORS:
logger.error("operator {} is invalid".format(operator))
return False
return {
"==": (lambda: abs(left_value - right_value) < EPS),
"!=": (lambda: abs(left_value - right_value) > EPS),
"<=": (lambda: left_value <= right_value),
">=": (lambda: left_value >= right_value),
"=": (lambda: abs(left_value - right_value) < EPS),
"<": (lambda: left_value < right_value),
">": (lambda: left_value > right_value)
}.get(operator, lambda: False)()
def start_job_handler(self, job):
logger.info("run job: {job}".format(job=job))
if job.get('id') in self.jobs.keys():
logger.warn("the job is running")
return
job_instance = Job(self.storage, job)
if not job_instance.valid():
logger.error("the job is invalid")
return
self.jobs[job.get('id')] = job_instance
t = Thread(target=job_instance.run)
t.start()
def get(self, job_id):
if job_id == "":
logger.error("job id is required")
self.finish({
"code": HTTP_MISS_ARGS,
"message": "job id is required"
})
return
try:
logger.info("close running job:{job_id}".format(job_id=job_id))
manager.stop_job(int(job_id))
except JobNotExistsException:
logger.error(
"close job:{job_id} failed: job is not exist".format(
job_id=job_id))
logger.exception("Exception Logged")
data = {
"code":
HTTP_FAIL,
"message":
"close job:{job_id} failed: job is not exist".format(
job_id=job_id)
}
except JobNotRunningException:
logger.error(
"close job:{job_id} failed: job is not running".format(
job_id=job_id))
logger.exception("Exception Logged")
data = {
"code":
HTTP_FAIL,
"message":
"close job:{job_id} failed: job is not running".format(
job_id=job_id)
}
except Exception as e:
logger.error("close job:{job_id} failed:{err}".format(
job_id=job_id, err=str(e)))
logger.exception("Exception Logged")
data = {
"code":
HTTP_FAIL,
"message":
"close job:{job_id} failed:{err}".format(job_id=job_id,
err=str(e))
}
else:
data = {"code": HTTP_OK, "message": "OK"}
finally:
self.finish(data)
def parse_rule(rules):
rules_list = []
for rule in rules:
match = re.match(FEATURE_RULE, rule)
if match:
try:
value = float(match.group(3))
except Exception as e:
logger.error("parse rule: {rule} failed, error: {err}"
.format(rule=rule, err=str(e)))
raise ParseRuleException(rule=rule)
rules_list.append({"expr": rule, "feature": match.group(1),
"operator": match.group(2),
"value": value})
else:
logger.error("parse rule: {rule} failed"
.format(rule=rule))
raise ParseRuleException(rule=rule)
return rules_list
def post(self):
try:
data = json.loads(self.request.body)
except Exception as e:
logger.error("load json: {json_data}\nfailed: {err}".format(
json_data=self.request.body, err=str(e)))
logger.exception("Exception Logged")
self.finish({
"code":
HTTP_FAIL,
"message":
"load json data failed: {err}".format(err=str(e))
})
return
slack_channel = data.get("slack_channel", DEFAULT_CHANNEL)
timeout = data.get("timeout", DEFAULT_JOB_TIMEOUT)
try:
job = manager.new_job(data["data_source"], data["models"],
slack_channel, timeout)
except KeyError as e:
logger.error("add new job failed: miss args %s" % e.args[0])
logger.exception("Exception Logged")
data = {
"code": HTTP_MISS_ARGS,
"message": "miss args %s" % e.args[0]
}
except NewJobException as e:
logger.error("add new job failed: {err}".format(err=str(e)))
logger.exception("Exception Logged")
data = {
"code": HTTP_FAIL,
"message": "add new job failed: {err}".format(err=str(e))
}
except Exception as e:
logger.error("add new job failed: {err}".format(err=str(e)))
logger.exception("Exception Logged")
data = {
"code": HTTP_FAIL,
"message": "add new job failed: {err}".format(err=str(e))
}
else:
data = {"code": HTTP_OK, "message": "OK", "data": job}
finally:
self.finish(data)
def get(self, job_id):
if job_id == "":
logger.error("job id is required")
self.finish({
"code": HTTP_MISS_ARGS,
"message": "job id is required"
})
return
try:
logger.info("get job:{job_id} detail".format(job_id=job_id))
job = manager.get_job(int(job_id))
except JobNotExistsException:
logger.error(
"get job:{job_id} detail failed: job is not exist".format(
job_id=job_id))
logger.exception("Exception Logged")
data = {
"code":
HTTP_NOT_FOUND,
"message":
"get job:{job_id} failed: job is not exist".format(
job_id=job_id)
}
except Exception as e:
logger.error("get job:{job_id} detail failed:{err}".format(
job_id=job_id, err=str(e)))
logger.exception("Exception Logged")
data = {
"code":
HTTP_FAIL,
"message":
"get job:{job_id} detail failed:{err}".format(
job_id=job_id, err=str(e))
}
else:
data = {"code": HTTP_OK, "message": "OK", "data": job}
finally:
self.finish(data)
