def write_snippet(snippet_id, snippet_name, snippet_src, snippet_descr,
snippet_when, job_path, job_id, user):
"""write python snippet"""
logger.debug('jobmodel - creating python snippet')
script = []
script.append("#!/usr/bin/python")
script.append("# -*- coding: utf-8 -*-")
script.append("")
script.append("def run_snippet(snippet):")
script.append(tab("import time"))
script.append(tab("import traceback"))
script.append(tab("from ea.automateactions.joblibrary import jobsnippet"))
script.append(tab("step_start_time = time.time()"))
script.append(tab('snippet.begin(description="%s")' % snippet_descr))
script.append(tab("try:"))
script.append(tab(write_snippet_import(snippet_id=snippet_id), nb_tab=2))
script.append(tab("snippet.done()", nb_tab=2))
script.append(tab("except jobsnippet.FailureException as e:"))
script.append(tab("snippet.error(message=e)", nb_tab=2))
script.append(tab("except Exception as e:"))
script.append(tab("tb = traceback.format_exc()", nb_tab=2))
script.append(tab("snippet.error(message=tb)", nb_tab=2))
script.append(tab("step_duration = time.time() - step_start_time"))
script.append(tab("snippet.ending(duration=step_duration)"))
with open(n("%s/snippet%s.py" % (job_path, snippet_id)), 'wb') as fd:
fd.write('\n'.join(script).encode('utf-8'))
write_snippet_code(snippet_id=snippet_id,
snippet_src=snippet_src,
job_path=job_path)
def kill(self):
"""kill the job"""
logger.debug("jobprocess - kill the job")
if self.process_id is None:
return constant.NOT_FOUND
success = constant.ERROR
if platform.system() == "Windows":
kill_cmd = ["taskkill"]
kill_cmd.append("/PID")
kill_cmd.append("%s" % self.process_id)
kill_cmd.append("/F")
p = subprocess.Popen(kill_cmd, stdout=subprocess.PIPE)
p.wait()
if not p.returncode:
success = constant.OK
else:
try:
os.kill(self.process_id, signal.SIGKILL)
success = constant.OK
except Exception as e:
logger.error("jobprocess - unable to kill %s" % e)
return success
def add_workspace(self, name):
"""create a new workspace"""
logger.debug("workspacesmanager - add workspace name=%s" % name)
wrk_path = "%s/%s" % (self.workspaces_path, name)
if os.path.exists(n(wrk_path)):
return (constant.ERROR, "workspace name must be unique")
# checking in cache if the name is free
if self.search_workspace(name=name):
return (constant.ALREADY_EXISTS,
"workspace name must be unique in cache")
# create main workspace folder
os.mkdir(n(wrk_path))
# create sub folders
snippets_path = "%s/snippets/" % wrk_path
actions_path = "%s/actions/" % wrk_path
os.mkdir(n(snippets_path))
os.mkdir(n(actions_path))
# add workspaces in the cache and save it
self.cache["workspaces"].append(name)
# save cache to file
self.save_workspaces()
return (constant.OK, "success")
def stop(self):
"""stop the scheduler"""
self.mutex.acquire()
logger.debug("scheduler - stopping scheduler")
self.running = False
self.event.set()
self.mutex.release()
def get_session(self, sess_id):
"""get session"""
logger.debug("sessionsmanager - get session id=%s" % sess_id)
if sess_id in self.sessions:
return self.sessions[sess_id]
return None
def set_state(self, state):
"""set state"""
logger.debug("jobprocess - state update %s" % state)
self.job_state = state
executionstorage.update_status(job_id=self.job_id,
status=self.to_dict())
def get_job(self, job_id):
"""Returns the job corresponding to the id
passed as argument, otherwise None"""
logger.debug("jobsmanager - get job (id=%s)" % job_id)
for job in self.jobs:
if job.job_id == job_id:
return job
return None
def gensalt(self):
"""generate salt"""
if settings.cfg['security']["salt"] is None:
logger.debug("sessionsmanager - generate salt")
settings.cfg['security']["salt"] = hexlify(os.urandom(20)).decode("utf8")
settings.save()
# generate hash password for all users
self.genpass_session()
def write_snippet_code(snippet_id, snippet_src, job_path):
"""create python snippet source"""
logger.debug('jobmodel - inject python source')
script = []
script.append("def run_snippet_code(snippet):")
script.append(tab(snippet_src))
with open(n("%s/snippet%s_code.py" % (job_path, snippet_id)), 'wb') as fd:
fd.write('\n'.join(script).encode('utf-8'))
def run(self):
"""
Run server
"""
logger.debug("restapi - server started")
try:
while not self.stop_event.isSet():
self.httpd.serve_forever()
except Exception as e:
logger.error("restapi - exception: " + str(e))
logger.debug("restapi - server stopped")
def cancel(self):
"""cancel the result"""
logger.debug("jobprocess - cancel the job")
# remove the reset storage
executionstorage.reset_storage(job_id=self.job_id)
# delete job from disk for recursive one
self.delete()
return (constant.OK, "success")
def get_user(self, login):
"""get user"""
logger.debug("usersmanager - get user=%s" % login)
# search the user
u = self.search_user(login=login)
if u is None:
error_str = "user=%s does not exist" % login
logger.error('usersmanager - %s' % error_str)
return (constant.NOT_FOUND, error_str)
return (constant.OK, [ u ])
def genpass_session(self):
"""generate session hash secret if the salt was generated"""
for u in self.cache["users"]:
# { "login": { "role": ..., "secrets": {...} } }
login = list(u.keys())[0]
logger.debug("sessionsmanager - generate password session for user=%s" % login)
u[login]["secrets"]["session"] = self.genpass_hash(password=u[login]["secrets"]["session"])
# save changes to file
self.save_users()
def search_user(self, login):
"""search user by login"""
logger.debug("usersmanager - search in "
"cache login=%s" % login)
found = None
for u in self.cache["users"]:
(l, _) = list(u.items())[0]
if l == login:
found = u
break
return found
def create_pyjob_runner(job_yaml, job_path, job_id, workspace, user):
"""create python job runner"""
logger.debug('jobmodel - creating python job runner')
# loading globals variables
globals_file = '%s/%s/%s/globals.yml' % (settings.get_app_path(
), settings.cfg['paths']['workspaces'], workspace)
globals_valid, yaml_globals = load_yamlfile(yaml_file=globals_file,
workspace=workspace,
user=user,
repo=constant.REPO_WORKSPACES)
if globals_valid != constant.OK:
logger.error('jobmodel - invalid globals variables')
return (constant.ERROR, {})
script = []
script.append("#!/usr/bin/python")
script.append("# -*- coding: utf-8 -*-")
script.append("")
script.append("import sys")
script.append("import os")
script.append("import time")
script.append("import json")
script.append("import traceback")
script.append("")
script.append("p = os.path.dirname(os.path.abspath(__file__))")
script.append("root_path = os.sep.join(p.split(os.sep)[:-5])")
script.append("sys.path.insert(0, root_path)")
script.append("")
script.append("from ea.automateactions.joblibrary import jobtracer")
script.append("from ea.automateactions.joblibrary import jobhandler")
script.append("from ea.automateactions.joblibrary import jobsnippet")
script.append("from ea.automateactions.joblibrary import datastore")
script.append("")
script.append("jobtracer.initialize(result_path=p)")
script.append("")
script.append("sys.stderr = jobtracer.StdWriter(mode_err=True)")
script.append("sys.stdout = jobtracer.StdWriter()")
script.append("")
script.append("jobhandler.initialize(globals=%s)" % yaml_globals)
script.append("datastore.initialize()")
script.append("")
script.append(write_snippets(job_path, job_yaml, job_id, workspace, user))
script.append("")
script.append("jobhandler.instance().start()")
script.append("jobhandler.finalize()")
script.append("ret_code = jobhandler.get_retcode()")
script.append("sys.exit(ret_code)")
with open(n("%s/jobrunner.py" % job_path), 'wb') as fd:
fd.write('\n'.join(script).encode('utf-8'))
return (constant.OK, "success")
def load_yamlstr(yaml_str):
"""open and read yaml string"""
logger.debug('jobmodel - loading yaml from string')
try:
yaml_job = yaml.safe_load(yaml_str)
except Exception as e:
error_str = "yaml loading error - %s" % e
logger.error('jobmodel - %s' % error_str)
return (constant.ERROR, error_str)
return (constant.OK, yaml_job)
def remove_event(self, event):
"""remove event from queue"""
self.mutex.acquire()
try:
if self.queue:
logger.debug("scheduler - remove event")
self.queue.remove(event)
heapq.heapify(self.queue)
del event
except Exception as e:
logger.error("scheduler - exception while "
"removing event %s: %s" % (event.ref, e) )
self.mutex.release()
def write_snippet_import(snippet_id):
"""write snippet python import"""
logger.debug('jobmodel - write python snippet import')
script = []
script.append("try:")
script.append(
tab("from snippet%s_code import run_snippet_code" % snippet_id))
script.append(tab("run_snippet_code(snippet=snippet)"))
script.append("except SyntaxError as err:")
script.append(tab("err.lineno = err.lineno - 1"))
script.append(tab("raise"))
return "\n".join(script)
def schedule_job(self, user, job_descr=None,
job_file=None, workspace="common",
sched_mode=0, sched_at=(0, 0, 0, 0, 0, 0),
sched_timestamp=0):
"""schedule a task to run an action"""
logger.debug("jobsmanager - schedule job")
# create the job
job = jobprocess.Job(job_mngr=self,
job_descr=job_descr,
job_file=job_file,
workspace=workspace,
sched_mode=sched_mode,
sched_at=sched_at,
user=user,
path_backups=self.path_bckps)
# prepare the job
success, details = job.init()
if success != constant.OK:
return (constant.ERROR, details)
success, details = job.build()
if success != constant.OK:
return (constant.ERROR, details)
# init start time of the job
if sched_timestamp > 0:
job.sched_timestamp = sched_timestamp
else:
job.init_start_time()
# save the job on the disk
success, details = job.save()
if success != constant.OK:
return (constant.ERROR, details)
# Register the job on the scheduler
logger.info("jobsmanager - adding job %s in scheduler" % job.job_id)
success, details = scheduler.add_event(ref=job.job_id,
timestamp=job.sched_timestamp,
callback=self.execute_job,
job=job)
if success != constant.OK:
return (constant.ERROR, "scheduler error")
job.set_event(event=details)
self.jobs.append(job)
return (constant.OK, job.job_id)
def delete(self):
"""delete the job from disk"""
logger.debug("jobprocess - delete the job from disk")
if self.is_recursive():
p = "%s/%s.json" % (self.path_backups, self.job_id)
p = os.path.normpath(p)
try:
os.remove(p)
except Exception:
pass
return (constant.OK, "success")
def save(self):
"""save the job on the disk"""
logger.debug("jobprocess - save the job on disk")
if self.is_recursive():
p = "%s/%s.json" % (self.path_backups, self.job_id)
p = os.path.normpath(p)
with open(p, "w") as fh:
job_dict = self.to_dict()
job_dict["job-file"] = self.job_file
job_dict["job-descr"] = self.job_descr
fh.write("%s" % json.dumps(job_dict))
return (constant.OK, "success")
def build(self):
"""build the job"""
logger.debug("jobprocess - build python job")
success, details = jobmodel.create_pyjob(yaml_file=self.job_file,
yaml_str=self.job_descr,
workspace=self.workspace,
user=self.user,
job_id=self.job_id)
if success != constant.OK:
return (constant.ERROR, details)
# all is OK
return (constant.OK, "success")
def get_entries(self, workspace):
"""get all entries according to the project id provided"""
logger.debug("globals - get entries")
env_file = '%s/%s/globals.yml' % (self.workspaces_path, workspace)
entries = ""
try:
with open( n(env_file) ) as f:
entries = f.read()
except FileNotFoundError:
logger.error("globals - globals file missing for workspace=%s" % workspace)
return (constant.OK, entries)
def tree_init(self):
"""init tree folders"""
logger.debug("workspacesmanager - creating folders if missing ")
for w in self.cache["workspaces"]:
# create the main folder if missing
wrk_path = "%s/%s" % (self.workspaces_path, w)
if not os.path.exists(n(wrk_path)):
os.mkdir(n(wrk_path))
snippets_path = "%s/snippets/" % wrk_path
actions_path = "%s/actions/" % wrk_path
os.mkdir(n(snippets_path))
os.mkdir(n(actions_path))
def do_ldap_auth(self, login, password):
"""do ldap auth"""
logger.debug("sessionsmanager - do ldap auth for login=%s" % login)
if ldap3 is None:
logger.error("auth failed - ldap library missing")
return False
# get ldap settings
ldap_host_list = settings.cfg['ldap']['host']
ldap_dn_list = settings.cfg['ldap']['dn']
# define ldap server(s)
servers_list = []
for host in ldap_host_list:
use_ssl = False
ldap_port = 386
# parse the url to extract scheme host and port
url_parsed = urllib.parse.urlparse(host)
if url_parsed.scheme == "ldaps":
use_ssl = True
ldap_port = 636
if ":" in url_parsed.netloc:
ldap_host, ldap_port = url_parsed.netloc.split(":")
else:
ldap_host = url_parsed.netloc
server = ldap3.Server(ldap_host,
port=int(ldap_port),
use_ssl=use_ssl)
servers_list.append(server)
last_auth_err = ""
for bind_dn in ldap_dn_list:
c = ldap3.Connection(servers_list,
user=bind_dn % login,
password=password)
# perform the Bind operation
auth_success = c.bind()
last_auth_err = c.result
if auth_success:
break
if not auth_success:
logger.debug("sessionsmanager - %s" % last_auth_err)
return auth_success
def init(self):
"""init result storage"""
logger.debug("jobprocess - init result storage")
success, details = executionstorage.init_storage(job_id=self.job_id)
if success != constant.OK:
return (constant.ERROR, details)
# save job status in result
success, details = executionstorage.init_status(job_id=self.job_id,
status=self.to_dict())
if success != constant.OK:
return (constant.ERROR, details)
return (constant.OK, "success")
def genpass_api(self):
"""generate basic secret if not provided"""
new_secrets = False
for u in self.cache["users"]:
(login, p) = list(u.items())[0]
if p["secrets"]["basic"] is None:
logger.debug("sessionsmanager - generate password api for user=%s" % login)
api_secret = hexlify(os.urandom(20)).decode("utf8")
p["secrets"]["basic"] = api_secret
new_secrets = True
# save changes to file
if new_secrets:
self.save_users()
def init_start_time(self):
"""get timestamp of the start"""
logger.debug("jobprocess - init start time")
y, m, d, h, mn, s = self.sched_at
cur_dt = time.localtime()
if self.sched_mode == constant.SCHED_NOW:
timestamp = time.time()
if self.sched_mode == constant.SCHED_AT:
dt = datetime.datetime(y, m, d, h, mn, s, 0)
timestamp = time.mktime(dt.timetuple())
if self.sched_mode == constant.SCHED_DAILY:
next_dt = datetime.datetime(cur_dt.tm_year, cur_dt.tm_mon,
cur_dt.tm_mday, h, mn, s, 0)
timestamp = time.mktime(next_dt.timetuple())
if self.sched_mode == constant.SCHED_HOURLY:
next_dt = datetime.datetime(cur_dt.tm_year, cur_dt.tm_mon,
cur_dt.tm_mday, cur_dt.tm_hour, mn, s,
0)
timestamp = time.mktime(next_dt.timetuple())
if self.sched_mode == constant.SCHED_EVERY_X:
next_dt = datetime.datetime(cur_dt.tm_year, cur_dt.tm_mon,
cur_dt.tm_mday, cur_dt.tm_hour,
cur_dt.tm_min, cur_dt.tm_sec, 0)
next_dt += datetime.timedelta(hours=h, minutes=mn, seconds=s)
timestamp = time.mktime(next_dt.timetuple())
if self.sched_mode == constant.SCHED_WEEKLY:
next_dt = datetime.datetime(cur_dt.tm_year, cur_dt.tm_mon,
cur_dt.tm_mday, h, mn, s, 0)
delta = datetime.timedelta(days=1)
while next_dt.weekday() != d:
next_dt = next_dt + delta
timestamp = time.mktime(next_dt.timetuple())
self.sched_timestamp = timestamp
# pershap the timestamp is too old
# compute the next start time
# only for recursive jobs
if self.is_recursive():
if timestamp < time.time():
self.sched_timestamp = self.get_next_start_time()
def update_role(self, login, role):
"""update role user"""
logger.debug('usersmanager - update role for user=%s' % login)
# search the user
u = self.search_user(login=login)
if u is None:
error_str = "user=%s does not exist" % login
logger.error('usersmanager - %s' % error_str)
return (constant.NOT_FOUND, error_str)
# update user in cache and save changes to file
u[login]["role"] = role
self.save_users()
return (constant.OK, "success")
def load_workspaces(self):
"""load workspaces"""
wrk_file = '%s/data/workspaces.yml' % (settings.get_app_path())
if not os.path.isfile(n(wrk_file)):
raise Exception("yaml workspaces file doesn't exist in data/")
# load yaml
with open(wrk_file, 'r') as fd:
wrks_str = fd.read()
try:
self.cache = yaml.safe_load(wrks_str)
except Exception as e:
raise Exception("bad yaml workspaces file provided: %s" % e)
logger.debug("workspacesmanager - workspaces cache "
"nb items: %s" % len(self.cache["workspaces"]))
