def schedule_job(self, user, job_descr=None,
job_file=None, workspace="common",
sched_mode=0, sched_at=(0, 0, 0, 0, 0, 0),
sched_timestamp=0):
"""schedule a task to run an action"""
logger.debug("jobsmanager - schedule job")
# create the job
job = jobprocess.Job(job_mngr=self,
job_descr=job_descr,
job_file=job_file,
workspace=workspace,
sched_mode=sched_mode,
sched_at=sched_at,
user=user,
path_backups=self.path_bckps)
# prepare the job
success, details = job.init()
if success != constant.OK:
return (constant.ERROR, details)
success, details = job.build()
if success != constant.OK:
return (constant.ERROR, details)
# init start time of the job
if sched_timestamp > 0:
job.sched_timestamp = sched_timestamp
else:
job.init_start_time()
# save the job on the disk
success, details = job.save()
if success != constant.OK:
return (constant.ERROR, details)
# Register the job on the scheduler
logger.info("jobsmanager - adding job %s in scheduler" % job.job_id)
success, details = scheduler.add_event(ref=job.job_id,
timestamp=job.sched_timestamp,
callback=self.execute_job,
job=job)
if success != constant.OK:
return (constant.ERROR, "scheduler error")
job.set_event(event=details)
self.jobs.append(job)
return (constant.OK, job.job_id)
def schedule_job(user, job_descr, job_file, workspace,
sched_mode, sched_at):
"""schedule a job"""
logger.info("scheduling new job "
"user=%s mode=%s at=%s" % (user["login"],
sched_mode,
sched_at) )
# all is ok, the schedule of the job can be done
return instance().schedule_job(user=user,
job_descr=job_descr,
job_file=job_file,
workspace=workspace,
sched_mode=sched_mode,
sched_at=sched_at)
def hup_handler(self, signum, frame):
"""
Hup handler
"""
logger.info('coreserver - reloading configuration...')
# reload settings ini
settings.finalize()
settings.initialize()
# reconfigure the level of log message
level = settings.cfg['log']['level']
logger.set_level(level=level)
# reload cache
UsersManager.instance().loadCache()
logger.info('coreserver - configuration reloaded!')
def reload_jobs(self):
"""reload recursive jobs"""
logger.info("jobsmanager - reloading jobs")
for fb in os.listdir(self.path_bckps):
# load the backup job file
with open( "%s/%s" % (self.path_bckps,fb), "r") as fh:
job = json.loads(fh.read())
# register the recursive backuped job
self.schedule_job(user=job["user"],
job_descr=job["job-descr"],
job_file=job["job-file"],
workspace=job["workspace"],
sched_mode=job["sched-mode"],
sched_at=job["sched-at"],
sched_timestamp=job["sched-timestamp"])
# remove old backup
try:
os.remove("%s/%s" % (self.path_bckps,fb))
except Exception as e:
pass
def delete_job(self, job_id, user):
"""kill or cancel a task"""
logger.info("jobsmanager - delete job (id=%s)" % job_id)
job = self.get_job(job_id=job_id)
if job is None:
return (constant.NOT_FOUND, 'job does not exists')
if user['role'] != constant.ROLE_ADMIN:
if job.user["login"] != user['login']:
return (constant.FORBIDDEN, 'access denied')
if job.job_state == constant.STATE_RUNNING:
logger.info("jobsmanager - killing job %s" % job.job_id)
job.kill()
if job.job_state == constant.STATE_WAITING:
logger.info("jobsmanager - cancelling job %s" % job.job_id)
job.cancel()
scheduler.remove_event(job.sched_event)
self.jobs.remove(job)
del job
return (constant.OK, 'job deleted')
def start(self):
"""start"""
if self.is_running():
logger.error("coreserver - server is already running")
sys.exit(1)
# run the server as daemon only for linux
if platform.system() == "Linux":
self.daemonize()
logger.info("coreserver - starting up server...")
try:
cliserver.initialize(coreserver=self)
logger.info("coreserver - cli [OK]")
workspacesmanager.initialize(workspaces_path=n(path_workspaces))
logger.info("coreserver - workspaces manager [OK]")
usersmanager.initialize()
logger.info("coreserver - users manager [OK]")
globalsmanager.initialize(workspaces_path=n(path_workspaces))
logger.info("coreserver - globals manager [OK]")
sessionsmanager.initialize()
logger.info("coreserver - sessions manager [OK]")
scheduler.initialize()
logger.info("coreserver - scheduler [OK]")
jobsmanager.initialize(path_bckps=n(path_backups))
logger.info("coreserver - jobs manager [OK]")
executionstorage.initialize(repo_path=n(path_results))
logger.info("coreserver - executions storage [OK]")
actionstorage.initialize(repo_path=n(path_actions))
logger.info("coreserver - actions storage [OK]")
snippetstorage.initialize(repo_path=n(path_snippets))
logger.info("coreserver - snippets storage [OK]")
bind_ip = settings.cfg['network']['api-bind-ip']
bind_port = settings.cfg['network']['api-bind-port']
restapi.initialize(bind_addr=(bind_ip, bind_port))
restapi.start()
logger.info("coreserver - rest api server [OK]")
jobsmanager.reload_jobs()
logger.info("coreserver - jobs reloaded [OK]")
except Exception:
tb = traceback.format_exc()
logger.error("coreserver - unable to start server: %s" % tb)
self.cleanup()
sys.exit(3)
msg_success = "server successfully started!"
logger.info(msg_success)
if platform.system() == "Windows":
print(msg_success)
self.run()
def run(self):
"""run thread"""
logger.debug("jobprocess - run the job")
# prepare next run if the job is recursive
if self.is_recursive():
# delete from disk the current backuped job
self.delete()
# register a new job with the same parameters
new_start_time = self.get_next_start_time()
self.job_mngr.schedule_job(user=self.user,
job_descr=self.job_descr,
job_file=self.job_file,
workspace=self.workspace,
sched_mode=self.sched_mode,
sched_at=self.sched_at,
sched_timestamp=new_start_time)
# keep the start time of the run
start_time = time.time()
# change state to running
self.set_state(state=constant.STATE_RUNNING)
# get python path according to the os
if platform.system() == "Windows":
executable = settings.cfg['paths']['python-windows']
else:
executable = settings.cfg['paths']['python-linux']
# prepare the path of the job
p = executionstorage.get_path(job_id=self.job_id)
job_path = "%s/jobrunner.py" % p
job_path = n(job_path)
jobtracer.initialize(result_path=n(p))
args = [executable]
args.append(job_path)
# run the job in a separate process
jobtracer.instance().log_job_started()
try:
p = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
self.process_id = p.pid
except Exception as e:
logger.error('jobprocess - unable to run job: %s' % e)
self.set_state(state=constant.STATE_FAILURE)
else:
# wait the process to complete
p.wait()
retcode = p.returncode
# compute the duration of the job
self.job_duration = time.time() - start_time
# set the final state of the job SUCCESS or FAILURE?
if retcode == 0:
self.set_state(state=constant.STATE_SUCCESS)
else:
err_str = p.stderr.read().decode("utf8")
if len(err_str):
jobtracer.instance().log_job_error(message=err_str)
self.set_state(state=constant.STATE_FAILURE)
job_result = constant.RETCODE_LIST.get(retcode, constant.STATE_FAILURE)
jobtracer.instance().log_job_stopped(result=job_result,
duration=self.job_duration)
jobtracer.finalize()
logger.info('jobprocess - job %s terminated' % self.job_id)
def execute_job(self, job):
"""execute the job"""
logger.info("jobsmanager - starting job %s" % job.job_id)
job_thread = threading.Thread(target=lambda: job.run())
job_thread.start()