def get_preparing_jobs_cluster_nb():
"""
Get number of jobs in preparation step (for cluster runs)
:return: number of jobs
:rtype: int
"""
with Job.connect():
return len(Job.select().where(Job.status == "preparing-cluster")), \
len(Job.select().where(Job.status == "prepare-scheduled"))
def parse_database(app_data, max_age, fake=False):
from dgenies.database import Job, Gallery
gallery_jobs = []
with Job.connect():
old_jobs = Job.select().where(
((Job.status == "success") & (Job.date_created < datetime.now() -
timedelta(days=max_age["data"])))
| ((Job.status != "success") & (Job.date_created < datetime.now() -
timedelta(days=max_age["error"]))))
for job in old_jobs:
id_job = job.id_job
is_gallery = len(
Gallery.select().join(Job).where(Job.id_job == id_job)) > 0
if is_gallery:
gallery_jobs.append(id_job)
else:
print("Removing job %s..." % id_job)
data_dir = os.path.join(app_data, id_job)
if os.path.exists(data_dir) and os.path.isdir(data_dir):
if not fake:
shutil.rmtree(data_dir)
else:
print("Job %s has no data folder!" % id_job)
if not fake:
job.delete_instance()
return gallery_jobs
def get_preparing_jobs_nb():
"""
Get number of jobs in preparation step (for local runs)
:return: number of jobs
:rtype: int
"""
with Job.connect():
return len(Job.select().where(Job.status == "preparing"))
def get_scheduled_cluster_jobs():
all_jobs = []
with Job.connect():
jobs = Job.select().where((Job.batch_type != "local") & ((Job.status == "prepared") | (Job.status == "scheduled"))).\
order_by(Job.date_created)
for job in jobs:
all_jobs.append({"job_id": job.id_job, "batch_type": job.batch_type})
job.status = "scheduled"
job.save()
return all_jobs
def get_prep_scheduled_jobs():
"""
Get list of jobs ready to be prepared (all data is downloaded and parsed)
:return: list of jobs
:rtype: list
"""
with Job.connect():
jobs = Job.select().where(Job.status == "waiting").order_by(
Job.date_created)
return [(j.id_job, j.batch_type) for j in jobs]
def get_scheduled_local_jobs():
"""
Get list of jobs ready to be started (for local runs)
:return: list of jobs
:rtype: list
"""
all_jobs = []
with Job.connect():
jobs = Job.select().where((Job.batch_type == "local") & ((Job.status == "prepared") | (Job.status == "scheduled"))).\
order_by(Job.date_created)
for job in jobs:
all_jobs.append(job.id_job)
job.status = "scheduled"
job.save()
return all_jobs
def parse_started_jobs():
with Job.connect():
jobs_started = [] # Only local jobs
cluster_jobs_started = [] # Only cluster jobs
jobs = Job.select().where((Job.status == "started") | (Job.status == "starting") | (Job.status == "succeed") |
(Job.status == "merging") | (Job.status == "scheduled-cluster") |
(Job.status == "prepare-scheduled") | (Job.status == "prepare-cluster"))
for job in jobs:
pid = job.id_process
if job.batch_type == "local":
if job.status != "started" or psutil.pid_exists(pid):
jobs_started.append(job.id_job)
else:
print("Job %s (pid: %d) has died!" % (job.id_job, job.id_process))
job.status = "fail"
job.error = "<p>Your job has failed for an unexpected reason. Please contact the support.</p>"
job.save()
# Todo: send mail about the error
else:
if job.status in ["started", "scheduled-cluster", "prepare-scheduled", "preparing-cluster"]:
s = DRMAA_SESSION.session
status = s.jobStatus(str(job.id_process))
if status not in [drmaa.JobState.RUNNING, drmaa.JobState.DONE, drmaa.JobState.QUEUED_ACTIVE,
drmaa.JobState.SYSTEM_ON_HOLD, drmaa.JobState.USER_ON_HOLD,
drmaa.JobState.USER_SYSTEM_ON_HOLD]:
if job.batch_type == "slurm":
os.system("scancel %s" % job.id_process)
elif job.batch_type == "sge":
os.system("qdel %s" % job.id_process)
print("Job %s (id on cluster: %d) has died!" % (job.id_job, job.id_process))
job.status = "fail"
job.error = "<p>Your job has failed for an unexpected reason. Please contact the support.</p>"
job.save()
# Todo: send mail about the error
else:
if job.status == "scheduled-cluster" and status == drmaa.JobState.RUNNING:
job.status = "started"
job.save()
cluster_jobs_started.append(job.id_job)
elif job.status == "prepare-scheduled" and status == drmaa.JobState.RUNNING:
job.status = "preparing-cluster"
job.save()
elif job.status == "started":
cluster_jobs_started.append(job.id_job)
else:
cluster_jobs_started.append(job.id_job)
return jobs_started, cluster_jobs_started
def parse_database(app_data, max_age, fake=False):
"""
Parse database and remove too old jobs (from database and from disk)
:param app_data: folder where jobs are stored
:type app_data: str
:param max_age: remove all files & folders older than this age. Define it for each category
(uploads, data, error, ...)
:type max_age: dict
:param fake: if True, just print files to delete, without delete them
:type fake: bool
:return: id jobs which are in the gallery (not removed independently of their age)
:rtype: list
"""
from dgenies.database import Job, Gallery
gallery_jobs = []
with Job.connect():
old_jobs = Job.select().where(
((Job.status == "success") & (Job.date_created < datetime.now() -
timedelta(days=max_age["data"])))
| ((Job.status != "success") & (Job.date_created < datetime.now() -
timedelta(days=max_age["error"]))))
for job in old_jobs:
id_job = job.id_job
is_gallery = len(
Gallery.select().join(Job).where(Job.id_job == id_job)) > 0
if is_gallery:
gallery_jobs.append(id_job)
else:
print("Removing job %s..." % id_job)
data_dir = os.path.join(app_data, id_job)
if os.path.exists(data_dir) and os.path.isdir(data_dir):
if not fake:
shutil.rmtree(data_dir)
else:
print("Job %s has no data folder!" % id_job)
if not fake:
job.delete_instance()
return gallery_jobs
def get_preparing_jobs_nb():
with Job.connect():
return len(Job.select().where(Job.status == "preparing"))
def get_prep_scheduled_jobs():
with Job.connect():
jobs = Job.select().where(Job.status == "waiting").order_by(Job.date_created)
return [(j.id_job, j.batch_type) for j in jobs]
def get_preparing_jobs_cluster_nb():
with Job.connect():
return len(Job.select().where(Job.status == "preparing-cluster")), \
len(Job.select().where(Job.status == "prepare-scheduled"))