def get_job_directory_listing(self, job_stage_id, directory):
stage = JobStages.GetJobStageByID(self.user, job_stage_id)
return self.RunUserProcess(
"%s/venv/bin/python %s/manage.py acl GET_DIR %s %s" %
(self.project_dir, self.project_dir, stage.WorkingDirectory,
directory),
user=stage.Job.User)
def get_job_output(self, job_stage_id):
stage = JobStages.GetJobStageByID(self.user, job_stage_id)
if stage.OutputLog != '':
data = self.RunUserProcess('cat %s' % stage.OutputLog,
user=stage.Job.User)
return data
else:
return "Output not available"
def RunCustomJob(self, job, commands, parsed_settings, files):
with transaction.atomic():
jobstage = JobStages.AddJobStage(self.user, job, Commands=commands)
#determine job directory
job_dir = os.path.join(
settings.JMS_SETTINGS["JMS_shared_directory"],
"users/%s/jobs/%d" % (self.user, jobstage.Job.JobID))
jobstage.WorkingDirectory = job_dir
jobstage.OutputLog = os.path.join(job_dir, "logs/output.log")
jobstage.ErrorLog = os.path.join(job_dir, "logs/error.log")
#parse settings
for s in parsed_settings:
#move to data layer
jsr = JobStageResource.objects.create(
ResourceManager=module_name,
JobStage=jobstage,
Key=s["Key"],
Value=s["Value"],
Label=s["Label"])
jobstage.save()
#create job directory
job_script = self.SetupJobDirectory(jobstage, files, 0)
r = ResourceManager(self.user)
cluster_id = r.ExecuteJobScript(job_script)
with open("/tmp/cluster_id.txt", 'w') as f:
print >> f, cluster_id
jobstage.ClusterJobID = cluster_id.strip()
jobstage.save()
j = r.GetJob(jobstage.ClusterJobID)
JobData = json.dumps(j.DataSections,
default=lambda o: o.__dict__,
sort_keys=True)
jobstage = JobStages.UpdateJobStage(jobstage, j.Status, j.ExitCode,
j.OutputLog, j.ErrorLog,
j.WorkingDir, JobData)
return jobstage
def get_tmp_job_file(self, job_stage_id, path):
#create a tmp file in an accessible area and return the path
filename = os.path.basename(path)
tmp_dir = self.make_tmp_directory()
tmp_path = os.path.join(tmp_dir, filename)
stage = JobStages.GetJobStageByID(self.user, job_stage_id)
filepath = path.lstrip("/")
if not filepath.startswith("/"):
abspath = os.path.join(stage.WorkingDirectory, filepath)
cmd = "%s/venv/bin/python %s/manage.py acl CREATE_TEMP_FILE %s %s" % (
self.project_dir, self.project_dir, abspath, tmp_path)
out = self.RunUserProcess(cmd, user=stage.Job.User)
if not out.startswith("ERROR:\n\n"):
return tmp_path
else:
raise Exception(out)
else:
raise Exception("Bad path.")
def GetJobByClusterJobID(self, id):
return JobStages.GetJobStageByClusterID(self.user, id).Job
def RunToolJob(self, job, user_parameters, files=[], stage=None,
stage_index=0):
with transaction.atomic():
#create db entries
jobstage = JobStages.AddJobStage(self.user, job, Stage=stage)
stage_dir = ""
if job.JobTypeID == 2:
version = job.ToolVersion
elif job.JobTypeID == 3:
version = stage.ToolVersion
stage_dir = str(stage.StageID)
stage_parameters = stage.StageParameters.all()
#determine job directory
job_dir = os.path.join(settings.JMS_SETTINGS["JMS_shared_directory"],
"users/%s/jobs/%d/%s" % (self.user, jobstage.Job.JobID, stage_dir)
)
jobstage.WorkingDirectory = job_dir
jobstage.OutputLog = os.path.join(job_dir, "logs/output.log")
jobstage.ErrorLog = os.path.join(job_dir, "logs/error.log")
#generate command for tool script
command = version.Command
#append parameters to command
#TODO: move database call to bottom tier
parameters = version.ToolParameters.all()
params = ""
for param in parameters:
#only add root parameters
if param.ParentParameter == None:
p = param.Context
val = param.Value
if param.InputBy == "user":
for up in user_parameters:
if up["ParameterID"] == param.ParameterID:
val = up["Value"]
#If parameter value comes from a previous stage, fetch the value
if jobstage.Stage != None:
try:
stage_parameter = stage_parameters.get(
Parameter__ParameterID=param.ParameterID
)
if stage_parameter.StageParameterType == 1:
val = stage_parameter.Value
elif stage_parameter.StageParameterType == 2:
try:
val = JobStageParameter.objects.get(JobStage__Job=job, Parameter_id=stage_parameter.Value).Value
except Exception, ex:
with open("/tmp/jobstageparameter.txt", 'w') as f:
print >> f, traceback.format_exc()
elif stage_parameter.StageParameterType == 3:
val = ExpectedOutput.objects.get(ExpectedOutputID=stage_parameter.Value).FileName
except Exception, ex:
pass
JobStageParameter.objects.create(JobStage=jobstage, Parameter=param,
ParameterName=param.ParameterName, Value=val)
if param.ParameterType.ParameterTypeID != 3 and len(str(val)) > 0:
try:
#check if the ${VALUE} variable is located in the string
num = p.index("${VALUE}")
p = p.replace("${VALUE}", str(val))
except Exception, e:
#if the ${VALUE} variable is not located in the string, append the value to the end
#File.print_to_file("/tmp/file.txt", str(e) + ": " + p)
p += " %s" % val
params += ' %s' % p
def UpdateJobHistory(self):
#f = open("/tmp/continue1.txt", 'w')
#print >> f, "here"
r = ResourceManager(self.user)
jobs = r.GetDetailedQueue()
new_status = None
emails = []
http = []
for job in jobs:
try:
JobData = json.dumps(job.DataSections,
default=lambda o: o.__dict__, sort_keys=True)
#update or create JobStage
jobstage = JobStages.GetJobStage(job.JobID)
new_status = job.Status
if new_status == 4 and job.ExitCode == None:
new_status = 6
if jobstage:
old_status = jobstage.Status.StatusID
if old_status < 5:
if new_status == 4 and jobstage.RequiresEditInd:
new_status = 5
jobstage = JobStages.UpdateJobStage(jobstage, new_status,
job.ExitCode, job.OutputLog, job.ErrorLog,
job.WorkingDir, JobData)
#if the status is held and the job is a workflow
#release job if dependencies have been satisfied
if new_status == 1 and jobstage.Job.JobTypeID < 4:
self.ReleaseJob(jobstage)
#if status has changed to completed state, send notifications
if old_status < 4 and new_status >= 4:
if jobstage.Job.JobStages.filter(Status_id__lt=4).count() == 0:
#if there are no jobstages in job with status < 4, the job must be complete
if jobstage.Job.NotificationEmail is not None:
emails.append(jobstage.Job)
if jobstage.Job.NotificationURL is not None:
http.append(jobstage.Job)
#if status has changed, get new status and send notifications
#if old_status != new_status:
#self.UpdateJobStatus(jobstage.Job)
else:
with transaction.atomic():
user, created = User.objects.get_or_create(
username=job.User.split("@")[0]
)
j = Jobs.AddJob(User=user, JobName=job.JobName,
Description="External submission - job was not submitted via JMS",
ToolVersion=None, WorkflowVersion=None, JobTypeID=4)
jobstage = JobStages.AddJobStage(user, j, StatusID=job.Status,
ClusterJobID=job.JobID, ExitCode=job.ExitCode,
ErrorLog=job.ErrorLog, OutputLog=job.OutputLog,
PWD=job.WorkingDir, JobData=JobData)
except Exception, ex:
File.print_to_file("/tmp/continue.txt", traceback.format_exc())
pass
job_script = self.SetupJobDirectory(jobstage, files, stage_index)
r = ResourceManager(self.user)
cluster_id = r.ExecuteJobScript(job_script)
with open("/tmp/job_id.txt", 'w') as f:
print >> f, job_script
print >> f, cluster_id
jobstage.ClusterJobID = cluster_id.strip()
jobstage.save()
j = r.GetJob(jobstage.ClusterJobID)
JobData = json.dumps(j.DataSections, default=lambda o: o.__dict__,
sort_keys=True)
jobstage = JobStages.UpdateJobStage(jobstage, j.Status,
j.ExitCode, j.OutputLog, j.ErrorLog,
j.WorkingDir, JobData)
return jobstage
def RunWorkflowJob(self, job, stages, files):
stage_levels = {}
executed_stages = {}
#create job stages
for i, s in enumerate(stages):
stage = Stages.GetStage(self.user, s["StageID"])
if not stage_levels.get(stage.StageLevel):
stage_levels[stage.StageLevel] = []
def UpdateJobHistory(self):
f = open("/tmp/continue1.txt", 'w')
r = ResourceManager(self.user)
queue = r.GetQueue()
new_status = None
#print >> f, queue
#print >> f, queue.rows
emails = []
http = []
for row in queue.rows:
try:
#update or create JobStage
jobstage = JobStages.GetJobStage(row.job_id)
#print >> f, "Working on %s" % row.job_id
#print >> f, jobstage
if jobstage: # if the job exists in the database and the status has changed
new_status = row.state
old_status = jobstage.Status.StatusID
#print >> f, old_status
if new_status == 4 and jobstage.RequiresEditInd:
new_status = 5
#print >> f, new_status
#if the job status has changed or the job is running
if old_status != new_status or new_status == 3:
#print >> f, "Update %s" % row.job_id
job = r.GetJob(row.job_id)
data = json.dumps(job.DataSections,
default=lambda o: o.__dict__,
sort_keys=True)
if new_status == 4 and job.ExitCode == None:
new_status = 6
jobstage = JobStages.UpdateJobStage(
jobstage, new_status, job.ExitCode, job.OutputLog,
job.ErrorLog, job.WorkingDir, data)
#if the status is held and the job is a workflow
#release job if dependencies have been satisfied.
#This should only happen if an error has occured
#when continuing the job
if new_status == 1 and jobstage.Job.JobTypeID < 4:
self.ReleaseJob(jobstage)
#if status has changed to completed state, send notifications
if new_status >= 4:
if jobstage.Job.JobStages.filter(
Status_id__lt=4).count() == 0:
#if there are no jobstages in job with status < 4, the job must be complete
if jobstage.Job.NotificationEmail is not None:
emails.append(jobstage.Job)
if jobstage.Job.NotificationURL is not None:
http.append(jobstage.Job)
else: # if the job doesnt exist in the database, add it
job = r.GetJob(row.job_id)
data = json.dumps(job.DataSections,
default=lambda o: o.__dict__,
sort_keys=True)
with transaction.atomic():
user, created = User.objects.get_or_create(
username=job.User.split("@")[0])
j = Jobs.AddJob(
User=user,
JobName=job.JobName,
Description=
"External submission - job was not submitted via JMS",
ToolVersion=None,
WorkflowVersion=None,
JobTypeID=4)
jobstage = JobStages.AddJobStage(
user,
j,
StatusID=job.Status,
ClusterJobID=job.JobID,
ExitCode=job.ExitCode,
ErrorLog=job.ErrorLog,
OutputLog=job.OutputLog,
PWD=job.WorkingDir,
JobData=data)
except Exception, ex:
File.print_to_file("/tmp/continue.txt", traceback.format_exc())
pass