def post_save_handler( sender, instance=False, **kwargs):
try:
logging.debug("Prestoprime POST save handler %s %s" % (instance, instance.file))
# Record Additional Usage
if instance.name is not None and not instance.initial_usage_recorded and instance.name.endswith(".mxf"):
logging.debug("Prestoprime POST save handler %s id:'%s' " % (instance,instance.id))
mxfframecounttask = mxfframecount.subtask([[instance.id],[]])
#count = result["frames"]
#logging.info("Recording ingest usage %s " % (count))
#usage_store.record(instance.id,pp_mxfframe_ingested_metric,count)
job = Job(name="Prestoprime Tasks",mfile=instance)
job.save()
output = JobOutput(name="Job Output",job=job,mimetype="text/plain")
output.save()
## Do Post Processing
mfiled10check = MFileD10Check(mfile=instance)
temp_handle = StringIO()
temp_handle.seek(0)
suf = SimpleUploadedFile("checkfile", temp_handle.read())
mfiled10check.checkfile.save(suf.name+'.txt', suf, save=False)
mfiled10check.save()
d10mxfchecksumtask = d10mxfchecksum.subtask([[instance.id],[output]])
ts = TaskSet(tasks=[d10mxfchecksumtask,mxfframecounttask])
tsr = ts.apply_async()
tsr.save()
job.taskset_id=tsr.taskset_id
job.save()
logging.debug("Prestoprime task %s " % task )
except Exception as e:
logging.error("Prestoprime POST save handler failed %s " % e)
def mfile_get_handler( sender, mfile=False, **kwargs):
logging.info("In Prestoprime handler" )
try:
if mfile.file.name.endswith(".mxf"):
mfiled10check=MFileD10Check.objects.get(mfile=mfile)
tmpfile = tempfile.NamedTemporaryFile(mode="w")
d10mxfchecksum([mfile.file.path],[tmpfile.name])
if not filecmp.cmp(tmpfile.name,mfiled10check.checkfile.path):
lines1 = open(tmpfile.name).readlines()
lines2 = open(mfiled10check.checkfile.path).readlines()
# TODO: Make this a celery task
corrupted = 0
i = 0
frames = []
job = Job(name="Extract Corrupted Frames",service=mfile.service)
job.save()
tasks = []
for line1 in lines1:
line2 = lines2[i]
if line1 != line2:
split = line1.split("\t")
frameS = split[0]
try:
frame = int(frameS)
frames.append(frame)
logging.info("frame %s corrupted " % frame )
output = JobOutput(name="Job 'Corrupted Frame %s'"%frame,job=job,mimetype="image/png")
output.save()
fname = "%s.%s" % ("corruptedframe","png")
outputpath = os.path.join( str(job.id) , fname)
output.file = outputpath
thumbfolder = os.path.join( settings.THUMB_ROOT, str(job.id))
if not os.path.exists(thumbfolder):
os.makedirs(thumbfolder)
(head,tail) = os.path.split(output.file.path)
if not os.path.isdir(head):
os.makedirs(head)
thumbfile= os.path.join( thumbfolder , "%s%s" % (fname,".thumb.png"))
thumbpath = os.path.join( str(job.id) , "%s%s" % (fname,".thumb.png"))
output.thumb = thumbpath
output.save()
thumboptions = {"width":settings.thumbsize[0],"height":settings.thumbsize[1]}
callback = thumbimage.subtask([output,thumbfile,thumboptions])
logging.info("Creating task %s %s " % (mfile.file.path,output.file.path))
inputs = [mfile.file.path]
outputs = [output.file.path]
options = {"frame":frame}
callbacks = [callback]
task = extractd10frame.subtask([inputs,outputs,options],callbacks=callbacks)
tasks.append(task)
logging.info("task created %s" % task)
corrupted += 1
except ValueError as e:
logging.info("PP handler %s " % e)
i = i+1
if len(tasks) > 0:
logging.info("tasks to execute created %s" % task)
job.save()
ts = TaskSet(tasks=tasks)
tsr = ts.apply_async()
tsr.save()
job.taskset_id=tsr.taskset_id
job.name="Extract Corrupted Frames %s" % frames
job.save()
else:
job.delete()
if corrupted > 0:
usage_store.record(mfile.id,pp_mxfframe_corrupted_metric,corrupted)
logging.info("Lines Corrupted = %s " % corrupted)
else:
logging.info("Files are same")
except MFileD10Check.DoesNotExist:
logging.info("Prestoprime mfile_get_handler ")
except Exception as e:
logging.info("Prestoprime mfile get handler failed %s " % e)
logging.info("Done Prestoprime handler" )
return
def create(self, request, mfileid=None):
if request.POST.has_key('jobtype'):
jobtype = request.POST['jobtype']
inputs = []
outputs = []
callbacks = []
options = {}
if mfileid != "":
mfile = MFile.objects.get(id=mfileid)
check = mfile.service.check_times()
if check:
return check
inputs.append(mfile.id)
else:
r = rc.BAD_REQUEST
r.write("\nRequest has no serviceid or mfileid - Creation Failed!")
return r
for v in request.POST:
options[v]=request.POST[v]
logging.info("Request for job type '%s' with options %s" % (jobtype, options) )
task_description = None
try:
task_description = get_task_description(jobtype)
except Exception as e:
logging.error("No job description for job type '%s' %s" % (jobtype, e) )
r = rc.BAD_REQUEST
r.write("\nJob has no description - Creation Failed!")
return r
nbinputs = task_description["nbinputs"]
nboutputs= task_description["nboutputs"]
jobname = "%s" % jobtype
job = mfile.do("POST","jobs",**{"name":jobname})
job.save()
for i in range(1,nbinputs):
mfileid = request.POST['input-%s'%i]
mfile = MFile.objects.get(id=mfileid)
inputs.append(mfile.id)
if job == None:
r = rc.BAD_REQUEST
r.write("Job Creation failed!")
return r
for i in range(0,nboutputs):
outputmimetype = task_description["output-%s"%i]["mimetype"]
output = JobOutput(name="Output %s '%s'"%(i,jobtype),job=job,mimetype=outputmimetype)
output.save()
outputs.append(output.id)
logging.info("Creating task %s inputs= %s outputs= %s options= %s" % (jobtype,inputs,outputs,options))
# TODO : Submit to correct Q options={"queue":"%s"%job.id}
prioritise = mfile.service.priority
q = "normal.%s"% (jobtype)
if prioritise:
q = "priority.%s"% (jobtype)
kwargs={"routing_key":q}
task = subtask(task=jobtype,args=[inputs,outputs,options],callbacks=callbacks,options=kwargs)
logging.info("Task created %s " % task )
tasks = [task]
ts = TaskSet(tasks=tasks)
tsr = ts.apply_async()
tsr.save()
job.taskset_id=tsr.taskset_id
job.save()
logging.info("Creating Job Type %s on file %s" % (jobtype,mfileid))
logging.info("Created Job %s" % (job))
return job
else:
r = rc.BAD_REQUEST
r.write("Invalid Request! no jobtype in request.")
return r
def create(self, request, mfileid=None):
if request.POST.has_key('jobtype'):
jobtype = request.POST['jobtype']
inputs = []
outputs = []
callbacks = []
options = {}
if mfileid != "":
mfile = MFile.objects.get(id=mfileid)
check = mfile.service.check_times()
if check:
return check
inputs.append(mfile.id)
else:
r = rc.BAD_REQUEST
r.write("\nRequest has no serviceid or mfileid - Creation Failed!")
return r
for v in request.POST:
options[v]=request.POST[v]
## Process request.FILES, save them and add to options
for k, f in request.FILES.items():
# Create temporary file object
temp_storage = os.path.join(settings.STORAGE_ROOT, "tmp")
self._ensure_dir_exists(temp_storage)
uploaded_name, uploaded_suffix = os.path.splitext(k)
uploaded_file = tempfile.NamedTemporaryFile('wb', suffix=uploaded_suffix, delete=False, dir=temp_storage)
# Write uploaded data to file
open(uploaded_file.name, 'wb+')
for chunk in f.chunks():
uploaded_file.write(chunk)
# Pass uploaded file to job as an option
# The job will delete the upload when it's finished
uploaded_file.close()
options[k] = uploaded_file.name
options["prioritise"] = mfile.service.priority
options["serviceid"] = mfile.service.id
logging.info("Request for job type '%s' with options %s" % (jobtype, options) )
try:
task_description = get_task_description(jobtype)
except Exception as e:
logging.error("No job description for job type '%s' %s" % (jobtype, e) )
r = rc.BAD_REQUEST
r.write("\nJob has no description - Creation Failed!")
return r
nbinputs = task_description["nbinputs"]
nboutputs= task_description["nboutputs"]
jobname = "%s" % jobtype
job = mfile.do("POST","jobs",**{"name":jobname})
job.save()
for i in range(1,nbinputs):
mfileid = request.POST['input-%s'%i]
mfile = MFile.objects.get(id=mfileid)
inputs.append(mfile.id)
if job is None:
r = rc.BAD_REQUEST
r.write("Job Creation failed!")
return r
for i in range(0,nboutputs):
outputmimetype = task_description["output-%s"%i]["mimetype"]
output = JobOutput(name="Output %s '%s'"%(i,jobtype),job=job,mimetype=outputmimetype)
output.save()
outputs.append(output.id)
logging.info("Creating task %s inputs= %s outputs= %s options= %s" % (jobtype,inputs,outputs,options))
task = subtask(task=jobtype,args=[inputs,outputs,options],callbacks=callbacks,options={"prioritise":mfile.service.priority})
logging.info("Task created %s " % task )
tasks = [task]
ts = TaskSet(tasks=tasks)
tsr = ts.apply_async()
tsr.save()
job.taskset_id=tsr.taskset_id
job.save()
logging.info("Creating Job Type %s on file %s" % (jobtype,mfileid))
logging.info("Created Job %s" % (job))
return job
else:
r = rc.BAD_REQUEST
r.write("Invalid Request! no jobtype in request.")
return r