def post_save_handler( sender, instance=False, **kwargs): try: logging.debug("Prestoprime POST save handler %s %s" % (instance, instance.file)) # Record Additional Usage if instance.name is not None and not instance.initial_usage_recorded and instance.name.endswith(".mxf"): logging.debug("Prestoprime POST save handler %s id:'%s' " % (instance,instance.id)) mxfframecounttask = mxfframecount.subtask([[instance.id],[]]) #count = result["frames"] #logging.info("Recording ingest usage %s " % (count)) #usage_store.record(instance.id,pp_mxfframe_ingested_metric,count) job = Job(name="Prestoprime Tasks",mfile=instance) job.save() output = JobOutput(name="Job Output",job=job,mimetype="text/plain") output.save() ## Do Post Processing mfiled10check = MFileD10Check(mfile=instance) temp_handle = StringIO() temp_handle.seek(0) suf = SimpleUploadedFile("checkfile", temp_handle.read()) mfiled10check.checkfile.save(suf.name+'.txt', suf, save=False) mfiled10check.save() d10mxfchecksumtask = d10mxfchecksum.subtask([[instance.id],[output]]) ts = TaskSet(tasks=[d10mxfchecksumtask,mxfframecounttask]) tsr = ts.apply_async() tsr.save() job.taskset_id=tsr.taskset_id job.save() logging.debug("Prestoprime task %s " % task ) except Exception as e: logging.error("Prestoprime POST save handler failed %s " % e)
def mfile_get_handler( sender, mfile=False, **kwargs): logging.info("In Prestoprime handler" ) try: if mfile.file.name.endswith(".mxf"): mfiled10check=MFileD10Check.objects.get(mfile=mfile) tmpfile = tempfile.NamedTemporaryFile(mode="w") d10mxfchecksum([mfile.file.path],[tmpfile.name]) if not filecmp.cmp(tmpfile.name,mfiled10check.checkfile.path): lines1 = open(tmpfile.name).readlines() lines2 = open(mfiled10check.checkfile.path).readlines() # TODO: Make this a celery task corrupted = 0 i = 0 frames = [] job = Job(name="Extract Corrupted Frames",service=mfile.service) job.save() tasks = [] for line1 in lines1: line2 = lines2[i] if line1 != line2: split = line1.split("\t") frameS = split[0] try: frame = int(frameS) frames.append(frame) logging.info("frame %s corrupted " % frame ) output = JobOutput(name="Job 'Corrupted Frame %s'"%frame,job=job,mimetype="image/png") output.save() fname = "%s.%s" % ("corruptedframe","png") outputpath = os.path.join( str(job.id) , fname) output.file = outputpath thumbfolder = os.path.join( settings.THUMB_ROOT, str(job.id)) if not os.path.exists(thumbfolder): os.makedirs(thumbfolder) (head,tail) = os.path.split(output.file.path) if not os.path.isdir(head): os.makedirs(head) thumbfile= os.path.join( thumbfolder , "%s%s" % (fname,".thumb.png")) thumbpath = os.path.join( str(job.id) , "%s%s" % (fname,".thumb.png")) output.thumb = thumbpath output.save() thumboptions = {"width":settings.thumbsize[0],"height":settings.thumbsize[1]} callback = thumbimage.subtask([output,thumbfile,thumboptions]) logging.info("Creating task %s %s " % (mfile.file.path,output.file.path)) inputs = [mfile.file.path] outputs = [output.file.path] options = {"frame":frame} callbacks = [callback] task = extractd10frame.subtask([inputs,outputs,options],callbacks=callbacks) tasks.append(task) logging.info("task created %s" % task) corrupted += 1 except ValueError as e: logging.info("PP handler %s " % e) i = i+1 if len(tasks) > 0: logging.info("tasks to execute created %s" % task) job.save() ts = TaskSet(tasks=tasks) tsr = ts.apply_async() tsr.save() job.taskset_id=tsr.taskset_id job.name="Extract Corrupted Frames %s" % frames job.save() else: job.delete() if corrupted > 0: usage_store.record(mfile.id,pp_mxfframe_corrupted_metric,corrupted) logging.info("Lines Corrupted = %s " % corrupted) else: logging.info("Files are same") except MFileD10Check.DoesNotExist: logging.info("Prestoprime mfile_get_handler ") except Exception as e: logging.info("Prestoprime mfile get handler failed %s " % e) logging.info("Done Prestoprime handler" ) return
def create(self, request, mfileid=None): if request.POST.has_key('jobtype'): jobtype = request.POST['jobtype'] inputs = [] outputs = [] callbacks = [] options = {} if mfileid != "": mfile = MFile.objects.get(id=mfileid) check = mfile.service.check_times() if check: return check inputs.append(mfile.id) else: r = rc.BAD_REQUEST r.write("\nRequest has no serviceid or mfileid - Creation Failed!") return r for v in request.POST: options[v]=request.POST[v] logging.info("Request for job type '%s' with options %s" % (jobtype, options) ) task_description = None try: task_description = get_task_description(jobtype) except Exception as e: logging.error("No job description for job type '%s' %s" % (jobtype, e) ) r = rc.BAD_REQUEST r.write("\nJob has no description - Creation Failed!") return r nbinputs = task_description["nbinputs"] nboutputs= task_description["nboutputs"] jobname = "%s" % jobtype job = mfile.do("POST","jobs",**{"name":jobname}) job.save() for i in range(1,nbinputs): mfileid = request.POST['input-%s'%i] mfile = MFile.objects.get(id=mfileid) inputs.append(mfile.id) if job == None: r = rc.BAD_REQUEST r.write("Job Creation failed!") return r for i in range(0,nboutputs): outputmimetype = task_description["output-%s"%i]["mimetype"] output = JobOutput(name="Output %s '%s'"%(i,jobtype),job=job,mimetype=outputmimetype) output.save() outputs.append(output.id) logging.info("Creating task %s inputs= %s outputs= %s options= %s" % (jobtype,inputs,outputs,options)) # TODO : Submit to correct Q options={"queue":"%s"%job.id} prioritise = mfile.service.priority q = "normal.%s"% (jobtype) if prioritise: q = "priority.%s"% (jobtype) kwargs={"routing_key":q} task = subtask(task=jobtype,args=[inputs,outputs,options],callbacks=callbacks,options=kwargs) logging.info("Task created %s " % task ) tasks = [task] ts = TaskSet(tasks=tasks) tsr = ts.apply_async() tsr.save() job.taskset_id=tsr.taskset_id job.save() logging.info("Creating Job Type %s on file %s" % (jobtype,mfileid)) logging.info("Created Job %s" % (job)) return job else: r = rc.BAD_REQUEST r.write("Invalid Request! no jobtype in request.") return r
def create(self, request, mfileid=None): if request.POST.has_key('jobtype'): jobtype = request.POST['jobtype'] inputs = [] outputs = [] callbacks = [] options = {} if mfileid != "": mfile = MFile.objects.get(id=mfileid) check = mfile.service.check_times() if check: return check inputs.append(mfile.id) else: r = rc.BAD_REQUEST r.write("\nRequest has no serviceid or mfileid - Creation Failed!") return r for v in request.POST: options[v]=request.POST[v] ## Process request.FILES, save them and add to options for k, f in request.FILES.items(): # Create temporary file object temp_storage = os.path.join(settings.STORAGE_ROOT, "tmp") self._ensure_dir_exists(temp_storage) uploaded_name, uploaded_suffix = os.path.splitext(k) uploaded_file = tempfile.NamedTemporaryFile('wb', suffix=uploaded_suffix, delete=False, dir=temp_storage) # Write uploaded data to file open(uploaded_file.name, 'wb+') for chunk in f.chunks(): uploaded_file.write(chunk) # Pass uploaded file to job as an option # The job will delete the upload when it's finished uploaded_file.close() options[k] = uploaded_file.name options["prioritise"] = mfile.service.priority options["serviceid"] = mfile.service.id logging.info("Request for job type '%s' with options %s" % (jobtype, options) ) try: task_description = get_task_description(jobtype) except Exception as e: logging.error("No job description for job type '%s' %s" % (jobtype, e) ) r = rc.BAD_REQUEST r.write("\nJob has no description - Creation Failed!") return r nbinputs = task_description["nbinputs"] nboutputs= task_description["nboutputs"] jobname = "%s" % jobtype job = mfile.do("POST","jobs",**{"name":jobname}) job.save() for i in range(1,nbinputs): mfileid = request.POST['input-%s'%i] mfile = MFile.objects.get(id=mfileid) inputs.append(mfile.id) if job is None: r = rc.BAD_REQUEST r.write("Job Creation failed!") return r for i in range(0,nboutputs): outputmimetype = task_description["output-%s"%i]["mimetype"] output = JobOutput(name="Output %s '%s'"%(i,jobtype),job=job,mimetype=outputmimetype) output.save() outputs.append(output.id) logging.info("Creating task %s inputs= %s outputs= %s options= %s" % (jobtype,inputs,outputs,options)) task = subtask(task=jobtype,args=[inputs,outputs,options],callbacks=callbacks,options={"prioritise":mfile.service.priority}) logging.info("Task created %s " % task ) tasks = [task] ts = TaskSet(tasks=tasks) tsr = ts.apply_async() tsr.save() job.taskset_id=tsr.taskset_id job.save() logging.info("Creating Job Type %s on file %s" % (jobtype,mfileid)) logging.info("Created Job %s" % (job)) return job else: r = rc.BAD_REQUEST r.write("Invalid Request! no jobtype in request.") return r