Ejemplo n.º 1
0
def post_save_handler( sender, instance=False, **kwargs):
    try:
        logging.debug("Prestoprime POST save handler %s %s" % (instance, instance.file))

        # Record Additional Usage
        if instance.name is not None and not instance.initial_usage_recorded and instance.name.endswith(".mxf"):
            logging.debug("Prestoprime POST save handler %s id:'%s' " % (instance,instance.id))

            mxfframecounttask = mxfframecount.subtask([[instance.id],[]])

            #count = result["frames"]
            #logging.info("Recording ingest usage %s " % (count))
            #usage_store.record(instance.id,pp_mxfframe_ingested_metric,count)

            job = Job(name="Prestoprime Tasks",mfile=instance)
            job.save()

            output = JobOutput(name="Job Output",job=job,mimetype="text/plain")
            output.save()

            ## Do Post Processing
            mfiled10check = MFileD10Check(mfile=instance)
            
            temp_handle = StringIO()
            temp_handle.seek(0)

            suf = SimpleUploadedFile("checkfile", temp_handle.read())
            mfiled10check.checkfile.save(suf.name+'.txt', suf, save=False)
            mfiled10check.save()

            d10mxfchecksumtask = d10mxfchecksum.subtask([[instance.id],[output]])

            ts = TaskSet(tasks=[d10mxfchecksumtask,mxfframecounttask])
            tsr = ts.apply_async()
            tsr.save()

            
            job.taskset_id=tsr.taskset_id
            job.save()
            
            logging.debug("Prestoprime task %s  " % task  )

    except Exception as e:
        logging.error("Prestoprime POST save handler failed %s " % e)
Ejemplo n.º 2
0
def mfile_get_handler( sender, mfile=False, **kwargs):
    logging.info("In Prestoprime handler"  )
    try:
        if mfile.file.name.endswith(".mxf"):
            mfiled10check=MFileD10Check.objects.get(mfile=mfile)
            tmpfile = tempfile.NamedTemporaryFile(mode="w")
            d10mxfchecksum([mfile.file.path],[tmpfile.name])
            
            if not filecmp.cmp(tmpfile.name,mfiled10check.checkfile.path):
                lines1 = open(tmpfile.name).readlines()
                lines2 = open(mfiled10check.checkfile.path).readlines()

                # TODO: Make this a celery task
                corrupted = 0
                i = 0
                frames = []
                job = Job(name="Extract Corrupted Frames",service=mfile.service)
                job.save()
                tasks = []
                for line1 in lines1:
                    line2 = lines2[i]
                    if line1 != line2:
                        split = line1.split("\t")
                        frameS = split[0]
                        try:
                            frame = int(frameS)
                            frames.append(frame)
                            logging.info("frame %s corrupted " % frame )

                            output = JobOutput(name="Job 'Corrupted Frame %s'"%frame,job=job,mimetype="image/png")
                            output.save()
                            fname = "%s.%s" % ("corruptedframe","png")
                            outputpath = os.path.join( str(job.id) , fname)
                            output.file = outputpath
                            thumbfolder = os.path.join( settings.THUMB_ROOT, str(job.id))
                            if not os.path.exists(thumbfolder):
                                os.makedirs(thumbfolder)

                            (head,tail) = os.path.split(output.file.path)
                            if not os.path.isdir(head):
                                os.makedirs(head)

                            thumbfile= os.path.join( thumbfolder , "%s%s" % (fname,".thumb.png"))
                            thumbpath = os.path.join( str(job.id) , "%s%s" % (fname,".thumb.png"))
                            output.thumb = thumbpath
                            output.save()
                            thumboptions = {"width":settings.thumbsize[0],"height":settings.thumbsize[1]}
                            callback = thumbimage.subtask([output,thumbfile,thumboptions])

                            logging.info("Creating task %s %s " % (mfile.file.path,output.file.path))
                            inputs    = [mfile.file.path]
                            outputs   = [output.file.path]
                            options   = {"frame":frame}
                            callbacks = [callback]
                            task = extractd10frame.subtask([inputs,outputs,options],callbacks=callbacks)
                            tasks.append(task)
                            
                            logging.info("task created %s" % task)

                            corrupted += 1
                        except ValueError as e:
                            logging.info("PP handler %s " % e)

                    i = i+1

                if len(tasks) > 0:


                    logging.info("tasks to execute created %s" % task)

                    job.save()
                    ts = TaskSet(tasks=tasks)
                    tsr = ts.apply_async()
                    tsr.save()

                    job.taskset_id=tsr.taskset_id
                    job.name="Extract Corrupted Frames %s" % frames
                    job.save()
                else:
                    job.delete()

                if corrupted > 0:
                    usage_store.record(mfile.id,pp_mxfframe_corrupted_metric,corrupted)

                logging.info("Lines Corrupted = %s " % corrupted)
            else:
                logging.info("Files are same")

    except MFileD10Check.DoesNotExist:
        logging.info("Prestoprime mfile_get_handler ")
    except Exception as e:
        logging.info("Prestoprime mfile get handler failed %s " % e)
    logging.info("Done Prestoprime handler"  )
    return 
Ejemplo n.º 3
0
    def create(self, request, mfileid=None):

        if request.POST.has_key('jobtype'):
            jobtype = request.POST['jobtype']

            inputs = []
            outputs = []
            callbacks = []
            options = {}

            if mfileid != "":
                mfile = MFile.objects.get(id=mfileid)

                check = mfile.service.check_times()
                if check:
                    return check

                inputs.append(mfile.id)
            else:
                r = rc.BAD_REQUEST
                r.write("\nRequest has no serviceid or mfileid - Creation Failed!")
                return r

            for v in request.POST:
                options[v]=request.POST[v]

            logging.info("Request for job type '%s' with options %s" % (jobtype, options) )

            task_description = None
            try:
                task_description = get_task_description(jobtype)
            except Exception as e:
                logging.error("No job description for job type '%s' %s" % (jobtype, e) )
                r = rc.BAD_REQUEST
                r.write("\nJob has no description - Creation Failed!")
                return r

            nbinputs = task_description["nbinputs"]
            nboutputs= task_description["nboutputs"]

            jobname = "%s" % jobtype
            job = mfile.do("POST","jobs",**{"name":jobname})
            job.save()

            for i in range(1,nbinputs):
                mfileid = request.POST['input-%s'%i]
                mfile = MFile.objects.get(id=mfileid)
                inputs.append(mfile.id)

            if job == None:
                r = rc.BAD_REQUEST
                r.write("Job Creation failed!")
                return r

            for i in range(0,nboutputs):
                outputmimetype = task_description["output-%s"%i]["mimetype"]
                output = JobOutput(name="Output %s '%s'"%(i,jobtype),job=job,mimetype=outputmimetype)
                output.save()
                outputs.append(output.id)

            logging.info("Creating task %s inputs= %s outputs= %s options= %s" % (jobtype,inputs,outputs,options))

            # TODO : Submit to correct Q options={"queue":"%s"%job.id}
            prioritise = mfile.service.priority
            q = "normal.%s"% (jobtype)
            if prioritise:
                q = "priority.%s"% (jobtype)
            kwargs={"routing_key":q}
            task = subtask(task=jobtype,args=[inputs,outputs,options],callbacks=callbacks,options=kwargs)
            logging.info("Task created %s " % task )

            tasks = [task]

            ts = TaskSet(tasks=tasks)
            tsr = ts.apply_async()
            tsr.save()

            job.taskset_id=tsr.taskset_id
            job.save()

            logging.info("Creating Job Type %s on file %s" % (jobtype,mfileid))

            logging.info("Created Job  %s" % (job))

            return job
        else:
            r = rc.BAD_REQUEST
            r.write("Invalid Request! no jobtype in request.")
            return r
Ejemplo n.º 4
0
    def create(self, request, mfileid=None):

        if request.POST.has_key('jobtype'):
            jobtype = request.POST['jobtype']

            inputs = []
            outputs = []
            callbacks = []
            options = {}

            if mfileid != "":
                mfile = MFile.objects.get(id=mfileid)

                check = mfile.service.check_times()
                if check:
                    return check

                inputs.append(mfile.id)
            else:
                r = rc.BAD_REQUEST
                r.write("\nRequest has no serviceid or mfileid - Creation Failed!")
                return r

            for v in request.POST:
                options[v]=request.POST[v]

            ## Process request.FILES, save them and add to options
            for k, f in request.FILES.items():
                # Create temporary file object
                temp_storage = os.path.join(settings.STORAGE_ROOT, "tmp")
                self._ensure_dir_exists(temp_storage)
                uploaded_name, uploaded_suffix = os.path.splitext(k)
                uploaded_file = tempfile.NamedTemporaryFile('wb', suffix=uploaded_suffix, delete=False, dir=temp_storage)
                # Write uploaded data to file
                open(uploaded_file.name, 'wb+')
                for chunk in f.chunks():
                    uploaded_file.write(chunk)
                # Pass uploaded file to job as an option
                # The job will delete the upload when it's finished
                uploaded_file.close()
                options[k] = uploaded_file.name

            options["prioritise"] = mfile.service.priority
            options["serviceid"] = mfile.service.id

            logging.info("Request for job type '%s' with options %s" % (jobtype, options) )

            try:
                task_description = get_task_description(jobtype)
            except Exception as e:
                logging.error("No job description for job type '%s' %s" % (jobtype, e) )
                r = rc.BAD_REQUEST
                r.write("\nJob has no description - Creation Failed!")
                return r

            nbinputs = task_description["nbinputs"]
            nboutputs= task_description["nboutputs"]

            jobname = "%s" % jobtype
            job = mfile.do("POST","jobs",**{"name":jobname})
            job.save()

            for i in range(1,nbinputs):
                mfileid = request.POST['input-%s'%i]
                mfile = MFile.objects.get(id=mfileid)
                inputs.append(mfile.id)

            if job is None:
                r = rc.BAD_REQUEST
                r.write("Job Creation failed!")
                return r

            for i in range(0,nboutputs):
                outputmimetype = task_description["output-%s"%i]["mimetype"]
                output = JobOutput(name="Output %s '%s'"%(i,jobtype),job=job,mimetype=outputmimetype)
                output.save()
                outputs.append(output.id)

            logging.info("Creating task %s inputs= %s outputs= %s options= %s" % (jobtype,inputs,outputs,options))
            task = subtask(task=jobtype,args=[inputs,outputs,options],callbacks=callbacks,options={"prioritise":mfile.service.priority})
            logging.info("Task created %s " % task )

            tasks = [task]
            ts = TaskSet(tasks=tasks)
            tsr = ts.apply_async()
            tsr.save()

            job.taskset_id=tsr.taskset_id
            job.save()

            logging.info("Creating Job Type %s on file %s" % (jobtype,mfileid))

            logging.info("Created Job  %s" % (job))

            return job
        else:
            r = rc.BAD_REQUEST
            r.write("Invalid Request! no jobtype in request.")
            return r