Exemplo n.º 1
0
def _drop_folder(filepath, inputfolder, outputfolder):
    try:
        uid = utils.unique_id()
        name = "%s_%s" % (uid, os.path.basename(filepath))
        inputfile = os.path.join(inputfolder, name)
        outputfile = os.path.join(outputfolder, name)
        shutil.copy(filepath, inputfile)

        exists = os.path.exists(outputfile)

        # Wait till file exists
        while not exists:
            logging.info("Output file %s doesnt exist, sleeping" % outputfile)
            time.sleep(10)
            exists = os.path.exists(outputfile)

        # Wait until file stops growing
        growing = True
        size = os.path.getsize(outputfile)
        logging.info("Size is %s" % str(size))
        while growing:
            time.sleep(10)
            newsize = os.path.getsize(outputfile)
            logging.info("Size is now %s" % str(newsize))
            growing = (size != newsize)
            size = newsize
            logging.info("File Growing  %s" % growing)

        return outputfile


    except Exception as e:
        logging.info("Error with watching drop folder %s" % e)
        raise e
Exemplo n.º 2
0
def fims_mews(inputs, outputs, options={}, callbacks=[]):

    # TODO: Create the correct paths for the FIMS MEWS service to be able to
    # see the MFile content

    bmcontent_locator = ""
    jobguid = utils.unique_id()
    transfer_locator = ""
    transfer_destination = ""

    transform_vars = {'bmcontent_locator': bmcontent_locator,
            'jobguid': jobguid,
            'transfer_locator':transfer_locator,
            'transfer_destination':transfer_destination }

    js = fims_transformrequest(settings.FIMS_MEWS_URL_TRANSFORM, transform_vars)
    jobGUID = js["transformAck"]["operationInfo"]["jobID"]["jobGUID"]

    # TODO: instead of having max_polls use the status field to break
    # when the job fails (dont know what these statuses are yet)
    max_polls=5
    js_resp = fims_job_queryrequest(settings.FIMS_MEWS_URL_JOBQUERY+jobGUID)
    status = js_resp ["queryJobRequest"]["queryJobInfo"]["jobInfo"]["status"]\
                    ["code"]
                    
    while status == "running" and max_polls >= 0:
        time.sleep(5)
        js_resp = fims_job_queryrequest(settings.FIMS_MEWS_URL_JOBQUERY+jobGUID)
        print js_resp
        status = js_resp ["queryJobRequest"]["queryJobInfo"]["jobInfo"]["status"]\
                    ["code"]
        max_polls = max_polls -1

    # TODO: Save the result from transfer_destination and transfer_locator?

    return js_resp
Exemplo n.º 3
0
def fims_mews(inputs, outputs, options={}, callbacks=[]):
    ''' Invokes the FIMS_MEWS service on remote mounted MServe storage '''

    try:
        mfileid = inputs[0]
        joboutput = outputs[0]
        from dataservice.models import MFile
        mf = MFile.objects.get(id=mfileid)
        job_uid = utils.unique_id()

        # We need to create the destination folder
        local_mount = settings.STORAGE_ROOT
        if mf.service.container.default_path:
            local_mount = os.path.join(settings.STORAGE_ROOT, mf.service.container.default_path)
        local_destination = os.path.join(local_mount, "fims_mews", job_uid)
        if not os.path.exists(local_destination):
            os.makedirs(local_destination)

        # Using the code below we need to mount the raw MServe storage on the fims mews machine.
        bm_content_locator = ntpath.normpath(_mfile_to_mount(settings.FIMS_MEWS_MOUNT, mf))
        mews_mount = settings.FIMS_MEWS_MOUNT
        transfer_destination = ntpath.normpath(ntpath.join(mews_mount, "fims_mews", job_uid))
        transfer_destination += "\\"
        
        transform_vars = {'bm_content_locator': bm_content_locator,
                          'job_uid': job_uid,
                          'transfer_destination': transfer_destination,
                          'output_wrapper': options["output_wrapper"],
                          'output_codec': options["output_codec"]}

        job_returned_js = fims_request(settings.FIMS_MEWS_URL_TRANSFORM, "transformRequest.xml", transform_vars)
        logging.debug("FIMS Transform Request returned: %s", job_returned_js)
        job_returned_uid = job_returned_js["transformAck"]["operationInfo"]["jobID"]["jobGUID"]

        status_returned_js = fims_job_status_request(settings.FIMS_MEWS_URL_JOBQUERY + job_returned_uid)
        logging.debug("FIMS Transform Service Status Request returned: %s", status_returned_js)
        status = status_returned_js["queryJobRequest"]["queryJobInfo"]["jobInfo"]["status"]["code"]

        while status != "completed":
            if status == "failed":
                # Failed job
                raise Exception("FIMS MEWS task failed")
            elif status == "canceled":
                # Canceled job
                raise Exception("FIMS MEWS task canceled")
            else:
                time.sleep(5)
                status_returned_js = fims_job_status_request(settings.FIMS_MEWS_URL_JOBQUERY + job_returned_uid)
                logging.debug("Service status returned: %s", status_returned_js)
                status = status_returned_js["queryJobRequest"]["queryJobInfo"]["jobInfo"]["status"]["code"]

        files = os.listdir(local_destination)
        temp_archive = tempfile.NamedTemporaryFile('wb')
        zip_files(temp_archive.name, local_destination, files)
        output_file = open(temp_archive.name, 'r')

        logging.info("output_file %s", output_file)

        if output_file:
            suf = SimpleUploadedFile("mfile", output_file.read(), content_type='application/octet-stream')

            if len(outputs) > 0:
                jo = JobOutput.objects.get(id=joboutput)
                jo.file.save('results.zip', suf, save=True)
            else:
                logging.error("Nowhere to save output")

            output_file.close()
        else:
            raise Exception("Unable to get output_file location")

        # Clean up
        clean_up_vars = {'job_uid': job_returned_uid,
                         'command': "cleanup"}
        clean_up_returned_js = fims_request(settings.FIMS_MEWS_URL_MANAGE, "manageJobRequest.xml", clean_up_vars)

        return {"success": True, "message": "FIMS_MEWS transcode of video successful"}

    except Exception as e:
        logging.info("Error with FIMS_MEWS %s" % e)
        raise e