def runjob(jobnumber, context, request):
    loginfo('bmu online job submission started', getJobfile(jobnumber), context, request)
    status = 'jobstarted'
    try:
        file_is_OK = True
        if INSTITUTION == 'cinefiles':
            # test file content
            input_file = getJobfile(jobnumber) + '.step1.csv'
            report_file = getJobfile(jobnumber) + '.check.csv'
            file_is_OK = doChecks(('', 'file', JOBDIR % '', input_file, report_file))
            # if ok continue
            # otherwise ... bail
            if file_is_OK:
                pass
            else:
                deletejob(request, jobnumber + '.step1.csv')
                loginfo('bmu ERROR:  process', jobnumber + " QC check failed.", context, request)
                status = 'jobfailed'
        if file_is_OK:
            p_object = subprocess.Popen([path.join(POSTBLOBPATH, 'postblob.sh'), INSTITUTION, getJobfile(jobnumber), BATCHPARAMETERS])
            pid = ''
            if p_object._child_created:
                pid = p_object.pid
                loginfo('bmu online job submitted:', jobnumber + f': Child returned {p_object.returncode}, pid {pid}', context, request)
            else:
                loginfo('bmu ERROR:', jobnumber + f': Child returned {p_object.returncode}, pid {pid}', context, request)
    except OSError as e:
        loginfo('error', "ERROR: Execution failed: %s" % e, context, request)
        status = 'jobfailed'
    loginfo('bmu online submission finished', getJobfile(jobnumber), context, request)
    return status
Пример #2
0
def showresults(request):
    elapsedtime = 0.0
    context = setConstants(request, im)
    try:
        status = request.GET['status']
    except:
        status = 'showfile'
    filename = request.GET['filename']
    context['filename'] = filename
    context['jobstatus'] = request.GET['status']
    f = open(getJobfile(filename), 'r')
    filecontent = f.read()
    if status == 'showmedia':
        context['derivativegrid'] = 'Medium'
        context['sizegrid'] = '240px'
        context['imageserver'] = prmz.IMAGESERVER
        context['items'] = rendermedia(filecontent)
    elif status == 'showinportal':
        pass
    else:
        context['filecontent'] = reformat(filecontent)
    elapsedtime = time.time() - elapsedtime
    context = setContext(context, elapsedtime)

    return render(request, 'uploadmedia.html', context)
def deletejob(request, filename):
    try:
        remove(getJobfile(filename))
        loginfo('bmu', '%s :: %s' % ('uploadmedia job deleted', filename), {}, {})
    except:
        loginfo('bmu', '%s :: %s' % ('ERROR: uploadmedia tried and failed to delete job', filename), {}, {})
    return showqueue(request)
Пример #4
0
def deletejob(request, filename):
    try:
        remove(getJobfile(filename))
        logger.info('%s :: %s' % ('uploadmedia job deleted', filename))

    except:
        logger.info('%s :: %s' %
                    ('uploadmedia tried and failed to delete job', filename))
    return showqueue(request)
Пример #5
0
def checkimagefilenames(request):
    elapsedtime = time.time()
    context = setConstants(request, im)
    try:
        filename = request.GET['filename']
        (jobnumber, step, csv) = filename.split('.')
        context['jobnumber'] = jobnumber
        context['filename'] = filename
        file_handle = open(getJobfile(filename), 'r')
        lines = file_handle.read().splitlines()
        recordtypes = [
            tuple(list(f.split("|")[i] for i in [0, 2, 7, 8])) for f in lines
        ]
        seen = {}
        checked_objects = []
        for objitems in recordtypes[1:]:
            if objitems[1] in seen:
                checked_objects.append(
                    objitems + upload_type_check(seen[objitems], objitems))
            else:
                asquery = '%s?as=%s_common:%s%%3D%%27%s%%27&wf_deleted=false&pgSz=%s' % (
                    'collectionobjects', 'collectionobjects', 'objectNumber',
                    urllib.parse.quote_plus(objitems[1]), 10)
                (objecturl, objectx, dummy,
                 itemtime) = getfromCSpace(asquery, request)
                if objectx is None:
                    totalItems = 0
                else:
                    objectx = fromstring(objectx)
                    totalItems = objectx.find('.//totalItems')
                    totalItems = int(totalItems.text)
                checked_objects.append(objitems +
                                       upload_type_check(totalItems, objitems))
                seen[objitems[1]] = totalItems
        file_handle.close()
    except:
        raise
        checked_objects = []
    elapsedtime = time.time() - elapsedtime
    context = setContext(context, elapsedtime)
    context['objectnumbers'] = checked_objects

    return render(request, 'uploadmedia.html', context)
Пример #6
0
def checkimagefilenames(request):
    elapsedtime = time.time()
    context = setConstants(request, im)
    try:
        filename = request.GET['filename']
        (jobnumber, step, csv) = filename.split('.')
        context['jobnumber'] = jobnumber
        context['filename'] = filename
        file_handle = open(getJobfile(filename), 'r')
        lines = file_handle.read().splitlines()
        recordtypes = [f.split("|") for f in lines]
        filenames = [r[0] for r in recordtypes[1:]]
        objectnumbers = []
        seen = {}
        for o in filenames:
            objitems = getNumber(o, INSTITUTION)
            if objitems[1] in seen:
                objectnumbers.append(objitems + (seen[objitems[1]], ))
            else:
                asquery = '%s?as=%s_common:%s%%3D%%27%s%%27&wf_deleted=false&pgSz=%s' % (
                    'collectionobjects', 'collectionobjects', 'objectNumber',
                    urllib.parse.quote_plus(objitems[1]), 10)
                (objecturl, objectx, dummy,
                 itemtime) = getfromCSpace(asquery, request)
                if objectx is None:
                    totalItems = 0
                else:
                    objectx = fromstring(objectx)
                    totalItems = objectx.find('.//totalItems')
                    totalItems = int(totalItems.text)
                #objectcsids = [e.text for e in objectx.findall('.//csid')]
                objectnumbers.append(objitems + (totalItems, ))
                seen[objitems[1]] = totalItems
        file_handle.close()
    except:
        raise
        objectnumbers = []
    elapsedtime = time.time() - elapsedtime
    context = setContext(context, elapsedtime)
    context['objectnumbers'] = objectnumbers

    return render(request, 'uploadmedia.html', context)
Пример #7
0
def prepareFiles(request, BMUoptions, context):

    validateonly = 'validateonly' in request.POST

    jobnumber = context['jobnumber']
    jobinfo = {}
    images = []
    for lineno, afile in enumerate(request.FILES.getlist('imagefiles')):
        try:
            loginfo(
                'bmu',
                ("%s %s: %s %s (%s %s)" %
                 ('id', lineno + 1, 'name', afile.name, 'size', afile.size)),
                context, request)
            image = get_exif(afile)
            filename, objectnumber, imagenumber, extra = getNumber(
                afile.name, INSTITUTION)
            datetimedigitized, dummy = assignValue('', 'ifblank', image,
                                                   'DateTimeDigitized', {})
            imageinfo = {
                'id': lineno,
                'name': afile.name,
                'size': afile.size,
                'objectnumber': objectnumber,
                'imagenumber': imagenumber,
                # 'objectCSID': objectCSID,
                'date': datetimedigitized,
                'extra': extra
            }
            for override in BMUoptions['overrides']:
                dname, refname = assignValue(context[override[2]][0],
                                             context[override[2]][1], image,
                                             override[3], override[4])
                imageinfo[override[2]] = refname
                # add the Displayname just in case...
                imageinfo['%sDisplayname' % override[2]] = dname

            if not validateonly:
                handle_uploaded_file(afile)

            for option in ['handling', 'approvedforweb']:
                if option in request.POST:
                    imageinfo[option] = request.POST[option]
                else:
                    imageinfo[option] = ''

            if 'handling' in request.POST:
                handling = request.POST['handling']
                for parms in BMUoptions['bmuconstants'][handling]:
                    imageinfo[parms] = BMUoptions['bmuconstants'][handling][
                        parms]

                # special case:
                # borndigital media have their mh id numbers unconditionally replaced with a sequence number
                if imageinfo['handling'] == 'borndigital':
                    # for these, we create a media handling number...
                    # options considered were:
                    # DP-2015-10-08-12-16-43-0001 length: 27
                    # DP-201510081216430001 length: 21
                    # DP-2CBE859E990BFB1 length: 18
                    # DP-2015-10-08-12-16-43-0001 length: 27 the winner! (most legible)
                    mhnumber = jobnumber + ("-%0.4d" % (lineno + 1))
                    #mhnumber = hex(int(mhnumber.replace('-','')))[2:]
                    imageinfo['objectnumber'] = 'DP-' + mhnumber

            specialhandling(imageinfo, context, BMUoptions, INSTITUTION)
            images.append(imageinfo)

        except:
            raise
            if not validateonly:
                # we still upload the file, anyway...
                try:
                    handle_uploaded_file(afile)
                except:
                    sys.stderr.write("error! file=%s %s" %
                                     (afile.name, traceback.format_exc()))

            images.append({
                'name':
                afile.name,
                'size':
                afile.size,
                'error':
                'problem uploading file or extracting image metadata, not processed'
            })

    if len(images) > 0:
        jobinfo['jobnumber'] = jobnumber

        if not validateonly:
            writeCsv(
                getJobfile(jobnumber) + '.step1.csv', images, FIELDS2WRITE)
        jobinfo['estimatedtime'] = '%8.1f' % (len(images) * 10 / 60.0)

        if 'createmedia' in request.POST:
            jobinfo['status'] = 'createmedia'
            if not validateonly:
                loginfo('bmu', getJobfile(jobnumber), context, request)
                try:
                    file_is_OK = True
                    if INSTITUTION == 'cinefiles':
                        # test file content
                        input_file = getJobfile(jobnumber) + '.step1.csv'
                        report_file = getJobfile(jobnumber) + '.check.csv'
                        file_is_OK = doChecks(
                            ('', 'file', JOBDIR % '', input_file, report_file))
                        # if ok continue
                        # otherwise ... bail
                        if file_is_OK:
                            pass
                        else:
                            images = []
                            deletejob(request, jobnumber + '.step1.csv')
                            jobinfo['status'] = 'jobfailed'
                            loginfo('bmu ERROR:  process',
                                    jobnumber + " QC check failed.", context,
                                    request)
                    if file_is_OK:
                        retcode = subprocess.call([
                            path.join(POSTBLOBPATH, 'postblob.sh'),
                            INSTITUTION,
                            getJobfile(jobnumber), BATCHPARAMETERS
                        ])
                        if retcode < 0:
                            loginfo(
                                'bmu ERROR: process', jobnumber +
                                " Child was terminated by signal %s" %
                                -retcode, context, request)
                        else:
                            loginfo(
                                'bmu ERROR: process',
                                jobnumber + ": Child returned %s" % retcode,
                                context, request)
                except OSError as e:
                    jobinfo['status'] = 'jobfailed'
                    loginfo('error', "ERROR: Execution failed: %s" % e,
                            context, request)
                loginfo('bmu finish', getJobfile(jobnumber), context, request)

        elif 'uploadmedia' in request.POST:
            jobinfo['status'] = 'uploadmedia'
        else:
            jobinfo['status'] = 'No status possible'

    return jobinfo, images
Пример #8
0
def downloadresults(request, filename):
    f = open(getJobfile(filename), 'r')
    response = HttpResponse(f, content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="%s"' % filename
    return response
def prepareFiles(request, BMUoptions, context):

    jobnumber = context['jobnumber']
    jobinfo = {}
    images = []
    for lineno, afile in enumerate(request.FILES.getlist('imagefiles')):
        try:
            loginfo('bmu', ("%s %s: %s %s (%s %s)" % ('id', lineno + 1, 'name', afile.name, 'size', afile.size)), context, request)
            image = get_exif(afile)
            filename, objectnumber, imagenumber, extra = getNumber(afile.name, INSTITUTION)
            datetimedigitized, dummy = assignValue('', 'ifblank', image, 'DateTimeDigitized', {})
            imageinfo = {'id': lineno, 'name': afile.name, 'size': afile.size,
                         'objectnumber': objectnumber,
                         'imagenumber': imagenumber,
                         # 'objectCSID': objectCSID,
                         'date': datetimedigitized,
                         'extra': extra}
            for override in BMUoptions['overrides']:
                dname,refname = assignValue(context[override[2]][0], context[override[2]][1], image, override[3], override[4])
                imageinfo[override[2]] = refname
                # add the Displayname just in case...
                imageinfo['%sDisplayname' % override[2]] = dname

            handle_uploaded_file(afile)

            for option in ['handling', 'approvedforweb']:
                if option in request.POST:
                    imageinfo[option] = request.POST[option]
                else:
                    imageinfo[option] = ''

            if 'handling' in request.POST:
                handling = request.POST['handling']
                for parms in BMUoptions['bmuconstants'][handling]:
                    imageinfo[parms] = BMUoptions['bmuconstants'][handling][parms]

                # special case:
                # borndigital media have their mh id numbers unconditionally replaced with a sequence number
                if imageinfo['handling'] == 'borndigital':
                    # for these, we create a media handling number...
                    # options considered were:
                    # DP-2015-10-08-12-16-43-0001 length: 27
                    # DP-201510081216430001 length: 21
                    # DP-2CBE859E990BFB1 length: 18
                    # DP-2015-10-08-12-16-43-0001 length: 27 the winner! (most legible)
                    mhnumber = jobnumber + ("-%0.4d" % (lineno + 1))
                    #mhnumber = hex(int(mhnumber.replace('-','')))[2:]
                    imageinfo['objectnumber'] = 'DP-' + mhnumber

            specialhandling(imageinfo, context, BMUoptions, INSTITUTION)
            images.append(imageinfo)

        except:
            raise
            # we still upload the file, anyway...
            try:
                handle_uploaded_file(afile)
            except:
                sys.stderr.write("error! file=%s %s" % (afile.name, traceback.format_exc()))

            images.append({'name': afile.name, 'size': afile.size,
                           'error': 'problem uploading file or extracting image metadata, not processed'})

    if len(images) > 0:
        jobinfo['jobnumber'] = jobnumber

        writeCsv(getJobfile(jobnumber) + '.step1.csv', images, FIELDS2WRITE)
        jobinfo['estimatedtime'] = '%8.1f' % (len(images) * 10 / 60.0)

        if 'createmedia' in request.POST:
            jobinfo['status'] = 'createmedia'
            jobinfo['status'] = runjob(jobnumber, context, request)

        elif 'uploadmedia' in request.POST:
            jobinfo['status'] = 'uploadmedia'
        else:
            jobinfo['status'] = 'No status possible'

    return jobinfo, images