def runjob(jobnumber, context, request):
loginfo('bmu online job submission started', getJobfile(jobnumber), context, request)
status = 'jobstarted'
try:
file_is_OK = True
if INSTITUTION == 'cinefiles':
# test file content
input_file = getJobfile(jobnumber) + '.step1.csv'
report_file = getJobfile(jobnumber) + '.check.csv'
file_is_OK = doChecks(('', 'file', JOBDIR % '', input_file, report_file))
# if ok continue
# otherwise ... bail
if file_is_OK:
pass
else:
deletejob(request, jobnumber + '.step1.csv')
loginfo('bmu ERROR: process', jobnumber + " QC check failed.", context, request)
status = 'jobfailed'
if file_is_OK:
p_object = subprocess.Popen([path.join(POSTBLOBPATH, 'postblob.sh'), INSTITUTION, getJobfile(jobnumber), BATCHPARAMETERS])
pid = ''
if p_object._child_created:
pid = p_object.pid
loginfo('bmu online job submitted:', jobnumber + f': Child returned {p_object.returncode}, pid {pid}', context, request)
else:
loginfo('bmu ERROR:', jobnumber + f': Child returned {p_object.returncode}, pid {pid}', context, request)
except OSError as e:
loginfo('error', "ERROR: Execution failed: %s" % e, context, request)
status = 'jobfailed'
loginfo('bmu online submission finished', getJobfile(jobnumber), context, request)
return status
def showresults(request):
elapsedtime = 0.0
context = setConstants(request, im)
try:
status = request.GET['status']
except:
status = 'showfile'
filename = request.GET['filename']
context['filename'] = filename
context['jobstatus'] = request.GET['status']
f = open(getJobfile(filename), 'r')
filecontent = f.read()
if status == 'showmedia':
context['derivativegrid'] = 'Medium'
context['sizegrid'] = '240px'
context['imageserver'] = prmz.IMAGESERVER
context['items'] = rendermedia(filecontent)
elif status == 'showinportal':
pass
else:
context['filecontent'] = reformat(filecontent)
elapsedtime = time.time() - elapsedtime
context = setContext(context, elapsedtime)
return render(request, 'uploadmedia.html', context)
def deletejob(request, filename):
try:
remove(getJobfile(filename))
loginfo('bmu', '%s :: %s' % ('uploadmedia job deleted', filename), {}, {})
except:
loginfo('bmu', '%s :: %s' % ('ERROR: uploadmedia tried and failed to delete job', filename), {}, {})
return showqueue(request)
def deletejob(request, filename):
try:
remove(getJobfile(filename))
logger.info('%s :: %s' % ('uploadmedia job deleted', filename))
except:
logger.info('%s :: %s' %
('uploadmedia tried and failed to delete job', filename))
return showqueue(request)
def checkimagefilenames(request):
elapsedtime = time.time()
context = setConstants(request, im)
try:
filename = request.GET['filename']
(jobnumber, step, csv) = filename.split('.')
context['jobnumber'] = jobnumber
context['filename'] = filename
file_handle = open(getJobfile(filename), 'r')
lines = file_handle.read().splitlines()
recordtypes = [
tuple(list(f.split("|")[i] for i in [0, 2, 7, 8])) for f in lines
]
seen = {}
checked_objects = []
for objitems in recordtypes[1:]:
if objitems[1] in seen:
checked_objects.append(
objitems + upload_type_check(seen[objitems], objitems))
else:
asquery = '%s?as=%s_common:%s%%3D%%27%s%%27&wf_deleted=false&pgSz=%s' % (
'collectionobjects', 'collectionobjects', 'objectNumber',
urllib.parse.quote_plus(objitems[1]), 10)
(objecturl, objectx, dummy,
itemtime) = getfromCSpace(asquery, request)
if objectx is None:
totalItems = 0
else:
objectx = fromstring(objectx)
totalItems = objectx.find('.//totalItems')
totalItems = int(totalItems.text)
checked_objects.append(objitems +
upload_type_check(totalItems, objitems))
seen[objitems[1]] = totalItems
file_handle.close()
except:
raise
checked_objects = []
elapsedtime = time.time() - elapsedtime
context = setContext(context, elapsedtime)
context['objectnumbers'] = checked_objects
return render(request, 'uploadmedia.html', context)
def checkimagefilenames(request):
elapsedtime = time.time()
context = setConstants(request, im)
try:
filename = request.GET['filename']
(jobnumber, step, csv) = filename.split('.')
context['jobnumber'] = jobnumber
context['filename'] = filename
file_handle = open(getJobfile(filename), 'r')
lines = file_handle.read().splitlines()
recordtypes = [f.split("|") for f in lines]
filenames = [r[0] for r in recordtypes[1:]]
objectnumbers = []
seen = {}
for o in filenames:
objitems = getNumber(o, INSTITUTION)
if objitems[1] in seen:
objectnumbers.append(objitems + (seen[objitems[1]], ))
else:
asquery = '%s?as=%s_common:%s%%3D%%27%s%%27&wf_deleted=false&pgSz=%s' % (
'collectionobjects', 'collectionobjects', 'objectNumber',
urllib.parse.quote_plus(objitems[1]), 10)
(objecturl, objectx, dummy,
itemtime) = getfromCSpace(asquery, request)
if objectx is None:
totalItems = 0
else:
objectx = fromstring(objectx)
totalItems = objectx.find('.//totalItems')
totalItems = int(totalItems.text)
#objectcsids = [e.text for e in objectx.findall('.//csid')]
objectnumbers.append(objitems + (totalItems, ))
seen[objitems[1]] = totalItems
file_handle.close()
except:
raise
objectnumbers = []
elapsedtime = time.time() - elapsedtime
context = setContext(context, elapsedtime)
context['objectnumbers'] = objectnumbers
return render(request, 'uploadmedia.html', context)
def prepareFiles(request, BMUoptions, context):
validateonly = 'validateonly' in request.POST
jobnumber = context['jobnumber']
jobinfo = {}
images = []
for lineno, afile in enumerate(request.FILES.getlist('imagefiles')):
try:
loginfo(
'bmu',
("%s %s: %s %s (%s %s)" %
('id', lineno + 1, 'name', afile.name, 'size', afile.size)),
context, request)
image = get_exif(afile)
filename, objectnumber, imagenumber, extra = getNumber(
afile.name, INSTITUTION)
datetimedigitized, dummy = assignValue('', 'ifblank', image,
'DateTimeDigitized', {})
imageinfo = {
'id': lineno,
'name': afile.name,
'size': afile.size,
'objectnumber': objectnumber,
'imagenumber': imagenumber,
# 'objectCSID': objectCSID,
'date': datetimedigitized,
'extra': extra
}
for override in BMUoptions['overrides']:
dname, refname = assignValue(context[override[2]][0],
context[override[2]][1], image,
override[3], override[4])
imageinfo[override[2]] = refname
# add the Displayname just in case...
imageinfo['%sDisplayname' % override[2]] = dname
if not validateonly:
handle_uploaded_file(afile)
for option in ['handling', 'approvedforweb']:
if option in request.POST:
imageinfo[option] = request.POST[option]
else:
imageinfo[option] = ''
if 'handling' in request.POST:
handling = request.POST['handling']
for parms in BMUoptions['bmuconstants'][handling]:
imageinfo[parms] = BMUoptions['bmuconstants'][handling][
parms]
# special case:
# borndigital media have their mh id numbers unconditionally replaced with a sequence number
if imageinfo['handling'] == 'borndigital':
# for these, we create a media handling number...
# options considered were:
# DP-2015-10-08-12-16-43-0001 length: 27
# DP-201510081216430001 length: 21
# DP-2CBE859E990BFB1 length: 18
# DP-2015-10-08-12-16-43-0001 length: 27 the winner! (most legible)
mhnumber = jobnumber + ("-%0.4d" % (lineno + 1))
#mhnumber = hex(int(mhnumber.replace('-','')))[2:]
imageinfo['objectnumber'] = 'DP-' + mhnumber
specialhandling(imageinfo, context, BMUoptions, INSTITUTION)
images.append(imageinfo)
except:
raise
if not validateonly:
# we still upload the file, anyway...
try:
handle_uploaded_file(afile)
except:
sys.stderr.write("error! file=%s %s" %
(afile.name, traceback.format_exc()))
images.append({
'name':
afile.name,
'size':
afile.size,
'error':
'problem uploading file or extracting image metadata, not processed'
})
if len(images) > 0:
jobinfo['jobnumber'] = jobnumber
if not validateonly:
writeCsv(
getJobfile(jobnumber) + '.step1.csv', images, FIELDS2WRITE)
jobinfo['estimatedtime'] = '%8.1f' % (len(images) * 10 / 60.0)
if 'createmedia' in request.POST:
jobinfo['status'] = 'createmedia'
if not validateonly:
loginfo('bmu', getJobfile(jobnumber), context, request)
try:
file_is_OK = True
if INSTITUTION == 'cinefiles':
# test file content
input_file = getJobfile(jobnumber) + '.step1.csv'
report_file = getJobfile(jobnumber) + '.check.csv'
file_is_OK = doChecks(
('', 'file', JOBDIR % '', input_file, report_file))
# if ok continue
# otherwise ... bail
if file_is_OK:
pass
else:
images = []
deletejob(request, jobnumber + '.step1.csv')
jobinfo['status'] = 'jobfailed'
loginfo('bmu ERROR: process',
jobnumber + " QC check failed.", context,
request)
if file_is_OK:
retcode = subprocess.call([
path.join(POSTBLOBPATH, 'postblob.sh'),
INSTITUTION,
getJobfile(jobnumber), BATCHPARAMETERS
])
if retcode < 0:
loginfo(
'bmu ERROR: process', jobnumber +
" Child was terminated by signal %s" %
-retcode, context, request)
else:
loginfo(
'bmu ERROR: process',
jobnumber + ": Child returned %s" % retcode,
context, request)
except OSError as e:
jobinfo['status'] = 'jobfailed'
loginfo('error', "ERROR: Execution failed: %s" % e,
context, request)
loginfo('bmu finish', getJobfile(jobnumber), context, request)
elif 'uploadmedia' in request.POST:
jobinfo['status'] = 'uploadmedia'
else:
jobinfo['status'] = 'No status possible'
return jobinfo, images
def downloadresults(request, filename):
f = open(getJobfile(filename), 'r')
response = HttpResponse(f, content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="%s"' % filename
return response
def prepareFiles(request, BMUoptions, context):
jobnumber = context['jobnumber']
jobinfo = {}
images = []
for lineno, afile in enumerate(request.FILES.getlist('imagefiles')):
try:
loginfo('bmu', ("%s %s: %s %s (%s %s)" % ('id', lineno + 1, 'name', afile.name, 'size', afile.size)), context, request)
image = get_exif(afile)
filename, objectnumber, imagenumber, extra = getNumber(afile.name, INSTITUTION)
datetimedigitized, dummy = assignValue('', 'ifblank', image, 'DateTimeDigitized', {})
imageinfo = {'id': lineno, 'name': afile.name, 'size': afile.size,
'objectnumber': objectnumber,
'imagenumber': imagenumber,
# 'objectCSID': objectCSID,
'date': datetimedigitized,
'extra': extra}
for override in BMUoptions['overrides']:
dname,refname = assignValue(context[override[2]][0], context[override[2]][1], image, override[3], override[4])
imageinfo[override[2]] = refname
# add the Displayname just in case...
imageinfo['%sDisplayname' % override[2]] = dname
handle_uploaded_file(afile)
for option in ['handling', 'approvedforweb']:
if option in request.POST:
imageinfo[option] = request.POST[option]
else:
imageinfo[option] = ''
if 'handling' in request.POST:
handling = request.POST['handling']
for parms in BMUoptions['bmuconstants'][handling]:
imageinfo[parms] = BMUoptions['bmuconstants'][handling][parms]
# special case:
# borndigital media have their mh id numbers unconditionally replaced with a sequence number
if imageinfo['handling'] == 'borndigital':
# for these, we create a media handling number...
# options considered were:
# DP-2015-10-08-12-16-43-0001 length: 27
# DP-201510081216430001 length: 21
# DP-2CBE859E990BFB1 length: 18
# DP-2015-10-08-12-16-43-0001 length: 27 the winner! (most legible)
mhnumber = jobnumber + ("-%0.4d" % (lineno + 1))
#mhnumber = hex(int(mhnumber.replace('-','')))[2:]
imageinfo['objectnumber'] = 'DP-' + mhnumber
specialhandling(imageinfo, context, BMUoptions, INSTITUTION)
images.append(imageinfo)
except:
raise
# we still upload the file, anyway...
try:
handle_uploaded_file(afile)
except:
sys.stderr.write("error! file=%s %s" % (afile.name, traceback.format_exc()))
images.append({'name': afile.name, 'size': afile.size,
'error': 'problem uploading file or extracting image metadata, not processed'})
if len(images) > 0:
jobinfo['jobnumber'] = jobnumber
writeCsv(getJobfile(jobnumber) + '.step1.csv', images, FIELDS2WRITE)
jobinfo['estimatedtime'] = '%8.1f' % (len(images) * 10 / 60.0)
if 'createmedia' in request.POST:
jobinfo['status'] = 'createmedia'
jobinfo['status'] = runjob(jobnumber, context, request)
elif 'uploadmedia' in request.POST:
jobinfo['status'] = 'uploadmedia'
else:
jobinfo['status'] = 'No status possible'
return jobinfo, images