def d10mxfchecksum(inputs,outputs,options={},callbacks=[]):
try:
mfileid = inputs[0]
joboutput = outputs[0]
inputfile = _get_mfile(mfileid)
outputfile = tempfile.NamedTemporaryFile()
logging.info("Processing d10mxfchecksum job on %s" % (inputfile))
if not os.path.exists(inputfile):
logging.info("Inputfile %s does not exist" % (inputfile))
return False
args = ["d10sumchecker","-i",inputfile,"-o",outputfile.name]
ret = subprocess.call(args)
if ret != 0:
raise Exception("d10mxfchecksum failed")
outputfile.seek(0)
suf = SimpleUploadedFile("mfile",outputfile.read(), content_type='text/plain')
from jobservice.models import JobOutput
jo = JobOutput.objects.get(id=joboutput)
jo.file.save('d10mxfchecksum.txt', suf, save=True)
for callback in callbacks:
subtask(callback).delay()
return {"success":True,"message":"d10mxfchecksum successful"}
except Exception as e:
logging.info("Error with d10mxfchecksum %s" % e)
raise e
def extractfragment(inputs,outputs,options={},callbacks=[]):
try:
mfileid=inputs[0]
videopath=_get_mfile(mfileid)
tempout=tempfile.NamedTemporaryFile(suffix=".mp4")
logging.info("temp file: %s" % tempout.name)
intime=options["intime"]
fragmentlength=options["fragmentlength"]
# extract 'fragmentlength' video fragment starting at 'intime' (seconds)
# ffmpeg -ss 00:00:30.0 -t 00:00:10.0 -i input.wmv -acodec copy -vcodec copy -async 1 output.wmv
args = ["ffmpeg -y -ss",intime,"-t",fragmentlength,"-i",videopath,"-acodec copy -vcodec copy -async 1",tempout.name]
cmd = " ".join(args)
logging.info(cmd)
p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, close_fds=True)
(stdout,stderr) = p.communicate()
logging.info(stdout)
if p.returncode != 0:
raise Exception("Command %s exited with code %d. Stderr: %s" % (cmd, p.returncode, stderr))
# make job outputs available
_save_joboutput(outputs[0], tempout)
for callback in callbacks:
subtask(callback).delay()
return {"success":True, "message":"extractfragment successful"}
except Exception as e:
logging.info("Error with extractfragment %s." % e)
raise e
def sha1file(inputs,outputs,options={},callbacks=[]):
"""Return hex sha1 digest for a Django FieldFile"""
try:
mfileid = inputs[0]
path = _get_mfile(mfileid)
file = open(path,'r')
sha1 = hashlib.sha1()
while True:
data = file.read(8192) # multiple of 128 bytes is best
if not data:
break
sha1.update(data)
file.close()
sha1string = sha1.hexdigest()
logging.info("SHA1 calculated %s" % (sha1string))
# TODO: move to dataservice and store checksum in file?
#from dataservice.models import MFile
#_mf = MFile.objects.get(id=mfileid)
#_mf.checksum = md5string
#_mf.save()
for callback in callbacks:
logging.info("Running Callback %s" % callback)
subtask(callback).delay()
return {"success":True,"message":"SHA1 successful", "sha1" : sha1string}
except Exception, e:
logging.info("Error with sha1 %s" % e)
raise e
def ffmbc(inputs,outputs,options={},callbacks=[]):
try:
mfileid=inputs[0]
videopath=_get_mfile(mfileid)
tempout=tempfile.NamedTemporaryFile()
logging.info("temp file: %s" % tempout.name)
ffmpeg_args=options["args"]
# extract all I frames that are no closer than 5 seconds apart
args = ["ffmbc -y -i",videopath,ffmpeg_args,tempout.name]
cmd = " ".join(args)
logging.info(cmd)
p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, close_fds=True)
(stdout,stderr) = p.communicate()
logging.info(stdout)
if p.returncode != 0:
raise Exception("Command %s exited with code %d. Stderr: %s" % (cmd, p.returncode, stderr))
# make job outputs available
_save_joboutput(outputs[0], tempout)
for callback in callbacks:
subtask(callback).delay()
return {"success":True, "message":"ffmbc successful"}
except Exception as e:
logging.info("Error with ffmbc %s." % e)
raise e
def mxftechmdextractor(inputs,outputs,options={},callbacks=[]):
try:
mfileid = inputs[0]
inputfile = _get_mfile(mfileid)
joboutput = outputs[0]
logging.info("Processing mxftechmdextractor job on %s" % (inputfile))
if not os.path.exists(inputfile):
logging.info("Inputfile %s does not exist" % (inputfile))
return False
tempout=tempfile.NamedTemporaryFile()
logging.info("temp file: %s" % tempout.name)
args = ["java -cp /opt/rai/mxftechmdextractor/mxftechmdextractor-last.jar eu.prestoprime.mxftools.test.JustRun",inputfile]
cmd = " ".join(args)
#p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, close_fds=True)
p = Popen(cmd, shell=True, stdout=tempout, stderr=PIPE, close_fds=True)
p.wait()
if p.returncode != 0:
raise Exception("Command %s exited with code %d. Stderr: %s" % (cmd, p.returncode, stderr))
#(stdout, stderr) = p.communicate()
#print("stdout"+stdout)
#print("stderr"+stderr)
#tempout.close()
tempout.flush()
import ConfigParser
# ensure case is maintained
config = ConfigParser.RawConfigParser()
config.optionxform = str
config.read(tempout.name)
mxfinfo = dict(config.items("INFO"))
print(mxfinfo)
from jobservice.models import JobOutput
jo = JobOutput.objects.get(id=joboutput)
#jo.file.save('mxftechmdextractor.txt', ContentFile(p.stdout.read()), save=True)
jo.file.save('mxftechmdextractor', ContentFile(json.dumps(mxfinfo,indent=4)), save=True)
for callback in callbacks:
subtask(callback).delay()
return {"success":True,"message":"MXFTechMDExtractorPlugin successful"}
except Exception as e:
logging.info("Error with mxftechmdextractor. %s" % e)
raise e
def dumbtask(inputs,outputs,options={},callbacks=[]):
logging.info("Processing dumb task")
try:
mfileid = inputs[0]
filepath = _get_mfile(mfileid)
toutfile = tempfile.NamedTemporaryFile(delete=False,suffix=".txt")
joboutput = outputs[0]
retcode = subprocess.call(["wc",filepath,toutfile.name])
_save_joboutput(joboutput,toutfile)
return {"success":True,"message":"Dumb task successful"}
except Exception as e:
logging.info("Error with dumb task %s" % e)
raise e
def ffprobe(inputs,outputs,options={},callbacks=[]):
try:
mfileid = inputs[0]
inputfile = _get_mfile(mfileid)
joboutput = outputs[0]
logging.info("Processing ffprobe job on %s" % (inputfile))
if not os.path.exists(inputfile):
logging.info("Inputfile %s does not exist" % (inputfile))
return False
print_format=options["format"]
if print_format not in ["default","compact","csv","json","xml"]:
print_format="json"
# Requires ffprobe >=0.9
args = ["ffprobe -v quiet -print_format",print_format,"-show_format -show_streams",inputfile]
cmd = " ".join(args)
p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, close_fds=True)
# communicate() is preferred to stdout.read()
(stdout, stderr) = p.communicate()
if p.returncode != 0:
raise Exception("Command %s exited with code %d. Stderr: %s" % (cmd, p.returncode, stderr))
from jobservice.models import JobOutput
jo = JobOutput.objects.get(id=joboutput)
jo.file.save('ffprobe.txt', ContentFile(stdout), save=True)
#(stdout, stderr) = p.communicate()
#print(stdout)
#print(stderr)
#ffprobe_output = json.loads(stdout)
#ret = {"success":True}
#ret["ffprobe"] = ffprobe_output
for callback in callbacks:
subtask(callback).delay()
return {"success":True, "message":"ffprobe successful"}
except Exception as e:
logging.info("Error with ffprobe. %s" % e)
raise e
def swirl(inputs,outputs,options={},callbacks=[]):
try:
mfileid = inputs[0]
path = _get_mfile(mfileid)
logging.info("Swirling image for %s (%s)" % (input, path))
img = PythonMagick.Image()
img.read(str(path))
img.swirl(90)
# Create swirled image as job output
toutfile = tempfile.NamedTemporaryFile(delete=False,suffix=".jpg")
img.write(toutfile.name)
joboutput = outputs[0]
_save_joboutput(joboutput,toutfile)
return {"success":True,"message":"Swirl successful"}
except Exception ,e:
logging.info("Error with swirl %s" % e)
def imodel(inputs,outputs,options={},callbacks=[]):
mfileid = inputs[0]
path = _get_mfile(mfileid)
logging.info("CWD: %s" % os.getcwd())
logging.info("Running imodel on %s" % (path))
# Run iModel in a temporary directory
logging.info(os.environ)
#imodel_home = "/opt/iModel-1.0-beta-3-SNAPSHOT"
#imodel_home = os.environ["IMODEL_HOME"]
imodel_home = settings.IMODEL_HOME
logging.info("Running iModel from %s" % imodel_home)
(mfile_dir, mfile_name) = os.path.split(path)
# XXX: configuration.txt must be in CWD
# XXX: Runtime arguments should not be provided in a file
tempdir = tempfile.mkdtemp()
logging.info("iModel temp dir: %s" % (tempdir))
shutil.copy("imodel/configuration.txt", tempdir)
shutil.copy(path, tempdir)
p = subprocess.Popen(["java", "-cp", imodel_home + ":" + imodel_home+"/lib/*:" + imodel_home+"/bin", "uk.ac.soton.itinnovation.prestoprime.imodel.batch.start.StartArchiveSystemModel", mfile_name], cwd=tempdir, stdout=subprocess.PIPE)
# save stdout
stdoutfile = open(tempdir+"/stdout", 'w')
logging.info("Temp file for stdout: %s" % (stdoutfile.name))
(stdout, stderr) = p.communicate()
stdoutfile.write(stdout)
stdoutfile.close()
_save_joboutput(outputs[1], stdoutfile)
# Process results
import sys
sys.path.append("imodel")
import parseimodel
processedresultsfilename = tempdir+"/data.csv"
parseimodel.parse(tempdir+"/outputSystemPerformance.log", processedresultsfilename)
joboutput = outputs[0]
processedresultsfile = open(processedresultsfilename, 'r')
_save_joboutput(joboutput, processedresultsfile)
return {"success":True,"message":"iModel simulation successful"}
def extractd10frame(inputs,outputs,options={},callbacks=[],**kwargs):
try:
mfileid = inputs[0]
inputfile = _get_mfile(mfileid)
joboutput = outputs[0]
frame = str(options['frame'])
logging.info("Processing extractd10frame job on %s" % (inputfile))
if not os.path.exists(inputfile):
logging.info("Inputfile %s does not exist" % (inputfile))
return False
import pyffmpeg
stream = pyffmpeg.VideoStream()
stream.open(inputfile)
image = stream.GetFrameNo(frame)
# Save the thumbnail
temp_handle = StringIO()
image.save(temp_handle, 'png')
temp_handle.seek(0)
# Save to the thumbnail field
suf = SimpleUploadedFile("mfile",temp_handle.read(), content_type='image/png')
from jobservice.models import JobOutput
jo = JobOutput.objects.get(id=joboutput.pk)
jo.file.save('extractd10frame.png', suf , save=False)
jo.save()
for callback in callbacks:
subtask(callback).delay()
return {"success":True,"message":"extractd10frame successful"}
except Exception as e:
logging.info("Error with extractd10frame %s" % e)
raise e
def mxfframecount(inputs,outputs,options={},callbacks=[]):
try:
mfileid = inputs[0]
inputfile = _get_mfile(mfileid)
outputfile = tempfile.NamedTemporaryFile()
logging.info("Processing mxfframecount job on %s" % (inputfile))
if not os.path.exists(inputfile):
logging.info("Inputfile %s does not exist" % (inputfile))
return False
args = ["d10sumchecker","-i",inputfile,"-o",outputfile.name]
logging.info(args)
ret = subprocess.call(args)
if ret != 0:
raise Exception("mxfframecount failed")
frames = 0
for line in open(outputfile.name):
frames += 1
# TODO: subtract 1 for additional output
frames = frames -1
import dataservice.usage_store as usage_store
usage_store.record(mfileid,"http://prestoprime/mxf_frames_ingested",frames)
for callback in callbacks:
subtask(callback).delay()
return {"success":True,"message":"mxfframecount successful", "frames": frames }
except Exception as e:
logging.info("Error with mxfframecount %s" % e)
raise e
def extractkeyframes(inputs,outputs,options={},callbacks=[]):
try:
mfileid=inputs[0]
videopath=_get_mfile(mfileid)
tempdir = tempfile.mkdtemp()
logging.info("temp dir: %s" % (tempdir))
interval = options['interval']
# extract all I frames that are no closer than 'interval' seconds apart
ffmpeg_args = "-vf select='eq(pict_type\,I)*(isnan(prev_selected_t)+gte(t-prev_selected_t\,%s))' -vsync 0 -f image2" % (interval)
args = ["ffmpeg -i",videopath, ffmpeg_args, tempdir+"/%09d.jpg"]
cmd = " ".join(args)
p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, close_fds=True)
(stdout,stderr) = p.communicate()
logging.info(stdout)
if p.returncode != 0:
raise Exception("Command %s exited with code %d. Stderr: %s" % (cmd, p.returncode, stderr))
results_file = tempdir+"/results.zip"
zipdir(tempdir, results_file)
results = open(results_file, 'r')
# make job outputs available
_save_joboutput(outputs[0], results)
results.close()
for callback in callbacks:
subtask(callback).delay()
return {"success":True, "message":"keyframe extraction successful"}
except Exception as e:
logging.info("Error with keyframe extraction %s" % e)
raise e
