def check_xml_submits():
"""
@rtype : Boolean
@return:
"""
submitfiles = fflock_globals.NFS_PATH
if submitfiles[-1:] != "/":
submitfiles += "/"
submitfiles += "*.xml"
for file in glob.glob(submitfiles):
submitted = 0
print "Submitting jobs from: ", file
# parse job file
xmldoc = minidom.parse(file)
joblist = xmldoc.getElementsByTagName('job')
for s in joblist:
submitted = 1
type = s.attributes['type'].value
input = s.attributes['input'].value
output = s.attributes['output'].value
preoptions = s.attributes['preoptions'].value
options = s.attributes['options'].value
joboptions = s.attributes['joboptions'].value
encodercmd = fflock_globals.ENCODER
job_options = joboptions.split(",")
for option in job_options:
if option == "encoder=ffmbc": encodercmd = "ffmbc"
if option == "encoder=avconv": encodercmd = "avconv"
commandstring = "%s -y -i %s %s %s" % (encodercmd, "%s", "%s",
"%s")
if type == "transcode":
storageuuid = fflock_utility.find_server_storage_UUIDs(_uuid)
fflock_utility.ensure_dir(output, storageuuid)
fflock_utility.submit_job("", "Master", "transcode",
commandstring, preoptions, options,
input, output, "", "", joboptions)
# delete the submitted xml file
if submitted == 1:
os.remove(file)
return True
def check_xml_submits():
"""
@rtype : Boolean
@return:
"""
submitfiles = fflock_globals.NFS_PATH
if submitfiles[-1:] != "/":
submitfiles += "/"
submitfiles += "*.xml"
for file in glob.glob(submitfiles):
submitted = 0
print "Submitting jobs from: ", file
# parse job file
xmldoc = minidom.parse(file)
joblist = xmldoc.getElementsByTagName('job')
for s in joblist:
submitted = 1
type = s.attributes['type'].value
input = s.attributes['input'].value
output = s.attributes['output'].value
preoptions = s.attributes['preoptions'].value
options = s.attributes['options'].value
joboptions = s.attributes['joboptions'].value
encodercmd = fflock_globals.ENCODER
job_options = joboptions.split(",")
for option in job_options:
if option == "encoder=ffmbc": encodercmd = "ffmbc"
if option == "encoder=avconv": encodercmd = "avconv"
commandstring = "%s -y -i %s %s %s" % (encodercmd, "%s", "%s", "%s")
if type == "transcode":
storageuuid = fflock_utility.find_server_storage_UUIDs(_uuid)
fflock_utility.ensure_dir(output, storageuuid)
fflock_utility.submit_job("", "Master", "transcode", commandstring, preoptions, options, input,
output, "", "", joboptions)
# delete the submitted xml file
if submitted == 1:
os.remove(file)
return True
def upload_external_destination(source, joboutput, dependencies, masteruuid):
"""
@rtype : boolean
"""
upload_uuid = " "
if joboutput.startswith("ftp://"):
upload_uuid = fflock_utility.get_uuid()
fflock_utility.submit_job(upload_uuid, "Storage", "ftp upload", " ", " ", " ", source, joboutput, dependencies,
masteruuid, "")
if joboutput.startswith("s3://"):
upload_uuid = fflock_utility.get_uuid()
fflock_utility.submit_job(upload_uuid, "Storage", "s3 upload", " ", " ", " ", source, joboutput, dependencies,
masteruuid, "")
return upload_uuid
def download_external_source(jobuuid, storageuuid, jobinput, masteruuid):
"""
@rtype : boolean
"""
download_uuid = " "
jobfileinput = jobinput
if jobinput.startswith("http://") or jobinput.startswith("https://"):
download_uuid = fflock_utility.get_uuid()
jobfileinput = jobinput.split('/')[-1]
fflock_utility.submit_job(download_uuid, "Storage", "http download", " ", " ", " ", jobinput, jobfileinput, " ",
masteruuid, "")
if jobinput.startswith("ftp://"):
download_uuid = fflock_utility.get_uuid()
jobfileinput = jobinput.split('/')[-1]
fflock_utility.submit_job(download_uuid, "Storage", "ftp download", " ", " ", " ", jobinput, jobfileinput, " ",
masteruuid, "")
if jobinput.startswith("s3://"):
download_uuid = fflock_utility.get_uuid()
jobfileinput = jobinput.split('/')[-1]
fflock_utility.submit_job(download_uuid, "Storage", "s3 download", " ", " ", " ", jobinput, jobfileinput, " ",
masteruuid, "")
return download_uuid, jobfileinput
def split_transcode_job(jobuuid, command, commandpreoptions, commandoptions, jobinput, joboutput, storageuuid,
masteruuid, resultsvalue1, resultsvalue2, joboptions):
"""
@rtype : boolean
"""
keyframes = resultsvalue1.split(",")
keyframes_diff = resultsvalue2.split(",")
# determine how many active slaves exist
num_slaves = fflock_utility.number_of_registered_slaves()
# determine length of each sub-clip
storage_nfs_path = fflock_utility.get_storage_nfs_folder_path(storageuuid)
outfilename, outfileextension = os.path.splitext(joboutput)
merge_textfile = joboutput + "_mergeinput.txt"
mergefiletext = ""
keyframe_index = 0
end_of_keyframes = 0
# create a uuid for the merge file creation job
mergefile_uuid = fflock_utility.get_uuid()
merge_dependencies = str(mergefile_uuid) + ","
# create transcode jobs for each sub-clip
for num in range(0, num_slaves):
print "Splitting Job", jobuuid, "into part", num
preoptions = commandpreoptions
options = commandoptions
# if last keyframe, dont specify time period
if keyframes_diff[keyframe_index] == "-1":
end_of_keyframes = 1
ffmpeg_startstop = "-ss %f -y" % float(keyframes[keyframe_index])
else:
ffmpeg_startstop = "-ss %f -t %f" % (
float(keyframes[keyframe_index]), float(keyframes_diff[keyframe_index]))
#preoptions += ffmpeg_startstop
options += "-an "
options += ffmpeg_startstop
encodercmd = fflock_globals.ENCODER
job_options = joboptions.split(",")
for option in job_options:
if option == "encoder=ffmbc": encodercmd = "ffmbc"
if option == "encoder=avconv": encodercmd = "avconv"
commandstring = "%s -y -i %s %s %s" % (encodercmd, "%s", "%s", "%s")
slavejobuuid = fflock_utility.submit_job("", "Slave", "transcode", commandstring, preoptions, options, jobinput,
joboutput + "_part" + str(num) + outfileextension, "", masteruuid,
joboptions)
keyframe_index += 1
merge_dependencies += str(slavejobuuid)
merge_dependencies += ","
# write the merge textfile for ffmpeg concat
#mergefiletext += 'file ' + '"' + storage_nfs_path + joboutput + '_part' + str(num) + outfileextension + '"' + "\n"
mergefiletext += "file " + storage_nfs_path + joboutput + "_part" + str(num) + outfileextension + "\n"
# if number of keyframes is less than number of slave servers, break
if end_of_keyframes == 1:
break
# trim off the last comma on the dependencies list
if merge_dependencies[-1:] == ",":
merge_dependencies = merge_dependencies[:-1]
fflock_utility.submit_job(mergefile_uuid, "Slave", "write mergefile", " ", " ", " ", mergefiletext, merge_textfile,
"", masteruuid, "")
return merge_dependencies, merge_textfile
def fetch_jobs():
"""
@rtype : boolean
"""
jobcursor = _db.cursor()
jobcursor.execute(
"SELECT UUID, JobType, JobSubType, Command, CommandPreOptions, CommandOptions, JobInput, JobOutput, Assigned, State, AssignedServerUUID, StorageUUID, Priority, Dependencies, MasterUUID, Progress, ResultValue1, ResultValue2, JobOptions FROM Jobs")
jobresults = jobcursor.fetchall()
for jobrow in jobresults:
jobuuid = jobrow[0]
jobtype = jobrow[1]
jobsubtype = jobrow[2]
command = jobrow[3]
commandpreoptions = jobrow[4]
commandoptions = jobrow[5]
jobinput = jobrow[6]
joboutput = jobrow[7]
jobassigned = jobrow[8]
jobstate = jobrow[9]
jobassignedserveruuid = jobrow[10]
storageuuid = jobrow[11]
jobpriority = jobrow[12]
jobdependencies = jobrow[13]
masteruuid = jobrow[14]
jobprogress = jobrow[15]
resultsvalue1 = jobrow[16]
resultsvalue2 = jobrow[17]
joboptions = jobrow[18]
source_dependencies = ""
# deal with master jobs
if jobtype == "Master":
if jobsubtype == "transcode":
if jobstate == 0:
source_dependencies, jobinput = download_external_source("", storageuuid, jobinput, jobuuid)
updatecursor = _db.cursor()
updatecursor.execute("UPDATE Jobs SET State='%s' WHERE UUID='%s'" % (1, jobuuid))
detect_frames_job_uuid = fflock_utility.get_uuid()
fflock_utility.submit_job(detect_frames_job_uuid, "Slave", "detect frames", " ", commandpreoptions,
commandoptions, jobinput, joboutput, source_dependencies, jobuuid,
joboptions)
# if master job is in progress, check state of child jobs and delete them if they are done
if jobstate == 1:
child_jobs = 0
childcursor = _db.cursor()
childcursor.execute(
"SELECT JobSubType, MasterUUID, State FROM Jobs WHERE MasterUUID='%s'" % (jobuuid))
childresults = childcursor.fetchall()
for row in childresults:
if row[0] == "detect frames":
child_jobs += 1
elif row[2] < 2:
child_jobs += 1
if child_jobs == 0:
childcursor.execute("DELETE FROM Jobs WHERE MasterUUID='%s' AND State='%s'" % (jobuuid, 2))
childcursor.execute("UPDATE Jobs SET State='%s' WHERE UUID='%s'" % (2, jobuuid))
# master job's children are all finished
if jobstate == 2:
print "\n------ Job", jobuuid, "finished successfully ---------"
print "------ Source:", jobinput
print "------ Output:", joboutput
job_options = joboptions.split(",")
for option in job_options:
if option == "confirm_framecount":
if resultsvalue1 < resultsvalue2:
print "------ Transcoded file has", int(resultsvalue2) - int(
resultsvalue1), "more frames than the source."
if resultsvalue1 > resultsvalue2:
print "------ Transcoded file has", int(resultsvalue1) - int(
resultsvalue2), "less frames than the source."
if resultsvalue1 == resultsvalue2:
print "------ Source and Transcoded file have the same number of frames:", resultsvalue1
cleanup_uuid = fflock_utility.get_uuid()
fflock_utility.submit_job(cleanup_uuid, "Storage", "cleanup", " ", storageuuid, jobuuid, jobinput,
joboutput, " ", jobuuid, "")
# if detect frames job is done, initiate split-stitch transcode process
if jobtype == "Slave" and jobsubtype == "detect frames" and jobstate == 2:
deletecursor = _db.cursor()
deletecursor.execute("DELETE FROM Jobs WHERE UUID = '%s'" % jobuuid)
mux_dependencies = ""
upload_dependencies = ""
joboutput_original = joboutput
joboutput = remap_output(joboutput)
# create audio demux job
demuxjob_uuid = fflock_utility.get_uuid()
audio_demuxed_filetype = ".mkv"
audio_demuxed_file = joboutput + "_audio" + audio_demuxed_filetype
encodercmd = fflock_globals.ENCODER
job_options = joboptions.split(",")
for option in job_options:
if option == "encoder=ffmbc": encodercmd = "ffmbc"
if option == "encoder=avconv": encodercmd = "avconv"
commandstring = "%s %s -i %s %s %s" % (encodercmd, "%s", "%s", "%s", "%s")
preoptions = commandpreoptions + "-y "
options = commandoptions + "-vn -strict -2"
fflock_utility.submit_job(demuxjob_uuid, "Slave", "audio demux", commandstring, preoptions, options,
jobinput,
audio_demuxed_file,
mux_dependencies, masteruuid, "")
mux_dependencies += str(demuxjob_uuid)
merge_dependencies, merge_textfile = split_transcode_job(jobuuid, command, commandpreoptions,
commandoptions, jobinput, joboutput, storageuuid,
masteruuid, resultsvalue1, resultsvalue2,
joboptions)
# create merge job
mergejob_uuid = fflock_utility.get_uuid()
outfilename, outfileextension = os.path.splitext(joboutput)
joboutput_video = joboutput + "_video" + outfileextension
fflock_utility.submit_job(mergejob_uuid, "Storage", "video merge", "ffmpeg %s -i %s %s %s", "-y -f concat",
"-c copy", merge_textfile, joboutput_video,
merge_dependencies, masteruuid, "")
if mux_dependencies[-1:] != ",":
mux_dependencies += ","
mux_dependencies += str(mergejob_uuid)
# create audio/video mux job
muxinput = joboutput_video + "," + audio_demuxed_file
muxjob_uuid = fflock_utility.get_uuid()
upload_dependencies += str(muxjob_uuid)
fflock_utility.submit_job(muxjob_uuid, "Storage", "a/v mux", "ffmpeg %s %s %s %s", "-y",
"-vcodec copy -acodec copy -strict -2", muxinput, joboutput, mux_dependencies,
masteruuid, "")
job_options = fflock_utility.find_job_options_for_job(masteruuid).split(",")
for option in job_options:
if option == "confirm_framecount":
framecount1_uuid = fflock_utility.get_uuid()
fflock_utility.submit_job(framecount1_uuid, "Slave", "count frames",
"ffprobe -show_frames %s | grep -c media_type=video", " ", "input",
jobinput,
" ", muxjob_uuid, masteruuid, "")
if len(str(upload_dependencies)) > 1 and upload_dependencies[-1:] != ",":
upload_dependencies += ","
upload_dependencies += str(framecount1_uuid)
framecount2_uuid = fflock_utility.get_uuid()
fflock_utility.submit_job(framecount2_uuid, "Slave", "count frames",
"ffprobe -show_frames %s | grep -c media_type=video", " ", "output",
joboutput,
" ", muxjob_uuid, masteruuid, "")
if upload_dependencies[-1:] != ",":
upload_dependencies += ","
upload_dependencies += str(framecount2_uuid)
upload_uuid = upload_external_destination(joboutput, joboutput_original,
upload_dependencies, masteruuid)
return True