def setupSequenceJob(qubeJobTemplate,
sequenceInitFile,
outputFile,
preset,
selfContained=True,
frameRange='ALL',
audioFile='',
smartUpdate=True,
fillMissingFrames=True,
transcoderFolder='',
segmentDuration=200,
maxSegmentsPerOutput=20,
maxSegmentTolerance=5):
'''
Setup a qube job dictionary based on the input.
Required Inputs:
qubeJobTemplate (dictionary)
Template qube job dictionary to build from.
sequenceInitFile (string)
One image from the input image sequence.
Can be any image from the sequence.
outputFile (string)
The destination file for the output
of the transcoder.
preset (string)
The blender file that serves as the template
for the transcoding process.
Optional:
selfContained (boolean)
Determines if the outputFile should be a
reference quicktime movie or self-contained.
Self-contained movies are much larger and take
more time to create. Referenced quicktimes
are much smaller, and much faster to create.
However, referenced quicktimes must maintain
their connectiong their associated inputs.
frameRange (string)
The output frame range to render from the input
image sequence. Ex: 1-10
audioFile (string)
The audio file to be added to the output file.
This audio should match the exact timeline of
the input image sequence.
smartUpdate (boolean)
Automatically update only the segments and outputs
that have been changed since the last transcode.
transcoderFolder (string)
The folder in which to store all files related
to the transcoding process. This includes the
segmented movies and the blender projects. If
creating a referenced output file, these are
the segments that movie will reference.
fillMissingFrames (boolean)
Automatically fill in missing frames with the
last frame that exists. This is useful for
creating quicktimes from sequences rendered on
every nth frame.
Advanced:
segmentDuration (integer)
Frame count for each segment.
maxSegmentsPerOutput (integer)
Maximum number of segments that can be in each
output file. If the number of segments needed
for the sequence exceeds this amount, the output
file is split into multiple segments of this
length.
maxSegmentTolerance (integer)
If the maxSegmentsPerOutput limit is reached,
check that the input sequence exceeds this tolerance
value as well. If not, keep the outputFile as one file.
Agenda
The agenda is setup in 3 main sections:
Initialization:
Purpose
This single subjobs loads the input sequence
into the provided blender scene preset.
This is done once, then all subsequent
jobs reference the resulting scene file.
Package
None
resultPackage
None
Naming
Initialize
Segments:
Purpose
These subjobs each create their assigned
segment of the image sequence.
Package
frameRange (string)
Range of frames to render for this segment.
segmentFile (string)
Destination path for the segment file.
resultPackage
changes (boolean)
Returns if any changes were made for
this segment.
segmentFile (string)
Destination path for the segment file
that actually rendered. Sometimes file
issues occur where the output file can't
be overwritten, so we automatically
compensate for this.
Naming
Segment: (frameRange)
Final Outputs:
Purpose
These subjobs render the output files.
They are split up based on the number of segments
and the max segments per output. They are placed
in the agenda right after their dependent segments
have been processed.
Package
segmentSubjobs (list of strings)
List of the names of the dependant
segment subjobs.
outputFile (string)
destination for the output
resultPackage
outputPaths (string)
Path to the final output file.
Naming
Output: (outputFile)
Callbacks
Callbacks are added to unblock subjobs when they are
ready to be processed.
Initialization subjob completion
Once the initialization is complete, all
segment subjobs are unblocked.
Segment subjobs complete.
Once all segments that pertain to a final
output are complete, that output subjob
is unblocked.
Job retried
If the job is retried
'''
''' ---- Pre-Processing For Agenda ---- '''
logger.debug('Setup Sequence: ' + str(locals()))
''' General '''
mySequence = sequenceTools.Sequence(sequenceInitFile, frameRange)
sequenceName = mySequence.getName()
if not transcoderFolder:
transcoderFolder = os.path.join(os.path.dirname(outputFile),
'_Transcoder/')
''' Initialize '''
init = qb.Work()
init['name'] = 'Initialize'
''' Segments
Use the qube chunk method to split up the frame range.
Then prep each segment:
Add the frameRange to the package.
Add the segmentFile to the package.
Change the subjob name to Segment: (frameRange)
Submit as blocked, because they will be unblocked
once the initialize command is completed.
'''
segments = qb.genchunks(segmentDuration,
'1-' + str(mySequence.getDuration()))
for segment in segments:
segment['package'] = {}
segment['package']['frameRange'] = segment['name']
outputFolder, outputName, outputExtension = splitPath(outputFile)
segmentFile = os.path.join(transcoderFolder, 'Segments/')
segmentFile += outputName + '/'
segmentFile += "Segment" + segment['name'].split(
'-')[0] + outputExtension
segment['package']['segmentFile'] = segmentFile
segment['status'] = 'blocked'
segment['name'] = 'Segment:' + segment['name']
''' Final Outputs '''
finalOutputSegments = chunkWithTolerance(segments, maxSegmentsPerOutput,
maxSegmentTolerance)
finalOutputs = []
count = 1
for outputSegment in finalOutputSegments:
output = qb.Work()
output['package'] = {}
segmentSubjobs = []
for segment in outputSegment:
segmentSubjobs.append(segment['name'])
output['package']['segmentSubjobs'] = segmentSubjobs
outputFolder, outputName, outputExtension = splitPath(outputFile)
finalOutputFile = outputFolder + outputName
if len(finalOutputSegments) > 1:
finalOutputFile += '_' + chr(64 + count)
finalOutputFile += outputExtension
output['package']['outputFile'] = finalOutputFile
output['status'] = 'blocked'
output['name'] = 'Output: ' + os.path.basename(finalOutputFile)
count += 1
finalOutputs.append(output)
'''
Callbacks
1 - Unblock the segments when the initialize command is completed.
2 - Unblock the outputs when the dependant segments are completed.
'''
callbacks = []
''' Unblock Segments '''
callback = {}
callback['triggers'] = 'complete-work-self-Initialize'
callback['language'] = 'python'
code = 'import qb\n'
for segment in segments:
code += '%s%s%s' % ('\nqb.workunblock(\'%s:', segment['name'],
'\' % qb.jobid())')
code += '\nqb.unblock(qb.jobid())'
callback['code'] = code
callbacks.append(callback)
''' Unblock Outputs '''
for finalOutput in finalOutputs:
callback = {}
triggers = []
for segment in finalOutput['package']['segmentSubjobs']:
triggers.append('complete-work-self-' + segment)
callback['triggers'] = ' and '.join(triggers)
callback['language'] = 'python'
code = 'import qb\n'
code += '%s%s%s' % ('\nqb.workunblock(\'%s:', finalOutput['name'],
'\' % qb.jobid())')
code += '\nqb.unblock(qb.jobid())'
callback['code'] = code
callbacks.append(callback)
''' ---- Now put the job together ---- '''
job = qubeJobTemplate
''' General '''
job['name'] = 'Quicktime: ' + sequenceName
job['prototype'] = 'Submit Transcoder'
''' Package '''
job['package'] = {}
job['package']['sequence'] = sequenceInitFile
job['package']['audioFile'] = audioFile
job['package']['outputFile'] = outputFile
job['package']['preset'] = preset
job['package']['selfContained'] = selfContained
job['package']['smartUpdate'] = smartUpdate
job['package']['fillMissingFrames'] = fillMissingFrames
job['package']['frameRange'] = '1-' + str(mySequence.getDuration())
job['package']['transcoderFolder'] = transcoderFolder
''' Agenda '''
job['agenda'] = []
job['agenda'].append(init)
job['agenda'].extend(segments)
''' Place the final outputs after their last segment. '''
for outputNum, output in enumerate(finalOutputs):
lastSegmentName = output['package']['segmentSubjobs'][-1]
lastSegmentIndex = None
for index, segment in enumerate(segments):
if segment['name'] == lastSegmentName:
lastSegmentIndex = index
break
if lastSegmentIndex != None:
job['agenda'].insert(
lastSegmentIndex + 2 + outputNum,
output) # +2 for Initialization and last segment
else:
logger.error("ERROR: Unable to find last segment for output " +
output['name'])
''' Callbacks '''
if not job.get('callbacks', None):
job['callbacks'] = []
job['callbacks'].extend(callbacks)
return job
def setupSequenceJob(qubeJobTemplate, sequenceInitFile, outputFile, preset,
selfContained=True, frameRange='ALL', audioFile='',
smartUpdate=True, fillMissingFrames=True, transcoderFolder='',
segmentDuration=200, maxSegmentsPerOutput=20, maxSegmentTolerance=5):
'''
Setup a qube job dictionary based on the input.
Required Inputs:
qubeJobTemplate (dictionary)
Template qube job dictionary to build from.
sequenceInitFile (string)
One image from the input image sequence.
Can be any image from the sequence.
outputFile (string)
The destination file for the output
of the transcoder.
preset (string)
The blender file that serves as the template
for the transcoding process.
Optional:
selfContained (boolean)
Determines if the outputFile should be a
reference quicktime movie or self-contained.
Self-contained movies are much larger and take
more time to create. Referenced quicktimes
are much smaller, and much faster to create.
However, referenced quicktimes must maintain
their connectiong their associated inputs.
frameRange (string)
The output frame range to render from the input
image sequence. Ex: 1-10
audioFile (string)
The audio file to be added to the output file.
This audio should match the exact timeline of
the input image sequence.
smartUpdate (boolean)
Automatically update only the segments and outputs
that have been changed since the last transcode.
transcoderFolder (string)
The folder in which to store all files related
to the transcoding process. This includes the
segmented movies and the blender projects. If
creating a referenced output file, these are
the segments that movie will reference.
fillMissingFrames (boolean)
Automatically fill in missing frames with the
last frame that exists. This is useful for
creating quicktimes from sequences rendered on
every nth frame.
Advanced:
segmentDuration (integer)
Frame count for each segment.
maxSegmentsPerOutput (integer)
Maximum number of segments that can be in each
output file. If the number of segments needed
for the sequence exceeds this amount, the output
file is split into multiple segments of this
length.
maxSegmentTolerance (integer)
If the maxSegmentsPerOutput limit is reached,
check that the input sequence exceeds this tolerance
value as well. If not, keep the outputFile as one file.
Agenda
The agenda is setup in 3 main sections:
Initialization:
Purpose
This single subjobs loads the input sequence
into the provided blender scene preset.
This is done once, then all subsequent
jobs reference the resulting scene file.
Package
None
resultPackage
None
Naming
Initialize
Segments:
Purpose
These subjobs each create their assigned
segment of the image sequence.
Package
frameRange (string)
Range of frames to render for this segment.
segmentFile (string)
Destination path for the segment file.
resultPackage
changes (boolean)
Returns if any changes were made for
this segment.
segmentFile (string)
Destination path for the segment file
that actually rendered. Sometimes file
issues occur where the output file can't
be overwritten, so we automatically
compensate for this.
Naming
Segment: (frameRange)
Final Outputs:
Purpose
These subjobs render the output files.
They are split up based on the number of segments
and the max segments per output. They are placed
in the agenda right after their dependent segments
have been processed.
Package
segmentSubjobs (list of strings)
List of the names of the dependant
segment subjobs.
outputFile (string)
destination for the output
resultPackage
outputPaths (string)
Path to the final output file.
Naming
Output: (outputFile)
Callbacks
Callbacks are added to unblock subjobs when they are
ready to be processed.
Initialization subjob completion
Once the initialization is complete, all
segment subjobs are unblocked.
Segment subjobs complete.
Once all segments that pertain to a final
output are complete, that output subjob
is unblocked.
Job retried
If the job is retried
'''
''' ---- Pre-Processing For Agenda ---- '''
logger.debug('Setup Sequence: ' + str(locals()))
''' General '''
mySequence = sequenceTools.Sequence(sequenceInitFile, frameRange)
sequenceName = mySequence.getName()
if not transcoderFolder:
transcoderFolder = os.path.join(os.path.dirname(outputFile), '_Transcoder/')
''' Initialize '''
init = qb.Work()
init['name'] = 'Initialize'
''' Segments
Use the qube chunk method to split up the frame range.
Then prep each segment:
Add the frameRange to the package.
Add the segmentFile to the package.
Change the subjob name to Segment: (frameRange)
Submit as blocked, because they will be unblocked
once the initialize command is completed.
'''
segments = qb.genchunks(segmentDuration, '1-' + str(mySequence.getDuration()))
for segment in segments:
segment['package']= {}
segment['package']['frameRange'] = segment['name']
outputFolder, outputName, outputExtension = splitPath(outputFile)
segmentFile = os.path.join(transcoderFolder, 'Segments/')
segmentFile += outputName + '/'
segmentFile += "Segment" + segment['name'].split('-')[0] + outputExtension
segment['package']['segmentFile'] = segmentFile
segment['status'] = 'blocked'
segment['name'] = 'Segment:' + segment['name']
''' Final Outputs '''
finalOutputSegments = chunkWithTolerance(segments, maxSegmentsPerOutput, maxSegmentTolerance)
finalOutputs = []
count = 1
for outputSegment in finalOutputSegments:
output = qb.Work()
output['package'] = {}
segmentSubjobs = []
for segment in outputSegment:
segmentSubjobs.append(segment['name'])
output['package']['segmentSubjobs'] = segmentSubjobs
outputFolder, outputName, outputExtension = splitPath(outputFile)
finalOutputFile = outputFolder + outputName
if len(finalOutputSegments) > 1:
finalOutputFile += '_' + chr(64+count)
finalOutputFile += outputExtension
output['package']['outputFile'] = finalOutputFile
output['status'] = 'blocked'
output['name'] = 'Output: ' + os.path.basename(finalOutputFile)
count += 1
finalOutputs.append(output)
'''
Callbacks
1 - Unblock the segments when the initialize command is completed.
2 - Unblock the outputs when the dependant segments are completed.
'''
callbacks = []
''' Unblock Segments '''
callback = {}
callback['triggers'] = 'complete-work-self-Initialize'
callback['language'] = 'python'
code = 'import qb\n'
for segment in segments:
code += '%s%s%s' % ('\nqb.workunblock(\'%s:', segment['name'], '\' % qb.jobid())')
code += '\nqb.unblock(qb.jobid())'
callback['code'] = code
callbacks.append(callback)
''' Unblock Outputs '''
for finalOutput in finalOutputs:
callback = {}
triggers = []
for segment in finalOutput['package']['segmentSubjobs']:
triggers.append('complete-work-self-' + segment)
callback['triggers'] = ' and '.join(triggers)
callback['language'] = 'python'
code = 'import qb\n'
code += '%s%s%s' % ('\nqb.workunblock(\'%s:', finalOutput['name'], '\' % qb.jobid())')
code += '\nqb.unblock(qb.jobid())'
callback['code'] = code
callbacks.append(callback)
''' ---- Now put the job together ---- '''
job = qubeJobTemplate
''' General '''
job['name'] = 'Quicktime: ' + sequenceName
job['prototype'] = 'Submit Transcoder'
''' Package '''
job['package'] = {}
job['package']['sequence'] = sequenceInitFile
job['package']['audioFile'] = audioFile
job['package']['outputFile'] = outputFile
job['package']['preset'] = preset
job['package']['selfContained'] = selfContained
job['package']['smartUpdate'] = smartUpdate
job['package']['fillMissingFrames'] = fillMissingFrames
job['package']['frameRange'] = '1-' + str(mySequence.getDuration())
job['package']['transcoderFolder'] = transcoderFolder
''' Agenda '''
job['agenda'] = []
job['agenda'].append(init)
job['agenda'].extend(segments)
''' Place the final outputs after their last segment. '''
for outputNum, output in enumerate(finalOutputs):
lastSegmentName = output['package']['segmentSubjobs'][-1]
lastSegmentIndex = None
for index, segment in enumerate(segments):
if segment['name'] == lastSegmentName:
lastSegmentIndex = index
break
if lastSegmentIndex != None:
job['agenda'].insert(lastSegmentIndex+2+outputNum, output) # +2 for Initialization and last segment
else:
logger.error("ERROR: Unable to find last segment for output " + output['name'])
''' Callbacks '''
if not job.get('callbacks', None):
job['callbacks'] = []
job['callbacks'].extend(callbacks)
return job
def postDialog(cmdjob, values):
'''
Prepare the output of the dialog.
Each rqitem is separated into its own job.
Store all history related info to a separate plist file
Project History
Email History
Contents of each job:
Universal:
sourceProjectPath - original AE project file
renderProjectPath - AE Project to render from
user - set to email address contents before @
email - set to @fellowshipchurch.com if @ not specified.
notes - user specified notes for the job
chunkSize - chunk size for the agenda
quality - render quality for the project, based on a custom script
callbacks:
mail
RQ Item Specific:
rqIndex - rqIndex to render
cpus - based on output type (only 1 if mov)
outputFiles - list of outputFiles for that rqItem
agenda - frames split up based on chunkSize for the job
frameCount - total number of frames to render
'''
valuesPkg = values.setdefault('package', {})
pValue = 0
pIncrement = 10
maxProgress = len(valuesPkg['aeProject']['rqItems']) * pIncrement * 2 + 50
pDlg = wx.ProgressDialog('Submitting Project...',
'Saving prefs...',
maximum=maxProgress)
pDlg.SetSize((300, -1))
try:
'''
----------------------------------------------------------------------------------------
First, save any history related items to the Elevate Plist
----------------------------------------------------------------------------------------
'''
elevatePrefs = {}
elevatePrefs['projectHistory'] = valuesPkg['aeProject'][
'projectHistory']
emailList = set(cmdjob.options['email']['choices'])
emailList.add(valuesPkg['email'])
if len(emailList) > 10:
emailList = emailList[0:9]
elevatePrefs['emailHistory'] = list(emailList)
logging.debug("emailHistory: %s" % elevatePrefs['emailHistory'])
plistlib.writePlist(elevatePrefs, ELEVATEPREFSFILE)
'''
----------------------------------------------------------------------------------------
Second, setup everything that will apply to all the rqItems
----------------------------------------------------------------------------------------
'''
pValue += pIncrement
pDlg.Update(pValue, "Copying original project...")
'''
Create a copy of the original project to render from
'''
sourceProjPath = valuesPkg['aeProject']['projectPath']
logging.debug("Making a copy of the project for rendering...")
#Create the time string to be placed on the end of the AE file
fileTimeStr = time.strftime("_%m%d%y_%H%M%S", time.gmtime())
#Copy the file to the project files folder and add the time on the end
sourceFolderPath, sourceProjName = os.path.split(sourceProjPath)
newFolderPath = os.path.join(sourceFolderPath,
QUBESUBMISSIONSFOLDERNAME)
newProjName = os.path.splitext(
sourceProjName)[0] + fileTimeStr + '.aep'
newProjPath = os.path.join(newFolderPath, newProjName)
try:
if not (os.path.exists(newFolderPath)):
os.mkdir(newFolderPath)
except:
raise ("Unable to create the folder %s" % newFolderPath)
try:
shutil.copy2(sourceProjPath, newProjPath)
logging.info("Project file copied to %s" % newProjPath)
except:
raise ("Unable to create a copy of the project under %s" %
newProjPath)
valuesPkg['sourceProjectPath'] = str(sourceProjPath)
logging.debug("sourceProjectPath: %s" % valuesPkg['sourceProjectPath'])
valuesPkg['renderProjectPath'] = str(newProjPath)
logging.debug("renderProjectPath: %s" % valuesPkg['renderProjectPath'])
'''
Setup the email, user, notes, chunkSize, quality, and callbacks.
'''
pValue += pIncrement
pDlg.Update(pValue, "Setting up qube jobs...")
if "@" not in valuesPkg['email']:
valuesPkg['email'] += "@fellowshipchurch.com"
values['mailaddress'] = valuesPkg['email']
values['user'] = valuesPkg['email'].split("@")[0]
values['notes'] = valuesPkg.get('notes', '').strip()
values['callbacks'] = [{
'triggers': 'done-job-self',
'language': 'mail'
}]
'''
----------------------------------------------------------------------------------------
Third, setup each rqItem's job
----------------------------------------------------------------------------------------
'''
'''
Setup the name, rqIndex, outputFiles, agenda and cpus.
'''
rqJobs = []
seqPattern = re.compile("\[#+\]")
for rqItem in valuesPkg['aeProject']['rqItems']:
outPaths = []
for item in rqItem['outFilePaths']:
outPaths.append(str(item))
sequence = True
if ".mov" in ",".join(outPaths):
sequence = False
pValue += pIncrement
pDlg.Update(pValue, "Setting up RQ Item: %s..." % rqItem['comp'])
rqiValues = copy.deepcopy(values)
rqiPkg = rqiValues.setdefault('package', {})
rqiPkg['rqIndex'] = str(rqItem['index'])
rqiValues['name'] = "%s %s-%s" % (values['name'], rqItem['index'],
rqItem['comp'])
rqiPkg['outputFiles'] = ",".join(outPaths)
logging.debug("Output File Paths: %s" % rqItem['outFilePaths'])
agendaRange = str(
"%s-%s" % (rqItem['startTime'], int(rqItem['stopTime']) - 1))
logging.debug("Agenda Range: %s" % agendaRange)
chunkSize = 30
if not sequence:
rqiValues['cpus'] = 1
chunkSize = 1000000
rqiValues['agenda'] = qb.genchunks(chunkSize, agendaRange)
logging.debug("Agenda: %s" % rqiValues['agenda'])
rqiValues['agenda'][-1]['resultpackage'] = {
'outputPaths': ",".join(outPaths)
}
rqiPkg['frameCount'] = int(rqItem['stopTime']) - int(
rqItem['startTime']) - 1
'''
If it's a sequence, mark all existing frames as complete.
'''
if sequence:
for path in outPaths:
pValue += pIncrement
pDlg.Update(
pValue, "Finding missing frames for %s..." %
os.path.basename(path))
seqPad = len(seqPattern.findall(path)[-1]) - 2
initFrame = sequenceTools.padFrame(rqItem['startTime'],
seqPad)
initPath = seqPattern.sub(initFrame, path)
logging.debug("initPath: %s" % initPath)
seq = sequenceTools.Sequence(initPath,
frameRange=agendaRange)
missingFrames = seq.getMissingFrames()
logging.debug("Missing Frames: %s" % missingFrames)
for task in rqiValues['agenda']:
logging.debug("Name: %s" % task['name'])
if "-" in task['name']:
tStart, tEnd = task['name'].split("-")
else:
tStart = tEnd = task['name']
tRange = range(int(tStart), int(tEnd) + 1)
found = False
for frame in tRange:
if frame in missingFrames:
found = True
if not found:
task['status'] = 'complete'
if task.has_key("resultPackage"):
task['resultpackage'][
'progress'] = '1' # 100% chunk progress
else:
task['resultpackage'] = {'progress': '1'}
logging.debug("Marking task as complete: %s" %
task)
'''
Delete any unecessary attributes
'''
rqiPkg['aeProject'] = None
del rqiPkg['aeProject']
rqiPkg['notes'] = None
del rqiPkg['notes']
rqiPkg['gui'] = None
del rqiPkg['gui']
rqJobs.append(rqiValues)
logging.debug("rqJobs: %s" % rqJobs)
pValue += pIncrement
pDlg.Update(pValue, "Submitting Jobs to qube...")
submittedJobs = qb.submit(rqJobs)
logging.debug("Submitted Jobs: %s" % submittedJobs)
pValue += pIncrement
pDlg.Update(pValue, "Refreshing Qube...")
# Update the Qube GUI
request = qbCache.QbServerRequest(
action="jobinfo",
value=[i['id'] for i in submittedJobs],
method='reload')
qbCache.QbServerRequestQueue.put(request)
pDlg.Update(maxProgress, "Complete!")
except Exception, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
dlg = wx.MessageDialog(None, "Unable to submit jobs %s" % e, "Error",
wx.OK | wx.ICON_ERROR)
logging.error(
repr(traceback.format_exception(exc_type, exc_value,
exc_traceback)))
dlg.ShowModal()
