def submit_ext(self, func, args=None, kwargs=None, **extra): """Same as :meth:`Executor.submit_ext <qbfutures.Executor.submit_ext>`, except extra keyword arguments are passed to the ``qb.Work``. """ work = qb.Work() work['name'] = extra.get( 'name', '%d: %s' % (len(self.futures) + 1, utils.get_func_name(func))) work['package'] = utils.pack( self.executor._base_work_package(func, args, kwargs, extra)) future = BatchFuture(work) self.futures.append(future) return future
def map(self, func, *iterables, **extra): """Same as :meth:`Executor.map <qbfutures.Executor.map>`, except extra keyword arguments are passed to the ``qb.Work``. """ futures = [] for i, args in enumerate(zip(*iterables)): work = qb.Work() package = self.executor._base_work_package(func, args, None, extra) work['name'] = package.pop('name', str(i + 1)) work['package'] = utils.pack(package) future = BatchFuture(work) futures.append(future) self.futures.append(future) return self.executor._map_iter(futures, extra.get('timeout'))
def submit_ext(self, func, args=None, kwargs=None, **extra): """Extended submission with more control over Qube job. :param func: The function to call. :param list args: The positional arguments to call with. :param dict kwargs: The keyword arguments to call with. :param **extra: Values to pass through to the ``qb.Job``. :returns: The :class:`~qbfutures.Future` linked to the submitted job. """ job = self._base_job(func, **extra) work = qb.Work() package = self._base_work_package(func, args, kwargs, extra) work['name'] = str(package.pop('name', '1')) work['package'] = utils.pack(package) job['agenda'] = [work] return self._submit(job)[0]
def map(self, func, *iterables, **extra): """Equivalent to ``map(func, *iterables)`` except ``func`` is executed asynchronously on Qube. :param timeout: The number of seconds to wait for results, or ``None``. Any other keyword arguments will be passed through to the ``qb.Job``:: >>> for result in Executor().map(my_function, range(10), cpus=4): ... print result """ job = self._base_job(func, **extra) for i, args in enumerate(zip(*iterables)): work = qb.Work() package = self._base_work_package(func, args, None, extra) work['name'] = package.pop('name', str(i + 1)) work['package'] = utils.pack(package) job['agenda'].append(work) futures = self._submit(job) return self._map_iter(futures, extra.get('timeout'))
def setupSequenceJob(qubeJobTemplate, sequenceInitFile, outputFile, preset, selfContained=True, frameRange='ALL', audioFile='', smartUpdate=True, fillMissingFrames=True, transcoderFolder='', segmentDuration=200, maxSegmentsPerOutput=20, maxSegmentTolerance=5): ''' Setup a qube job dictionary based on the input. Required Inputs: qubeJobTemplate (dictionary) Template qube job dictionary to build from. sequenceInitFile (string) One image from the input image sequence. Can be any image from the sequence. outputFile (string) The destination file for the output of the transcoder. preset (string) The blender file that serves as the template for the transcoding process. Optional: selfContained (boolean) Determines if the outputFile should be a reference quicktime movie or self-contained. Self-contained movies are much larger and take more time to create. Referenced quicktimes are much smaller, and much faster to create. However, referenced quicktimes must maintain their connectiong their associated inputs. frameRange (string) The output frame range to render from the input image sequence. Ex: 1-10 audioFile (string) The audio file to be added to the output file. This audio should match the exact timeline of the input image sequence. smartUpdate (boolean) Automatically update only the segments and outputs that have been changed since the last transcode. transcoderFolder (string) The folder in which to store all files related to the transcoding process. This includes the segmented movies and the blender projects. If creating a referenced output file, these are the segments that movie will reference. fillMissingFrames (boolean) Automatically fill in missing frames with the last frame that exists. This is useful for creating quicktimes from sequences rendered on every nth frame. Advanced: segmentDuration (integer) Frame count for each segment. maxSegmentsPerOutput (integer) Maximum number of segments that can be in each output file. If the number of segments needed for the sequence exceeds this amount, the output file is split into multiple segments of this length. maxSegmentTolerance (integer) If the maxSegmentsPerOutput limit is reached, check that the input sequence exceeds this tolerance value as well. If not, keep the outputFile as one file. Agenda The agenda is setup in 3 main sections: Initialization: Purpose This single subjobs loads the input sequence into the provided blender scene preset. This is done once, then all subsequent jobs reference the resulting scene file. Package None resultPackage None Naming Initialize Segments: Purpose These subjobs each create their assigned segment of the image sequence. Package frameRange (string) Range of frames to render for this segment. segmentFile (string) Destination path for the segment file. resultPackage changes (boolean) Returns if any changes were made for this segment. segmentFile (string) Destination path for the segment file that actually rendered. Sometimes file issues occur where the output file can't be overwritten, so we automatically compensate for this. Naming Segment: (frameRange) Final Outputs: Purpose These subjobs render the output files. They are split up based on the number of segments and the max segments per output. They are placed in the agenda right after their dependent segments have been processed. Package segmentSubjobs (list of strings) List of the names of the dependant segment subjobs. outputFile (string) destination for the output resultPackage outputPaths (string) Path to the final output file. Naming Output: (outputFile) Callbacks Callbacks are added to unblock subjobs when they are ready to be processed. Initialization subjob completion Once the initialization is complete, all segment subjobs are unblocked. Segment subjobs complete. Once all segments that pertain to a final output are complete, that output subjob is unblocked. Job retried If the job is retried ''' ''' ---- Pre-Processing For Agenda ---- ''' logger.debug('Setup Sequence: ' + str(locals())) ''' General ''' mySequence = sequenceTools.Sequence(sequenceInitFile, frameRange) sequenceName = mySequence.getName() if not transcoderFolder: transcoderFolder = os.path.join(os.path.dirname(outputFile), '_Transcoder/') ''' Initialize ''' init = qb.Work() init['name'] = 'Initialize' ''' Segments Use the qube chunk method to split up the frame range. Then prep each segment: Add the frameRange to the package. Add the segmentFile to the package. Change the subjob name to Segment: (frameRange) Submit as blocked, because they will be unblocked once the initialize command is completed. ''' segments = qb.genchunks(segmentDuration, '1-' + str(mySequence.getDuration())) for segment in segments: segment['package'] = {} segment['package']['frameRange'] = segment['name'] outputFolder, outputName, outputExtension = splitPath(outputFile) segmentFile = os.path.join(transcoderFolder, 'Segments/') segmentFile += outputName + '/' segmentFile += "Segment" + segment['name'].split( '-')[0] + outputExtension segment['package']['segmentFile'] = segmentFile segment['status'] = 'blocked' segment['name'] = 'Segment:' + segment['name'] ''' Final Outputs ''' finalOutputSegments = chunkWithTolerance(segments, maxSegmentsPerOutput, maxSegmentTolerance) finalOutputs = [] count = 1 for outputSegment in finalOutputSegments: output = qb.Work() output['package'] = {} segmentSubjobs = [] for segment in outputSegment: segmentSubjobs.append(segment['name']) output['package']['segmentSubjobs'] = segmentSubjobs outputFolder, outputName, outputExtension = splitPath(outputFile) finalOutputFile = outputFolder + outputName if len(finalOutputSegments) > 1: finalOutputFile += '_' + chr(64 + count) finalOutputFile += outputExtension output['package']['outputFile'] = finalOutputFile output['status'] = 'blocked' output['name'] = 'Output: ' + os.path.basename(finalOutputFile) count += 1 finalOutputs.append(output) ''' Callbacks 1 - Unblock the segments when the initialize command is completed. 2 - Unblock the outputs when the dependant segments are completed. ''' callbacks = [] ''' Unblock Segments ''' callback = {} callback['triggers'] = 'complete-work-self-Initialize' callback['language'] = 'python' code = 'import qb\n' for segment in segments: code += '%s%s%s' % ('\nqb.workunblock(\'%s:', segment['name'], '\' % qb.jobid())') code += '\nqb.unblock(qb.jobid())' callback['code'] = code callbacks.append(callback) ''' Unblock Outputs ''' for finalOutput in finalOutputs: callback = {} triggers = [] for segment in finalOutput['package']['segmentSubjobs']: triggers.append('complete-work-self-' + segment) callback['triggers'] = ' and '.join(triggers) callback['language'] = 'python' code = 'import qb\n' code += '%s%s%s' % ('\nqb.workunblock(\'%s:', finalOutput['name'], '\' % qb.jobid())') code += '\nqb.unblock(qb.jobid())' callback['code'] = code callbacks.append(callback) ''' ---- Now put the job together ---- ''' job = qubeJobTemplate ''' General ''' job['name'] = 'Quicktime: ' + sequenceName job['prototype'] = 'Submit Transcoder' ''' Package ''' job['package'] = {} job['package']['sequence'] = sequenceInitFile job['package']['audioFile'] = audioFile job['package']['outputFile'] = outputFile job['package']['preset'] = preset job['package']['selfContained'] = selfContained job['package']['smartUpdate'] = smartUpdate job['package']['fillMissingFrames'] = fillMissingFrames job['package']['frameRange'] = '1-' + str(mySequence.getDuration()) job['package']['transcoderFolder'] = transcoderFolder ''' Agenda ''' job['agenda'] = [] job['agenda'].append(init) job['agenda'].extend(segments) ''' Place the final outputs after their last segment. ''' for outputNum, output in enumerate(finalOutputs): lastSegmentName = output['package']['segmentSubjobs'][-1] lastSegmentIndex = None for index, segment in enumerate(segments): if segment['name'] == lastSegmentName: lastSegmentIndex = index break if lastSegmentIndex != None: job['agenda'].insert( lastSegmentIndex + 2 + outputNum, output) # +2 for Initialization and last segment else: logger.error("ERROR: Unable to find last segment for output " + output['name']) ''' Callbacks ''' if not job.get('callbacks', None): job['callbacks'] = [] job['callbacks'].extend(callbacks) return job