Ejemplo n.º 1
0
def testBibleStory():
    # Set basic job properties
    job = {}
    job['cpus'] = 50
    job['requirements'] = ''
    job['reservations'] = 'host.processors=1'
    job['flagsstring'] = 'auto_wrangling'
    # job['hosts'] = 'bchapman.local'
    job['priority'] = 100
    job['hostorder'] = '+host.processors.avail'
    sequenceFile = '/Volumes/theGrill/Elevate_Series/Port_Of_Call/Kids/Art_Anim/_Renders_and_Exports/Image_Sequences/L2/POC_BS_L2_00000.png'
    outputFile = '/Volumes/theGrill/Staff-Directories/Brennan/Testing/Transcoder/POC_BS_L2_Test.mov'
    preset = '/Volumes/theGrill/.qube/Jobtypes/Submit Transcoder/Presets/720p_wAlpha.blend'
    audioFile = '/Volumes/theGrill/Staff-Directories/Brennan/Testing/Transcoder/POC_BS_L2.wav'
    job = setupSequenceJob(job,
                           sequenceFile,
                           outputFile,
                           preset,
                           audioFile=audioFile,
                           maxSegmentsPerOutput=5,
                           fillMissingFrames=True,
                           maxSegmentTolerance=2,
                           segmentDuration=200)
    logger.info(job)
    qb.submit([job])
Ejemplo n.º 2
0
def testLessSegments():
    # Set basic job properties
    job = {}
    job['cpus'] = 50
    job['requirements'] = ''
    job['reservations'] = 'host.processors=1'
    job['flagsstring'] = 'auto_wrangling'
    # job['hosts'] = 'bchapman.local'
    job['priority'] = 100
    job['hostorder'] = '+host.processors.avail'
    sequenceFile = '/Volumes/theGrill/Elevate_NonSeries_Projects/ElevateAtHome_Leadership/Renders/_Proxies/testIS/IS/LeadershipWeb_00000.tga'
    outputFile = '/Volumes/theGrill/Elevate_NonSeries_Projects/ElevateAtHome_Leadership/Renders/_Proxies/testIS/LeadershipWeb_LessSegments.mov'
    preset = '/Volumes/theGrill/.qube/Jobtypes/Submit Transcoder/Presets/1280x720-29.97-ProRes4444.blend'
    audioFile = '/Volumes/theGrill/Elevate_NonSeries_Projects/ElevateAtHome_Leadership/Renders/_Proxies/testIS/LeadershipWeb.wav'
    job = setupSequenceJob(job,
                           sequenceFile,
                           outputFile,
                           preset,
                           audioFile=audioFile,
                           maxSegmentsPerOutput=1,
                           fillMissingFrames=True,
                           maxSegmentTolerance=0,
                           segmentDuration=1000)
    logger.info(job)
    qb.submit([job])
Ejemplo n.º 3
0
    def _submit(self, job):

        # Final chance for cleanup! Qube does not like None names.
        try:
            if job['user'] is None:
                job.pop('user')
        except KeyError:
            pass

        # Assign a default user from the environment.
        try:
            job.setdefault('user', os.environ['QBFUTURES_USER'])
        except KeyError:
            pass

        job_id = qb.submit([job])[0]['id']

        futures = []
        for work_id, work in enumerate(job['agenda']):
            future = Future(job_id, work_id)
            futures.append(future)
            _poller.add(future)

        _poller.trigger()
        return futures
Ejemplo n.º 4
0
    def submit(my):

        job_type = my.get_option('job_type')
        if not job_type:
            job_type = "tactic"

        # build the appropriate package
        if job_type == "tactic":
            qube_job = QubeTacticJob(my.render_package, my.queue)
        #elif job_type == "xsi":
        #    qube_job = QubeXSIJob(my.render_package, my.queue)
        else:
            qube_job = QubeSimpleJob(my.render_package, my.queue)


        # build a qube job and submit command
        my.job_list = []

        # get the job from 
        my.job_list.append( qube_job.get_job() )

        print "job_list: ", my.job_list

        # append the checkin job as a callback
        #checkin_job = QubeCheckinJob(my.render_package, queue)
        #my.job_list.append( checkin_job.get_job() )

        # get the id for the submitted job
        submitted = qb.submit(my.job_list)
        dispatcher_id = submitted[0]['id']

        return dispatcher_id
Ejemplo n.º 5
0
def prepareJobsFromDlg(qubejob):

    tJobs = qubejob.get('package', {}).get('transcodeJobs', [])

    jobsToSubmit = []
    for tJob in tJobs:    
        logger.info("job from array: " + str(tJob))
        sequenceFile = tJob['imageSequence']
        logger.info("imageSequence: " + str(sequenceFile))
        frameRange = tJob['frameRange']
        logger.info("frameRange: " + str(frameRange))
        outputFile = tJob['outputMovie']
        logger.info("outputMovie: " + str(outputFile))
        preset = tJob['outputPreset']
        logger.info("preset: " + str(preset))
        audioFile = tJob['audioFile']
        selfContained = tJob['selfContained']
        logger.info("selfContained: " + str(preset))
        smartUpdate = tJob['smartUpdate']
        logger.info("smartUpdate: " + str(smartUpdate))
        smartUpdate = tJob['smartUpdate']
        logger.info("smartUpdate: " + str(smartUpdate))
        fillMissingFrames = tJob['fillMissingFrames']
        logger.info("fillMissingFrames: " + str(fillMissingFrames))
        transcodeJob = setupSequenceJob(qubejob, sequenceFile, outputFile, preset, audioFile=audioFile, maxSegmentsPerOutput=5, frameRange=frameRange,
            fillMissingFrames=fillMissingFrames, maxSegmentTolerance=2, segmentDuration=100, selfContained=selfContained, smartUpdate=smartUpdate)
        logger.info("Setup Sequence Job: " + str(transcodeJob))
        jobsToSubmit.append(transcodeJob)

    submittedJobs = qb.submit(jobsToSubmit)

    request = qbCache.QbServerRequest(action="jobinfo", value=[i['id'] for i in submittedJobs], method='reload')
    qbCache.QbServerRequestQueue.put(request)
Ejemplo n.º 6
0
    def submit(self, depend=None):
        self.reservations = ["host.processors=1+"]

        # add license registration
        for each in self.licenses:
            self.reservations.append("license.%s" % each)

        from sys import path

        path.insert(0, pipe.apps.qube().path("qube-core/local/pfx/qube/api/python/"))
        import qb

        # print dir(qb)
        self.qb = qb.Job()
        self.qb["name"] = self.name
        self.qb["prototype"] = "cmdrange"
        self.qb["package"] = {
            #            'cmdline' : 'echo "Frame QB_FRAME_NUMBER" ; echo $HOME ; echo $USER ; cd ; echo `pwd` ; sleep 10',
            "cmdline": self.cmd,
            "padding": self.pad,
            "range": str(self.range())[1:-1],
        }
        self.qb["agenda"] = qb.genframes(self.qb["package"]["range"])
        self.qb["cpus"] = self.cpus
        self.qb["hosts"] = ""
        self.qb["groups"] = self.group
        self.qb["cluster"] = "/pipe"
        self.qb["priority"] = self.priority
        self.qb["reservations"] = ",".join(self.reservations)
        self.qb["retrywork"] = self.retry
        self.qb["retrywork_delay"] = 15
        self.qb["retrysubjob"] = self.retry
        self.qb["flags"] = "auto_wrangling,migrate_on_frame_retry"

        return qb.submit(self.qb)[0]["id"]
Ejemplo n.º 7
0
    def submit(my):

        job_type = my.get_option('job_type')
        if not job_type:
            job_type = "tactic"

        # build the appropriate package
        if job_type == "tactic":
            qube_job = QubeTacticJob(my.render_package, my.queue)
        #elif job_type == "xsi":
        #    qube_job = QubeXSIJob(my.render_package, my.queue)
        else:
            qube_job = QubeSimpleJob(my.render_package, my.queue)

        # build a qube job and submit command
        my.job_list = []

        # get the job from
        my.job_list.append(qube_job.get_job())

        print "job_list: ", my.job_list

        # append the checkin job as a callback
        #checkin_job = QubeCheckinJob(my.render_package, queue)
        #my.job_list.append( checkin_job.get_job() )

        # get the id for the submitted job
        submitted = qb.submit(my.job_list)
        dispatcher_id = submitted[0]['id']

        return dispatcher_id
Ejemplo n.º 8
0
    def _submit(self, job):
        
        # Final chance for cleanup! Qube does not like None names.
        try:
            if job['user'] is None:
                job.pop('user')
        except KeyError:
            pass

        # Assign a default user from the environment.
        try:
            job.setdefault('user', os.environ['QBFUTURES_USER'])
        except KeyError:
            pass

        job_id = qb.submit([job])[0]['id']
        
        futures = []
        for work_id, work in enumerate(job['agenda']):
            future = Future(job_id, work_id)
            futures.append(future)
            _poller.add(future)
        
        _poller.trigger()
        return futures
Ejemplo n.º 9
0
def prepareJobsFromDlg(qubejob):
    
    pValue = 0
    pIncrement = 10
    pDlg = wx.ProgressDialog ( 'Submitting Transcoder...', 'Preparing...', maximum = 100)
    pDlg.SetSize((300, -1))
    
    try:

        pValue += pIncrement
        pDlg.Update(pValue, "Setting up transcode jobs...")

        tJobs = qubejob.get('package', {}).get('transcodeJobs', [])

        jobsToSubmit = []
        for tJob in tJobs:    
            logger.info("job from array: " + str(tJob))
            sequenceFile = tJob['imageSequence']
            logger.info("imageSequence: " + str(sequenceFile))
            frameRange = tJob['frameRange']
            logger.info("frameRange: " + str(frameRange))
            outputFile = tJob['outputMovie']
            logger.info("outputMovie: " + str(outputFile))
            preset = tJob['outputPreset']
            logger.info("preset: " + str(preset))
            audioFile = tJob['audioFile']
            selfContained = tJob['selfContained']
            logger.info("selfContained: " + str(preset))
            smartUpdate = tJob['smartUpdate']
            logger.info("smartUpdate: " + str(smartUpdate))
            smartUpdate = tJob['smartUpdate']
            logger.info("smartUpdate: " + str(smartUpdate))
            fillMissingFrames = tJob['fillMissingFrames']
            logger.info("fillMissingFrames: " + str(fillMissingFrames))
            transcodeJob = setupSequenceJob(qubejob, sequenceFile, outputFile, preset, audioFile=audioFile, maxSegmentsPerOutput=5, frameRange=frameRange,
                fillMissingFrames=fillMissingFrames, maxSegmentTolerance=2, segmentDuration=100, selfContained=selfContained, smartUpdate=smartUpdate)
            logger.info("Setup Sequence Job: " + str(transcodeJob))
            jobsToSubmit.append(transcodeJob)

        pValue += pIncrement
        pDlg.Update(pValue, "Submitting Jobs to qube...")

        submittedJobs = qb.submit(jobsToSubmit)

        pValue += pIncrement
        pDlg.Update(pValue, "Refreshing Qube...")

        request = qbCache.QbServerRequest(action="jobinfo", value=[i['id'] for i in submittedJobs], method='reload')
        qbCache.QbServerRequestQueue.put(request)

        pDlg.Update(100, "Complete!")

    except Exception, e:
        exc_type, exc_value, exc_traceback = sys.exc_info()
        dlg = wx.MessageDialog(None, "Unable to submit jobs %s" % e, "Error", wx.OK | wx.ICON_ERROR)
        logging.error(repr(traceback.format_exception(exc_type, exc_value,exc_traceback)))
        dlg.ShowModal()
Ejemplo n.º 10
0
def testLessSegments():
    # Set basic job properties
    job = {}
    job['cpus'] = 50
    job['requirements'] = ''
    job['reservations'] = 'host.processors=1'
    job['flagsstring'] = 'auto_wrangling'
    # job['hosts'] = 'bchapman.local'
    job['priority'] = 100
    job['hostorder'] = '+host.processors.avail'
    sequenceFile = '/Volumes/theGrill/Elevate_NonSeries_Projects/ElevateAtHome_Leadership/Renders/_Proxies/testIS/IS/LeadershipWeb_00000.tga'
    outputFile = '/Volumes/theGrill/Elevate_NonSeries_Projects/ElevateAtHome_Leadership/Renders/_Proxies/testIS/LeadershipWeb_LessSegments.mov'
    preset = '/Volumes/theGrill/.qube/Jobtypes/Submit Transcoder/Presets/1280x720-29.97-ProRes4444.blend'
    audioFile = '/Volumes/theGrill/Elevate_NonSeries_Projects/ElevateAtHome_Leadership/Renders/_Proxies/testIS/LeadershipWeb.wav'
    job = setupSequenceJob(job, sequenceFile, outputFile, preset, audioFile=audioFile, maxSegmentsPerOutput=1,
        fillMissingFrames=True, maxSegmentTolerance=0, segmentDuration=1000)
    logger.info(job)
    qb.submit([job])
Ejemplo n.º 11
0
def testBibleStory():
    # Set basic job properties
    job = {}
    job['cpus'] = 50
    job['requirements'] = ''
    job['reservations'] = 'host.processors=1'
    job['flagsstring'] = 'auto_wrangling'
    # job['hosts'] = 'bchapman.local'
    job['priority'] = 100
    job['hostorder'] = '+host.processors.avail'
    sequenceFile = '/Volumes/theGrill/Elevate_Series/Port_Of_Call/Kids/Art_Anim/_Renders_and_Exports/Image_Sequences/L2/POC_BS_L2_00000.png'
    outputFile = '/Volumes/theGrill/Staff-Directories/Brennan/Testing/Transcoder/POC_BS_L2_Test.mov'
    preset = '/Volumes/theGrill/.qube/Jobtypes/Submit Transcoder/Presets/720p_wAlpha.blend'
    audioFile = '/Volumes/theGrill/Staff-Directories/Brennan/Testing/Transcoder/POC_BS_L2.wav'
    job = setupSequenceJob(job, sequenceFile, outputFile, preset, audioFile=audioFile, maxSegmentsPerOutput=5,
        fillMissingFrames=True, maxSegmentTolerance=2, segmentDuration=200)
    logger.info(job)
    qb.submit([job])
Ejemplo n.º 12
0
 def _submit_job(self, job):
     '''Submit the *job* to Qube.
     
     Return the job id.
     
     '''
     # Import here so that subclasses can avoid qb import if desired.
     import qb
     
     return qb.submit(job)
Ejemplo n.º 13
0
    def _submit_job(self, job):
        '''Submit the *job* to Qube.
        
        Return the job id.
        
        '''
        # Import here so that subclasses can avoid qb import if desired.
        import qb

        return qb.submit(job)
Ejemplo n.º 14
0
 def commit(self):
     """Perform the actual job submittion. Called automatically if used as
     a context manager."""
     if not self.futures:
         return []
     self.job['agenda'] = [future.work for future in self.futures]
     submitted = qb.submit([self.job])
     assert len(submitted) == 1
     for i, future in enumerate(self.futures):
         future.job_id = submitted[0]['id']
         future.work_id = i
         _poller.add(future)
     _poller.trigger()
     return self.futures
Ejemplo n.º 15
0
 def commit(self):
     """Perform the actual job submittion. Called automatically if used as
     a context manager."""
     if not self.futures:
         return []
     self.job['agenda'] = [future.work for future in self.futures]
     submitted = qb.submit([self.job])
     assert len(submitted) == 1
     for i, future in enumerate(self.futures):
         future.job_id = submitted[0]['id']
         future.work_id = i
         _poller.add(future)
     _poller.trigger()
     return self.futures
Ejemplo n.º 16
0
def main():
    # The parameters here are the same as before, with exceptions noted
    job = {}
    job['name'] = 'python test job - echo the frame number'

    # This time, we will request 4 instances (previously known as subjobs).
    # By requesting 4 instances, assuming there are 4 open slots on the farm,
    # up to 4 agenda items will be processed simultaneously.
    job['cpus'] = 4

    # In the last example, we used the prototype 'cmdline' which implied a single
    # command being run on the farm.  This time, we will use the 'cmdrange' prototype
    # which tells Qube that we are running a command per agenda item.
    job['prototype'] = 'cmdrange'

    package = {}

    # Just like the last example, we create a package parameter called 'cmdline'.
    # This is the command that will be run for every agenda item.  QB_FRAME_NUMBER,
    # however, is unique to cmdrange.  The text QB_FRAME_NUMBER will be replaced with
    # the actual frame number at run time.
    package['cmdline'] = 'echo QB_FRAME_NUMBER'

    job['package'] = package

    # Now we must create our agenda list.  This is an absolutely essential part of
    # submitting jobs with agenda items (i.e. frames).
    # First we define a range.  The range is in typical number range format where:
    #  1-5 means frames 1,2,3,4,5
    #  1,3,5 means frames 1,3, and 5
    #  1-5,7 means frames 1,2,3,4,5,7
    #  1-10x3 means frames 1,4,7,10
    agendaRange = '0-60x10'  # will evaluate to 0,10,20,30,40,50,60

    # Using the given range, we will create an agenda list using qb.genframes
    agenda = qb.genframes(agendaRange)

    # Now that we have a properly formatted agenda, assign it to the job
    job['agenda'] = agenda

    # As before, we create a list of 1 job, then submit the list.  Again, we
    # could submit just the single job w/o the list, but submitting a list is
    # good form.
    listOfJobsToSubmit = []
    listOfJobsToSubmit.append(job)
    listOfSubmittedJobs = qb.submit(listOfJobsToSubmit)
    for job in listOfSubmittedJobs:
        print job['id']
Ejemplo n.º 17
0
def prepareJobsFromDlg(qubejob):

    tJobs = qubejob.get('package', {}).get('transcodeJobs', [])

    jobsToSubmit = []
    for tJob in tJobs:
        logger.info("job from array: " + str(tJob))
        sequenceFile = tJob['imageSequence']
        logger.info("imageSequence: " + str(sequenceFile))
        frameRange = tJob['frameRange']
        logger.info("frameRange: " + str(frameRange))
        outputFile = tJob['outputMovie']
        logger.info("outputMovie: " + str(outputFile))
        preset = tJob['outputPreset']
        logger.info("preset: " + str(preset))
        audioFile = tJob['audioFile']
        selfContained = tJob['selfContained']
        logger.info("selfContained: " + str(preset))
        smartUpdate = tJob['smartUpdate']
        logger.info("smartUpdate: " + str(smartUpdate))
        smartUpdate = tJob['smartUpdate']
        logger.info("smartUpdate: " + str(smartUpdate))
        fillMissingFrames = tJob['fillMissingFrames']
        logger.info("fillMissingFrames: " + str(fillMissingFrames))
        transcodeJob = setupSequenceJob(qubejob,
                                        sequenceFile,
                                        outputFile,
                                        preset,
                                        audioFile=audioFile,
                                        maxSegmentsPerOutput=5,
                                        frameRange=frameRange,
                                        fillMissingFrames=fillMissingFrames,
                                        maxSegmentTolerance=2,
                                        segmentDuration=100,
                                        selfContained=selfContained,
                                        smartUpdate=smartUpdate)
        logger.info("Setup Sequence Job: " + str(transcodeJob))
        jobsToSubmit.append(transcodeJob)

    submittedJobs = qb.submit(jobsToSubmit)

    request = qbCache.QbServerRequest(action="jobinfo",
                                      value=[i['id'] for i in submittedJobs],
                                      method='reload')
    qbCache.QbServerRequestQueue.put(request)
Ejemplo n.º 18
0
    def submitJob(self):
        print 'submitting job'
        if self.startFrame.value() >= self.endFrame.value():
            msg = QMessageBox()
            msg.setIcon(QMessageBox.Information)
            msg.setText("Range Error")
            msg.setInformativeText("Start frame must be less than end frame")
            return msg.exec_()

        # The first few parameters are the same as the previous examples
        job = {}
        job['name'] = str(self.jobName.text())
        job['cpus'] = min(self.endFrame.value() - self.startFrame.value(), 40)
        job['priority'] = 9999
        job['cwd'] = str(self.cwd.text())

        # Below creates an empty package dictionary
        package = {}
        # Below instructs the Qube! GUI which submission UI to use for resubmission
        if self.renderRange.isChecked() == True:
            package['simpleCmdType'] = 'cmdrange'
            job['prototype'] = 'cmdrange'
            # doing a range
            # todo add sanity check for range
            package['range'] = '%d-%d' % (self.startFrame.value(),
                                          self.endFrame.value())
            renderString = '$RMANTREE/bin/render '
            renderString = renderString + self.cwd.text() + '/'
            # todo add sanity check and instructions on file formats
            file = str(self.ribFile.text())
            # some regex magic!
            fileName = re.sub(r'(\d+)', 'QB_FRAME_NUMBER', file, 1)
            renderString = renderString + fileName

            package['cmdline'] = str(renderString)
            package['padding'] = self.padding.value()
            job['package'] = package
            # Using the given range, we will create an agenda list using qb.genframes
            agenda = qb.genframes(package['range'])
            # Now that we have a properly formatted agenda, assign it to the job
            job['agenda'] = agenda

        else:
            # single file
            print "single"
            package['simpleCmdType'] = 'cmdline'
            job['prototype'] = 'cmdline'
            renderString = '$RMANTREE/bin/render '
            renderString = renderString + self.cwd.text() + '/'
            # todo add sanity check and instructions on file formats
            file = str(self.ribFile.text())
            # some regex magic!
            renderString = renderString + file
            package['rangeExecution'] = 'individual'
            package['cmdline'] = str(renderString)
            job['package'] = package

        # Below sets the job's package to the package dictionary we just created
        job['env'] = {
            'RMANTREE': '/opt/software/pixar/RenderManProServer-20.10/'
        }

        listOfJobsToSubmit = []
        listOfJobsToSubmit.append(job)

        # As before, we create a list of 1 job, then submit the list.  Again, we
        # could submit just the single job w/o the list, but submitting a list is
        # good form.
        listOfSubmittedJobs = qb.submit(listOfJobsToSubmit)
        for job in listOfSubmittedJobs:
            print job['id']

        print job, package
Ejemplo n.º 19
0
def prepareJobsFromDlg(qubejob):

    pValue = 0
    pIncrement = 10
    pDlg = wx.ProgressDialog('Submitting Transcoder...',
                             'Preparing...',
                             maximum=100)
    pDlg.SetSize((300, -1))

    try:

        pValue += pIncrement
        pDlg.Update(pValue, "Setting up transcode jobs...")

        tJobs = qubejob.get('package', {}).get('transcodeJobs', [])

        jobsToSubmit = []
        for tJob in tJobs:
            logger.info("job from array: " + str(tJob))
            sequenceFile = tJob['imageSequence']
            logger.info("imageSequence: " + str(sequenceFile))
            frameRange = tJob['frameRange']
            logger.info("frameRange: " + str(frameRange))
            outputFile = tJob['outputMovie']
            logger.info("outputMovie: " + str(outputFile))
            preset = tJob['outputPreset']
            logger.info("preset: " + str(preset))
            audioFile = tJob['audioFile']
            selfContained = tJob['selfContained']
            logger.info("selfContained: " + str(preset))
            smartUpdate = tJob['smartUpdate']
            logger.info("smartUpdate: " + str(smartUpdate))
            smartUpdate = tJob['smartUpdate']
            logger.info("smartUpdate: " + str(smartUpdate))
            fillMissingFrames = tJob['fillMissingFrames']
            logger.info("fillMissingFrames: " + str(fillMissingFrames))
            transcodeJob = setupSequenceJob(
                qubejob,
                sequenceFile,
                outputFile,
                preset,
                audioFile=audioFile,
                maxSegmentsPerOutput=5,
                frameRange=frameRange,
                fillMissingFrames=fillMissingFrames,
                maxSegmentTolerance=2,
                segmentDuration=100,
                selfContained=selfContained,
                smartUpdate=smartUpdate)
            logger.info("Setup Sequence Job: " + str(transcodeJob))
            jobsToSubmit.append(transcodeJob)

        pValue += pIncrement
        pDlg.Update(pValue, "Submitting Jobs to qube...")

        submittedJobs = qb.submit(jobsToSubmit)

        pValue += pIncrement
        pDlg.Update(pValue, "Refreshing Qube...")

        request = qbCache.QbServerRequest(
            action="jobinfo",
            value=[i['id'] for i in submittedJobs],
            method='reload')
        qbCache.QbServerRequestQueue.put(request)

        pDlg.Update(100, "Complete!")

    except Exception, e:
        exc_type, exc_value, exc_traceback = sys.exc_info()
        dlg = wx.MessageDialog(None, "Unable to submit jobs %s" % e, "Error",
                               wx.OK | wx.ICON_ERROR)
        logging.error(
            repr(traceback.format_exception(exc_type, exc_value,
                                            exc_traceback)))
        dlg.ShowModal()
Ejemplo n.º 20
0
def postDialog(cmdjob, values):
    '''
    Prepare the output of the dialog.
    Each rqitem is separated into its own job.
    Store all history related info to a separate plist file
        Project History
        Email History
    
    Contents of each job:
        Universal:
            sourceProjectPath - original AE project file
            renderProjectPath - AE Project to render from
            user - set to email address contents before @
            email - set to @fellowshipchurch.com if @ not specified.
            notes - user specified notes for the job
            chunkSize - chunk size for the agenda
            quality - render quality for the project, based on a custom script
            callbacks:
                mail

        RQ Item Specific:
            rqIndex - rqIndex to render
            cpus - based on output type (only 1 if mov)
            outputFiles - list of outputFiles for that rqItem
            agenda - frames split up based on chunkSize for the job
            frameCount - total number of frames to render
    '''
    valuesPkg = values.setdefault('package', {})

    pValue = 0
    pIncrement = 10
    maxProgress = len(valuesPkg['aeProject']['rqItems']) * pIncrement * 2 + 50
    pDlg = wx.ProgressDialog('Submitting Project...',
                             'Saving prefs...',
                             maximum=maxProgress)
    pDlg.SetSize((300, -1))

    try:
        '''
        ----------------------------------------------------------------------------------------
        First, save any history related items to the Elevate Plist
        ----------------------------------------------------------------------------------------
        '''
        elevatePrefs = {}

        elevatePrefs['projectHistory'] = valuesPkg['aeProject'][
            'projectHistory']
        emailList = set(cmdjob.options['email']['choices'])
        emailList.add(valuesPkg['email'])
        if len(emailList) > 10:
            emailList = emailList[0:9]
        elevatePrefs['emailHistory'] = list(emailList)
        logging.debug("emailHistory: %s" % elevatePrefs['emailHistory'])
        plistlib.writePlist(elevatePrefs, ELEVATEPREFSFILE)
        '''
        ----------------------------------------------------------------------------------------
        Second, setup everything that will apply to all the rqItems
        ----------------------------------------------------------------------------------------
        '''
        pValue += pIncrement
        pDlg.Update(pValue, "Copying original project...")
        '''
        Create a copy of the original project to render from
        '''
        sourceProjPath = valuesPkg['aeProject']['projectPath']

        logging.debug("Making a copy of the project for rendering...")

        #Create the time string to be placed on the end of the AE file
        fileTimeStr = time.strftime("_%m%d%y_%H%M%S", time.gmtime())

        #Copy the file to the project files folder and add the time on the end
        sourceFolderPath, sourceProjName = os.path.split(sourceProjPath)
        newFolderPath = os.path.join(sourceFolderPath,
                                     QUBESUBMISSIONSFOLDERNAME)
        newProjName = os.path.splitext(
            sourceProjName)[0] + fileTimeStr + '.aep'
        newProjPath = os.path.join(newFolderPath, newProjName)

        try:
            if not (os.path.exists(newFolderPath)):
                os.mkdir(newFolderPath)
        except:
            raise ("Unable to create the folder %s" % newFolderPath)

        try:
            shutil.copy2(sourceProjPath, newProjPath)
            logging.info("Project file copied to %s" % newProjPath)
        except:
            raise ("Unable to create a copy of the project under %s" %
                   newProjPath)

        valuesPkg['sourceProjectPath'] = str(sourceProjPath)
        logging.debug("sourceProjectPath: %s" % valuesPkg['sourceProjectPath'])
        valuesPkg['renderProjectPath'] = str(newProjPath)
        logging.debug("renderProjectPath: %s" % valuesPkg['renderProjectPath'])
        '''
        Setup the email, user, notes, chunkSize, quality, and callbacks.
        '''
        pValue += pIncrement
        pDlg.Update(pValue, "Setting up qube jobs...")

        if "@" not in valuesPkg['email']:
            valuesPkg['email'] += "@fellowshipchurch.com"
        values['mailaddress'] = valuesPkg['email']
        values['user'] = valuesPkg['email'].split("@")[0]
        values['notes'] = valuesPkg.get('notes', '').strip()
        values['callbacks'] = [{
            'triggers': 'done-job-self',
            'language': 'mail'
        }]
        '''
        ----------------------------------------------------------------------------------------
        Third, setup each rqItem's job
        ----------------------------------------------------------------------------------------
        '''
        '''
        Setup the name, rqIndex, outputFiles, agenda and cpus.
        '''
        rqJobs = []
        seqPattern = re.compile("\[#+\]")
        for rqItem in valuesPkg['aeProject']['rqItems']:

            outPaths = []
            for item in rqItem['outFilePaths']:
                outPaths.append(str(item))

            sequence = True
            if ".mov" in ",".join(outPaths):
                sequence = False

            pValue += pIncrement
            pDlg.Update(pValue, "Setting up RQ Item: %s..." % rqItem['comp'])

            rqiValues = copy.deepcopy(values)
            rqiPkg = rqiValues.setdefault('package', {})
            rqiPkg['rqIndex'] = str(rqItem['index'])
            rqiValues['name'] = "%s %s-%s" % (values['name'], rqItem['index'],
                                              rqItem['comp'])
            rqiPkg['outputFiles'] = ",".join(outPaths)
            logging.debug("Output File Paths: %s" % rqItem['outFilePaths'])
            agendaRange = str(
                "%s-%s" % (rqItem['startTime'], int(rqItem['stopTime']) - 1))
            logging.debug("Agenda Range: %s" % agendaRange)
            chunkSize = 30
            if not sequence:
                rqiValues['cpus'] = 1
                chunkSize = 1000000
            rqiValues['agenda'] = qb.genchunks(chunkSize, agendaRange)
            logging.debug("Agenda: %s" % rqiValues['agenda'])
            rqiValues['agenda'][-1]['resultpackage'] = {
                'outputPaths': ",".join(outPaths)
            }
            rqiPkg['frameCount'] = int(rqItem['stopTime']) - int(
                rqItem['startTime']) - 1
            '''
            If it's a sequence, mark all existing frames as complete.
            '''
            if sequence:

                for path in outPaths:
                    pValue += pIncrement
                    pDlg.Update(
                        pValue, "Finding missing frames for %s..." %
                        os.path.basename(path))

                    seqPad = len(seqPattern.findall(path)[-1]) - 2
                    initFrame = sequenceTools.padFrame(rqItem['startTime'],
                                                       seqPad)
                    initPath = seqPattern.sub(initFrame, path)
                    logging.debug("initPath: %s" % initPath)
                    seq = sequenceTools.Sequence(initPath,
                                                 frameRange=agendaRange)
                    missingFrames = seq.getMissingFrames()
                    logging.debug("Missing Frames: %s" % missingFrames)

                    for task in rqiValues['agenda']:
                        logging.debug("Name: %s" % task['name'])
                        if "-" in task['name']:
                            tStart, tEnd = task['name'].split("-")
                        else:
                            tStart = tEnd = task['name']
                        tRange = range(int(tStart), int(tEnd) + 1)
                        found = False
                        for frame in tRange:
                            if frame in missingFrames:
                                found = True
                        if not found:
                            task['status'] = 'complete'
                            if task.has_key("resultPackage"):
                                task['resultpackage'][
                                    'progress'] = '1'  # 100% chunk progress
                            else:
                                task['resultpackage'] = {'progress': '1'}
                            logging.debug("Marking task as complete: %s" %
                                          task)
            '''
            Delete any unecessary attributes
            '''
            rqiPkg['aeProject'] = None
            del rqiPkg['aeProject']
            rqiPkg['notes'] = None
            del rqiPkg['notes']
            rqiPkg['gui'] = None
            del rqiPkg['gui']

            rqJobs.append(rqiValues)

        logging.debug("rqJobs: %s" % rqJobs)

        pValue += pIncrement
        pDlg.Update(pValue, "Submitting Jobs to qube...")

        submittedJobs = qb.submit(rqJobs)
        logging.debug("Submitted Jobs: %s" % submittedJobs)

        pValue += pIncrement
        pDlg.Update(pValue, "Refreshing Qube...")

        # Update the Qube GUI
        request = qbCache.QbServerRequest(
            action="jobinfo",
            value=[i['id'] for i in submittedJobs],
            method='reload')
        qbCache.QbServerRequestQueue.put(request)

        pDlg.Update(maxProgress, "Complete!")
    except Exception, e:
        exc_type, exc_value, exc_traceback = sys.exc_info()
        dlg = wx.MessageDialog(None, "Unable to submit jobs %s" % e, "Error",
                               wx.OK | wx.ICON_ERROR)
        logging.error(
            repr(traceback.format_exception(exc_type, exc_value,
                                            exc_traceback)))
        dlg.ShowModal()