def genBlock(self, hipfilename): if VERBOSE: if self.ropnode: print('Generating block for "%s" from "%s"' % (self.ropnode.path(), self.afnode.path())) else: print('Generating command block from "%s"' % (self.afnode.path())) block = af.Block(self.name, self.type) block.setParser(self.parser) block.setCommand(self.cmd % vars(), self.cmd_useprefix) if self.preview != '': block.setFiles([self.preview]) if self.numeric: block.setNumeric(self.frame_first, self.frame_last, self.frame_pertask, self.frame_inc) else: t = 0 for cmd in self.tasks_cmds: task = af.Task(self.tasks_names[t]) task.setCommand(cmd) if len(self.tasks_previews): task.setFiles([self.tasks_previews[t]]) block.tasks.append(task) t += 1 block.setFramesPerTask(self.frame_pertask) block.setSequential(self.frame_sequential) block.setCapacity(self.capacity) if self.capacity_min != -1 or self.capacity_max != -1: block.setVariableCapacity(self.capacity_min, self.capacity_max) block.setTasksMaxRunTime(self.maxruntime) if self.subblock: if self.max_runtasks > -1: block.setMaxRunningTasks(self.max_runtasks) if self.maxperhost > -1: block.setMaxRunTasksPerHost(self.maxperhost) if self.hosts_mask != '': block.setHostsMask(self.hosts_mask) if self.hosts_mask_exclude != '': block.setHostsMaskExclude(self.hosts_mask_exclude) if self.dependmask != '': if self.fullrangedepend: block.setDependMask(self.dependmask) else: block.setTasksDependMask(self.dependmask) if self.subtaskdepend: block.setDependSubTask() return block
def submitAsJob(self, graph_file, node_path): """ [virtual] Called when the scheduler should cook the entire TOP Network as a standalone job. by pressing the 'Submit as Job' button on the scheduler node UI. Creates a job which cooks that TOP graph using hython. Returns the status URI for the submitted job - just to open manager Web GUI. graph_file Path to a .hip file containing the TOP Network, relative to $PDG_DIR. node_path Op path to the TOP Network """ self._log("submitAsJob({},{})".format(graph_file, node_path)) # Constuct a command for hython + topcook script cmd = 'hython' # Use PDG licence cmd += ' --pdg' # Specify script that cooks graph cmd += ' "%s/pdgjob/topcook.py"' % os.getenv('HHP') # Set verbosity level cmd += ' --verbosity 2' # Set hip file: cmd += ' --hip "%s"' % hou.hipFile.path() # Set top network to cook cmd += ' --toppath "%s"' % node_path # Constuct a job: job = af.Job(self['gj_name'].evaluateString()) job.setBranch(self['job_branch'].evaluateString()) job.setPriority(self['gj_priority'].evaluateInt()) job.setDependMask(self['gj_depend_mask'].evaluateString()) job.setDependMaskGlobal(self['gj_depend_mask_global'].evaluateString()) job.setHostsMask(self['gj_hosts_mask'].evaluateString()) job.setHostsMaskExclude(self['gj_hosts_mask_exclude'].evaluateString()) if self['gj_start_paused'].evaluateInt(): job.setPaused() # Block block = af.Block('PDG-GRAPH', self['gj_service'].evaluateString()) block.setCapacity(self['gj_capacity'].evaluateInt()) block.addTicket(self['gj_ticket'].evaluateString(), 1) # Task task = af.Task(node_path) task.setCommand(cmd) task.setEnv('AF_USERNAME', cgruconfig.VARS['USERNAME']) # Append task and block and send job block.tasks.append(task) job.blocks.append(block) job.send() return None
def _constructTask(self, work_item): task = af.Task(work_item.name) task.setCommand(self.expandCommandTokens(work_item.command, work_item)) # Set environment variables task.setEnv('PDG_RESULT_SERVER', str(self.workItemResultServerAddr())) task.setEnv('PDG_ITEM_NAME', str(work_item.name)) task.setEnv('PDG_DIR', str(self.workingDir(False))) task.setEnv('PDG_TEMP', str(self.tempDir(False))) task.setEnv('PDG_SHARED_TEMP', str(self.tempDir(False))) task.setEnv('PDG_INDEX', str(work_item.index)) task.setEnv('PDG_INDEX4', "{:04d}".format(work_item.index)) task.setEnv('PDG_SCRIPTDIR', str(self.scriptDir(False))) # This adds paths mapping data to environment self.resolvePathMapping(task.data['environment']) # We need to set it after task.setEnv, # as on task.setEnv CGRU paths mapping applied. # And better not to apply CGRU paths mapping on PDG paths mapping data. return task
def execute(self, context): sce = context.scene cgru_props = sce.cgru rd = context.scene.render images = None engine_string = sce.render.engine sceneModified = False # if the opriginal scene modified checker # set selected pool (static) CGRU_Submit.selected_pool = cgru_props.pools # Import Afanasy module: import af # Calculate temporary scene path: scenefile = bpy.data.filepath if scenefile.endswith('.blend'): scenefile = scenefile[:-6] renderscenefile = "%s.%s.blend" % (scenefile, time.strftime('%Y%m%d%H%M%S')) # Make all Local and pack all textures and objects if cgru_props.packLinkedObjects: bpy.ops.object.make_local(type='ALL') sceneModified = True if cgru_props.relativePaths: bpy.ops.file.make_paths_relative() sceneModified = True if cgru_props.packTextures: bpy.ops.file.pack_all() sceneModified = True # Get job name: jobname = cgru_props.jobname # If job name is empty use scene file name: if not jobname: jobname = os.path.basename(scenefile) # Try to cut standart '.blend' extension: if jobname.endswith('.blend'): jobname = jobname[:-6] # Get frames settings: fstart = sce.frame_start fend = sce.frame_end finc = sce.frame_step fpertask = cgru_props.fpertask sequential = cgru_props.sequential # Check frames settings: if fpertask < 1: fpertask = 1 if fend < fstart: fend = fstart # Create a job: job = af.Job(jobname) servicename = 'blender' renderlayer_names = [] layers = bpy.context.scene.render.layers if cgru_props.splitRenderLayers and len(layers) > 1: for layer in layers: if layer.use: renderlayer_names.append(layer.name) else: renderlayer_names.append('') for renderlayer_name in renderlayer_names: block = None images = None # Create block if cgru_props.splitRenderLayers and len(layers) > 1: txt_block = bpy.data.texts.new("layer_%s" % renderlayer_name) txt_block.write(LAYER_TEXT_BLOCK.format(renderlayer_name)) block = af.Block("layer_%s" % renderlayer_name, servicename) else: block = af.Block(engine_string, servicename) # Check current render engine if engine_string == 'BLENDER_RENDER': block.setParser('blender_render') elif engine_string == 'CYCLES': block.setParser('blender_cycles') if cgru_props.filepath != '': pos = cgru_props.filepath.find('#') if pos != -1: if cgru_props.filepath[pos - 1] in '._- ': images = "{0}{1}{2}".format( cgru_props.filepath[:pos - 1], renderlayer_name, cgru_props.filepath[pos - 1:]) else: images = "{0}{1}{2}".format(cgru_props.filepath[:pos], renderlayer_name, cgru_props.filepath[pos:]) else: images = "{0}{1}".format(cgru_props.filepath, renderlayer_name) output_images = re.sub(r'(#+)', r'@\1@', images) if output_images.startswith('//'): output_images = os.path.join( os.path.dirname(renderscenefile), output_images.replace('//', '')) if rd.file_extension not in output_images: block.setFiles([output_images + rd.file_extension]) else: block.setFiles([output_images]) if cgru_props.splitRenderLayers and len(layers) > 1: python_options = ' --python-text "layer_%s"' % renderlayer_name else: python_options = '' cmd = CMD_TEMPLATE.format(blend_scene=renderscenefile, render_engine=engine_string, python_options=python_options, output_options=' -o "%s" ' % images if images else '', frame_inc=finc) block.setCommand(cmd) block.setNumeric(fstart, fend, fpertask, finc) block.setSequential(sequential) block.setHostsMaskExclude(getHostsMaskExclude()) job.blocks.append(block) if cgru_props.make_movie: movie_block = af.Block(cgru_props.mov_name + '-movie', 'movgen') movie_block.setDependMask(job.blocks[-1]) movie_task = af.Task(cgru_props.mov_name) movie_block.tasks.append(movie_task) cmd = os.getenv('CGRU_LOCATION') cmd = os.path.join(cmd, 'utilities', 'moviemaker', 'makemovie.py') cmd = 'python "%s"' % cmd cmd += ' --codec "%s"' % cgru_props.mov_codecs cmd += ' -r "%sx%s"' % (cgru_props.mov_width, cgru_props.mov_height) cmd += ' "%s"' % images.replace('@#', '#').replace('#@', '#') cmd += ' "%s"' % cgru_props.mov_name movie_task.setCommand(cmd) job.blocks.append(movie_block) # Set job running parameters: if cgru_props.maxruntasks > -1: job.setMaxRunningTasks(cgru_props.maxruntasks) if cgru_props.priority > -1: job.setPriority(cgru_props.priority) if cgru_props.dependmask != '': job.setDependMask(cgru_props.dependmask) if cgru_props.dependmaskglobal != '': job.setDependMaskGlobal(cgru_props.dependmaskglobal) if cgru_props.hostsmask != '': job.setHostsMask(cgru_props.hostsmask) if cgru_props.hostsmaskexclude != '': job.setHostsMaskExclude(cgru_props.hostsmaskexclude) if cgru_props.pause: job.offLine() if cgru_props.previewPendingApproval: job.setPPApproval() # Make server to delete temporary file after job deletion: job.setCmdPost('deletefiles "%s"' % os.path.abspath(renderscenefile)) # Print job information: job.output(True) # Save Temporary file bpy.ops.wm.save_as_mainfile(filepath=renderscenefile, copy=True) # Clean up temp text blocks if cgru_props.splitRenderLayers and len(layers) > 1: for text in bpy.data.texts: if "layer_" in text: bpy.data.texts.remove(text) # Send job to server: result = job.send() if not result[0]: msg = ("An error occurred when submitting job to Afanasy." "Check console.") self.report({'ERROR'}, msg) else: msg = "Job id:%s successfully submit to Afanasy." self.report({'INFO'}, msg % result[1]['id']) # if opriginal scene is modified - we need to reload the scene file if sceneModified: bpy.ops.wm.open_mainfile(filepath=scenefile + ".blend") return {'FINISHED'}
def genBlock(self, hipfilename): if VERBOSE: if self.ropnode: print('Generating block for "%s" from "%s"' % (self.ropnode.path(), self.afnode.path())) else: print('Generating command block from "%s"' % (self.afnode.path())) auxargs = self.auxargs # Place hipfilename and auxargs cmd = self.cmd % vars() block = af.Block(self.name, self.service) block.setParser(self.parser) block.setCommand(cmd, self.cmd_useprefix) if self.preview != '': block.setFiles([self.preview]) if self.numeric: block.setNumeric(self.frame_first, self.frame_last, self.frame_pertask, self.frame_inc) else: t = 0 for cmd in self.tasks_cmds: task = af.Task(self.tasks_names[t]) task.setCommand(cmd) if len(self.tasks_previews): task.setFiles([self.tasks_previews[t]]) block.tasks.append(task) t += 1 block.setFramesPerTask(self.frame_pertask) block.setSequential(self.frame_sequential) block.setCapacity(self.capacity) if self.capacity_min != -1 or self.capacity_max != -1: block.setVariableCapacity(self.capacity_min, self.capacity_max) block.setTaskMaxRunTime(self.maxruntime) block.setTaskMinRunTime(self.minruntime) # Delete files in a block post command: if len(self.delete_files): post_cmd = 'deletefiles' for files in self.delete_files: post_cmd += ' "%s"' % re.sub('@#*@', '*', files) block.setCmdPost(post_cmd) if self.subblock: if self.max_runtasks > -1: block.setMaxRunningTasks(self.max_runtasks) if self.maxperhost > -1: block.setMaxRunTasksPerHost(self.maxperhost) if self.hosts_mask != '': block.setHostsMask(self.hosts_mask) if self.hosts_mask_exclude != '': block.setHostsMaskExclude(self.hosts_mask_exclude) if self.dependmask != '': if self.fullrangedepend: block.setDependMask(self.dependmask) else: block.setTasksDependMask(self.dependmask) if self.subtaskdepend: block.setDependSubTask() if self.min_memory > -1: block.setNeedMemory(self.min_memory) return block
]) else: block.setFiles(['file_a.@#@.@####@', 'file_b.@#@.@####@']) else: block.setCommand( 'python task.py%(str_capacity)s @#@ -v %(verbose)d' % vars(), False) block.setTasksName('task @#@') block.setFiles(['file_a.@#@', 'file_b.@#@']) if options.frames != '': fr = frames[b].split('/') block.setFramesPerTask(int(fr[2])) numtasks = int(fr[1]) - int(fr[0]) + 1 for t in range(numtasks): timesec_task = timesec + randtime * random.random() task = af.Task('#' + str(t)) task.setCommand('-s %(t)d -e %(t)d -t %(timesec_task)g' % vars()) task.setFiles(['%04d' % t]) block.tasks.append(task) if options.cmdpre != '': job.setCmdPre(options.cmdpre) if options.cmdpost != '': job.setCmdPost(options.cmdpost) if options.waittime: job.setWaitTime(int(time.time()) + options.waittime) if options.user != '': job.setUserName(options.user)
if Options.move: if Options.verbose: print('%s -> %s' % (src,dst)) if not Options.test: try: shutil.move(src, dst) except: errExit('Can`t move to "%s"' % dst) continue cmd = Put cmd += ' -s "%s"' % src cmd += ' -d "%s"' % dst cmd += ' -n "%s"' % os.path.basename(src) if Options.test: continue if Options.afanasy: task = af.Task(os.path.basename(src)) task.setCommand(cmd) block.tasks.append(task) else: os.system(cmd) if Options.afanasy and not Options.move and not Options.test: job.send() print(json.dumps({'deploy': Out}, indent=4))
if Options.afcap != -1: block.setCapacity(Options.afcap) if Options.afmph != -1: block.setMaxRunTasksPerHost(Options.afmph) if Options.afmrt != -1: block.setTasksMaxRunTime(Options.afmrt) for j in range(0, len(Jobs[i])): if Options.verbose or Options.debug: print(Jobs[i][j]) if Options.afanasy: task = af.Task(os.path.basename(FilesIn[i][j])) block.tasks.append(task) task.setCommand(Jobs[i][j]) if not Options.afanasy and not Options.debug: os.system(Jobs[i][j]) if Options.afanasy: if Options.verbose or Options.debug: job.output() if not Options.debug: job.send() if Options.json: print(json.dumps(OUT))
cmd_encode += ' "%s"' % os.path.join(OutDir, TmpFiles) cmd_encode += ' "%s"' % movie_name # Afanasy job creation: job = af.Job('CUT ' + CutName) job.setMaxRunningTasks(Options.afmaxtasks) job.setMaxRunTasksPerHost(Options.afperhost) if Options.afuser != '': job.setUserName(Options.afuser) # Delete previous sequence block: delete_name = None if os.path.isdir(OutDir): delete_name = 'delete' block = af.Block(delete_name) block.setCapacity(1) task = af.Task(delete_name + ' ' + os.path.basename(OutDir)) task.setCommand('deletefiles "%s"' % OutDir) block.tasks.append(task) job.blocks.append(block) # Convert block: block = af.Block('convert', Options.afservice) if delete_name: block.setDependMask(delete_name) counter = 0 for cmd in commands: task = af.Task(task_names[counter]) task.setCommand(cmd) block.tasks.append(task) counter += 1 if Options.verbose: print(cmd) block.setCapacity(Options.afcapacity)
if images != '': block.setFiles( afcommon.patternFromDigits( afcommon.patternFromStdC(images) ).split(';') ) blocks.append(block) if mname != '' and images != '': block = af.Block(mname + '-movie', 'movgen') blocks.append(block) block.setWorkingDirectory(pwd) block.setDependMask(blocknames[-1]) task = af.Task(mname) block.tasks.append(task) cmd = os.getenv('CGRU_LOCATION') cmd = os.path.join(cmd, 'utilities') cmd = os.path.join(cmd, 'moviemaker') cmd = os.path.join(cmd, 'makemovie.py') cmd = 'python "%s"' % cmd if mcodec != '': cmd += ' --codec "%s"' % mcodec if mres != '': cmd += ' -r "%s"' % mres cmd += ' "%s"' % images.replace('@#', '#').replace('#@', '#') if not os.path.isabs(mname):
def genBlock(self, hipfilename): if VERBOSE: if self.ropnode: print('Generating block for "%s" from "%s"' % (self.ropnode.path(), self.afnode.path())) else: print('Generating command block from "%s"' % (self.afnode.path())) auxargs = self.auxargs # Place hipfilename and auxargs cmd = self.cmd % vars() block = af.Block(self.name, self.service) block.setParser(self.parser) block.setCommand(cmd, self.cmd_useprefix) if self.preview != '' and self.generate_previews: block.setFiles([self.preview]) if self.numeric: block.setNumeric(self.frame_first, self.frame_last, self.frame_pertask, self.frame_inc) else: t = 0 for cmd in self.tasks_cmds: task = af.Task(self.tasks_names[t]) task.setCommand(cmd) if len(self.tasks_previews) and self.generate_previews: task.setFiles([self.tasks_previews[t]]) block.tasks.append(task) t += 1 block.setFramesPerTask(self.frame_pertask) block.setSequential(self.frame_sequential) block.setCapacity(self.capacity) if self.capacity_min != -1 or self.capacity_max != -1: block.setVariableCapacity(self.capacity_min, self.capacity_max) if self.minruntime > 0.01: block.setTaskMinRunTime(self.minruntime) if self.maxruntime > 0.01: block.setTaskMaxRunTime(int(self.maxruntime * 3600.0)) if self.progress_timeout > 0.001: block.setTaskProgressChangeTimeout( int(self.progress_timeout * 3600.0)) if self.file_check_enable: block.checkRenderedFiles(self.file_check_size_mb_min, self.file_check_size_mb_max) if self.file_check_skip_existing: block.skipExistingFiles() # Delete files in a block post command: if len(self.delete_files): post_cmd = 'deletefiles' for files in self.delete_files: post_cmd += ' "%s"' % re.sub('@#*@', '*', files) block.setCmdPost(post_cmd) if self.subblock: if self.max_runtasks > -1: block.setMaxRunningTasks(self.max_runtasks) if self.maxperhost > -1: block.setMaxRunTasksPerHost(self.maxperhost) if self.hosts_mask != '': block.setHostsMask(self.hosts_mask) if self.hosts_mask_exclude != '': block.setHostsMaskExclude(self.hosts_mask_exclude) if self.dependmask != '': if self.fullrangedepend: block.setDependMask(self.dependmask) else: block.setTasksDependMask(self.dependmask) if self.subtaskdepend: block.setDependSubTask() if self.min_memory > -1: block.setNeedMemory(self.min_memory * 1024) # Process Tickets if self.tickets_use: if self.tickets_auto: for ticket in self.tickets: block.addTicket(ticket, self.tickets[ticket]) if self.ticket_mem: block.addTicket('MEM', self.ticket_mem) if self.tickets_aux_use and self.tickets_aux_data is not None and len( self.tickets_aux_data): for ticket in self.tickets_aux_data.split(','): ticket = ticket.strip().split(':') if len(ticket) != 2: hou.ui.displayMessage('Invalid ticket data: "%s".' % ticket) continue block.addTicket(ticket[0], int(ticket[1])) return block
JobName = Command.split(' ')[0] if Options.verbose: print('JobName: %s' % JobName) print('Renders: %s' % (','.join(Renders))) print('Command: %s' % Command) job = af.Job(JobName) job.setMaintenance() block = af.Block('Maintenance', Options.service) block.setParser(Options.parser) block.setCommand(Command) for render in Renders: task = af.Task(render) block.tasks.append(task) job.blocks.append(block) if Options.maxruntasks: job.setMaxRunningTasks(Options.maxruntasks) if Options.offline: job.offline() if Options.ignore: job.setIgnoreNimby() job.setIgnorePaused() if Options.verbose:
tiles = divx * divy job = af.Job('Houdini Test: Tile Render') b_genifd = af.Block('generate ifd', 'hbatch') b_genifd.setCommand('hrender_af -s @#@ -e @#@ %s %s' % (scene, rop)) b_genifd.setNumeric(f_start, f_finish) b_render = af.Block('render tiles', 'mantra') b_render.setCommand('mantrarender tc %(divx)d %(divy)d @#@' % vars()) b_render.setTasksDependMask('generate ifd') b_render.setFramesPerTask(-tiles) for f in range(f_start, f_finish + 1): cmd = ' -R -f ' + ifdd % f for t in range(0, tiles): task = af.Task('%d tile %d' % (f, t)) task.setCommand(str(t) + cmd) task.setFiles((imgd % f) + ('.tile_%d.exr' % t)) b_render.tasks.append(task) cmd = 'exrjoin %(divx)d %(divy)d %(img)s d && deletefiles %(ifd)s' % vars() b_exrjoin = af.Block('join tiles') b_exrjoin.setCommand(cmd, False) b_exrjoin.setFiles(img) b_exrjoin.setNumeric(f_start, f_finish) b_exrjoin.setTasksDependMask('render tiles') job.blocks.append(b_exrjoin) job.blocks.append(b_render) job.blocks.append(b_genifd)
# -*- coding: utf-8 -*- import af job = af.Job('example job') block = af.Block('block of tasks') block.setWorkingDirectory('/home') task = af.Task('simple task') task.setCommand('ls -l') block.tasks.append(task) job.blocks.append(block) job.send()
print('{"progress":"%d sequences found"},' % len(Shots)) print('{"progress":"%d files found"},' % file_counter) cmd_encode = os.path.join(os.path.dirname(sys.argv[0]), 'makemovie.py') cmd_encode = 'python "%s"' % os.path.normpath(cmd_encode) cmd_encode += ' -f %s' % Options.fps cmd_encode += ' -c %s' % Options.codec cmd_encode += ' "%s"' % os.path.join(OutDir, TmpFiles) cmd_encode += ' "%s"' % movie_name job = af.Job('CUT ' + CutName) block = af.Block('convert') counter = 0 for cmd in commands: task = af.Task(task_names[counter]) task.setCommand(cmd) block.tasks.append(task) counter += 1 block.setCapacity(100) block.setMaxRunTasksPerHost(2) block.setTasksMaxRunTime(20) job.blocks.append(block) block = af.Block('encode') block.setDependMask('convert') task = af.Task('encode') task.setCommand(cmd_encode) block.tasks.append(task) job.blocks.append(block)
(seq['first'], seq['last'], seq['count'])) block.setFramesPerTask(Options.affpt) first = seq['first'] while first <= seq['last']: last = first + Options.affpt if last > seq['last']: last = seq['last'] scene = first if Options.renumfirst or Options.renumpad: scene = 1 if Options.renumfirst: scene = Options.renumfirst scene = first - seq['first'] + scene task = af.Task(seq['cmd_name'] + ('[%d-%d]' % (scene, scene + last - first - 1))) cmd = seq['cmd'] cmd = cmd.replace('@FIRST@', str(first)) cmd = cmd.replace('@LAST@', str(last)) cmd = cmd.replace('@SCENE@', str(scene)) task.setCommand(cmd) block.tasks.append(task) first += Options.affpt else: task = af.Task(seq['cmd_name']) task.setCommand(seq['cmd']) block.tasks.append(task) if Options.afcap != -1: