def genBlock(self, hipfilename):
if VERBOSE:
if self.ropnode:
print('Generating block for "%s" from "%s"' %
(self.ropnode.path(), self.afnode.path()))
else:
print('Generating command block from "%s"' %
(self.afnode.path()))
block = af.Block(self.name, self.type)
block.setParser(self.parser)
block.setCommand(self.cmd % vars(), self.cmd_useprefix)
if self.preview != '':
block.setFiles([self.preview])
if self.numeric:
block.setNumeric(self.frame_first, self.frame_last,
self.frame_pertask, self.frame_inc)
else:
t = 0
for cmd in self.tasks_cmds:
task = af.Task(self.tasks_names[t])
task.setCommand(cmd)
if len(self.tasks_previews):
task.setFiles([self.tasks_previews[t]])
block.tasks.append(task)
t += 1
block.setFramesPerTask(self.frame_pertask)
block.setSequential(self.frame_sequential)
block.setCapacity(self.capacity)
if self.capacity_min != -1 or self.capacity_max != -1:
block.setVariableCapacity(self.capacity_min, self.capacity_max)
block.setTasksMaxRunTime(self.maxruntime)
if self.subblock:
if self.max_runtasks > -1:
block.setMaxRunningTasks(self.max_runtasks)
if self.maxperhost > -1:
block.setMaxRunTasksPerHost(self.maxperhost)
if self.hosts_mask != '':
block.setHostsMask(self.hosts_mask)
if self.hosts_mask_exclude != '':
block.setHostsMaskExclude(self.hosts_mask_exclude)
if self.dependmask != '':
if self.fullrangedepend:
block.setDependMask(self.dependmask)
else:
block.setTasksDependMask(self.dependmask)
if self.subtaskdepend:
block.setDependSubTask()
return block
def submitAsJob(self, graph_file, node_path):
"""
[virtual]
Called when the scheduler should cook the entire TOP Network as a standalone job.
by pressing the 'Submit as Job' button on the scheduler node UI.
Creates a job which cooks that TOP graph using hython.
Returns the status URI for the submitted job - just to open manager Web GUI.
graph_file Path to a .hip file containing the TOP Network, relative to $PDG_DIR.
node_path Op path to the TOP Network
"""
self._log("submitAsJob({},{})".format(graph_file, node_path))
# Constuct a command for hython + topcook script
cmd = 'hython'
# Use PDG licence
cmd += ' --pdg'
# Specify script that cooks graph
cmd += ' "%s/pdgjob/topcook.py"' % os.getenv('HHP')
# Set verbosity level
cmd += ' --verbosity 2'
# Set hip file:
cmd += ' --hip "%s"' % hou.hipFile.path()
# Set top network to cook
cmd += ' --toppath "%s"' % node_path
# Constuct a job:
job = af.Job(self['gj_name'].evaluateString())
job.setBranch(self['job_branch'].evaluateString())
job.setPriority(self['gj_priority'].evaluateInt())
job.setDependMask(self['gj_depend_mask'].evaluateString())
job.setDependMaskGlobal(self['gj_depend_mask_global'].evaluateString())
job.setHostsMask(self['gj_hosts_mask'].evaluateString())
job.setHostsMaskExclude(self['gj_hosts_mask_exclude'].evaluateString())
if self['gj_start_paused'].evaluateInt():
job.setPaused()
# Block
block = af.Block('PDG-GRAPH', self['gj_service'].evaluateString())
block.setCapacity(self['gj_capacity'].evaluateInt())
block.addTicket(self['gj_ticket'].evaluateString(), 1)
# Task
task = af.Task(node_path)
task.setCommand(cmd)
task.setEnv('AF_USERNAME', cgruconfig.VARS['USERNAME'])
# Append task and block and send job
block.tasks.append(task)
job.blocks.append(block)
job.send()
return None
def _constructTask(self, work_item):
task = af.Task(work_item.name)
task.setCommand(self.expandCommandTokens(work_item.command, work_item))
# Set environment variables
task.setEnv('PDG_RESULT_SERVER', str(self.workItemResultServerAddr()))
task.setEnv('PDG_ITEM_NAME', str(work_item.name))
task.setEnv('PDG_DIR', str(self.workingDir(False)))
task.setEnv('PDG_TEMP', str(self.tempDir(False)))
task.setEnv('PDG_SHARED_TEMP', str(self.tempDir(False)))
task.setEnv('PDG_INDEX', str(work_item.index))
task.setEnv('PDG_INDEX4', "{:04d}".format(work_item.index))
task.setEnv('PDG_SCRIPTDIR', str(self.scriptDir(False)))
# This adds paths mapping data to environment
self.resolvePathMapping(task.data['environment'])
# We need to set it after task.setEnv,
# as on task.setEnv CGRU paths mapping applied.
# And better not to apply CGRU paths mapping on PDG paths mapping data.
return task
def execute(self, context):
sce = context.scene
cgru_props = sce.cgru
rd = context.scene.render
images = None
engine_string = sce.render.engine
sceneModified = False # if the opriginal scene modified checker
# set selected pool (static)
CGRU_Submit.selected_pool = cgru_props.pools
# Import Afanasy module:
import af
# Calculate temporary scene path:
scenefile = bpy.data.filepath
if scenefile.endswith('.blend'):
scenefile = scenefile[:-6]
renderscenefile = "%s.%s.blend" % (scenefile,
time.strftime('%Y%m%d%H%M%S'))
# Make all Local and pack all textures and objects
if cgru_props.packLinkedObjects:
bpy.ops.object.make_local(type='ALL')
sceneModified = True
if cgru_props.relativePaths:
bpy.ops.file.make_paths_relative()
sceneModified = True
if cgru_props.packTextures:
bpy.ops.file.pack_all()
sceneModified = True
# Get job name:
jobname = cgru_props.jobname
# If job name is empty use scene file name:
if not jobname:
jobname = os.path.basename(scenefile)
# Try to cut standart '.blend' extension:
if jobname.endswith('.blend'):
jobname = jobname[:-6]
# Get frames settings:
fstart = sce.frame_start
fend = sce.frame_end
finc = sce.frame_step
fpertask = cgru_props.fpertask
sequential = cgru_props.sequential
# Check frames settings:
if fpertask < 1:
fpertask = 1
if fend < fstart:
fend = fstart
# Create a job:
job = af.Job(jobname)
servicename = 'blender'
renderlayer_names = []
layers = bpy.context.scene.render.layers
if cgru_props.splitRenderLayers and len(layers) > 1:
for layer in layers:
if layer.use:
renderlayer_names.append(layer.name)
else:
renderlayer_names.append('')
for renderlayer_name in renderlayer_names:
block = None
images = None
# Create block
if cgru_props.splitRenderLayers and len(layers) > 1:
txt_block = bpy.data.texts.new("layer_%s" % renderlayer_name)
txt_block.write(LAYER_TEXT_BLOCK.format(renderlayer_name))
block = af.Block("layer_%s" % renderlayer_name, servicename)
else:
block = af.Block(engine_string, servicename)
# Check current render engine
if engine_string == 'BLENDER_RENDER':
block.setParser('blender_render')
elif engine_string == 'CYCLES':
block.setParser('blender_cycles')
if cgru_props.filepath != '':
pos = cgru_props.filepath.find('#')
if pos != -1:
if cgru_props.filepath[pos - 1] in '._- ':
images = "{0}{1}{2}".format(
cgru_props.filepath[:pos - 1], renderlayer_name,
cgru_props.filepath[pos - 1:])
else:
images = "{0}{1}{2}".format(cgru_props.filepath[:pos],
renderlayer_name,
cgru_props.filepath[pos:])
else:
images = "{0}{1}".format(cgru_props.filepath,
renderlayer_name)
output_images = re.sub(r'(#+)', r'@\1@', images)
if output_images.startswith('//'):
output_images = os.path.join(
os.path.dirname(renderscenefile),
output_images.replace('//', ''))
if rd.file_extension not in output_images:
block.setFiles([output_images + rd.file_extension])
else:
block.setFiles([output_images])
if cgru_props.splitRenderLayers and len(layers) > 1:
python_options = ' --python-text "layer_%s"' % renderlayer_name
else:
python_options = ''
cmd = CMD_TEMPLATE.format(blend_scene=renderscenefile,
render_engine=engine_string,
python_options=python_options,
output_options=' -o "%s" ' %
images if images else '',
frame_inc=finc)
block.setCommand(cmd)
block.setNumeric(fstart, fend, fpertask, finc)
block.setSequential(sequential)
block.setHostsMaskExclude(getHostsMaskExclude())
job.blocks.append(block)
if cgru_props.make_movie:
movie_block = af.Block(cgru_props.mov_name + '-movie',
'movgen')
movie_block.setDependMask(job.blocks[-1])
movie_task = af.Task(cgru_props.mov_name)
movie_block.tasks.append(movie_task)
cmd = os.getenv('CGRU_LOCATION')
cmd = os.path.join(cmd, 'utilities', 'moviemaker',
'makemovie.py')
cmd = 'python "%s"' % cmd
cmd += ' --codec "%s"' % cgru_props.mov_codecs
cmd += ' -r "%sx%s"' % (cgru_props.mov_width,
cgru_props.mov_height)
cmd += ' "%s"' % images.replace('@#', '#').replace('#@', '#')
cmd += ' "%s"' % cgru_props.mov_name
movie_task.setCommand(cmd)
job.blocks.append(movie_block)
# Set job running parameters:
if cgru_props.maxruntasks > -1:
job.setMaxRunningTasks(cgru_props.maxruntasks)
if cgru_props.priority > -1:
job.setPriority(cgru_props.priority)
if cgru_props.dependmask != '':
job.setDependMask(cgru_props.dependmask)
if cgru_props.dependmaskglobal != '':
job.setDependMaskGlobal(cgru_props.dependmaskglobal)
if cgru_props.hostsmask != '':
job.setHostsMask(cgru_props.hostsmask)
if cgru_props.hostsmaskexclude != '':
job.setHostsMaskExclude(cgru_props.hostsmaskexclude)
if cgru_props.pause:
job.offLine()
if cgru_props.previewPendingApproval:
job.setPPApproval()
# Make server to delete temporary file after job deletion:
job.setCmdPost('deletefiles "%s"' % os.path.abspath(renderscenefile))
# Print job information:
job.output(True)
# Save Temporary file
bpy.ops.wm.save_as_mainfile(filepath=renderscenefile, copy=True)
# Clean up temp text blocks
if cgru_props.splitRenderLayers and len(layers) > 1:
for text in bpy.data.texts:
if "layer_" in text:
bpy.data.texts.remove(text)
# Send job to server:
result = job.send()
if not result[0]:
msg = ("An error occurred when submitting job to Afanasy."
"Check console.")
self.report({'ERROR'}, msg)
else:
msg = "Job id:%s successfully submit to Afanasy."
self.report({'INFO'}, msg % result[1]['id'])
# if opriginal scene is modified - we need to reload the scene file
if sceneModified:
bpy.ops.wm.open_mainfile(filepath=scenefile + ".blend")
return {'FINISHED'}
def genBlock(self, hipfilename):
if VERBOSE:
if self.ropnode:
print('Generating block for "%s" from "%s"' %
(self.ropnode.path(), self.afnode.path()))
else:
print('Generating command block from "%s"' %
(self.afnode.path()))
auxargs = self.auxargs
# Place hipfilename and auxargs
cmd = self.cmd % vars()
block = af.Block(self.name, self.service)
block.setParser(self.parser)
block.setCommand(cmd, self.cmd_useprefix)
if self.preview != '':
block.setFiles([self.preview])
if self.numeric:
block.setNumeric(self.frame_first, self.frame_last,
self.frame_pertask, self.frame_inc)
else:
t = 0
for cmd in self.tasks_cmds:
task = af.Task(self.tasks_names[t])
task.setCommand(cmd)
if len(self.tasks_previews):
task.setFiles([self.tasks_previews[t]])
block.tasks.append(task)
t += 1
block.setFramesPerTask(self.frame_pertask)
block.setSequential(self.frame_sequential)
block.setCapacity(self.capacity)
if self.capacity_min != -1 or self.capacity_max != -1:
block.setVariableCapacity(self.capacity_min, self.capacity_max)
block.setTaskMaxRunTime(self.maxruntime)
block.setTaskMinRunTime(self.minruntime)
# Delete files in a block post command:
if len(self.delete_files):
post_cmd = 'deletefiles'
for files in self.delete_files:
post_cmd += ' "%s"' % re.sub('@#*@', '*', files)
block.setCmdPost(post_cmd)
if self.subblock:
if self.max_runtasks > -1:
block.setMaxRunningTasks(self.max_runtasks)
if self.maxperhost > -1:
block.setMaxRunTasksPerHost(self.maxperhost)
if self.hosts_mask != '':
block.setHostsMask(self.hosts_mask)
if self.hosts_mask_exclude != '':
block.setHostsMaskExclude(self.hosts_mask_exclude)
if self.dependmask != '':
if self.fullrangedepend:
block.setDependMask(self.dependmask)
else:
block.setTasksDependMask(self.dependmask)
if self.subtaskdepend:
block.setDependSubTask()
if self.min_memory > -1:
block.setNeedMemory(self.min_memory)
return block
])
else:
block.setFiles(['file_a.@#@.@####@', 'file_b.@#@.@####@'])
else:
block.setCommand(
'python task.py%(str_capacity)s @#@ -v %(verbose)d' % vars(),
False)
block.setTasksName('task @#@')
block.setFiles(['file_a.@#@', 'file_b.@#@'])
if options.frames != '':
fr = frames[b].split('/')
block.setFramesPerTask(int(fr[2]))
numtasks = int(fr[1]) - int(fr[0]) + 1
for t in range(numtasks):
timesec_task = timesec + randtime * random.random()
task = af.Task('#' + str(t))
task.setCommand('-s %(t)d -e %(t)d -t %(timesec_task)g' % vars())
task.setFiles(['%04d' % t])
block.tasks.append(task)
if options.cmdpre != '':
job.setCmdPre(options.cmdpre)
if options.cmdpost != '':
job.setCmdPost(options.cmdpost)
if options.waittime:
job.setWaitTime(int(time.time()) + options.waittime)
if options.user != '':
job.setUserName(options.user)
if Options.move:
if Options.verbose:
print('%s -> %s' % (src,dst))
if not Options.test:
try:
shutil.move(src, dst)
except:
errExit('Can`t move to "%s"' % dst)
continue
cmd = Put
cmd += ' -s "%s"' % src
cmd += ' -d "%s"' % dst
cmd += ' -n "%s"' % os.path.basename(src)
if Options.test:
continue
if Options.afanasy:
task = af.Task(os.path.basename(src))
task.setCommand(cmd)
block.tasks.append(task)
else:
os.system(cmd)
if Options.afanasy and not Options.move and not Options.test:
job.send()
print(json.dumps({'deploy': Out}, indent=4))
if Options.afcap != -1:
block.setCapacity(Options.afcap)
if Options.afmph != -1:
block.setMaxRunTasksPerHost(Options.afmph)
if Options.afmrt != -1:
block.setTasksMaxRunTime(Options.afmrt)
for j in range(0, len(Jobs[i])):
if Options.verbose or Options.debug:
print(Jobs[i][j])
if Options.afanasy:
task = af.Task(os.path.basename(FilesIn[i][j]))
block.tasks.append(task)
task.setCommand(Jobs[i][j])
if not Options.afanasy and not Options.debug:
os.system(Jobs[i][j])
if Options.afanasy:
if Options.verbose or Options.debug:
job.output()
if not Options.debug:
job.send()
if Options.json:
print(json.dumps(OUT))
cmd_encode += ' "%s"' % os.path.join(OutDir, TmpFiles)
cmd_encode += ' "%s"' % movie_name
# Afanasy job creation:
job = af.Job('CUT ' + CutName)
job.setMaxRunningTasks(Options.afmaxtasks)
job.setMaxRunTasksPerHost(Options.afperhost)
if Options.afuser != '': job.setUserName(Options.afuser)
# Delete previous sequence block:
delete_name = None
if os.path.isdir(OutDir):
delete_name = 'delete'
block = af.Block(delete_name)
block.setCapacity(1)
task = af.Task(delete_name + ' ' + os.path.basename(OutDir))
task.setCommand('deletefiles "%s"' % OutDir)
block.tasks.append(task)
job.blocks.append(block)
# Convert block:
block = af.Block('convert', Options.afservice)
if delete_name: block.setDependMask(delete_name)
counter = 0
for cmd in commands:
task = af.Task(task_names[counter])
task.setCommand(cmd)
block.tasks.append(task)
counter += 1
if Options.verbose: print(cmd)
block.setCapacity(Options.afcapacity)
if images != '':
block.setFiles(
afcommon.patternFromDigits(
afcommon.patternFromStdC(images)
).split(';')
)
blocks.append(block)
if mname != '' and images != '':
block = af.Block(mname + '-movie', 'movgen')
blocks.append(block)
block.setWorkingDirectory(pwd)
block.setDependMask(blocknames[-1])
task = af.Task(mname)
block.tasks.append(task)
cmd = os.getenv('CGRU_LOCATION')
cmd = os.path.join(cmd, 'utilities')
cmd = os.path.join(cmd, 'moviemaker')
cmd = os.path.join(cmd, 'makemovie.py')
cmd = 'python "%s"' % cmd
if mcodec != '':
cmd += ' --codec "%s"' % mcodec
if mres != '':
cmd += ' -r "%s"' % mres
cmd += ' "%s"' % images.replace('@#', '#').replace('#@', '#')
if not os.path.isabs(mname):
def genBlock(self, hipfilename):
if VERBOSE:
if self.ropnode:
print('Generating block for "%s" from "%s"' %
(self.ropnode.path(), self.afnode.path()))
else:
print('Generating command block from "%s"' %
(self.afnode.path()))
auxargs = self.auxargs
# Place hipfilename and auxargs
cmd = self.cmd % vars()
block = af.Block(self.name, self.service)
block.setParser(self.parser)
block.setCommand(cmd, self.cmd_useprefix)
if self.preview != '' and self.generate_previews:
block.setFiles([self.preview])
if self.numeric:
block.setNumeric(self.frame_first, self.frame_last,
self.frame_pertask, self.frame_inc)
else:
t = 0
for cmd in self.tasks_cmds:
task = af.Task(self.tasks_names[t])
task.setCommand(cmd)
if len(self.tasks_previews) and self.generate_previews:
task.setFiles([self.tasks_previews[t]])
block.tasks.append(task)
t += 1
block.setFramesPerTask(self.frame_pertask)
block.setSequential(self.frame_sequential)
block.setCapacity(self.capacity)
if self.capacity_min != -1 or self.capacity_max != -1:
block.setVariableCapacity(self.capacity_min, self.capacity_max)
if self.minruntime > 0.01:
block.setTaskMinRunTime(self.minruntime)
if self.maxruntime > 0.01:
block.setTaskMaxRunTime(int(self.maxruntime * 3600.0))
if self.progress_timeout > 0.001:
block.setTaskProgressChangeTimeout(
int(self.progress_timeout * 3600.0))
if self.file_check_enable:
block.checkRenderedFiles(self.file_check_size_mb_min,
self.file_check_size_mb_max)
if self.file_check_skip_existing:
block.skipExistingFiles()
# Delete files in a block post command:
if len(self.delete_files):
post_cmd = 'deletefiles'
for files in self.delete_files:
post_cmd += ' "%s"' % re.sub('@#*@', '*', files)
block.setCmdPost(post_cmd)
if self.subblock:
if self.max_runtasks > -1:
block.setMaxRunningTasks(self.max_runtasks)
if self.maxperhost > -1:
block.setMaxRunTasksPerHost(self.maxperhost)
if self.hosts_mask != '':
block.setHostsMask(self.hosts_mask)
if self.hosts_mask_exclude != '':
block.setHostsMaskExclude(self.hosts_mask_exclude)
if self.dependmask != '':
if self.fullrangedepend:
block.setDependMask(self.dependmask)
else:
block.setTasksDependMask(self.dependmask)
if self.subtaskdepend:
block.setDependSubTask()
if self.min_memory > -1:
block.setNeedMemory(self.min_memory * 1024)
# Process Tickets
if self.tickets_use:
if self.tickets_auto:
for ticket in self.tickets:
block.addTicket(ticket, self.tickets[ticket])
if self.ticket_mem:
block.addTicket('MEM', self.ticket_mem)
if self.tickets_aux_use and self.tickets_aux_data is not None and len(
self.tickets_aux_data):
for ticket in self.tickets_aux_data.split(','):
ticket = ticket.strip().split(':')
if len(ticket) != 2:
hou.ui.displayMessage('Invalid ticket data: "%s".' %
ticket)
continue
block.addTicket(ticket[0], int(ticket[1]))
return block
JobName = Command.split(' ')[0]
if Options.verbose:
print('JobName: %s' % JobName)
print('Renders: %s' % (','.join(Renders)))
print('Command: %s' % Command)
job = af.Job(JobName)
job.setMaintenance()
block = af.Block('Maintenance', Options.service)
block.setParser(Options.parser)
block.setCommand(Command)
for render in Renders:
task = af.Task(render)
block.tasks.append(task)
job.blocks.append(block)
if Options.maxruntasks:
job.setMaxRunningTasks(Options.maxruntasks)
if Options.offline:
job.offline()
if Options.ignore:
job.setIgnoreNimby()
job.setIgnorePaused()
if Options.verbose:
tiles = divx * divy
job = af.Job('Houdini Test: Tile Render')
b_genifd = af.Block('generate ifd', 'hbatch')
b_genifd.setCommand('hrender_af -s @#@ -e @#@ %s %s' % (scene, rop))
b_genifd.setNumeric(f_start, f_finish)
b_render = af.Block('render tiles', 'mantra')
b_render.setCommand('mantrarender tc %(divx)d %(divy)d @#@' % vars())
b_render.setTasksDependMask('generate ifd')
b_render.setFramesPerTask(-tiles)
for f in range(f_start, f_finish + 1):
cmd = ' -R -f ' + ifdd % f
for t in range(0, tiles):
task = af.Task('%d tile %d' % (f, t))
task.setCommand(str(t) + cmd)
task.setFiles((imgd % f) + ('.tile_%d.exr' % t))
b_render.tasks.append(task)
cmd = 'exrjoin %(divx)d %(divy)d %(img)s d && deletefiles %(ifd)s' % vars()
b_exrjoin = af.Block('join tiles')
b_exrjoin.setCommand(cmd, False)
b_exrjoin.setFiles(img)
b_exrjoin.setNumeric(f_start, f_finish)
b_exrjoin.setTasksDependMask('render tiles')
job.blocks.append(b_exrjoin)
job.blocks.append(b_render)
job.blocks.append(b_genifd)
# -*- coding: utf-8 -*-
import af
job = af.Job('example job')
block = af.Block('block of tasks')
block.setWorkingDirectory('/home')
task = af.Task('simple task')
task.setCommand('ls -l')
block.tasks.append(task)
job.blocks.append(block)
job.send()
print('{"progress":"%d sequences found"},' % len(Shots))
print('{"progress":"%d files found"},' % file_counter)
cmd_encode = os.path.join(os.path.dirname(sys.argv[0]), 'makemovie.py')
cmd_encode = 'python "%s"' % os.path.normpath(cmd_encode)
cmd_encode += ' -f %s' % Options.fps
cmd_encode += ' -c %s' % Options.codec
cmd_encode += ' "%s"' % os.path.join(OutDir, TmpFiles)
cmd_encode += ' "%s"' % movie_name
job = af.Job('CUT ' + CutName)
block = af.Block('convert')
counter = 0
for cmd in commands:
task = af.Task(task_names[counter])
task.setCommand(cmd)
block.tasks.append(task)
counter += 1
block.setCapacity(100)
block.setMaxRunTasksPerHost(2)
block.setTasksMaxRunTime(20)
job.blocks.append(block)
block = af.Block('encode')
block.setDependMask('convert')
task = af.Task('encode')
task.setCommand(cmd_encode)
block.tasks.append(task)
job.blocks.append(block)
(seq['first'], seq['last'], seq['count']))
block.setFramesPerTask(Options.affpt)
first = seq['first']
while first <= seq['last']:
last = first + Options.affpt
if last > seq['last']:
last = seq['last']
scene = first
if Options.renumfirst or Options.renumpad:
scene = 1
if Options.renumfirst:
scene = Options.renumfirst
scene = first - seq['first'] + scene
task = af.Task(seq['cmd_name'] +
('[%d-%d]' %
(scene, scene + last - first - 1)))
cmd = seq['cmd']
cmd = cmd.replace('@FIRST@', str(first))
cmd = cmd.replace('@LAST@', str(last))
cmd = cmd.replace('@SCENE@', str(scene))
task.setCommand(cmd)
block.tasks.append(task)
first += Options.affpt
else:
task = af.Task(seq['cmd_name'])
task.setCommand(seq['cmd'])
block.tasks.append(task)
if Options.afcap != -1:
