def submit(job, poolName=None):
g = Graph(job.name, job, username(), poolName)
print g.submit(gethost(), getport())
help='Number of task by group.')
parser.add_argument('nbCmds',
metavar='NB_CMD',
type=int,
help='Number of commands by task')
parser.add_argument('server',
metavar='SERVER',
type=str,
help='Name of the server on which to submit job')
parser.add_argument('--sleep',
type=int,
default=5,
help='Sleep duration in s')
args = parser.parse_args()
# Create graph
runner = "puliclient.contrib.generic.GenericRunner"
name = "Job_Test"
tags = {"prod": "mikros_test", "comment": "JOB TEST", "nbFrames": 1}
puliGraph = Graph(name, poolName='mik', tags=tags, maxRN=5)
print "NB_GROUP = " + str(args.nbGroups)
print "NB_TASK = " + str(args.nbTasks)
print "NB_CMD = " + str(args.nbCmds)
print "SERVER = " + args.server
createFolderNodes(puliGraph, args.nbGroups, args.nbTasks, args.nbCmds, args.sleep)
puliGraph.submit(args.server, 8004)
if __name__ == "__main__":
#
# Submission script
#
tags = {
"prod": "prod_name",
"shot": "shot_code",
# Add any valuable info relative to the job here: type, step, version, iteration...
}
# First we create a graph
# Added to the graph is a dict of tags that will be used to clarify the job process
graph = Graph('simple job', tags=tags)
# To define a Task, we need 3 arguments :
# - its name
# - a runner is a python class that defines the workflow execution for a given job type.
# Here, we will use MyRunner which has been declared previously
# - an arguments dict
name = "wait 10s"
runner = "example.MyRunner"
arguments = {"wait": 10}
# Then add a new task to the graph
graph.addNewTask(name, runner=runner, arguments=arguments)
# Finally submit the graph to the server
graph.submit("pulitest", 8004)
def submit(job, poolName=None, maxRN=-1):
g = Graph(job.name, job, username(), poolName, maxRN)
print g.submit(gethost(), getport())
if __name__ == "__main__":
#
# Submission script
#
tags = {
"prod": "prod_name",
"shot": "shot_code",
# Add any valuable info relative to the job here: type, step, version, iteration...
}
# First we create a graph
# Added to the graph is a dict of tags that will be used to clarify the job process
graph = Graph("simple job", tags=tags)
# To define a Task, we need 3 arguments :
# - its name
# - a runner is a python class that defines the workflow execution for a given job type.
# Here, we will use MyRunner which has been declared previously
# - an arguments dict
name = "wait 10s"
runner = "example.MyRunner"
arguments = {"wait": 10}
# Then add a new task to the graph
graph.addNewTask(name, runner=runner, arguments=arguments)
# Finally submit the graph to the server
graph.submit("pulitest", 8004)
parser.add_argument('nbTasks',
metavar='NB_TASK',
type=int,
help='Number of task by group.')
parser.add_argument('nbCmds',
metavar='NB_CMD',
type=int,
help='Number of commands by task')
parser.add_argument('server',
metavar='SERVER',
type=str,
help='Name of the server on which to submit job')
parser.add_argument('--sleep', type=int, default=5, help='Sleep duration in s')
args = parser.parse_args()
# Create graph
runner = "puliclient.contrib.generic.GenericRunner"
name = "Job_Test"
tags = {"prod": "mikros_test", "comment": "JOB TEST", "nbFrames": 1}
puliGraph = Graph(name, poolName='mik', tags=tags, maxRN=5)
print "NB_GROUP = " + str(args.nbGroups)
print "NB_TASK = " + str(args.nbTasks)
print "NB_CMD = " + str(args.nbCmds)
print "SERVER = " + args.server
createFolderNodes(puliGraph, args.nbGroups, args.nbTasks, args.nbCmds,
args.sleep)
puliGraph.submit(args.server, 8004)
if __name__ == '__main__':
(options, args) = process_args()
command = "sleep `shuf -i %d-%d -n 1`" % (options.min, options.max )
# command = "sleep %%MI_FRAME%%"
# command = "bash /tmp/test.bash"
# command = "/datas/jsa/01_test/memcrash/mem.py -l %s -k 0.01" % options.min
args = { "cmd":command, "delay": options.min, "start":1, "end":options.num, "packetSize":1, "scriptTimeOut":45 }
tags = { "prod":"test", "shot":"test", "nbFrames":options.num }
#
# Create custom graph
#
simpleTask = Task( name=options.jobname, arguments=args, tags=tags, runner="puliclient.contrib.commandlinerunner.CommandLineRunner", lic="katana" )
# simpleTask = Task( name="T-Generic", arguments=args, tags=tags, runner="puliclient.contrib.debug.WaitRunner" )
graph = Graph( options.jobname, simpleTask, tags=tags, poolName='default' )
# graph.addNewTask( name="T1", arguments={ "args": command, "start":1, "end":5, "packetSize":1 }, tags={ "prod":"test", "shot":"test", "nbFrames":5}, runner=runner )
#
#
# g1 = graph.addNewTaskGroup( name="group1", tags=tags )
# g1.addTask( simpleTask )
# g1.addNewTask( name="T2", arguments={ "args": command, "start":1, "end":1, "packetSize":1 }, tags={ "prod":"test", "shot":"test", "nbFrames":1}, runner=runner )
# g1.addNewTask( name="T3", arguments={ "args": command, "start":1, "end":1, "packetSize":1 }, tags={ "prod":"test", "shot":"test", "nbFrames":1}, runner=runner )
#
# graph.addNewTask( name="T4", arguments={ "args": command, "start":1, "end":10, "packetSize":1 }, tags={ "prod":"test", "shot":"test", "nbFrames":10}, runner=runner )
if options.dump:
print graph
#
#!/usr/bin/python
# coding: utf-8
from puliclient import Graph
tags = {
"prod": "prod_name",
"shot": "shot_code",
# Add any valuable info relative to the job here: type, step, version, iteration...
}
# First we create a graph
# Added to the graph is a dict of tags that will be used to clarify the job process
graph = Graph('simple_job', tags=tags)
# In this example we will create 2 tasks with multiple frames
# The first task executes one process per command, the second will group several processes by commands
# We still need to define its name and arguments dict
# The default runner automatically handles the following arguments:
# - cmd: a string representing the command to start
# - start: start frame number
# - end: end frame number
# First task
name = "multiple commands"
arguments = {
"cmd": "sleep %%MI_FRAME%%",
"start": 1,
"end": 10,
if __name__ == '__main__':
# In this example we will create 2 tasks to execute custom callables
# Internally we will use a predefined runner: CallableRunner
# The function or class method we are asking to execute must be accessible to the render nodes.
tags = {
"prod": "prod_name",
"shot": "shot_code",
# Add any valuable info relative to the job here: type, step, version, iteration...
}
# First we create a graph
# Added to the graph is a dict of tags that will be used to clarify the job process
graph = Graph('job with callable', tags=tags)
# First task
name = "callable function"
graph.addNewCallable(
myFunction,
"callable function",
user_args=("param1", "param2"),
user_kwargs={"wait": 10},
)
# Second task
name = "callable method"
graph.addNewCallable(
#!/usr/bin/python
# coding: utf-8
from puliclient import Graph
tags = {
"prod": "prod_name",
"shot": "shot_code",
# Add any valuable info relative to the job here: type, step, version, iteration...
}
# First we create a graph
# Added to the graph is a dict of tags that will be used to clarify the job process
graph = Graph('simple_job', tags=tags)
# Then we define a task. A task is basically a group of one or several commands all of the same kind.
# It means, each command will start the same process but might have different params. For instance you will
# create a task to execute a single process or to exec several times the same process like when rendering
# an image sequence.
# In this example we will create a simple task with only one command
# Therefore we need to define only 2 attributes:
# - its name
# - a dict of its arguments
name = "my command"
# The arguments dict can handle many arguments.
# When creating a classic command (like a shell command) we internally use a "runner" called "CommandLineRunner".
# This default runner automatically handles the following arguments:
# - cmd: a string representing the command to start
# - start: start frame number
1. a task waits the end of a taskgroup --> ok
2. a taskgroup waits the end of a task --> ok
3. a taskgroup waits the end of a taskgroup --> ok
"""
from puliclient import Task, Graph, GraphDumper
if __name__ == '__main__':
args = { "cmd":"sleep 30", "start":1, "end":10, "packetSize":1 }
tags = { "prod":"test", "shot":"test" }
decomposer = "puliclient.contrib.generic.GenericDecomposer"
#
# 1. a task waits the end of a taskgroup
#
graph = Graph('1st case', tags=tags)
tg1 = graph.addNewTaskGroup( name="TG1" )
tg1.addNewTask( name="task1", arguments=args, tags=tags, decomposer=decomposer )
tg1.addNewTask( name="task2", arguments=args, tags=tags, decomposer=decomposer )
task3 = graph.addNewTask( name="task3", arguments=args, tags=tags, decomposer=decomposer )
graph.addEdges( [(tg1, task3)] )
graph.submit("pulitest", 8004)
#
# 2. a taskgroup waits the end of a task
#
graph = Graph('2nd case', tags=tags)
tg1 = graph.addNewTaskGroup( name="TG1" )
tg1.addNewTask( name="task1", arguments=args, tags=tags, decomposer=decomposer )
tg1.addNewTask( name="task2", arguments=args, tags=tags, decomposer=decomposer )
#
# Create custom graph
#
command = "sleep `shuf -i %d-%d -n 1`" % (options.min, options.max )
args = { "args": command, "start":1, "end":options.num, "packetSize":1 }
tags = { "prod":"test", "shot":"test", "nbFrames":options.num }
runner = "puliclient.contrib.commandlinerunner.CommandLineRunner"
if options.lic != "":
lic=options.lic
else:
lic=None
simpleTask = Task( name="NoLic", arguments=args, tags=tags, runner=runner, lic=None )
graph = Graph(options.jobname, poolName='default', tags=tags)
tg = graph.addNewTaskGroup( name="TG")
tg.addNewTask( name="Lic", arguments=args, tags=tags, runner=runner, lic=lic )
tg2 = graph.addNewTaskGroup( name="TG2")
tg2.addTask( simpleTask )
if options.dump:
print graph
#
# Execute
#
if options.execute:
graph.execute()
else:
options, args = parser.parse_args()
return options, args
pass
if __name__ == '__main__':
(options, args) = process_args()
tags = {"prod": "zaza", "shot": "zuzu", "nbFrames": options.num}
#
# Create custom graph
#
try:
graph = Graph(options.jobname, tags=tags)
# graph.addNewCallableTaskRAW(
# targetCall=myFunction,
# name="une_fonction",
# user_args=(1, tags)
# )
# command = "sleep `shuf -i 20-30 -n 1`"
# args = {"args": command, "start": 1, "end": 10, "packetSize": 1}
# runner = "puliclient.contrib.commandlinerunner.CommandLineRunner"
# graph.addNewTask(name="Timer", arguments=args, tags=tags, runner=runner, timer=time.time()+600)
graph.addNewCallable(
MyClass.myMethod,
def compile(job):
g = Graph(job.name, job, username())
from pprint import pprint
pprint(g.toRepresentation())
def compile(job):
g = Graph(job.name, job, username())
from pprint import pprint
pprint(g.toRepresentation())
#!/usr/bin/python
# coding: utf-8
import sys
sys.path.insert(0,"/s/apps/lin/vfx_test_apps/OpenRenderManagement/Puli/src")
from puliclient import Task, Graph
tags = { "prod":"test", "shot":"test" }
arguments={ 'cmd':'echo RUUUUUNN !!!' }
graph = Graph('debug', tags=tags, poolName='default' )
graph.addNewTask( "DO_SOMETHING", tags=tags, arguments=arguments )
# graph.submit("puliserver", 8004)
graph.execute()
#!/usr/bin/python
# coding: utf-8
import sys
sys.path.insert(0,"/s/apps/lin/vfx_test_apps/OpenRenderManagement/Puli/src")
from puliclient import Task, Graph
# command = "nuke -x -F %%MI_FRAME%% ma_comp.nk"
# command = "nuke -x -F %%MI_START%%-%%MI_END%% ma_comp.nk"
# command = "echo currFrame=%%MI_FRAME%% in [%%MI_START%%-%%MI_END%%]"
command = "sleep %%MI_FRAME%%s"
arguments={ 'cmd':command, 'start':1, 'end':50 , 'packetSize':10}
tags = { "prod":"test", "shot":"test" }
graph = Graph('Testing DefaultRunner', tags=tags, poolName='default' )
task1 = graph.addNewTask( "TASK_1", tags=tags, arguments=arguments )
graph.submit( "puliserver", 8004)
#
# Pour une execution en local
#
#graph.execute()
#!/usr/bin/python
# coding: utf-8
from puliclient import Graph
tags = {
"prod": "prod_name",
"shot": "shot_code",
# Add any valuable info relative to the job here: type, step, version, iteration...
}
# First we create a graph
# Added to the graph is a dict of tags that will be used to clarify the job process
graph = Graph("simple_job", tags=tags)
# In this example we will create 2 tasks with multiple frames
# The first task executes one process per command, the second will group several processes by commands
# We still need to define its name and arguments dict
# The default runner automatically handles the following arguments:
# - cmd: a string representing the command to start
# - start: start frame number
# - end: end frame number
# First task
name = "multiple commands"
arguments = {"cmd": "sleep %%MI_FRAME%%", "start": 1, "end": 10}
graph.addNewTask(name, arguments=arguments)
from puliclient import Task, Graph
if __name__ == '__main__':
simpleTask = Task(name="simpleTask",
arguments={},
decomposer="puliclient.contrib.generic.GenericDecomposer")
graph = Graph('simpleGraph', simpleTask)
host, port = ("127.0.0.1", 8004)
graph.submit(host, port)
- Graph's method addNewTask() will create a task and attach it to the graph
- A chain of dependencies is added to the graph, the default end status is [DONE]
task1 > task4 > task3 > task2 > task5 > task6
- Classic submission
"""
from puliclient import Task, TaskGroup, Graph, DONE
if __name__ == '__main__':
args = { "cmd":"sleep 30", "start":1, "end":10, "packetSize":1 }
tags = { "prod":"test", "shot":"test" }
decomposer = "puliclient.contrib.generic.GenericDecomposer"
graph = Graph('simpleGraph', tags=tags)
task1 = graph.addNewTask(name="task1", arguments=args, tags=tags)
task2 = graph.addNewTask(name="task2", arguments=args, tags=tags)
task3 = graph.addNewTask(name="task3", arguments=args, tags=tags)
task4 = graph.addNewTask(name="task4", arguments=args, tags=tags)
task5 = graph.addNewTask(name="task5", arguments=args, tags=tags)
task6 = graph.addNewTask(name="task6", arguments=args, tags=tags)
# Create a chain of dependencies, execution order will be:
# task1 > task4 > task3 > task2 > task5 > task6
graph.addChain( [task1, task4, task3, task2, task5, task6] )
graph.submit(host="vfxpc64")
# PREVIOUS METHOD (still valid)
'proj': mayaProjPath,
'scene': mayaScenePath,
's': startFrame,
'e': endFrame,
'packetSize': packetSize,
'framesList': framesList,
'rx': resX,
'ry': resY,
'cam': camera,
'as': '1',
'o': outFileFormat,
'pad': paddingSize,
'f': renderDirPath,
'p': outFileNamePrefix
}
tags = { "prod":"test", "shot":"test" }
decomposer='puliclient.contrib.puliDbg.mtoa.MtoaDecomposer'
graph = Graph('mtoa_graph', tags=tags, poolName='default' )
graph.addNewTask( "mtoa_task", tags=tags, arguments=arguments, decomposer=decomposer, lic="shave&mtoa" )
graph.submit("vfxpc64", 8004)
# graph.execute()
if __name__ == "__main__":
# In this example we will create 2 tasks to execute custom callables
# Internally we will use a predefined runner: CallableRunner
# The function or class method we are asking to execute must be accessible to the render nodes.
tags = {
"prod": "prod_name",
"shot": "shot_code",
# Add any valuable info relative to the job here: type, step, version, iteration...
}
# First we create a graph
# Added to the graph is a dict of tags that will be used to clarify the job process
graph = Graph("job with callable", tags=tags)
# First task
name = "callable function"
graph.addNewCallable(myFunction, "callable function", user_args=("param1", "param2"), user_kwargs={"wait": 10})
# Second task
name = "callable method"
graph.addNewCallable(
MyClass.myMethod,
"callable method",
user_args=(),
user_kwargs={"param1": "first_param", "param2": "second_param", "wait": 10},
)
