def submit(job, poolName=None): g = Graph(job.name, job, username(), poolName) print g.submit(gethost(), getport())
help='Number of task by group.') parser.add_argument('nbCmds', metavar='NB_CMD', type=int, help='Number of commands by task') parser.add_argument('server', metavar='SERVER', type=str, help='Name of the server on which to submit job') parser.add_argument('--sleep', type=int, default=5, help='Sleep duration in s') args = parser.parse_args() # Create graph runner = "puliclient.contrib.generic.GenericRunner" name = "Job_Test" tags = {"prod": "mikros_test", "comment": "JOB TEST", "nbFrames": 1} puliGraph = Graph(name, poolName='mik', tags=tags, maxRN=5) print "NB_GROUP = " + str(args.nbGroups) print "NB_TASK = " + str(args.nbTasks) print "NB_CMD = " + str(args.nbCmds) print "SERVER = " + args.server createFolderNodes(puliGraph, args.nbGroups, args.nbTasks, args.nbCmds, args.sleep) puliGraph.submit(args.server, 8004)
if __name__ == "__main__": # # Submission script # tags = { "prod": "prod_name", "shot": "shot_code", # Add any valuable info relative to the job here: type, step, version, iteration... } # First we create a graph # Added to the graph is a dict of tags that will be used to clarify the job process graph = Graph('simple job', tags=tags) # To define a Task, we need 3 arguments : # - its name # - a runner is a python class that defines the workflow execution for a given job type. # Here, we will use MyRunner which has been declared previously # - an arguments dict name = "wait 10s" runner = "example.MyRunner" arguments = {"wait": 10} # Then add a new task to the graph graph.addNewTask(name, runner=runner, arguments=arguments) # Finally submit the graph to the server graph.submit("pulitest", 8004)
def submit(job, poolName=None, maxRN=-1): g = Graph(job.name, job, username(), poolName, maxRN) print g.submit(gethost(), getport())
if __name__ == "__main__": # # Submission script # tags = { "prod": "prod_name", "shot": "shot_code", # Add any valuable info relative to the job here: type, step, version, iteration... } # First we create a graph # Added to the graph is a dict of tags that will be used to clarify the job process graph = Graph("simple job", tags=tags) # To define a Task, we need 3 arguments : # - its name # - a runner is a python class that defines the workflow execution for a given job type. # Here, we will use MyRunner which has been declared previously # - an arguments dict name = "wait 10s" runner = "example.MyRunner" arguments = {"wait": 10} # Then add a new task to the graph graph.addNewTask(name, runner=runner, arguments=arguments) # Finally submit the graph to the server graph.submit("pulitest", 8004)
parser.add_argument('nbTasks', metavar='NB_TASK', type=int, help='Number of task by group.') parser.add_argument('nbCmds', metavar='NB_CMD', type=int, help='Number of commands by task') parser.add_argument('server', metavar='SERVER', type=str, help='Name of the server on which to submit job') parser.add_argument('--sleep', type=int, default=5, help='Sleep duration in s') args = parser.parse_args() # Create graph runner = "puliclient.contrib.generic.GenericRunner" name = "Job_Test" tags = {"prod": "mikros_test", "comment": "JOB TEST", "nbFrames": 1} puliGraph = Graph(name, poolName='mik', tags=tags, maxRN=5) print "NB_GROUP = " + str(args.nbGroups) print "NB_TASK = " + str(args.nbTasks) print "NB_CMD = " + str(args.nbCmds) print "SERVER = " + args.server createFolderNodes(puliGraph, args.nbGroups, args.nbTasks, args.nbCmds, args.sleep) puliGraph.submit(args.server, 8004)
if __name__ == '__main__': (options, args) = process_args() command = "sleep `shuf -i %d-%d -n 1`" % (options.min, options.max ) # command = "sleep %%MI_FRAME%%" # command = "bash /tmp/test.bash" # command = "/datas/jsa/01_test/memcrash/mem.py -l %s -k 0.01" % options.min args = { "cmd":command, "delay": options.min, "start":1, "end":options.num, "packetSize":1, "scriptTimeOut":45 } tags = { "prod":"test", "shot":"test", "nbFrames":options.num } # # Create custom graph # simpleTask = Task( name=options.jobname, arguments=args, tags=tags, runner="puliclient.contrib.commandlinerunner.CommandLineRunner", lic="katana" ) # simpleTask = Task( name="T-Generic", arguments=args, tags=tags, runner="puliclient.contrib.debug.WaitRunner" ) graph = Graph( options.jobname, simpleTask, tags=tags, poolName='default' ) # graph.addNewTask( name="T1", arguments={ "args": command, "start":1, "end":5, "packetSize":1 }, tags={ "prod":"test", "shot":"test", "nbFrames":5}, runner=runner ) # # # g1 = graph.addNewTaskGroup( name="group1", tags=tags ) # g1.addTask( simpleTask ) # g1.addNewTask( name="T2", arguments={ "args": command, "start":1, "end":1, "packetSize":1 }, tags={ "prod":"test", "shot":"test", "nbFrames":1}, runner=runner ) # g1.addNewTask( name="T3", arguments={ "args": command, "start":1, "end":1, "packetSize":1 }, tags={ "prod":"test", "shot":"test", "nbFrames":1}, runner=runner ) # # graph.addNewTask( name="T4", arguments={ "args": command, "start":1, "end":10, "packetSize":1 }, tags={ "prod":"test", "shot":"test", "nbFrames":10}, runner=runner ) if options.dump: print graph #
#!/usr/bin/python # coding: utf-8 from puliclient import Graph tags = { "prod": "prod_name", "shot": "shot_code", # Add any valuable info relative to the job here: type, step, version, iteration... } # First we create a graph # Added to the graph is a dict of tags that will be used to clarify the job process graph = Graph('simple_job', tags=tags) # In this example we will create 2 tasks with multiple frames # The first task executes one process per command, the second will group several processes by commands # We still need to define its name and arguments dict # The default runner automatically handles the following arguments: # - cmd: a string representing the command to start # - start: start frame number # - end: end frame number # First task name = "multiple commands" arguments = { "cmd": "sleep %%MI_FRAME%%", "start": 1, "end": 10,
if __name__ == '__main__': # In this example we will create 2 tasks to execute custom callables # Internally we will use a predefined runner: CallableRunner # The function or class method we are asking to execute must be accessible to the render nodes. tags = { "prod": "prod_name", "shot": "shot_code", # Add any valuable info relative to the job here: type, step, version, iteration... } # First we create a graph # Added to the graph is a dict of tags that will be used to clarify the job process graph = Graph('job with callable', tags=tags) # First task name = "callable function" graph.addNewCallable( myFunction, "callable function", user_args=("param1", "param2"), user_kwargs={"wait": 10}, ) # Second task name = "callable method" graph.addNewCallable(
#!/usr/bin/python # coding: utf-8 from puliclient import Graph tags = { "prod": "prod_name", "shot": "shot_code", # Add any valuable info relative to the job here: type, step, version, iteration... } # First we create a graph # Added to the graph is a dict of tags that will be used to clarify the job process graph = Graph('simple_job', tags=tags) # Then we define a task. A task is basically a group of one or several commands all of the same kind. # It means, each command will start the same process but might have different params. For instance you will # create a task to execute a single process or to exec several times the same process like when rendering # an image sequence. # In this example we will create a simple task with only one command # Therefore we need to define only 2 attributes: # - its name # - a dict of its arguments name = "my command" # The arguments dict can handle many arguments. # When creating a classic command (like a shell command) we internally use a "runner" called "CommandLineRunner". # This default runner automatically handles the following arguments: # - cmd: a string representing the command to start # - start: start frame number
1. a task waits the end of a taskgroup --> ok 2. a taskgroup waits the end of a task --> ok 3. a taskgroup waits the end of a taskgroup --> ok """ from puliclient import Task, Graph, GraphDumper if __name__ == '__main__': args = { "cmd":"sleep 30", "start":1, "end":10, "packetSize":1 } tags = { "prod":"test", "shot":"test" } decomposer = "puliclient.contrib.generic.GenericDecomposer" # # 1. a task waits the end of a taskgroup # graph = Graph('1st case', tags=tags) tg1 = graph.addNewTaskGroup( name="TG1" ) tg1.addNewTask( name="task1", arguments=args, tags=tags, decomposer=decomposer ) tg1.addNewTask( name="task2", arguments=args, tags=tags, decomposer=decomposer ) task3 = graph.addNewTask( name="task3", arguments=args, tags=tags, decomposer=decomposer ) graph.addEdges( [(tg1, task3)] ) graph.submit("pulitest", 8004) # # 2. a taskgroup waits the end of a task # graph = Graph('2nd case', tags=tags) tg1 = graph.addNewTaskGroup( name="TG1" ) tg1.addNewTask( name="task1", arguments=args, tags=tags, decomposer=decomposer ) tg1.addNewTask( name="task2", arguments=args, tags=tags, decomposer=decomposer )
# # Create custom graph # command = "sleep `shuf -i %d-%d -n 1`" % (options.min, options.max ) args = { "args": command, "start":1, "end":options.num, "packetSize":1 } tags = { "prod":"test", "shot":"test", "nbFrames":options.num } runner = "puliclient.contrib.commandlinerunner.CommandLineRunner" if options.lic != "": lic=options.lic else: lic=None simpleTask = Task( name="NoLic", arguments=args, tags=tags, runner=runner, lic=None ) graph = Graph(options.jobname, poolName='default', tags=tags) tg = graph.addNewTaskGroup( name="TG") tg.addNewTask( name="Lic", arguments=args, tags=tags, runner=runner, lic=lic ) tg2 = graph.addNewTaskGroup( name="TG2") tg2.addTask( simpleTask ) if options.dump: print graph # # Execute # if options.execute: graph.execute() else:
options, args = parser.parse_args() return options, args pass if __name__ == '__main__': (options, args) = process_args() tags = {"prod": "zaza", "shot": "zuzu", "nbFrames": options.num} # # Create custom graph # try: graph = Graph(options.jobname, tags=tags) # graph.addNewCallableTaskRAW( # targetCall=myFunction, # name="une_fonction", # user_args=(1, tags) # ) # command = "sleep `shuf -i 20-30 -n 1`" # args = {"args": command, "start": 1, "end": 10, "packetSize": 1} # runner = "puliclient.contrib.commandlinerunner.CommandLineRunner" # graph.addNewTask(name="Timer", arguments=args, tags=tags, runner=runner, timer=time.time()+600) graph.addNewCallable( MyClass.myMethod,
def compile(job): g = Graph(job.name, job, username()) from pprint import pprint pprint(g.toRepresentation())
#!/usr/bin/python # coding: utf-8 import sys sys.path.insert(0,"/s/apps/lin/vfx_test_apps/OpenRenderManagement/Puli/src") from puliclient import Task, Graph tags = { "prod":"test", "shot":"test" } arguments={ 'cmd':'echo RUUUUUNN !!!' } graph = Graph('debug', tags=tags, poolName='default' ) graph.addNewTask( "DO_SOMETHING", tags=tags, arguments=arguments ) # graph.submit("puliserver", 8004) graph.execute()
#!/usr/bin/python # coding: utf-8 import sys sys.path.insert(0,"/s/apps/lin/vfx_test_apps/OpenRenderManagement/Puli/src") from puliclient import Task, Graph # command = "nuke -x -F %%MI_FRAME%% ma_comp.nk" # command = "nuke -x -F %%MI_START%%-%%MI_END%% ma_comp.nk" # command = "echo currFrame=%%MI_FRAME%% in [%%MI_START%%-%%MI_END%%]" command = "sleep %%MI_FRAME%%s" arguments={ 'cmd':command, 'start':1, 'end':50 , 'packetSize':10} tags = { "prod":"test", "shot":"test" } graph = Graph('Testing DefaultRunner', tags=tags, poolName='default' ) task1 = graph.addNewTask( "TASK_1", tags=tags, arguments=arguments ) graph.submit( "puliserver", 8004) # # Pour une execution en local # #graph.execute()
#!/usr/bin/python # coding: utf-8 from puliclient import Graph tags = { "prod": "prod_name", "shot": "shot_code", # Add any valuable info relative to the job here: type, step, version, iteration... } # First we create a graph # Added to the graph is a dict of tags that will be used to clarify the job process graph = Graph("simple_job", tags=tags) # In this example we will create 2 tasks with multiple frames # The first task executes one process per command, the second will group several processes by commands # We still need to define its name and arguments dict # The default runner automatically handles the following arguments: # - cmd: a string representing the command to start # - start: start frame number # - end: end frame number # First task name = "multiple commands" arguments = {"cmd": "sleep %%MI_FRAME%%", "start": 1, "end": 10} graph.addNewTask(name, arguments=arguments)
from puliclient import Task, Graph if __name__ == '__main__': simpleTask = Task(name="simpleTask", arguments={}, decomposer="puliclient.contrib.generic.GenericDecomposer") graph = Graph('simpleGraph', simpleTask) host, port = ("127.0.0.1", 8004) graph.submit(host, port)
- Graph's method addNewTask() will create a task and attach it to the graph - A chain of dependencies is added to the graph, the default end status is [DONE] task1 > task4 > task3 > task2 > task5 > task6 - Classic submission """ from puliclient import Task, TaskGroup, Graph, DONE if __name__ == '__main__': args = { "cmd":"sleep 30", "start":1, "end":10, "packetSize":1 } tags = { "prod":"test", "shot":"test" } decomposer = "puliclient.contrib.generic.GenericDecomposer" graph = Graph('simpleGraph', tags=tags) task1 = graph.addNewTask(name="task1", arguments=args, tags=tags) task2 = graph.addNewTask(name="task2", arguments=args, tags=tags) task3 = graph.addNewTask(name="task3", arguments=args, tags=tags) task4 = graph.addNewTask(name="task4", arguments=args, tags=tags) task5 = graph.addNewTask(name="task5", arguments=args, tags=tags) task6 = graph.addNewTask(name="task6", arguments=args, tags=tags) # Create a chain of dependencies, execution order will be: # task1 > task4 > task3 > task2 > task5 > task6 graph.addChain( [task1, task4, task3, task2, task5, task6] ) graph.submit(host="vfxpc64") # PREVIOUS METHOD (still valid)
'proj': mayaProjPath, 'scene': mayaScenePath, 's': startFrame, 'e': endFrame, 'packetSize': packetSize, 'framesList': framesList, 'rx': resX, 'ry': resY, 'cam': camera, 'as': '1', 'o': outFileFormat, 'pad': paddingSize, 'f': renderDirPath, 'p': outFileNamePrefix } tags = { "prod":"test", "shot":"test" } decomposer='puliclient.contrib.puliDbg.mtoa.MtoaDecomposer' graph = Graph('mtoa_graph', tags=tags, poolName='default' ) graph.addNewTask( "mtoa_task", tags=tags, arguments=arguments, decomposer=decomposer, lic="shave&mtoa" ) graph.submit("vfxpc64", 8004) # graph.execute()
if __name__ == "__main__": # In this example we will create 2 tasks to execute custom callables # Internally we will use a predefined runner: CallableRunner # The function or class method we are asking to execute must be accessible to the render nodes. tags = { "prod": "prod_name", "shot": "shot_code", # Add any valuable info relative to the job here: type, step, version, iteration... } # First we create a graph # Added to the graph is a dict of tags that will be used to clarify the job process graph = Graph("job with callable", tags=tags) # First task name = "callable function" graph.addNewCallable(myFunction, "callable function", user_args=("param1", "param2"), user_kwargs={"wait": 10}) # Second task name = "callable method" graph.addNewCallable( MyClass.myMethod, "callable method", user_args=(), user_kwargs={"param1": "first_param", "param2": "second_param", "wait": 10}, )