def submit(args): gpus = args.gpus.strip().split(',') """Submit function of local jobs.""" def mthread_submit(nworker, nserver, envs): """ customized submit script, that submit nslave jobs, each must contain args as parameter note this can be a lambda function containing additional parameters in input Parameters ---------- nworker: number of slave process to start up nserver: number of server nodes to start up envs: enviroment variables to be added to the starting programs """ procs = {} for i, gpu in enumerate(gpus): for j in range(args.num_threads): procs[i] = Thread(target=exec_cmd, args=(args.command + ['--gpus=%s'%gpu], 'worker', i*args.num_threads+j, envs)) procs[i].setDaemon(True) procs[i].start() for i in range(len(gpus)*args.num_threads, len(gpus)*args.num_threads + nserver): procs[i] = Thread(target=exec_cmd, args=(args.command, 'server', i, envs)) procs[i].setDaemon(True) procs[i].start() # call submit, with nslave, the commands to run each job and submit function tracker.submit(args.num_threads*len(gpus), args.num_servers, fun_submit=mthread_submit, pscmd=(' '.join(args.command)))