def execute_single_task(task): # get the executables bunch = get_task_name(task) execlist = [] try: execlist = args.config["bunch"][bunch]["exec"].split(",") except: pass timer = perf.Timer(logging) timer.start('provision-execlist') # provision execlist on disk for item in execlist: execpath = os.path.join(execdir, item) # check if the program exists and its mtime mtime = None try: stat = os.stat(execpath) mtime = int(stat.st_mtime) except: pass if not mtime or mtime < tnow: # if the file does not exist or it is older than current time, # contact the head task content = client.getexec(args.master,item,mtime) swc = "None" if content: swc = str(len(content)) logging.debug("Host received %s" % (swc)) if content: if len(content) > 0: logging.debug("Host saving to %s" % (execpath)) f = open(execpath,"w") f.write(content) f.close() os.utime(execpath,(tnow, tnow)) timer.stop('provision-execlist') prog = execlist[0] logging.debug("Task %s, exec %s" % (prog, execlist)) progpath = os.path.join(execdir, prog) if not os.path.exists(progpath): logging.error("task %s not started, program %s not found" % (task, progpath)) return taskdir = "snapw.%d/tasks/%s" % (args.pid, task) config.mkdir_p(taskdir) qdir = os.path.join(args.workdir, args.qactname, task) tdir = os.path.join(args.workdir, taskdir) logging.info("starting task %s, prog %s, workdir %s, qdir %s\n" % (task, prog, tdir, qdir)) # get server information host = args.server.host port = args.server.port # construct a command line for worker cmd = python + " %s -t %s -h %s:%d -q %s" % ( progpath, task, host, port, qdir) logging.info("starting cmd %s" % (cmd)) # start the work process p = subprocess.Popen(cmd.split(), cwd=tdir, close_fds=True) return p, prog
def execute_single_task(task): # get the executables bunch = "%s" % (task.split("-",1)[0]) execlist = [] try: execlist = args.config["bunch"][bunch]["exec"].split(",") except: pass for item in execlist: execpath = os.path.join(execdir, item) # check if the program exists and its mtime mtime = None try: stat = os.stat(execpath) mtime = int(stat.st_mtime) except: pass if not mtime or mtime < tnow: # the file does not exist or it is older than current time, # contact the head task content = client.getexec(args.master,item,mtime) swc = "None" if content: swc = str(len(content)) print "Host received %s" % (swc) if content: if len(content) > 0: print "Host saving to %s" % (execpath) f = open(execpath,"w") f.write(content) f.close() os.utime(execpath,(tnow, tnow)) prog = execlist[0] print "Task %s, exec %s" % (prog, execlist) progpath = os.path.join(execdir, prog) if not os.path.exists(progpath): line = "*** Error: task %s not started, program %s not found\n" % ( task, progpath) args.flog.write(line) args.flog.flush() return taskdir = "snapw.%d/tasks/%s" % (args.pid, task) config.mkdir_p(taskdir) qdir = os.path.join(args.workdir, args.qactname, task) tdir = os.path.join(args.workdir, taskdir) line = "starting task %s, prog %s, workdir %s, qdir %s\n" % ( task, prog, tdir, qdir) args.flog.write(line) args.flog.flush() # get server information host = args.server.host port = args.server.port # construct a command line cmd = python + " %s -t %s -h %s:%d -q %s" % ( progpath, task, host, port, qdir) args.flog.write("starting cmd %s\n" % (cmd)) args.flog.flush() # start the work process p = pexec.Exec(tdir,cmd) return p
def Execute(args): args.flog.write("Execute " + str(args.active) + "\n") tnow = time.time() if len(args.active) > 0: execdir = os.path.join(args.workdir, "snapw.%d/exec" % (args.pid)) config.mkdir_p(execdir) # execute the tasks sequentially for task in args.active: # get the executables bunch = "%s" % (task.split("-",1)[0]) execlist = [] try: execlist = args.config["bunch"][bunch]["exec"].split(",") except: pass for item in execlist: execpath = os.path.join(execdir, item) # check if the program exists and its mtime mtime = None try: stat = os.stat(execpath) mtime = int(stat.st_mtime) except: pass if not mtime or mtime < tnow: # the file does not exist or it is older than current time, # contact the head task content = client.getexec(args.master,item,mtime) swc = "None" if content: swc = str(len(content)) print "Host received %s" % (swc) if content: if len(content) > 0: print "Host saving to %s" % (execpath) f = open(execpath,"w") f.write(content) f.close() os.utime(execpath,(tnow, tnow)) prog = execlist[0] print "Task %s, exec %s" % (prog, execlist) progpath = os.path.join(execdir, prog) if not os.path.exists(progpath): line = "*** Error: task %s not started, program %s not found\n" % ( task, progpath) args.flog.write(line) args.flog.flush() continue taskdir = "snapw.%d/tasks/%s" % (args.pid, task) config.mkdir_p(taskdir) qdir = os.path.join(args.workdir, args.qactname, task) tdir = os.path.join(args.workdir, taskdir) line = "starting task %s, prog %s, workdir %s, qdir %s\n" % ( task, prog, tdir, qdir) args.flog.write(line) args.flog.flush() # get server information host = args.server.host port = args.server.port # construct a command line cmd = python + " %s -t %s -h %s:%d -q %s" % ( progpath, task, host, port, qdir) args.flog.write("starting cmd %s\n" % (cmd)) args.flog.flush() # start the work process p = pexec.Exec(tdir,cmd) # wait for the process to complete while True: args.flog.write("polling\n") args.flog.flush() status = pexec.Poll(p) if status != None: break time.sleep(0.1) args.flog.write("finished\n") args.flog.flush() # send done to master client.done(args.master, args.id)
def execute_single_task(task): # get the executables bunch = "%s" % (task.split("-", 1)[0]) execlist = [] try: execlist = args.config["bunch"][bunch]["exec"].split(",") except: pass for item in execlist: execpath = os.path.join(execdir, item) # check if the program exists and its mtime mtime = None try: stat = os.stat(execpath) mtime = int(stat.st_mtime) except: pass if not mtime or mtime < tnow: # the file does not exist or it is older than current time, # contact the head task content = client.getexec(args.master, item, mtime) swc = "None" if content: swc = str(len(content)) print "Host received %s" % (swc) if content: if len(content) > 0: print "Host saving to %s" % (execpath) f = open(execpath, "w") f.write(content) f.close() os.utime(execpath, (tnow, tnow)) prog = execlist[0] print "Task %s, exec %s" % (prog, execlist) progpath = os.path.join(execdir, prog) if not os.path.exists(progpath): line = "*** Error: task %s not started, program %s not found\n" % ( task, progpath) args.flog.write(line) args.flog.flush() return taskdir = "snapw.%d/tasks/%s" % (args.pid, task) config.mkdir_p(taskdir) qdir = os.path.join(args.workdir, args.qactname, task) tdir = os.path.join(args.workdir, taskdir) line = "starting task %s, prog %s, workdir %s, qdir %s\n" % ( task, prog, tdir, qdir) args.flog.write(line) args.flog.flush() # get server information host = args.server.host port = args.server.port # construct a command line cmd = python + " %s -t %s -h %s:%d -q %s" % (progpath, task, host, port, qdir) args.flog.write("starting cmd %s\n" % (cmd)) args.flog.flush() # start the work process p = pexec.Exec(tdir, cmd) return p