def execute_single_task(task):
# get the executables
bunch = get_task_name(task)
execlist = []
try:
execlist = args.config["bunch"][bunch]["exec"].split(",")
except:
pass
timer = perf.Timer(logging)
timer.start('provision-execlist')
# provision execlist on disk
for item in execlist:
execpath = os.path.join(execdir, item)
# check if the program exists and its mtime
mtime = None
try:
stat = os.stat(execpath)
mtime = int(stat.st_mtime)
except:
pass
if not mtime or mtime < tnow:
# if the file does not exist or it is older than current time,
# contact the head task
content = client.getexec(args.master,item,mtime)
swc = "None"
if content:
swc = str(len(content))
logging.debug("Host received %s" % (swc))
if content:
if len(content) > 0:
logging.debug("Host saving to %s" % (execpath))
f = open(execpath,"w")
f.write(content)
f.close()
os.utime(execpath,(tnow, tnow))
timer.stop('provision-execlist')
prog = execlist[0]
logging.debug("Task %s, exec %s" % (prog, execlist))
progpath = os.path.join(execdir, prog)
if not os.path.exists(progpath):
logging.error("task %s not started, program %s not found" % (task, progpath))
return
taskdir = "snapw.%d/tasks/%s" % (args.pid, task)
config.mkdir_p(taskdir)
qdir = os.path.join(args.workdir, args.qactname, task)
tdir = os.path.join(args.workdir, taskdir)
logging.info("starting task %s, prog %s, workdir %s, qdir %s\n" % (task, prog, tdir, qdir))
# get server information
host = args.server.host
port = args.server.port
# construct a command line for worker
cmd = python + " %s -t %s -h %s:%d -q %s" % (
progpath, task, host, port, qdir)
logging.info("starting cmd %s" % (cmd))
# start the work process
p = subprocess.Popen(cmd.split(), cwd=tdir, close_fds=True)
return p, prog
def execute_single_task(task):
# get the executables
bunch = get_task_name(task)
execlist = []
try:
execlist = args.config["bunch"][bunch]["exec"].split(",")
except:
pass
timer = perf.Timer(logging)
timer.start('provision-execlist')
# provision execlist on disk
for item in execlist:
execpath = os.path.join(execdir, item)
# check if the program exists and its mtime
mtime = None
try:
stat = os.stat(execpath)
mtime = int(stat.st_mtime)
except:
pass
if not mtime or mtime < tnow:
# if the file does not exist or it is older than current time,
# contact the head task
content = client.getexec(args.master,item,mtime)
swc = "None"
if content:
swc = str(len(content))
logging.debug("Host received %s" % (swc))
if content:
if len(content) > 0:
logging.debug("Host saving to %s" % (execpath))
f = open(execpath,"w")
f.write(content)
f.close()
os.utime(execpath,(tnow, tnow))
timer.stop('provision-execlist')
prog = execlist[0]
logging.debug("Task %s, exec %s" % (prog, execlist))
progpath = os.path.join(execdir, prog)
if not os.path.exists(progpath):
logging.error("task %s not started, program %s not found" % (task, progpath))
return
taskdir = "snapw.%d/tasks/%s" % (args.pid, task)
config.mkdir_p(taskdir)
qdir = os.path.join(args.workdir, args.qactname, task)
tdir = os.path.join(args.workdir, taskdir)
logging.info("starting task %s, prog %s, workdir %s, qdir %s\n" % (task, prog, tdir, qdir))
# get server information
host = args.server.host
port = args.server.port
# construct a command line for worker
cmd = python + " %s -t %s -h %s:%d -q %s" % (
progpath, task, host, port, qdir)
logging.info("starting cmd %s" % (cmd))
# start the work process
p = subprocess.Popen(cmd.split(), cwd=tdir, close_fds=True)
return p, prog
def execute_single_task(task):
# get the executables
bunch = "%s" % (task.split("-",1)[0])
execlist = []
try:
execlist = args.config["bunch"][bunch]["exec"].split(",")
except:
pass
for item in execlist:
execpath = os.path.join(execdir, item)
# check if the program exists and its mtime
mtime = None
try:
stat = os.stat(execpath)
mtime = int(stat.st_mtime)
except:
pass
if not mtime or mtime < tnow:
# the file does not exist or it is older than current time,
# contact the head task
content = client.getexec(args.master,item,mtime)
swc = "None"
if content:
swc = str(len(content))
print "Host received %s" % (swc)
if content:
if len(content) > 0:
print "Host saving to %s" % (execpath)
f = open(execpath,"w")
f.write(content)
f.close()
os.utime(execpath,(tnow, tnow))
prog = execlist[0]
print "Task %s, exec %s" % (prog, execlist)
progpath = os.path.join(execdir, prog)
if not os.path.exists(progpath):
line = "*** Error: task %s not started, program %s not found\n" % (
task, progpath)
args.flog.write(line)
args.flog.flush()
return
taskdir = "snapw.%d/tasks/%s" % (args.pid, task)
config.mkdir_p(taskdir)
qdir = os.path.join(args.workdir, args.qactname, task)
tdir = os.path.join(args.workdir, taskdir)
line = "starting task %s, prog %s, workdir %s, qdir %s\n" % (
task, prog, tdir, qdir)
args.flog.write(line)
args.flog.flush()
# get server information
host = args.server.host
port = args.server.port
# construct a command line
cmd = python + " %s -t %s -h %s:%d -q %s" % (
progpath, task, host, port, qdir)
args.flog.write("starting cmd %s\n" % (cmd))
args.flog.flush()
# start the work process
p = pexec.Exec(tdir,cmd)
return p
def Execute(args):
args.flog.write("Execute " + str(args.active) + "\n")
tnow = time.time()
if len(args.active) > 0:
execdir = os.path.join(args.workdir, "snapw.%d/exec" % (args.pid))
config.mkdir_p(execdir)
# execute the tasks sequentially
for task in args.active:
# get the executables
bunch = "%s" % (task.split("-",1)[0])
execlist = []
try:
execlist = args.config["bunch"][bunch]["exec"].split(",")
except:
pass
for item in execlist:
execpath = os.path.join(execdir, item)
# check if the program exists and its mtime
mtime = None
try:
stat = os.stat(execpath)
mtime = int(stat.st_mtime)
except:
pass
if not mtime or mtime < tnow:
# the file does not exist or it is older than current time,
# contact the head task
content = client.getexec(args.master,item,mtime)
swc = "None"
if content:
swc = str(len(content))
print "Host received %s" % (swc)
if content:
if len(content) > 0:
print "Host saving to %s" % (execpath)
f = open(execpath,"w")
f.write(content)
f.close()
os.utime(execpath,(tnow, tnow))
prog = execlist[0]
print "Task %s, exec %s" % (prog, execlist)
progpath = os.path.join(execdir, prog)
if not os.path.exists(progpath):
line = "*** Error: task %s not started, program %s not found\n" % (
task, progpath)
args.flog.write(line)
args.flog.flush()
continue
taskdir = "snapw.%d/tasks/%s" % (args.pid, task)
config.mkdir_p(taskdir)
qdir = os.path.join(args.workdir, args.qactname, task)
tdir = os.path.join(args.workdir, taskdir)
line = "starting task %s, prog %s, workdir %s, qdir %s\n" % (
task, prog, tdir, qdir)
args.flog.write(line)
args.flog.flush()
# get server information
host = args.server.host
port = args.server.port
# construct a command line
cmd = python + " %s -t %s -h %s:%d -q %s" % (
progpath, task, host, port, qdir)
args.flog.write("starting cmd %s\n" % (cmd))
args.flog.flush()
# start the work process
p = pexec.Exec(tdir,cmd)
# wait for the process to complete
while True:
args.flog.write("polling\n")
args.flog.flush()
status = pexec.Poll(p)
if status != None:
break
time.sleep(0.1)
args.flog.write("finished\n")
args.flog.flush()
# send done to master
client.done(args.master, args.id)
def execute_single_task(task):
# get the executables
bunch = "%s" % (task.split("-", 1)[0])
execlist = []
try:
execlist = args.config["bunch"][bunch]["exec"].split(",")
except:
pass
for item in execlist:
execpath = os.path.join(execdir, item)
# check if the program exists and its mtime
mtime = None
try:
stat = os.stat(execpath)
mtime = int(stat.st_mtime)
except:
pass
if not mtime or mtime < tnow:
# the file does not exist or it is older than current time,
# contact the head task
content = client.getexec(args.master, item, mtime)
swc = "None"
if content:
swc = str(len(content))
print "Host received %s" % (swc)
if content:
if len(content) > 0:
print "Host saving to %s" % (execpath)
f = open(execpath, "w")
f.write(content)
f.close()
os.utime(execpath, (tnow, tnow))
prog = execlist[0]
print "Task %s, exec %s" % (prog, execlist)
progpath = os.path.join(execdir, prog)
if not os.path.exists(progpath):
line = "*** Error: task %s not started, program %s not found\n" % (
task, progpath)
args.flog.write(line)
args.flog.flush()
return
taskdir = "snapw.%d/tasks/%s" % (args.pid, task)
config.mkdir_p(taskdir)
qdir = os.path.join(args.workdir, args.qactname, task)
tdir = os.path.join(args.workdir, taskdir)
line = "starting task %s, prog %s, workdir %s, qdir %s\n" % (
task, prog, tdir, qdir)
args.flog.write(line)
args.flog.flush()
# get server information
host = args.server.host
port = args.server.port
# construct a command line
cmd = python + " %s -t %s -h %s:%d -q %s" % (progpath, task, host,
port, qdir)
args.flog.write("starting cmd %s\n" % (cmd))
args.flog.flush()
# start the work process
p = pexec.Exec(tdir, cmd)
return p
