def Execute(args):
logging.info("Execute " + str(args.active) + "")
tnow = time.time()
overall_timer = perf.Timer(logging)
task_name = "overall: "
if len(args.active) > 0:
# Get's the task name
task_name = "overall: %s" % get_task_name(args.active[0])
overall_timer.start("superstep-%d-%s" % \
(args.server.superstep_count, task_name))
if len(args.active) > 0:
execdir = os.path.join(args.workdir, "snapw.%d/exec" % (args.pid))
config.mkdir_p(execdir)
def execute_single_task(task):
# get the executables
bunch = get_task_name(task)
execlist = []
try:
execlist = args.config["bunch"][bunch]["exec"].split(",")
except:
pass
timer = perf.Timer(logging)
timer.start('provision-execlist')
# provision execlist on disk
for item in execlist:
execpath = os.path.join(execdir, item)
# check if the program exists and its mtime
mtime = None
try:
stat = os.stat(execpath)
mtime = int(stat.st_mtime)
except:
pass
if not mtime or mtime < tnow:
# if the file does not exist or it is older than current time,
# contact the head task
content = client.getexec(args.master,item,mtime)
swc = "None"
if content:
swc = str(len(content))
logging.debug("Host received %s" % (swc))
if content:
if len(content) > 0:
logging.debug("Host saving to %s" % (execpath))
f = open(execpath,"w")
f.write(content)
f.close()
os.utime(execpath,(tnow, tnow))
timer.stop('provision-execlist')
prog = execlist[0]
logging.debug("Task %s, exec %s" % (prog, execlist))
progpath = os.path.join(execdir, prog)
if not os.path.exists(progpath):
logging.error("task %s not started, program %s not found" % (task, progpath))
return
taskdir = "snapw.%d/tasks/%s" % (args.pid, task)
config.mkdir_p(taskdir)
qdir = os.path.join(args.workdir, args.qactname, task)
tdir = os.path.join(args.workdir, taskdir)
logging.info("starting task %s, prog %s, workdir %s, qdir %s\n" % (task, prog, tdir, qdir))
# get server information
host = args.server.host
port = args.server.port
# construct a command line for worker
cmd = python + " %s -t %s -h %s:%d -q %s" % (
progpath, task, host, port, qdir)
logging.info("starting cmd %s" % (cmd))
# start the work process
p = subprocess.Popen(cmd.split(), cwd=tdir, close_fds=True)
return p, prog
# Dynamically check what the number of processors we have on each host
# In any error, default to 1.
max_tasks = 1
var_par_tasks = int(args.config['par_tasks'])
if var_par_tasks <= 0:
try:
max_tasks = os.sysconf('SC_NPROCESSORS_ONLN')
except:
max_tasks = 1
else:
max_tasks = var_par_tasks
# execute the tasks in a parallel fashion by running
# at most max_tasks processes at any point.
task_list = args.active[:]
procs = []
logging.info("Running %d tasks with %d-way parallelism: %s" % \
(len(task_list), max_tasks, str(task_list)))
timer = perf.Timer(logging)
pcounter = 0
counter_map = {}
while True:
while task_list and len(procs) < max_tasks:
task = task_list.pop()
timer.start("prog-%d" % pcounter)
p, prog = execute_single_task(task)
timer.update_extra("prog-%d" % pcounter, "step: %d, pid: %d, prog: %s" \
% (args.server.superstep_count, p.pid, prog))
counter_map[p.pid] = pcounter
pcounter += 1
procs.append(p)
for p in procs:
# wait for the process to complete
pid = p.pid
logging.debug("polling %d" % pid)
status = p.poll()
if status is not None:
timer.stop("prog-%d" % counter_map[p.pid])
del counter_map[p.pid]
logging.debug("finished %d with status %s" % (pid, str(status)))
# error reporting
if status <> 0:
msg = "Pid %d terminated unexpectedly with status %d" % (pid, status)
logging.error(msg)
client.error(args.master, args.id, msg)
procs.remove(p)
if not procs and not task_list:
break
else:
time.sleep(0.1)
overall_timer.stop("superstep-%d-%s" % \
(args.server.superstep_count, task_name))
# send done to master
client.done(args.master, args.id)
def execute_single_task(task):
# get the executables
bunch = get_task_name(task)
execlist = []
try:
execlist = args.config["bunch"][bunch]["exec"].split(",")
except:
pass
timer = perf.Timer(logging)
timer.start('provision-execlist')
# provision execlist on disk
for item in execlist:
execpath = os.path.join(execdir, item)
# check if the program exists and its mtime
mtime = None
try:
stat = os.stat(execpath)
mtime = int(stat.st_mtime)
except:
pass
if not mtime or mtime < tnow:
# if the file does not exist or it is older than current time,
# contact the head task
content = client.getexec(args.master,item,mtime)
swc = "None"
if content:
swc = str(len(content))
logging.debug("Host received %s" % (swc))
if content:
if len(content) > 0:
logging.debug("Host saving to %s" % (execpath))
f = open(execpath,"w")
f.write(content)
f.close()
os.utime(execpath,(tnow, tnow))
timer.stop('provision-execlist')
prog = execlist[0]
logging.debug("Task %s, exec %s" % (prog, execlist))
progpath = os.path.join(execdir, prog)
if not os.path.exists(progpath):
logging.error("task %s not started, program %s not found" % (task, progpath))
return
taskdir = "snapw.%d/tasks/%s" % (args.pid, task)
config.mkdir_p(taskdir)
qdir = os.path.join(args.workdir, args.qactname, task)
tdir = os.path.join(args.workdir, taskdir)
logging.info("starting task %s, prog %s, workdir %s, qdir %s\n" % (task, prog, tdir, qdir))
# get server information
host = args.server.host
port = args.server.port
# construct a command line for worker
cmd = python + " %s -t %s -h %s:%d -q %s" % (
progpath, task, host, port, qdir)
logging.info("starting cmd %s" % (cmd))
# start the work process
p = subprocess.Popen(cmd.split(), cwd=tdir, close_fds=True)
return p, prog
def do_POST(self):
#print "POST path", self.path
parsed_path = urlparse.urlparse(self.path)
message_parts = [
'CLIENT VALUES:',
'client_address=%s (%s)' % (self.client_address,
self.address_string()),
'command=%s' % self.command,
'path=%s' % self.path,
'real path=%s' % parsed_path.path,
'query=%s' % parsed_path.query,
'request_version=%s' % self.request_version,
'',
'SERVER VALUES:',
'server_type=%s' % "host server",
'server_version=%s' % self.server_version,
'sys_version=%s' % self.sys_version,
'protocol_version=%s' % self.protocol_version,
'',
'HEADERS RECEIVED:',
]
for name, value in sorted(self.headers.items()):
message_parts.append('%s=%s' % (name, value.rstrip()))
message_parts.append('')
message = '\r\n'.join(message_parts)
length = int(self.headers.get("Content-Length"))
body = ""
subpath = self.path.split("/")
if subpath[1] == "msg":
dst = subpath[2]
src = subpath[3]
dirname = "snapw.%d/qin/%s" % (self.pid, dst)
config.mkdir_p(dirname)
fname = "%s/%s" % (dirname, src)
f, fnew = config.uniquefile(fname)
if fname <> fnew:
logging.warn("uniquefile?: %s; %s; %s; %s;" % (dirname, src, fname, fnew))
length = int(self.headers.get("Content-Length"))
#print "Content-Length", length
try:
nleft = length
while nleft > 0:
nread = min(102400, nleft)
body = self.rfile.read(nread)
f.write(body)
nleft -= len(body)
except Exception as e:
logging.warn("file stream error: %s" % str(e))
try:
f.close()
os.remove(fnew)
except:
pass
try:
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
except:
pass
return
f.close()
logging.info("message %s length %d" % (fnew, length))
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
return
length = int(self.headers.get("Content-Length"))
#print "Content-Length", length
body = ""
if length and length > 0:
body = self.rfile.read(length)
# Begin the response
self.send_response(200)
self.end_headers()
self.wfile.write('Client: %s\n' % str(self.client_address))
self.wfile.write('User-agent: %s\n' % str(self.headers['user-agent']))
self.wfile.write('Path: %s\n' % self.path)
def do_GET(self):
parsed_path = urlparse.urlparse(self.path)
message_parts = [
'CLIENT VALUES:',
'client_address=%s (%s)' % (self.client_address,
self.address_string()),
'command=%s' % self.command,
'path=%s' % self.path,
'real path=%s' % parsed_path.path,
'query=%s' % parsed_path.query,
'request_version=%s' % self.request_version,
'',
'SERVER VALUES:',
'server_type=%s' % "host server",
'server_version=%s' % self.server_version,
'sys_version=%s' % self.sys_version,
'protocol_version=%s' % self.protocol_version,
'',
'HEADERS RECEIVED:',
]
for name, value in sorted(self.headers.items()):
message_parts.append('%s=%s' % (name, value.rstrip()))
message_parts.append('')
message = '\r\n'.join(message_parts)
subpath = self.path.split("/")
if self.path == "/prepare":
prepare_timer = perf.Timer(logging)
self.server.superstep_count += 1
prepare_timer.start("prepare-%d" % self.server.superstep_count)
# move qin to qact
qinname = "snapw.%d/qin" % (self.pid)
qactname = "snapw.%d/qact" % (self.pid)
# rename an existing qact
if os.path.exists(qactname):
removed = False
if not self.config['debug']:
try:
shutil.rmtree(qactname)
logging.debug("removed dir %s" % qactname)
removed = True
except:
logging.error("error on removing dir %s" % qactname)
if not removed:
t = time.time()
s = time.strftime("%Y%m%d-%H%M%S", time.localtime(t))
mus = "%06d" % (t*1000000 - int(t)*1000000)
qactnewname = "%s-%s-%s" % (qactname, s, mus)
os.rename(qactname, qactnewname)
logging.debug("renamed %s to %s" % (qactname, qactnewname))
# get the number of active tasks, rename existing qin
numtasks = 0
if os.path.exists(qinname):
os.rename(qinname, qactname)
active = os.listdir(qactname)
numtasks = len(active)
# create new qin
config.mkdir_p(qinname)
logging.info("preparing next step: %s, %s" % \
(qinname, qactname))
# send ready to master
client.ready(self.master, self.id, numtasks)
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
prepare_timer.stop("prepare-%d" % self.server.superstep_count)
return
elif self.path == "/quit":
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
SYS_STATS = False
# set the flag to terminate the server
self.server.running = False
self.server.self_dummy()
return
elif self.path == "/getkv":
logging.debug("getting kv file")
self.send_response(200)
if self.server.superstep_count > 1:
body = json.dumps(get_kv_file("supervisor"))
self.send_header('Content-Length', len(body))
self.end_headers()
self.wfile.write(body)
else:
self.send_header('Content-Length', len("None"))
self.end_headers()
self.wfile.write("None")
return
elif self.path == "/dummy":
logging.debug("dummy request")
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
return
elif self.path == "/step":
logging.info("execute next step")
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
# TODO, implement null action,
# skip execution if there are no tasks to execute,
# qact does not exist
# get the tasks to execute
qactname = "snapw.%d/qact" % (self.pid)
active = []
if os.path.exists(qactname):
active = os.listdir(qactname)
logging.debug("active tasks %s" % (str(active)))
self.qactname = qactname
self.active = active # the task list
# start a thread to execute the work tasks
t = threading.Thread(target=Execute, args=(self, ))
t.start()
return
elif self.path == "/config":
logging.debug("get configuration")
body = json.dumps(self.config)
self.send_response(200)
self.send_header('Content-Length', len(body))
self.end_headers()
self.wfile.write(body)
return
elif self.path == "/quit":
logging.info("terminate execution")
SYS_STATS = False
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
sys.exit(0)
self.send_response(200)
self.end_headers()
self.wfile.write(message)
return
def do_GET(self):
#print "GET path", self.path
parsed_path = urlparse.urlparse(self.path)
message_parts = [
'CLIENT VALUES:',
'client_address=%s (%s)' %
(self.client_address, self.address_string()),
'command=%s' % self.command,
'path=%s' % self.path,
'real path=%s' % parsed_path.path,
'query=%s' % parsed_path.query,
'request_version=%s' % self.request_version,
'',
'SERVER VALUES:',
'server_type=%s' % "host server",
'server_version=%s' % self.server_version,
'sys_version=%s' % self.sys_version,
'protocol_version=%s' % self.protocol_version,
'',
'HEADERS RECEIVED:',
]
for name, value in sorted(self.headers.items()):
message_parts.append('%s=%s' % (name, value.rstrip()))
message_parts.append('')
message = '\r\n'.join(message_parts)
#print message
#self.flog.write(message)
#self.flog.flush()
subpath = self.path.split("/")
#print subpath
if self.path == "/prepare":
# move qin to qact
qinname = "snapw.%d/qin" % (self.pid)
qactname = "snapw.%d/qact" % (self.pid)
# rename an existing qact
qactnewname = "none"
if os.path.exists(qactname):
t = time.time()
s = time.strftime("%Y%m%d-%H%M%S", time.localtime(t))
mus = "%06d" % (t * 1000000 - int(t) * 1000000)
qactnewname = "%s-%s-%s" % (qactname, s, mus)
os.rename(qactname, qactnewname)
# get the number of active tasks, rename existing qin
numtasks = 0
if os.path.exists(qinname):
os.rename(qinname, qactname)
active = os.listdir(qactname)
numtasks = len(active)
# create new qin
config.mkdir_p(qinname)
line = "preparing next step: %s, %s, %s\n" % (qinname, qactname,
qactnewname)
self.flog.write(line)
self.flog.flush()
# send ready to master
client.ready(self.master, self.id, numtasks)
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
return
elif self.path == "/quit":
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
# set the flag to terminate the server
self.server.running = False
self.server.self_dummy()
return
elif self.path == "/dummy":
print "dummy request"
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
return
elif self.path == "/step":
line = "execute next step\n"
self.flog.write(line)
self.flog.flush()
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
# TODO, implement null action,
# skip execution if there are no tasks to execute,
# qact does not exist
# get the tasks to execute
qactname = "snapw.%d/qact" % (self.pid)
active = []
if os.path.exists(qactname):
active = os.listdir(qactname)
line = "active tasks %s\n" % (str(active))
self.flog.write(line)
self.flog.flush()
self.qactname = qactname
self.active = active
# start a thread to execute the work tasks
t = threading.Thread(target=Execute, args=(self, ))
t.start()
return
elif self.path == "/config":
line = "get configuration\n"
self.flog.write(line)
self.flog.flush()
body = simplejson.dumps(self.config)
self.send_response(200)
self.send_header('Content-Length', len(body))
self.end_headers()
self.wfile.write(body)
return
elif self.path == "/quit":
line = "terminate execution\n"
self.flog.write(line)
self.flog.flush()
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
sys.exit(0)
self.send_response(200)
#self.send_header('Last-Modified', self.date_time_string(time.time()))
self.end_headers()
self.wfile.write(message)
return
def Execute(args):
args.flog.write("Execute " + str(args.active) + "\n")
tnow = time.time()
if len(args.active) > 0:
execdir = os.path.join(args.workdir, "snapw.%d/exec" % (args.pid))
config.mkdir_p(execdir)
# execute the tasks sequentially
for task in args.active:
# get the executables
bunch = "%s" % (task.split("-",1)[0])
execlist = []
try:
execlist = args.config["bunch"][bunch]["exec"].split(",")
except:
pass
for item in execlist:
execpath = os.path.join(execdir, item)
# check if the program exists and its mtime
mtime = None
try:
stat = os.stat(execpath)
mtime = int(stat.st_mtime)
except:
pass
if not mtime or mtime < tnow:
# the file does not exist or it is older than current time,
# contact the head task
content = client.getexec(args.master,item,mtime)
swc = "None"
if content:
swc = str(len(content))
print "Host received %s" % (swc)
if content:
if len(content) > 0:
print "Host saving to %s" % (execpath)
f = open(execpath,"w")
f.write(content)
f.close()
os.utime(execpath,(tnow, tnow))
prog = execlist[0]
print "Task %s, exec %s" % (prog, execlist)
progpath = os.path.join(execdir, prog)
if not os.path.exists(progpath):
line = "*** Error: task %s not started, program %s not found\n" % (
task, progpath)
args.flog.write(line)
args.flog.flush()
continue
taskdir = "snapw.%d/tasks/%s" % (args.pid, task)
config.mkdir_p(taskdir)
qdir = os.path.join(args.workdir, args.qactname, task)
tdir = os.path.join(args.workdir, taskdir)
line = "starting task %s, prog %s, workdir %s, qdir %s\n" % (
task, prog, tdir, qdir)
args.flog.write(line)
args.flog.flush()
# get server information
host = args.server.host
port = args.server.port
# construct a command line
cmd = python + " %s -t %s -h %s:%d -q %s" % (
progpath, task, host, port, qdir)
args.flog.write("starting cmd %s\n" % (cmd))
args.flog.flush()
# start the work process
p = pexec.Exec(tdir,cmd)
# wait for the process to complete
while True:
args.flog.write("polling\n")
args.flog.flush()
status = pexec.Poll(p)
if status != None:
break
time.sleep(0.1)
args.flog.write("finished\n")
args.flog.flush()
# send done to master
client.done(args.master, args.id)
def execute_single_task(task):
# get the executables
bunch = "%s" % (task.split("-",1)[0])
execlist = []
try:
execlist = args.config["bunch"][bunch]["exec"].split(",")
except:
pass
for item in execlist:
execpath = os.path.join(execdir, item)
# check if the program exists and its mtime
mtime = None
try:
stat = os.stat(execpath)
mtime = int(stat.st_mtime)
except:
pass
if not mtime or mtime < tnow:
# the file does not exist or it is older than current time,
# contact the head task
content = client.getexec(args.master,item,mtime)
swc = "None"
if content:
swc = str(len(content))
print "Host received %s" % (swc)
if content:
if len(content) > 0:
print "Host saving to %s" % (execpath)
f = open(execpath,"w")
f.write(content)
f.close()
os.utime(execpath,(tnow, tnow))
prog = execlist[0]
print "Task %s, exec %s" % (prog, execlist)
progpath = os.path.join(execdir, prog)
if not os.path.exists(progpath):
line = "*** Error: task %s not started, program %s not found\n" % (
task, progpath)
args.flog.write(line)
args.flog.flush()
return
taskdir = "snapw.%d/tasks/%s" % (args.pid, task)
config.mkdir_p(taskdir)
qdir = os.path.join(args.workdir, args.qactname, task)
tdir = os.path.join(args.workdir, taskdir)
line = "starting task %s, prog %s, workdir %s, qdir %s\n" % (
task, prog, tdir, qdir)
args.flog.write(line)
args.flog.flush()
# get server information
host = args.server.host
port = args.server.port
# construct a command line
cmd = python + " %s -t %s -h %s:%d -q %s" % (
progpath, task, host, port, qdir)
args.flog.write("starting cmd %s\n" % (cmd))
args.flog.flush()
# start the work process
p = pexec.Exec(tdir,cmd)
return p
def Execute(args):
logging.info("Execute " + str(args.active) + "")
tnow = time.time()
overall_timer = perf.Timer(logging)
task_name = "overall: "
if len(args.active) > 0:
# Get's the task name
task_name = "overall: %s" % get_task_name(args.active[0])
overall_timer.start("superstep-%d-%s" % \
(args.server.superstep_count, task_name))
if len(args.active) > 0:
execdir = os.path.join(args.workdir, "snapw.%d/exec" % (args.pid))
config.mkdir_p(execdir)
def execute_single_task(task):
# get the executables
bunch = get_task_name(task)
execlist = []
try:
execlist = args.config["bunch"][bunch]["exec"].split(",")
except:
pass
timer = perf.Timer(logging)
timer.start('provision-execlist')
# provision execlist on disk
for item in execlist:
execpath = os.path.join(execdir, item)
# check if the program exists and its mtime
mtime = None
try:
stat = os.stat(execpath)
mtime = int(stat.st_mtime)
except:
pass
if not mtime or mtime < tnow:
# if the file does not exist or it is older than current time,
# contact the head task
content = client.getexec(args.master,item,mtime)
swc = "None"
if content:
swc = str(len(content))
logging.debug("Host received %s" % (swc))
if content:
if len(content) > 0:
logging.debug("Host saving to %s" % (execpath))
f = open(execpath,"w")
f.write(content)
f.close()
os.utime(execpath,(tnow, tnow))
timer.stop('provision-execlist')
prog = execlist[0]
logging.debug("Task %s, exec %s" % (prog, execlist))
progpath = os.path.join(execdir, prog)
if not os.path.exists(progpath):
logging.error("task %s not started, program %s not found" % (task, progpath))
return
taskdir = "snapw.%d/tasks/%s" % (args.pid, task)
config.mkdir_p(taskdir)
qdir = os.path.join(args.workdir, args.qactname, task)
tdir = os.path.join(args.workdir, taskdir)
logging.info("starting task %s, prog %s, workdir %s, qdir %s\n" % (task, prog, tdir, qdir))
# get server information
host = args.server.host
port = args.server.port
# construct a command line for worker
cmd = python + " %s -t %s -h %s:%d -q %s" % (
progpath, task, host, port, qdir)
logging.info("starting cmd %s" % (cmd))
# start the work process
p = subprocess.Popen(cmd.split(), cwd=tdir, close_fds=True)
return p, prog
# Dynamically check what the number of processors we have on each host
# In any error, default to 1.
max_tasks = 1
var_par_tasks = int(args.config['par_tasks'])
if var_par_tasks <= 0:
try:
max_tasks = os.sysconf('SC_NPROCESSORS_ONLN')
except:
max_tasks = 1
else:
max_tasks = var_par_tasks
# execute the tasks in a parallel fashion by running
# at most max_tasks processes at any point.
task_list = args.active[:]
procs = []
logging.info("Running %d tasks with %d-way parallelism: %s" % \
(len(task_list), max_tasks, str(task_list)))
timer = perf.Timer(logging)
pcounter = 0
counter_map = {}
while True:
while task_list and len(procs) < max_tasks:
task = task_list.pop()
timer.start("prog-%d" % pcounter)
p, prog = execute_single_task(task)
timer.update_extra("prog-%d" % pcounter, "step: %d, pid: %d, prog: %s" \
% (args.server.superstep_count, p.pid, prog))
counter_map[p.pid] = pcounter
pcounter += 1
procs.append(p)
for p in procs:
# wait for the process to complete
pid = p.pid
logging.debug("polling %d" % pid)
status = p.poll()
if status is not None:
timer.stop("prog-%d" % counter_map[p.pid])
del counter_map[p.pid]
logging.debug("finished %d with status %s" % (pid, str(status)))
# error reporting
if status <> 0:
msg = "Pid %d terminated unexpectedly with status %d" % (pid, status)
logging.error(msg)
client.error(args.master, args.id, msg)
procs.remove(p)
if not procs and not task_list:
break
else:
time.sleep(0.1)
overall_timer.stop("superstep-%d-%s" % \
(args.server.superstep_count, task_name))
# send done to master
client.done(args.master, args.id)
def resample_msg(h5files,
gribfiles,
outdir='./'): #,accstep=1,acc = True, start = 0):
'''Finds SAF MSG data in a hdf5 file,
resamples it to fc grid and puts it in a grib message '''
if h5files == [] or gribfiles == []:
print 'resample_msg: Files not found'
return 0
import find_date, gribtools, config
# make dir to output files
config.mkdir_p(outdir)
# get necessary lats,lons ONCE:
glats, glons = gribtools.get_latlon(gribfiles[0])
rlats, rlons = init_MSG_h5data(h5files[0])
def tofile(gfile, date, data, param=71, level=0):
'''take grib file and hdf5 data, resample and insert into new grib file'''
resampled_data = resample_radar(data, rlats, rlons, glats, glons)
dtg = dt.datetime.strftime(date, '%Y%m%d%H')
newfile = os.path.join(outdir, 'msg_' + str(dtg) + '+000.grib')
gribtools.insert_grib(gfile,
newfile,
resampled_data,
parameter=param,
level=level,
force=True)
#gribtools.insert_grib(gfile, newfile, resampled_data,parameter = 0, level = level)
return newfile
msgfiles = set() #[]
for gfile in gribfiles:
alreadythere = 0 # check below if accumulated observations have been made already
dtg, ltg = find_date.datetest(gfile, pattern='fc')
gdate = dtg + ltg
print 'In resample_msg', os.path.split(gfile)[1], dtg, ltg, gdate
#check if corresponding file already exists!
resampled_files = glob.glob(os.path.join(outdir, 'msg*grib'))
for rf in resampled_files:
dtr, ltr = find_date.datetest(rf, pattern='fc')
if dtr == None:
print 'not a valid filename ', os.path.split(rf)[1]
alreadythere = 1
break
elif dtr + ltr == gdate:
print 'MSG file available', rf
alreadythere = 1
#radfiles.append(rf)
msgfiles.add(rf)
break
if not alreadythere: # do matching, resampling, write to file
for afile in h5files:
hdate = find_date.datetest(afile, pattern='hdf')[0]
if hdate == gdate: # unlike radar, just take instantaneous cloud mask
wa = get_h5data(afile, h5source={'model': 'MSG'})
# sometimes file is corrupt or something
if wa == None: break
msgfile = tofile(gfile, gdate, wa)
print msgfile
msgfiles.add(msgfile)
return msgfiles
def Execute(args):
args.flog.write("Execute " + str(args.active) + "\n")
tnow = time.time()
if len(args.active) > 0:
execdir = os.path.join(args.workdir, "snapw.%d/exec" % (args.pid))
config.mkdir_p(execdir)
def execute_single_task(task):
# get the executables
bunch = "%s" % (task.split("-",1)[0])
execlist = []
try:
execlist = args.config["bunch"][bunch]["exec"].split(",")
except:
pass
for item in execlist:
execpath = os.path.join(execdir, item)
# check if the program exists and its mtime
mtime = None
try:
stat = os.stat(execpath)
mtime = int(stat.st_mtime)
except:
pass
if not mtime or mtime < tnow:
# the file does not exist or it is older than current time,
# contact the head task
content = client.getexec(args.master,item,mtime)
swc = "None"
if content:
swc = str(len(content))
print "Host received %s" % (swc)
if content:
if len(content) > 0:
print "Host saving to %s" % (execpath)
f = open(execpath,"w")
f.write(content)
f.close()
os.utime(execpath,(tnow, tnow))
prog = execlist[0]
print "Task %s, exec %s" % (prog, execlist)
progpath = os.path.join(execdir, prog)
if not os.path.exists(progpath):
line = "*** Error: task %s not started, program %s not found\n" % (
task, progpath)
args.flog.write(line)
args.flog.flush()
return
taskdir = "snapw.%d/tasks/%s" % (args.pid, task)
config.mkdir_p(taskdir)
qdir = os.path.join(args.workdir, args.qactname, task)
tdir = os.path.join(args.workdir, taskdir)
line = "starting task %s, prog %s, workdir %s, qdir %s\n" % (
task, prog, tdir, qdir)
args.flog.write(line)
args.flog.flush()
# get server information
host = args.server.host
port = args.server.port
# construct a command line
cmd = python + " %s -t %s -h %s:%d -q %s" % (
progpath, task, host, port, qdir)
args.flog.write("starting cmd %s\n" % (cmd))
args.flog.flush()
# start the work process
p = pexec.Exec(tdir,cmd)
return p
# Dynamically check what the number of processors we have on each host
# In any error, default to 1.
try:
max_tasks = os.sysconf('SC_NPROCESSORS_ONLN')
except:
max_tasks = 1
args.flog.write("Running tasks with " + str(max_tasks) + "-way parallelism\n")
# execute the tasks in a parallel fashion by running
# at most max_tasks processes at any point.
task_list = args.active[:]
procs = []
while True:
while task_list and len(procs) < max_tasks:
task = task_list.pop()
procs.append(execute_single_task(task))
for p in procs:
# wait for the process to complete
pid = pexec.GetPid(p)
args.flog.write("polling " + str(pid) + "\n")
args.flog.flush()
status = pexec.Poll(p)
if status is not None:
args.flog.write("finished " + str(pid) + "\n")
args.flog.flush()
procs.remove(p)
if not procs and not task_list:
break
else:
time.sleep(1.0)
# send done to master
client.done(args.master, args.id)
def accumulate_radar(h5files,
gribfiles,
outdir='./',
accstep=1,
acc=True,
start=0):
'''Finds reflectivity(!) data in a hdf5 file,
resamples it to fc grid and puts it in a grib message '''
if h5files == [] or gribfiles == []:
print 'Accumulate radar: Files not found'
return 0
import find_date, gribtools, config
config.mkdir_p(outdir)
glats, glons = gribtools.get_latlon(gribfiles[0])
rlats, rlons = init_h5data(h5files[0])
def tofile(gfile, date, data, step, param=61, level=457):
resampled_data = resample_radar(data, rlats, rlons, glats, glons)
#print 'now resampled: ',resampled_data.max(), resampled_data.mean(), resampled_data.min()
dtg = dt.datetime.strftime(date, '%Y%m%d%H')
newfile = os.path.join(
outdir, 'rad_' + str(int(step)) + '_' + str(dtg) + '+000.grib')
gribtools.insert_grib(gfile,
newfile,
resampled_data,
parameter=param,
level=level)
#gribtools.insert_grib(gfile, newfile, resampled_data,parameter = 0, level = level)
return newfile
def accumulate(gfile, h5files):
w = None #?
for afile in h5files:
hdate = find_date.datetest(afile, pattern='hdf')[0]
print 'files:', os.path.split(gfile)[1], os.path.split(afile)[1]
print gdate, hdate, acctime.seconds / 3.6e3, gdate - acctime < hdate <= gdate
if gdate - acctime < hdate <= gdate: # obs time between fctime and an accumulation time before
wa = get_h5data(afile)
if acc:
try:
w = w + wa
nr += 1
except: # first
w = wa
nr = 1
print 'now: nr, max, mean, min: ', nr, w.max(), w.mean(
), w.min()
else: # for the RACfiles: 3h accumulation every hour available!
#print 'now tot: ',wa.max(), wa.min()
if gdate - dt.timedelta(hours=1) < hdate <= gdate:
accfile = tofile(gfile, gdate, wa,
acctime.seconds / 3.6e3)
#radfiles.append( accfile )
return accfile
else:
pass
elif hdate > gdate or hdate <= gdate - acctime: # time to write to file
if acc:
#try:
if w not in [
[], None
]: # check if data to write to file is not None
accfile = tofile(gfile, gdate, w / nr,
acctime.seconds / 3.6e3)
#reset: (obsolete?)
w = []
nr = 0
#radfiles.append( accfile )
return accfile
else: #except:
pass
#pass
else: # do nothing
pass
else: # date strange
print 'strange, obdate, fcdate: ', hdate, gdate
wa = get_h5data(afile)
accfile = tofile(gfile, gdate, wa, acctime.seconds / 3.6e3)
return accfile
#last bit, check...
if acc:
try:
if w not in [[], None]:
print 'eh ', afile, gfile, hdate, gdate
accfile = tofile(gfile, gdate, w / nr,
int(acctime.seconds / 3.6e3))
return accfile
except:
print 'Not in accumulated file: ', gfile
print('strange, obdate, fcdate: ', hdate, gdate)
return None #accfile #(Done?)
acctime = dt.timedelta(hours=accstep)
dtg0, lt0 = find_date.datetest(gribfiles[0], pattern='fc')
dtg1, lt1 = find_date.datetest(gribfiles[1], pattern='fc')
deltime = (dtg1 + lt1) - (dtg0 + lt0)
if deltime > acctime: acctime = deltime
#sdate = find_date.datetest(h5files[0],pattern='hdf')[0] # start at first file
#sdate = dt.datetime(1900,1,1,12,0) # arbitrary date in far history
radfiles = set() #[]
for gfile in sorted(gribfiles):
alreadythere = 0 # check below if accumulated observations have been made already
dtg, ltg = find_date.datetest(gfile, pattern='fc')
gdate = dtg + ltg
#print 'In accumulate radar',os.path.split(gfile)[1], dtg,ltg,gdate
#check if corresponding file already exists!
resampled_files = glob.glob(os.path.join(outdir, 'rad*grib'))
for rf in sorted(resampled_files):
dtr, ltr = find_date.datetest(rf, pattern='fc')
if dtr == None:
print 'not a valid filename ', os.path.split(rf)[1]
alreadythere = 1
break
elif dtr + ltr == gdate:
#print 'radar file available'
alreadythere = 1
radfiles.add(rf)
break
if not alreadythere:
accfile = accumulate(gfile, h5files)
print accfile
radfiles.add(accfile)
return radfiles
def execute_single_task(task):
# get the executables
bunch = "%s" % (task.split("-", 1)[0])
execlist = []
try:
execlist = args.config["bunch"][bunch]["exec"].split(",")
except:
pass
for item in execlist:
execpath = os.path.join(execdir, item)
# check if the program exists and its mtime
mtime = None
try:
stat = os.stat(execpath)
mtime = int(stat.st_mtime)
except:
pass
if not mtime or mtime < tnow:
# the file does not exist or it is older than current time,
# contact the head task
content = client.getexec(args.master, item, mtime)
swc = "None"
if content:
swc = str(len(content))
print "Host received %s" % (swc)
if content:
if len(content) > 0:
print "Host saving to %s" % (execpath)
f = open(execpath, "w")
f.write(content)
f.close()
os.utime(execpath, (tnow, tnow))
prog = execlist[0]
print "Task %s, exec %s" % (prog, execlist)
progpath = os.path.join(execdir, prog)
if not os.path.exists(progpath):
line = "*** Error: task %s not started, program %s not found\n" % (
task, progpath)
args.flog.write(line)
args.flog.flush()
return
taskdir = "snapw.%d/tasks/%s" % (args.pid, task)
config.mkdir_p(taskdir)
qdir = os.path.join(args.workdir, args.qactname, task)
tdir = os.path.join(args.workdir, taskdir)
line = "starting task %s, prog %s, workdir %s, qdir %s\n" % (
task, prog, tdir, qdir)
args.flog.write(line)
args.flog.flush()
# get server information
host = args.server.host
port = args.server.port
# construct a command line
cmd = python + " %s -t %s -h %s:%d -q %s" % (progpath, task, host,
port, qdir)
args.flog.write("starting cmd %s\n" % (cmd))
args.flog.flush()
# start the work process
p = pexec.Exec(tdir, cmd)
return p
def Execute(args):
args.flog.write("Execute " + str(args.active) + "\n")
tnow = time.time()
if len(args.active) > 0:
execdir = os.path.join(args.workdir, "snapw.%d/exec" % (args.pid))
config.mkdir_p(execdir)
def execute_single_task(task):
# get the executables
bunch = "%s" % (task.split("-", 1)[0])
execlist = []
try:
execlist = args.config["bunch"][bunch]["exec"].split(",")
except:
pass
for item in execlist:
execpath = os.path.join(execdir, item)
# check if the program exists and its mtime
mtime = None
try:
stat = os.stat(execpath)
mtime = int(stat.st_mtime)
except:
pass
if not mtime or mtime < tnow:
# the file does not exist or it is older than current time,
# contact the head task
content = client.getexec(args.master, item, mtime)
swc = "None"
if content:
swc = str(len(content))
print "Host received %s" % (swc)
if content:
if len(content) > 0:
print "Host saving to %s" % (execpath)
f = open(execpath, "w")
f.write(content)
f.close()
os.utime(execpath, (tnow, tnow))
prog = execlist[0]
print "Task %s, exec %s" % (prog, execlist)
progpath = os.path.join(execdir, prog)
if not os.path.exists(progpath):
line = "*** Error: task %s not started, program %s not found\n" % (
task, progpath)
args.flog.write(line)
args.flog.flush()
return
taskdir = "snapw.%d/tasks/%s" % (args.pid, task)
config.mkdir_p(taskdir)
qdir = os.path.join(args.workdir, args.qactname, task)
tdir = os.path.join(args.workdir, taskdir)
line = "starting task %s, prog %s, workdir %s, qdir %s\n" % (
task, prog, tdir, qdir)
args.flog.write(line)
args.flog.flush()
# get server information
host = args.server.host
port = args.server.port
# construct a command line
cmd = python + " %s -t %s -h %s:%d -q %s" % (progpath, task, host,
port, qdir)
args.flog.write("starting cmd %s\n" % (cmd))
args.flog.flush()
# start the work process
p = pexec.Exec(tdir, cmd)
return p
# Dynamically check what the number of processors we have on each host
# In any error, default to 1.
try:
max_tasks = os.sysconf('SC_NPROCESSORS_ONLN')
except:
max_tasks = 1
args.flog.write("Running tasks with " + str(max_tasks) +
"-way parallelism\n")
# execute the tasks in a parallel fashion by running
# at most max_tasks processes at any point.
task_list = args.active[:]
procs = []
while True:
while task_list and len(procs) < max_tasks:
task = task_list.pop()
procs.append(execute_single_task(task))
for p in procs:
# wait for the process to complete
pid = pexec.GetPid(p)
args.flog.write("polling " + str(pid) + "\n")
args.flog.flush()
status = pexec.Poll(p)
if status is not None:
args.flog.write("finished " + str(pid) + "\n")
args.flog.flush()
procs.remove(p)
if not procs and not task_list:
break
else:
time.sleep(1.0)
# send done to master
client.done(args.master, args.id)
def do_POST(self):
#print "POST path", self.path
parsed_path = urlparse.urlparse(self.path)
message_parts = [
'CLIENT VALUES:',
'client_address=%s (%s)' % (self.client_address,
self.address_string()),
'command=%s' % self.command,
'path=%s' % self.path,
'real path=%s' % parsed_path.path,
'query=%s' % parsed_path.query,
'request_version=%s' % self.request_version,
'',
'SERVER VALUES:',
'server_type=%s' % "host server",
'server_version=%s' % self.server_version,
'sys_version=%s' % self.sys_version,
'protocol_version=%s' % self.protocol_version,
'',
'HEADERS RECEIVED:',
]
for name, value in sorted(self.headers.items()):
message_parts.append('%s=%s' % (name, value.rstrip()))
message_parts.append('')
message = '\r\n'.join(message_parts)
length = int(self.headers.get("Content-Length"))
body = ""
subpath = self.path.split("/")
if subpath[1] == "msg":
dst = subpath[2]
src = subpath[3]
dirname = "snapw.%d/qin/%s" % (self.pid, dst)
config.mkdir_p(dirname)
fname = "%s/%s" % (dirname, src)
f, fnew = config.uniquefile(fname)
if fname <> fnew:
logging.warn("uniquefile?: %s; %s; %s; %s;" % (dirname, src, fname, fnew))
length = int(self.headers.get("Content-Length"))
#print "Content-Length", length
try:
nleft = length
while nleft > 0:
nread = min(102400, nleft)
body = self.rfile.read(nread)
f.write(body)
nleft -= len(body)
except Exception as e:
logging.warn("file stream error: %s" % str(e))
try:
f.close()
os.remove(fnew)
except:
pass
try:
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
except:
pass
return
f.close()
logging.info("message %s length %d" % (fnew, length))
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
return
length = int(self.headers.get("Content-Length"))
#print "Content-Length", length
body = ""
if length and length > 0:
body = self.rfile.read(length)
# Begin the response
self.send_response(200)
self.end_headers()
self.wfile.write('Client: %s\n' % str(self.client_address))
self.wfile.write('User-agent: %s\n' % str(self.headers['user-agent']))
self.wfile.write('Path: %s\n' % self.path)
def do_POST(self):
#print "POST path", self.path
parsed_path = urlparse.urlparse(self.path)
message_parts = [
'CLIENT VALUES:',
'client_address=%s (%s)' % (self.client_address,
self.address_string()),
'command=%s' % self.command,
'path=%s' % self.path,
'real path=%s' % parsed_path.path,
'query=%s' % parsed_path.query,
'request_version=%s' % self.request_version,
'',
'SERVER VALUES:',
'server_type=%s' % "host server",
'server_version=%s' % self.server_version,
'sys_version=%s' % self.sys_version,
'protocol_version=%s' % self.protocol_version,
'',
'HEADERS RECEIVED:',
]
for name, value in sorted(self.headers.items()):
message_parts.append('%s=%s' % (name, value.rstrip()))
message_parts.append('')
message = '\r\n'.join(message_parts)
#print message
length = int(self.headers.get("Content-Length"))
#print "Content-Length", length
body = ""
if length and length > 0:
body = self.rfile.read(length)
#print "length", length
#print "body"
#print body
subpath = self.path.split("/")
#print subpath
if subpath[1] == "msg":
dst = subpath[2]
src = subpath[3]
#print "msg", dst, src
#print "body", body
dirname = "snapw.%d/qin/%s" % (self.pid, dst)
config.mkdir_p(dirname)
fname = "%s/%s" % (dirname, src)
f,fnew = config.uniquefile(fname)
f.write(body)
f.close()
line = "message %s length %d\n" % (fnew, length)
self.flog.write(line)
self.flog.flush()
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
return
# Begin the response
self.send_response(200)
self.end_headers()
self.wfile.write('Client: %s\n' % str(self.client_address))
self.wfile.write('User-agent: %s\n' % str(self.headers['user-agent']))
self.wfile.write('Path: %s\n' % self.path)
def do_GET(self):
parsed_path = urlparse.urlparse(self.path)
message_parts = [
'CLIENT VALUES:',
'client_address=%s (%s)' % (self.client_address,
self.address_string()),
'command=%s' % self.command,
'path=%s' % self.path,
'real path=%s' % parsed_path.path,
'query=%s' % parsed_path.query,
'request_version=%s' % self.request_version,
'',
'SERVER VALUES:',
'server_type=%s' % "host server",
'server_version=%s' % self.server_version,
'sys_version=%s' % self.sys_version,
'protocol_version=%s' % self.protocol_version,
'',
'HEADERS RECEIVED:',
]
for name, value in sorted(self.headers.items()):
message_parts.append('%s=%s' % (name, value.rstrip()))
message_parts.append('')
message = '\r\n'.join(message_parts)
subpath = self.path.split("/")
if self.path == "/prepare":
prepare_timer = perf.Timer(logging)
self.server.superstep_count += 1
prepare_timer.start("prepare-%d" % self.server.superstep_count)
# move qin to qact
qinname = "snapw.%d/qin" % (self.pid)
qactname = "snapw.%d/qact" % (self.pid)
# rename an existing qact
if os.path.exists(qactname):
removed = False
if not self.config['debug']:
try:
shutil.rmtree(qactname)
logging.debug("removed dir %s" % qactname)
removed = True
except:
logging.error("error on removing dir %s" % qactname)
if not removed:
t = time.time()
s = time.strftime("%Y%m%d-%H%M%S", time.localtime(t))
mus = "%06d" % (t*1000000 - int(t)*1000000)
qactnewname = "%s-%s-%s" % (qactname, s, mus)
os.rename(qactname, qactnewname)
logging.debug("renamed %s to %s" % (qactname, qactnewname))
# get the number of active tasks, rename existing qin
numtasks = 0
if os.path.exists(qinname):
os.rename(qinname, qactname)
active = os.listdir(qactname)
numtasks = len(active)
# create new qin
config.mkdir_p(qinname)
logging.info("preparing next step: %s, %s" % \
(qinname, qactname))
# send ready to master
client.ready(self.master, self.id, numtasks)
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
prepare_timer.stop("prepare-%d" % self.server.superstep_count)
return
elif self.path == "/quit":
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
SYS_STATS = False
# set the flag to terminate the server
self.server.running = False
self.server.self_dummy()
return
elif self.path == "/getkv":
logging.debug("getting kv file")
self.send_response(200)
if self.server.superstep_count > 1:
body = json.dumps(get_kv_file("supervisor"))
self.send_header('Content-Length', len(body))
self.end_headers()
self.wfile.write(body)
else:
self.send_header('Content-Length', len("None"))
self.end_headers()
self.wfile.write("None")
return
elif self.path == "/dummy":
logging.debug("dummy request")
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
return
elif self.path == "/step":
logging.info("execute next step")
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
# TODO, implement null action,
# skip execution if there are no tasks to execute,
# qact does not exist
# get the tasks to execute
qactname = "snapw.%d/qact" % (self.pid)
active = []
if os.path.exists(qactname):
active = os.listdir(qactname)
logging.debug("active tasks %s" % (str(active)))
self.qactname = qactname
self.active = active # the task list
# start a thread to execute the work tasks
t = threading.Thread(target=Execute, args=(self, ))
t.start()
return
elif self.path == "/config":
logging.debug("get configuration")
body = json.dumps(self.config)
self.send_response(200)
self.send_header('Content-Length', len(body))
self.end_headers()
self.wfile.write(body)
return
elif self.path == "/quit":
logging.info("terminate execution")
SYS_STATS = False
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
sys.exit(0)
self.send_response(200)
self.end_headers()
self.wfile.write(message)
return
def do_GET(self):
#print "GET path", self.path
parsed_path = urlparse.urlparse(self.path)
message_parts = [
'CLIENT VALUES:',
'client_address=%s (%s)' % (self.client_address,
self.address_string()),
'command=%s' % self.command,
'path=%s' % self.path,
'real path=%s' % parsed_path.path,
'query=%s' % parsed_path.query,
'request_version=%s' % self.request_version,
'',
'SERVER VALUES:',
'server_type=%s' % "host server",
'server_version=%s' % self.server_version,
'sys_version=%s' % self.sys_version,
'protocol_version=%s' % self.protocol_version,
'',
'HEADERS RECEIVED:',
]
for name, value in sorted(self.headers.items()):
message_parts.append('%s=%s' % (name, value.rstrip()))
message_parts.append('')
message = '\r\n'.join(message_parts)
#print message
#self.flog.write(message)
#self.flog.flush()
subpath = self.path.split("/")
#print subpath
if self.path == "/prepare":
# move qin to qact
qinname = "snapw.%d/qin" % (self.pid)
qactname = "snapw.%d/qact" % (self.pid)
# rename an existing qact
qactnewname = "none"
if os.path.exists(qactname):
t = time.time()
s = time.strftime("%Y%m%d-%H%M%S", time.localtime(t))
mus = "%06d" % (t*1000000 - int(t)*1000000)
qactnewname = "%s-%s-%s" % (qactname, s, mus)
os.rename(qactname, qactnewname)
# get the number of active tasks, rename existing qin
numtasks = 0
if os.path.exists(qinname):
os.rename(qinname, qactname)
active = os.listdir(qactname)
numtasks = len(active)
# create new qin
config.mkdir_p(qinname)
line = "preparing next step: %s, %s, %s\n" % (
qinname, qactname, qactnewname)
self.flog.write(line)
self.flog.flush()
# send ready to master
client.ready(self.master, self.id, numtasks)
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
return
elif self.path == "/quit":
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
# set the flag to terminate the server
self.server.running = False
self.server.self_dummy()
return
elif self.path == "/dummy":
print "dummy request"
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
return
elif self.path == "/step":
line = "execute next step\n"
self.flog.write(line)
self.flog.flush()
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
# TODO, implement null action,
# skip execution if there are no tasks to execute,
# qact does not exist
# get the tasks to execute
qactname = "snapw.%d/qact" % (self.pid)
active = []
if os.path.exists(qactname):
active = os.listdir(qactname)
line = "active tasks %s\n" % (str(active))
self.flog.write(line)
self.flog.flush()
self.qactname = qactname
self.active = active
# start a thread to execute the work tasks
t = threading.Thread(target=Execute, args=(self, ))
t.start()
return
elif self.path == "/config":
line = "get configuration\n"
self.flog.write(line)
self.flog.flush()
body = simplejson.dumps(self.config)
self.send_response(200)
self.send_header('Content-Length', len(body))
self.end_headers()
self.wfile.write(body)
return
elif self.path == "/quit":
line = "terminate execution\n"
self.flog.write(line)
self.flog.flush()
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
sys.exit(0)
self.send_response(200)
#self.send_header('Last-Modified', self.date_time_string(time.time()))
self.end_headers()
self.wfile.write(message)
return
def execute_single_task(task):
# get the executables
bunch = get_task_name(task)
execlist = []
try:
execlist = args.config["bunch"][bunch]["exec"].split(",")
except:
pass
timer = perf.Timer(logging)
timer.start('provision-execlist')
# provision execlist on disk
for item in execlist:
execpath = os.path.join(execdir, item)
# check if the program exists and its mtime
mtime = None
try:
stat = os.stat(execpath)
mtime = int(stat.st_mtime)
except:
pass
if not mtime or mtime < tnow:
# if the file does not exist or it is older than current time,
# contact the head task
content = client.getexec(args.master,item,mtime)
swc = "None"
if content:
swc = str(len(content))
logging.debug("Host received %s" % (swc))
if content:
if len(content) > 0:
logging.debug("Host saving to %s" % (execpath))
f = open(execpath,"w")
f.write(content)
f.close()
os.utime(execpath,(tnow, tnow))
timer.stop('provision-execlist')
prog = execlist[0]
logging.debug("Task %s, exec %s" % (prog, execlist))
progpath = os.path.join(execdir, prog)
if not os.path.exists(progpath):
logging.error("task %s not started, program %s not found" % (task, progpath))
return
taskdir = "snapw.%d/tasks/%s" % (args.pid, task)
config.mkdir_p(taskdir)
qdir = os.path.join(args.workdir, args.qactname, task)
tdir = os.path.join(args.workdir, taskdir)
logging.info("starting task %s, prog %s, workdir %s, qdir %s\n" % (task, prog, tdir, qdir))
# get server information
host = args.server.host
port = args.server.port
# construct a command line for worker
cmd = python + " %s -t %s -h %s:%d -q %s" % (
progpath, task, host, port, qdir)
logging.info("starting cmd %s" % (cmd))
# start the work process
p = subprocess.Popen(cmd.split(), cwd=tdir, close_fds=True)
return p, prog
def do_POST(self):
#print "POST path", self.path
parsed_path = urlparse.urlparse(self.path)
message_parts = [
'CLIENT VALUES:',
'client_address=%s (%s)' %
(self.client_address, self.address_string()),
'command=%s' % self.command,
'path=%s' % self.path,
'real path=%s' % parsed_path.path,
'query=%s' % parsed_path.query,
'request_version=%s' % self.request_version,
'',
'SERVER VALUES:',
'server_type=%s' % "host server",
'server_version=%s' % self.server_version,
'sys_version=%s' % self.sys_version,
'protocol_version=%s' % self.protocol_version,
'',
'HEADERS RECEIVED:',
]
for name, value in sorted(self.headers.items()):
message_parts.append('%s=%s' % (name, value.rstrip()))
message_parts.append('')
message = '\r\n'.join(message_parts)
#print message
#print "length", length
#print "body"
#print body
subpath = self.path.split("/")
#print subpath
if subpath[1] == "msg":
dst = subpath[2]
src = subpath[3]
#print "msg", dst, src
#print "body", body
dirname = "snapw.%d/qin/%s" % (self.pid, dst)
config.mkdir_p(dirname)
fname = "%s/%s" % (dirname, src)
f, fnew = config.uniquefile(fname)
length = int(self.headers.get("Content-Length"))
#print "Content-Length", length
nleft = length
while nleft > 0:
nread = 102400
if nleft < nread:
nread = nleft
body = self.rfile.read(nread)
f.write(body)
nleft -= nread
f.close()
line = "message %s length %d\n" % (fnew, length)
self.flog.write(line)
self.flog.flush()
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
return
length = int(self.headers.get("Content-Length"))
#print "Content-Length", length
body = ""
if length and length > 0:
body = self.rfile.read(length)
# Begin the response
self.send_response(200)
self.end_headers()
self.wfile.write('Client: %s\n' % str(self.client_address))
self.wfile.write('User-agent: %s\n' % str(self.headers['user-agent']))
self.wfile.write('Path: %s\n' % self.path)
