def GetConfig(self):
# get configuration from the host
sconf = None
if self.configfile:
try:
f = open(self.configfile)
sconf = f.read()
f.close()
except:
pass
elif self.host:
sconf = client.config(self.host)
if not sconf:
print "*** no configuration"
return None
self.config = simplejson.loads(sconf)
self.var = self.config.get("var")
self.route = self.config.get("route")
self.tasks = self.config.get("tasks")
dbunch = self.config.get("bunch")
hostlist = self.config.get("hosts")
if dbunch:
dinfo = dbunch.get(self.name)
if dinfo:
self.range = int(dinfo.get("range"))
if self.route:
for key, routes in self.route.iteritems():
dest = routes.get(self.name)
if dest:
if not self.target:
self.target = {}
self.target[key] = dest
if (self.var == None or self.route == None or hostlist == None
or self.tasks == None or self.range == None
or self.target == None):
return None
self.hosts = {}
for h in hostlist:
self.hosts[h["id"]] = h["host"] + ":" + h["port"]
self.flog.write(str(self.config))
self.flog.write("\n")
self.flog.write("Got configuration size %d\n" %
(len(str(self.config))))
self.flog.flush()
return self.config
def GetConfig(self):
# get configuration from the host
sconf = None
if self.configfile:
try:
f = open(self.configfile)
sconf = f.read()
f.close()
except:
pass
elif self.host:
sconf = client.config(self.host)
if not sconf:
print "*** no configuration"
return None
self.config = simplejson.loads(sconf)
self.var = self.config.get("var")
self.route = self.config.get("route")
self.tasks = self.config.get("tasks")
dbunch = self.config.get("bunch")
hostlist = self.config.get("hosts")
if dbunch:
dinfo = dbunch.get(self.name)
if dinfo:
self.range = int(dinfo.get("range"))
if self.route:
for key, routes in self.route.iteritems():
dest = routes.get(self.name)
if dest:
if not self.target:
self.target = {}
self.target[key] = dest
if (self.var == None or self.route == None or
hostlist == None or self.tasks == None or
self.range == None or self.target == None):
return None
self.hosts = {}
for h in hostlist:
self.hosts[h["id"]] = h["host"] + ":" + h["port"]
self.flog.write(str(self.config))
self.flog.write("\n")
self.flog.write("Got configuration size %d\n" % (len(str(self.config))))
self.flog.flush()
return self.config
server.running = True
# no locking required...
server.superstep_count = 0
handler = BaseHTTPServer.BaseHTTPRequestHandler
handler.port = port
handler.id = id
handler.pid = os.getpid()
handler.workdir = workdir
handler.master = master
if master != None:
SYS_STATS = True
# get configuration from master
sconf = client.config(master)
dconf = json.loads(sconf)
handler.config = dconf
logging.debug("Config size: %d" % (len(sconf)))
logging.debug(str(dconf))
# send done to master
client.done(master, id)
# NOTE: Time conflict. The master might already send 'step' request,
# before server_forever() is started, so 'step' might be lost.
# Delay done until the server is up and running.
# Check out Asynchronous Mixins example for SocketServer
# Comment: the constructor might already activate the server,
server.running = True
# no locking required...
server.superstep_count = 0
handler = BaseHTTPServer.BaseHTTPRequestHandler
handler.port = port
handler.id = id
handler.pid = os.getpid()
handler.workdir = workdir
handler.master = master
if master != None:
# SYS_STATS = True
# get configuration from master
sconf = client.config(master)
dconf = json.loads(sconf)
handler.config = dconf
logging.debug("Config size: %d" % (len(sconf)))
logging.debug(str(dconf))
# send done to master
client.done(master, id)
# NOTE: Time conflict. The master might already send 'step' request,
# before server_forever() is started, so 'step' might be lost.
# Delay done until the server is up and running.
# Check out Asynchronous Mixins example for SocketServer
# Comment: the constructor might already activate the server,
def GetConfig(self):
# get configuration from the host
sconf = None
if self.configfile:
try:
f = open(self.configfile)
sconf = f.read()
f.close()
except:
pass
elif self.host:
sconf = client.config(self.host)
if not sconf:
print "*** no configuration"
return None
self.config = json.loads(sconf)
self.var = self.config.get("var")
self.route = self.config.get("route")
self.tasks = self.config.get("tasks")
dbunch = self.config.get("bunch")
hostlist = self.config.get("hosts")
if dbunch:
dinfo = dbunch.get(self.name)
if dinfo:
self.range = int(dinfo.get("range"))
seg_bits = self.var.get('seg_bits')
if seg_bits is not None:
seg_bits = int(seg_bits)
if seg_bits <= 0:
print 'Error! for node-segmented BFS, the number of segment bits must be positive!'
sys.exit(1)
if (1<<seg_bits) % self.range != 0:
print 'Error! for node-segmented BFS, the segment size must be a multiple of the range!'
sys.exit(1)
if self.route:
for key, routes in self.route.iteritems():
dest = routes.get(self.name)
if dest:
if not self.target:
self.target = {}
self.target[key] = dest
if (self.var == None or self.route == None or
hostlist == None or self.tasks == None or
self.range == None or self.target == None):
return None
self.hosts = {}
for h in hostlist:
self.hosts[h["id"]] = h["host"] + ":" + h["port"]
self.log.debug("Configuration: %s" % str(self.config))
self.log.debug("config size %d" % (len(str(self.config))))
return self.config
def GetConfig(self):
# get configuration from the host
sconf = None
if self.configfile:
try:
f = open(self.configfile)
sconf = f.read()
f.close()
except:
pass
elif self.host:
sconf = client.config(self.host)
if not sconf:
print "*** no configuration"
return None
self.config = json.loads(sconf)
self.var = self.config.get("var")
self.route = self.config.get("route")
self.tasks = self.config.get("tasks")
dbunch = self.config.get("bunch")
hostlist = self.config.get("hosts")
if dbunch:
dinfo = dbunch.get(self.name)
if dinfo:
self.range = int(dinfo.get("range"))
seg_bits = self.var.get('seg_bits')
if seg_bits is not None:
seg_bits = int(seg_bits)
if seg_bits <= 0:
print 'Error! for node-segmented BFS, the number of segment bits must be positive!'
sys.exit(1)
if (1 << seg_bits) % self.range != 0:
print 'Error! for node-segmented BFS, the segment size must be a multiple of the range!'
sys.exit(1)
if self.route:
for key, routes in self.route.iteritems():
dest = routes.get(self.name)
if dest:
if not self.target:
self.target = {}
self.target[key] = dest
if (self.var == None or self.route == None or hostlist == None
or self.tasks == None or self.range == None
or self.target == None):
return None
self.hosts = {}
for h in hostlist:
self.hosts[h["id"]] = h["host"] + ":" + h["port"]
self.log.debug("Configuration: %s" % str(self.config))
self.log.debug("config size %d" % (len(str(self.config))))
return self.config
def do_GET(self):
parsed_path = urlparse.urlparse(self.path)
message_parts = [
'CLIENT VALUES:',
'client_address=%s (%s)' % (self.client_address,
self.address_string()),
'command=%s' % self.command,
'path=%s' % self.path,
'real path=%s' % parsed_path.path,
'query=%s' % parsed_path.query,
'request_version=%s' % self.request_version,
'',
'SERVER VALUES:',
'server_type=%s' % "host server",
'server_version=%s' % self.server_version,
'sys_version=%s' % self.sys_version,
'protocol_version=%s' % self.protocol_version,
'',
'HEADERS RECEIVED:',
]
for name, value in sorted(self.headers.items()):
message_parts.append('%s=%s' % (name, value.rstrip()))
message_parts.append('')
message = '\r\n'.join(message_parts)
subpath = self.path.split("/")
if self.path == "/prepare":
# move qin to qact
qinname = "snapw.%d/qin" % (self.pid)
qactname = "snapw.%d/qact" % (self.pid)
# rename an existing qact
qactnewname = "none"
if os.path.exists(qactname):
t = time.time()
s = time.strftime("%Y%m%d-%H%M%S", time.localtime(t))
mus = "%06d" % (t*1000000 - int(t)*1000000)
qactnewname = "%s-%s-%s" % (qactname, s, mus)
os.rename(qactname, qactnewname)
# get the number of active tasks, rename existing qin
numtasks = 0
if os.path.exists(qinname):
os.rename(qinname, qactname)
active = os.listdir(qactname)
numtasks = len(active)
# create new qin
config.mkdir_p(qinname)
line = "preparing next step: %s, %s, %s\n" % (
qinname, qactname, qactnewname)
self.flog.write(line)
self.flog.flush()
# initially just get the whole config all over again -- Extremely inefficient
sconf = client.config(master)
dconf = simplejson.loads(sconf)
handler.config = dconf
flog.write("Got configuration size %d\n" % (len(sconf)))
flog.write(str(dconf))
flog.write("\n")
flog.flush()
# send ready to master
client.ready(self.master, self.id, numtasks)
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
return
elif self.path == "/quit":
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
# set the flag to terminate the server
self.server.running = False
self.server.self_dummy()
return
elif self.path == "/dummy":
print "dummy request"
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
return
elif self.path == "/step":
line = "execute next step\n"
self.flog.write(line)
self.flog.flush()
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
# TODO, implement null action,
# skip execution if there are no tasks to execute,
# qact does not exist
# get the tasks to execute
qactname = "snapw.%d/qact" % (self.pid)
active = []
if os.path.exists(qactname):
active = os.listdir(qactname)
line = "active tasks %s\n" % (str(active))
self.flog.write(line)
self.flog.flush()
self.qactname = qactname
self.active = active
# start a thread to execute the work tasks
t = threading.Thread(target=Execute, args=(self, ))
t.start()
return
elif self.path == "/config":
line = "get configuration\n"
self.flog.write(line)
self.flog.flush()
body = simplejson.dumps(self.config)
self.send_response(200)
self.send_header('Content-Length', len(body))
self.end_headers()
self.wfile.write(body)
return
elif self.path == "/quit":
line = "terminate execution\n"
self.flog.write(line)
self.flog.flush()
self.send_response(200)
self.send_header('Content-Length', 0)
self.end_headers()
sys.exit(0)
self.send_response(200)
#self.send_header('Last-Modified', self.date_time_string(time.time()))
self.end_headers()
self.wfile.write(message)
return
