def _publisher_thread(self, pub_addrs, pull_address, push_addr):
#FIXME aaaaahhh pub_addresses are set here, not in the main thread
# (which reads them in _register method)
pub_sock, self.pub_addresses = self._init_socket(
pub_addrs, zmq.PUB)
pull_sock = self.ctx.socket(zmq.PULL)
pull_sock.bind(pull_address)
push_sock = self.ctx.socket(zmq.PUSH)
push_sock.connect(push_addr)
send_msg(push_sock, u'1')
po = PollingObject()
while not self._stop_publishing:
try:
to_publish, det = po.poll_recv(pull_sock, 500)
if to_publish:
send_msg(pub_sock, to_publish)
except:
#print self.name, '.Publisher -- STOP.'
break
# self.logger.info( "close sock %s %s", pub_addrs, pub_sock)
pub_sock.close()
pull_sock.close()
push_sock.close()
def handle(self):
print "SERVER REQUEST", self.__class__, "ME: ", self.request.getsockname(
)
print "FROM :", self.request.getpeername()
message = recv_unicode_netstring(self.rfile)
print message
pl = PollingObject()
parsed = self.server.mtool.unpack_msg(message)
if parsed.type == 'find_eeg_experiments' or parsed.type == 'find_eeg_amplifiers':
pull_addr = 'tcp://' + socket.gethostname() + ':' + str(
self.server.pull_port)
parsed.client_push_address = pull_addr
srv_sock = self.make_srv_sock()
try:
send_msg(srv_sock, parsed.SerializeToString())
response, det = pl.poll_recv(srv_sock, timeout=5000)
finally:
srv_sock.close()
print "passed msg and got result: ", response
if not response:
self.bad_response(self.wfile, det)
return
if parsed.type == 'find_eeg_experiments' or parsed.type == 'find_eeg_amplifiers':
response, det = pl.poll_recv(self.server.pull_sock, timeout=20000)
if not response:
self.bad_response(self.wfile, det)
return
data = make_unicode_netstring(response)
self.wfile.write(data)
def start_eeg_signal_experiment(ctx, srv_addrs, rq_message):
client = OBCIClient(srv_addrs, ctx)
# server_req_socket = ctx.socket(zmq.REQ)
# for addr in srv_addrs:
# server_req_socket.connect(addr)
amp_params = {}
amp_params.update(rq_message.amplifier_params['additional_params'])
del rq_message.amplifier_params['additional_params']
amp_params.update(rq_message.amplifier_params)
par_list = ['--peer', 'amplifier']
for par, val in amp_params.iteritems():
par_list += ['-p', par, unicode(val)]
overwrites = peer_cmd.peer_overwrites_pack(par_list)
result = client.launch(rq_message.launch_file, None, rq_message.name, overwrites)
LOGGER.info("START EEG signal! return to: " + rq_message.client_push_address)
mtool = OBCIMessageTool(message_templates)
to_client = ctx.socket(zmq.PUSH)
to_client.connect(rq_message.client_push_address)
if result is None:
send_msg(to_client, mtool.fill_msg("rq_error", err_code="launch_failed",
details="No response from server or experiment"))
else:
send_msg(to_client, result.raw())
LOGGER.info("sent eeg launch result" + str(result)[:500] )
time.sleep(0.1)
def _launch_process(self, path, args, proc_type, name, env=None, capture_io=NO_STDIO):
proc, details = self.subprocess_mgr.new_local_process(
path, args, proc_type=proc_type, name=name, monitoring_optflags=RETURNCODE, capture_io=capture_io, env=env
)
if proc is None:
print self.name, "[", self.type, "]", "process launch FAILED:", path, args
send_msg(
self._publish_socket,
self.mtool.fill_msg(
"launch_error",
sender=self.uuid,
details=dict(machine=self.machine, path=path, args=args, error=details),
),
)
else:
print self.name, "[", self.type, "]", "process launch success:", path, args, proc.pid
send_msg(
self._publish_socket,
self.mtool.fill_msg(
"launched_process_info",
sender=self.uuid,
machine=self.machine,
pid=proc.pid,
proc_type=proc_type,
name=name,
path=path,
args=args,
),
)
return proc, details
def _handle_match_name(self, message, sock, this_machine=False):
matches = self.exp_matching(message.strname)
match = None
msg = None
if not matches:
msg = self.mtool.fill_msg("rq_error",
request=vars(message),
err_code='experiment_not_found')
elif len(matches) > 1:
matches = [(exp.uuid, exp.name) for exp in matches]
msg = self.mtool.fill_msg("rq_error",
request=vars(message),
err_code='ambiguous_exp_name',
details=matches)
else:
match = matches.pop()
if this_machine and match.origin_machine != self.machine:
msg = self.mtool.fill_msg("rq_error",
request=vars(message),
err_code='exp_not_on_this_machine',
details=match.origin_machine)
match = None
if msg and sock.socket_type in [zmq.REP, zmq.ROUTER]:
send_msg(sock, msg)
return match
def handle_create_experiment(self, message, sock):
if not self.network_ready() and self._nearby_servers.dict_snapshot():
send_msg(sock, self.mtool.fill_msg("rq_error",
err_code='server_network_not_ready'))
return
launch_file = message.launch_file
sandbox = message.sandbox_dir
name = message.name
overwrites = message.overwrites
sandbox = sandbox if sandbox else settings.DEFAULT_SANDBOX_DIR
exp, details = self.start_experiment_process(
sandbox, launch_file, name, overwrites)
if exp is None:
self.logger.error("failed to launch experiment "
"process, request: " + str(vars(message)))
send_msg(sock, self.mtool.fill_msg("rq_error",
request=vars(message),
err_code='launch_error', details=details))
else:
self.logger.info("experiment process "
"launched: {0}".format(exp.pid))
if sock.socket_type in [zmq.REP, zmq.ROUTER]:
self.client_rq = (message, sock)
def find_new_experiments_and_push_results(ctx, rq_message):
LOGGER = logger.get_logger("eeg_AMPLIFIER_finder", "info")
if not rq_message.amplifier_types:
LOGGER.info("AMPLIFIER TYPES NOT SET, FINDING ALL...")
driv = find_drivers()
else:
driv = []
for amptype in rq_message.amplifier_types:
if amptype == 'bt' or amptype == 'bluetooth':
driv += find_bluetooth_amps()
elif amptype == 'usb':
driv += find_usb_amps()
elif amptype == 'virtual':
driv += find_virtual_amps()
LOGGER.info("amplifiers! return to: " + rq_message.client_push_address)
mtool = OBCIMessageTool(message_templates)
to_client = ctx.socket(zmq.PUSH)
to_client.connect(rq_message.client_push_address)
send_msg(
to_client,
mtool.fill_msg('eeg_amplifiers',
sender_ip=socket.gethostname(),
amplifier_list=driv))
LOGGER.info("sent amplifier data... " + str(driv)[:500] + ' [...]')
time.sleep(0.1)
def find_new_experiments_and_push_results(ctx, rq_message):
LOGGER = logger.get_logger("eeg_AMPLIFIER_finder", "info")
if not rq_message.amplifier_types:
LOGGER.info("AMPLIFIER TYPES NOT SET, FINDING ALL...")
driv = find_drivers()
else:
driv = []
for amptype in rq_message.amplifier_types:
if amptype == 'bt' or amptype == 'bluetooth':
driv += find_bluetooth_amps()
elif amptype == 'usb':
driv += find_usb_amps()
elif amptype == 'virtual':
driv += find_virtual_amps()
LOGGER.info("amplifiers! return to: " + rq_message.client_push_address)
mtool = OBCIMessageTool(message_templates)
to_client = ctx.socket(zmq.PUSH)
to_client.connect(rq_message.client_push_address)
send_msg(to_client, mtool.fill_msg('eeg_amplifiers', sender_ip=socket.gethostname(),
amplifier_list=driv))
LOGGER.info("sent amplifier data... " + str(driv)[:500] + ' [...]')
time.sleep(0.1)
def _publisher_thread(self, pub_addrs, pull_address, push_addr):
# FIXME aaaaahhh pub_addresses are set here, not in the main thread
# (which reads them in _register method)
pub_sock, self.pub_addresses = self._init_socket(
pub_addrs, zmq.PUB)
pull_sock = self.ctx.socket(zmq.PULL)
pull_sock.bind(pull_address)
push_sock = self.ctx.socket(zmq.PUSH)
push_sock.connect(push_addr)
send_msg(push_sock, b'1')
po = PollingObject()
while not self._stop_publishing:
try:
to_publish, det = po.poll_recv(pull_sock, 500)
if to_publish:
send_msg(pub_sock, to_publish)
except:
# print self.name, '.Publisher -- STOP.'
break
# self.logger.info( "close sock %s %s", pub_addrs, pub_sock)
pub_sock.close()
pull_sock.close()
push_sock.close()
def _launch_processes(self, launch_data, restore_config=[]):
proc, details, info_obj = None, None, None
success = True
path, args = None, None
self.status = launcher_tools.LAUNCHING
ldata = []
if 'amplifier' in launch_data:
ldata.append(('amplifier', launch_data['amplifier']))
for peer, data in launch_data.iteritems():
if (peer, data) not in ldata and peer != 'config_server':
ldata.append((peer, data))
for peer, data in ldata:#self.launch_data.iteritems():
if peer.startswith('mx'):
continue
proc, details, wait, info_obj = self.launch_process(peer, data, restore_config=restore_config)
time.sleep(wait)
if proc is None:
success = False
break
if success:
send_msg(self._publish_socket, self.mtool.fill_msg("all_peers_launched",
machine=self.machine))
def morph(self,
exp_strname,
launch_file,
name=None,
overwrites=None,
leave_on=None):
response = self.get_experiment_contact(exp_strname)
exp_sock = self.ctx.socket(zmq.REQ)
try:
if response.type == "rq_error" or response.type == "no_data":
return response
for addr in response.rep_addrs:
exp_sock.connect(addr)
msg = self.mtool.fill_msg('morph_to_new_scenario',
launch_file=launch_file,
name=name,
overwrites=overwrites,
leave_on=leave_on)
send_msg(exp_sock, msg)
response, details = self.poll_recv(exp_sock, 6000)
return response
finally:
exp_sock.close()
def _launch_processes(self, launch_data, restore_config=[]):
proc, details, info_obj = None, None, None
success = True
self.status = launcher_tools.LAUNCHING
ldata = []
if 'amplifier' in launch_data:
ldata.append(('amplifier', launch_data['amplifier']))
for peer, data in launch_data.items():
if (peer, data) not in ldata and peer != 'config_server':
ldata.append((peer, data))
for peer, data in ldata: # self.launch_data.iteritems():
if peer.startswith('mx'):
continue
proc, details, wait, info_obj = self.launch_process(peer, data, restore_config=restore_config)
time.sleep(wait)
if proc is None:
success = False
break
if success:
send_msg(self._publish_socket, self.mtool.fill_msg("all_peers_launched",
machine=self.machine))
def configure_peer(self,
exp_strname,
peer_id,
config_overrides,
override_files=None):
response = self.get_experiment_contact(exp_strname)
if response.type == "rq_error" or response.type == "no_data":
return response
sock = self.ctx.socket(zmq.REQ)
try:
for addr in response.rep_addrs:
sock.connect(addr)
if override_files:
send_msg(sock,
self.mtool.fill_msg("get_peer_info", peer_id=peer_id))
response, details = self.poll_recv(sock, 2000)
if response.type is 'rq_error':
return response
msg = self.mtool.fill_msg("update_peer_config",
peer_id=peer_id,
**config_overrides)
send_msg(sock, msg)
#print msg
response, details = self.poll_recv(sock, 2000)
return response
finally:
sock.close()
def add_peer(self,
strname,
peer_id,
path,
machine,
param_overwrites=None,
custom_config_path=None,
config_sources=None,
launch_dependencies=None,
apply_globals=True):
response = self.get_experiment_contact(strname)
if response.type == "rq_error" or response.type == "no_data":
return response
sock = self.ctx.socket(zmq.REQ)
try:
self._connect(sock, response.rep_addrs)
send_msg(
sock,
self.mtool.fill_msg("add_peer",
peer_id=peer_id,
peer_path=path,
peer_type='obci_peer',
machine=machine,
param_overwrites=param_overwrites,
custom_config_path=custom_config_path,
config_sources=config_sources,
launch_dependencies=launch_dependencies,
apply_globals=apply_globals))
response, details = self.poll_recv(sock, 5000)
return response
finally:
sock.close()
def configure_peer(self, exp_strname, peer_id, config_overrides, override_files=None):
response = self.get_experiment_contact(exp_strname)
if response.type == "rq_error" or response.type == "no_data":
return response
sock = self.ctx.socket(zmq.REQ)
try:
for addr in response.rep_addrs:
sock.connect(addr)
if override_files:
send_msg(sock, self.mtool.fill_msg("get_peer_info", peer_id=peer_id))
response, details = self.poll_recv(sock, 2000)
if response.type is 'rq_error':
return response
msg = self.mtool.fill_msg("update_peer_config", peer_id=peer_id,
**config_overrides)
send_msg(sock, msg)
# print msg
response, details = self.poll_recv(sock, 2000)
return response
finally:
sock.close()
def get_experiment_contact(self, strname):
send_msg(
self.server_req_socket,
self.mtool.fill_msg("get_experiment_contact", strname=strname))
response, details = self.poll_recv(self.server_req_socket,
self.default_timeout)
return response
def _launch_process(self, path, args, proc_type, name,
env=None, capture_io=NO_STDIO):
self.logger.debug("launching..... %s %s", path, args)
proc, details = self.subprocess_mgr.new_local_process(path, args,
proc_type=proc_type,
name=name,
monitoring_optflags=RETURNCODE,
capture_io=capture_io,
env=env)
if proc is None:
self.logger.error("process launch FAILED: %s --- %s",
path, str(args))
send_msg(self._publish_socket, self.mtool.fill_msg("launch_error",
sender=self.uuid,
details=dict(machine=self.machine, path=path, args=args,
error=details, peer_id=name)))
else:
self.logger.info("process launch success:" +\
path + str(args) + str(proc.pid))
msg = self.mtool.fill_msg("launched_process_info",
sender=self.uuid,
machine=self.machine,
pid=proc.pid,
proc_type=proc_type, name=name,
path=path,
args=args)
if name == "config_server":
self.__cfg_launch_info = msg
else:
send_msg(self._publish_socket, msg)
return proc, details
def _launch_process(self, path, args, proc_type, name,
env=None, capture_io=NO_STDIO):
self.logger.debug("launching..... %s %s", path, args)
proc, details = self.subprocess_mgr.new_local_process(path, args,
proc_type=proc_type,
name=name,
monitoring_optflags=RETURNCODE,
capture_io=capture_io,
env=env)
if proc is None:
self.logger.error("process launch FAILED: %s --- %s",
path, str(args))
send_msg(self._publish_socket, self.mtool.fill_msg("launch_error",
sender=self.uuid,
details=dict(machine=self.machine, path=path, args=args,
error=details, peer_id=name)))
else:
self.logger.info("process launch success:" +
path + str(args) + str(proc.pid))
msg = self.mtool.fill_msg("launched_process_info",
sender=self.uuid,
machine=self.machine,
pid=proc.pid,
proc_type=proc_type, name=name,
path=path,
args=args)
if name == "config_server":
self.__cfg_launch_info = msg
else:
send_msg(self._publish_socket, msg)
return proc, details
def handle(self):
print "SERVER REQUEST", self.__class__, "ME: ", self.request.getsockname()
print "FROM :", self.request.getpeername()
message = recv_unicode_netstring(self.rfile)
print message
pl = PollingObject()
parsed = self.server.mtool.unpack_msg(message)
if parsed.type == "find_eeg_experiments" or parsed.type == "find_eeg_amplifiers":
pull_addr = "tcp://" + socket.gethostname() + ":" + str(self.server.pull_port)
parsed.client_push_address = pull_addr
srv_sock = self.make_srv_sock()
try:
send_msg(srv_sock, parsed.SerializeToString())
response, det = pl.poll_recv(srv_sock, timeout=5000)
finally:
srv_sock.close()
print "passed msg and got result: ", response
if not response:
self.bad_response(self.wfile, det)
return
if parsed.type == "find_eeg_experiments" or parsed.type == "find_eeg_amplifiers":
response, det = pl.poll_recv(self.server.pull_sock, timeout=20000)
if not response:
self.bad_response(self.wfile, det)
return
data = make_unicode_netstring(response)
self.wfile.write(data)
def handle_create_experiment(self, message, sock):
if not self.network_ready() and self._nearby_servers.dict_snapshot():
send_msg(
sock,
self.mtool.fill_msg("rq_error",
err_code='server_network_not_ready'))
return
launch_file = message.launch_file
sandbox = message.sandbox_dir
name = message.name
overwrites = message.overwrites
sandbox = sandbox if sandbox else settings.DEFAULT_SANDBOX_DIR
exp, details = self.start_experiment_process(sandbox, launch_file,
name, overwrites)
if exp is None:
self.logger.error("failed to launch experiment "
"process, request: " + str(vars(message)))
send_msg(
sock,
self.mtool.fill_msg("rq_error",
request=vars(message),
err_code='launch_error',
details=details))
else:
self.logger.info("experiment process "
"launched: {0}".format(exp.pid))
if sock.socket_type in [zmq.REP, zmq.ROUTER]:
self.client_rq = (message, sock)
def handle_start_eeg_signal(self, message, sock):
if not self.network_ready() and self._nearby_servers.dict_snapshot():
send_msg(sock, self.mtool.fill_msg("rq_error", err_code="server_network_not_ready"))
return
send_msg(sock, self.mtool.fill_msg("rq_ok"))
start_thr = threading.Thread(target=start_eeg_signal_experiment, args=[self.ctx, self.rep_addresses, message])
start_thr.daemon = True
start_thr.start()
def cleanup_before_net_shutdown(self, kill_message, sock=None):
send_msg(self._publish_socket, # self.exp_pub,
self.mtool.fill_msg("kill", receiver=""))
send_msg(self._publish_socket, self.mtool.fill_msg("launcher_shutdown",
sender=self.uuid))
for sup in self.exp_process_supervisors:
self.exp_process_supervisors[sup].kill()
self.logger.info('sent KILL to experiments')
def send_create_experiment(self, launch_file=None, sandbox_dir=None, name=None, overwrites=None):
send_msg(self.server_req_socket, self.mtool.fill_msg("create_experiment",
launch_file=launch_file, sandbox_dir=sandbox_dir, name=name,
overwrites=overwrites))
response, details = self.poll_recv(self.server_req_socket, 5000)
return response
def send_create_experiment(self, launch_file=None, sandbox_dir=None, name=None, overwrites=None):
send_msg(self.server_req_socket, self.mtool.fill_msg("create_experiment",
launch_file=launch_file, sandbox_dir=sandbox_dir, name=name,
overwrites=overwrites))
response, details = self.poll_recv(self.server_req_socket, 5000)
return response
def handle_register_peer(self, message, sock):
"""Register peer"""
if message.peer_type == "obci_client":
send_msg(sock, self.mtool.fill_msg("rq_ok"))
elif message.peer_type == "obci_experiment":
self.handle_register_experiment(message, sock)
else:
super(OBCIServer, self).handle_register_peer(message, sock)
def handle_register_peer(self, message, sock):
"""Register peer"""
if message.peer_type == "obci_client":
send_msg(sock, self.mtool.fill_msg("rq_ok"))
elif message.peer_type == "obci_experiment":
self.handle_register_experiment(message, sock)
else:
super(OBCIServer, self).handle_register_peer(message, sock)
def _gather_other_server_results(ctx, this_addr, ip_list):
exps = []
if not ip_list:
return exps
mtool = OBCIMessageTool(message_templates)
other_exps_pull = ctx.socket(zmq.PULL)
port = other_exps_pull.bind_to_random_port('tcp://*',
min_port=PORT_RANGE[0],
max_port=PORT_RANGE[1],
max_tries=500)
my_push_addr = this_addr + ':' + str(port)
LOGGER.info("my exp_data pull: " + my_push_addr)
harvester = zmq.Poller()
harvester.register(other_exps_pull)
reqs = {}
for srv_ip in ip_list:
addr = 'tcp://' + srv_ip + ':' + net.server_rep_port()
req = ctx.socket(zmq.REQ)
req.connect(addr)
send_msg(
req,
mtool.fill_msg('find_eeg_experiments',
client_push_address='tcp://' + my_push_addr,
checked_srvs=[this_addr]))
harvester.register(req, zmq.POLLIN)
reqs[req] = addr
srv_responses = 0
for i in range(len(reqs) * 50):
socks = dict(harvester.poll(timeout=300))
for req in socks:
msg = recv_msg(req)
msg = mtool.unpack_msg(msg)
if msg.type == 'rq_ok':
LOGGER.info("waiting for experiments from server: " +
str(reqs[req]))
harvester.unregister(req)
req.close()
elif msg.type == 'eeg_experiments':
LOGGER.info("GOT EXPERIMENTS from: " + msg.sender_ip)
exps += msg.experiment_list
srv_responses += 1
else:
LOGGER.warning('strange msg: ' + str(msg))
if srv_responses == len(ip_list):
print "GOT ALL eeg_experiment RESPONSES :-) [addr list: ", ip_list, "]"
break
other_exps_pull.close()
return exps
def handle_find_new_eeg_amplifiers(self, message, sock):
if not self.network_ready() and self._nearby_servers.dict_snapshot():
send_msg(sock, self.mtool.fill_msg("rq_error", err_code="server_network_not_ready"))
return
send_msg(sock, self.mtool.fill_msg("rq_ok"))
amp_thr = threading.Thread(target=find_new_experiments_and_push_results, args=[self.ctx, message])
amp_thr.daemon = True
amp_thr.start()
def cleanup_before_net_shutdown(self, kill_message, sock=None):
send_msg(
self._publish_socket, #self.exp_pub,
self.mtool.fill_msg("kill", receiver=""))
send_msg(self._publish_socket,
self.mtool.fill_msg("launcher_shutdown", sender=self.uuid))
for sup in self.exp_process_supervisors:
self.exp_process_supervisors[sup].kill()
self.logger.info('sent KILL to experiments')
def update_nearby_servers(srv_data,
bcast_port,
ctx=None,
update_push_addr=None):
mtool = OBCIMessageTool(message_templates)
loops_to_update = _LOOPS
s = socket(AF_INET, SOCK_DGRAM)
s.bind(('', bcast_port))
notify_sock = None
if update_push_addr is not None:
ctx = ctx if ctx else zmq.Context()
notify_sock = ctx.socket(zmq.PUSH)
notify_sock.connect(update_push_addr)
while 1:
changed = False
try:
inp, out, exc = select.select([s], [], [],
UPDATE_INTERVAL / _LOOPS)
except Exception, e:
srv_data.logger.critical("nearby_servers_update - exception: %s",
str(e))
srv_data.logger.critical("nearby_servers - aborting")
return
if s in inp:
data, wherefrom = s.recvfrom(1500, 0)
# sys.stderr.write(repr(wherefrom) + '\n')
# sys.stdout.write(data)
msg = unicode(data, encoding='utf-8')
msg = msg[:-1]
message = mtool.unpack_msg(msg)
changed = srv_data.update(ip=wherefrom[0],
hostname=message.sender_ip,
uuid=message.sender,
rep_port=message.rep_port,
pub_port=message.pub_port)
else:
# print "no data"
pass
loops_to_update -= 1
if loops_to_update == 0:
loops_to_update = _LOOPS
changed = srv_data.clean_silent()
if changed:
send_msg(
notify_sock,
mtool.fill_msg('nearby_machines',
nearby_machines=srv_data.dict_snapshot()))
def launch_process(self, peer, launch_data, restore_config=[]):
data = launch_data
wait = 0
p = os.path.expanduser(data['path'])
if not os.path.isabs(p):
path = os.path.join(launcher_tools.obci_root(), p)
path = os.path.abspath(path)
else:
path = os.path.realpath(p)
dirname = os.path.dirname(path)
if not launcher_tools.obci_root() in dirname:
launcher_tools.update_pythonpath(dirname)
launcher_tools.update_obci_syspath(dirname)
self.env.update({"PYTHONPATH" : os.environ["PYTHONPATH"]})
self.logger.info("PYTHONPATH UPDATED for " + peer +\
"!!!!!!!! " + str(self.env["PYTHONPATH"]))
args = data['args']
args = self._attach_base_config_path(path, args)
args += ['-p', 'experiment_uuid', self.experiment_uuid]
if peer.startswith('config_server'):
args += ['-p', 'launcher_socket_addr', self.cs_addr]
if restore_config:
args += ['-p', 'restore_peers', ' '.join(restore_config)]
# wait = 0.5
if "log_dir" in args:
idx = args.index("log_dir") + 1
log_dir = args[idx]
log_dir = os.path.join(log_dir, self.name)
args[idx] = log_dir
else:
log_dir = os.path.join(CONFIG_DEFAULTS["log_dir"], self.name)
args += ['-p', 'log_dir', log_dir]
if not os.path.exists(log_dir):
os.makedirs(log_dir)
proc, details = self._launch_process(path, args, data['peer_type'],
peer, env=self.env, capture_io=NO_STDIO)
info_obj = {
"path": path,
"args": args,
"peer": peer
}
if proc is not None:
self.processes[peer] = proc
else:
self.logger.error("OBCI LAUNCH FAILED")
send_msg(self._publish_socket, self.mtool.fill_msg("obci_launch_failed",
machine=self.machine, path=info_obj['path'],
args=info_obj['args'], details=details))
self.processes = {}
self.subprocess_mgr.killall(force=True)
return proc, details, wait, info_obj
def _exp_connect(self, exp_data):
for addr in exp_data['pub_addrs']:
if not addr.startswith(
'tcp://localhost') and self._addr_connectable(
addr, exp_data['origin_machine']):
send_msg(
self.monitor_push,
self.mtool.fill_msg('_launcher_engine_msg',
task='connect',
pub_addr=addr))
def _gather_other_server_results(ctx, this_addr, ip_list):
exps = []
if not ip_list:
return exps
mtool = OBCIMessageTool(message_templates)
other_exps_pull = ctx.socket(zmq.PULL)
port = other_exps_pull.bind_to_random_port('tcp://*',
min_port=PORT_RANGE[0],
max_port=PORT_RANGE[1], max_tries=500)
my_push_addr = this_addr + ':' + str(port)
LOGGER.info("my exp_data pull: " + my_push_addr)
harvester = zmq.Poller()
harvester.register(other_exps_pull)
reqs = {}
for srv_ip in ip_list:
addr = 'tcp://' + srv_ip + ':' + net.server_rep_port()
req = ctx.socket(zmq.REQ)
req.connect(addr)
send_msg(req, mtool.fill_msg('find_eeg_experiments',
client_push_address='tcp://' + my_push_addr,
checked_srvs=[this_addr]))
harvester.register(req, zmq.POLLIN)
reqs[req] = addr
srv_responses = 0
for i in range(len(reqs) * 50):
socks = dict(harvester.poll(timeout=300))
for req in socks:
msg = recv_msg(req)
msg = mtool.unpack_msg(msg)
if msg.type == 'rq_ok':
LOGGER.info("waiting for experiments from server: " + str(reqs[req]))
harvester.unregister(req)
req.close()
elif msg.type == 'eeg_experiments':
LOGGER.info("GOT EXPERIMENTS from: " + msg.sender_ip)
exps += msg.experiment_list
srv_responses += 1
else:
LOGGER.warning('strange msg: ' + str(msg))
if srv_responses == len(ip_list):
print("GOT ALL eeg_experiment RESPONSES :-) [addr list: ", ip_list, "]")
break
other_exps_pull.close()
return exps
def handle_kill_process_supervisor(self, message, sock):
proc = self.subprocess_mgr.process(message.machine, message.pid)
if not proc:
send_msg(sock, self.mtool.fill_msg("rq_error", err_code="process_not_found"))
else:
# TODO
# name = proc.name
proc.kill()
proc.mark_delete()
send_msg(sock, self.mtool.fill_msg("rq_ok"))
del self.exp_process_supervisors[proc.name]
def handle_get_tail(self, message, sock):
lines = message.len if message.len else DEFAULT_TAIL_RQ
peer = message.peer_id
if peer not in self.launch_data:
return
experiment_id = self.launch_data[peer]["experiment_id"]
txt = self.processes[peer].tail_stdout(lines=lines)
send_msg(
self._publish_socket,
self.mtool.fill_msg("tail", txt=txt, sender=self.uuid, experiment_id=experiment_id, peer_id=peer),
)
def handle_get_tail(self, message, sock):
lines = message.len if message.len else DEFAULT_TAIL_RQ
peer = message.peer_id
if peer not in self.launch_data:
return
experiment_id = self.launch_data[peer]['experiment_id']
txt = self.processes[peer].tail_stdout(lines=lines)
send_msg(self._publish_socket, self.mtool.fill_msg("tail", txt=txt,
sender=self.uuid,
experiment_id=experiment_id,
peer_id=peer))
def handle(self):
message = recv_unicode_netstring(self.rfile)
srv_sock = self.make_srv_sock()
try:
send_msg(srv_sock, message)
pl = PollingObject()
response, det = pl.poll_recv(srv_sock, timeout=5000)
finally:
srv_sock.close()
if not response:
self.bad_response(self.wfile, det)
self.rfile.write(make_unicode_netstring(response))
def handle(self):
message = recv_unicode_netstring(self.rfile)
srv_sock = self.make_srv_sock()
try:
send_msg(srv_sock, message)
pl = PollingObject()
response, det = pl.poll_recv(srv_sock, timeout=5000)
finally:
srv_sock.close()
if not response:
self.bad_response(self.wfile, det)
self.rfile.write(make_unicode_netstring(response))
def comm_exp(self, msg):
send_msg(self.exp_req, msg)
response, _ = self.poller.poll_recv(self.exp_req, timeout=3000)
if not response:
print "!!!!!!!!!!!!!!!!!!!!!!!!!!!1 no response to ", msg
self.exp_req.close()
self.exp_req = self.ctx.socket(zmq.REQ)
for addr in self.launcher_data['rep_addrs']:
if self._addr_connectable(addr, self.launcher_data['origin_machine']):
self.exp_req.connect(addr)
return None
return self.mtool.unpack_msg(response)
def comm_exp(self, msg):
send_msg(self.exp_req, msg)
response, _ = self.poller.poll_recv(self.exp_req, timeout=3000)
if not response:
print "!!!!!!!!!!!!!!!!!!!!!!!!!!!1 no response to ", msg
self.exp_req.close()
self.exp_req = self.ctx.socket(zmq.REQ)
for addr in self.launcher_data['rep_addrs']:
if self._addr_connectable(addr, self.launcher_data['origin_machine']):
self.exp_req.connect(addr)
return None
return self.mtool.unpack_msg(response)
def _handle_register_experiment_timeout(self, exp):
self.logger.error("New experiment process failed to "
"register before timeout" + str(exp.pid))
if exp.returncode is None:
exp.kill()
exp.wait()
# msg_type = self.client_rq[0].type
rq_sock = self.client_rq[1]
send_msg(rq_sock, self.mtool.fill_msg("rq_error",
err_code="create_experiment_error",
request=vars(self.client_rq[0])))
def send_start_experiment(self, exp_addrs):
exp_sock = self.ctx.socket(zmq.REQ)
try:
for addr in exp_addrs:
exp_sock.connect(addr)
send_msg(exp_sock, self.mtool.fill_msg("start_experiment"))
reply, details = self.poll_recv(exp_sock, 20000)
# print reply
return reply
finally:
exp_sock.close()
def send_start_experiment(self, exp_addrs):
exp_sock = self.ctx.socket(zmq.REQ)
try:
for addr in exp_addrs:
exp_sock.connect(addr)
send_msg(exp_sock, self.mtool.fill_msg("start_experiment"))
reply, details = self.poll_recv(exp_sock, 20000)
#print reply
return reply
finally:
exp_sock.close()
def handle_kill_process_supervisor(self, message, sock):
proc = self.subprocess_mgr.process(message.machine, message.pid)
if not proc:
send_msg(
sock,
self.mtool.fill_msg("rq_error", err_code="process_not_found"))
else:
#TODO
name = proc.name
proc.kill()
proc.mark_delete()
send_msg(sock, self.mtool.fill_msg("rq_ok"))
del self.exp_process_supervisors[proc.name]
def handle_start_eeg_signal(self, message, sock):
if not self.network_ready() and self._nearby_servers.dict_snapshot():
send_msg(
sock,
self.mtool.fill_msg("rq_error",
err_code='server_network_not_ready'))
return
send_msg(sock, self.mtool.fill_msg("rq_ok"))
start_thr = threading.Thread(
target=start_eeg_signal_experiment,
args=[self.ctx, self.rep_addresses, message])
start_thr.daemon = True
start_thr.start()
def _subprocess_info(self, push_addr):
push_sock = self.ctx.socket(zmq.PUSH)
push_sock.connect(push_addr)
send_msg(push_sock, u'1')
while not self._stop_monitoring:
dead = self.subprocess_mgr.not_running_processes()
if dead:
# self.logger.warning("DEAD process" + str(dead))
for key, status in dead.iteritems():
send_msg(push_sock, self.mtool.fill_msg('dead_process', machine=key[0],
pid=key[1], status=status))
time.sleep(0.5)
push_sock.close()
def handle_message(self, mxmsg):
# handle something
if mxmsg.type == types.DICT_GET_REQUEST_MESSAGE:
# print '*',
self.queries += 1
self.send_message(message=str(self.queries), to=int(mxmsg.from_),
type=types.DICT_GET_RESPONSE_MESSAGE, flush=True)
send_msg(self.push, str(self.queries))
if self.queries % 10000 == 0:
print "[mx srv]: sent ", self.queries, "messages"
if self.queries == SEND:
print "[mx srv]:", SEND, "queries"
def handle_kill_experiment(self, message, sock):
match = self._handle_match_name(message, sock, this_machine=True)
if match:
if match.kill_timer is not None:
send_msg(
sock,
self.mtool.fill_msg(
"rq_error",
err_code="already_killed",
details="Experiment already shutting down"))
elif not message.force:
self.logger.info("sending kill to experiment "
"{0} ({1})".format(match.uuid, match.name))
send_msg(
self._publish_socket, #self.exp_pub,
self.mtool.fill_msg("kill", receiver=match.uuid))
send_msg(
sock,
self.mtool.fill_msg("kill_sent", experiment_id=match.uuid))
pid = match.experiment_pid
uid = match.uuid
self.logger.info(
"Waiting for experiment process {0} to terminate".format(
uid))
match.kill_timer = threading.Timer(1.1,
self._handle_killing_exp,
args=[pid, uid])
match.kill_timer.start()
send_msg(
self._publish_socket,
self.mtool.fill_msg('kill_sent', experiment_id=match.uuid))
def handle_find_new_eeg_amplifiers(self, message, sock):
if not self.network_ready() and self._nearby_servers.dict_snapshot():
send_msg(
sock,
self.mtool.fill_msg("rq_error",
err_code='server_network_not_ready'))
return
send_msg(sock, self.mtool.fill_msg("rq_ok"))
amp_thr = threading.Thread(
target=find_new_experiments_and_push_results,
args=[self.ctx, message])
amp_thr.daemon = True
amp_thr.start()
def _subprocess_info(self, push_addr):
push_sock = self.ctx.socket(zmq.PUSH)
push_sock.connect(push_addr)
send_msg(push_sock, b'1')
while not self._stop_monitoring:
dead = self.subprocess_mgr.not_running_processes()
if dead:
# self.logger.warning("DEAD process" + str(dead))
for key, status in dead.items():
send_msg(push_sock, self.mtool.fill_msg('dead_process', machine=key[0],
pid=key[1], status=status))
time.sleep(0.5)
push_sock.close()
def _running_experiments(self):
send_msg(self.server_req_socket, self.mtool.fill_msg("list_experiments"))
exp_list, details = self.poll_recv(self.server_req_socket, 2000)
if not exp_list:
LOGGER.error("Connection to obci_server failed. (list_experiments)")
return None
exps = exp_list.exp_data
running = []
for exp in exps.values():
if exp['status_name'] == launcher_tools.RUNNING or \
exp['status_name'] == launcher_tools.LAUNCHING:
running.append(exp)
return running
def handle_find_eeg_experiments(self, message, sock):
if not self.network_ready() and self._nearby_servers.dict_snapshot():
send_msg(sock, self.mtool.fill_msg("rq_error",
err_code='server_network_not_ready'))
return
send_msg(sock, self.mtool.fill_msg("rq_ok"))
finder_thr = threading.Thread(target=find_eeg_experiments_and_push_results,
args=[self.ctx, self.rep_addresses,
message,
self._nearby_servers.copy()])
finder_thr.daemon = True
finder_thr.start()
def handle_get_experiment_contact(self, message, sock):
self.logger.debug("##### rq contact for: %s", message.strname)
info = self._handle_match_name(message, sock)
if info:
send_msg(sock, self.mtool.fill_msg("experiment_contact",
uuid=info.uuid,
name=info.name,
rep_addrs=info.rep_addrs,
pub_addrs=info.pub_addrs,
tcp_addrs=info.tcp_addrs,
machine=info.origin_machine,
status_name=info.status_name,
details=info.details))
def start_experiment(self, msg, store_options=None):
print "START EXPERIMENT!!!!"
uid = str(msg)
index = self.index_of(uid)
if index is None:
print "experiment uuid not found: ", uid
return
exp = self.experiments[index]
if exp.launcher_data:
print "already running"
return
if store_options:
exp.enable_signal_storing(store_options)
jsoned = serialize_scenario_json(exp.exp_config)
result = self.client.send_create_experiment(name=exp.name)
ok = self._process_response(result)
if not ok:
return result
print result
machine = result.origin_machine
addrs = [
addr for addr in result.rep_addrs
if self._addr_connectable(addr, machine)
]
exp_sock = self.ctx.socket(zmq.REQ)
for addr in addrs:
exp_sock.connect(addr)
send_msg(
exp_sock,
self.mtool.fill_msg(
"set_experiment_scenario",
scenario=jsoned,
launch_file_path=exp.exp_config.launch_file_path))
reply, details = self.client.poll_recv(exp_sock, 20000)
ok = self._process_response(reply)
if not ok:
exp_sock.close()
return
send_msg(exp_sock, self.mtool.fill_msg("start_experiment"))
reply, details = self.client.poll_recv(exp_sock, 20000)
self._process_response(reply)
exp_sock.close()
def _handle_register_experiment_timeout(self, exp):
self.logger.error("New experiment process failed to "
"register before timeout" + str(exp.pid))
if exp.returncode is None:
exp.kill()
exp.wait()
msg_type = self.client_rq[0].type
rq_sock = self.client_rq[1]
send_msg(
rq_sock,
self.mtool.fill_msg("rq_error",
err_code="create_experiment_error",
request=vars(self.client_rq[0])))
def _register(self, rep_addrs, pub_addrs, params):
message = self.mtool.fill_msg("register_peer", peer_type=self.type,
uuid=self.uuid,
rep_addrs=rep_addrs,
pub_addrs=pub_addrs,
name=self.name,
other_params=params)
self.logger.debug("_register() " + str(message))
send_msg(self.source_req_socket, message)
response_str = recv_msg(self.source_req_socket)
response = self.mtool.unpack_msg(response_str)
if response.type == "rq_error":
self.logger.critical("Registration failed: {0}".format(response_str))
sys.exit(2)
return response
def _get_amp_info(self, exp_sock, peer_id):
send_msg(exp_sock, self.mtool.fill_msg('get_peer_info', peer_id=peer_id))
info, details = self.poll_recv(exp_sock, 4000)
send_msg(exp_sock, self.mtool.fill_msg('get_peer_param_values', peer_id=peer_id))
params, details = self.poll_recv(exp_sock, 4000)
if not info or not params:
LOGGER.error("get_peer_info failed " + str(peer_id) + " " + str(details))
return None, None
info = info.dict()
params = params.dict()
for field in ["sender", "sender_ip", "receiver", "type", "local_params", "external_params",
"config_sources", "launch_dependencies"]:
del info[field]
return info, params
def _register(self, rep_addrs, pub_addrs, params):
message = self.mtool.fill_msg("register_peer", peer_type=self.type,
uuid=self.uuid,
rep_addrs=rep_addrs,
pub_addrs=pub_addrs,
name=self.name,
other_params=params)
self.logger.debug("_register() " + str(message))
send_msg(self.source_req_socket, message)
response_str = recv_msg(self.source_req_socket)
response = self.mtool.unpack_msg(response_str)
if response.type == "rq_error":
self.logger.critical("Registration failed: {0}".format(response_str))
sys.exit(2)
return response
def kill_peer(self, exp_strname, peer_id, remove_config=False):
response = self.get_experiment_contact(exp_strname)
if response.type == "rq_error" or response.type == "no_data":
return response
try:
sock = self.ctx.socket(zmq.REQ)
self._connect(sock, response.rep_addrs)
send_msg(sock, self.mtool.fill_msg("kill_peer",
peer_id=peer_id, remove_config=remove_config))
response, details = self.poll_recv(sock, 5000)
return response
finally:
sock.close()
