def broadcast_server(server_uuid, rep_port, pub_port, bcast_port): mtool = OBCIMessageTool(message_templates) str_msg = mtool.fill_msg("server_broadcast", sender_ip=socket.gethostname(), sender=server_uuid, rep_port=rep_port, pub_port=pub_port) str_msg += b'\n' s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.bind(('', 0)) s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) # introduction for i in range(10): try: s.sendto(str_msg, ('<broadcast>', bcast_port)) except socket.error as e: pass time.sleep(0.3) # updates while True: try: s.sendto(str_msg, ('<broadcast>', bcast_port)) except socket.error as e: print("[obci_server] Cannot broadcast obci_server, will try again in 1min:", str(e)) time.sleep(53) time.sleep(7) s.close()
def start_eeg_signal_experiment(ctx, srv_addrs, rq_message): client = OBCIClient(srv_addrs, ctx) # server_req_socket = ctx.socket(zmq.REQ) # for addr in srv_addrs: # server_req_socket.connect(addr) amp_params = {} amp_params.update(rq_message.amplifier_params['additional_params']) del rq_message.amplifier_params['additional_params'] amp_params.update(rq_message.amplifier_params) par_list = ['--peer', 'amplifier'] for par, val in amp_params.iteritems(): par_list += ['-p', par, unicode(val)] overwrites = peer_cmd.peer_overwrites_pack(par_list) result = client.launch(rq_message.launch_file, None, rq_message.name, overwrites) LOGGER.info("START EEG signal! return to: " + rq_message.client_push_address) mtool = OBCIMessageTool(message_templates) to_client = ctx.socket(zmq.PUSH) to_client.connect(rq_message.client_push_address) if result is None: send_msg(to_client, mtool.fill_msg("rq_error", err_code="launch_failed", details="No response from server or experiment")) else: send_msg(to_client, result.raw()) LOGGER.info("sent eeg launch result" + str(result)[:500] ) time.sleep(0.1)
def find_new_experiments_and_push_results(ctx, rq_message): LOGGER = logger.get_logger("eeg_AMPLIFIER_finder", "info") if not rq_message.amplifier_types: LOGGER.info("AMPLIFIER TYPES NOT SET, FINDING ALL...") driv = find_drivers() else: driv = [] for amptype in rq_message.amplifier_types: if amptype == 'bt' or amptype == 'bluetooth': driv += find_bluetooth_amps() elif amptype == 'usb': driv += find_usb_amps() elif amptype == 'virtual': driv += find_virtual_amps() LOGGER.info("amplifiers! return to: " + rq_message.client_push_address) mtool = OBCIMessageTool(message_templates) to_client = ctx.socket(zmq.PUSH) to_client.connect(rq_message.client_push_address) send_msg(to_client, mtool.fill_msg('eeg_amplifiers', sender_ip=socket.gethostname(), amplifier_list=driv)) LOGGER.info("sent amplifier data... " + str(driv)[:500] + ' [...]') time.sleep(0.1)
def __init__(self, proc_description, reg_timeout_desc=None, monitoring_optflags=PING, logger=None): self.desc = proc_description self.must_register = reg_timeout_desc is not None self._status_lock = threading.RLock() self._status = UNKNOWN if self.must_register else RUNNING self._status_details = None self.ping_it = monitoring_optflags & PING self.check_returncode = monitoring_optflags & RETURNCODE if \ self.desc.pid is not None else False self.logger = logger or get_logger( 'subprocess_monitor' + '-' + self.desc.name + '-' + str(self.desc.pid), stream_level='info') self.set_registration_timeout_handler(reg_timeout_desc) self.registration_data = None self._stop_monitoring = False self._ping_thread = None self._ping_retries = 8 self._returncode_thread = None self._mtool = OBCIMessageTool(message_templates) self._ctx = None self.rq_sock = None self._poller = PollingObject() self.delete = False
def find_new_experiments_and_push_results(ctx, rq_message): LOGGER = logger.get_logger("eeg_AMPLIFIER_finder", "info") if not rq_message.amplifier_types: LOGGER.info("AMPLIFIER TYPES NOT SET, FINDING ALL...") driv = find_drivers() else: driv = [] for amptype in rq_message.amplifier_types: if amptype == 'bt' or amptype == 'bluetooth': driv += find_bluetooth_amps() elif amptype == 'usb': driv += find_usb_amps() elif amptype == 'virtual': driv += find_virtual_amps() LOGGER.info("amplifiers! return to: " + rq_message.client_push_address) mtool = OBCIMessageTool(message_templates) to_client = ctx.socket(zmq.PUSH) to_client.connect(rq_message.client_push_address) send_msg( to_client, mtool.fill_msg('eeg_amplifiers', sender_ip=socket.gethostname(), amplifier_list=driv)) LOGGER.info("sent amplifier data... " + str(driv)[:500] + ' [...]') time.sleep(0.1)
def _gather_other_server_results(ctx, this_addr, ip_list): exps = [] if not ip_list: return exps mtool = OBCIMessageTool(message_templates) other_exps_pull = ctx.socket(zmq.PULL) port = other_exps_pull.bind_to_random_port('tcp://*', min_port=PORT_RANGE[0], max_port=PORT_RANGE[1], max_tries=500) my_push_addr = this_addr + ':' + str(port) LOGGER.info("my exp_data pull: " + my_push_addr) harvester = zmq.Poller() harvester.register(other_exps_pull) reqs = {} for srv_ip in ip_list: addr = 'tcp://' + srv_ip + ':' + net.server_rep_port() req = ctx.socket(zmq.REQ) req.connect(addr) send_msg( req, mtool.fill_msg('find_eeg_experiments', client_push_address='tcp://' + my_push_addr, checked_srvs=[this_addr])) harvester.register(req, zmq.POLLIN) reqs[req] = addr srv_responses = 0 for i in range(len(reqs) * 50): socks = dict(harvester.poll(timeout=300)) for req in socks: msg = recv_msg(req) msg = mtool.unpack_msg(msg) if msg.type == 'rq_ok': LOGGER.info("waiting for experiments from server: " + str(reqs[req])) harvester.unregister(req) req.close() elif msg.type == 'eeg_experiments': LOGGER.info("GOT EXPERIMENTS from: " + msg.sender_ip) exps += msg.experiment_list srv_responses += 1 else: LOGGER.warning('strange msg: ' + str(msg)) if srv_responses == len(ip_list): print "GOT ALL eeg_experiment RESPONSES :-) [addr list: ", ip_list, "]" break other_exps_pull.close() return exps
def update_nearby_servers(srv_data, bcast_port, ctx=None, update_push_addr=None): mtool = OBCIMessageTool(message_templates) loops_to_update = _LOOPS s = socket(AF_INET, SOCK_DGRAM) s.bind(('', bcast_port)) notify_sock = None if update_push_addr is not None: ctx = ctx if ctx else zmq.Context() notify_sock = ctx.socket(zmq.PUSH) notify_sock.connect(update_push_addr) while 1: changed = False try: inp, out, exc = select.select([s], [], [], UPDATE_INTERVAL / _LOOPS) except Exception, e: srv_data.logger.critical("nearby_servers_update - exception: %s", str(e)) srv_data.logger.critical("nearby_servers - aborting") return if s in inp: data, wherefrom = s.recvfrom(1500, 0) # sys.stderr.write(repr(wherefrom) + '\n') # sys.stdout.write(data) msg = unicode(data, encoding='utf-8') msg = msg[:-1] message = mtool.unpack_msg(msg) changed = srv_data.update(ip=wherefrom[0], hostname=message.sender_ip, uuid=message.sender, rep_port=message.rep_port, pub_port=message.pub_port) else: # print "no data" pass loops_to_update -= 1 if loops_to_update == 0: loops_to_update = _LOOPS changed = srv_data.clean_silent() if changed: send_msg( notify_sock, mtool.fill_msg('nearby_machines', nearby_machines=srv_data.dict_snapshot()))
def __init__(self, srv_addrs, ctx, client_push_address, nearby_servers): self.ctx = ctx self.server_req_socket = self.ctx.socket(zmq.REQ) for addr in srv_addrs: self.server_req_socket.connect(addr) self.poller = PollingObject() self.mtool = OBCIMessageTool(message_templates) self.nearby_servers = nearby_servers self._amplified_cache = {}
def __init__(self, zmq_ctx, uuid, logger=None, obci_dns=None): self._processes = {} self._ctx = zmq_ctx self.uuid = uuid self.logger = logger or get_logger('subprocess_monitor', stream_level='warning') self.obci_dns = obci_dns self._mtool = OBCIMessageTool(message_templates) self.poller = PollingObject() self._proc_lock = threading.RLock()
def _gather_other_server_results(ctx, this_addr, ip_list): exps = [] if not ip_list: return exps mtool = OBCIMessageTool(message_templates) other_exps_pull = ctx.socket(zmq.PULL) port = other_exps_pull.bind_to_random_port('tcp://*', min_port=PORT_RANGE[0], max_port=PORT_RANGE[1], max_tries=500) my_push_addr = this_addr + ':' + str(port) LOGGER.info("my exp_data pull: " + my_push_addr) harvester = zmq.Poller() harvester.register(other_exps_pull) reqs = {} for srv_ip in ip_list: addr = 'tcp://' + srv_ip + ':' + net.server_rep_port() req = ctx.socket(zmq.REQ) req.connect(addr) send_msg(req, mtool.fill_msg('find_eeg_experiments', client_push_address='tcp://' + my_push_addr, checked_srvs=[this_addr])) harvester.register(req, zmq.POLLIN) reqs[req] = addr srv_responses = 0 for i in range(len(reqs) * 50): socks = dict(harvester.poll(timeout=300)) for req in socks: msg = recv_msg(req) msg = mtool.unpack_msg(msg) if msg.type == 'rq_ok': LOGGER.info("waiting for experiments from server: " + str(reqs[req])) harvester.unregister(req) req.close() elif msg.type == 'eeg_experiments': LOGGER.info("GOT EXPERIMENTS from: " + msg.sender_ip) exps += msg.experiment_list srv_responses += 1 else: LOGGER.warning('strange msg: ' + str(msg)) if srv_responses == len(ip_list): print("GOT ALL eeg_experiment RESPONSES :-) [addr list: ", ip_list, "]") break other_exps_pull.close() return exps
def __init__(self, address): self.context = zmq.Context.instance() templates = launcher_messages.message_templates self.msg_factory = OBCIMessageTool(msg_templates=templates) self.socket = self.context.socket(zmq.REQ) self.socket.connect(address) # disabled bc Ubuntu LTS uses archaic zmq version # self.socket.RCVTIMEO = 7000 self.poller = zmq.Poller() self.poller.register(self.socket, zmq.POLLIN) self.open = True
def __init__(self, server_addresses, zmq_context=None): self.ctx = zmq_context if zmq_context else zmq.Context() self.server_addresses = server_addresses self.server_req_socket = None self.init_server_socket(server_addresses) #self = zmq.Poller() #self.register(self.server_req_socket, zmq.POLLIN) self.poller = PollingObject() self.mtool = OBCIMessageTool(message_templates) self.dns = net.DNS()
class ObciBaseClient(object): """ Base class for communicating with OBCI peers. """ def __init__(self, address): self.context = zmq.Context.instance() templates = launcher_messages.message_templates self.msg_factory = OBCIMessageTool(msg_templates=templates) self.socket = self.context.socket(zmq.REQ) self.socket.connect(address) # disabled bc Ubuntu LTS uses archaic zmq version # self.socket.RCVTIMEO = 7000 self.poller = zmq.Poller() self.poller.register(self.socket, zmq.POLLIN) self.open = True def close(self): if self.open: self.socket.close() self.open = False def send_recv(self, message_name, **kwargs): data = self.msg_factory.fill_msg(message_name, **kwargs) self.socket.send(data) poll_results = self.poller.poll(7000) response = self.socket.recv(zmq.NOBLOCK) if poll_results else None return json.loads(response)
def __init__(self, proc_description, reg_timeout_desc=None, monitoring_optflags=PING, logger=None): self.desc = proc_description self.must_register = reg_timeout_desc is not None self._status_lock = threading.RLock() self._status = UNKNOWN if self.must_register else RUNNING self._status_details = None self.ping_it = monitoring_optflags & PING self.check_returncode = monitoring_optflags & RETURNCODE if \ self.desc.pid is not None else False self.logger = logger or get_logger( 'subprocess_monitor'+'-'+self.desc.name+'-'+str(self.desc.pid), stream_level='info') self.set_registration_timeout_handler(reg_timeout_desc) self.registration_data = None self._stop_monitoring = False self._ping_thread = None self._ping_retries = 8 self._returncode_thread = None self._mtool = OBCIMessageTool(message_templates) self._ctx = None self.rq_sock = None self._poller = PollingObject() self.delete = False
def update_nearby_servers(srv_data, bcast_port, ctx=None, update_push_addr=None): mtool = OBCIMessageTool(message_templates) loops_to_update = _LOOPS s = socket(AF_INET, SOCK_DGRAM) s.bind(('', bcast_port)) notify_sock = None if update_push_addr is not None: ctx = ctx if ctx else zmq.Context() notify_sock = ctx.socket(zmq.PUSH) notify_sock.connect(update_push_addr) while 1: changed = False try: inp, out, exc = select.select([s], [], [], UPDATE_INTERVAL / _LOOPS) except Exception, e: srv_data.logger.critical("nearby_servers_update - exception: %s", str(e)) srv_data.logger.critical("nearby_servers - aborting") return if s in inp: data, wherefrom = s.recvfrom(1500, 0) # sys.stderr.write(repr(wherefrom) + '\n') # sys.stdout.write(data) msg = unicode(data, encoding='utf-8') msg = msg[:-1] message = mtool.unpack_msg(msg) changed = srv_data.update(ip=wherefrom[0], hostname=message.sender_ip, uuid=message.sender, rep_port=message.rep_port, pub_port=message.pub_port) else: # print "no data" pass loops_to_update -= 1 if loops_to_update == 0: loops_to_update = _LOOPS changed = srv_data.clean_silent() if changed: send_msg(notify_sock, mtool.fill_msg('nearby_machines', nearby_machines=srv_data.dict_snapshot()))
def __init__(self, addresses): super(ConfigServer, self).__init__(addresses=addresses, type=peers.CONFIG_SERVER) self._configs = {} self._ext_configs = {} self._ready_peers = [] self.__to_all = False self.mtool = OBCIMessageTool(message_templates) self.launcher_sock = None params, other_params = PeerCmd().parse_cmd() self.addr = params['local_params'].get('launcher_socket_addr', '') self.exp_uuid = params['local_params'].get('experiment_uuid', '') self.log_dir = params['local_params'].get('log_dir', None) self.logger = get_logger('config_server', log_dir=self.log_dir, conn=self.conn, file_level=params['local_params'].get('file_log_level', None), mx_level=params['local_params'].get('mx_log_level', None), stream_level=params['local_params'].get('console_log_level', None)) self._old_configs = self._stored_config() self._restore_peers = params['local_params'].get('restore_peers', '').split() for peer in self._restore_peers: if peer in self._old_configs["local"]: self._configs[peer] = dict(self._old_configs["local"][peer]) self._ready_peers.append(peer) if peer in self._old_configs["ext"]: self._ext_configs[peer] = dict(self._old_configs["ext"][peer]) if self.addr != '': self.ctx = zmq.Context() self.launcher_sock = self.ctx.socket(zmq.PUSH) try: self.launcher_sock.connect(self.addr) except Exception, e: self.logger.error("failed to connect to address " +\ self.addr + " !!!") self.launcher_sock = None else: self.logger.info("OK: connected to " + self.addr) send_msg(self.launcher_sock, self.mtool.fill_msg("config_server_ready"))
def broadcast_server(server_uuid, rep_port, pub_port, bcast_port): mtool = OBCIMessageTool(message_templates) msg = mtool.fill_msg("server_broadcast", sender_ip=gethostname(), sender=server_uuid, rep_port=rep_port, pub_port=pub_port) msg += u'\n' str_msg = msg.encode('utf-8') s = socket(AF_INET, SOCK_DGRAM) s.bind(('', 0)) s.setsockopt(SOL_SOCKET, SO_BROADCAST, 1) # introduction for i in range(10): try: s.sendto(str_msg, ('<broadcast>', bcast_port)) except error, e: pass time.sleep(0.3)
def kill(self): # send "kill" to the process or kill request to its supervisor? self.stop_monitoring() if not self._ctx: self._ctx = zmq.Context() rq_sock = self._ctx.socket(zmq.REQ) try: rq_sock.connect(self.rq_address) mtool = OBCIMessageTool(message_templates) poller = PollingObject() send_msg(rq_sock, mtool.fill_msg("kill_process", pid=self.pid, machine=self.machine_ip)) res, _ = poller.poll_recv(rq_sock, timeout=5000) finally: rq_sock.close() if res: res = mtool.unpack_msg(res) print "Response to kill request: ", res with self._status_lock: self._status = TERMINATED
def __init__(self, server_addresses, zmq_context=None): self.ctx = zmq_context if zmq_context else zmq.Context() self.server_addresses = server_addresses self.server_req_socket = None self.init_server_socket(server_addresses) # self = zmq.Poller() # self.register(self.server_req_socket, zmq.POLLIN) self.poller = PollingObject() self.mtool = OBCIMessageTool(message_templates) self.dns = net.DNS()
def __init__(self, server_address, handler_class, bind_and_activate=True, zmq_ctx=None, zmq_rep_addr=None): daemon_threads = True server_timeout = 45 SocketServer.TCPServer.__init__(self, server_address, handler_class, bind_and_activate) self.mtool = OBCIMessageTool(message_templates) self.pl = PollingObject() self.ctx = zmq_ctx self.rep_addr = zmq_rep_addr
def start_eeg_signal_experiment(ctx, srv_addrs, rq_message): client = OBCIClient(srv_addrs, ctx) # server_req_socket = ctx.socket(zmq.REQ) # for addr in srv_addrs: # server_req_socket.connect(addr) amp_params = {} amp_params.update(rq_message.amplifier_params['additional_params']) del rq_message.amplifier_params['additional_params'] amp_params.update(rq_message.amplifier_params) par_list = ['--peer', 'amplifier'] for par, val in amp_params.iteritems(): par_list += ['-p', par, unicode(val)] for par, val in CONFIG_DEFAULTS.iteritems(): if par not in par_list: par_list += ['-p', par, unicode(val)] overwrites = peer_cmd.peer_overwrites_pack(par_list) result = client.launch(rq_message.launch_file, None, rq_message.name, overwrites) LOGGER.info("START EEG signal! return to: " + rq_message.client_push_address) mtool = OBCIMessageTool(message_templates) to_client = ctx.socket(zmq.PUSH) to_client.connect(rq_message.client_push_address) if result is None or result.type == 'no_data': send_msg( to_client, mtool.fill_msg("rq_error", err_code="launch_failed", details="No response from server or experiment")) else: send_msg(to_client, result.raw()) LOGGER.info("sent eeg launch result" + str(result)[:500]) time.sleep(0.1)
def __init__(self, address, zmq_ctx, zmq_rep_addr): self.srv_address = address self.ctx = zmq_ctx self.zmq_rep_addr = zmq_rep_addr.replace("*", socket.gethostname()) self.long_rqs = {} for msgtype in [ "find_eeg_experiments", "find_eeg_amplifiers", #"join_experiment", "start_eeg_signal" ]: self.long_rqs[msgtype] = self._make_pull_sock() self.mtool = OBCIMessageTool(message_templates)
class ConfigServer(BaseMultiplexerServer): @log_crash def __init__(self, addresses): super(ConfigServer, self).__init__(addresses=addresses, type=peers.CONFIG_SERVER) self._configs = {} self._ext_configs = {} self._ready_peers = [] self.__to_all = False self.mtool = OBCIMessageTool(message_templates) self.launcher_sock = None params, other_params = PeerCmd().parse_cmd() self.addr = params['local_params'].get('launcher_socket_addr', '') self.exp_uuid = params['local_params'].get('experiment_uuid', '') self.log_dir = params['local_params'].get('log_dir', None) self.logger = get_logger('config_server', log_dir=self.log_dir, conn=self.conn, file_level=params['local_params'].get('file_log_level', None), mx_level=params['local_params'].get('mx_log_level', None), stream_level=params['local_params'].get('console_log_level', None)) self._old_configs = self._stored_config() self._restore_peers = params['local_params'].get('restore_peers', '').split() for peer in self._restore_peers: if peer in self._old_configs["local"]: self._configs[peer] = dict(self._old_configs["local"][peer]) self._ready_peers.append(peer) if peer in self._old_configs["ext"]: self._ext_configs[peer] = dict(self._old_configs["ext"][peer]) if self.addr != '': self.ctx = zmq.Context() self.launcher_sock = self.ctx.socket(zmq.PUSH) try: self.launcher_sock.connect(self.addr) except Exception, e: self.logger.error("failed to connect to address " +\ self.addr + " !!!") self.launcher_sock = None else: self.logger.info("OK: connected to " + self.addr) send_msg(self.launcher_sock, self.mtool.fill_msg("config_server_ready")) self.logger.info("connections count ::::::::: %s", self.conn.connections_count())
def __init__(self, preset_data=None, launcher_data=None, ctx=None): self.preset_data = preset_data self.launch_file = None self.launcher_data = launcher_data self.name = None self.info = "" self.public_params = [] self.origin_machine = '' self.unsupervised_peers = {} self.old_uid = None self.ctx = ctx self.exp_req = None self.mtool = OBCIMessageTool(message_templates) self.poller = PollingObject() self.category = DEFAULT_CATEGORY if preset_data is not None: self.setup_from_preset(preset_data) elif launcher_data is not None: self.setup_from_launcher(launcher_data) super(ExperimentEngineInfo, self).__init__()
class SubprocessMonitor(object): def __init__(self, zmq_ctx, uuid, logger=None, obci_dns=None): self._processes = {} self._ctx = zmq_ctx self.uuid = uuid self.logger = logger or get_logger('subprocess_monitor', stream_level='warning') self.obci_dns = obci_dns self._mtool = OBCIMessageTool(message_templates) self.poller = PollingObject() self._proc_lock = threading.RLock() def not_running_processes(self): status = {} with self._proc_lock: for key, proc in self._processes.iteritems(): st = proc.status() if st[0] in [FINISHED, FAILED, TERMINATED ] and not proc.marked_delete(): status[key] = st return status def unknown_status_processes(self): with self._proc_lock: return [ proc for proc in self._processes.values() if proc.status()[0] == UNKNOWN ] def process(self, machine_ip, pid): with self._proc_lock: return self._processes.get((machine_ip, pid), None) def killall(self, force=False): with self._proc_lock: for proc in self._processes.values(): kill_method = proc.kill if not force else proc.kill_with_force if proc.status()[0] not in [FINISHED, FAILED, TERMINATED]: kill_method() def delete(self, machine, pid): proc = self._processes.get((machine, pid), None) if proc is None: raise Exception("Process not found: " + str((machine, pid))) if not proc.running(): del self._processes[(machine, pid)] return True else: self.logger.error("Process is running, will not delete! " + str((machine, pid))) return False def delete_all(self): with self._proc_lock: for proc in self._processes.values(): del proc self._processes = {} def stop_monitoring(self): with self._proc_lock: for proc in self._processes.values(): proc.stop_monitoring() def _launch_args(self, path, args): # TODO fix interpreter calls // only python is supported if path.endswith('.py'): launch_args = PYTHON_CALL + [path] + args else: launch_args = [path] + args return launch_args def _stdio_actions(self, io_flags): out = subprocess.PIPE if io_flags & STDOUT else None if io_flags & STDERR: err = subprocess.PIPE elif out is not None: err = subprocess.STDOUT else: err = None stdin = subprocess.PIPE if io_flags & STDIN else None return (out, err, stdin) def _local_launch(self, launch_args, stdio_actions, env): ON_POSIX = 'posix' in sys.builtin_module_names out, err, stdin = stdio_actions try: if sys.platform == "win32": crflags = subprocess.CREATE_NEW_PROCESS_GROUP popen_obj = subprocess.Popen(launch_args, stdout=out, stderr=err, stdin=stdin, bufsize=1, close_fds=ON_POSIX, env=env, creationflags=crflags) else: popen_obj = subprocess.Popen(launch_args, stdout=out, stderr=err, stdin=stdin, bufsize=1, close_fds=ON_POSIX, env=env) details = "Popen constructor finished for " +\ str(launch_args[:3]) + "(...)" self.logger.info(details) return popen_obj, details except OSError as e: details = "Unable to spawn process {0} [{1}]".format( launch_args, e.args) self.logger.error(details) return None, details except ValueError as e: details = "Unable to spawn process (bad arguments) {0} [{1}]".format( launch_args, e.args) self.logger.error(details) return None, details except Exception as e: details = "Process launch Error: " + str(e) + str(e.args) + str( vars(e)) self.logger.error(details) return None, details def new_local_process(self, path, args, proc_type='', name='', capture_io=STDOUT | STDIN, stdout_log=None, stderr_log=None, register_timeout_desc=None, monitoring_optflags=RETURNCODE | PING, machine_ip=None, env=None): launch_args = self._launch_args(path, args) self.logger.debug(proc_type + " local path: " + path) machine = machine_ip if machine_ip else socket.gethostname() std_actions = self._stdio_actions(capture_io) timeout_desc = register_timeout_desc self.logger.debug('process launch arg list: %s', launch_args) popen_obj, details = self._local_launch(launch_args, std_actions, env) if popen_obj is None: return None, details if popen_obj.returncode is not None: det = "opened process already terminated" + popen_obj.communicate() self.logger.warning(det) if not name: name = os.path.basename(path) process_desc = ProcessDescription(proc_type=proc_type, name=name, path=path, args=args, machine_ip=machine, pid=popen_obj.pid) # io_handler will be None if no stdio is captured io_handler = start_stdio_handler(popen_obj, std_actions, ':'.join([machine, path, name]), stdout_log, stderr_log) new_proc = LocalProcess(process_desc, popen_obj, io_handler=io_handler, reg_timeout_desc=timeout_desc, monitoring_optflags=monitoring_optflags, logger=self.logger) if monitoring_optflags & PING: new_proc._ctx = self._ctx with self._proc_lock: self._processes[(machine, popen_obj.pid)] = new_proc new_proc.start_monitoring() return new_proc, None def new_remote_process(self, path, args, proc_type, name, machine_ip, conn_addr, capture_io=STDOUT | STDIN, stdout_log=None, stderr_log=None, register_timeout_desc=None, monitoring_optflags=PING): """Send a request to conn_addr for a process launch. By default the process will be monitored with ping requests and locally by the remote peer.""" timeout_desc = register_timeout_desc rq_message = self._mtool.fill_msg('launch_process', path=path, args=args, proc_type=proc_type, name=name, machine_ip=machine_ip, capture_io=capture_io, stdout_log=stdout_log, stderr_log=stderr_log) rq_sock = self._ctx.socket(zmq.REQ) try: rq_sock.connect(conn_addr) except zmq.ZMQError as e: det = "Could not connect to {0}, err: {1}, {2}".format( conn_addr, e, e.args) self.logger.error(det) return None, det self.logger.info("SENDING LAUNCH REQUEST {0} {1} {2} {3}".format( machine_ip, _DEFAULT_TIMEOUT_MS, 'ms', conn_addr)) send_msg(rq_sock, rq_message) result, details = self.poller.poll_recv(rq_sock, _DEFAULT_TIMEOUT_MS) rq_sock.close() if not result: details = details + " [address was: {0}]".format(conn_addr) self.logger.error(details) return None, details else: result = self._mtool.unpack_msg(result) if result.type == 'rq_error': det = "REQUEST FAILED" + str(result.err_code) + ':' + str( result.details) self.logger.error(det) return None, det elif result.type == 'launched_process_info': self.logger.info("REQUEST SUCCESS %s", result.dict()) process_desc = ProcessDescription(proc_type=result.proc_type, name=result.name, path=result.path, args=args, machine_ip=result.machine, pid=result.pid) new_proc = RemoteProcess(process_desc, conn_addr, reg_timeout_desc=timeout_desc, monitoring_optflags=monitoring_optflags, logger=self.logger) if monitoring_optflags & PING: new_proc._ctx = self._ctx with self._proc_lock: self._processes[(result.machine, result.pid)] = new_proc new_proc.start_monitoring() return new_proc, None
class EEGExperimentFinder(object): def __init__(self, srv_addrs, ctx, client_push_address, nearby_servers): self.ctx = ctx self.server_req_socket = self.ctx.socket(zmq.REQ) for addr in srv_addrs: self.server_req_socket.connect(addr) self.poller = PollingObject() self.mtool = OBCIMessageTool(message_templates) self.nearby_servers = nearby_servers self._amplified_cache = {} def _running_experiments(self): send_msg(self.server_req_socket, self.mtool.fill_msg("list_experiments")) exp_list, details = self.poll_recv(self.server_req_socket, 2000) if not exp_list: LOGGER.error("Connection to obci_server failed. (list_experiments)") return None exps = exp_list.exp_data running = [] for exp in exps.values(): if exp['status_name'] == launcher_tools.RUNNING or \ exp['status_name'] == launcher_tools.LAUNCHING: running.append(exp) return running def find_amplified_experiments(self): running_exps = self._running_experiments() amplified = [] for exp in running_exps: LOGGER.info("Found running experiment: " + str(exp['name'])) infos = self._info_amplified(exp) if infos is not None: print("Found experiments...", str(infos)[:500]) amplified += infos return amplified def _info_amplified(self, exp_desc): amp_options = [] LOGGER.info("Processing experiment " + str(exp_desc['name']) + "w/ addr: " + str(exp_desc['rep_addrs'])) tcp_addrs = exp_desc['tcp_addrs'] rep_addrs = net.choose_not_local(exp_desc['rep_addrs']) if not rep_addrs: rep_addrs = net.choose_local(exp_desc['rep_addrs'], ip=True) rep_addr = rep_addrs.pop() pub_addrs = net.choose_not_local(exp_desc['pub_addrs']) if not pub_addrs: pub_addrs = net.choose_local(exp_desc['pub_addrs'], ip=True) pub_addr = pub_addrs.pop() tcp_addr = tcp_addrs.pop() LOGGER.info("Chosen experiment addresses: REP -- " + str(rep_addr) + ", PUB -- " + str(pub_addr)) req_sock = self.ctx.socket(zmq.REQ) try: req_sock.connect(rep_addr) send_msg(req_sock, self.mtool.fill_msg('get_experiment_info')) res, details = self.poll_recv(req_sock, 4000) finally: req_sock.close() if not res: LOGGER.error("Connection failed (experiment " + exp_desc['name'] + "), get_experiment_info") return None exp_info = res.dict() # json.loads(res) for field in ["peers_status", "details"]: del exp_info["experiment_status"][field] peer_list = exp_info["peers"] if not self._has_mx(peer_list): LOGGER.info("Experiment " + exp_desc['name'] + " does not have a multiplexer.") return None maybe_amps = self._amp_like_peers(peer_list) if not maybe_amps: LOGGER.info("Experiment " + exp_desc['name'] + " -- no amplifier.") return None req_sock = self.ctx.socket(zmq.REQ) try: req_sock.connect(rep_addr) for peer in maybe_amps: info, params = self._get_amp_info(req_sock, peer) if not self._is_amplifier(info, params): LOGGER.info("Experiment " + exp_desc['name'] + " -- peer " + str(peer) + "is not an amplifier.") continue else: exp_data = self._create_exp_data(exp_info, info, params['param_values'], rep_addr, pub_addr, tcp_addr) amp_options.append(exp_data) finally: req_sock.close() return amp_options def _get_amp_info(self, exp_sock, peer_id): send_msg(exp_sock, self.mtool.fill_msg('get_peer_info', peer_id=peer_id)) info, details = self.poll_recv(exp_sock, 4000) send_msg(exp_sock, self.mtool.fill_msg('get_peer_param_values', peer_id=peer_id)) params, details = self.poll_recv(exp_sock, 4000) if not info or not params: LOGGER.error("get_peer_info failed " + str(peer_id) + " " + str(details)) return None, None info = info.dict() params = params.dict() for field in ["sender", "sender_ip", "receiver", "type", "local_params", "external_params", "config_sources", "launch_dependencies"]: del info[field] return info, params def _is_amplifier(self, peer_info, peer_params): info = peer_info peer_id = info['peer_id'] if not info['peer_type'] == 'obci_peer': LOGGER.info("Peer " + str(peer_id) + " not obci_peer") return False params = peer_params['param_values'] if 'channels_info' not in params or\ 'active_channels' not in params: LOGGER.info('Peer ' + str(peer_id) + " no channels_info param.") return False return True def _create_exp_data(self, exp_info, peer_info, params, rep_addr, pub_addr, tcp_addr): data = {} data['amplifier_params'] = params data['amplifier_peer_info'] = peer_info data['experiment_info'] = exp_info data['rep_addrs'] = [rep_addr] data['pub_addrs'] = [pub_addr] data['tcp_addrs'] = [tcp_addr] return data def _has_mx(self, peer_list): return [peer for peer in peer_list if peer.startswith('mx')] != [] def _amp_like_peers(self, peer_list): return [peer for peer in peer_list if peer.startswith('amplifier')] def poll_recv(self, socket, timeout): result, details = self.poller.poll_recv(socket, timeout) if result: result = self.mtool.unpack_msg(result) return result, details
#!/usr/bin/python # -*- coding: utf-8 -*- import json import zmq import sys import socket from obci.control.common.message import OBCIMessageTool, send_msg, recv_msg, PollingObject from obci.control.launcher.launcher_messages import message_templates, error_codes from obci.control.common.obci_control_settings import PORT_RANGE import obci.control.common.net_tools as net if __name__ == '__main__': mtool = OBCIMessageTool(message_templates) pl = PollingObject() # ifname = net.server_ifname() my_addr = 'tcp://' + 'localhost' ctx = zmq.Context() server_req = ctx.socket(zmq.REQ) server_req.connect(my_addr + ':' + net.server_rep_port()) exp_info_pull = ctx.socket(zmq.PULL) port = exp_info_pull.bind_to_random_port('tcp://*', min_port=PORT_RANGE[0], max_port=PORT_RANGE[1],
def message_tool(self): return OBCIMessageTool(message_templates)
#!/usr/bin/python3 import json import zmq import sys from obci.control.common.message import OBCIMessageTool, send_msg, PollingObject from obci.control.launcher.launcher_messages import message_templates from obci.control.common.obci_control_settings import PORT_RANGE import obci.control.common.net_tools as net if __name__ == '__main__': mtool = OBCIMessageTool(message_templates) pl = PollingObject() # ifname = net.server_ifname() my_addr = 'tcp://' + 'localhost' ctx = zmq.Context() server_req = ctx.socket(zmq.REQ) server_req.connect(my_addr + ':' + net.server_rep_port()) exp_info_pull = ctx.socket(zmq.PULL) port = exp_info_pull.bind_to_random_port('tcp://*', min_port=PORT_RANGE[0], max_port=PORT_RANGE[1], max_tries=500)
class SubprocessMonitor(object): def __init__(self, zmq_ctx, uuid, logger=None, obci_dns=None): self._processes = {} self._ctx = zmq_ctx self.uuid = uuid self.logger = logger or get_logger('subprocess_monitor', stream_level='warning') self.obci_dns = obci_dns self._mtool = OBCIMessageTool(message_templates) self.poller = PollingObject() self._proc_lock = threading.RLock() def not_running_processes(self): status = {} with self._proc_lock: for key, proc in self._processes.iteritems(): st = proc.status() if st[0] in [FINISHED, FAILED, TERMINATED] and not proc.marked_delete(): status[key] = st return status def unknown_status_processes(self): with self._proc_lock: return [proc for proc in self._processes.values()\ if proc.status()[0] == UNKNOWN] def process(self, machine_ip, pid): with self._proc_lock: return self._processes.get((machine_ip, pid), None) def killall(self, force=False): with self._proc_lock: for proc in self._processes.values(): kill_method = proc.kill if not force else proc.kill_with_force if proc.status()[0] not in [FINISHED, FAILED, TERMINATED]: kill_method() def delete(self, machine, pid): proc = self._processes.get((machine, pid), None) if proc is None: raise Exception("Process not found: " + str((machine, pid))) if not proc.running(): del self._processes[(machine, pid)] return True else: self.logger.error("Process is running, will not delete! " + str((machine, pid))) return False def delete_all(self): with self._proc_lock: for proc in self._processes.values(): del proc self._processes = {} def stop_monitoring(self): with self._proc_lock: for proc in self._processes.values(): proc.stop_monitoring() def _launch_args(self, path, args): #TODO fix interpreter calls // only python is supported if path.endswith('.py'): launch_args = PYTHON_CALL +[path] + args else: launch_args = [path] + args return launch_args def _stdio_actions(self, io_flags): out = subprocess.PIPE if io_flags & STDOUT else None if io_flags & STDERR: err = subprocess.PIPE elif out is not None: err = subprocess.STDOUT else: err = None stdin = subprocess.PIPE if io_flags & STDIN else None return (out, err, stdin) def _local_launch(self, launch_args, stdio_actions, env): ON_POSIX = 'posix' in sys.builtin_module_names out, err, stdin = stdio_actions try: if sys.platform == "win32": crflags = subprocess.CREATE_NEW_PROCESS_GROUP popen_obj = subprocess.Popen(launch_args, stdout=out, stderr=err, stdin=stdin, bufsize=1, close_fds=ON_POSIX, env=env, creationflags=crflags) else: popen_obj = subprocess.Popen(launch_args, stdout=out, stderr=err, stdin=stdin, bufsize=1, close_fds=ON_POSIX, env=env) details = "Popen constructor finished for " +\ str(launch_args[:3]) + "(...)" self.logger.info(details) return popen_obj, details except OSError as e: details = "Unable to spawn process {0} [{1}]".format(launch_args, e.args) self.logger.error(details) return None, details except ValueError as e: details = "Unable to spawn process (bad arguments) {0} [{1}]".format(launch_args, e.args) self.logger.error(details) return None, details except Exception as e: details = "Process launch Error: " + str(e) + str(e.args) + str(vars(e)) self.logger.error(details) return None, details def new_local_process(self, path, args, proc_type='', name='', capture_io= STDOUT | STDIN, stdout_log=None, stderr_log=None, register_timeout_desc=None, monitoring_optflags=RETURNCODE | PING, machine_ip=None, env=None): launch_args = self._launch_args(path, args) self.logger.debug(proc_type + " local path: " + path) machine = machine_ip if machine_ip else socket.gethostname() std_actions = self._stdio_actions(capture_io) timeout_desc = register_timeout_desc self.logger.debug('process launch arg list: %s', launch_args) popen_obj, details = self._local_launch(launch_args, std_actions, env) if popen_obj is None: return None, details if popen_obj.returncode is not None: det = "opened process already terminated" + popen_obj.communicate() self.logger.warning(det) if not name: name = os.path.basename(path) process_desc = ProcessDescription(proc_type=proc_type, name=name, path=path, args=args, machine_ip=machine, pid=popen_obj.pid) # io_handler will be None if no stdio is captured io_handler = start_stdio_handler(popen_obj, std_actions, ':'.join([machine, path, name]), stdout_log, stderr_log) new_proc = LocalProcess(process_desc, popen_obj, io_handler=io_handler, reg_timeout_desc=timeout_desc, monitoring_optflags=monitoring_optflags, logger=self.logger) if monitoring_optflags & PING: new_proc._ctx = self._ctx with self._proc_lock: self._processes[(machine, popen_obj.pid)] = new_proc new_proc.start_monitoring() return new_proc, None def new_remote_process(self, path, args, proc_type, name, machine_ip, conn_addr, capture_io= STDOUT | STDIN, stdout_log=None, stderr_log=None, register_timeout_desc=None, monitoring_optflags=PING): """Send a request to conn_addr for a process launch. By default the process will be monitored with ping requests and locally by the remote peer.""" timeout_desc = register_timeout_desc rq_message = self._mtool.fill_msg('launch_process', path=path, args=args, proc_type=proc_type, name=name, machine_ip=machine_ip, capture_io=capture_io, stdout_log=stdout_log, stderr_log=stderr_log) rq_sock = self._ctx.socket(zmq.REQ) try: rq_sock.connect(conn_addr) except zmq.ZMQError as e: det = "Could not connect to {0}, err: {1}, {2}".format( conn_addr, e, e.args) self.logger.error(det) return None, det self.logger.info("SENDING LAUNCH REQUEST {0} {1} {2} {3}".format( machine_ip, _DEFAULT_TIMEOUT_MS, 'ms', conn_addr)) send_msg(rq_sock, rq_message) result, details = self.poller.poll_recv(rq_sock, _DEFAULT_TIMEOUT_MS) rq_sock.close() if not result: details = details + " [address was: {0}]".format(conn_addr) self.logger.error(details) return None, details else: result = self._mtool.unpack_msg(result) if result.type == 'rq_error': det = "REQUEST FAILED" + str(result.err_code) + ':' + str(result.details) self.logger.error(det) return None, det elif result.type == 'launched_process_info': self.logger.info("REQUEST SUCCESS %s", result.dict()) process_desc = ProcessDescription(proc_type=result.proc_type, name=result.name, path=result.path, args=args, machine_ip=result.machine, pid=result.pid) new_proc = RemoteProcess(process_desc, conn_addr, reg_timeout_desc=timeout_desc, monitoring_optflags=monitoring_optflags, logger=self.logger) if monitoring_optflags & PING: new_proc._ctx = self._ctx with self._proc_lock: self._processes[(result.machine, result.pid)] = new_proc new_proc.start_monitoring() return new_proc, None
class OBCIClient(object): default_timeout = 5000 def __init__(self, server_addresses, zmq_context=None): self.ctx = zmq_context if zmq_context else zmq.Context() self.server_addresses = server_addresses self.server_req_socket = None self.init_server_socket(server_addresses) # self = zmq.Poller() # self.register(self.server_req_socket, zmq.POLLIN) self.poller = PollingObject() self.mtool = OBCIMessageTool(message_templates) self.dns = net.DNS() def init_server_socket(self, srv_addrs): if self.server_req_socket is not None: print("server socket restart") self.server_req_socket.close() self.server_req_socket = self.ctx.socket(zmq.REQ) for addr in srv_addrs: print(addr) self.server_req_socket.connect(addr) def launch(self, launch_file=None, sandbox_dir=None, name=None, overwrites=None): result = self.send_create_experiment(launch_file, sandbox_dir, name, overwrites) print("create result:", result) if not result: self.init_server_socket(self.server_addresses) return result if result.type != "experiment_created": return result print(result) machine = result.origin_machine addrs = [addr for addr in result.rep_addrs if self._addr_connectable(addr, machine)] return self.send_start_experiment(addrs) def morph(self, exp_strname, launch_file, name=None, overwrites=None, leave_on=None): response = self.get_experiment_contact(exp_strname) exp_sock = self.ctx.socket(zmq.REQ) try: if response.type == "rq_error" or response.type == "no_data": return response for addr in response.rep_addrs: exp_sock.connect(addr) msg = self.mtool.fill_msg('morph_to_new_scenario', launch_file=launch_file, name=name, overwrites=overwrites, leave_on=leave_on) send_msg(exp_sock, msg) response, details = self.poll_recv(exp_sock, 6000) return response finally: exp_sock.close() def _addr_connectable(self, addr, machine): return machine == socket.gethostname() or \ (net.is_ip(addr) and not net.addr_is_local(addr)) def start_chosen_experiment(self, exp_strname): response = self.get_experiment_contact(exp_strname) if response.type == "rq_error" or response.type == "no_data": return response return self.send_start_experiment(response.rep_addrs) def send_start_experiment(self, exp_addrs): exp_sock = self.ctx.socket(zmq.REQ) try: for addr in exp_addrs: exp_sock.connect(addr) send_msg(exp_sock, self.mtool.fill_msg("start_experiment")) reply, details = self.poll_recv(exp_sock, 20000) # print reply return reply finally: exp_sock.close() def force_kill_experiment(self, strname): pass def get_experiment_contact(self, strname): send_msg(self.server_req_socket, self.mtool.fill_msg("get_experiment_contact", strname=strname)) response, details = self.poll_recv(self.server_req_socket, self.default_timeout) return response def ping_server(self, timeout=50): send_msg(self.server_req_socket, self.mtool.fill_msg("ping")) response, details = self.poll_recv(self.server_req_socket, timeout) return response def retry_ping(self, timeout=50): response, details = self.poll_recv(self.server_req_socket, timeout) return response def send_create_experiment(self, launch_file=None, sandbox_dir=None, name=None, overwrites=None): send_msg( self.server_req_socket, self.mtool.fill_msg( "create_experiment", launch_file=launch_file, sandbox_dir=sandbox_dir, name=name, overwrites=overwrites)) response, details = self.poll_recv(self.server_req_socket, 5000) return response def send_list_experiments(self): send_msg(self.server_req_socket, self.mtool.fill_msg("list_experiments")) response, details = self.poll_recv(self.server_req_socket, 4000) return response def send_list_nearby_machines(self): send_msg(self.server_req_socket, self.mtool.fill_msg("list_nearby_machines")) response, details = self.poll_recv(self.server_req_socket, 4000) return response def get_experiment_details(self, strname, peer_id=None): response = self.get_experiment_contact(strname) if response.type == "rq_error" or response.type == "no_data": return response sock = self.ctx.socket(zmq.REQ) try: for addr in response.rep_addrs: sock.connect(addr) if peer_id: send_msg(sock, self.mtool.fill_msg("get_peer_info", peer_id=peer_id)) else: send_msg(sock, self.mtool.fill_msg("get_experiment_info")) response, details = self.poll_recv(sock, 2000) return response finally: sock.close() def configure_peer(self, exp_strname, peer_id, config_overrides, override_files=None): response = self.get_experiment_contact(exp_strname) if response.type == "rq_error" or response.type == "no_data": return response sock = self.ctx.socket(zmq.REQ) try: for addr in response.rep_addrs: sock.connect(addr) if override_files: send_msg(sock, self.mtool.fill_msg("get_peer_info", peer_id=peer_id)) response, details = self.poll_recv(sock, 2000) if response.type is 'rq_error': return response msg = self.mtool.fill_msg("update_peer_config", peer_id=peer_id, **config_overrides) send_msg(sock, msg) # print msg response, details = self.poll_recv(sock, 2000) return response finally: sock.close() # "update_peer_config" : dict(peer_id='', local_params='', # external_params='', launch_dependencies='', config_sources=''), def kill_exp(self, strname, force=False): send_msg(self.server_req_socket, self.mtool.fill_msg("kill_experiment", strname=strname)) return self.poll_recv(self.server_req_socket, 2000)[0] def srv_kill(self): send_msg(self.server_req_socket, self.mtool.fill_msg("kill")) return self.poll_recv(self.server_req_socket, 2000)[0] def join_experiment(self, strname, peer_id, path): response = self.get_experiment_contact(strname) if response.type == "rq_error" or response.type == "no_data": return response sock = self.ctx.socket(zmq.REQ) try: self._connect(sock, response.rep_addrs) send_msg(sock, self.mtool.fill_msg("join_experiment", peer_id=peer_id, path=path, peer_type='obci_peer')) response, details = self.poll_recv(sock, 5000) sock.close() return response finally: sock.close() def add_peer(self, strname, peer_id, path, machine, param_overwrites=None, custom_config_path=None, config_sources=None, launch_dependencies=None, apply_globals=True): response = self.get_experiment_contact(strname) if response.type == "rq_error" or response.type == "no_data": return response sock = self.ctx.socket(zmq.REQ) try: self._connect(sock, response.rep_addrs) send_msg(sock, self.mtool.fill_msg("add_peer", peer_id=peer_id, peer_path=path, peer_type='obci_peer', machine=machine, param_overwrites=param_overwrites, custom_config_path=custom_config_path, config_sources=config_sources, launch_dependencies=launch_dependencies, apply_globals=apply_globals)) response, details = self.poll_recv(sock, 5000) return response finally: sock.close() def kill_peer(self, exp_strname, peer_id, remove_config=False): response = self.get_experiment_contact(exp_strname) if response.type == "rq_error" or response.type == "no_data": return response try: sock = self.ctx.socket(zmq.REQ) self._connect(sock, response.rep_addrs) send_msg(sock, self.mtool.fill_msg("kill_peer", peer_id=peer_id, remove_config=remove_config)) response, details = self.poll_recv(sock, 5000) return response finally: sock.close() def _connect(self, sock, addr_list): print("**** ", addr_list) this = self._is_this_machine(addr_list) connected = False for addr in addr_list: if not this and ('localhost' in addr or '127.0.0.1' in addr): continue try: sock.connect(addr) connected = True except zmq.ZMQError as e: print(addr, " ::: ", str(e)) if not connected: raise Exception("Could not connect to any of the addresses: " + str(addr_list)) def _is_this_machine(self, addr_list): for addr in addr_list: if self.dns.is_this_machine(addr): return True return False def leave_experiment(self, strname, peer_id): response = self.get_experiment_contact(strname) if response.type == "rq_error" or response.type == "no_data": return response try: sock = self.ctx.socket(zmq.REQ) self._connect(sock, response.rep_addrs) send_msg(sock, self.mtool.fill_msg("leave_experiment", peer_id=peer_id)) response, details = self.poll_recv(sock, 5000) return response finally: sock.close() def get_tail(self, strname, peer_id, len_): response = self.get_experiment_contact(strname) if response.type == "rq_error" or response.type == "no_data": return response sock = self.ctx.socket(zmq.REQ) try: for addr in response.rep_addrs: sock.connect(addr) send_msg(sock, self.mtool.fill_msg("get_tail", peer_id=peer_id, len=len_)) response, details = self.poll_recv(sock, 4000) return response finally: sock.close() def poll_recv(self, socket, timeout): result, details = self.poller.poll_recv(socket, timeout) if result: result = self.mtool.unpack_msg(result) else: result = EmptyResponse(details) return result, details
class Process(object): def __init__(self, proc_description, reg_timeout_desc=None, monitoring_optflags=PING, logger=None): self.desc = proc_description self.must_register = reg_timeout_desc is not None self._status_lock = threading.RLock() self._status = UNKNOWN if self.must_register else RUNNING self._status_details = None self.ping_it = monitoring_optflags & PING self.check_returncode = monitoring_optflags & RETURNCODE if \ self.desc.pid is not None else False self.logger = logger or get_logger( 'subprocess_monitor'+'-'+self.desc.name+'-'+str(self.desc.pid), stream_level='info') self.set_registration_timeout_handler(reg_timeout_desc) self.registration_data = None self._stop_monitoring = False self._ping_thread = None self._ping_retries = 8 self._returncode_thread = None self._mtool = OBCIMessageTool(message_templates) self._ctx = None self.rq_sock = None self._poller = PollingObject() self.delete = False @property def machine_ip(self): return self.desc.machine_ip @property def pid(self): return self.desc.pid @property def path(self): return self.desc.path @property def proc_type(self): return self.desc.proc_type @property def name(self): return self.desc.name def status(self): with self._status_lock: return self._status, self._status_details def set_registration_timeout_handler(self, reg_timeout_desc): with self._status_lock: self._status = UNKNOWN self._status_details = None self.must_register = reg_timeout_desc is not None self.reg_timeout_desc = reg_timeout_desc self.reg_timer = None if not self.must_register else \ self.new_timer(self.reg_timeout_desc, REG_TIMER) if self.must_register: self.reg_timer.start() def is_local(self): raise NotImplementedError() def timeout_handler(self, custom_method, args, type_): self._do_handle_timeout(type_) custom_method(*args) def _do_handle_timeout(self, type_): raise NotImplementedError() def new_timer(self, tim_desc, type_): return threading.Timer(tim_desc.timeout, self.timeout_handler, [tim_desc.timeout_method, tim_desc.timeout_args, type_]) def registered(self, reg_data): if self.reg_timer is not None: self.reg_timer.cancel() self.logger.info("{0} [{1}] REGISTERED!!! {2}".format( self.name, self.proc_type, reg_data.machine_ip)) #print "ping:", self.ping_it, "ret:", self.check_returncode with self._status_lock: self._status = RUNNING #TODO validate registration data self.registration_data = reg_data self.logger.info("reg_data" + str(vars(reg_data))) if self.ping_it: if not self._ctx: self._ctx = zmq.Context() self.rq_sock = self._ctx.socket(zmq.REQ) for addr in reg_data.rep_addrs: if reg_data.machine_ip != socket.gethostname() and\ net.addr_is_local(addr): continue self.logger.debug(self.name + "connecting to " + addr) self.rq_sock.connect(addr) def stop_monitoring(self): if self.reg_timer: self.reg_timer.cancel() self.reg_timer = None self._stop_monitoring = True if self._ping_thread is not None: self.logger.info("%s, %s, %s", self.proc_type, self.name ,"Joining ping thread") self._ping_thread.join() if self._returncode_thread is not None: self.logger.info("%s %s %s", self.proc_type,self.name, "joining returncode thread") self._returncode_thread.join() self.logger.info("monitor for: %s, %s, %s", self.proc_type,self.name, " ...monitoring threads stopped.") def finished(self): finished = True if self._ping_thread is not None: finished = not self._ping_thread.is_alive() if self._returncode_thread is not None: finished = finished and not self._returncode_thread.is_alive() return finished def process_is_running(self): running = True if self._ping_thread is not None: running = self._ping_thread.is_alive() if self._returncode_thread is not None: running = running and self._returncode_thread.is_alive() return running def start_monitoring(self): if self.ping_it: self._ping_thread = threading.Thread(target=self.ping_monitor, args=()) self._ping_thread.daemon = True self._ping_thread.start() if self.check_returncode: self._returncode_thread = threading.Thread(target=self.returncode_monitor, args=()) self._returncode_thread.daemon = True self._returncode_thread.start() def ping_monitor(self): is_alive = True try: while not self._stop_monitoring and is_alive: time.sleep(2) if self.rq_sock is not None: send_msg(self.rq_sock, self._mtool.fill_msg('ping')) result = None while self._ping_retries and not result and not self._stop_monitoring: result, det = self._poller.poll_recv(socket=self.rq_sock, timeout=1500) if not result and not self._stop_monitoring: self.logger.info("%s %s %s", self.proc_type, self.name, "NO RESPONSE TO PING!") with self._status_lock: if self._status not in [FAILED, FINISHED]: self._status = NON_RESPONSIVE self._status_details = 'ping response timeout' print "status:", self._status is_alive = False finally: self.rq_sock.close(linger=0) def returncode_monitor(self): raise NotImplementedError() def kill(self): raise NotImplementedError() def mark_delete(self): with self._status_lock: self.delete = True def marked_delete(self): with self._status_lock: return self.delete
class OBCIConnection(object): """ Synchronous OBCI connection. """ def __init__(self, address): templates = launcher_messages.message_templates self.msg_factory = OBCIMessageTool(msg_templates=templates) self.netstring_codec = NetstringCodec() self.address = address self.experiment_uuid = None def close(self): self.connection.close() def open(self): self.connection = socket.create_connection(self.address, timeout=30) def send_recv(self, message_name, **kwargs): message_json = self.msg_factory.fill_msg(message_name, **kwargs) message_netstring = self.netstring_codec.encode(message_json) self.open() self.connection.send(message_netstring) response_json = self.netstring_codec.decode(self.connection) self.close() response_dict = self.msg_factory.decode_msg(response_json) return response_dict def get_amp_list(self): message = self.msg_factory.fill_msg("find_eeg_amplifiers") message = self.netstring_codec.encode(message) self.open() self.connection.send(message) response_text = self.netstring_codec.decode(self.connection) response = self.msg_factory.decode_msg(response_text) self.close() amp_list = response["amplifier_list"] return amp_list def start_eeg_signal(self, name, launch_file, amplifier_params): message = self.msg_factory.fill_msg("start_eeg_signal", launch_file=launch_file, amplifier_params=amplifier_params, name=name) message_text = self.netstring_codec.encode(message) self.open() self.connection.send(message_text) response_text = self.netstring_codec.decode(self.connection) response = self.msg_factory.decode_msg(response_text) self.close() return response["sender"] def get_experiment_contact(self, name): response = self.send_recv("get_experiment_contact", strname=name) self.experiment_uuid = response["uuid"] return self.experiment_uuid def stop_experiment(self, experiment_uuid=None): if not experiment_uuid: experiment_uuid = self.experiment_uuid message = self.msg_factory.fill_msg("kill_experiment", strname=experiment_uuid) message_text = self.netstring_codec.encode(message) self.open() self.connection.send(message_text) response_text = self.netstring_codec.decode(self.connection) self.close() response = self.msg_factory.decode_msg(response_text) return response def create_experiment(self, name="unnamed"): message = self.msg_factory.fill_msg("create_experiment", name=name) message_text = self.netstring_codec.encode(message) self.open() self.connection.send(message_text) response_text = self.netstring_codec.decode(self.connection) self.close() return self.msg_factory.decode_msg(response_text) def set_experiment_scenario(self, launch_file_path="", scenario=""): return self.send_recv("set_experiment_scenario", launch_file_path=launch_file_path, scenario=scenario) def start_experiment(self): return self.send_recv("start_experiment") def get_nearby_servers(self): response = self.send_recv("list_nearby_machines") return response
def __init__(self, address): templates = launcher_messages.message_templates self.msg_factory = OBCIMessageTool(msg_templates=templates) self.netstring_codec = NetstringCodec() self.address = address self.experiment_uuid = None
class OBCIClient(object): default_timeout = 5000 def __init__(self, server_addresses, zmq_context=None): self.ctx = zmq_context if zmq_context else zmq.Context() self.server_addresses = server_addresses self.server_req_socket = None self.init_server_socket(server_addresses) #self = zmq.Poller() #self.register(self.server_req_socket, zmq.POLLIN) self.poller = PollingObject() self.mtool = OBCIMessageTool(message_templates) self.dns = net.DNS() def init_server_socket(self, srv_addrs): if self.server_req_socket is not None: print "server socket restart" self.server_req_socket.close() self.server_req_socket = self.ctx.socket(zmq.REQ) for addr in srv_addrs: print addr self.server_req_socket.connect(addr) def launch(self, launch_file=None, sandbox_dir=None, name=None, overwrites=None): result = self.send_create_experiment(launch_file, sandbox_dir, name, overwrites) print "create result:", result if not result: self.init_server_socket(self.server_addresses) return result if result.type != "experiment_created": return result print result machine = result.origin_machine addrs = [ addr for addr in result.rep_addrs if self._addr_connectable(addr, machine) ] return self.send_start_experiment(addrs) def morph(self, exp_strname, launch_file, name=None, overwrites=None, leave_on=None): response = self.get_experiment_contact(exp_strname) exp_sock = self.ctx.socket(zmq.REQ) try: if response.type == "rq_error" or response.type == "no_data": return response for addr in response.rep_addrs: exp_sock.connect(addr) msg = self.mtool.fill_msg('morph_to_new_scenario', launch_file=launch_file, name=name, overwrites=overwrites, leave_on=leave_on) send_msg(exp_sock, msg) response, details = self.poll_recv(exp_sock, 6000) return response finally: exp_sock.close() def _addr_connectable(self, addr, machine): return machine == socket.gethostname() or \ (net.is_ip(addr) and not net.addr_is_local(addr)) def start_chosen_experiment(self, exp_strname): response = self.get_experiment_contact(exp_strname) if response.type == "rq_error" or response.type == "no_data": return response return self.send_start_experiment(response.rep_addrs) def send_start_experiment(self, exp_addrs): exp_sock = self.ctx.socket(zmq.REQ) try: for addr in exp_addrs: exp_sock.connect(addr) send_msg(exp_sock, self.mtool.fill_msg("start_experiment")) reply, details = self.poll_recv(exp_sock, 20000) #print reply return reply finally: exp_sock.close() def force_kill_experiment(self, strname): pass def get_experiment_contact(self, strname): send_msg( self.server_req_socket, self.mtool.fill_msg("get_experiment_contact", strname=strname)) response, details = self.poll_recv(self.server_req_socket, self.default_timeout) return response def ping_server(self, timeout=50): send_msg(self.server_req_socket, self.mtool.fill_msg("ping")) response, details = self.poll_recv(self.server_req_socket, timeout) return response def retry_ping(self, timeout=50): response, details = self.poll_recv(self.server_req_socket, timeout) return response def send_create_experiment(self, launch_file=None, sandbox_dir=None, name=None, overwrites=None): send_msg( self.server_req_socket, self.mtool.fill_msg("create_experiment", launch_file=launch_file, sandbox_dir=sandbox_dir, name=name, overwrites=overwrites)) response, details = self.poll_recv(self.server_req_socket, 5000) return response def send_list_experiments(self): send_msg(self.server_req_socket, self.mtool.fill_msg("list_experiments")) response, details = self.poll_recv(self.server_req_socket, 4000) return response def send_list_nearby_machines(self): send_msg(self.server_req_socket, self.mtool.fill_msg("list_nearby_machines")) response, details = self.poll_recv(self.server_req_socket, 4000) return response def get_experiment_details(self, strname, peer_id=None): response = self.get_experiment_contact(strname) if response.type == "rq_error" or response.type == "no_data": return response sock = self.ctx.socket(zmq.REQ) try: for addr in response.rep_addrs: sock.connect(addr) if peer_id: send_msg(sock, self.mtool.fill_msg("get_peer_info", peer_id=peer_id)) else: send_msg(sock, self.mtool.fill_msg("get_experiment_info")) response, details = self.poll_recv(sock, 2000) return response finally: sock.close() def configure_peer(self, exp_strname, peer_id, config_overrides, override_files=None): response = self.get_experiment_contact(exp_strname) if response.type == "rq_error" or response.type == "no_data": return response sock = self.ctx.socket(zmq.REQ) try: for addr in response.rep_addrs: sock.connect(addr) if override_files: send_msg(sock, self.mtool.fill_msg("get_peer_info", peer_id=peer_id)) response, details = self.poll_recv(sock, 2000) if response.type is 'rq_error': return response msg = self.mtool.fill_msg("update_peer_config", peer_id=peer_id, **config_overrides) send_msg(sock, msg) #print msg response, details = self.poll_recv(sock, 2000) return response finally: sock.close() # "update_peer_config" : dict(peer_id='', local_params='', # external_params='', launch_dependencies='', config_sources=''), def kill_exp(self, strname, force=False): send_msg(self.server_req_socket, self.mtool.fill_msg("kill_experiment", strname=strname)) return self.poll_recv(self.server_req_socket, 2000)[0] def srv_kill(self): send_msg(self.server_req_socket, self.mtool.fill_msg("kill")) return self.poll_recv(self.server_req_socket, 2000)[0] def join_experiment(self, strname, peer_id, path): response = self.get_experiment_contact(strname) if response.type == "rq_error" or response.type == "no_data": return response sock = self.ctx.socket(zmq.REQ) try: self._connect(sock, response.rep_addrs) send_msg( sock, self.mtool.fill_msg("join_experiment", peer_id=peer_id, path=path, peer_type='obci_peer')) response, details = self.poll_recv(sock, 5000) sock.close() return response finally: sock.close() def add_peer(self, strname, peer_id, path, machine, param_overwrites=None, custom_config_path=None, config_sources=None, launch_dependencies=None, apply_globals=True): response = self.get_experiment_contact(strname) if response.type == "rq_error" or response.type == "no_data": return response sock = self.ctx.socket(zmq.REQ) try: self._connect(sock, response.rep_addrs) send_msg( sock, self.mtool.fill_msg("add_peer", peer_id=peer_id, peer_path=path, peer_type='obci_peer', machine=machine, param_overwrites=param_overwrites, custom_config_path=custom_config_path, config_sources=config_sources, launch_dependencies=launch_dependencies, apply_globals=apply_globals)) response, details = self.poll_recv(sock, 5000) return response finally: sock.close() def kill_peer(self, exp_strname, peer_id, remove_config=False): response = self.get_experiment_contact(exp_strname) if response.type == "rq_error" or response.type == "no_data": return response try: sock = self.ctx.socket(zmq.REQ) self._connect(sock, response.rep_addrs) send_msg( sock, self.mtool.fill_msg("kill_peer", peer_id=peer_id, remove_config=remove_config)) response, details = self.poll_recv(sock, 5000) return response finally: sock.close() def _connect(self, sock, addr_list): print "**** ", addr_list this = self._is_this_machine(addr_list) connected = False for addr in addr_list: if not this and ('localhost' in addr or '127.0.0.1' in addr): continue try: sock.connect(addr) connected = True except zmq.ZMQError, e: print addr, " ::: ", str(e) if not connected: raise Exception("Could not connect to any of the addresses: " + str(addr_list))
class Process(object): def __init__(self, proc_description, reg_timeout_desc=None, monitoring_optflags=PING, logger=None): self.desc = proc_description self.must_register = reg_timeout_desc is not None self._status_lock = threading.RLock() self._status = UNKNOWN if self.must_register else RUNNING self._status_details = None self.ping_it = monitoring_optflags & PING self.check_returncode = monitoring_optflags & RETURNCODE if \ self.desc.pid is not None else False self.logger = logger or get_logger( 'subprocess_monitor' + '-' + self.desc.name + '-' + str(self.desc.pid), stream_level='info') self.set_registration_timeout_handler(reg_timeout_desc) self.registration_data = None self._stop_monitoring = False self._ping_thread = None self._ping_retries = 8 self._returncode_thread = None self._mtool = OBCIMessageTool(message_templates) self._ctx = None self.rq_sock = None self._poller = PollingObject() self.delete = False @property def machine_ip(self): return self.desc.machine_ip @property def pid(self): return self.desc.pid @property def path(self): return self.desc.path @property def proc_type(self): return self.desc.proc_type @property def name(self): return self.desc.name def status(self): with self._status_lock: return self._status, self._status_details def set_registration_timeout_handler(self, reg_timeout_desc): with self._status_lock: self._status = UNKNOWN self._status_details = None self.must_register = reg_timeout_desc is not None self.reg_timeout_desc = reg_timeout_desc self.reg_timer = None if not self.must_register else \ self.new_timer(self.reg_timeout_desc, REG_TIMER) if self.must_register: self.reg_timer.start() def is_local(self): raise NotImplementedError() def timeout_handler(self, custom_method, args, type_): self._do_handle_timeout(type_) custom_method(*args) def _do_handle_timeout(self, type_): raise NotImplementedError() def new_timer(self, tim_desc, type_): return threading.Timer( tim_desc.timeout, self.timeout_handler, [tim_desc.timeout_method, tim_desc.timeout_args, type_]) def registered(self, reg_data): if self.reg_timer is not None: self.reg_timer.cancel() self.logger.info("{0} [{1}] REGISTERED!!! {2}".format( self.name, self.proc_type, reg_data.machine_ip)) # print "ping:", self.ping_it, "ret:", self.check_returncode with self._status_lock: self._status = RUNNING # TODO validate registration data self.registration_data = reg_data self.logger.info("reg_data" + str(vars(reg_data))) if self.ping_it: if not self._ctx: self._ctx = zmq.Context() self.rq_sock = self._ctx.socket(zmq.REQ) for addr in reg_data.rep_addrs: if reg_data.machine_ip != socket.gethostname() and\ net.addr_is_local(addr): continue self.logger.debug(self.name + "connecting to " + addr) self.rq_sock.connect(addr) def stop_monitoring(self): if self.reg_timer: self.reg_timer.cancel() self.reg_timer = None self._stop_monitoring = True if self._ping_thread is not None: self.logger.info("%s, %s, %s", self.proc_type, self.name, "Joining ping thread") self._ping_thread.join() if self._returncode_thread is not None: self.logger.info("%s %s %s", self.proc_type, self.name, "joining returncode thread") self._returncode_thread.join() self.logger.info("monitor for: %s, %s, %s", self.proc_type, self.name, " ...monitoring threads stopped.") def finished(self): finished = True if self._ping_thread is not None: finished = not self._ping_thread.is_alive() if self._returncode_thread is not None: finished = finished and not self._returncode_thread.is_alive() return finished def process_is_running(self): running = True if self._ping_thread is not None: running = self._ping_thread.is_alive() if self._returncode_thread is not None: running = running and self._returncode_thread.is_alive() return running def start_monitoring(self): if self.ping_it: self._ping_thread = threading.Thread(target=self.ping_monitor, args=()) self._ping_thread.daemon = True self._ping_thread.start() if self.check_returncode: self._returncode_thread = threading.Thread( target=self.returncode_monitor, args=()) self._returncode_thread.daemon = True self._returncode_thread.start() def ping_monitor(self): is_alive = True try: while not self._stop_monitoring and is_alive: time.sleep(2) if self.rq_sock is not None: send_msg(self.rq_sock, self._mtool.fill_msg('ping')) result = None while self._ping_retries and not result and not self._stop_monitoring: result, det = self._poller.poll_recv( socket=self.rq_sock, timeout=1500) if not result and not self._stop_monitoring: self.logger.info("%s %s %s", self.proc_type, self.name, "NO RESPONSE TO PING!") with self._status_lock: if self._status not in [FAILED, FINISHED]: self._status = NON_RESPONSIVE self._status_details = 'ping response timeout' print "status:", self._status is_alive = False finally: if self.rq_sock is not None: self.rq_sock.close(linger=0) def returncode_monitor(self): raise NotImplementedError() def kill(self): raise NotImplementedError() def mark_delete(self): with self._status_lock: self.delete = True def marked_delete(self): with self._status_lock: return self.delete
class ExperimentEngineInfo(QtCore.QObject): exp_saver_msg = QtCore.pyqtSignal(object) def __init__(self, preset_data=None, launcher_data=None, ctx=None): self.preset_data = preset_data self.launch_file = None self.launcher_data = launcher_data self.name = None self.info = "" self.public_params = [] self.origin_machine = '' self.unsupervised_peers = {} self.old_uid = None self.ctx = ctx self.exp_req = None self.mtool = OBCIMessageTool(message_templates) self.poller = PollingObject() self.category = DEFAULT_CATEGORY if preset_data is not None: self.setup_from_preset(preset_data) elif launcher_data is not None: self.setup_from_launcher(launcher_data) super(ExperimentEngineInfo, self).__init__() def cleanup(self): if self.exp_req: self.exp_req.close() # linger=0) def setup_from_preset(self, preset_data, launcher=False): self.preset_data = preset_data self.overwrites = {} self.runtime_changes = {} self.status = launcher_tools.ExperimentStatus() self.exp_config = system_config.OBCIExperimentConfig() self.name = preset_data['name'] self.launch_file = preset_data['launch_file'] self.info = preset_data['info'] self.public_params = [p.strip() for p in preset_data['public_params'].split(',')] self.exp_config.uuid = self.name + '--' + self.launch_file self.category = preset_data['category'] result, details = self._make_config() self.status.details = details self._set_public_params() def _addr_connectable(self, addr, machine): return machine == socket.gethostname() or \ (net.is_ip(addr) and not net.addr_is_local(addr)) def setup_from_launcher(self, launcher_data, preset=False, transform=False): self.launcher_data = launcher_data self.runtime_changes = {} if preset: self.old_uid = self.exp_config.uuid if not preset or transform: self.overwrites = {} self.status = launcher_tools.ExperimentStatus() self.exp_config = system_config.OBCIExperimentConfig() self.name = launcher_data['name'] # if not preset else self.old_uid if not preset: self.launch_file = launcher_data['launch_file_path'] connected = False if not transform: self.ctx = self.ctx if self.ctx is not None else zmq.Context() self.exp_req = self.ctx.socket(zmq.REQ) machine = launcher_data['origin_machine'] for addr in launcher_data['rep_addrs']: if self._addr_connectable(addr, machine): try: self.exp_req.connect(addr) except Exception as e: print(addr, False) else: connected = True if not connected: print("Connection to experiment ", self.name, "UNSUCCESFUL!!!!!!") return self.exp_config.uuid = launcher_data['uuid'] self.exp_config.origin_machine = launcher_data['origin_machine'] self.uuid = self.exp_config.uuid self.exp_config.launch_file_path = self.launch_file result, details = self._get_experiment_scenario() self.exp_config.status(self.status) self.status.set_status(launcher_data['status_name'], details=launcher_data['details']) self._get_experiment_details() self._set_public_params() def update_scenario(self, launch_file_path, scenario): self.exp_config.launch_file_path = launch_file_path self._process_experiment_scenario(scenario) self.exp_config.status(self.status) def _set_public_params(self): for par in self.public_params: if len(par.split('.')) == 2: [peer, param] = par.split('.') self.exp_config.peers[peer].public_params.append(param) def _make_config(self): self.exp_config.launch_file_path = self.launch_file self.uuid = self.exp_config.uuid result, details = self.make_experiment_config() self.exp_config.status(self.status) return result, details # FIXME !!! copy-paste from obci_experiment def make_experiment_config(self): launch_parser = launch_file_parser.LaunchFileParser( launcher_tools.obci_root(), settings.DEFAULT_SCENARIO_DIR) if not self.launch_file: return False, "Empty scenario." try: with open(launcher_tools.expand_path(self.launch_file)) as f: launch_parser.parse(f, self.exp_config, apply_globals=True) except Exception as e: self.status.set_status(launcher_tools.NOT_READY, details=str(e)) print("config errror ", str(e)) return False, str(e) rd, details = self.exp_config.config_ready() if rd: self.status.set_status(launcher_tools.READY_TO_LAUNCH) else: self.status.set_status(launcher_tools.NOT_READY, details=details) print(rd, details) return True, None def _get_experiment_scenario(self): if not self.exp_req: return False, "No experiment socket" response = self.comm_exp(self.mtool.fill_msg("get_experiment_scenario")) if not response: return False, "No response from experient" print("GOT SCENARIO", response.scenario) return self._process_experiment_scenario(response.scenario) def _process_experiment_scenario(self, json_scenario): jsonpar = launch_file_parser.LaunchJSONParser( launcher_tools.obci_root(), settings.DEFAULT_SCENARIO_DIR) inbuf = io.StringIO(json_scenario) jsonpar.parse(inbuf, self.exp_config) print("MY PEEEEERS:", self.exp_config.peers.keys()) rd, details = self.exp_config.config_ready() if rd: self.status.set_status(launcher_tools.READY_TO_LAUNCH) else: self.status.set_status(launcher_tools.NOT_READY, details=details) print(rd, details) return True, None def _get_experiment_details(self): if not self.exp_req: return exp_msg = self.comm_exp(self.mtool.fill_msg("get_experiment_info")) if not exp_msg: return self.origin_machine = exp_msg.origin_machine for peer, short_info in exp_msg.peers.items(): # self.exp_config.set_peer_machine(peer, short_info['machine']) msg = self.comm_exp(self.mtool.fill_msg("get_peer_info", peer_id=peer)) if not msg: return ext_defs = {} for name, defi in msg.external_params.items(): ext_defs[name] = defi[0] + '.' + defi[1] self.exp_config.update_peer_config(peer, dict(config_sources=msg.config_sources, launch_dependencies=msg.launch_dependencies, local_params=msg.local_params, external_params=ext_defs)) for peer, status in exp_msg.experiment_status['peers_status'].items(): self.status.peer_status(peer).set_status( status['status_name'], details=status['details']) def parameters(self, peer_id, mode): params = {} peer = self.exp_config.peers[peer_id] if mode == MODE_BASIC: for par in peer.public_params: params[par] = (self.exp_config.param_value(peer_id, par), None) else: params = peer.config.local_params for param in peer.config.local_params: params[param] = (self.exp_config.param_value(peer_id, param), None) for param, defi in peer.config.ext_param_defs.items(): source_symbol = defi[0] source = peer.config.config_sources[source_symbol] params[param] = (self.exp_config.param_value(peer_id, param), source + '.' + defi[1]) return params def comm_exp(self, msg): send_msg(self.exp_req, msg) response, _ = self.poller.poll_recv(self.exp_req, timeout=3000) if not response: print("!!!!!!!!!!!!!!!!!!!!!!!!!!!1 no response to ", msg) self.exp_req.close() self.exp_req = self.ctx.socket(zmq.REQ) for addr in self.launcher_data['rep_addrs']: if self._addr_connectable(addr, self.launcher_data['origin_machine']): self.exp_req.connect(addr) return None return self.mtool.unpack_msg(response) def updatable(self, peer_id, config_part, **kwargs): return False def update_peer_param(self, peer_id, param, value, runtime=False): changes = self.overwrites if not runtime else self.runtime_changes ovr = changes.get(peer_id, None) ovr = ovr if ovr is not None else {} if param not in ovr: old = self.exp_config.param_value(peer_id, param) if old != value: ovr[param] = old changes[peer_id] = ovr self.exp_config.update_local_param(peer_id, param, value) def get_launch_args(self): d = dict(launch_file=self.launch_file, name=self.name) args = ['--ovr'] if self.overwrites: for peer_id in self.overwrites: args.append('--peer') args.append(peer_id) for arg in self.overwrites[peer_id]: args += ['-p', arg, self.exp_config.param_value(peer_id, arg)] pack = peer_cmd.peer_overwrites_pack(args) d['overwrites'] = pack print("overwrites pack!!!!!!!!!!!!!!!!!!!!! ", pack) return d def peer_info(self, peer_id): return self.exp_config.peers[peer_id] def add_peer(self, peer_id, peer_path, config_sources=None, launch_deps=None, custom_config_path=None, param_overwrites=None, machine=None): return launch_file_parser.extend_experiment_config( self.exp_config, peer_id, peer_path, config_sources, launch_deps, custom_config_path, param_overwrites, machine, apply_globals=True) def enable_signal_storing(self, store_options): if not store_options: return if int(store_options['append_timestamp']): store_options = dict(store_options) store_options['save_file_name'] = store_options['save_file_name'] + "_" + str(time.time()) for peer, peer_path in SIGNAL_STORAGE_PEERS.items(): if peer not in self.exp_config.peers: self.add_peer(peer, peer_path) saver = self.exp_config.peers['signal_saver'] params = saver.config.param_values for opt, val in store_options.items(): if opt in saver.config.param_values: params[opt] = val def stop_storing(self, client): join_response = client.join_experiment(self.uuid, "dummy_module_" + str(time.time()), "") if join_response is None: print("experiment engine info - ERROR - connection timeout on stop signal storing!") return if not join_response.type == "rq_ok": print("experiment engine info - ERROR - join error on stop signal storing!") return mx_addr = join_response.params["mx_addr"].split(':') # hang and wait ... acquisition_helper.finish_saving([(mx_addr[0], int(mx_addr[1]))])
class EEGExperimentFinder(object): def __init__(self, srv_addrs, ctx, client_push_address, nearby_servers): self.ctx = ctx self.server_req_socket = self.ctx.socket(zmq.REQ) for addr in srv_addrs: self.server_req_socket.connect(addr) self.poller = PollingObject() self.mtool = OBCIMessageTool(message_templates) self.nearby_servers = nearby_servers self._amplified_cache = {} def _running_experiments(self): send_msg(self.server_req_socket, self.mtool.fill_msg("list_experiments")) exp_list, details = self.poll_recv(self.server_req_socket, 2000) if not exp_list: LOGGER.error( "Connection to obci_server failed. (list_experiments)") return None exps = exp_list.exp_data running = [] for exp in exps.values(): if exp['status_name'] == launcher_tools.RUNNING or \ exp['status_name'] == launcher_tools.LAUNCHING: running.append(exp) return running def find_amplified_experiments(self): running_exps = self._running_experiments() amplified = [] for exp in running_exps: LOGGER.info("Found running experiment: " + str(exp['name'])) infos = self._info_amplified(exp) if infos is not None: print "Found experiments...", str(infos)[:500] amplified += infos return amplified def _info_amplified(self, exp_desc): amp_options = [] LOGGER.info("Processing experiment "+ str(exp_desc['name']) +\ "w/ addr: " + str(exp_desc['rep_addrs'])) tcp_addrs = exp_desc['tcp_addrs'] rep_addrs = net.choose_not_local(exp_desc['rep_addrs']) if not rep_addrs: rep_addrs = net.choose_local(exp_desc['rep_addrs'], ip=True) rep_addr = rep_addrs.pop() pub_addrs = net.choose_not_local(exp_desc['pub_addrs']) if not pub_addrs: pub_addrs = net.choose_local(exp_desc['pub_addrs'], ip=True) pub_addr = pub_addrs.pop() tcp_addr = tcp_addrs.pop() LOGGER.info("Chosen experiment addresses: REP -- " + \ str(rep_addr) + ", PUB -- " + str(pub_addr)) req_sock = self.ctx.socket(zmq.REQ) try: req_sock.connect(rep_addr) send_msg(req_sock, self.mtool.fill_msg('get_experiment_info')) res, details = self.poll_recv(req_sock, 4000) finally: req_sock.close() if not res: LOGGER.error("Connection failed (experiment " + exp_desc['name'] + \ "), get_experiment_info") return None exp_info = res.dict() #json.loads(res) for field in ["peers_status", "details"]: del exp_info["experiment_status"][field] peer_list = exp_info["peers"] if not self._has_mx(peer_list): LOGGER.info("Experiment " + exp_desc['name'] + \ " does not have a multiplexer.") return None maybe_amps = self._amp_like_peers(peer_list) if not maybe_amps: LOGGER.info("Experiment "+ exp_desc['name'] + \ " -- no amplifier.") return None req_sock = self.ctx.socket(zmq.REQ) try: req_sock.connect(rep_addr) for peer in maybe_amps: info, params = self._get_amp_info(req_sock, peer) if not self._is_amplifier(info, params): LOGGER.info("Experiment "+ exp_desc['name'] + \ " -- peer " + str(peer) + "is not an amplifier.") continue else: exp_data = self._create_exp_data(exp_info, info, params['param_values'], rep_addr, pub_addr, tcp_addr) amp_options.append(exp_data) finally: req_sock.close() return amp_options def _get_amp_info(self, exp_sock, peer_id): send_msg(exp_sock, self.mtool.fill_msg('get_peer_info', peer_id=peer_id)) info, details = self.poll_recv(exp_sock, 4000) send_msg(exp_sock, self.mtool.fill_msg('get_peer_param_values', peer_id=peer_id)) params, details = self.poll_recv(exp_sock, 4000) if not info or not params: LOGGER.error("get_peer_info failed " + str(peer_id) + " " + str(details)) return None, None info = info.dict() params = params.dict() for field in [ "sender", "sender_ip", "receiver", "type", "local_params", "external_params", "config_sources", "launch_dependencies" ]: del info[field] return info, params def _is_amplifier(self, peer_info, peer_params): info = peer_info peer_id = info['peer_id'] if not info['peer_type'] == 'obci_peer': LOGGER.info("Peer " + str(peer_id) + " not obci_peer") return False params = peer_params['param_values'] if not 'channels_info' in params or\ not 'active_channels' in params: LOGGER.info('Peer ' + str(peer_id) + " no channels_info param.") return False return True def _create_exp_data(self, exp_info, peer_info, params, rep_addr, pub_addr, tcp_addr): data = {} data['amplifier_params'] = params data['amplifier_peer_info'] = peer_info data['experiment_info'] = exp_info data['rep_addrs'] = [rep_addr] data['pub_addrs'] = [pub_addr] data['tcp_addrs'] = [tcp_addr] return data def _has_mx(self, peer_list): return [peer for peer in peer_list if peer.startswith('mx')] != [] def _amp_like_peers(self, peer_list): return [peer for peer in peer_list if peer.startswith('amplifier')] def poll_recv(self, socket, timeout): result, details = self.poller.poll_recv(socket, timeout) if result: result = self.mtool.unpack_msg(result) return result, details
#!/usr/bin/python # -*- coding: utf-8 -*- import json import socket import time from obci.control.common.message import OBCIMessageTool import obci.control.common.net_tools as net from obci.control.launcher.launcher_messages import message_templates from obci.control.launcher.plain_tcp_handling import make_unicode_netstring, recv_unicode_netstring mtool = OBCIMessageTool(message_templates) def send_and_receive(host, port, msg): netstr = make_unicode_netstring(msg) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) wfile = None rfile = None rec_msg = None try: # Connect to server and send data sock.connect((host, port)) rfile = sock.makefile('rb', -1) wfile = sock.makefile('wb', 0) wfile.write(netstr)