Beispiel #1
0
def broadcast_server(server_uuid, rep_port, pub_port, bcast_port):
    mtool = OBCIMessageTool(message_templates)

    str_msg = mtool.fill_msg("server_broadcast", sender_ip=socket.gethostname(), sender=server_uuid,
                             rep_port=rep_port, pub_port=pub_port)
    str_msg += b'\n'

    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.bind(('', 0))
    s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)

    # introduction
    for i in range(10):
        try:
            s.sendto(str_msg, ('<broadcast>', bcast_port))
        except socket.error as e:
            pass
        time.sleep(0.3)

    # updates
    while True:
        try:
            s.sendto(str_msg, ('<broadcast>', bcast_port))
        except socket.error as e:
            print("[obci_server] Cannot broadcast obci_server, will try again in 1min:", str(e))
            time.sleep(53)
        time.sleep(7)

    s.close()
def start_eeg_signal_experiment(ctx, srv_addrs, rq_message):
    client = OBCIClient(srv_addrs, ctx)
    # server_req_socket = ctx.socket(zmq.REQ)
    #     for addr in srv_addrs:
    #         server_req_socket.connect(addr)

    amp_params = {}
    amp_params.update(rq_message.amplifier_params['additional_params'])
    del rq_message.amplifier_params['additional_params']
    amp_params.update(rq_message.amplifier_params)

    par_list = ['--peer', 'amplifier']
    for par, val in amp_params.iteritems():
        par_list += ['-p', par, unicode(val)]

    overwrites = peer_cmd.peer_overwrites_pack(par_list)
    result = client.launch(rq_message.launch_file, None, rq_message.name, overwrites)


    LOGGER.info("START EEG signal! return to:  " + rq_message.client_push_address)
    mtool = OBCIMessageTool(message_templates)
    to_client = ctx.socket(zmq.PUSH)
    to_client.connect(rq_message.client_push_address)
    if result is None:
        send_msg(to_client, mtool.fill_msg("rq_error", err_code="launch_failed",
                    details="No response from server or experiment"))
    else:
        send_msg(to_client, result.raw())
    LOGGER.info("sent eeg launch result"  + str(result)[:500] )
    time.sleep(0.1)
Beispiel #3
0
def find_new_experiments_and_push_results(ctx, rq_message):
    LOGGER = logger.get_logger("eeg_AMPLIFIER_finder", "info")

    if not rq_message.amplifier_types:
        LOGGER.info("AMPLIFIER TYPES NOT SET, FINDING ALL...")
        driv = find_drivers()
    else:
        driv = []
        for amptype in rq_message.amplifier_types:

            if amptype == 'bt' or amptype == 'bluetooth':
                driv += find_bluetooth_amps()
            elif amptype == 'usb':
                driv += find_usb_amps()
            elif amptype == 'virtual':
                driv += find_virtual_amps()

    LOGGER.info("amplifiers! return to:  " + rq_message.client_push_address)
    mtool = OBCIMessageTool(message_templates)
    to_client = ctx.socket(zmq.PUSH)
    to_client.connect(rq_message.client_push_address)

    send_msg(to_client, mtool.fill_msg('eeg_amplifiers', sender_ip=socket.gethostname(),
                                       amplifier_list=driv))
    LOGGER.info("sent amplifier data... " + str(driv)[:500] + ' [...]')
    time.sleep(0.1)
Beispiel #4
0
    def __init__(self,
                 proc_description,
                 reg_timeout_desc=None,
                 monitoring_optflags=PING,
                 logger=None):

        self.desc = proc_description

        self.must_register = reg_timeout_desc is not None
        self._status_lock = threading.RLock()
        self._status = UNKNOWN if self.must_register else RUNNING
        self._status_details = None

        self.ping_it = monitoring_optflags & PING
        self.check_returncode = monitoring_optflags & RETURNCODE if \
            self.desc.pid is not None else False

        self.logger = logger or get_logger(
            'subprocess_monitor' + '-' + self.desc.name + '-' +
            str(self.desc.pid),
            stream_level='info')
        self.set_registration_timeout_handler(reg_timeout_desc)
        self.registration_data = None

        self._stop_monitoring = False
        self._ping_thread = None
        self._ping_retries = 8
        self._returncode_thread = None
        self._mtool = OBCIMessageTool(message_templates)
        self._ctx = None
        self.rq_sock = None
        self._poller = PollingObject()

        self.delete = False
def find_new_experiments_and_push_results(ctx, rq_message):
    LOGGER = logger.get_logger("eeg_AMPLIFIER_finder", "info")

    if not rq_message.amplifier_types:
        LOGGER.info("AMPLIFIER TYPES NOT SET, FINDING ALL...")
        driv = find_drivers()
    else:
        driv = []
        for amptype in rq_message.amplifier_types:

            if amptype == 'bt' or amptype == 'bluetooth':
                driv += find_bluetooth_amps()
            elif amptype == 'usb':
                driv += find_usb_amps()
            elif amptype == 'virtual':
                driv += find_virtual_amps()

    LOGGER.info("amplifiers! return to:  " + rq_message.client_push_address)
    mtool = OBCIMessageTool(message_templates)
    to_client = ctx.socket(zmq.PUSH)
    to_client.connect(rq_message.client_push_address)

    send_msg(
        to_client,
        mtool.fill_msg('eeg_amplifiers',
                       sender_ip=socket.gethostname(),
                       amplifier_list=driv))
    LOGGER.info("sent amplifier data... " + str(driv)[:500] + ' [...]')
    time.sleep(0.1)
def _gather_other_server_results(ctx, this_addr, ip_list):
    exps = []
    if not ip_list:
        return exps
    mtool = OBCIMessageTool(message_templates)
    other_exps_pull = ctx.socket(zmq.PULL)
    port = other_exps_pull.bind_to_random_port('tcp://*',
                                               min_port=PORT_RANGE[0],
                                               max_port=PORT_RANGE[1],
                                               max_tries=500)

    my_push_addr = this_addr + ':' + str(port)
    LOGGER.info("my exp_data pull: " + my_push_addr)

    harvester = zmq.Poller()
    harvester.register(other_exps_pull)

    reqs = {}
    for srv_ip in ip_list:

        addr = 'tcp://' + srv_ip + ':' + net.server_rep_port()
        req = ctx.socket(zmq.REQ)
        req.connect(addr)
        send_msg(
            req,
            mtool.fill_msg('find_eeg_experiments',
                           client_push_address='tcp://' + my_push_addr,
                           checked_srvs=[this_addr]))
        harvester.register(req, zmq.POLLIN)
        reqs[req] = addr

    srv_responses = 0

    for i in range(len(reqs) * 50):
        socks = dict(harvester.poll(timeout=300))

        for req in socks:
            msg = recv_msg(req)
            msg = mtool.unpack_msg(msg)

            if msg.type == 'rq_ok':
                LOGGER.info("waiting for experiments from server: " +
                            str(reqs[req]))
                harvester.unregister(req)
                req.close()

            elif msg.type == 'eeg_experiments':
                LOGGER.info("GOT EXPERIMENTS from: " + msg.sender_ip)
                exps += msg.experiment_list
                srv_responses += 1
            else:
                LOGGER.warning('strange msg:  ' + str(msg))
        if srv_responses == len(ip_list):
            print "GOT ALL eeg_experiment RESPONSES :-) [addr list:  ", ip_list, "]"
            break

    other_exps_pull.close()
    return exps
Beispiel #7
0
def update_nearby_servers(srv_data,
                          bcast_port,
                          ctx=None,
                          update_push_addr=None):
    mtool = OBCIMessageTool(message_templates)

    loops_to_update = _LOOPS

    s = socket(AF_INET, SOCK_DGRAM)
    s.bind(('', bcast_port))

    notify_sock = None
    if update_push_addr is not None:
        ctx = ctx if ctx else zmq.Context()
        notify_sock = ctx.socket(zmq.PUSH)
        notify_sock.connect(update_push_addr)

    while 1:
        changed = False
        try:
            inp, out, exc = select.select([s], [], [],
                                          UPDATE_INTERVAL / _LOOPS)
        except Exception, e:
            srv_data.logger.critical("nearby_servers_update - exception: %s",
                                     str(e))
            srv_data.logger.critical("nearby_servers - aborting")
            return

        if s in inp:
            data, wherefrom = s.recvfrom(1500, 0)
            # sys.stderr.write(repr(wherefrom) + '\n')
            # sys.stdout.write(data)
            msg = unicode(data, encoding='utf-8')
            msg = msg[:-1]
            message = mtool.unpack_msg(msg)
            changed = srv_data.update(ip=wherefrom[0],
                                      hostname=message.sender_ip,
                                      uuid=message.sender,
                                      rep_port=message.rep_port,
                                      pub_port=message.pub_port)

        else:
            # print "no data"
            pass

        loops_to_update -= 1

        if loops_to_update == 0:
            loops_to_update = _LOOPS
            changed = srv_data.clean_silent()

        if changed:
            send_msg(
                notify_sock,
                mtool.fill_msg('nearby_machines',
                               nearby_machines=srv_data.dict_snapshot()))
    def __init__(self, srv_addrs, ctx, client_push_address, nearby_servers):
        self.ctx = ctx
        self.server_req_socket = self.ctx.socket(zmq.REQ)
        for addr in srv_addrs:
            self.server_req_socket.connect(addr)

        self.poller = PollingObject()
        self.mtool = OBCIMessageTool(message_templates)
        self.nearby_servers = nearby_servers
        self._amplified_cache = {}
Beispiel #9
0
 def __init__(self, zmq_ctx, uuid, logger=None, obci_dns=None):
     self._processes = {}
     self._ctx = zmq_ctx
     self.uuid = uuid
     self.logger = logger or get_logger('subprocess_monitor',
                                        stream_level='warning')
     self.obci_dns = obci_dns
     self._mtool = OBCIMessageTool(message_templates)
     self.poller = PollingObject()
     self._proc_lock = threading.RLock()
Beispiel #10
0
def _gather_other_server_results(ctx, this_addr, ip_list):
    exps = []
    if not ip_list:
        return exps
    mtool = OBCIMessageTool(message_templates)
    other_exps_pull = ctx.socket(zmq.PULL)
    port = other_exps_pull.bind_to_random_port('tcp://*',
                                               min_port=PORT_RANGE[0],
                                               max_port=PORT_RANGE[1], max_tries=500)

    my_push_addr = this_addr + ':' + str(port)
    LOGGER.info("my exp_data pull: " + my_push_addr)

    harvester = zmq.Poller()
    harvester.register(other_exps_pull)

    reqs = {}
    for srv_ip in ip_list:

        addr = 'tcp://' + srv_ip + ':' + net.server_rep_port()
        req = ctx.socket(zmq.REQ)
        req.connect(addr)
        send_msg(req, mtool.fill_msg('find_eeg_experiments',
                                     client_push_address='tcp://' + my_push_addr,
                                     checked_srvs=[this_addr]))
        harvester.register(req, zmq.POLLIN)
        reqs[req] = addr

    srv_responses = 0

    for i in range(len(reqs) * 50):
        socks = dict(harvester.poll(timeout=300))

        for req in socks:
            msg = recv_msg(req)
            msg = mtool.unpack_msg(msg)

            if msg.type == 'rq_ok':
                LOGGER.info("waiting for experiments from server: " + str(reqs[req]))
                harvester.unregister(req)
                req.close()

            elif msg.type == 'eeg_experiments':
                LOGGER.info("GOT EXPERIMENTS from: " + msg.sender_ip)
                exps += msg.experiment_list
                srv_responses += 1
            else:
                LOGGER.warning('strange msg:  ' + str(msg))
        if srv_responses == len(ip_list):
            print("GOT ALL eeg_experiment RESPONSES :-) [addr list:  ", ip_list, "]")
            break

    other_exps_pull.close()
    return exps
Beispiel #11
0
    def __init__(self, address):
        self.context = zmq.Context.instance()
        templates = launcher_messages.message_templates
        self.msg_factory = OBCIMessageTool(msg_templates=templates)
        self.socket = self.context.socket(zmq.REQ)
        self.socket.connect(address)

        # disabled bc Ubuntu LTS uses archaic zmq version
        # self.socket.RCVTIMEO = 7000
        self.poller = zmq.Poller()
        self.poller.register(self.socket, zmq.POLLIN)

        self.open = True
Beispiel #12
0
    def __init__(self, server_addresses, zmq_context=None):
        self.ctx = zmq_context if zmq_context else zmq.Context()

        self.server_addresses = server_addresses
        self.server_req_socket = None
        self.init_server_socket(server_addresses)

        #self = zmq.Poller()
        #self.register(self.server_req_socket, zmq.POLLIN)
        self.poller = PollingObject()

        self.mtool = OBCIMessageTool(message_templates)
        self.dns = net.DNS()
Beispiel #13
0
class ObciBaseClient(object):
    """
    Base class for communicating with OBCI peers.
    """
    def __init__(self, address):
        self.context = zmq.Context.instance()
        templates = launcher_messages.message_templates
        self.msg_factory = OBCIMessageTool(msg_templates=templates)
        self.socket = self.context.socket(zmq.REQ)
        self.socket.connect(address)
        
        # disabled bc Ubuntu LTS uses archaic zmq version
        # self.socket.RCVTIMEO = 7000
        self.poller = zmq.Poller()
        self.poller.register(self.socket, zmq.POLLIN)
        
        self.open = True

    def close(self):
        if self.open:
            self.socket.close()
            self.open = False

    def send_recv(self, message_name, **kwargs):
        data = self.msg_factory.fill_msg(message_name, **kwargs)
        self.socket.send(data)
        poll_results = self.poller.poll(7000)
        response = self.socket.recv(zmq.NOBLOCK) if poll_results else None
        return json.loads(response)
Beispiel #14
0
class ObciBaseClient(object):
    """
    Base class for communicating with OBCI peers.
    """
    def __init__(self, address):
        self.context = zmq.Context.instance()
        templates = launcher_messages.message_templates
        self.msg_factory = OBCIMessageTool(msg_templates=templates)
        self.socket = self.context.socket(zmq.REQ)
        self.socket.connect(address)

        # disabled bc Ubuntu LTS uses archaic zmq version
        # self.socket.RCVTIMEO = 7000
        self.poller = zmq.Poller()
        self.poller.register(self.socket, zmq.POLLIN)

        self.open = True

    def close(self):
        if self.open:
            self.socket.close()
            self.open = False

    def send_recv(self, message_name, **kwargs):
        data = self.msg_factory.fill_msg(message_name, **kwargs)
        self.socket.send(data)
        poll_results = self.poller.poll(7000)
        response = self.socket.recv(zmq.NOBLOCK) if poll_results else None
        return json.loads(response)
Beispiel #15
0
    def __init__(self, proc_description,
                                reg_timeout_desc=None,
                                monitoring_optflags=PING,
                                logger=None):

        self.desc = proc_description

        self.must_register = reg_timeout_desc is not None
        self._status_lock = threading.RLock()
        self._status = UNKNOWN if self.must_register else RUNNING
        self._status_details = None

        self.ping_it = monitoring_optflags & PING
        self.check_returncode = monitoring_optflags & RETURNCODE if \
                                        self.desc.pid is not None else False

        self.logger = logger or get_logger(
                                'subprocess_monitor'+'-'+self.desc.name+'-'+str(self.desc.pid),
                                stream_level='info')
        self.set_registration_timeout_handler(reg_timeout_desc)
        self.registration_data = None

        self._stop_monitoring = False
        self._ping_thread = None
        self._ping_retries = 8
        self._returncode_thread = None
        self._mtool = OBCIMessageTool(message_templates)
        self._ctx = None
        self.rq_sock = None
        self._poller = PollingObject()

        self.delete = False
Beispiel #16
0
def update_nearby_servers(srv_data, bcast_port, ctx=None, update_push_addr=None):
    mtool = OBCIMessageTool(message_templates)

    loops_to_update = _LOOPS

    s = socket(AF_INET, SOCK_DGRAM)
    s.bind(('', bcast_port))

    notify_sock = None
    if update_push_addr is not None:
        ctx = ctx if ctx else zmq.Context()
        notify_sock = ctx.socket(zmq.PUSH)
        notify_sock.connect(update_push_addr)

    while 1:
        changed = False
        try:
            inp, out, exc = select.select([s], [], [], UPDATE_INTERVAL / _LOOPS)
        except Exception, e:
            srv_data.logger.critical("nearby_servers_update - exception: %s", str(e))
            srv_data.logger.critical("nearby_servers - aborting")
            return

        if s in inp:
            data, wherefrom = s.recvfrom(1500, 0)
            # sys.stderr.write(repr(wherefrom) + '\n')
            # sys.stdout.write(data)
            msg = unicode(data, encoding='utf-8')
            msg = msg[:-1]
            message = mtool.unpack_msg(msg)
            changed = srv_data.update(ip=wherefrom[0], hostname=message.sender_ip,
                                      uuid=message.sender, rep_port=message.rep_port,
                                      pub_port=message.pub_port)

        else:
            # print "no data"
            pass

        loops_to_update -= 1

        if loops_to_update == 0:
            loops_to_update = _LOOPS
            changed = srv_data.clean_silent()

        if changed:
            send_msg(notify_sock, mtool.fill_msg('nearby_machines',
                                                 nearby_machines=srv_data.dict_snapshot()))
Beispiel #17
0
    def __init__(self, addresses):
        super(ConfigServer, self).__init__(addresses=addresses, type=peers.CONFIG_SERVER)
        self._configs = {}
        self._ext_configs = {}
        self._ready_peers = []
        self.__to_all = False
        self.mtool = OBCIMessageTool(message_templates)
        self.launcher_sock = None
        params, other_params = PeerCmd().parse_cmd()

        self.addr = params['local_params'].get('launcher_socket_addr', '')

        self.exp_uuid = params['local_params'].get('experiment_uuid', '')
        self.log_dir = params['local_params'].get('log_dir', None)
        self.logger = get_logger('config_server', log_dir=self.log_dir,
                                conn=self.conn,
                                file_level=params['local_params'].get('file_log_level', None),
                                mx_level=params['local_params'].get('mx_log_level', None),
                                stream_level=params['local_params'].get('console_log_level', None))
        self._old_configs = self._stored_config()
        self._restore_peers = params['local_params'].get('restore_peers', '').split()

        for peer in self._restore_peers:
            if peer in self._old_configs["local"]:
                self._configs[peer] = dict(self._old_configs["local"][peer])
                self._ready_peers.append(peer)
            if peer in self._old_configs["ext"]:
                self._ext_configs[peer] = dict(self._old_configs["ext"][peer])

        if self.addr != '':
            self.ctx = zmq.Context()
            self.launcher_sock = self.ctx.socket(zmq.PUSH)
            try:
                self.launcher_sock.connect(self.addr)

            except Exception, e:
                self.logger.error("failed to connect to address " +\
                                             self.addr + " !!!")
                self.launcher_sock = None
            else:
                self.logger.info("OK: connected to " + self.addr)

                send_msg(self.launcher_sock, self.mtool.fill_msg("config_server_ready"))
Beispiel #18
0
 def __init__(self, zmq_ctx, uuid, logger=None, obci_dns=None):
     self._processes = {}
     self._ctx = zmq_ctx
     self.uuid = uuid
     self.logger = logger or get_logger('subprocess_monitor',
                             stream_level='warning')
     self.obci_dns = obci_dns
     self._mtool = OBCIMessageTool(message_templates)
     self.poller = PollingObject()
     self._proc_lock = threading.RLock()
Beispiel #19
0
    def __init__(self, srv_addrs, ctx, client_push_address, nearby_servers):
        self.ctx = ctx
        self.server_req_socket = self.ctx.socket(zmq.REQ)
        for addr in srv_addrs:
            self.server_req_socket.connect(addr)

        self.poller = PollingObject()
        self.mtool = OBCIMessageTool(message_templates)
        self.nearby_servers = nearby_servers
        self._amplified_cache = {}
Beispiel #20
0
def broadcast_server(server_uuid, rep_port, pub_port, bcast_port):
    mtool = OBCIMessageTool(message_templates)

    msg = mtool.fill_msg("server_broadcast", sender_ip=gethostname(), sender=server_uuid,
                         rep_port=rep_port, pub_port=pub_port)
    msg += u'\n'
    str_msg = msg.encode('utf-8')

    s = socket(AF_INET, SOCK_DGRAM)
    s.bind(('', 0))
    s.setsockopt(SOL_SOCKET, SO_BROADCAST, 1)

    # introduction
    for i in range(10):
        try:
            s.sendto(str_msg, ('<broadcast>', bcast_port))
        except error, e:
            pass
        time.sleep(0.3)
Beispiel #21
0
    def kill(self):
        # send "kill" to the process or kill request to its supervisor?
        self.stop_monitoring()
        if not self._ctx:
            self._ctx = zmq.Context()
        rq_sock = self._ctx.socket(zmq.REQ)
        try:
            rq_sock.connect(self.rq_address)
            mtool = OBCIMessageTool(message_templates)
            poller = PollingObject()
            send_msg(rq_sock, mtool.fill_msg("kill_process", pid=self.pid, machine=self.machine_ip))
            res, _ = poller.poll_recv(rq_sock, timeout=5000)
        finally:
            rq_sock.close()
        if res:
            res = mtool.unpack_msg(res)
            print "Response to kill request: ", res

            with self._status_lock:
                self._status = TERMINATED
Beispiel #22
0
    def __init__(self, server_addresses, zmq_context=None):
        self.ctx = zmq_context if zmq_context else zmq.Context()

        self.server_addresses = server_addresses
        self.server_req_socket = None
        self.init_server_socket(server_addresses)

        # self = zmq.Poller()
        # self.register(self.server_req_socket, zmq.POLLIN)
        self.poller = PollingObject()

        self.mtool = OBCIMessageTool(message_templates)
        self.dns = net.DNS()
Beispiel #23
0
def broadcast_server(server_uuid, rep_port, pub_port, bcast_port):
    mtool = OBCIMessageTool(message_templates)

    msg = mtool.fill_msg("server_broadcast",
                         sender_ip=gethostname(),
                         sender=server_uuid,
                         rep_port=rep_port,
                         pub_port=pub_port)
    msg += u'\n'
    str_msg = msg.encode('utf-8')

    s = socket(AF_INET, SOCK_DGRAM)
    s.bind(('', 0))
    s.setsockopt(SOL_SOCKET, SO_BROADCAST, 1)

    # introduction
    for i in range(10):
        try:
            s.sendto(str_msg, ('<broadcast>', bcast_port))
        except error, e:
            pass
        time.sleep(0.3)
Beispiel #24
0
 def __init__(self, address):
     self.context = zmq.Context.instance()
     templates = launcher_messages.message_templates
     self.msg_factory = OBCIMessageTool(msg_templates=templates)
     self.socket = self.context.socket(zmq.REQ)
     self.socket.connect(address)
     
     # disabled bc Ubuntu LTS uses archaic zmq version
     # self.socket.RCVTIMEO = 7000
     self.poller = zmq.Poller()
     self.poller.register(self.socket, zmq.POLLIN)
     
     self.open = True
Beispiel #25
0
 def __init__(self,
              server_address,
              handler_class,
              bind_and_activate=True,
              zmq_ctx=None,
              zmq_rep_addr=None):
     daemon_threads = True
     server_timeout = 45
     SocketServer.TCPServer.__init__(self, server_address, handler_class,
                                     bind_and_activate)
     self.mtool = OBCIMessageTool(message_templates)
     self.pl = PollingObject()
     self.ctx = zmq_ctx
     self.rep_addr = zmq_rep_addr
Beispiel #26
0
def start_eeg_signal_experiment(ctx, srv_addrs, rq_message):
    client = OBCIClient(srv_addrs, ctx)
    # server_req_socket = ctx.socket(zmq.REQ)
    #     for addr in srv_addrs:
    #         server_req_socket.connect(addr)

    amp_params = {}
    amp_params.update(rq_message.amplifier_params['additional_params'])
    del rq_message.amplifier_params['additional_params']
    amp_params.update(rq_message.amplifier_params)

    par_list = ['--peer', 'amplifier']
    for par, val in amp_params.iteritems():
        par_list += ['-p', par, unicode(val)]
    for par, val in CONFIG_DEFAULTS.iteritems():
        if par not in par_list:
            par_list += ['-p', par, unicode(val)]

    overwrites = peer_cmd.peer_overwrites_pack(par_list)
    result = client.launch(rq_message.launch_file, None, rq_message.name,
                           overwrites)

    LOGGER.info("START EEG signal! return to:  " +
                rq_message.client_push_address)
    mtool = OBCIMessageTool(message_templates)
    to_client = ctx.socket(zmq.PUSH)
    to_client.connect(rq_message.client_push_address)
    if result is None or result.type == 'no_data':
        send_msg(
            to_client,
            mtool.fill_msg("rq_error",
                           err_code="launch_failed",
                           details="No response from server or experiment"))
    else:
        send_msg(to_client, result.raw())
    LOGGER.info("sent eeg launch result" + str(result)[:500])
    time.sleep(0.1)
Beispiel #27
0
    def __init__(self, address, zmq_ctx, zmq_rep_addr):
        self.srv_address = address
        self.ctx = zmq_ctx
        self.zmq_rep_addr = zmq_rep_addr.replace("*", socket.gethostname())
        self.long_rqs = {}

        for msgtype in [
                "find_eeg_experiments",
                "find_eeg_amplifiers",
                #"join_experiment",
                "start_eeg_signal"
        ]:
            self.long_rqs[msgtype] = self._make_pull_sock()

        self.mtool = OBCIMessageTool(message_templates)
Beispiel #28
0
class ConfigServer(BaseMultiplexerServer):
    @log_crash
    def __init__(self, addresses):
        super(ConfigServer, self).__init__(addresses=addresses, type=peers.CONFIG_SERVER)
        self._configs = {}
        self._ext_configs = {}
        self._ready_peers = []
        self.__to_all = False
        self.mtool = OBCIMessageTool(message_templates)
        self.launcher_sock = None
        params, other_params = PeerCmd().parse_cmd()

        self.addr = params['local_params'].get('launcher_socket_addr', '')

        self.exp_uuid = params['local_params'].get('experiment_uuid', '')
        self.log_dir = params['local_params'].get('log_dir', None)
        self.logger = get_logger('config_server', log_dir=self.log_dir,
                                conn=self.conn,
                                file_level=params['local_params'].get('file_log_level', None),
                                mx_level=params['local_params'].get('mx_log_level', None),
                                stream_level=params['local_params'].get('console_log_level', None))
        self._old_configs = self._stored_config()
        self._restore_peers = params['local_params'].get('restore_peers', '').split()

        for peer in self._restore_peers:
            if peer in self._old_configs["local"]:
                self._configs[peer] = dict(self._old_configs["local"][peer])
                self._ready_peers.append(peer)
            if peer in self._old_configs["ext"]:
                self._ext_configs[peer] = dict(self._old_configs["ext"][peer])

        if self.addr != '':
            self.ctx = zmq.Context()
            self.launcher_sock = self.ctx.socket(zmq.PUSH)
            try:
                self.launcher_sock.connect(self.addr)

            except Exception, e:
                self.logger.error("failed to connect to address " +\
                                             self.addr + " !!!")
                self.launcher_sock = None
            else:
                self.logger.info("OK: connected to " + self.addr)

                send_msg(self.launcher_sock, self.mtool.fill_msg("config_server_ready"))
        self.logger.info("connections count ::::::::: %s", self.conn.connections_count())
Beispiel #29
0
    def __init__(self, preset_data=None, launcher_data=None, ctx=None):
        self.preset_data = preset_data
        self.launch_file = None
        self.launcher_data = launcher_data
        self.name = None
        self.info = ""
        self.public_params = []
        self.origin_machine = ''
        self.unsupervised_peers = {}
        self.old_uid = None

        self.ctx = ctx
        self.exp_req = None
        self.mtool = OBCIMessageTool(message_templates)
        self.poller = PollingObject()

        self.category = DEFAULT_CATEGORY

        if preset_data is not None:
            self.setup_from_preset(preset_data)

        elif launcher_data is not None:
            self.setup_from_launcher(launcher_data)
        super(ExperimentEngineInfo, self).__init__()
Beispiel #30
0
    def __init__(self, preset_data=None, launcher_data=None, ctx=None):
        self.preset_data = preset_data
        self.launch_file = None
        self.launcher_data = launcher_data
        self.name = None
        self.info = ""
        self.public_params = []
        self.origin_machine = ''
        self.unsupervised_peers = {}
        self.old_uid = None

        self.ctx = ctx
        self.exp_req = None
        self.mtool = OBCIMessageTool(message_templates)
        self.poller = PollingObject()

        self.category = DEFAULT_CATEGORY

        if preset_data is not None:
            self.setup_from_preset(preset_data)

        elif launcher_data is not None:
            self.setup_from_launcher(launcher_data)
        super(ExperimentEngineInfo, self).__init__()
Beispiel #31
0
class SubprocessMonitor(object):
    def __init__(self, zmq_ctx, uuid, logger=None, obci_dns=None):
        self._processes = {}
        self._ctx = zmq_ctx
        self.uuid = uuid
        self.logger = logger or get_logger('subprocess_monitor',
                                           stream_level='warning')
        self.obci_dns = obci_dns
        self._mtool = OBCIMessageTool(message_templates)
        self.poller = PollingObject()
        self._proc_lock = threading.RLock()

    def not_running_processes(self):
        status = {}
        with self._proc_lock:
            for key, proc in self._processes.iteritems():
                st = proc.status()
                if st[0] in [FINISHED, FAILED, TERMINATED
                             ] and not proc.marked_delete():
                    status[key] = st

        return status

    def unknown_status_processes(self):
        with self._proc_lock:
            return [
                proc for proc in self._processes.values()
                if proc.status()[0] == UNKNOWN
            ]

    def process(self, machine_ip, pid):
        with self._proc_lock:
            return self._processes.get((machine_ip, pid), None)

    def killall(self, force=False):
        with self._proc_lock:
            for proc in self._processes.values():
                kill_method = proc.kill if not force else proc.kill_with_force
                if proc.status()[0] not in [FINISHED, FAILED, TERMINATED]:
                    kill_method()

    def delete(self, machine, pid):
        proc = self._processes.get((machine, pid), None)
        if proc is None:
            raise Exception("Process not found: " + str((machine, pid)))
        if not proc.running():
            del self._processes[(machine, pid)]
            return True
        else:
            self.logger.error("Process is running, will not delete! " +
                              str((machine, pid)))
            return False

    def delete_all(self):
        with self._proc_lock:
            for proc in self._processes.values():
                del proc
            self._processes = {}

    def stop_monitoring(self):
        with self._proc_lock:
            for proc in self._processes.values():
                proc.stop_monitoring()

    def _launch_args(self, path, args):
        # TODO fix interpreter calls // only python is supported
        if path.endswith('.py'):
            launch_args = PYTHON_CALL + [path] + args
        else:
            launch_args = [path] + args
        return launch_args

    def _stdio_actions(self, io_flags):
        out = subprocess.PIPE if io_flags & STDOUT else None

        if io_flags & STDERR:
            err = subprocess.PIPE
        elif out is not None:
            err = subprocess.STDOUT
        else:
            err = None

        stdin = subprocess.PIPE if io_flags & STDIN else None
        return (out, err, stdin)

    def _local_launch(self, launch_args, stdio_actions, env):
        ON_POSIX = 'posix' in sys.builtin_module_names
        out, err, stdin = stdio_actions
        try:
            if sys.platform == "win32":
                crflags = subprocess.CREATE_NEW_PROCESS_GROUP
                popen_obj = subprocess.Popen(launch_args,
                                             stdout=out,
                                             stderr=err,
                                             stdin=stdin,
                                             bufsize=1,
                                             close_fds=ON_POSIX,
                                             env=env,
                                             creationflags=crflags)
            else:
                popen_obj = subprocess.Popen(launch_args,
                                             stdout=out,
                                             stderr=err,
                                             stdin=stdin,
                                             bufsize=1,
                                             close_fds=ON_POSIX,
                                             env=env)

            details = "Popen constructor finished for " +\
                str(launch_args[:3]) + "(...)"
            self.logger.info(details)
            return popen_obj, details
        except OSError as e:
            details = "Unable to spawn process {0} [{1}]".format(
                launch_args, e.args)
            self.logger.error(details)
            return None, details
        except ValueError as e:
            details = "Unable to spawn process (bad arguments) {0} [{1}]".format(
                launch_args, e.args)
            self.logger.error(details)
            return None, details
        except Exception as e:
            details = "Process launch Error: " + str(e) + str(e.args) + str(
                vars(e))
            self.logger.error(details)
            return None, details

    def new_local_process(self,
                          path,
                          args,
                          proc_type='',
                          name='',
                          capture_io=STDOUT | STDIN,
                          stdout_log=None,
                          stderr_log=None,
                          register_timeout_desc=None,
                          monitoring_optflags=RETURNCODE | PING,
                          machine_ip=None,
                          env=None):

        launch_args = self._launch_args(path, args)
        self.logger.debug(proc_type + " local path:  " + path)
        machine = machine_ip if machine_ip else socket.gethostname()
        std_actions = self._stdio_actions(capture_io)
        timeout_desc = register_timeout_desc

        self.logger.debug('process launch arg list:  %s', launch_args)
        popen_obj, details = self._local_launch(launch_args, std_actions, env)

        if popen_obj is None:
            return None, details

        if popen_obj.returncode is not None:
            det = "opened process already terminated" + popen_obj.communicate()
            self.logger.warning(det)

        if not name:
            name = os.path.basename(path)

        process_desc = ProcessDescription(proc_type=proc_type,
                                          name=name,
                                          path=path,
                                          args=args,
                                          machine_ip=machine,
                                          pid=popen_obj.pid)

        # io_handler will be None if no stdio is captured
        io_handler = start_stdio_handler(popen_obj, std_actions,
                                         ':'.join([machine, path, name]),
                                         stdout_log, stderr_log)

        new_proc = LocalProcess(process_desc,
                                popen_obj,
                                io_handler=io_handler,
                                reg_timeout_desc=timeout_desc,
                                monitoring_optflags=monitoring_optflags,
                                logger=self.logger)

        if monitoring_optflags & PING:
            new_proc._ctx = self._ctx

        with self._proc_lock:
            self._processes[(machine, popen_obj.pid)] = new_proc

        new_proc.start_monitoring()

        return new_proc, None

    def new_remote_process(self,
                           path,
                           args,
                           proc_type,
                           name,
                           machine_ip,
                           conn_addr,
                           capture_io=STDOUT | STDIN,
                           stdout_log=None,
                           stderr_log=None,
                           register_timeout_desc=None,
                           monitoring_optflags=PING):
        """Send a request to conn_addr for a process launch. By default
        the process will be monitored with ping requests and locally by the
        remote peer."""

        timeout_desc = register_timeout_desc

        rq_message = self._mtool.fill_msg('launch_process',
                                          path=path,
                                          args=args,
                                          proc_type=proc_type,
                                          name=name,
                                          machine_ip=machine_ip,
                                          capture_io=capture_io,
                                          stdout_log=stdout_log,
                                          stderr_log=stderr_log)

        rq_sock = self._ctx.socket(zmq.REQ)

        try:
            rq_sock.connect(conn_addr)
        except zmq.ZMQError as e:
            det = "Could not connect to {0}, err: {1}, {2}".format(
                conn_addr, e, e.args)
            self.logger.error(det)
            return None, det

        self.logger.info("SENDING LAUNCH REQUEST  {0}  {1}  {2} {3}".format(
            machine_ip, _DEFAULT_TIMEOUT_MS, 'ms', conn_addr))

        send_msg(rq_sock, rq_message)
        result, details = self.poller.poll_recv(rq_sock, _DEFAULT_TIMEOUT_MS)

        rq_sock.close()

        if not result:
            details = details + "  [address was: {0}]".format(conn_addr)
            self.logger.error(details)
            return None, details
        else:
            result = self._mtool.unpack_msg(result)

        if result.type == 'rq_error':
            det = "REQUEST FAILED" + str(result.err_code) + ':' + str(
                result.details)
            self.logger.error(det)
            return None, det
        elif result.type == 'launched_process_info':
            self.logger.info("REQUEST SUCCESS  %s", result.dict())
            process_desc = ProcessDescription(proc_type=result.proc_type,
                                              name=result.name,
                                              path=result.path,
                                              args=args,
                                              machine_ip=result.machine,
                                              pid=result.pid)

            new_proc = RemoteProcess(process_desc,
                                     conn_addr,
                                     reg_timeout_desc=timeout_desc,
                                     monitoring_optflags=monitoring_optflags,
                                     logger=self.logger)

            if monitoring_optflags & PING:
                new_proc._ctx = self._ctx

            with self._proc_lock:
                self._processes[(result.machine, result.pid)] = new_proc

            new_proc.start_monitoring()

            return new_proc, None
Beispiel #32
0
class EEGExperimentFinder(object):

    def __init__(self, srv_addrs, ctx, client_push_address, nearby_servers):
        self.ctx = ctx
        self.server_req_socket = self.ctx.socket(zmq.REQ)
        for addr in srv_addrs:
            self.server_req_socket.connect(addr)

        self.poller = PollingObject()
        self.mtool = OBCIMessageTool(message_templates)
        self.nearby_servers = nearby_servers
        self._amplified_cache = {}

    def _running_experiments(self):
        send_msg(self.server_req_socket, self.mtool.fill_msg("list_experiments"))
        exp_list, details = self.poll_recv(self.server_req_socket, 2000)
        if not exp_list:
            LOGGER.error("Connection to obci_server failed. (list_experiments)")
            return None

        exps = exp_list.exp_data
        running = []
        for exp in exps.values():
            if exp['status_name'] == launcher_tools.RUNNING or \
                    exp['status_name'] == launcher_tools.LAUNCHING:
                running.append(exp)
        return running

    def find_amplified_experiments(self):
        running_exps = self._running_experiments()
        amplified = []
        for exp in running_exps:
            LOGGER.info("Found running experiment: " + str(exp['name']))
            infos = self._info_amplified(exp)

            if infos is not None:
                print("Found experiments...", str(infos)[:500])
                amplified += infos

        return amplified

    def _info_amplified(self, exp_desc):
        amp_options = []
        LOGGER.info("Processing experiment " + str(exp_desc['name']) +
                    "w/ addr: " + str(exp_desc['rep_addrs']))
        tcp_addrs = exp_desc['tcp_addrs']
        rep_addrs = net.choose_not_local(exp_desc['rep_addrs'])

        if not rep_addrs:
            rep_addrs = net.choose_local(exp_desc['rep_addrs'], ip=True)

        rep_addr = rep_addrs.pop()

        pub_addrs = net.choose_not_local(exp_desc['pub_addrs'])
        if not pub_addrs:
            pub_addrs = net.choose_local(exp_desc['pub_addrs'], ip=True)
        pub_addr = pub_addrs.pop()
        tcp_addr = tcp_addrs.pop()

        LOGGER.info("Chosen experiment addresses: REP -- " +
                    str(rep_addr) + ", PUB -- " + str(pub_addr))

        req_sock = self.ctx.socket(zmq.REQ)
        try:
            req_sock.connect(rep_addr)

            send_msg(req_sock, self.mtool.fill_msg('get_experiment_info'))
            res, details = self.poll_recv(req_sock, 4000)
        finally:
            req_sock.close()

        if not res:
            LOGGER.error("Connection failed (experiment " + exp_desc['name'] +
                         "), get_experiment_info")
            return None
        exp_info = res.dict()  # json.loads(res)
        for field in ["peers_status", "details"]:
            del exp_info["experiment_status"][field]

        peer_list = exp_info["peers"]
        if not self._has_mx(peer_list):
            LOGGER.info("Experiment " + exp_desc['name'] +
                        " does not have a multiplexer.")
            return None

        maybe_amps = self._amp_like_peers(peer_list)
        if not maybe_amps:
            LOGGER.info("Experiment " + exp_desc['name'] +
                        " -- no amplifier.")
            return None

        req_sock = self.ctx.socket(zmq.REQ)
        try:
            req_sock.connect(rep_addr)
            for peer in maybe_amps:
                info, params = self._get_amp_info(req_sock, peer)
                if not self._is_amplifier(info, params):
                    LOGGER.info("Experiment " + exp_desc['name'] +
                                " -- peer " + str(peer) + "is not an amplifier.")
                    continue
                else:
                    exp_data = self._create_exp_data(exp_info, info, params['param_values'],
                                                     rep_addr, pub_addr, tcp_addr)
                    amp_options.append(exp_data)
        finally:
            req_sock.close()

        return amp_options

    def _get_amp_info(self, exp_sock, peer_id):
        send_msg(exp_sock, self.mtool.fill_msg('get_peer_info', peer_id=peer_id))
        info, details = self.poll_recv(exp_sock, 4000)

        send_msg(exp_sock, self.mtool.fill_msg('get_peer_param_values', peer_id=peer_id))
        params, details = self.poll_recv(exp_sock, 4000)
        if not info or not params:
            LOGGER.error("get_peer_info failed " + str(peer_id) + "  " + str(details))
            return None, None
        info = info.dict()
        params = params.dict()
        for field in ["sender", "sender_ip", "receiver", "type", "local_params", "external_params",
                      "config_sources", "launch_dependencies"]:
            del info[field]
        return info, params

    def _is_amplifier(self, peer_info, peer_params):
        info = peer_info
        peer_id = info['peer_id']

        if not info['peer_type'] == 'obci_peer':
            LOGGER.info("Peer " + str(peer_id) + "  not obci_peer")
            return False

        params = peer_params['param_values']
        if 'channels_info' not in params or\
                'active_channels' not in params:
            LOGGER.info('Peer  ' + str(peer_id) + "  no channels_info param.")
            return False
        return True

    def _create_exp_data(self, exp_info, peer_info, params, rep_addr, pub_addr, tcp_addr):
        data = {}
        data['amplifier_params'] = params
        data['amplifier_peer_info'] = peer_info
        data['experiment_info'] = exp_info
        data['rep_addrs'] = [rep_addr]
        data['pub_addrs'] = [pub_addr]
        data['tcp_addrs'] = [tcp_addr]
        return data

    def _has_mx(self, peer_list):
        return [peer for peer in peer_list if peer.startswith('mx')] != []

    def _amp_like_peers(self, peer_list):
        return [peer for peer in peer_list if peer.startswith('amplifier')]

    def poll_recv(self, socket, timeout):
        result, details = self.poller.poll_recv(socket, timeout)
        if result:
            result = self.mtool.unpack_msg(result)

        return result, details
Beispiel #33
0
#!/usr/bin/python
# -*- coding: utf-8 -*-

import json
import zmq
import sys
import socket

from obci.control.common.message import OBCIMessageTool, send_msg, recv_msg, PollingObject
from obci.control.launcher.launcher_messages import message_templates, error_codes

from obci.control.common.obci_control_settings import PORT_RANGE
import obci.control.common.net_tools as net

if __name__ == '__main__':
    mtool = OBCIMessageTool(message_templates)
    pl = PollingObject()

    # ifname = net.server_ifname()
    my_addr = 'tcp://' + 'localhost'

    ctx = zmq.Context()

    server_req = ctx.socket(zmq.REQ)
    server_req.connect(my_addr + ':' + net.server_rep_port())

    exp_info_pull = ctx.socket(zmq.PULL)

    port = exp_info_pull.bind_to_random_port('tcp://*',
                                             min_port=PORT_RANGE[0],
                                             max_port=PORT_RANGE[1],
Beispiel #34
0
 def message_tool(self):
     return OBCIMessageTool(message_templates)
#!/usr/bin/python3


import json
import zmq
import sys

from obci.control.common.message import OBCIMessageTool, send_msg, PollingObject
from obci.control.launcher.launcher_messages import message_templates

from obci.control.common.obci_control_settings import PORT_RANGE
import obci.control.common.net_tools as net

if __name__ == '__main__':
    mtool = OBCIMessageTool(message_templates)
    pl = PollingObject()

    # ifname = net.server_ifname()
    my_addr = 'tcp://' + 'localhost'

    ctx = zmq.Context()

    server_req = ctx.socket(zmq.REQ)
    server_req.connect(my_addr + ':' + net.server_rep_port())

    exp_info_pull = ctx.socket(zmq.PULL)

    port = exp_info_pull.bind_to_random_port('tcp://*',
                                             min_port=PORT_RANGE[0],
                                             max_port=PORT_RANGE[1], max_tries=500)
Beispiel #36
0
class SubprocessMonitor(object):
    def __init__(self, zmq_ctx, uuid, logger=None, obci_dns=None):
        self._processes = {}
        self._ctx = zmq_ctx
        self.uuid = uuid
        self.logger = logger or get_logger('subprocess_monitor',
                                stream_level='warning')
        self.obci_dns = obci_dns
        self._mtool = OBCIMessageTool(message_templates)
        self.poller = PollingObject()
        self._proc_lock = threading.RLock()

    def not_running_processes(self):
        status = {}
        with self._proc_lock:
            for key, proc in self._processes.iteritems():
                st = proc.status()
                if st[0] in [FINISHED, FAILED, TERMINATED] and not proc.marked_delete():
                    status[key] = st

        return status

    def unknown_status_processes(self):
        with self._proc_lock:
            return [proc for proc in self._processes.values()\
                            if proc.status()[0] == UNKNOWN]

    def process(self, machine_ip, pid):
        with self._proc_lock:
            return self._processes.get((machine_ip, pid), None)

    def killall(self, force=False):
        with self._proc_lock:
            for proc in self._processes.values():
                kill_method = proc.kill if not force else proc.kill_with_force
                if proc.status()[0] not in [FINISHED, FAILED, TERMINATED]:
                    kill_method()

    def delete(self, machine, pid):
        proc = self._processes.get((machine, pid), None)
        if proc is None:
            raise Exception("Process not found: " + str((machine, pid)))
        if not proc.running():
            del self._processes[(machine, pid)]
            return True
        else:
            self.logger.error("Process is running, will not delete! " + str((machine, pid)))
            return False

    def delete_all(self):
        with self._proc_lock:
            for proc in self._processes.values():
                del proc
            self._processes = {}

    def stop_monitoring(self):
        with self._proc_lock:
            for proc in self._processes.values():
                proc.stop_monitoring()

    def _launch_args(self, path, args):
        #TODO fix interpreter calls // only python is supported
        if path.endswith('.py'):
            launch_args = PYTHON_CALL +[path] + args
        else:
            launch_args = [path] + args
        return launch_args

    def _stdio_actions(self, io_flags):
        out = subprocess.PIPE if io_flags & STDOUT else None

        if io_flags & STDERR:
            err = subprocess.PIPE
        elif out is not None:
            err = subprocess.STDOUT
        else: err = None

        stdin = subprocess.PIPE if io_flags & STDIN else None
        return (out, err, stdin)

    def _local_launch(self, launch_args, stdio_actions, env):
        ON_POSIX = 'posix' in sys.builtin_module_names
        out, err, stdin = stdio_actions
        try:
            if sys.platform == "win32":
                crflags = subprocess.CREATE_NEW_PROCESS_GROUP
                popen_obj = subprocess.Popen(launch_args,
                                             stdout=out, stderr=err, stdin=stdin,
                                             bufsize=1, close_fds=ON_POSIX, env=env,
                                             creationflags=crflags)
            else:
                popen_obj = subprocess.Popen(launch_args,
                                             stdout=out, stderr=err, stdin=stdin,
                                             bufsize=1, close_fds=ON_POSIX, env=env)

            details = "Popen constructor finished for " +\
                                                 str(launch_args[:3]) + "(...)"
            self.logger.info(details)
            return popen_obj, details
        except OSError as e:
            details = "Unable to spawn process {0} [{1}]".format(launch_args, e.args)
            self.logger.error(details)
            return None, details
        except ValueError as e:
            details = "Unable to spawn process (bad arguments) {0} [{1}]".format(launch_args, e.args)
            self.logger.error(details)
            return None, details
        except Exception as e:
            details = "Process launch Error: " + str(e) + str(e.args) + str(vars(e))
            self.logger.error(details)
            return None, details


    def new_local_process(self, path, args, proc_type='', name='',
                                capture_io= STDOUT | STDIN,
                                stdout_log=None,
                                stderr_log=None,
                                register_timeout_desc=None,
                                monitoring_optflags=RETURNCODE | PING,
                                machine_ip=None,
                                env=None):

        launch_args = self._launch_args(path, args)
        self.logger.debug(proc_type + " local path:  " + path)
        machine = machine_ip if machine_ip else socket.gethostname()
        std_actions = self._stdio_actions(capture_io)
        timeout_desc = register_timeout_desc

        self.logger.debug('process launch arg list:  %s', launch_args)
        popen_obj, details = self._local_launch(launch_args, std_actions, env)

        if popen_obj is None:
            return None, details

        if popen_obj.returncode is not None:
            det = "opened process already terminated" + popen_obj.communicate()
            self.logger.warning(det)

        if not name:
            name = os.path.basename(path)

        process_desc = ProcessDescription(proc_type=proc_type,
                                        name=name,
                                        path=path,
                                        args=args,
                                        machine_ip=machine,
                                        pid=popen_obj.pid)

        # io_handler will be None if no stdio is captured
        io_handler = start_stdio_handler(popen_obj, std_actions,
                                                    ':'.join([machine, path, name]),
                                                    stdout_log, stderr_log)

        new_proc = LocalProcess(process_desc, popen_obj, io_handler=io_handler,
                            reg_timeout_desc=timeout_desc,
                            monitoring_optflags=monitoring_optflags,
                            logger=self.logger)

        if monitoring_optflags & PING:
            new_proc._ctx = self._ctx

        with self._proc_lock:
            self._processes[(machine, popen_obj.pid)] = new_proc

        new_proc.start_monitoring()

        return new_proc, None

    def new_remote_process(self, path, args, proc_type, name,
                                machine_ip, conn_addr,
                                capture_io= STDOUT | STDIN,
                                stdout_log=None,
                                stderr_log=None,
                                register_timeout_desc=None,
                                monitoring_optflags=PING):
        """Send a request to conn_addr for a process launch. By default
        the process will be monitored with ping requests and locally by the
        remote peer."""


        timeout_desc = register_timeout_desc

        rq_message = self._mtool.fill_msg('launch_process',
                                path=path,
                                args=args, proc_type=proc_type,
                                name=name,
                                machine_ip=machine_ip,
                                capture_io=capture_io,
                                stdout_log=stdout_log,
                                stderr_log=stderr_log)

        rq_sock = self._ctx.socket(zmq.REQ)

        try:
            rq_sock.connect(conn_addr)
        except zmq.ZMQError as e:
            det = "Could not connect to {0}, err: {1}, {2}".format(
                                                            conn_addr, e, e.args)
            self.logger.error(det)
            return None, det

        self.logger.info("SENDING LAUNCH REQUEST  {0}  {1}  {2} {3}".format(
                                machine_ip, _DEFAULT_TIMEOUT_MS, 'ms', conn_addr))

        send_msg(rq_sock, rq_message)
        result, details = self.poller.poll_recv(rq_sock, _DEFAULT_TIMEOUT_MS)

        rq_sock.close()

        if not result:
            details = details + "  [address was: {0}]".format(conn_addr)
            self.logger.error(details)
            return None, details
        else:
            result = self._mtool.unpack_msg(result)

        if result.type == 'rq_error':
            det = "REQUEST FAILED" + str(result.err_code) + ':' + str(result.details)
            self.logger.error(det)
            return None, det
        elif result.type == 'launched_process_info':
            self.logger.info("REQUEST SUCCESS  %s", result.dict())
            process_desc = ProcessDescription(proc_type=result.proc_type,
                                            name=result.name,
                                            path=result.path,
                                            args=args,
                                            machine_ip=result.machine,
                                            pid=result.pid)

            new_proc = RemoteProcess(process_desc, conn_addr,
                                reg_timeout_desc=timeout_desc,
                                monitoring_optflags=monitoring_optflags,
                                logger=self.logger)

            if monitoring_optflags & PING:
                new_proc._ctx = self._ctx

            with self._proc_lock:
                self._processes[(result.machine, result.pid)] = new_proc

            new_proc.start_monitoring()

            return new_proc, None
Beispiel #37
0
class OBCIClient(object):
    default_timeout = 5000

    def __init__(self, server_addresses, zmq_context=None):
        self.ctx = zmq_context if zmq_context else zmq.Context()

        self.server_addresses = server_addresses
        self.server_req_socket = None
        self.init_server_socket(server_addresses)

        # self = zmq.Poller()
        # self.register(self.server_req_socket, zmq.POLLIN)
        self.poller = PollingObject()

        self.mtool = OBCIMessageTool(message_templates)
        self.dns = net.DNS()

    def init_server_socket(self, srv_addrs):
        if self.server_req_socket is not None:
            print("server socket restart")
            self.server_req_socket.close()

        self.server_req_socket = self.ctx.socket(zmq.REQ)

        for addr in srv_addrs:
            print(addr)
            self.server_req_socket.connect(addr)

    def launch(self, launch_file=None, sandbox_dir=None, name=None, overwrites=None):
        result = self.send_create_experiment(launch_file, sandbox_dir, name, overwrites)
        print("create result:", result)
        if not result:
            self.init_server_socket(self.server_addresses)
            return result
        if result.type != "experiment_created":
            return result
        print(result)
        machine = result.origin_machine
        addrs = [addr for addr in result.rep_addrs if self._addr_connectable(addr, machine)]

        return self.send_start_experiment(addrs)

    def morph(self, exp_strname, launch_file, name=None, overwrites=None, leave_on=None):
        response = self.get_experiment_contact(exp_strname)
        exp_sock = self.ctx.socket(zmq.REQ)

        try:
            if response.type == "rq_error" or response.type == "no_data":
                return response
            for addr in response.rep_addrs:
                exp_sock.connect(addr)

            msg = self.mtool.fill_msg('morph_to_new_scenario', launch_file=launch_file,
                                      name=name, overwrites=overwrites, leave_on=leave_on)
            send_msg(exp_sock, msg)

            response, details = self.poll_recv(exp_sock, 6000)
            return response
        finally:
            exp_sock.close()

    def _addr_connectable(self, addr, machine):
        return machine == socket.gethostname() or \
            (net.is_ip(addr) and not net.addr_is_local(addr))

    def start_chosen_experiment(self, exp_strname):
        response = self.get_experiment_contact(exp_strname)

        if response.type == "rq_error" or response.type == "no_data":
            return response

        return self.send_start_experiment(response.rep_addrs)

    def send_start_experiment(self, exp_addrs):
        exp_sock = self.ctx.socket(zmq.REQ)
        try:
            for addr in exp_addrs:
                exp_sock.connect(addr)

                send_msg(exp_sock, self.mtool.fill_msg("start_experiment"))
                reply, details = self.poll_recv(exp_sock, 20000)

                # print reply
                return reply
        finally:
            exp_sock.close()

    def force_kill_experiment(self, strname):
        pass

    def get_experiment_contact(self, strname):
        send_msg(self.server_req_socket, self.mtool.fill_msg("get_experiment_contact",
                                                             strname=strname))
        response, details = self.poll_recv(self.server_req_socket, self.default_timeout)
        return response

    def ping_server(self, timeout=50):
        send_msg(self.server_req_socket, self.mtool.fill_msg("ping"))
        response, details = self.poll_recv(self.server_req_socket, timeout)

        return response

    def retry_ping(self, timeout=50):
        response, details = self.poll_recv(self.server_req_socket, timeout)
        return response

    def send_create_experiment(self, launch_file=None, sandbox_dir=None, name=None, overwrites=None):

        send_msg(
            self.server_req_socket,
            self.mtool.fill_msg(
                "create_experiment",
                launch_file=launch_file,
                sandbox_dir=sandbox_dir,
                name=name,
                overwrites=overwrites))

        response, details = self.poll_recv(self.server_req_socket, 5000)
        return response

    def send_list_experiments(self):
        send_msg(self.server_req_socket, self.mtool.fill_msg("list_experiments"))

        response, details = self.poll_recv(self.server_req_socket, 4000)
        return response

    def send_list_nearby_machines(self):
        send_msg(self.server_req_socket, self.mtool.fill_msg("list_nearby_machines"))
        response, details = self.poll_recv(self.server_req_socket, 4000)
        return response

    def get_experiment_details(self, strname, peer_id=None):
        response = self.get_experiment_contact(strname)

        if response.type == "rq_error" or response.type == "no_data":
            return response

        sock = self.ctx.socket(zmq.REQ)
        try:
            for addr in response.rep_addrs:
                sock.connect(addr)

            if peer_id:
                send_msg(sock, self.mtool.fill_msg("get_peer_info", peer_id=peer_id))
            else:
                send_msg(sock, self.mtool.fill_msg("get_experiment_info"))
            response, details = self.poll_recv(sock, 2000)

            return response
        finally:
            sock.close()

    def configure_peer(self, exp_strname, peer_id, config_overrides, override_files=None):
        response = self.get_experiment_contact(exp_strname)

        if response.type == "rq_error" or response.type == "no_data":
            return response

        sock = self.ctx.socket(zmq.REQ)
        try:
            for addr in response.rep_addrs:
                sock.connect(addr)

            if override_files:
                send_msg(sock, self.mtool.fill_msg("get_peer_info", peer_id=peer_id))
                response, details = self.poll_recv(sock, 2000)
                if response.type is 'rq_error':
                    return response

            msg = self.mtool.fill_msg("update_peer_config", peer_id=peer_id,
                                      **config_overrides)

            send_msg(sock, msg)
            # print msg
            response, details = self.poll_recv(sock, 2000)
            return response
        finally:
            sock.close()

            # "update_peer_config" : dict(peer_id='', local_params='',
            #                 external_params='', launch_dependencies='', config_sources=''),

    def kill_exp(self, strname, force=False):
        send_msg(self.server_req_socket,
                 self.mtool.fill_msg("kill_experiment", strname=strname))
        return self.poll_recv(self.server_req_socket, 2000)[0]

    def srv_kill(self):
        send_msg(self.server_req_socket,
                 self.mtool.fill_msg("kill"))
        return self.poll_recv(self.server_req_socket, 2000)[0]

    def join_experiment(self, strname, peer_id, path):
        response = self.get_experiment_contact(strname)

        if response.type == "rq_error" or response.type == "no_data":
            return response

        sock = self.ctx.socket(zmq.REQ)

        try:
            self._connect(sock, response.rep_addrs)

            send_msg(sock, self.mtool.fill_msg("join_experiment",
                                               peer_id=peer_id, path=path, peer_type='obci_peer'))
            response, details = self.poll_recv(sock, 5000)
            sock.close()
            return response
        finally:
            sock.close()

    def add_peer(self, strname, peer_id, path, machine, param_overwrites=None,
                 custom_config_path=None, config_sources=None, launch_dependencies=None,
                 apply_globals=True):
        response = self.get_experiment_contact(strname)

        if response.type == "rq_error" or response.type == "no_data":
            return response

        sock = self.ctx.socket(zmq.REQ)
        try:
            self._connect(sock, response.rep_addrs)
            send_msg(sock, self.mtool.fill_msg("add_peer",
                                               peer_id=peer_id, peer_path=path, peer_type='obci_peer',
                                               machine=machine, param_overwrites=param_overwrites,
                                               custom_config_path=custom_config_path, config_sources=config_sources,
                                               launch_dependencies=launch_dependencies, apply_globals=apply_globals))
            response, details = self.poll_recv(sock, 5000)
            return response
        finally:
            sock.close()

    def kill_peer(self, exp_strname, peer_id, remove_config=False):
        response = self.get_experiment_contact(exp_strname)

        if response.type == "rq_error" or response.type == "no_data":
            return response

        try:
            sock = self.ctx.socket(zmq.REQ)
            self._connect(sock, response.rep_addrs)
            send_msg(sock, self.mtool.fill_msg("kill_peer",
                                               peer_id=peer_id, remove_config=remove_config))
            response, details = self.poll_recv(sock, 5000)
            return response
        finally:
            sock.close()

    def _connect(self, sock, addr_list):
        print("****  ", addr_list)
        this = self._is_this_machine(addr_list)
        connected = False
        for addr in addr_list:
            if not this and ('localhost' in addr or '127.0.0.1' in addr):
                continue
            try:
                sock.connect(addr)
                connected = True
            except zmq.ZMQError as e:
                print(addr, " ::: ", str(e))
        if not connected:
            raise Exception("Could not connect to any of the addresses: " + str(addr_list))

    def _is_this_machine(self, addr_list):
        for addr in addr_list:
            if self.dns.is_this_machine(addr):
                return True
        return False

    def leave_experiment(self, strname, peer_id):
        response = self.get_experiment_contact(strname)

        if response.type == "rq_error" or response.type == "no_data":
            return response

        try:
            sock = self.ctx.socket(zmq.REQ)
            self._connect(sock, response.rep_addrs)

            send_msg(sock, self.mtool.fill_msg("leave_experiment",
                                               peer_id=peer_id))
            response, details = self.poll_recv(sock, 5000)
            return response
        finally:
            sock.close()

    def get_tail(self, strname, peer_id, len_):
        response = self.get_experiment_contact(strname)

        if response.type == "rq_error" or response.type == "no_data":
            return response

        sock = self.ctx.socket(zmq.REQ)
        try:
            for addr in response.rep_addrs:
                sock.connect(addr)

            send_msg(sock, self.mtool.fill_msg("get_tail", peer_id=peer_id, len=len_))
            response, details = self.poll_recv(sock, 4000)

            return response
        finally:
            sock.close()

    def poll_recv(self, socket, timeout):
        result, details = self.poller.poll_recv(socket, timeout)
        if result:
            result = self.mtool.unpack_msg(result)
        else:
            result = EmptyResponse(details)

        return result, details
Beispiel #38
0
class Process(object):
    def __init__(self, proc_description,
                                reg_timeout_desc=None,
                                monitoring_optflags=PING,
                                logger=None):

        self.desc = proc_description

        self.must_register = reg_timeout_desc is not None
        self._status_lock = threading.RLock()
        self._status = UNKNOWN if self.must_register else RUNNING
        self._status_details = None

        self.ping_it = monitoring_optflags & PING
        self.check_returncode = monitoring_optflags & RETURNCODE if \
                                        self.desc.pid is not None else False

        self.logger = logger or get_logger(
                                'subprocess_monitor'+'-'+self.desc.name+'-'+str(self.desc.pid),
                                stream_level='info')
        self.set_registration_timeout_handler(reg_timeout_desc)
        self.registration_data = None

        self._stop_monitoring = False
        self._ping_thread = None
        self._ping_retries = 8
        self._returncode_thread = None
        self._mtool = OBCIMessageTool(message_templates)
        self._ctx = None
        self.rq_sock = None
        self._poller = PollingObject()

        self.delete = False


    @property
    def machine_ip(self):
        return self.desc.machine_ip

    @property
    def pid(self):
        return self.desc.pid

    @property
    def path(self):
        return self.desc.path

    @property
    def proc_type(self):
        return self.desc.proc_type

    @property
    def name(self):
        return self.desc.name

    def status(self):
        with self._status_lock:
            return self._status, self._status_details

    def set_registration_timeout_handler(self, reg_timeout_desc):
        with self._status_lock:
            self._status = UNKNOWN
            self._status_details = None
        self.must_register = reg_timeout_desc is not None
        self.reg_timeout_desc = reg_timeout_desc
        self.reg_timer = None if not self.must_register else \
                                        self.new_timer(self.reg_timeout_desc, REG_TIMER)

        if self.must_register:
            self.reg_timer.start()

    def is_local(self):
        raise NotImplementedError()

    def timeout_handler(self, custom_method, args, type_):
        self._do_handle_timeout(type_)
        custom_method(*args)

    def _do_handle_timeout(self, type_):
        raise NotImplementedError()

    def new_timer(self, tim_desc, type_):
        return threading.Timer(tim_desc.timeout, self.timeout_handler,
                            [tim_desc.timeout_method, tim_desc.timeout_args, type_])

    def registered(self, reg_data):
        if self.reg_timer is not None:
            self.reg_timer.cancel()

        self.logger.info("{0} [{1}]  REGISTERED!!! {2}".format(
                                            self.name, self.proc_type, reg_data.machine_ip))
        #print "ping:", self.ping_it, "ret:", self.check_returncode
        with self._status_lock:
            self._status = RUNNING
        #TODO validate registration data
        self.registration_data = reg_data
        self.logger.info("reg_data" + str(vars(reg_data)))
        if self.ping_it:
            if not self._ctx:
                self._ctx = zmq.Context()
            self.rq_sock = self._ctx.socket(zmq.REQ)
            for addr in reg_data.rep_addrs:
                if reg_data.machine_ip != socket.gethostname() and\
                                                        net.addr_is_local(addr):
                    continue
                self.logger.debug(self.name + "connecting to " + addr)
                self.rq_sock.connect(addr)


    def stop_monitoring(self):
        if self.reg_timer:
            self.reg_timer.cancel()
            self.reg_timer = None
        self._stop_monitoring = True

        if self._ping_thread is not None:
            self.logger.info("%s, %s, %s",
                            self.proc_type, self.name ,"Joining ping thread")
            
            self._ping_thread.join()
        if self._returncode_thread is not None:
            self.logger.info("%s  %s  %s",
                            self.proc_type,self.name, "joining returncode thread")
            self._returncode_thread.join()
        self.logger.info("monitor for: %s, %s, %s", 
                    self.proc_type,self.name, "  ...monitoring threads stopped.")

    def finished(self):
        finished = True
        if self._ping_thread is not None:
            finished = not self._ping_thread.is_alive()
        if self._returncode_thread is not None:
            finished = finished and not self._returncode_thread.is_alive()
        return finished

    def process_is_running(self):
        running = True
        if self._ping_thread is not None:
            running = self._ping_thread.is_alive()
        if self._returncode_thread is not None:
            running = running and self._returncode_thread.is_alive()
        return running

    def start_monitoring(self):
        if self.ping_it:
            self._ping_thread = threading.Thread(target=self.ping_monitor, args=())
            self._ping_thread.daemon = True
            self._ping_thread.start()
        if self.check_returncode:
            self._returncode_thread = threading.Thread(target=self.returncode_monitor, args=())
            self._returncode_thread.daemon = True
            self._returncode_thread.start()

    def ping_monitor(self):
        is_alive = True
        try:
            while not self._stop_monitoring and is_alive:
                time.sleep(2)
                if self.rq_sock is not None:
                    send_msg(self.rq_sock, self._mtool.fill_msg('ping'))
                    result = None
                    while self._ping_retries and not result and not self._stop_monitoring:
                        result, det = self._poller.poll_recv(socket=self.rq_sock, timeout=1500)
                    if not result and not self._stop_monitoring:
                        self.logger.info("%s %s %s", 
                                self.proc_type, self.name, "NO RESPONSE TO PING!")
                        with self._status_lock:
                            if self._status not in [FAILED, FINISHED]:
                                self._status = NON_RESPONSIVE
                                self._status_details = 'ping response timeout'
                            print "status:", self._status
                            is_alive = False
        finally:
            self.rq_sock.close(linger=0)


    def returncode_monitor(self):
        raise NotImplementedError()

    def kill(self):
        raise NotImplementedError()

    def mark_delete(self):
        with self._status_lock:
            self.delete = True

    def marked_delete(self):
        with self._status_lock:
            return self.delete
Beispiel #39
0
class OBCIConnection(object):
    """
    Synchronous OBCI connection.
    """
    def __init__(self, address):
        templates = launcher_messages.message_templates
        self.msg_factory = OBCIMessageTool(msg_templates=templates)
        self.netstring_codec = NetstringCodec()
        self.address = address
        self.experiment_uuid = None
    
    def close(self):
        self.connection.close()
    
    def open(self):
        self.connection = socket.create_connection(self.address, timeout=30)
    
    def send_recv(self, message_name, **kwargs):
        message_json = self.msg_factory.fill_msg(message_name, **kwargs)
        message_netstring = self.netstring_codec.encode(message_json)
        self.open()
        self.connection.send(message_netstring)
        response_json = self.netstring_codec.decode(self.connection)
        self.close()
        response_dict = self.msg_factory.decode_msg(response_json)
        return response_dict
    
    def get_amp_list(self):
        message = self.msg_factory.fill_msg("find_eeg_amplifiers")
        message = self.netstring_codec.encode(message)
        self.open()
        self.connection.send(message)
        response_text = self.netstring_codec.decode(self.connection)
        response = self.msg_factory.decode_msg(response_text)
        self.close()
        amp_list = response["amplifier_list"]
        return amp_list
    
    def start_eeg_signal(self, name, launch_file, amplifier_params):
        message = self.msg_factory.fill_msg("start_eeg_signal", launch_file=launch_file, amplifier_params=amplifier_params, name=name)
        message_text = self.netstring_codec.encode(message)
        self.open()
        self.connection.send(message_text)
        response_text = self.netstring_codec.decode(self.connection)
        response = self.msg_factory.decode_msg(response_text)
        self.close()
        return response["sender"]
    
    def get_experiment_contact(self, name):
        response = self.send_recv("get_experiment_contact", strname=name)
        self.experiment_uuid = response["uuid"]
        return self.experiment_uuid
    
    def stop_experiment(self, experiment_uuid=None):
        if not experiment_uuid:
            experiment_uuid = self.experiment_uuid
        message = self.msg_factory.fill_msg("kill_experiment", strname=experiment_uuid)
        message_text = self.netstring_codec.encode(message)
        self.open()
        self.connection.send(message_text)
        response_text = self.netstring_codec.decode(self.connection)
        self.close()
        response = self.msg_factory.decode_msg(response_text)
        return response
    
    def create_experiment(self, name="unnamed"):
        message = self.msg_factory.fill_msg("create_experiment", name=name)
        message_text = self.netstring_codec.encode(message)
        self.open()
        self.connection.send(message_text)
        response_text = self.netstring_codec.decode(self.connection)
        self.close()
        return self.msg_factory.decode_msg(response_text)
    
    def set_experiment_scenario(self, launch_file_path="", scenario=""):
        return self.send_recv("set_experiment_scenario", launch_file_path=launch_file_path, scenario=scenario)
    
    def start_experiment(self):
        return self.send_recv("start_experiment")
    
    def get_nearby_servers(self):
        response = self.send_recv("list_nearby_machines")
        return response
Beispiel #40
0
 def __init__(self, address):
     templates = launcher_messages.message_templates
     self.msg_factory = OBCIMessageTool(msg_templates=templates)
     self.netstring_codec = NetstringCodec()
     self.address = address
     self.experiment_uuid = None
Beispiel #41
0
class OBCIClient(object):
    default_timeout = 5000

    def __init__(self, server_addresses, zmq_context=None):
        self.ctx = zmq_context if zmq_context else zmq.Context()

        self.server_addresses = server_addresses
        self.server_req_socket = None
        self.init_server_socket(server_addresses)

        #self = zmq.Poller()
        #self.register(self.server_req_socket, zmq.POLLIN)
        self.poller = PollingObject()

        self.mtool = OBCIMessageTool(message_templates)
        self.dns = net.DNS()

    def init_server_socket(self, srv_addrs):
        if self.server_req_socket is not None:
            print "server socket restart"
            self.server_req_socket.close()

        self.server_req_socket = self.ctx.socket(zmq.REQ)

        for addr in srv_addrs:
            print addr
            self.server_req_socket.connect(addr)

    def launch(self,
               launch_file=None,
               sandbox_dir=None,
               name=None,
               overwrites=None):
        result = self.send_create_experiment(launch_file, sandbox_dir, name,
                                             overwrites)
        print "create result:", result
        if not result:
            self.init_server_socket(self.server_addresses)
            return result
        if result.type != "experiment_created":
            return result
        print result
        machine = result.origin_machine
        addrs = [
            addr for addr in result.rep_addrs
            if self._addr_connectable(addr, machine)
        ]

        return self.send_start_experiment(addrs)

    def morph(self,
              exp_strname,
              launch_file,
              name=None,
              overwrites=None,
              leave_on=None):
        response = self.get_experiment_contact(exp_strname)
        exp_sock = self.ctx.socket(zmq.REQ)

        try:
            if response.type == "rq_error" or response.type == "no_data":
                return response
            for addr in response.rep_addrs:
                exp_sock.connect(addr)

            msg = self.mtool.fill_msg('morph_to_new_scenario',
                                      launch_file=launch_file,
                                      name=name,
                                      overwrites=overwrites,
                                      leave_on=leave_on)
            send_msg(exp_sock, msg)

            response, details = self.poll_recv(exp_sock, 6000)
            return response
        finally:
            exp_sock.close()

    def _addr_connectable(self, addr, machine):
        return machine == socket.gethostname() or \
                    (net.is_ip(addr) and not net.addr_is_local(addr))

    def start_chosen_experiment(self, exp_strname):
        response = self.get_experiment_contact(exp_strname)

        if response.type == "rq_error" or response.type == "no_data":
            return response

        return self.send_start_experiment(response.rep_addrs)

    def send_start_experiment(self, exp_addrs):
        exp_sock = self.ctx.socket(zmq.REQ)
        try:
            for addr in exp_addrs:
                exp_sock.connect(addr)

                send_msg(exp_sock, self.mtool.fill_msg("start_experiment"))
                reply, details = self.poll_recv(exp_sock, 20000)

                #print reply
                return reply
        finally:
            exp_sock.close()

    def force_kill_experiment(self, strname):
        pass

    def get_experiment_contact(self, strname):
        send_msg(
            self.server_req_socket,
            self.mtool.fill_msg("get_experiment_contact", strname=strname))
        response, details = self.poll_recv(self.server_req_socket,
                                           self.default_timeout)
        return response

    def ping_server(self, timeout=50):
        send_msg(self.server_req_socket, self.mtool.fill_msg("ping"))
        response, details = self.poll_recv(self.server_req_socket, timeout)

        return response

    def retry_ping(self, timeout=50):
        response, details = self.poll_recv(self.server_req_socket, timeout)
        return response

    def send_create_experiment(self,
                               launch_file=None,
                               sandbox_dir=None,
                               name=None,
                               overwrites=None):

        send_msg(
            self.server_req_socket,
            self.mtool.fill_msg("create_experiment",
                                launch_file=launch_file,
                                sandbox_dir=sandbox_dir,
                                name=name,
                                overwrites=overwrites))

        response, details = self.poll_recv(self.server_req_socket, 5000)
        return response

    def send_list_experiments(self):
        send_msg(self.server_req_socket,
                 self.mtool.fill_msg("list_experiments"))

        response, details = self.poll_recv(self.server_req_socket, 4000)
        return response

    def send_list_nearby_machines(self):
        send_msg(self.server_req_socket,
                 self.mtool.fill_msg("list_nearby_machines"))
        response, details = self.poll_recv(self.server_req_socket, 4000)
        return response

    def get_experiment_details(self, strname, peer_id=None):
        response = self.get_experiment_contact(strname)

        if response.type == "rq_error" or response.type == "no_data":
            return response

        sock = self.ctx.socket(zmq.REQ)
        try:
            for addr in response.rep_addrs:
                sock.connect(addr)

            if peer_id:
                send_msg(sock,
                         self.mtool.fill_msg("get_peer_info", peer_id=peer_id))
            else:
                send_msg(sock, self.mtool.fill_msg("get_experiment_info"))
            response, details = self.poll_recv(sock, 2000)

            return response
        finally:
            sock.close()

    def configure_peer(self,
                       exp_strname,
                       peer_id,
                       config_overrides,
                       override_files=None):
        response = self.get_experiment_contact(exp_strname)

        if response.type == "rq_error" or response.type == "no_data":
            return response

        sock = self.ctx.socket(zmq.REQ)
        try:
            for addr in response.rep_addrs:
                sock.connect(addr)

            if override_files:
                send_msg(sock,
                         self.mtool.fill_msg("get_peer_info", peer_id=peer_id))
                response, details = self.poll_recv(sock, 2000)
                if response.type is 'rq_error':
                    return response

            msg = self.mtool.fill_msg("update_peer_config",
                                      peer_id=peer_id,
                                      **config_overrides)

            send_msg(sock, msg)
            #print msg
            response, details = self.poll_recv(sock, 2000)
            return response
        finally:
            sock.close()

            # "update_peer_config" : dict(peer_id='', local_params='',
            #                 external_params='', launch_dependencies='', config_sources=''),

    def kill_exp(self, strname, force=False):
        send_msg(self.server_req_socket,
                 self.mtool.fill_msg("kill_experiment", strname=strname))
        return self.poll_recv(self.server_req_socket, 2000)[0]

    def srv_kill(self):
        send_msg(self.server_req_socket, self.mtool.fill_msg("kill"))
        return self.poll_recv(self.server_req_socket, 2000)[0]

    def join_experiment(self, strname, peer_id, path):
        response = self.get_experiment_contact(strname)

        if response.type == "rq_error" or response.type == "no_data":
            return response

        sock = self.ctx.socket(zmq.REQ)

        try:
            self._connect(sock, response.rep_addrs)

            send_msg(
                sock,
                self.mtool.fill_msg("join_experiment",
                                    peer_id=peer_id,
                                    path=path,
                                    peer_type='obci_peer'))
            response, details = self.poll_recv(sock, 5000)
            sock.close()
            return response
        finally:
            sock.close()

    def add_peer(self,
                 strname,
                 peer_id,
                 path,
                 machine,
                 param_overwrites=None,
                 custom_config_path=None,
                 config_sources=None,
                 launch_dependencies=None,
                 apply_globals=True):
        response = self.get_experiment_contact(strname)

        if response.type == "rq_error" or response.type == "no_data":
            return response

        sock = self.ctx.socket(zmq.REQ)
        try:
            self._connect(sock, response.rep_addrs)
            send_msg(
                sock,
                self.mtool.fill_msg("add_peer",
                                    peer_id=peer_id,
                                    peer_path=path,
                                    peer_type='obci_peer',
                                    machine=machine,
                                    param_overwrites=param_overwrites,
                                    custom_config_path=custom_config_path,
                                    config_sources=config_sources,
                                    launch_dependencies=launch_dependencies,
                                    apply_globals=apply_globals))
            response, details = self.poll_recv(sock, 5000)
            return response
        finally:
            sock.close()

    def kill_peer(self, exp_strname, peer_id, remove_config=False):
        response = self.get_experiment_contact(exp_strname)

        if response.type == "rq_error" or response.type == "no_data":
            return response

        try:
            sock = self.ctx.socket(zmq.REQ)
            self._connect(sock, response.rep_addrs)
            send_msg(
                sock,
                self.mtool.fill_msg("kill_peer",
                                    peer_id=peer_id,
                                    remove_config=remove_config))
            response, details = self.poll_recv(sock, 5000)
            return response
        finally:
            sock.close()

    def _connect(self, sock, addr_list):
        print "****  ", addr_list
        this = self._is_this_machine(addr_list)
        connected = False
        for addr in addr_list:
            if not this and ('localhost' in addr or '127.0.0.1' in addr):
                continue
            try:
                sock.connect(addr)
                connected = True
            except zmq.ZMQError, e:
                print addr, " ::: ", str(e)
        if not connected:
            raise Exception("Could not connect to any of the addresses: " +
                            str(addr_list))
Beispiel #42
0
class Process(object):
    def __init__(self,
                 proc_description,
                 reg_timeout_desc=None,
                 monitoring_optflags=PING,
                 logger=None):

        self.desc = proc_description

        self.must_register = reg_timeout_desc is not None
        self._status_lock = threading.RLock()
        self._status = UNKNOWN if self.must_register else RUNNING
        self._status_details = None

        self.ping_it = monitoring_optflags & PING
        self.check_returncode = monitoring_optflags & RETURNCODE if \
            self.desc.pid is not None else False

        self.logger = logger or get_logger(
            'subprocess_monitor' + '-' + self.desc.name + '-' +
            str(self.desc.pid),
            stream_level='info')
        self.set_registration_timeout_handler(reg_timeout_desc)
        self.registration_data = None

        self._stop_monitoring = False
        self._ping_thread = None
        self._ping_retries = 8
        self._returncode_thread = None
        self._mtool = OBCIMessageTool(message_templates)
        self._ctx = None
        self.rq_sock = None
        self._poller = PollingObject()

        self.delete = False

    @property
    def machine_ip(self):
        return self.desc.machine_ip

    @property
    def pid(self):
        return self.desc.pid

    @property
    def path(self):
        return self.desc.path

    @property
    def proc_type(self):
        return self.desc.proc_type

    @property
    def name(self):
        return self.desc.name

    def status(self):
        with self._status_lock:
            return self._status, self._status_details

    def set_registration_timeout_handler(self, reg_timeout_desc):
        with self._status_lock:
            self._status = UNKNOWN
            self._status_details = None
        self.must_register = reg_timeout_desc is not None
        self.reg_timeout_desc = reg_timeout_desc
        self.reg_timer = None if not self.must_register else \
            self.new_timer(self.reg_timeout_desc, REG_TIMER)

        if self.must_register:
            self.reg_timer.start()

    def is_local(self):
        raise NotImplementedError()

    def timeout_handler(self, custom_method, args, type_):
        self._do_handle_timeout(type_)
        custom_method(*args)

    def _do_handle_timeout(self, type_):
        raise NotImplementedError()

    def new_timer(self, tim_desc, type_):
        return threading.Timer(
            tim_desc.timeout, self.timeout_handler,
            [tim_desc.timeout_method, tim_desc.timeout_args, type_])

    def registered(self, reg_data):
        if self.reg_timer is not None:
            self.reg_timer.cancel()

        self.logger.info("{0} [{1}]  REGISTERED!!! {2}".format(
            self.name, self.proc_type, reg_data.machine_ip))
        # print "ping:", self.ping_it, "ret:", self.check_returncode
        with self._status_lock:
            self._status = RUNNING
        # TODO validate registration data
        self.registration_data = reg_data
        self.logger.info("reg_data" + str(vars(reg_data)))
        if self.ping_it:
            if not self._ctx:
                self._ctx = zmq.Context()
            self.rq_sock = self._ctx.socket(zmq.REQ)
            for addr in reg_data.rep_addrs:
                if reg_data.machine_ip != socket.gethostname() and\
                        net.addr_is_local(addr):
                    continue
                self.logger.debug(self.name + "connecting to " + addr)
                self.rq_sock.connect(addr)

    def stop_monitoring(self):
        if self.reg_timer:
            self.reg_timer.cancel()
            self.reg_timer = None
        self._stop_monitoring = True

        if self._ping_thread is not None:
            self.logger.info("%s, %s, %s", self.proc_type, self.name,
                             "Joining ping thread")

            self._ping_thread.join()
        if self._returncode_thread is not None:
            self.logger.info("%s  %s  %s", self.proc_type, self.name,
                             "joining returncode thread")
            self._returncode_thread.join()
        self.logger.info("monitor for: %s, %s, %s", self.proc_type, self.name,
                         "  ...monitoring threads stopped.")

    def finished(self):
        finished = True
        if self._ping_thread is not None:
            finished = not self._ping_thread.is_alive()
        if self._returncode_thread is not None:
            finished = finished and not self._returncode_thread.is_alive()
        return finished

    def process_is_running(self):
        running = True
        if self._ping_thread is not None:
            running = self._ping_thread.is_alive()
        if self._returncode_thread is not None:
            running = running and self._returncode_thread.is_alive()
        return running

    def start_monitoring(self):
        if self.ping_it:
            self._ping_thread = threading.Thread(target=self.ping_monitor,
                                                 args=())
            self._ping_thread.daemon = True
            self._ping_thread.start()
        if self.check_returncode:
            self._returncode_thread = threading.Thread(
                target=self.returncode_monitor, args=())
            self._returncode_thread.daemon = True
            self._returncode_thread.start()

    def ping_monitor(self):
        is_alive = True
        try:
            while not self._stop_monitoring and is_alive:
                time.sleep(2)
                if self.rq_sock is not None:
                    send_msg(self.rq_sock, self._mtool.fill_msg('ping'))
                    result = None
                    while self._ping_retries and not result and not self._stop_monitoring:
                        result, det = self._poller.poll_recv(
                            socket=self.rq_sock, timeout=1500)
                    if not result and not self._stop_monitoring:
                        self.logger.info("%s %s %s", self.proc_type, self.name,
                                         "NO RESPONSE TO PING!")
                        with self._status_lock:
                            if self._status not in [FAILED, FINISHED]:
                                self._status = NON_RESPONSIVE
                                self._status_details = 'ping response timeout'
                            print "status:", self._status
                            is_alive = False
        finally:
            if self.rq_sock is not None:
                self.rq_sock.close(linger=0)

    def returncode_monitor(self):
        raise NotImplementedError()

    def kill(self):
        raise NotImplementedError()

    def mark_delete(self):
        with self._status_lock:
            self.delete = True

    def marked_delete(self):
        with self._status_lock:
            return self.delete
Beispiel #43
0
class OBCIConnection(object):
    """
    Synchronous OBCI connection.
    """
    def __init__(self, address):
        templates = launcher_messages.message_templates
        self.msg_factory = OBCIMessageTool(msg_templates=templates)
        self.netstring_codec = NetstringCodec()
        self.address = address
        self.experiment_uuid = None

    def close(self):
        self.connection.close()

    def open(self):
        self.connection = socket.create_connection(self.address, timeout=30)

    def send_recv(self, message_name, **kwargs):
        message_json = self.msg_factory.fill_msg(message_name, **kwargs)
        message_netstring = self.netstring_codec.encode(message_json)
        self.open()
        self.connection.send(message_netstring)
        response_json = self.netstring_codec.decode(self.connection)
        self.close()
        response_dict = self.msg_factory.decode_msg(response_json)
        return response_dict

    def get_amp_list(self):
        message = self.msg_factory.fill_msg("find_eeg_amplifiers")
        message = self.netstring_codec.encode(message)
        self.open()
        self.connection.send(message)
        response_text = self.netstring_codec.decode(self.connection)
        response = self.msg_factory.decode_msg(response_text)
        self.close()
        amp_list = response["amplifier_list"]
        return amp_list

    def start_eeg_signal(self, name, launch_file, amplifier_params):
        message = self.msg_factory.fill_msg("start_eeg_signal",
                                            launch_file=launch_file,
                                            amplifier_params=amplifier_params,
                                            name=name)
        message_text = self.netstring_codec.encode(message)
        self.open()
        self.connection.send(message_text)
        response_text = self.netstring_codec.decode(self.connection)
        response = self.msg_factory.decode_msg(response_text)
        self.close()
        return response["sender"]

    def get_experiment_contact(self, name):
        response = self.send_recv("get_experiment_contact", strname=name)
        self.experiment_uuid = response["uuid"]
        return self.experiment_uuid

    def stop_experiment(self, experiment_uuid=None):
        if not experiment_uuid:
            experiment_uuid = self.experiment_uuid
        message = self.msg_factory.fill_msg("kill_experiment",
                                            strname=experiment_uuid)
        message_text = self.netstring_codec.encode(message)
        self.open()
        self.connection.send(message_text)
        response_text = self.netstring_codec.decode(self.connection)
        self.close()
        response = self.msg_factory.decode_msg(response_text)
        return response

    def create_experiment(self, name="unnamed"):
        message = self.msg_factory.fill_msg("create_experiment", name=name)
        message_text = self.netstring_codec.encode(message)
        self.open()
        self.connection.send(message_text)
        response_text = self.netstring_codec.decode(self.connection)
        self.close()
        return self.msg_factory.decode_msg(response_text)

    def set_experiment_scenario(self, launch_file_path="", scenario=""):
        return self.send_recv("set_experiment_scenario",
                              launch_file_path=launch_file_path,
                              scenario=scenario)

    def start_experiment(self):
        return self.send_recv("start_experiment")

    def get_nearby_servers(self):
        response = self.send_recv("list_nearby_machines")
        return response
Beispiel #44
0
class ExperimentEngineInfo(QtCore.QObject):
    exp_saver_msg = QtCore.pyqtSignal(object)

    def __init__(self, preset_data=None, launcher_data=None, ctx=None):
        self.preset_data = preset_data
        self.launch_file = None
        self.launcher_data = launcher_data
        self.name = None
        self.info = ""
        self.public_params = []
        self.origin_machine = ''
        self.unsupervised_peers = {}
        self.old_uid = None

        self.ctx = ctx
        self.exp_req = None
        self.mtool = OBCIMessageTool(message_templates)
        self.poller = PollingObject()

        self.category = DEFAULT_CATEGORY

        if preset_data is not None:
            self.setup_from_preset(preset_data)

        elif launcher_data is not None:
            self.setup_from_launcher(launcher_data)
        super(ExperimentEngineInfo, self).__init__()

    def cleanup(self):
        if self.exp_req:
            self.exp_req.close()  # linger=0)

    def setup_from_preset(self, preset_data, launcher=False):
        self.preset_data = preset_data
        self.overwrites = {}
        self.runtime_changes = {}

        self.status = launcher_tools.ExperimentStatus()
        self.exp_config = system_config.OBCIExperimentConfig()

        self.name = preset_data['name']
        self.launch_file = preset_data['launch_file']
        self.info = preset_data['info']
        self.public_params = [p.strip() for p in preset_data['public_params'].split(',')]
        self.exp_config.uuid = self.name + '--' + self.launch_file

        self.category = preset_data['category']

        result, details = self._make_config()

        self.status.details = details
        self._set_public_params()

    def _addr_connectable(self, addr, machine):
        return machine == socket.gethostname() or \
            (net.is_ip(addr) and not net.addr_is_local(addr))

    def setup_from_launcher(self, launcher_data, preset=False, transform=False):
        self.launcher_data = launcher_data
        self.runtime_changes = {}
        if preset:
            self.old_uid = self.exp_config.uuid

        if not preset or transform:
            self.overwrites = {}
            self.status = launcher_tools.ExperimentStatus()
            self.exp_config = system_config.OBCIExperimentConfig()

        self.name = launcher_data['name']  # if not preset else self.old_uid
        if not preset:
            self.launch_file = launcher_data['launch_file_path']

        connected = False
        if not transform:
            self.ctx = self.ctx if self.ctx is not None else zmq.Context()
            self.exp_req = self.ctx.socket(zmq.REQ)
            machine = launcher_data['origin_machine']

            for addr in launcher_data['rep_addrs']:
                if self._addr_connectable(addr, machine):
                    try:
                        self.exp_req.connect(addr)
                    except Exception as e:
                        print(addr, False)
                    else:
                        connected = True
            if not connected:
                print("Connection to experiment ", self.name, "UNSUCCESFUL!!!!!!")
                return

        self.exp_config.uuid = launcher_data['uuid']
        self.exp_config.origin_machine = launcher_data['origin_machine']

        self.uuid = self.exp_config.uuid

        self.exp_config.launch_file_path = self.launch_file
        result, details = self._get_experiment_scenario()
        self.exp_config.status(self.status)
        self.status.set_status(launcher_data['status_name'], details=launcher_data['details'])

        self._get_experiment_details()
        self._set_public_params()

    def update_scenario(self, launch_file_path, scenario):
        self.exp_config.launch_file_path = launch_file_path
        self._process_experiment_scenario(scenario)
        self.exp_config.status(self.status)

    def _set_public_params(self):
        for par in self.public_params:
            if len(par.split('.')) == 2:
                [peer, param] = par.split('.')
                self.exp_config.peers[peer].public_params.append(param)

    def _make_config(self):

        self.exp_config.launch_file_path = self.launch_file
        self.uuid = self.exp_config.uuid

        result, details = self.make_experiment_config()

        self.exp_config.status(self.status)
        return result, details

    # FIXME  !!! copy-paste from obci_experiment
    def make_experiment_config(self):
        launch_parser = launch_file_parser.LaunchFileParser(
            launcher_tools.obci_root(), settings.DEFAULT_SCENARIO_DIR)
        if not self.launch_file:
            return False, "Empty scenario."

        try:
            with open(launcher_tools.expand_path(self.launch_file)) as f:
                launch_parser.parse(f, self.exp_config, apply_globals=True)
        except Exception as e:
            self.status.set_status(launcher_tools.NOT_READY, details=str(e))
            print("config errror   ", str(e))
            return False, str(e)

        rd, details = self.exp_config.config_ready()
        if rd:
            self.status.set_status(launcher_tools.READY_TO_LAUNCH)
        else:
            self.status.set_status(launcher_tools.NOT_READY, details=details)
            print(rd, details)

        return True, None

    def _get_experiment_scenario(self):
        if not self.exp_req:
            return False, "No experiment socket"
        response = self.comm_exp(self.mtool.fill_msg("get_experiment_scenario"))
        if not response:
            return False, "No response from experient"
        print("GOT SCENARIO", response.scenario)
        return self._process_experiment_scenario(response.scenario)

    def _process_experiment_scenario(self, json_scenario):
        jsonpar = launch_file_parser.LaunchJSONParser(
            launcher_tools.obci_root(), settings.DEFAULT_SCENARIO_DIR)
        inbuf = io.StringIO(json_scenario)
        jsonpar.parse(inbuf, self.exp_config)
        print("MY PEEEEERS:", self.exp_config.peers.keys())
        rd, details = self.exp_config.config_ready()
        if rd:
            self.status.set_status(launcher_tools.READY_TO_LAUNCH)
        else:
            self.status.set_status(launcher_tools.NOT_READY, details=details)
            print(rd, details)
        return True, None

    def _get_experiment_details(self):
        if not self.exp_req:
            return
        exp_msg = self.comm_exp(self.mtool.fill_msg("get_experiment_info"))
        if not exp_msg:
            return

        self.origin_machine = exp_msg.origin_machine

        for peer, short_info in exp_msg.peers.items():
            # self.exp_config.set_peer_machine(peer, short_info['machine'])

            msg = self.comm_exp(self.mtool.fill_msg("get_peer_info",
                                                    peer_id=peer))
            if not msg:
                return

            ext_defs = {}
            for name, defi in msg.external_params.items():
                ext_defs[name] = defi[0] + '.' + defi[1]
            self.exp_config.update_peer_config(peer, dict(config_sources=msg.config_sources,
                                                          launch_dependencies=msg.launch_dependencies,
                                                          local_params=msg.local_params,
                                                          external_params=ext_defs))

        for peer, status in exp_msg.experiment_status['peers_status'].items():
            self.status.peer_status(peer).set_status(
                status['status_name'],
                details=status['details'])

    def parameters(self, peer_id, mode):
        params = {}
        peer = self.exp_config.peers[peer_id]
        if mode == MODE_BASIC:
            for par in peer.public_params:
                params[par] = (self.exp_config.param_value(peer_id, par), None)
        else:
            params = peer.config.local_params
            for param in peer.config.local_params:
                params[param] = (self.exp_config.param_value(peer_id, param), None)
            for param, defi in peer.config.ext_param_defs.items():
                source_symbol = defi[0]
                source = peer.config.config_sources[source_symbol]
                params[param] = (self.exp_config.param_value(peer_id, param), source + '.' + defi[1])
        return params

    def comm_exp(self, msg):
        send_msg(self.exp_req, msg)
        response, _ = self.poller.poll_recv(self.exp_req, timeout=3000)
        if not response:
            print("!!!!!!!!!!!!!!!!!!!!!!!!!!!1 no response to ", msg)
            self.exp_req.close()
            self.exp_req = self.ctx.socket(zmq.REQ)
            for addr in self.launcher_data['rep_addrs']:
                if self._addr_connectable(addr, self.launcher_data['origin_machine']):
                    self.exp_req.connect(addr)
            return None
        return self.mtool.unpack_msg(response)

    def updatable(self, peer_id, config_part, **kwargs):
        return False

    def update_peer_param(self, peer_id, param, value, runtime=False):
        changes = self.overwrites if not runtime else self.runtime_changes

        ovr = changes.get(peer_id, None)
        ovr = ovr if ovr is not None else {}
        if param not in ovr:
            old = self.exp_config.param_value(peer_id, param)
            if old != value:
                ovr[param] = old
                changes[peer_id] = ovr

        self.exp_config.update_local_param(peer_id, param, value)

    def get_launch_args(self):
        d = dict(launch_file=self.launch_file, name=self.name)
        args = ['--ovr']
        if self.overwrites:
            for peer_id in self.overwrites:
                args.append('--peer')
                args.append(peer_id)

                for arg in self.overwrites[peer_id]:
                    args += ['-p', arg, self.exp_config.param_value(peer_id, arg)]
        pack = peer_cmd.peer_overwrites_pack(args)
        d['overwrites'] = pack
        print("overwrites pack!!!!!!!!!!!!!!!!!!!!!  ", pack)
        return d

    def peer_info(self, peer_id):
        return self.exp_config.peers[peer_id]

    def add_peer(self, peer_id, peer_path, config_sources=None, launch_deps=None,
                 custom_config_path=None, param_overwrites=None, machine=None):

        return launch_file_parser.extend_experiment_config(
            self.exp_config,
            peer_id,
            peer_path,
            config_sources,
            launch_deps,
            custom_config_path,
            param_overwrites,
            machine,
            apply_globals=True)

    def enable_signal_storing(self, store_options):
        if not store_options:
            return

        if int(store_options['append_timestamp']):
            store_options = dict(store_options)
            store_options['save_file_name'] = store_options['save_file_name'] + "_" + str(time.time())

        for peer, peer_path in SIGNAL_STORAGE_PEERS.items():
            if peer not in self.exp_config.peers:
                self.add_peer(peer, peer_path)

        saver = self.exp_config.peers['signal_saver']
        params = saver.config.param_values
        for opt, val in store_options.items():
            if opt in saver.config.param_values:
                params[opt] = val

    def stop_storing(self, client):
        join_response = client.join_experiment(self.uuid, "dummy_module_" + str(time.time()), "")
        if join_response is None:
            print("experiment engine info - ERROR - connection timeout on stop signal storing!")
            return
        if not join_response.type == "rq_ok":
            print("experiment engine info - ERROR - join error on stop signal storing!")
            return
        mx_addr = join_response.params["mx_addr"].split(':')
        # hang and wait ...
        acquisition_helper.finish_saving([(mx_addr[0], int(mx_addr[1]))])
class EEGExperimentFinder(object):
    def __init__(self, srv_addrs, ctx, client_push_address, nearby_servers):
        self.ctx = ctx
        self.server_req_socket = self.ctx.socket(zmq.REQ)
        for addr in srv_addrs:
            self.server_req_socket.connect(addr)

        self.poller = PollingObject()
        self.mtool = OBCIMessageTool(message_templates)
        self.nearby_servers = nearby_servers
        self._amplified_cache = {}

    def _running_experiments(self):
        send_msg(self.server_req_socket,
                 self.mtool.fill_msg("list_experiments"))
        exp_list, details = self.poll_recv(self.server_req_socket, 2000)
        if not exp_list:
            LOGGER.error(
                "Connection to obci_server failed. (list_experiments)")
            return None

        exps = exp_list.exp_data
        running = []
        for exp in exps.values():
            if exp['status_name'] == launcher_tools.RUNNING or \
                    exp['status_name'] == launcher_tools.LAUNCHING:
                running.append(exp)
        return running

    def find_amplified_experiments(self):
        running_exps = self._running_experiments()
        amplified = []
        for exp in running_exps:
            LOGGER.info("Found running experiment: " + str(exp['name']))
            infos = self._info_amplified(exp)

            if infos is not None:
                print "Found experiments...", str(infos)[:500]
                amplified += infos

        return amplified

    def _info_amplified(self, exp_desc):
        amp_options = []
        LOGGER.info("Processing experiment "+ str(exp_desc['name']) +\
                                     "w/ addr: " + str(exp_desc['rep_addrs']))
        tcp_addrs = exp_desc['tcp_addrs']
        rep_addrs = net.choose_not_local(exp_desc['rep_addrs'])

        if not rep_addrs:
            rep_addrs = net.choose_local(exp_desc['rep_addrs'], ip=True)

        rep_addr = rep_addrs.pop()

        pub_addrs = net.choose_not_local(exp_desc['pub_addrs'])
        if not pub_addrs:
            pub_addrs = net.choose_local(exp_desc['pub_addrs'], ip=True)
        pub_addr = pub_addrs.pop()
        tcp_addr = tcp_addrs.pop()

        LOGGER.info("Chosen experiment addresses: REP -- " + \
                                str(rep_addr) + ", PUB -- " + str(pub_addr))

        req_sock = self.ctx.socket(zmq.REQ)
        try:
            req_sock.connect(rep_addr)

            send_msg(req_sock, self.mtool.fill_msg('get_experiment_info'))
            res, details = self.poll_recv(req_sock, 4000)
        finally:
            req_sock.close()

        if not res:
            LOGGER.error("Connection failed (experiment " + exp_desc['name'] + \
                                                    "), get_experiment_info")
            return None
        exp_info = res.dict()  #json.loads(res)
        for field in ["peers_status", "details"]:
            del exp_info["experiment_status"][field]

        peer_list = exp_info["peers"]
        if not self._has_mx(peer_list):
            LOGGER.info("Experiment " + exp_desc['name'] + \
                                                " does not have a multiplexer.")
            return None

        maybe_amps = self._amp_like_peers(peer_list)
        if not maybe_amps:
            LOGGER.info("Experiment "+ exp_desc['name'] + \
                                                " -- no amplifier.")
            return None

        req_sock = self.ctx.socket(zmq.REQ)
        try:
            req_sock.connect(rep_addr)
            for peer in maybe_amps:
                info, params = self._get_amp_info(req_sock, peer)
                if not self._is_amplifier(info, params):
                    LOGGER.info("Experiment "+ exp_desc['name'] + \
                                    " -- peer " + str(peer) + "is not an amplifier.")
                    continue
                else:
                    exp_data = self._create_exp_data(exp_info, info,
                                                     params['param_values'],
                                                     rep_addr, pub_addr,
                                                     tcp_addr)
                    amp_options.append(exp_data)
        finally:
            req_sock.close()

        return amp_options

    def _get_amp_info(self, exp_sock, peer_id):
        send_msg(exp_sock, self.mtool.fill_msg('get_peer_info',
                                               peer_id=peer_id))
        info, details = self.poll_recv(exp_sock, 4000)

        send_msg(exp_sock,
                 self.mtool.fill_msg('get_peer_param_values', peer_id=peer_id))
        params, details = self.poll_recv(exp_sock, 4000)
        if not info or not params:
            LOGGER.error("get_peer_info failed " + str(peer_id) + "  " +
                         str(details))
            return None, None
        info = info.dict()
        params = params.dict()
        for field in [
                "sender", "sender_ip", "receiver", "type", "local_params",
                "external_params", "config_sources", "launch_dependencies"
        ]:
            del info[field]
        return info, params

    def _is_amplifier(self, peer_info, peer_params):
        info = peer_info
        peer_id = info['peer_id']

        if not info['peer_type'] == 'obci_peer':
            LOGGER.info("Peer " + str(peer_id) + "  not obci_peer")
            return False

        params = peer_params['param_values']
        if not 'channels_info' in params or\
            not 'active_channels' in params:
            LOGGER.info('Peer  ' + str(peer_id) + "  no channels_info param.")
            return False
        return True

    def _create_exp_data(self, exp_info, peer_info, params, rep_addr, pub_addr,
                         tcp_addr):
        data = {}
        data['amplifier_params'] = params
        data['amplifier_peer_info'] = peer_info
        data['experiment_info'] = exp_info
        data['rep_addrs'] = [rep_addr]
        data['pub_addrs'] = [pub_addr]
        data['tcp_addrs'] = [tcp_addr]
        return data

    def _has_mx(self, peer_list):
        return [peer for peer in peer_list if peer.startswith('mx')] != []

    def _amp_like_peers(self, peer_list):
        return [peer for peer in peer_list if peer.startswith('amplifier')]

    def poll_recv(self, socket, timeout):
        result, details = self.poller.poll_recv(socket, timeout)
        if result:
            result = self.mtool.unpack_msg(result)

        return result, details
#!/usr/bin/python
# -*- coding: utf-8 -*-

import json
import socket
import time

from obci.control.common.message import OBCIMessageTool
import obci.control.common.net_tools as net

from obci.control.launcher.launcher_messages import message_templates

from obci.control.launcher.plain_tcp_handling import make_unicode_netstring, recv_unicode_netstring

mtool = OBCIMessageTool(message_templates)


def send_and_receive(host, port, msg):
    netstr = make_unicode_netstring(msg)
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    wfile = None
    rfile = None

    rec_msg = None
    try:
        # Connect to server and send data
        sock.connect((host, port))
        rfile = sock.makefile('rb', -1)
        wfile = sock.makefile('wb', 0)
        wfile.write(netstr)
Beispiel #47
0
 def __init__(self, address):
     templates = launcher_messages.message_templates
     self.msg_factory = OBCIMessageTool(msg_templates=templates)
     self.netstring_codec = NetstringCodec()
     self.address = address
     self.experiment_uuid = None