Пример #1
0
    def sync_mapping(self):
        """TODO."""
        self.map_to_minions = messages.compute_map_master(
            self.vid, self.var_pt.view(np.int8))
        print(self.fid[self.pf_list])

        # send mapping to minions
        tag = messages.SYNC_MAPPING
        data = {
            "map": messages.serialize(self.map_to_minions),
            "pf": messages.serialize(self.fid[self.pf_to_send])
        }
        if self.num_minions != 0:
            pub_func = partial(send_to_minion, data, tag)
            self.clientPool.imap(pub_func, self.minion2host.values())
        #newEvent = self.local_client.cmd(self.minions,
        #                                 'event.fire',
        #                                 [data, tag],
        #                                 expr_form='list')

        self.map_from_minion = [None for i in range(len(self.minions))]
        self.pf_from_minion = [None for i in range(len(self.minions))]
        self.ufo_from_minion = [None for i in range(len(self.minions))]
        self.ufo_map_from_minion = self.ufo_recv.copy()
        self.ufo_map_from_minion["vid"] = self.vid[
            self.ufo_map_from_minion["vid"]]
        resp = 0
        while resp < len(self.minions):
            # receive map and save
            evdata = self.event_bus.get_event(wait=5,
                                              tag=messages.SYNC_MAPPING_RES,
                                              full=True)
            if evdata:
                tag, data = evdata['tag'], evdata['data']['data']
                pid = data["pid"]
                self.map_from_minion[pid] = \
                    messages.deserialize(data["map"], np.int64)
                messages.apply_inverse_map(self.vid, self.map_from_minion[pid])
                self.pf_from_minion[pid] = messages.deserialize(
                    data["pf"], np.int64)
                messages.apply_inverse_map(self.fid, self.pf_from_minion[pid])
                self.ufo_from_minion[pid] = messages.ufo_to_factor(
                    messages.deserialize(data["ufo"], UnaryFactorOpt),
                    self.ufo_map_from_minion, len(self.factor_pt))
                resp += 1
        print("DONE WITH SENDING MAPPING")

        messages.apply_inverse_map(self.vid, self.map_to_minions)
Пример #2
0
    async def serve(self, socket, path):
        user = User(coolname.generate_slug(2), socket)
        logger.info("%s connected", user)

        while True:
            try:
                data = await user.socket.recv()
            except (
                websockets.exceptions.ConnectionClosedError,
                websockets.exceptions.ConnectionClosedOK,
            ) as e:
                logger.info("%s closed connection (%s)", user, e)
                await self.stage.on_disconnect(user)
                break

            message = messages.deserialize(data)
            logger.info("Received %s from %s", message, user)

            if isinstance(message, messages.AuthCode):
                await self.stage.on_auth_code(user, message)
            elif isinstance(message, messages.CountCode):
                await self.stage.on_count_code(user, message)
            elif isinstance(message, messages.QuestionAnswers):
                await self.stage.on_question_answers(user, message)
            else:
                raise NotImplementedError(f"Unsupported message {message}")
Пример #3
0
    def run(self):

        self.connect()

        while self.end is not True:
            message = protocol.recv_one_message(self.socket)
            self.handle_message(messages.deserialize(message))
Пример #4
0
def demander(q, j):
    informer("QUESTION : " + q)
    protocol.send_one_message(clients[j.numero], messages.Question(q).toJson())
    r = protocol.recv_one_message(clients[j.numero])
    r = messages.deserialize(r)
    informer("REPONSE DONNEE : " + str(r.content))
    return str(r.content)
Пример #5
0
    def send(self, sender, host_port, message):
        print "TRANSPORT SENDS", messages.deserialize(message)
        for cb in self.on_send_cbs:
            cb(sender, host_port, message)

        f = self.protocols[host_port].receive
        gevent.spawn_later(0.0001, f, message)
Пример #6
0
    def read_all(self, filename):
        """
        Given a filename, it opens it and read all data into memory and return
        """
        with open(filename, 'rb') as f:
            d = f.read()

        return deserialize(d)
Пример #7
0
    def receive(self, data):
        assert len(data) < self.max_message_size

        # check if we handled this message already, if so repeat Ack
        h = sha3(data)
        if h in self.sent_acks:
            # assert False, "DEACTIVATED ACK RESENTS"
            return self.send_ack(*self.sent_acks[h])

        # note, we ignore the sending endpoint, as this can not be known w/ UDP
        msg = messages.deserialize(data)
        # handle Acks
        if isinstance(msg, Ack):
            del self.tries[msg.echo]
            return

        assert isinstance(msg, Secret) or msg.sender
        self.raiden.on_message(msg)
Пример #8
0
    def __handleReadAvailable(self):
        try:
            bytes = self.socket.recv(4096)

            if len(bytes) == 0:
                log.log("Detected socket close by other side; closing this side too")
                self.close()

            self.__readBuffer += bytes
            if self.protocolVersion == None:
                self.__tryReadProtocolVersion()
            if self.protocolVersion == None:
                return

                # There can be multiple messages in the read buffer,
                # so repeat until there is no more complete message
            while True:

                # Message detection:
                if len(self.__readBuffer) < 4:
                    return

                    # 4-byte unsigned int in network byte order:
                msgLen = struct.unpack("!I", self.__readBuffer[:4])[0]
                if len(self.__readBuffer) < msgLen + 4:
                    return

                msg = self.__readBuffer[4 : msgLen + 4]
                self.__readBuffer = self.__readBuffer[msgLen + 4 :]

                # print "Received serialized message: ", repr(msg)

                # de-serialize
                msg = messages.deserialize(msg)

                self.__handleMessage(msg)

        except:
            log.logException()
            self.close()
Пример #9
0
    def run(self):
        self.client.connect()
        self.authenticate(self.username, self.password)
        players = self.get_player_list()
        if players is not None:
            for player in players:
                self.add_player_info(player['Login'], player)
            #
        #

        self.plugins = Plugins(self)

        delay = 0.25
        prev_tick = time.time()
        prev_second = prev_tick
        while True:
            cbs = self.client.load_callbacks()

            for cb in cbs:
                object = messages.deserialize(cb)
                self.process_callback(object)
            #

            self.client.reset_callbacks()

            self.raise_event('tick', None)

            now = time.time()
            if now - prev_second >= 1.0:
                self.raise_event('second_passed', None)
                prev_second = now
            #

            now = time.time()
            duration = now - prev_tick
            if duration < delay:
                time.sleep(delay - duration)
            #
            prev_tick = now
Пример #10
0
 def loadFG(self, data):
     """TODO."""
     try:
         weight = np.fromstring(data['weight'], dtype=Weight)
         variable = np.fromstring(data['variable'], dtype=Variable)
         factor = messages.deserialize(data['factor'], Factor)
         fmap = np.fromstring(data['fmap'], dtype=FactorToVar)
         vmap = np.fromstring(data['vmap'], dtype=VarToFactor)
         factor_index = np.fromstring(data['factor_index'], dtype=np.int64)
     except:
         return 'FAILED LOADING', None
     try:
         self.ns.loadFactorGraphRaw(weight, variable, factor, fmap, vmap,
                                    factor_index)
         fg = self.ns.factorGraphs[-1]
         meta = {}
         meta['weights'] = fg.weight.shape[0]
         meta['variables'] = fg.variable.shape[0]
         meta['factors'] = fg.factor.shape[0]
         return 'SUCCESS', meta
     except:
         return 'FAILED', None
Пример #11
0
def start():
    """TODO."""
    log.debug('Initializing Numbskull Minion Engine')
    ns_minion = NumbskullMinion()
    event_bus = salt.utils.event.get_event('minion',
                                           transport=__opts__['transport'],
                                           opts=__opts__,
                                           sock_dir=__opts__['sock_dir'],
                                           listen=True)
    log.debug('Starting Numbskull Minion Engine')
    partition_id = -1
    for evdata in event_bus.iter_events(full=True):
        loop_begin = time.time()
        tag, data = evdata['tag'], evdata['data']

        if tag == messages.ASSIGN_ID:
            partition_id = data['id']
            print("Assigned partition id #", partition_id)
            # TODO: respond to master
        elif tag == messages.INIT_NS:
            try:
                ns_minion.init_numbskull(data['argv'])
                # Respond OK to master
                data = {'status': 'OK'}
                __salt__['event.send'](messages.INIT_NS_RES, data)
            except:
                # Respond FAIL to master
                data = {'status': 'FAIL'}
                __salt__['event.send'](messages.INIT_NS_RES, data)
        elif tag == messages.LOAD_FG:
            # Connect to an existing database
            # http://stackoverflow.com/questions/15634092/connect-to-an-uri-in-postgres
            db_url = data["db_url"]
            url = urlparse.urlparse(db_url)
            username = url.username
            password = url.password
            database = url.path[1:]
            hostname = url.hostname
            port = url.port
            conn = psycopg2.connect(database=database,
                                    user=username,
                                    password=password,
                                    host=hostname,
                                    port=port)

            # Open a cursor to perform database operations
            cur = conn.cursor()

            # TODO: the Au filter should only be for variables
            # For partition scheme PPB with UFO,
            # variables are missing from the minion
            # and the minion needs to know the cardinality
            minion_filter = "   partition_key similar to 'Au' " \
                            "or partition_key similar to 'B(|u)' " \
                            "or partition_key similar to 'C(|u){partition_id}' " \
                            "or partition_key similar to 'D(|u){partition_id}' " \
                            "or partition_key similar to 'E(|u){partition_id}' " \
                            "or partition_key similar to 'F(|u){partition_id}' " \
                            "or partition_key similar to 'G(|um|uw|umw){partition_id}' " \
                            "or partition_key similar to 'H(|u)' "
            minion_filter = minion_filter.format(partition_id=partition_id)

            (weight, variable, factor, fmap, domain_mask, edges, var_pt,
             factor_pt, var_ufo, factor_ufo, fid, vid, ufo_send, ufo_recv, ufo_start, ufo_map, ufo_var_begin, pf_list, factors_to_skip, pf_to_send) = \
                messages.get_fg_data(cur, minion_filter, False)

            # Close communication with the database
            cur.close()
            conn.close()

            variable[var_pt == "B"]["isEvidence"] = 4  # not owned var type

            ns_minion.ns.loadFactorGraph(weight, variable, factor, fmap,
                                         domain_mask, edges, 1, 1,
                                         factors_to_skip)

            # Respond to master
            data = {}
            __salt__['event.send'](messages.LOAD_FG_RES, data)
            log.debug("DONE LOADFG")
        elif tag == messages.SYNC_MAPPING:
            # receive map from master
            map_from_master = messages.deserialize(data["map"], np.int64)
            pf_from_master = messages.deserialize(data["pf"], np.int64)
            messages.apply_loose_inverse_map(fid, pf_from_master)

            # compute map
            map_to_master = messages.compute_map_minion(
                vid, var_pt.view(np.int8))

            ufo_map_to_master = ufo_send.copy()
            ufo_map_to_master["vid"] = vid[ufo_map_to_master["vid"]]
            data = {
                "pid": partition_id,
                "map": messages.serialize(map_to_master),
                "pf": messages.serialize(fid[pf_to_send]),
                "ufo": messages.serialize(ufo_map_to_master)
            }
            __salt__['event.send'](messages.SYNC_MAPPING_RES, data)

            messages.apply_inverse_map(vid, map_from_master)
            messages.apply_inverse_map(vid, map_to_master)

            variables_to_master = np.zeros(map_to_master.size, np.int64)
            var_evid_to_master = np.zeros(map_to_master.size, np.int64)

            pf_to_master = np.zeros(pf_to_send.size, np.int64)
            pf_evid_to_master = np.zeros(pf_to_send.size, np.int64)

            m_factors, m_fmap, m_var = messages.extra_space(
                vid, variable, ufo_send)
            ufo_to_master = np.empty(m_var, np.int64)
            ufo_evid_to_master = np.empty(m_var, np.int64)

            log.debug("DONE SYNC_MAPPING")
        elif tag == messages.INFER or tag == messages.LEARN:
            variables_from_master = \
                messages.deserialize(data["values"], np.int64)
            messages.process_received_vars(
                map_from_master, variables_from_master,
                ns_minion.ns.factorGraphs[-1].var_value[0])
            messages.apply_pf_values(
                factor, fmap, ns_minion.ns.factorGraphs[-1].var_value[0],
                variable, pf_from_master,
                messages.deserialize(data["pf"], np.int64))

            if tag == messages.LEARN:
                var_evid_from_master = \
                    messages.deserialize(data["v_evid"], np.int64)
                messages.process_received_vars(
                    map_from_master, var_evid_from_master,
                    ns_minion.ns.factorGraphs[-1].var_value_evid[0])
                messages.apply_pf_values(
                    factor, fmap,
                    ns_minion.ns.factorGraphs[-1].var_value_evid[0], variable,
                    pf_from_master,
                    messages.deserialize(data["pf_evid"], np.int64))

                ns_minion.ns.factorGraphs[-1].weight_value[0] = \
                        messages.deserialize(data["weight"], np.float64)
                w0 = ns_minion.ns.factorGraphs[-1].weight_value[0]

            begin = time.time()

            fgID = 0
            if tag == messages.LEARN:
                ns_minion.ns.learning(fgID, False)
            else:
                ns_minion.ns.inference(fgID, False)

            end = time.time()
            log.debug("INFERENCE LOOP TOOK " + str(end - begin))

            # Respond to master
            messages.compute_vars_to_send(
                map_to_master, variables_to_master,
                ns_minion.ns.factorGraphs[-1].var_value[0])
            messages.compute_pf_values(factor, fmap,
                                       ns_minion.ns.factorGraphs[-1].var_value,
                                       variable, pf_to_send, pf_to_master)
            messages.compute_ufo_values(
                factor, fmap, ns_minion.ns.factorGraphs[-1].var_value,
                variable, var_ufo, ufo_send, ufo_start, ufo_map, ufo_to_master)
            print(80 * "*")
            print(ns_minion.ns.factorGraphs[-1].var_value)
            print(ufo_to_master)

            if tag == messages.INFER:
                data = {
                    "pid": partition_id,
                    "values": messages.serialize(variables_to_master),
                    "pf": messages.serialize(pf_to_master),
                    "ufo": messages.serialize(ufo_to_master)
                }
                __salt__['event.send'](messages.INFER_RES, data)
            else:
                messages.compute_vars_to_send(
                    map_to_master, var_evid_to_master,
                    ns_minion.ns.factorGraphs[-1].var_value_evid[0])
                messages.compute_pf_values(
                    factor, fmap, ns_minion.ns.factorGraphs[-1].var_value_evid,
                    variable, pf_to_send, pf_evid_to_master)
                messages.compute_ufo_values(
                    factor, fmap, ns_minion.ns.factorGraphs[-1].var_value_evid,
                    variable, var_ufo, ufo_send, ufo_start, ufo_map,
                    ufo_evid_to_master)
                dweight = ns_minion.ns.factorGraphs[-1].weight_value[0] - w0

                data = {
                    "pid": partition_id,
                    "values": messages.serialize(variables_to_master),
                    "v_evid": messages.serialize(var_evid_to_master),
                    "pf": messages.serialize(pf_to_master),
                    "pf_evid": messages.serialize(pf_evid_to_master),
                    "ufo": messages.serialize(ufo_to_master),
                    "ufo_evid": messages.serialize(ufo_to_master),
                    "dw": messages.serialize(dweight)
                }
                __salt__['event.send'](messages.LEARN_RES, data)
        loop_end = time.time()
        print("*****" + tag + " took " + str(loop_end - loop_begin) + "*****")
Пример #12
0
    def inference(self, epochs=1, learn=False):
        """TODO."""
        mode = "Learning" if learn else "Inference"
        print("BEGINNING " + mode.upper())
        begin = time.time()
        variables_to_minions = np.zeros(self.map_to_minions.size, np.int64)
        var_evid_to_minions = np.zeros(self.map_to_minions.size, np.int64)
        pf_to_minions = np.zeros(self.pf_to_send.size, np.int64)
        pf_evid_to_minions = np.zeros(self.pf_to_send.size, np.int64)

        print("***LENGTHS***")
        print("Variables to minions:         ", len(variables_to_minions))
        print("Partial Factors to minions:   ", len(pf_to_minions))
        print("Variables from Minions:       ",
              sum([len(i) for i in self.map_from_minion]))
        print("Partial Factors from Minions: ",
              sum([len(i) for i in self.pf_from_minion]))
        print("UFO from Minions:             ",
              sum([len(i) for i in self.ufo_from_minion]))
        print("*************")

        for i in range(epochs):
            print(mode + " loop " + str(i))
            # sample own variables
            begin1 = time.time()
            fgID = 0

            if learn:
                self.ns.learning(fgID, False)
            else:
                self.ns.inference(fgID, False)

            end1 = time.time()
            print(mode + " LOOP TOOK " + str(end1 - begin1))

            # gather values to ship to minions
            # TODO: handle multiple copies
            messages.compute_vars_to_send(
                self.map_to_minions, variables_to_minions,
                self.ns.factorGraphs[-1].var_value[0])
            messages.compute_pf_values(self.factor, self.fmap,
                                       self.ns.factorGraphs[-1].var_value,
                                       self.variable, self.pf_to_send,
                                       pf_to_minions)

            if learn:
                messages.compute_vars_to_send(
                    self.map_to_minions, var_evid_to_minions,
                    self.ns.factorGraphs[-1].var_value_evid[0])
                messages.compute_pf_values(
                    self.factor, self.fmap,
                    self.ns.factorGraphs[-1].var_value_evid, self.variable,
                    self.pf_to_send, pf_evid_to_minions)

            # Tell minions to sample
            beginTest = time.time()
            if learn:
                tag = messages.LEARN
                # TODO: which copy of weight to use when multiple
                weight_value = self.ns.factorGraphs[-1].weight_value[0]
                data = {
                    "values": messages.serialize(variables_to_minions),
                    "v_evid": messages.serialize(var_evid_to_minions),
                    "pf": messages.serialize(pf_to_minions),
                    "pf_evid": messages.serialize(pf_evid_to_minions),
                    "weight": messages.serialize(weight_value)
                }
            else:
                tag = messages.INFER
                data = {
                    "values": messages.serialize(variables_to_minions),
                    "pf": messages.serialize(pf_to_minions)
                }

            if self.num_minions != 0:
                pub_func = partial(send_to_minion, data, tag)
                self.clientPool.imap(pub_func, self.minion2host.values())

            endTest = time.time()
            print("EVENT FIRE LOOP TOOK " + str(endTest - beginTest))

            messages.clear_ufo_values(self.ns.factorGraphs[-1].var_value[0],
                                      self.ufo_var_begin)
            if learn:
                messages.clear_ufo_values(
                    self.ns.factorGraphs[-1].var_value_evid[0],
                    self.ufo_var_begin)

            resp = 0
            while resp < len(self.minions):
                tag = messages.LEARN_RES if learn else messages.INFER_RES
                evdata = self.event_bus.get_event(wait=5, tag=tag, full=True)
                if evdata:
                    resp += 1
                    data = evdata['data']['data']
                    pid = data["pid"]
                    # Process variables from minions
                    vfmin = messages.deserialize(data["values"], np.int64)
                    messages.process_received_vars(
                        self.map_from_minion[pid], vfmin,
                        self.ns.factorGraphs[-1].var_value[0])
                    messages.apply_pf_values(
                        self.factor, self.fmap,
                        self.ns.factorGraphs[-1].var_value[0], self.variable,
                        self.pf_from_minion[pid],
                        messages.deserialize(data["pf"], np.int64))
                    messages.apply_ufo_values(
                        self.factor, self.fmap,
                        self.ns.factorGraphs[-1].var_value[0],
                        self.ufo_from_minion[pid],
                        messages.deserialize(data["ufo"], np.int64))

                    if learn:
                        vfmin = messages.deserialize(data["v_evid"], np.int64)
                        messages.process_received_vars(
                            self.map_from_minion[pid], vfmin,
                            self.ns.factorGraphs[-1].var_value_evid[0])
                        messages.apply_pf_values(
                            self.factor, self.fmap,
                            self.ns.factorGraphs[-1].var_value[0],
                            self.variable, self.pf_from_minion[pid],
                            messages.deserialize(data["pf_evid"], np.int64))
                        messages.apply_ufo_values(
                            self.factor, self.fmap,
                            self.ns.factorGraphs[-1].var_value_evid[0],
                            self.ufo_from_minion[pid],
                            messages.deserialize(data["ufo_evid"], np.int64))

                        self.ns.factorGraphs[-1].weight_value[0] += \
                            messages.deserialize(data["dw"], np.float64)

            end1 = time.time()
            print("FULL " + mode + " LOOP TOOK " + str(end1 - begin1) + "\n")

        # TODO: get and return marginals
        # TODO: switch to proper probs
        end = time.time()
        print(mode + " TOOK", end - begin)
        return end - begin