def sync_mapping(self):
"""TODO."""
self.map_to_minions = messages.compute_map_master(
self.vid, self.var_pt.view(np.int8))
print(self.fid[self.pf_list])
# send mapping to minions
tag = messages.SYNC_MAPPING
data = {
"map": messages.serialize(self.map_to_minions),
"pf": messages.serialize(self.fid[self.pf_to_send])
}
if self.num_minions != 0:
pub_func = partial(send_to_minion, data, tag)
self.clientPool.imap(pub_func, self.minion2host.values())
#newEvent = self.local_client.cmd(self.minions,
# 'event.fire',
# [data, tag],
# expr_form='list')
self.map_from_minion = [None for i in range(len(self.minions))]
self.pf_from_minion = [None for i in range(len(self.minions))]
self.ufo_from_minion = [None for i in range(len(self.minions))]
self.ufo_map_from_minion = self.ufo_recv.copy()
self.ufo_map_from_minion["vid"] = self.vid[
self.ufo_map_from_minion["vid"]]
resp = 0
while resp < len(self.minions):
# receive map and save
evdata = self.event_bus.get_event(wait=5,
tag=messages.SYNC_MAPPING_RES,
full=True)
if evdata:
tag, data = evdata['tag'], evdata['data']['data']
pid = data["pid"]
self.map_from_minion[pid] = \
messages.deserialize(data["map"], np.int64)
messages.apply_inverse_map(self.vid, self.map_from_minion[pid])
self.pf_from_minion[pid] = messages.deserialize(
data["pf"], np.int64)
messages.apply_inverse_map(self.fid, self.pf_from_minion[pid])
self.ufo_from_minion[pid] = messages.ufo_to_factor(
messages.deserialize(data["ufo"], UnaryFactorOpt),
self.ufo_map_from_minion, len(self.factor_pt))
resp += 1
print("DONE WITH SENDING MAPPING")
messages.apply_inverse_map(self.vid, self.map_to_minions)
async def serve(self, socket, path):
user = User(coolname.generate_slug(2), socket)
logger.info("%s connected", user)
while True:
try:
data = await user.socket.recv()
except (
websockets.exceptions.ConnectionClosedError,
websockets.exceptions.ConnectionClosedOK,
) as e:
logger.info("%s closed connection (%s)", user, e)
await self.stage.on_disconnect(user)
break
message = messages.deserialize(data)
logger.info("Received %s from %s", message, user)
if isinstance(message, messages.AuthCode):
await self.stage.on_auth_code(user, message)
elif isinstance(message, messages.CountCode):
await self.stage.on_count_code(user, message)
elif isinstance(message, messages.QuestionAnswers):
await self.stage.on_question_answers(user, message)
else:
raise NotImplementedError(f"Unsupported message {message}")
def run(self):
self.connect()
while self.end is not True:
message = protocol.recv_one_message(self.socket)
self.handle_message(messages.deserialize(message))
def demander(q, j):
informer("QUESTION : " + q)
protocol.send_one_message(clients[j.numero], messages.Question(q).toJson())
r = protocol.recv_one_message(clients[j.numero])
r = messages.deserialize(r)
informer("REPONSE DONNEE : " + str(r.content))
return str(r.content)
def send(self, sender, host_port, message):
print "TRANSPORT SENDS", messages.deserialize(message)
for cb in self.on_send_cbs:
cb(sender, host_port, message)
f = self.protocols[host_port].receive
gevent.spawn_later(0.0001, f, message)
def read_all(self, filename):
"""
Given a filename, it opens it and read all data into memory and return
"""
with open(filename, 'rb') as f:
d = f.read()
return deserialize(d)
def receive(self, data):
assert len(data) < self.max_message_size
# check if we handled this message already, if so repeat Ack
h = sha3(data)
if h in self.sent_acks:
# assert False, "DEACTIVATED ACK RESENTS"
return self.send_ack(*self.sent_acks[h])
# note, we ignore the sending endpoint, as this can not be known w/ UDP
msg = messages.deserialize(data)
# handle Acks
if isinstance(msg, Ack):
del self.tries[msg.echo]
return
assert isinstance(msg, Secret) or msg.sender
self.raiden.on_message(msg)
def __handleReadAvailable(self):
try:
bytes = self.socket.recv(4096)
if len(bytes) == 0:
log.log("Detected socket close by other side; closing this side too")
self.close()
self.__readBuffer += bytes
if self.protocolVersion == None:
self.__tryReadProtocolVersion()
if self.protocolVersion == None:
return
# There can be multiple messages in the read buffer,
# so repeat until there is no more complete message
while True:
# Message detection:
if len(self.__readBuffer) < 4:
return
# 4-byte unsigned int in network byte order:
msgLen = struct.unpack("!I", self.__readBuffer[:4])[0]
if len(self.__readBuffer) < msgLen + 4:
return
msg = self.__readBuffer[4 : msgLen + 4]
self.__readBuffer = self.__readBuffer[msgLen + 4 :]
# print "Received serialized message: ", repr(msg)
# de-serialize
msg = messages.deserialize(msg)
self.__handleMessage(msg)
except:
log.logException()
self.close()
def run(self):
self.client.connect()
self.authenticate(self.username, self.password)
players = self.get_player_list()
if players is not None:
for player in players:
self.add_player_info(player['Login'], player)
#
#
self.plugins = Plugins(self)
delay = 0.25
prev_tick = time.time()
prev_second = prev_tick
while True:
cbs = self.client.load_callbacks()
for cb in cbs:
object = messages.deserialize(cb)
self.process_callback(object)
#
self.client.reset_callbacks()
self.raise_event('tick', None)
now = time.time()
if now - prev_second >= 1.0:
self.raise_event('second_passed', None)
prev_second = now
#
now = time.time()
duration = now - prev_tick
if duration < delay:
time.sleep(delay - duration)
#
prev_tick = now
def loadFG(self, data):
"""TODO."""
try:
weight = np.fromstring(data['weight'], dtype=Weight)
variable = np.fromstring(data['variable'], dtype=Variable)
factor = messages.deserialize(data['factor'], Factor)
fmap = np.fromstring(data['fmap'], dtype=FactorToVar)
vmap = np.fromstring(data['vmap'], dtype=VarToFactor)
factor_index = np.fromstring(data['factor_index'], dtype=np.int64)
except:
return 'FAILED LOADING', None
try:
self.ns.loadFactorGraphRaw(weight, variable, factor, fmap, vmap,
factor_index)
fg = self.ns.factorGraphs[-1]
meta = {}
meta['weights'] = fg.weight.shape[0]
meta['variables'] = fg.variable.shape[0]
meta['factors'] = fg.factor.shape[0]
return 'SUCCESS', meta
except:
return 'FAILED', None
def start():
"""TODO."""
log.debug('Initializing Numbskull Minion Engine')
ns_minion = NumbskullMinion()
event_bus = salt.utils.event.get_event('minion',
transport=__opts__['transport'],
opts=__opts__,
sock_dir=__opts__['sock_dir'],
listen=True)
log.debug('Starting Numbskull Minion Engine')
partition_id = -1
for evdata in event_bus.iter_events(full=True):
loop_begin = time.time()
tag, data = evdata['tag'], evdata['data']
if tag == messages.ASSIGN_ID:
partition_id = data['id']
print("Assigned partition id #", partition_id)
# TODO: respond to master
elif tag == messages.INIT_NS:
try:
ns_minion.init_numbskull(data['argv'])
# Respond OK to master
data = {'status': 'OK'}
__salt__['event.send'](messages.INIT_NS_RES, data)
except:
# Respond FAIL to master
data = {'status': 'FAIL'}
__salt__['event.send'](messages.INIT_NS_RES, data)
elif tag == messages.LOAD_FG:
# Connect to an existing database
# http://stackoverflow.com/questions/15634092/connect-to-an-uri-in-postgres
db_url = data["db_url"]
url = urlparse.urlparse(db_url)
username = url.username
password = url.password
database = url.path[1:]
hostname = url.hostname
port = url.port
conn = psycopg2.connect(database=database,
user=username,
password=password,
host=hostname,
port=port)
# Open a cursor to perform database operations
cur = conn.cursor()
# TODO: the Au filter should only be for variables
# For partition scheme PPB with UFO,
# variables are missing from the minion
# and the minion needs to know the cardinality
minion_filter = " partition_key similar to 'Au' " \
"or partition_key similar to 'B(|u)' " \
"or partition_key similar to 'C(|u){partition_id}' " \
"or partition_key similar to 'D(|u){partition_id}' " \
"or partition_key similar to 'E(|u){partition_id}' " \
"or partition_key similar to 'F(|u){partition_id}' " \
"or partition_key similar to 'G(|um|uw|umw){partition_id}' " \
"or partition_key similar to 'H(|u)' "
minion_filter = minion_filter.format(partition_id=partition_id)
(weight, variable, factor, fmap, domain_mask, edges, var_pt,
factor_pt, var_ufo, factor_ufo, fid, vid, ufo_send, ufo_recv, ufo_start, ufo_map, ufo_var_begin, pf_list, factors_to_skip, pf_to_send) = \
messages.get_fg_data(cur, minion_filter, False)
# Close communication with the database
cur.close()
conn.close()
variable[var_pt == "B"]["isEvidence"] = 4 # not owned var type
ns_minion.ns.loadFactorGraph(weight, variable, factor, fmap,
domain_mask, edges, 1, 1,
factors_to_skip)
# Respond to master
data = {}
__salt__['event.send'](messages.LOAD_FG_RES, data)
log.debug("DONE LOADFG")
elif tag == messages.SYNC_MAPPING:
# receive map from master
map_from_master = messages.deserialize(data["map"], np.int64)
pf_from_master = messages.deserialize(data["pf"], np.int64)
messages.apply_loose_inverse_map(fid, pf_from_master)
# compute map
map_to_master = messages.compute_map_minion(
vid, var_pt.view(np.int8))
ufo_map_to_master = ufo_send.copy()
ufo_map_to_master["vid"] = vid[ufo_map_to_master["vid"]]
data = {
"pid": partition_id,
"map": messages.serialize(map_to_master),
"pf": messages.serialize(fid[pf_to_send]),
"ufo": messages.serialize(ufo_map_to_master)
}
__salt__['event.send'](messages.SYNC_MAPPING_RES, data)
messages.apply_inverse_map(vid, map_from_master)
messages.apply_inverse_map(vid, map_to_master)
variables_to_master = np.zeros(map_to_master.size, np.int64)
var_evid_to_master = np.zeros(map_to_master.size, np.int64)
pf_to_master = np.zeros(pf_to_send.size, np.int64)
pf_evid_to_master = np.zeros(pf_to_send.size, np.int64)
m_factors, m_fmap, m_var = messages.extra_space(
vid, variable, ufo_send)
ufo_to_master = np.empty(m_var, np.int64)
ufo_evid_to_master = np.empty(m_var, np.int64)
log.debug("DONE SYNC_MAPPING")
elif tag == messages.INFER or tag == messages.LEARN:
variables_from_master = \
messages.deserialize(data["values"], np.int64)
messages.process_received_vars(
map_from_master, variables_from_master,
ns_minion.ns.factorGraphs[-1].var_value[0])
messages.apply_pf_values(
factor, fmap, ns_minion.ns.factorGraphs[-1].var_value[0],
variable, pf_from_master,
messages.deserialize(data["pf"], np.int64))
if tag == messages.LEARN:
var_evid_from_master = \
messages.deserialize(data["v_evid"], np.int64)
messages.process_received_vars(
map_from_master, var_evid_from_master,
ns_minion.ns.factorGraphs[-1].var_value_evid[0])
messages.apply_pf_values(
factor, fmap,
ns_minion.ns.factorGraphs[-1].var_value_evid[0], variable,
pf_from_master,
messages.deserialize(data["pf_evid"], np.int64))
ns_minion.ns.factorGraphs[-1].weight_value[0] = \
messages.deserialize(data["weight"], np.float64)
w0 = ns_minion.ns.factorGraphs[-1].weight_value[0]
begin = time.time()
fgID = 0
if tag == messages.LEARN:
ns_minion.ns.learning(fgID, False)
else:
ns_minion.ns.inference(fgID, False)
end = time.time()
log.debug("INFERENCE LOOP TOOK " + str(end - begin))
# Respond to master
messages.compute_vars_to_send(
map_to_master, variables_to_master,
ns_minion.ns.factorGraphs[-1].var_value[0])
messages.compute_pf_values(factor, fmap,
ns_minion.ns.factorGraphs[-1].var_value,
variable, pf_to_send, pf_to_master)
messages.compute_ufo_values(
factor, fmap, ns_minion.ns.factorGraphs[-1].var_value,
variable, var_ufo, ufo_send, ufo_start, ufo_map, ufo_to_master)
print(80 * "*")
print(ns_minion.ns.factorGraphs[-1].var_value)
print(ufo_to_master)
if tag == messages.INFER:
data = {
"pid": partition_id,
"values": messages.serialize(variables_to_master),
"pf": messages.serialize(pf_to_master),
"ufo": messages.serialize(ufo_to_master)
}
__salt__['event.send'](messages.INFER_RES, data)
else:
messages.compute_vars_to_send(
map_to_master, var_evid_to_master,
ns_minion.ns.factorGraphs[-1].var_value_evid[0])
messages.compute_pf_values(
factor, fmap, ns_minion.ns.factorGraphs[-1].var_value_evid,
variable, pf_to_send, pf_evid_to_master)
messages.compute_ufo_values(
factor, fmap, ns_minion.ns.factorGraphs[-1].var_value_evid,
variable, var_ufo, ufo_send, ufo_start, ufo_map,
ufo_evid_to_master)
dweight = ns_minion.ns.factorGraphs[-1].weight_value[0] - w0
data = {
"pid": partition_id,
"values": messages.serialize(variables_to_master),
"v_evid": messages.serialize(var_evid_to_master),
"pf": messages.serialize(pf_to_master),
"pf_evid": messages.serialize(pf_evid_to_master),
"ufo": messages.serialize(ufo_to_master),
"ufo_evid": messages.serialize(ufo_to_master),
"dw": messages.serialize(dweight)
}
__salt__['event.send'](messages.LEARN_RES, data)
loop_end = time.time()
print("*****" + tag + " took " + str(loop_end - loop_begin) + "*****")
def inference(self, epochs=1, learn=False):
"""TODO."""
mode = "Learning" if learn else "Inference"
print("BEGINNING " + mode.upper())
begin = time.time()
variables_to_minions = np.zeros(self.map_to_minions.size, np.int64)
var_evid_to_minions = np.zeros(self.map_to_minions.size, np.int64)
pf_to_minions = np.zeros(self.pf_to_send.size, np.int64)
pf_evid_to_minions = np.zeros(self.pf_to_send.size, np.int64)
print("***LENGTHS***")
print("Variables to minions: ", len(variables_to_minions))
print("Partial Factors to minions: ", len(pf_to_minions))
print("Variables from Minions: ",
sum([len(i) for i in self.map_from_minion]))
print("Partial Factors from Minions: ",
sum([len(i) for i in self.pf_from_minion]))
print("UFO from Minions: ",
sum([len(i) for i in self.ufo_from_minion]))
print("*************")
for i in range(epochs):
print(mode + " loop " + str(i))
# sample own variables
begin1 = time.time()
fgID = 0
if learn:
self.ns.learning(fgID, False)
else:
self.ns.inference(fgID, False)
end1 = time.time()
print(mode + " LOOP TOOK " + str(end1 - begin1))
# gather values to ship to minions
# TODO: handle multiple copies
messages.compute_vars_to_send(
self.map_to_minions, variables_to_minions,
self.ns.factorGraphs[-1].var_value[0])
messages.compute_pf_values(self.factor, self.fmap,
self.ns.factorGraphs[-1].var_value,
self.variable, self.pf_to_send,
pf_to_minions)
if learn:
messages.compute_vars_to_send(
self.map_to_minions, var_evid_to_minions,
self.ns.factorGraphs[-1].var_value_evid[0])
messages.compute_pf_values(
self.factor, self.fmap,
self.ns.factorGraphs[-1].var_value_evid, self.variable,
self.pf_to_send, pf_evid_to_minions)
# Tell minions to sample
beginTest = time.time()
if learn:
tag = messages.LEARN
# TODO: which copy of weight to use when multiple
weight_value = self.ns.factorGraphs[-1].weight_value[0]
data = {
"values": messages.serialize(variables_to_minions),
"v_evid": messages.serialize(var_evid_to_minions),
"pf": messages.serialize(pf_to_minions),
"pf_evid": messages.serialize(pf_evid_to_minions),
"weight": messages.serialize(weight_value)
}
else:
tag = messages.INFER
data = {
"values": messages.serialize(variables_to_minions),
"pf": messages.serialize(pf_to_minions)
}
if self.num_minions != 0:
pub_func = partial(send_to_minion, data, tag)
self.clientPool.imap(pub_func, self.minion2host.values())
endTest = time.time()
print("EVENT FIRE LOOP TOOK " + str(endTest - beginTest))
messages.clear_ufo_values(self.ns.factorGraphs[-1].var_value[0],
self.ufo_var_begin)
if learn:
messages.clear_ufo_values(
self.ns.factorGraphs[-1].var_value_evid[0],
self.ufo_var_begin)
resp = 0
while resp < len(self.minions):
tag = messages.LEARN_RES if learn else messages.INFER_RES
evdata = self.event_bus.get_event(wait=5, tag=tag, full=True)
if evdata:
resp += 1
data = evdata['data']['data']
pid = data["pid"]
# Process variables from minions
vfmin = messages.deserialize(data["values"], np.int64)
messages.process_received_vars(
self.map_from_minion[pid], vfmin,
self.ns.factorGraphs[-1].var_value[0])
messages.apply_pf_values(
self.factor, self.fmap,
self.ns.factorGraphs[-1].var_value[0], self.variable,
self.pf_from_minion[pid],
messages.deserialize(data["pf"], np.int64))
messages.apply_ufo_values(
self.factor, self.fmap,
self.ns.factorGraphs[-1].var_value[0],
self.ufo_from_minion[pid],
messages.deserialize(data["ufo"], np.int64))
if learn:
vfmin = messages.deserialize(data["v_evid"], np.int64)
messages.process_received_vars(
self.map_from_minion[pid], vfmin,
self.ns.factorGraphs[-1].var_value_evid[0])
messages.apply_pf_values(
self.factor, self.fmap,
self.ns.factorGraphs[-1].var_value[0],
self.variable, self.pf_from_minion[pid],
messages.deserialize(data["pf_evid"], np.int64))
messages.apply_ufo_values(
self.factor, self.fmap,
self.ns.factorGraphs[-1].var_value_evid[0],
self.ufo_from_minion[pid],
messages.deserialize(data["ufo_evid"], np.int64))
self.ns.factorGraphs[-1].weight_value[0] += \
messages.deserialize(data["dw"], np.float64)
end1 = time.time()
print("FULL " + mode + " LOOP TOOK " + str(end1 - begin1) + "\n")
# TODO: get and return marginals
# TODO: switch to proper probs
end = time.time()
print(mode + " TOOK", end - begin)
return end - begin