def main():
text = "a dead dad ceded a bad babe a beaded abaca bed"
# on analyse les fréquences d'occurrence dans text
# pour fabriquer un arbre binaire
builder = TreeBuilder(text)
binary_tree = builder.tree()
# on passe l'arbre binaire à un encodeur/décodeur
codec = Codec(binary_tree)
# qui permet d'encoder
encoded = codec.encode_bin(text)
# et de décoder
decoded = codec.decode_bin(encoded)
# si cette assertion est fausse il y a un gros problème avec le code
assert text == decoded
# on affiche le résultat
print(f"{text}\n{encoded}")
if decoded != text:
print("OOPS")
text = 'ab'
compress_encode = []
compress_huffman = []
compress_binary = []
for rang in range(300):
texte = text * rang # Création d'un texte
taille = sys.getsizeof(texte) # Espace système de ce texte
compress_encode.append(taille / sys.getsizeof(codec.encode(texte)))
compress_huffman.append(taille /
sys.getsizeof(codec.encode_bin(texte)))
compress_binary.append(
sys.getsizeof(codec.encode(texte)) /
sys.getsizeof(codec.encode_bin(texte)))
# Affichage
plt.close()
plt.title('Facteurs de compression')
plt.plot(compress_encode,
label="Facteur de compression par \n Codec.encode sortie str")
plt.plot(
compress_huffman,
label=
"Facteur de compression de \n l'algorithme de Huffman (Avec Binaire)")
plt.plot(compress_binary,
label="Facteur de compression \n d'une str encoded en binaire ")
plt.legend()
plt.show()
def main():
with open(CONFIG_FN) as f:
conf = json.load(f)
global sent_groups
with open(conf["sent_groups"]) as f:
sent_groups = json.load(f)["groups"]
kb = load_kb(conf["kb"], 'name')
sys_vocab, sys_word2idx = load_sys_vocab(conf["sys_vocab"])
sys_codec = Codec(sys_vocab, sys_word2idx)
onto, onto_idx = load_ontology(conf["ontology"])
word2idx, embed = load_embed(**conf)
usr_codec = Codec([], word2idx)
trk_model, slot_len_sum = load_tracker_model(onto, embed, conf, kb)
trk_model.eval()
hidden = trk_model.state_tracker.init_hidden()
kb_vec = Variable(torch.zeros(1, conf["kb_indicator_len"]))
sentence_generator = SentenceGenerator(kb, onto, sent_groups)
for line in iter(sys.stdin.readline, ''):
inp = usr_codec.encode(line.strip())
inp = Variable(torch.LongTensor([
inp,
]))
sentvecs, states_reps, states_preds, hidden, sent_grp_preds = trk_model(
inp, None, hidden)
criteria = to_search_criteria(states_preds, onto)
ret, kb_vec = get_kb_result(kb, criteria, conf["kb_indicator_len"])
# print criteria, kb_vec
sentvecs = sentvecs.view(1, -1)
states_reps = states_reps.view(1, -1)
print_ret(states_preds, sent_grp_preds, onto, sentence_generator)
from codec import TreeBuilder, Codec
text = "a dead dad ceded a bad babe a beaded abaca bed"
builder = TreeBuilder(text)
binary_tree = builder.tree()
# on passe l'arbre binaire à un encodeur/décodeur
codec = Codec(binary_tree)
# qui permet d'encoder
encoded = codec.encode(text)
# et de décoder
decoded = codec.decode(encoded)
# si cette assertion est fausse il y a un gros problème avec le code
# on affiche le résultat
print(f"{text}\n{decoded}")
if decoded != text:
print("OOPS")
def main():
with open(CONFIG_FN) as f:
conf = json.load(f)
args = parse_args()
kb = load_kb(conf["kb"], 'name')
sys_vocab, sys_word2idx = load_sys_vocab(conf["sys_vocab"])
sys_codec = Codec(sys_vocab, sys_word2idx)
onto, onto_idx = load_ontology(conf["ontology"])
word2idx, embed = load_embed(**conf)
usr_codec = Codec([], word2idx)
trk_model, slot_len_sum = load_tracker_model(onto, embed, conf)
cond_net, generator = load_generator_model(conf, args.cond_net,
args.gen_net, slot_len_sum,
len(sys_vocab))
cond_net.eval()
generator.eval()
trk_model.eval()
hidden = trk_model.state_tracker.init_hidden()
kb_found = Variable(torch.zeros(1, conf["kb_indicator_len"]))
def gen_sent(cond):
'''train one sentence'''
hidden = generator.init_hidden()
inp = Variable(torch.LongTensor([[sys_word2idx['<sos>']]]))
sent_out = []
for i in range(MAX_SENT_LEN):
out, hidden = generator(inp, cond, hidden)
topv, topi = out.data.topk(1)
out_word = int(topi[0][0])
if out_word == sys_word2idx['<eos>']:
break
inp = Variable(torch.LongTensor([[out_word]]))
sent_out.append(out_word)
return sys_codec.decode(sent_out)
for line in iter(sys.stdin.readline, ''):
inp = usr_codec.encode(line.strip())
inp = Variable(torch.LongTensor([
inp,
]))
sentvecs, states_reps, states_preds, hidden = trk_model(
inp, kb_found, hidden)
# print_ret(states_preds)
criteria = to_search_criteria(states_preds, onto)
ret, kb_vec = get_kb_result(kb, criteria, conf["kb_indicator_len"])
sentvecs = sentvecs.view(1, -1)
states_reps = states_reps.view(1, -1)
for slot in states_preds:
states_preds[slot] = states_preds[slot].view(1, -1)
cond = cond_net(sentvecs, states_reps, states_preds, kb_found)
print gen_sent(cond)
codec = Codec(layer_sizes, symbols, rho=.999)
controller = Controller(layer_sizes, pathways, hidden_size, plastic)
# controller = Controller(layer_sizes, pathways, hidden_size, input_keys=["m","rinp"])
# controller.rnn.weight_hh_l0.data = 1*tr.eye(hidden_size) # favor repeated actions
# Sanity check
ghu = GatedHebbianUnit(layer_sizes,
pathways,
controller,
codec,
plastic=plastic)
ghu.associate(associations)
for p, s, t in associations:
q, r = ghu.pathways[p]
assert (codec.decode(q, tr.mv(ghu.W[p], codec.encode(r, s))) == t)
ghu_init = ghu
# # Pre-train for uniform action distribution
# tol=.25
# max_time=4
# max_iters=500
# learning_rate=0.01
# for itr in range(max_iters):
# ghu = GatedHebbianUnit(layer_sizes, pathways, controller, codec, plastic=plastic)
# ghu.associate(associations)
# loss = 0.
# for t in range(max_time):
# for k in layer_sizes.keys():
# ghu.v[t][k] = codec.encode(k, np.random.choice(symbols))
# ghu.tick()
# a = codec.encode("r0","0")
# b = codec.encode("r0","1")
# print(a)
# x = codec.decode("r0",a)
# print(x == "0")
# print(x == "1")
# y = codec.decode("r0",b)
# print(y == "0")
# print(y == "1")
# print(codec.parameters())
for t in [-1, 0]:
for k in ["r0", "r1"]:
ghu.v[t][k] = tr.repeat_interleave(codec.encode(k,
str(0 - t)).view(
1, -1),
batch_size,
dim=0)
# ghu.v[-1]["r0"] = codec.encode("r0", str(1))
# ghu.v[-1]["r1"] = codec.encode("r1", str(1))
# ghu.v[0]["r0"] = codec.encode("r0", str(0))
# ghu.v[0]["r1"] = codec.encode("r1", str(0))
ghu.tick(num_steps=2)
# e = ghu.g[len(ghu.g)-1].sum()
# e.backward()
# print("codec")
# print(codec)
# for p in codec.parameters():
class FIXConnectionHandler(object):
def __init__(self, engine, protocol, sock=None, addr=None, observer=None):
self.codec = Codec(protocol)
self.engine = engine
self.connectionState = ConnectionState.CONNECTED
self.session = None
self.addr = addr
self.observer = observer
self.msgBuffer = b''
self.heartbeatPeriod = 30.0
self.msgHandlers = []
self.sock = sock
self.heartbeatTimerRegistration = None
self.expectedHeartbeatRegistration = None
self.socketEvent = FileDescriptorEventRegistration(
self.handle_read, sock, EventType.READ)
self.engine.eventManager.registerHandler(self.socketEvent)
def address(self):
return self.addr
def disconnect(self):
self.handle_close()
def _notifyMessageObservers(self, msg, direction, persistMessage=True):
if persistMessage is True:
self.engine.journaller.persistMsg(msg, self.session, direction)
for handler in filter(
lambda x: (x[1] is None or x[1] == direction) and
(x[2] is None or x[2] == msg.msgType), self.msgHandlers):
handler[0](self, msg)
def addMessageHandler(self, handler, direction=None, msgType=None):
self.msgHandlers.append((handler, direction, msgType))
def removeMessageHandler(self, handler, direction=None, msgType=None):
remove = filter(
lambda x: x[0] == handler and
(x[1] == direction or direction is None) and
(x[2] == msgType or msgType is None), self.msgHandlers)
for h in remove:
self.msgHandlers.remove(h)
def _sendHeartbeat(self):
self.sendMsg(self.codec.protocol.messages.Messages.heartbeat())
def _expectedHeartbeat(self, type, closure):
logging.warning("Expected heartbeat from peer %s" %
(self.expectedHeartbeatRegistration, ))
self.sendMsg(self.codec.protocol.messages.Messages.test_request())
def registerLoggedIn(self):
self.heartbeatTimerRegistration = TimerEventRegistration(
lambda type, closure: self._sendHeartbeat(), self.heartbeatPeriod)
self.engine.eventManager.registerHandler(
self.heartbeatTimerRegistration)
# register timeout for 10% more than we expect
self.expectedHeartbeatRegistration = TimerEventRegistration(
self._expectedHeartbeat, self.heartbeatPeriod * 1.10)
self.engine.eventManager.registerHandler(
self.expectedHeartbeatRegistration)
def registerLoggedOut(self):
if self.heartbeatTimerRegistration is not None:
self.engine.eventManager.unregisterHandler(
self.heartbeatTimerRegistration)
self.heartbeatTimerRegistration = None
if self.expectedHeartbeatRegistration is not None:
self.engine.eventManager.unregisterHandler(
self.expectedHeartbeatRegistration)
self.expectedHeartbeatRegistration = None
def _handleResendRequest(self, msg):
protocol = self.codec.protocol
responses = []
beginSeqNo = msg[protocol.fixtags.BeginSeqNo]
endSeqNo = msg[protocol.fixtags.EndSeqNo]
if int(endSeqNo) == 0:
endSeqNo = sys.maxsize
logging.info("Received resent request from %s to %s", beginSeqNo,
endSeqNo)
replayMsgs = self.engine.journaller.recoverMsgs(
self.session, MessageDirection.OUTBOUND, beginSeqNo, endSeqNo)
gapFillBegin = int(beginSeqNo)
gapFillEnd = int(beginSeqNo)
for replayMsg in replayMsgs:
msgSeqNum = int(replayMsg[protocol.fixtags.MsgSeqNum])
if replayMsg[protocol.fixtags.
MsgType] in protocol.msgtype.sessionMessageTypes:
gapFillEnd = msgSeqNum + 1
else:
if self.engine.shouldResendMessage(self.session, replayMsg):
if gapFillBegin < gapFillEnd:
# we need to send a gap fill message
gapFillMsg = FIXMessage(protocol.msgtype.SEQUENCERESET)
gapFillMsg.setField(protocol.fixtags.GapFillFlag, 'Y')
gapFillMsg.setField(protocol.fixtags.MsgSeqNum,
gapFillBegin)
gapFillMsg.setField(protocol.fixtags.NewSeqNo,
str(gapFillEnd))
responses.append(gapFillMsg)
# and then resent the replayMsg
replayMsg.removeField(protocol.fixtags.BeginString)
replayMsg.removeField(protocol.fixtags.BodyLength)
replayMsg.removeField(protocol.fixtags.SendingTime)
replayMsg.removeField(protocol.fixtags.SenderCompID)
replayMsg.removeField(protocol.fixtags.TargetCompID)
replayMsg.removeField(protocol.fixtags.CheckSum)
replayMsg.setField(protocol.fixtags.PossDupFlag, "Y")
responses.append(replayMsg)
gapFillBegin = msgSeqNum + 1
else:
gapFillEnd = msgSeqNum + 1
responses.append(replayMsg)
if gapFillBegin < gapFillEnd:
# we need to send a gap fill message
gapFillMsg = FIXMessage(protocol.msgtype.SEQUENCERESET)
gapFillMsg.setField(protocol.fixtags.GapFillFlag, 'Y')
gapFillMsg.setField(protocol.fixtags.MsgSeqNum, gapFillBegin)
gapFillMsg.setField(protocol.fixtags.NewSeqNo, str(gapFillEnd))
responses.append(gapFillMsg)
return responses
def handle_read(self, type, closure):
protocol = self.codec.protocol
try:
msg = self.sock.recv(8192)
if msg:
self.msgBuffer = self.msgBuffer + msg
(decodedMsg, parsedLength) = self.codec.decode(self.msgBuffer)
self.msgBuffer = self.msgBuffer[parsedLength:]
while decodedMsg is not None and self.connectionState != ConnectionState.DISCONNECTED:
self.processMessage(decodedMsg)
(decodedMsg,
parsedLength) = self.codec.decode(self.msgBuffer)
self.msgBuffer = self.msgBuffer[parsedLength:]
if self.expectedHeartbeatRegistration is not None:
self.expectedHeartbeatRegistration.reset()
else:
logging.debug("Connection has been closed")
self.disconnect()
except ConnectionError as why:
logging.debug("Connection has been closed %s" % (why, ))
self.disconnect()
def handleSessionMessage(self, msg):
return -1
def processMessage(self, decodedMsg):
protocol = self.codec.protocol
beginString = decodedMsg[protocol.fixtags.BeginString]
if beginString != protocol.beginstring:
logging.warning(
"FIX BeginString is incorrect (expected: %s received: %s)",
(protocol.beginstring, beginString))
self.disconnect()
return
msgType = decodedMsg[protocol.fixtags.MsgType]
try:
responses = []
if msgType in protocol.msgtype.sessionMessageTypes:
(recvSeqNo, responses) = self.handleSessionMessage(decodedMsg)
else:
recvSeqNo = decodedMsg[protocol.fixtags.MsgSeqNum]
# validate the seq number
(seqNoState,
lastKnownSeqNo) = self.session.validateRecvSeqNo(recvSeqNo)
if seqNoState is False:
# We should send a resend request
logging.info("Requesting resend of messages: %s to %s" %
(lastKnownSeqNo, 0))
responses.append(
protocol.messages.Messages.resend_request(
lastKnownSeqNo, 0))
# we still need to notify if we are processing Logon message
if msgType == protocol.msgtype.LOGON:
self._notifyMessageObservers(decodedMsg,
MessageDirection.INBOUND,
False)
else:
self.session.setRecvSeqNo(recvSeqNo)
self._notifyMessageObservers(decodedMsg,
MessageDirection.INBOUND)
for m in responses:
self.sendMsg(m)
except SessionWarning as sw:
logging.warning(sw)
except SessionError as se:
logging.error(se)
self.disconnect()
except DuplicateSeqNoError:
try:
if decodedMsg[protocol.fixtags.PossDupFlag] == "Y":
logging.debug(
"Received duplicate message with PossDupFlag set")
except KeyError:
pass
finally:
logging.error(
"Failed to process message with duplicate seq no (MsgSeqNum: %s) (and no PossDupFlag='Y') - disconnecting"
% (recvSeqNo, ))
self.disconnect()
def handle_close(self):
if self.connectionState != ConnectionState.DISCONNECTED:
logging.info("Client disconnected")
self.registerLoggedOut()
self.sock.close()
self.connectionState = ConnectionState.DISCONNECTED
self.msgHandlers.clear()
if self.observer is not None:
self.observer.notifyDisconnect(self)
self.engine.eventManager.unregisterHandler(self.socketEvent)
def sendMsg(self, msg):
if self.connectionState != ConnectionState.CONNECTED and self.connectionState != ConnectionState.LOGGED_IN:
raise FIXException(FIXException.FIXExceptionReason.NOT_CONNECTED)
encodedMsg = self.codec.encode(msg, self.session).encode('utf-8')
self.sock.send(encodedMsg)
if self.heartbeatTimerRegistration is not None:
self.heartbeatTimerRegistration.reset()
decodedMsg, junk = self.codec.decode(encodedMsg)
try:
self._notifyMessageObservers(decodedMsg, MessageDirection.OUTBOUND)
except DuplicateSeqNoError:
logging.error(
"We have sent a message with a duplicate seq no, failed to persist it (MsgSeqNum: %s)"
% (decodedMsg[self.codec.protocol.fixtags.MsgSeqNum]))
plastic = []
symbols = [str(a) for a in range(num_symbols)]
pathways, associations = default_initializer( # all to all
layer_sizes.keys(), symbols)
codec = Codec(layer_sizes, symbols, rho=.9)
controller = Controller(layer_sizes, pathways, hidden_size, plastic)
# Sanity check
ghu = GatedHebbianUnit(layer_sizes,
pathways,
controller,
codec,
plastic=plastic)
ghu.associate(associations)
for p, s, t in associations:
q, r = ghu.pathways[p]
assert (codec.decode(q, tr.mv(ghu.W[p], codec.encode(r, s))) == t)
# Initialize layers
ghu.v[0]["rinp"] = codec.encode("rinp", "0")
ghu.v[0]["rout"] = codec.encode("rout", "1")
# Run GHU
ghu.tick() # Take a step
ghu.tick() # Take a step
print(ghu.ag)
print(gradtree(ghu.ag[1]["rout"][2]))
