def main(): text = "a dead dad ceded a bad babe a beaded abaca bed" # on analyse les fréquences d'occurrence dans text # pour fabriquer un arbre binaire builder = TreeBuilder(text) binary_tree = builder.tree() # on passe l'arbre binaire à un encodeur/décodeur codec = Codec(binary_tree) # qui permet d'encoder encoded = codec.encode_bin(text) # et de décoder decoded = codec.decode_bin(encoded) # si cette assertion est fausse il y a un gros problème avec le code assert text == decoded # on affiche le résultat print(f"{text}\n{encoded}") if decoded != text: print("OOPS") text = 'ab' compress_encode = [] compress_huffman = [] compress_binary = [] for rang in range(300): texte = text * rang # Création d'un texte taille = sys.getsizeof(texte) # Espace système de ce texte compress_encode.append(taille / sys.getsizeof(codec.encode(texte))) compress_huffman.append(taille / sys.getsizeof(codec.encode_bin(texte))) compress_binary.append( sys.getsizeof(codec.encode(texte)) / sys.getsizeof(codec.encode_bin(texte))) # Affichage plt.close() plt.title('Facteurs de compression') plt.plot(compress_encode, label="Facteur de compression par \n Codec.encode sortie str") plt.plot( compress_huffman, label= "Facteur de compression de \n l'algorithme de Huffman (Avec Binaire)") plt.plot(compress_binary, label="Facteur de compression \n d'une str encoded en binaire ") plt.legend() plt.show()
def main(): with open(CONFIG_FN) as f: conf = json.load(f) global sent_groups with open(conf["sent_groups"]) as f: sent_groups = json.load(f)["groups"] kb = load_kb(conf["kb"], 'name') sys_vocab, sys_word2idx = load_sys_vocab(conf["sys_vocab"]) sys_codec = Codec(sys_vocab, sys_word2idx) onto, onto_idx = load_ontology(conf["ontology"]) word2idx, embed = load_embed(**conf) usr_codec = Codec([], word2idx) trk_model, slot_len_sum = load_tracker_model(onto, embed, conf, kb) trk_model.eval() hidden = trk_model.state_tracker.init_hidden() kb_vec = Variable(torch.zeros(1, conf["kb_indicator_len"])) sentence_generator = SentenceGenerator(kb, onto, sent_groups) for line in iter(sys.stdin.readline, ''): inp = usr_codec.encode(line.strip()) inp = Variable(torch.LongTensor([ inp, ])) sentvecs, states_reps, states_preds, hidden, sent_grp_preds = trk_model( inp, None, hidden) criteria = to_search_criteria(states_preds, onto) ret, kb_vec = get_kb_result(kb, criteria, conf["kb_indicator_len"]) # print criteria, kb_vec sentvecs = sentvecs.view(1, -1) states_reps = states_reps.view(1, -1) print_ret(states_preds, sent_grp_preds, onto, sentence_generator)
from codec import TreeBuilder, Codec text = "a dead dad ceded a bad babe a beaded abaca bed" builder = TreeBuilder(text) binary_tree = builder.tree() # on passe l'arbre binaire à un encodeur/décodeur codec = Codec(binary_tree) # qui permet d'encoder encoded = codec.encode(text) # et de décoder decoded = codec.decode(encoded) # si cette assertion est fausse il y a un gros problème avec le code # on affiche le résultat print(f"{text}\n{decoded}") if decoded != text: print("OOPS")
def main(): with open(CONFIG_FN) as f: conf = json.load(f) args = parse_args() kb = load_kb(conf["kb"], 'name') sys_vocab, sys_word2idx = load_sys_vocab(conf["sys_vocab"]) sys_codec = Codec(sys_vocab, sys_word2idx) onto, onto_idx = load_ontology(conf["ontology"]) word2idx, embed = load_embed(**conf) usr_codec = Codec([], word2idx) trk_model, slot_len_sum = load_tracker_model(onto, embed, conf) cond_net, generator = load_generator_model(conf, args.cond_net, args.gen_net, slot_len_sum, len(sys_vocab)) cond_net.eval() generator.eval() trk_model.eval() hidden = trk_model.state_tracker.init_hidden() kb_found = Variable(torch.zeros(1, conf["kb_indicator_len"])) def gen_sent(cond): '''train one sentence''' hidden = generator.init_hidden() inp = Variable(torch.LongTensor([[sys_word2idx['<sos>']]])) sent_out = [] for i in range(MAX_SENT_LEN): out, hidden = generator(inp, cond, hidden) topv, topi = out.data.topk(1) out_word = int(topi[0][0]) if out_word == sys_word2idx['<eos>']: break inp = Variable(torch.LongTensor([[out_word]])) sent_out.append(out_word) return sys_codec.decode(sent_out) for line in iter(sys.stdin.readline, ''): inp = usr_codec.encode(line.strip()) inp = Variable(torch.LongTensor([ inp, ])) sentvecs, states_reps, states_preds, hidden = trk_model( inp, kb_found, hidden) # print_ret(states_preds) criteria = to_search_criteria(states_preds, onto) ret, kb_vec = get_kb_result(kb, criteria, conf["kb_indicator_len"]) sentvecs = sentvecs.view(1, -1) states_reps = states_reps.view(1, -1) for slot in states_preds: states_preds[slot] = states_preds[slot].view(1, -1) cond = cond_net(sentvecs, states_reps, states_preds, kb_found) print gen_sent(cond)
codec = Codec(layer_sizes, symbols, rho=.999) controller = Controller(layer_sizes, pathways, hidden_size, plastic) # controller = Controller(layer_sizes, pathways, hidden_size, input_keys=["m","rinp"]) # controller.rnn.weight_hh_l0.data = 1*tr.eye(hidden_size) # favor repeated actions # Sanity check ghu = GatedHebbianUnit(layer_sizes, pathways, controller, codec, plastic=plastic) ghu.associate(associations) for p, s, t in associations: q, r = ghu.pathways[p] assert (codec.decode(q, tr.mv(ghu.W[p], codec.encode(r, s))) == t) ghu_init = ghu # # Pre-train for uniform action distribution # tol=.25 # max_time=4 # max_iters=500 # learning_rate=0.01 # for itr in range(max_iters): # ghu = GatedHebbianUnit(layer_sizes, pathways, controller, codec, plastic=plastic) # ghu.associate(associations) # loss = 0. # for t in range(max_time): # for k in layer_sizes.keys(): # ghu.v[t][k] = codec.encode(k, np.random.choice(symbols)) # ghu.tick()
# a = codec.encode("r0","0") # b = codec.encode("r0","1") # print(a) # x = codec.decode("r0",a) # print(x == "0") # print(x == "1") # y = codec.decode("r0",b) # print(y == "0") # print(y == "1") # print(codec.parameters()) for t in [-1, 0]: for k in ["r0", "r1"]: ghu.v[t][k] = tr.repeat_interleave(codec.encode(k, str(0 - t)).view( 1, -1), batch_size, dim=0) # ghu.v[-1]["r0"] = codec.encode("r0", str(1)) # ghu.v[-1]["r1"] = codec.encode("r1", str(1)) # ghu.v[0]["r0"] = codec.encode("r0", str(0)) # ghu.v[0]["r1"] = codec.encode("r1", str(0)) ghu.tick(num_steps=2) # e = ghu.g[len(ghu.g)-1].sum() # e.backward() # print("codec") # print(codec) # for p in codec.parameters():
class FIXConnectionHandler(object): def __init__(self, engine, protocol, sock=None, addr=None, observer=None): self.codec = Codec(protocol) self.engine = engine self.connectionState = ConnectionState.CONNECTED self.session = None self.addr = addr self.observer = observer self.msgBuffer = b'' self.heartbeatPeriod = 30.0 self.msgHandlers = [] self.sock = sock self.heartbeatTimerRegistration = None self.expectedHeartbeatRegistration = None self.socketEvent = FileDescriptorEventRegistration( self.handle_read, sock, EventType.READ) self.engine.eventManager.registerHandler(self.socketEvent) def address(self): return self.addr def disconnect(self): self.handle_close() def _notifyMessageObservers(self, msg, direction, persistMessage=True): if persistMessage is True: self.engine.journaller.persistMsg(msg, self.session, direction) for handler in filter( lambda x: (x[1] is None or x[1] == direction) and (x[2] is None or x[2] == msg.msgType), self.msgHandlers): handler[0](self, msg) def addMessageHandler(self, handler, direction=None, msgType=None): self.msgHandlers.append((handler, direction, msgType)) def removeMessageHandler(self, handler, direction=None, msgType=None): remove = filter( lambda x: x[0] == handler and (x[1] == direction or direction is None) and (x[2] == msgType or msgType is None), self.msgHandlers) for h in remove: self.msgHandlers.remove(h) def _sendHeartbeat(self): self.sendMsg(self.codec.protocol.messages.Messages.heartbeat()) def _expectedHeartbeat(self, type, closure): logging.warning("Expected heartbeat from peer %s" % (self.expectedHeartbeatRegistration, )) self.sendMsg(self.codec.protocol.messages.Messages.test_request()) def registerLoggedIn(self): self.heartbeatTimerRegistration = TimerEventRegistration( lambda type, closure: self._sendHeartbeat(), self.heartbeatPeriod) self.engine.eventManager.registerHandler( self.heartbeatTimerRegistration) # register timeout for 10% more than we expect self.expectedHeartbeatRegistration = TimerEventRegistration( self._expectedHeartbeat, self.heartbeatPeriod * 1.10) self.engine.eventManager.registerHandler( self.expectedHeartbeatRegistration) def registerLoggedOut(self): if self.heartbeatTimerRegistration is not None: self.engine.eventManager.unregisterHandler( self.heartbeatTimerRegistration) self.heartbeatTimerRegistration = None if self.expectedHeartbeatRegistration is not None: self.engine.eventManager.unregisterHandler( self.expectedHeartbeatRegistration) self.expectedHeartbeatRegistration = None def _handleResendRequest(self, msg): protocol = self.codec.protocol responses = [] beginSeqNo = msg[protocol.fixtags.BeginSeqNo] endSeqNo = msg[protocol.fixtags.EndSeqNo] if int(endSeqNo) == 0: endSeqNo = sys.maxsize logging.info("Received resent request from %s to %s", beginSeqNo, endSeqNo) replayMsgs = self.engine.journaller.recoverMsgs( self.session, MessageDirection.OUTBOUND, beginSeqNo, endSeqNo) gapFillBegin = int(beginSeqNo) gapFillEnd = int(beginSeqNo) for replayMsg in replayMsgs: msgSeqNum = int(replayMsg[protocol.fixtags.MsgSeqNum]) if replayMsg[protocol.fixtags. MsgType] in protocol.msgtype.sessionMessageTypes: gapFillEnd = msgSeqNum + 1 else: if self.engine.shouldResendMessage(self.session, replayMsg): if gapFillBegin < gapFillEnd: # we need to send a gap fill message gapFillMsg = FIXMessage(protocol.msgtype.SEQUENCERESET) gapFillMsg.setField(protocol.fixtags.GapFillFlag, 'Y') gapFillMsg.setField(protocol.fixtags.MsgSeqNum, gapFillBegin) gapFillMsg.setField(protocol.fixtags.NewSeqNo, str(gapFillEnd)) responses.append(gapFillMsg) # and then resent the replayMsg replayMsg.removeField(protocol.fixtags.BeginString) replayMsg.removeField(protocol.fixtags.BodyLength) replayMsg.removeField(protocol.fixtags.SendingTime) replayMsg.removeField(protocol.fixtags.SenderCompID) replayMsg.removeField(protocol.fixtags.TargetCompID) replayMsg.removeField(protocol.fixtags.CheckSum) replayMsg.setField(protocol.fixtags.PossDupFlag, "Y") responses.append(replayMsg) gapFillBegin = msgSeqNum + 1 else: gapFillEnd = msgSeqNum + 1 responses.append(replayMsg) if gapFillBegin < gapFillEnd: # we need to send a gap fill message gapFillMsg = FIXMessage(protocol.msgtype.SEQUENCERESET) gapFillMsg.setField(protocol.fixtags.GapFillFlag, 'Y') gapFillMsg.setField(protocol.fixtags.MsgSeqNum, gapFillBegin) gapFillMsg.setField(protocol.fixtags.NewSeqNo, str(gapFillEnd)) responses.append(gapFillMsg) return responses def handle_read(self, type, closure): protocol = self.codec.protocol try: msg = self.sock.recv(8192) if msg: self.msgBuffer = self.msgBuffer + msg (decodedMsg, parsedLength) = self.codec.decode(self.msgBuffer) self.msgBuffer = self.msgBuffer[parsedLength:] while decodedMsg is not None and self.connectionState != ConnectionState.DISCONNECTED: self.processMessage(decodedMsg) (decodedMsg, parsedLength) = self.codec.decode(self.msgBuffer) self.msgBuffer = self.msgBuffer[parsedLength:] if self.expectedHeartbeatRegistration is not None: self.expectedHeartbeatRegistration.reset() else: logging.debug("Connection has been closed") self.disconnect() except ConnectionError as why: logging.debug("Connection has been closed %s" % (why, )) self.disconnect() def handleSessionMessage(self, msg): return -1 def processMessage(self, decodedMsg): protocol = self.codec.protocol beginString = decodedMsg[protocol.fixtags.BeginString] if beginString != protocol.beginstring: logging.warning( "FIX BeginString is incorrect (expected: %s received: %s)", (protocol.beginstring, beginString)) self.disconnect() return msgType = decodedMsg[protocol.fixtags.MsgType] try: responses = [] if msgType in protocol.msgtype.sessionMessageTypes: (recvSeqNo, responses) = self.handleSessionMessage(decodedMsg) else: recvSeqNo = decodedMsg[protocol.fixtags.MsgSeqNum] # validate the seq number (seqNoState, lastKnownSeqNo) = self.session.validateRecvSeqNo(recvSeqNo) if seqNoState is False: # We should send a resend request logging.info("Requesting resend of messages: %s to %s" % (lastKnownSeqNo, 0)) responses.append( protocol.messages.Messages.resend_request( lastKnownSeqNo, 0)) # we still need to notify if we are processing Logon message if msgType == protocol.msgtype.LOGON: self._notifyMessageObservers(decodedMsg, MessageDirection.INBOUND, False) else: self.session.setRecvSeqNo(recvSeqNo) self._notifyMessageObservers(decodedMsg, MessageDirection.INBOUND) for m in responses: self.sendMsg(m) except SessionWarning as sw: logging.warning(sw) except SessionError as se: logging.error(se) self.disconnect() except DuplicateSeqNoError: try: if decodedMsg[protocol.fixtags.PossDupFlag] == "Y": logging.debug( "Received duplicate message with PossDupFlag set") except KeyError: pass finally: logging.error( "Failed to process message with duplicate seq no (MsgSeqNum: %s) (and no PossDupFlag='Y') - disconnecting" % (recvSeqNo, )) self.disconnect() def handle_close(self): if self.connectionState != ConnectionState.DISCONNECTED: logging.info("Client disconnected") self.registerLoggedOut() self.sock.close() self.connectionState = ConnectionState.DISCONNECTED self.msgHandlers.clear() if self.observer is not None: self.observer.notifyDisconnect(self) self.engine.eventManager.unregisterHandler(self.socketEvent) def sendMsg(self, msg): if self.connectionState != ConnectionState.CONNECTED and self.connectionState != ConnectionState.LOGGED_IN: raise FIXException(FIXException.FIXExceptionReason.NOT_CONNECTED) encodedMsg = self.codec.encode(msg, self.session).encode('utf-8') self.sock.send(encodedMsg) if self.heartbeatTimerRegistration is not None: self.heartbeatTimerRegistration.reset() decodedMsg, junk = self.codec.decode(encodedMsg) try: self._notifyMessageObservers(decodedMsg, MessageDirection.OUTBOUND) except DuplicateSeqNoError: logging.error( "We have sent a message with a duplicate seq no, failed to persist it (MsgSeqNum: %s)" % (decodedMsg[self.codec.protocol.fixtags.MsgSeqNum]))
plastic = [] symbols = [str(a) for a in range(num_symbols)] pathways, associations = default_initializer( # all to all layer_sizes.keys(), symbols) codec = Codec(layer_sizes, symbols, rho=.9) controller = Controller(layer_sizes, pathways, hidden_size, plastic) # Sanity check ghu = GatedHebbianUnit(layer_sizes, pathways, controller, codec, plastic=plastic) ghu.associate(associations) for p, s, t in associations: q, r = ghu.pathways[p] assert (codec.decode(q, tr.mv(ghu.W[p], codec.encode(r, s))) == t) # Initialize layers ghu.v[0]["rinp"] = codec.encode("rinp", "0") ghu.v[0]["rout"] = codec.encode("rout", "1") # Run GHU ghu.tick() # Take a step ghu.tick() # Take a step print(ghu.ag) print(gradtree(ghu.ag[1]["rout"][2]))