def __init__(self, config): Backend = import_backend(config).ServerBackend self.backend = Backend(**config['backend']) self.new_frontend = import_frontend(config) self.key = config['key'] # tunnels dictionary, in which values are dictionaries of the # connections belong to it. Those dictionaries' key is the # Connection ID and value is the frontend instance. self.tunnels = ObjectDict() # dictionary of frontend instances, in which keys are the # frontend instances and values are tuples of their # corresponding Connection IDs and tunnel one belongs to. self.frontends = ObjectDict()
def generate(self, **kwargs): """Generate this template with the given arguments.""" namespace = { "escape": escape.xhtml_escape, "xhtml_escape": escape.xhtml_escape, "url_escape": escape.url_escape, "json_encode": escape.json_encode, "squeeze": escape.squeeze, "linkify": escape.linkify, "datetime": datetime, "_tt_utf8": escape.utf8, # for internal use "_tt_string_types": (unicode_type, bytes), # __name__ and __loader__ allow the traceback mechanism to find # the generated source code. "__name__": self.name.replace('.', '_'), "__loader__": ObjectDict(get_source=lambda name: self.code), } namespace.update(self.namespace) namespace.update(kwargs) exec_in(self.compiled, namespace) execute = namespace["_tt_execute"] # Clear the traceback module's cache of source data now that # we've generated a new template (mainly for this module's # unittests, where different tests reuse the same name). linecache.clearcache() return execute()
def __init__(self, data_type='seg'): conf_path = _get_abs_path(data_type + '_model/conf/') args = { 'tag_type': data_type, 'word_dict_path': conf_path + "/word.dic", 'label_dict_path': conf_path + "/tag.dic", 'word_rep_dict_path': conf_path + "/q2b.dic", 'batch_size': 120 } Dataset.__init__(self, ObjectDict(args), dev_count=10)
class TunnelServer(object): def __init__(self, config): Backend = import_backend(config).ServerBackend self.backend = Backend(**config['backend']) self.new_frontend = import_frontend(config) self.key = config['key'] # tunnels dictionary, in which values are dictionaries of the # connections belong to it. Those dictionaries' key is the # Connection ID and value is the frontend instance. self.tunnels = ObjectDict() # dictionary of frontend instances, in which keys are the # frontend instances and values are tuples of their # corresponding Connection IDs and tunnel one belongs to. self.frontends = ObjectDict() def run(self): self.running = True while self.running: try: self._process() except Exception as e: exc_type, _, exc_tb = sys.exc_info() exc_type = exc_type.__name__ exc_tb = traceback.extract_tb(exc_tb) msg = "unknown exception occurred: {0}, {1}; {2}"\ .format(exc_type, str(e), repr(exc_tb)) error(msg, 'tunnel', None) # close connections self.backend.close() for tunnel in self.tunnels.keys(): self._close_tunnel(tunnel) while True: wlist, wdict = get_select_list('get_wlist', self.tunnels.iterkeys()) if not wlist: break _, wlist, _ = select.select([], wlist, []) for fileno in wlist: conn = wdict[fileno] if conn in self.tunnels: self._process_tunnel_sending(conn) def _process(self): rlist, rdict = get_select_list( 'get_rlist', self.backend, self.tunnels.iterkeys(), (frontend for frontend, (conn_id, tunnel) in self.frontends.iteritems() if tunnel.available)) wlist, wdict = get_select_list('get_wlist', self.tunnels.iterkeys(), self.frontends.iterkeys()) try: rlist, wlist, _ = select.select(rlist, wlist, []) except select.error as e: if e[0] == errno.EINTR: return raise for fileno in rlist: conn = rdict[fileno] if conn is self.backend: self._process_backend() elif conn in self.tunnels: self._process_tunnel(conn) elif conn in self.frontends: self._process_frontend(conn) written_conns = ObjectSet() for fileno in wlist: conn = wdict[fileno] if conn in written_conns: continue written_conns.add(conn) if conn in self.tunnels: self._process_tunnel_sending(conn) else: conn.send() def _process_backend(self): inst = self.backend.accept() if not inst: return record_conn = RecordConnection(self.key, inst) tunnel = TunnelConnection(record_conn) tunnel.address = inst.address self.tunnels[tunnel] = {} info("connected", 'backend', inst.address) def _process_tunnel(self, tunnel): try: for packet in tunnel.receive_packets(): self._process_tunnel_packet(tunnel, *packet) except record.ConnectionClosedException: self._close_tunnel(tunnel) info("disconnected", 'record', tunnel.address) except record.CriticalException as e: # logging message if isinstance(e, record.HashfailError): msg = "detect a wrong hash" elif isinstance(e, record.InvalidHeaderError): msg = "detect an invalid header" elif isinstance(e, record.RemoteResetException): msg = "remote host reset the connection" elif isinstance(e, record.InsecureClosingError): msg = "detect an insecure closing" elif isinstance(e, record.FirstPacketIncorrectError): msg = "first packet is incorrect, protocol incompatible" else: msg = "detect a critical exception" # log the exception warning(msg, 'record', tunnel.address) self._close_tunnel(tunnel) def _process_tunnel_packet(self, tunnel, conn_id, control, data): frontends = self.tunnels[tunnel] # RST flag is set if control & StatusControl.rst: self._close_frontend(frontends[conn_id], True) # SYN flag is set if control & StatusControl.syn: if conn_id in frontends: self._close_frontend(frontends[conn_id], True) try: frontend = self.new_frontend() except FrontendUnavailableError: error("unavailable", 'frontend', tunnel.address) tunnel.reset_connection(conn_id) return frontends[conn_id] = frontend self.frontends[frontend] = conn_id, tunnel # DAT flag is set if control & StatusControl.dat: frontends[conn_id].send(data) # FIN flag is set if control & StatusControl.fin: self._close_frontend(frontends[conn_id]) def _process_frontend(self, frontend): conn_id, tunnel = self.frontends[frontend] try: data = frontend.recv() except Exception as e: msg = "unknown error: " + str(e) error(msg, 'frontend', tunnel.address) tunnel.reset_connection(conn_id) self._close_frontend(frontend) return if data: tunnel.send_packet(conn_id, data) elif data is None: tunnel.close_connection(conn_id) self._close_frontend(frontend) def _process_tunnel_sending(self, tunnel): tunnel.continue_sending() if tunnel.record_conn.closed: if not tunnel.get_wlist(): tunnel.record_conn.backend.close() del self.tunnels[tunnel] def _close_tunnel(self, tunnel): for frontend in self.tunnels[tunnel].values(): self._close_frontend(frontend) tunnel.record_conn.close() self._process_tunnel_sending(tunnel) def _close_frontend(self, frontend, reset=False): if reset: frontend.reset() else: frontend.close() conn_id, tunnel = self.frontends[frontend] del self.frontends[frontend] del self.tunnels[tunnel][conn_id]
class TunnelServer(object): def __init__(self, config): Backend = import_backend(config).ServerBackend self.backend = Backend(**config['backend']) self.new_frontend = import_frontend(config) self.key = config['key'] # tunnels dictionary, in which values are dictionaries of the # connections belong to it. Those dictionaries' key is the # Connection ID and value is the frontend instance. self.tunnels = ObjectDict() # dictionary of frontend instances, in which keys are the # frontend instances and values are tuples of their # corresponding Connection IDs and tunnel one belongs to. self.frontends = ObjectDict() def run(self): self.running = True while self.running: try: self._process() except Exception as e: exc_type, _, exc_tb = sys.exc_info() exc_type = exc_type.__name__ exc_tb = traceback.extract_tb(exc_tb) msg = "unknown exception occurred: {0}, {1}; {2}"\ .format(exc_type, str(e), repr(exc_tb)) error(msg, 'tunnel', None) # close connections self.backend.close() for tunnel in self.tunnels.keys(): self._close_tunnel(tunnel) while True: wlist, wdict = get_select_list( 'get_wlist', self.tunnels.iterkeys()) if not wlist: break _, wlist, _ = select.select([], wlist, []) for fileno in wlist: conn = wdict[fileno] if conn in self.tunnels: self._process_tunnel_sending(conn) def _process(self): rlist, rdict = get_select_list('get_rlist', self.backend, self.tunnels.iterkeys(), (frontend for frontend, (conn_id, tunnel) in self.frontends.iteritems() if tunnel.available)) wlist, wdict = get_select_list('get_wlist', self.tunnels.iterkeys(), self.frontends.iterkeys()) try: rlist, wlist, _ = select.select(rlist, wlist, []) except select.error as e: if e[0] == errno.EINTR: return raise for fileno in rlist: conn = rdict[fileno] if conn is self.backend: self._process_backend() elif conn in self.tunnels: self._process_tunnel(conn) elif conn in self.frontends: self._process_frontend(conn) written_conns = ObjectSet() for fileno in wlist: conn = wdict[fileno] if conn in written_conns: continue written_conns.add(conn) if conn in self.tunnels: self._process_tunnel_sending(conn) else: conn.send() def _process_backend(self): inst = self.backend.accept() if not inst: return record_conn = RecordConnection(self.key, inst) tunnel = TunnelConnection(record_conn) tunnel.address = inst.address self.tunnels[tunnel] = {} info("connected", 'backend', inst.address) def _process_tunnel(self, tunnel): try: for packet in tunnel.receive_packets(): self._process_tunnel_packet(tunnel, *packet) except record.ConnectionClosedException: self._close_tunnel(tunnel) info("disconnected", 'record', tunnel.address) except record.CriticalException as e: # logging message if isinstance(e, record.HashfailError): msg = "detect a wrong hash" elif isinstance(e, record.InvalidHeaderError): msg = "detect an invalid header" elif isinstance(e, record.RemoteResetException): msg = "remote host reset the connection" elif isinstance(e, record.InsecureClosingError): msg = "detect an insecure closing" elif isinstance(e, record.FirstPacketIncorrectError): msg = "first packet is incorrect, protocol incompatible" else: msg = "detect a critical exception" # log the exception warning(msg, 'record', tunnel.address) self._close_tunnel(tunnel) def _process_tunnel_packet(self, tunnel, conn_id, control, data): frontends = self.tunnels[tunnel] # RST flag is set if control & StatusControl.rst: self._close_frontend(frontends[conn_id], True) # SYN flag is set if control & StatusControl.syn: if conn_id in frontends: self._close_frontend(frontends[conn_id], True) try: frontend = self.new_frontend() except FrontendUnavailableError: error("unavailable", 'frontend', tunnel.address) tunnel.reset_connection(conn_id) return frontends[conn_id] = frontend self.frontends[frontend] = conn_id, tunnel # DAT flag is set if control & StatusControl.dat: frontends[conn_id].send(data) # FIN flag is set if control & StatusControl.fin: self._close_frontend(frontends[conn_id]) def _process_frontend(self, frontend): conn_id, tunnel = self.frontends[frontend] try: data = frontend.recv() except Exception as e: msg = "unknown error: " + str(e) error(msg, 'frontend', tunnel.address) tunnel.reset_connection(conn_id) self._close_frontend(frontend) return if data: tunnel.send_packet(conn_id, data) elif data is None: tunnel.close_connection(conn_id) self._close_frontend(frontend) def _process_tunnel_sending(self, tunnel): tunnel.continue_sending() if tunnel.record_conn.closed: if not tunnel.get_wlist(): tunnel.record_conn.backend.close() del self.tunnels[tunnel] def _close_tunnel(self, tunnel): for frontend in self.tunnels[tunnel].values(): self._close_frontend(frontend) tunnel.record_conn.close() self._process_tunnel_sending(tunnel) def _close_frontend(self, frontend, reset=False): if reset: frontend.reset() else: frontend.close() conn_id, tunnel = self.frontends[frontend] del self.frontends[frontend] del self.tunnels[tunnel][conn_id]