def __init__(self, config):
Backend = import_backend(config).ServerBackend
self.backend = Backend(**config['backend'])
self.new_frontend = import_frontend(config)
self.key = config['key']
# tunnels dictionary, in which values are dictionaries of the
# connections belong to it. Those dictionaries' key is the
# Connection ID and value is the frontend instance.
self.tunnels = ObjectDict()
# dictionary of frontend instances, in which keys are the
# frontend instances and values are tuples of their
# corresponding Connection IDs and tunnel one belongs to.
self.frontends = ObjectDict()
def __init__(self, config):
Backend = import_backend(config).ServerBackend
self.backend = Backend(**config['backend'])
self.new_frontend = import_frontend(config)
self.key = config['key']
# tunnels dictionary, in which values are dictionaries of the
# connections belong to it. Those dictionaries' key is the
# Connection ID and value is the frontend instance.
self.tunnels = ObjectDict()
# dictionary of frontend instances, in which keys are the
# frontend instances and values are tuples of their
# corresponding Connection IDs and tunnel one belongs to.
self.frontends = ObjectDict()
def generate(self, **kwargs):
"""Generate this template with the given arguments."""
namespace = {
"escape": escape.xhtml_escape,
"xhtml_escape": escape.xhtml_escape,
"url_escape": escape.url_escape,
"json_encode": escape.json_encode,
"squeeze": escape.squeeze,
"linkify": escape.linkify,
"datetime": datetime,
"_tt_utf8": escape.utf8, # for internal use
"_tt_string_types": (unicode_type, bytes),
# __name__ and __loader__ allow the traceback mechanism to find
# the generated source code.
"__name__": self.name.replace('.', '_'),
"__loader__": ObjectDict(get_source=lambda name: self.code),
}
namespace.update(self.namespace)
namespace.update(kwargs)
exec_in(self.compiled, namespace)
execute = namespace["_tt_execute"]
# Clear the traceback module's cache of source data now that
# we've generated a new template (mainly for this module's
# unittests, where different tests reuse the same name).
linecache.clearcache()
return execute()
def __init__(self, data_type='seg'):
conf_path = _get_abs_path(data_type + '_model/conf/')
args = {
'tag_type': data_type,
'word_dict_path': conf_path + "/word.dic",
'label_dict_path': conf_path + "/tag.dic",
'word_rep_dict_path': conf_path + "/q2b.dic",
'batch_size': 120
}
Dataset.__init__(self, ObjectDict(args), dev_count=10)
class TunnelServer(object):
def __init__(self, config):
Backend = import_backend(config).ServerBackend
self.backend = Backend(**config['backend'])
self.new_frontend = import_frontend(config)
self.key = config['key']
# tunnels dictionary, in which values are dictionaries of the
# connections belong to it. Those dictionaries' key is the
# Connection ID and value is the frontend instance.
self.tunnels = ObjectDict()
# dictionary of frontend instances, in which keys are the
# frontend instances and values are tuples of their
# corresponding Connection IDs and tunnel one belongs to.
self.frontends = ObjectDict()
def run(self):
self.running = True
while self.running:
try:
self._process()
except Exception as e:
exc_type, _, exc_tb = sys.exc_info()
exc_type = exc_type.__name__
exc_tb = traceback.extract_tb(exc_tb)
msg = "unknown exception occurred: {0}, {1}; {2}"\
.format(exc_type, str(e), repr(exc_tb))
error(msg, 'tunnel', None)
# close connections
self.backend.close()
for tunnel in self.tunnels.keys():
self._close_tunnel(tunnel)
while True:
wlist, wdict = get_select_list('get_wlist',
self.tunnels.iterkeys())
if not wlist:
break
_, wlist, _ = select.select([], wlist, [])
for fileno in wlist:
conn = wdict[fileno]
if conn in self.tunnels:
self._process_tunnel_sending(conn)
def _process(self):
rlist, rdict = get_select_list(
'get_rlist', self.backend, self.tunnels.iterkeys(),
(frontend
for frontend, (conn_id, tunnel) in self.frontends.iteritems()
if tunnel.available))
wlist, wdict = get_select_list('get_wlist', self.tunnels.iterkeys(),
self.frontends.iterkeys())
try:
rlist, wlist, _ = select.select(rlist, wlist, [])
except select.error as e:
if e[0] == errno.EINTR:
return
raise
for fileno in rlist:
conn = rdict[fileno]
if conn is self.backend:
self._process_backend()
elif conn in self.tunnels:
self._process_tunnel(conn)
elif conn in self.frontends:
self._process_frontend(conn)
written_conns = ObjectSet()
for fileno in wlist:
conn = wdict[fileno]
if conn in written_conns:
continue
written_conns.add(conn)
if conn in self.tunnels:
self._process_tunnel_sending(conn)
else:
conn.send()
def _process_backend(self):
inst = self.backend.accept()
if not inst:
return
record_conn = RecordConnection(self.key, inst)
tunnel = TunnelConnection(record_conn)
tunnel.address = inst.address
self.tunnels[tunnel] = {}
info("connected", 'backend', inst.address)
def _process_tunnel(self, tunnel):
try:
for packet in tunnel.receive_packets():
self._process_tunnel_packet(tunnel, *packet)
except record.ConnectionClosedException:
self._close_tunnel(tunnel)
info("disconnected", 'record', tunnel.address)
except record.CriticalException as e:
# logging message
if isinstance(e, record.HashfailError):
msg = "detect a wrong hash"
elif isinstance(e, record.InvalidHeaderError):
msg = "detect an invalid header"
elif isinstance(e, record.RemoteResetException):
msg = "remote host reset the connection"
elif isinstance(e, record.InsecureClosingError):
msg = "detect an insecure closing"
elif isinstance(e, record.FirstPacketIncorrectError):
msg = "first packet is incorrect, protocol incompatible"
else:
msg = "detect a critical exception"
# log the exception
warning(msg, 'record', tunnel.address)
self._close_tunnel(tunnel)
def _process_tunnel_packet(self, tunnel, conn_id, control, data):
frontends = self.tunnels[tunnel]
# RST flag is set
if control & StatusControl.rst:
self._close_frontend(frontends[conn_id], True)
# SYN flag is set
if control & StatusControl.syn:
if conn_id in frontends:
self._close_frontend(frontends[conn_id], True)
try:
frontend = self.new_frontend()
except FrontendUnavailableError:
error("unavailable", 'frontend', tunnel.address)
tunnel.reset_connection(conn_id)
return
frontends[conn_id] = frontend
self.frontends[frontend] = conn_id, tunnel
# DAT flag is set
if control & StatusControl.dat:
frontends[conn_id].send(data)
# FIN flag is set
if control & StatusControl.fin:
self._close_frontend(frontends[conn_id])
def _process_frontend(self, frontend):
conn_id, tunnel = self.frontends[frontend]
try:
data = frontend.recv()
except Exception as e:
msg = "unknown error: " + str(e)
error(msg, 'frontend', tunnel.address)
tunnel.reset_connection(conn_id)
self._close_frontend(frontend)
return
if data:
tunnel.send_packet(conn_id, data)
elif data is None:
tunnel.close_connection(conn_id)
self._close_frontend(frontend)
def _process_tunnel_sending(self, tunnel):
tunnel.continue_sending()
if tunnel.record_conn.closed:
if not tunnel.get_wlist():
tunnel.record_conn.backend.close()
del self.tunnels[tunnel]
def _close_tunnel(self, tunnel):
for frontend in self.tunnels[tunnel].values():
self._close_frontend(frontend)
tunnel.record_conn.close()
self._process_tunnel_sending(tunnel)
def _close_frontend(self, frontend, reset=False):
if reset:
frontend.reset()
else:
frontend.close()
conn_id, tunnel = self.frontends[frontend]
del self.frontends[frontend]
del self.tunnels[tunnel][conn_id]
class TunnelServer(object):
def __init__(self, config):
Backend = import_backend(config).ServerBackend
self.backend = Backend(**config['backend'])
self.new_frontend = import_frontend(config)
self.key = config['key']
# tunnels dictionary, in which values are dictionaries of the
# connections belong to it. Those dictionaries' key is the
# Connection ID and value is the frontend instance.
self.tunnels = ObjectDict()
# dictionary of frontend instances, in which keys are the
# frontend instances and values are tuples of their
# corresponding Connection IDs and tunnel one belongs to.
self.frontends = ObjectDict()
def run(self):
self.running = True
while self.running:
try:
self._process()
except Exception as e:
exc_type, _, exc_tb = sys.exc_info()
exc_type = exc_type.__name__
exc_tb = traceback.extract_tb(exc_tb)
msg = "unknown exception occurred: {0}, {1}; {2}"\
.format(exc_type, str(e), repr(exc_tb))
error(msg, 'tunnel', None)
# close connections
self.backend.close()
for tunnel in self.tunnels.keys():
self._close_tunnel(tunnel)
while True:
wlist, wdict = get_select_list(
'get_wlist', self.tunnels.iterkeys())
if not wlist:
break
_, wlist, _ = select.select([], wlist, [])
for fileno in wlist:
conn = wdict[fileno]
if conn in self.tunnels:
self._process_tunnel_sending(conn)
def _process(self):
rlist, rdict = get_select_list('get_rlist',
self.backend, self.tunnels.iterkeys(),
(frontend for frontend, (conn_id, tunnel)
in self.frontends.iteritems()
if tunnel.available))
wlist, wdict = get_select_list('get_wlist',
self.tunnels.iterkeys(), self.frontends.iterkeys())
try:
rlist, wlist, _ = select.select(rlist, wlist, [])
except select.error as e:
if e[0] == errno.EINTR:
return
raise
for fileno in rlist:
conn = rdict[fileno]
if conn is self.backend:
self._process_backend()
elif conn in self.tunnels:
self._process_tunnel(conn)
elif conn in self.frontends:
self._process_frontend(conn)
written_conns = ObjectSet()
for fileno in wlist:
conn = wdict[fileno]
if conn in written_conns:
continue
written_conns.add(conn)
if conn in self.tunnels:
self._process_tunnel_sending(conn)
else:
conn.send()
def _process_backend(self):
inst = self.backend.accept()
if not inst:
return
record_conn = RecordConnection(self.key, inst)
tunnel = TunnelConnection(record_conn)
tunnel.address = inst.address
self.tunnels[tunnel] = {}
info("connected", 'backend', inst.address)
def _process_tunnel(self, tunnel):
try:
for packet in tunnel.receive_packets():
self._process_tunnel_packet(tunnel, *packet)
except record.ConnectionClosedException:
self._close_tunnel(tunnel)
info("disconnected", 'record', tunnel.address)
except record.CriticalException as e:
# logging message
if isinstance(e, record.HashfailError):
msg = "detect a wrong hash"
elif isinstance(e, record.InvalidHeaderError):
msg = "detect an invalid header"
elif isinstance(e, record.RemoteResetException):
msg = "remote host reset the connection"
elif isinstance(e, record.InsecureClosingError):
msg = "detect an insecure closing"
elif isinstance(e, record.FirstPacketIncorrectError):
msg = "first packet is incorrect, protocol incompatible"
else:
msg = "detect a critical exception"
# log the exception
warning(msg, 'record', tunnel.address)
self._close_tunnel(tunnel)
def _process_tunnel_packet(self, tunnel, conn_id, control, data):
frontends = self.tunnels[tunnel]
# RST flag is set
if control & StatusControl.rst:
self._close_frontend(frontends[conn_id], True)
# SYN flag is set
if control & StatusControl.syn:
if conn_id in frontends:
self._close_frontend(frontends[conn_id], True)
try:
frontend = self.new_frontend()
except FrontendUnavailableError:
error("unavailable", 'frontend', tunnel.address)
tunnel.reset_connection(conn_id)
return
frontends[conn_id] = frontend
self.frontends[frontend] = conn_id, tunnel
# DAT flag is set
if control & StatusControl.dat:
frontends[conn_id].send(data)
# FIN flag is set
if control & StatusControl.fin:
self._close_frontend(frontends[conn_id])
def _process_frontend(self, frontend):
conn_id, tunnel = self.frontends[frontend]
try:
data = frontend.recv()
except Exception as e:
msg = "unknown error: " + str(e)
error(msg, 'frontend', tunnel.address)
tunnel.reset_connection(conn_id)
self._close_frontend(frontend)
return
if data:
tunnel.send_packet(conn_id, data)
elif data is None:
tunnel.close_connection(conn_id)
self._close_frontend(frontend)
def _process_tunnel_sending(self, tunnel):
tunnel.continue_sending()
if tunnel.record_conn.closed:
if not tunnel.get_wlist():
tunnel.record_conn.backend.close()
del self.tunnels[tunnel]
def _close_tunnel(self, tunnel):
for frontend in self.tunnels[tunnel].values():
self._close_frontend(frontend)
tunnel.record_conn.close()
self._process_tunnel_sending(tunnel)
def _close_frontend(self, frontend, reset=False):
if reset:
frontend.reset()
else:
frontend.close()
conn_id, tunnel = self.frontends[frontend]
del self.frontends[frontend]
del self.tunnels[tunnel][conn_id]
