def listenSSLonExistingFD(reactor, fd, factory, contextFactory): try: from twisted.protocols import tls except ImportError: raise Exception("tor2web: ssl hack for listenSSLonExistingFD not implemented (tls only)") tlsFactory = tls.TLSMemoryBIOFactory(contextFactory, False, factory) port = reactor.listenTCPonExistingFD(reactor, fd, tlsFactory) port._type = 'TLS' return port
def daemon_main(self): reactor.listenTCPonExistingFD = listenTCPonExistingFD reactor.listenTCPonExistingFD(reactor, fd=self.socket_rpc.fileno(), factory=pb.PBServerFactory(rpc_server)) for i in range(config.processes): subprocess = spawnT2W(self.childFDs, self.fds_https, self.fds_http) subprocesses.append(subprocess.pid) if config.debugmode: if config.debugtostdout: log.startLogging(sys.stdout) else: log.startLogging(log.NullFile) sys.excepthook = MailException reactor.run()
def start_worker(): global antanistaticmap global templates global pool global rexp global ports lc = LoopingCall(updateListsTask) lc.start(600) rexp = { 'body': re.compile(r'(<body.*?\s*>)', re.I), 'w2t': re.compile(r'(http.?:)?//([a-z0-9]{16}).' + config.basehost + '(?!:\d+)', re.I), 't2w': re.compile(r'(http.?:)?//([a-z0-9]{16}).(?!' + config.basehost + ')onion(?!:\d+)', re.I) } ############################################################################### # Static Data loading # Here we make a file caching to not handle I/O # at run-time and achieve better performance ############################################################################### antanistaticmap = {} # system default static files sys_static_dir = os.path.join(config.sysdatadir, "static/") if os.path.exists(sys_static_dir): for root, dirs, files in os.walk(os.path.join(sys_static_dir)): for basename in files: filename = os.path.join(root, basename) f = FilePath(filename) antanistaticmap[filename.replace(sys_static_dir, "")] = f.getContent() # user defined static files usr_static_dir = os.path.join(config.datadir, "static/") if usr_static_dir != sys_static_dir and os.path.exists(usr_static_dir): for root, dirs, files in os.walk(os.path.join(usr_static_dir)): for basename in files: filename = os.path.join(root, basename) f = FilePath(filename) antanistaticmap[filename.replace(usr_static_dir, "")] = f.getContent() ############################################################################### ############################################################################### # Templates loading # Here we make a templates cache in order to not handle I/O # at run-time and achieve better performance ############################################################################### templates = {} # system default templates sys_tpl_dir = os.path.join(config.sysdatadir, "templates/") if os.path.exists(sys_tpl_dir): files = FilePath(sys_tpl_dir).globChildren("*.tpl") for f in files: f = FilePath(config.t2w_file_path(os.path.join('templates', f.basename()))) templates[f.basename()] = PageTemplate(XMLString(f.getContent())) # user defined templates usr_tpl_dir = os.path.join(config.datadir, "templates/") if usr_tpl_dir != sys_tpl_dir and os.path.exists(usr_tpl_dir): files = FilePath(usr_tpl_dir).globChildren("*.tpl") for f in files: f = FilePath(config.t2w_file_path(os.path.join('templates', f.basename()))) templates[f.basename()] = PageTemplate(XMLString(f.getContent())) ############################################################################### pool = HTTPConnectionPool(reactor, True, config.sockmaxpersistentperhost, config.sockcachedconnectiontimeout, config.sockretryautomatically) factory = T2WProxyFactory() # we do not want all workers to die in the same moment requests_countdown = config.requests_per_process / random.randint(1, 3) factory = T2WLimitedRequestsFactory(factory, requests_countdown) context_factory = T2WSSLContextFactory(os.path.join(config.datadir, "certs/tor2web-key.pem"), os.path.join(config.datadir, "certs/tor2web-intermediate.pem"), os.path.join(config.datadir, "certs/tor2web-dh.pem"), config.cipher_list) fds_https = [] if 'T2W_FDS_HTTPS' in os.environ: fds_https = filter(None, os.environ['T2W_FDS_HTTPS'].split(",")) fds_https = [int(i) for i in fds_https] fds_http = [] if 'T2W_FDS_HTTP' in os.environ: fds_http = filter(None, os.environ['T2W_FDS_HTTP'].split(",")) fds_http = [int(i) for i in fds_http] reactor.listenTCPonExistingFD = listenTCPonExistingFD reactor.listenSSLonExistingFD = listenSSLonExistingFD for fd in fds_https: ports.append(reactor.listenSSLonExistingFD(reactor, fd=fd, factory=factory, contextFactory=context_factory)) for fd in fds_http: ports.append(reactor.listenTCPonExistingFD(reactor, fd=fd, factory=factory)) def MailException(etype, value, tb): sendexceptionmail(config, etype, value, tb) sys.excepthook = MailException
def start(): global config global antanistaticmap global templates global pool global rexp global fds_https global fds_http global ports global requests_countdown config = yield rpc("get_config") rexp = { 'body': re.compile(r'(<body.*?\s*>)', re.I), 'w2t': re.compile(r'(https:)?//([a-z0-9]{16}).' + config['basehost'] + '(:443)?', re.I), 't2w': re.compile(r'(http:)?//([a-z0-9]{16}).onion(:80)?', re.I) } ############################################################################### # Templates loading ############################################################################### antanistaticmap = {} files = FilePath('/usr/share/tor2web/static/').globChildren("*") for file in files: file = FilePath(t2w_file_path(config['datadir'], os.path.join('static', file.basename()))) antanistaticmap[file.basename()] = file.getContent() # we add additional files eventually written in datadir/static # and not already loaded by previos lines. if os.path.exists(os.path.join(config['datadir'], "static/")): for file in files: if file.basename() not in antanistaticmap: antanistaticmap[file.basename()] = file.getContent() ############################################################################### ############################################################################### # Templates loading ############################################################################### templates = {} files = FilePath('/usr/share/tor2web/templates/').globChildren("*.tpl") for file in files: file = FilePath(t2w_file_path(config['datadir'], os.path.join('templates', file.basename()))) templates[file.basename()] = PageTemplate(XMLString(file.getContent())) ############################################################################### pool = HTTPConnectionPool(reactor, True, config['sockmaxpersistentperhost'], config['sockcachedconnectiontimeout'], config['sockretryautomatically']) factory = T2WProxyFactory() factory.requestCountdown = config['requests_per_process'] context_factory = T2WSSLContextFactory(os.path.join(config['datadir'], "certs/tor2web-key.pem"), os.path.join(config['datadir'], "certs/tor2web-intermediate.pem"), os.path.join(config['datadir'], "certs/tor2web-dh.pem"), config['cipher_list']) if config['debugmode']: if config['debugtostdout']: log.startLogging(sys.stdout) else: log.startLogging(log.NullFile) fds_https = filter(None, args[0].split(",")) fds_https = [int(i) for i in fds_https] fds_http = filter(None, args[1].split(",")) fds_http = [int(i) for i in fds_http] reactor.listenTCPonExistingFD = listenTCPonExistingFD reactor.listenSSLonExistingFD = listenSSLonExistingFD for fd in fds_https: ports.append(reactor.listenSSLonExistingFD(reactor, fd=fd, factory=factory, contextFactory=context_factory)) for fd in fds_http: ports.append(reactor.listenTCPonExistingFD(reactor, fd=fd, factory=factory)) # we do not want all workers to die in the same moment requests_countdown = config['requests_per_process'] / random.randint(3, 5) sys.excepthook = MailException
def start(): global config global antanistaticmap global templates global pool global rexp global fds_https global fds_http global ports config = yield rpc("get_config") lc = LoopingCall(updateTask) lc.start(600) rexp = { 'body': re.compile(r'(<body.*?\s*>)', re.I), 'w2t': re.compile(r'(http.?:)?//([a-z0-9]{16}).' + config['basehost'] + '(?!:\d+)', re.I), 't2w': re.compile(r'(http.?:)?//([a-z0-9]{16}).(?!' + config['basehost'] + ')onion(?!:\d+)', re.I) } ############################################################################### # Static Data loading # Here we make a file caching to not handle I/O # at run-time and achieve better performance ############################################################################### antanistaticmap = {} files = FilePath(static_path).globChildren("*") for f in files: f = FilePath(t2w_file_path(config['datadir'], os.path.join('static', f.basename()))) antanistaticmap[f.basename()] = f.getContent() # we add also user defined data allowing also the user to override tor2web defaults. userstaticdir = os.path.join(config['datadir'], "static/") if os.path.exists(userstaticdir): for root, dirs, files in os.walk(os.path.join(userstaticdir)): for basename in files: filename = os.path.join(root, basename) f = FilePath(filename) antanistaticmap[filename.replace(userstaticdir, "")] = f.getContent() ############################################################################### ############################################################################### # Templates loading # Here we make a templates caching to not handle I/O # at run-time and achieve better performance ############################################################################### templates = {} files = FilePath(templates_path).globChildren("*.tpl") for f in files: f = FilePath(t2w_file_path(config['datadir'], os.path.join('templates', f.basename()))) templates[f.basename()] = PageTemplate(XMLString(f.getContent())) ############################################################################### pool = HTTPConnectionPool(reactor, True, config['sockmaxpersistentperhost'], config['sockcachedconnectiontimeout'], config['sockretryautomatically']) factory = T2WProxyFactory() # we do not want all workers to die in the same moment requests_countdown = config['requests_per_process'] / random.randint(1, 3) factory = T2WLimitedRequestsFactory(factory, requests_countdown) context_factory = T2WSSLContextFactory(os.path.join(config['datadir'], "certs/tor2web-key.pem"), os.path.join(config['datadir'], "certs/tor2web-intermediate.pem"), os.path.join(config['datadir'], "certs/tor2web-dh.pem"), config['cipher_list']) if config['debugmode'] and config['debugtostdout']: log.startLogging(sys.stdout) else: log.startLogging(log.NullFile) fds_https = filter(None, args[0].split(",")) fds_https = [int(i) for i in fds_https] fds_http = filter(None, args[1].split(",")) fds_http = [int(i) for i in fds_http] reactor.listenTCPonExistingFD = listenTCPonExistingFD reactor.listenSSLonExistingFD = listenSSLonExistingFD for fd in fds_https: ports.append(reactor.listenSSLonExistingFD(reactor, fd=fd, factory=factory, contextFactory=context_factory)) for fd in fds_http: ports.append(reactor.listenTCPonExistingFD(reactor, fd=fd, factory=factory)) sys.excepthook = MailException
def start(): global config global antanistaticmap global templates global pool global rexp global fds_https global fds_http global ports config = yield rpc("get_config") lc = LoopingCall(updateTask) lc.start(600) rexp = { 'body': re.compile(r'(<body.*?\s*>)', re.I), 'w2t': re.compile( r'(https:)?//([a-z0-9]{16}).' + config['basehost'] + '(:443)?', re.I), 't2w': re.compile(r'(http:)?//([a-z0-9]{16}).onion(:80)?', re.I) } ############################################################################### # Templates loading ############################################################################### antanistaticmap = {} files = FilePath('/usr/share/tor2web/static/').globChildren("*") for file in files: file = FilePath( t2w_file_path(config['datadir'], os.path.join('static', file.basename()))) antanistaticmap[file.basename()] = file.getContent() # we add additional files eventually written in datadir/static # and not already loaded by previos lines. if os.path.exists(os.path.join(config['datadir'], "static/")): for file in files: if file.basename() not in antanistaticmap: antanistaticmap[file.basename()] = file.getContent() ############################################################################### ############################################################################### # Templates loading ############################################################################### templates = {} files = FilePath('/usr/share/tor2web/templates/').globChildren("*.tpl") for file in files: file = FilePath( t2w_file_path(config['datadir'], os.path.join('templates', file.basename()))) templates[file.basename()] = PageTemplate(XMLString(file.getContent())) ############################################################################### pool = HTTPConnectionPool(reactor, True, config['sockmaxpersistentperhost'], config['sockcachedconnectiontimeout'], config['sockretryautomatically']) factory = T2WProxyFactory() # we do not want all workers to die in the same moment requests_countdown = config['requests_per_process'] / random.randint(1, 3) factory = T2WLimitedRequestsFactory(factory, requests_countdown) context_factory = T2WSSLContextFactory( os.path.join(config['datadir'], "certs/tor2web-key.pem"), os.path.join(config['datadir'], "certs/tor2web-intermediate.pem"), os.path.join(config['datadir'], "certs/tor2web-dh.pem"), config['cipher_list']) if config['debugmode'] and config['debugtostdout']: log.startLogging(sys.stdout) else: log.startLogging(log.NullFile) fds_https = filter(None, args[0].split(",")) fds_https = [int(i) for i in fds_https] fds_http = filter(None, args[1].split(",")) fds_http = [int(i) for i in fds_http] reactor.listenTCPonExistingFD = listenTCPonExistingFD reactor.listenSSLonExistingFD = listenSSLonExistingFD for fd in fds_https: ports.append( reactor.listenSSLonExistingFD(reactor, fd=fd, factory=factory, contextFactory=context_factory)) for fd in fds_http: ports.append( reactor.listenTCPonExistingFD(reactor, fd=fd, factory=factory)) sys.excepthook = MailException
def start(): global config global antanistaticmap global templates global pool global rexp global fds_https global fds_http global ports config = yield rpc("get_config") lc = LoopingCall(updateTask) lc.start(600) rexp = { 'body': re.compile(r'(<body.*?\s*>)', re.I), 'w2t': re.compile( r'(http.?:)?//([a-z0-9]{16}).' + config['basehost'] + '(?!:\d+)', re.I), 't2w': re.compile( r'(http.?:)?//([a-z0-9]{16}).(?!' + config['basehost'] + ')onion(?!:\d+)', re.I) } ############################################################################### # Static Data loading # Here we make a file caching to not handle I/O # at run-time and achieve better performance ############################################################################### antanistaticmap = {} files = FilePath('/usr/share/tor2web/static/').globChildren("*") for f in files: f = FilePath( t2w_file_path(config['datadir'], os.path.join('static', f.basename()))) antanistaticmap[f.basename()] = f.getContent() # we add also user defined data allowing also the user to override tor2web defaults. userstaticdir = os.path.join(config['datadir'], "static/") if os.path.exists(userstaticdir): for root, dirs, files in os.walk(os.path.join(userstaticdir)): for basename in files: filename = os.path.join(root, basename) f = FilePath(filename) antanistaticmap[filename.replace(userstaticdir, "")] = f.getContent() ############################################################################### ############################################################################### # Templates loading # Here we make a templates caching to not handle I/O # at run-time and achieve better performance ############################################################################### templates = {} files = FilePath('/usr/share/tor2web/templates/').globChildren("*.tpl") for f in files: f = FilePath( t2w_file_path(config['datadir'], os.path.join('templates', f.basename()))) templates[f.basename()] = PageTemplate(XMLString(f.getContent())) ############################################################################### pool = HTTPConnectionPool(reactor, True, config['sockmaxpersistentperhost'], config['sockcachedconnectiontimeout'], config['sockretryautomatically']) factory = T2WProxyFactory() # we do not want all workers to die in the same moment requests_countdown = config['requests_per_process'] / random.randint(1, 3) factory = T2WLimitedRequestsFactory(factory, requests_countdown) context_factory = T2WSSLContextFactory( os.path.join(config['datadir'], "certs/tor2web-key.pem"), os.path.join(config['datadir'], "certs/tor2web-intermediate.pem"), os.path.join(config['datadir'], "certs/tor2web-dh.pem"), config['cipher_list']) if config['debugmode'] and config['debugtostdout']: log.startLogging(sys.stdout) else: log.startLogging(log.NullFile) fds_https = filter(None, args[0].split(",")) fds_https = [int(i) for i in fds_https] fds_http = filter(None, args[1].split(",")) fds_http = [int(i) for i in fds_http] reactor.listenTCPonExistingFD = listenTCPonExistingFD reactor.listenSSLonExistingFD = listenSSLonExistingFD for fd in fds_https: ports.append( reactor.listenSSLonExistingFD(reactor, fd=fd, factory=factory, contextFactory=context_factory)) for fd in fds_http: ports.append( reactor.listenTCPonExistingFD(reactor, fd=fd, factory=factory)) sys.excepthook = MailException
def start_worker(): global antanistaticmap global templates global pool global rexp global ports lc = LoopingCall(updateListsTask) lc.start(600) rexp = { 'head': re.compile(r'(<head.*?\s*>)', re.I), 'w2t': re.compile(r'(http.?:)?//([a-z0-9]{16}).' + config.basehost + '(?!:\d+)', re.I), 't2w': re.compile(r'(http.?:)?//([a-z0-9]{16}).(?!' + config.basehost + ')onion(?!:\d+)', re.I) } ############################################################################### # Static Data loading # Here we make a file caching to not handle I/O # at run-time and achieve better performance ############################################################################### antanistaticmap = {} # system default static files sys_static_dir = os.path.join(config.sysdatadir, "static/") if os.path.exists(sys_static_dir): for root, dirs, files in os.walk(os.path.join(sys_static_dir)): for basename in files: filename = os.path.join(root, basename) f = FilePath(filename) antanistaticmap[filename.replace(sys_static_dir, "")] = f.getContent() # user defined static files usr_static_dir = os.path.join(config.datadir, "static/") if usr_static_dir != sys_static_dir and os.path.exists(usr_static_dir): for root, dirs, files in os.walk(os.path.join(usr_static_dir)): for basename in files: filename = os.path.join(root, basename) f = FilePath(filename) antanistaticmap[filename.replace(usr_static_dir, "")] = f.getContent() ############################################################################### ############################################################################### # Templates loading # Here we make a templates cache in order to not handle I/O # at run-time and achieve better performance ############################################################################### templates = {} # system default templates sys_tpl_dir = os.path.join(config.sysdatadir, "templates/") if os.path.exists(sys_tpl_dir): files = FilePath(sys_tpl_dir).globChildren("*.tpl") for f in files: f = FilePath(config.t2w_file_path(os.path.join('templates', f.basename()))) templates[f.basename()] = PageTemplate(XMLString(f.getContent())) # user defined templates usr_tpl_dir = os.path.join(config.datadir, "templates/") if usr_tpl_dir != sys_tpl_dir and os.path.exists(usr_tpl_dir): files = FilePath(usr_tpl_dir).globChildren("*.tpl") for f in files: f = FilePath(config.t2w_file_path(os.path.join('templates', f.basename()))) templates[f.basename()] = PageTemplate(XMLString(f.getContent())) ############################################################################### pool = HTTPConnectionPool(reactor, True, config.sockmaxpersistentperhost, config.sockcachedconnectiontimeout, config.sockretryautomatically) factory = T2WProxyFactory() # we do not want all workers to die in the same moment requests_countdown = config.requests_per_process / random.randint(1, 3) factory = T2WLimitedRequestsFactory(factory, requests_countdown) context_factory = T2WSSLContextFactory(os.path.join(config.datadir, "certs/tor2web-key.pem"), os.path.join(config.datadir, "certs/tor2web-intermediate.pem"), os.path.join(config.datadir, "certs/tor2web-dh.pem"), config.cipher_list) fds_https = [] if 'T2W_FDS_HTTPS' in os.environ: fds_https = filter(None, os.environ['T2W_FDS_HTTPS'].split(",")) fds_https = [int(i) for i in fds_https] fds_http = [] if 'T2W_FDS_HTTP' in os.environ: fds_http = filter(None, os.environ['T2W_FDS_HTTP'].split(",")) fds_http = [int(i) for i in fds_http] reactor.listenTCPonExistingFD = listenTCPonExistingFD reactor.listenSSLonExistingFD = listenSSLonExistingFD for fd in fds_https: ports.append(reactor.listenSSLonExistingFD(reactor, fd=fd, factory=factory, contextFactory=context_factory)) for fd in fds_http: ports.append(reactor.listenTCPonExistingFD(reactor, fd=fd, factory=factory)) def MailException(etype, value, tb): sendexceptionmail(config, etype, value, tb) sys.excepthook = MailException