def start_worker():
global antanistaticmap
global templates
global pool
global rexp
global ports
lc = LoopingCall(updateListsTask)
lc.start(600)
rexp = {
'body': re.compile(r'(<body.*?\s*>)', re.I),
'w2t': re.compile(r'(http.?:)?//([a-z0-9]{16}).' + config.basehost + '(?!:\d+)', re.I),
't2w': re.compile(r'(http.?:)?//([a-z0-9]{16}).(?!' + config.basehost + ')onion(?!:\d+)', re.I)
}
###############################################################################
# Static Data loading
# Here we make a file caching to not handle I/O
# at run-time and achieve better performance
###############################################################################
antanistaticmap = {}
# system default static files
sys_static_dir = os.path.join(config.sysdatadir, "static/")
if os.path.exists(sys_static_dir):
for root, dirs, files in os.walk(os.path.join(sys_static_dir)):
for basename in files:
filename = os.path.join(root, basename)
f = FilePath(filename)
antanistaticmap[filename.replace(sys_static_dir, "")] = f.getContent()
# user defined static files
usr_static_dir = os.path.join(config.datadir, "static/")
if usr_static_dir != sys_static_dir and os.path.exists(usr_static_dir):
for root, dirs, files in os.walk(os.path.join(usr_static_dir)):
for basename in files:
filename = os.path.join(root, basename)
f = FilePath(filename)
antanistaticmap[filename.replace(usr_static_dir, "")] = f.getContent()
###############################################################################
###############################################################################
# Templates loading
# Here we make a templates cache in order to not handle I/O
# at run-time and achieve better performance
###############################################################################
templates = {}
# system default templates
sys_tpl_dir = os.path.join(config.sysdatadir, "templates/")
if os.path.exists(sys_tpl_dir):
files = FilePath(sys_tpl_dir).globChildren("*.tpl")
for f in files:
f = FilePath(config.t2w_file_path(os.path.join('templates', f.basename())))
templates[f.basename()] = PageTemplate(XMLString(f.getContent()))
# user defined templates
usr_tpl_dir = os.path.join(config.datadir, "templates/")
if usr_tpl_dir != sys_tpl_dir and os.path.exists(usr_tpl_dir):
files = FilePath(usr_tpl_dir).globChildren("*.tpl")
for f in files:
f = FilePath(config.t2w_file_path(os.path.join('templates', f.basename())))
templates[f.basename()] = PageTemplate(XMLString(f.getContent()))
###############################################################################
pool = HTTPConnectionPool(reactor, True,
config.sockmaxpersistentperhost,
config.sockcachedconnectiontimeout,
config.sockretryautomatically)
factory = T2WProxyFactory()
# we do not want all workers to die in the same moment
requests_countdown = config.requests_per_process / random.randint(1, 3)
factory = T2WLimitedRequestsFactory(factory, requests_countdown)
context_factory = T2WSSLContextFactory(os.path.join(config.datadir, "certs/tor2web-key.pem"),
os.path.join(config.datadir, "certs/tor2web-intermediate.pem"),
os.path.join(config.datadir, "certs/tor2web-dh.pem"),
config.cipher_list)
fds_https = []
if 'T2W_FDS_HTTPS' in os.environ:
fds_https = filter(None, os.environ['T2W_FDS_HTTPS'].split(","))
fds_https = [int(i) for i in fds_https]
fds_http = []
if 'T2W_FDS_HTTP' in os.environ:
fds_http = filter(None, os.environ['T2W_FDS_HTTP'].split(","))
fds_http = [int(i) for i in fds_http]
reactor.listenTCPonExistingFD = listenTCPonExistingFD
reactor.listenSSLonExistingFD = listenSSLonExistingFD
for fd in fds_https:
ports.append(reactor.listenSSLonExistingFD(reactor,
fd=fd,
factory=factory,
contextFactory=context_factory))
for fd in fds_http:
ports.append(reactor.listenTCPonExistingFD(reactor,
fd=fd,
factory=factory))
def MailException(etype, value, tb):
sendexceptionmail(config, etype, value, tb)
sys.excepthook = MailException
def start():
global config
global antanistaticmap
global templates
global pool
global rexp
global fds_https
global fds_http
global ports
global requests_countdown
config = yield rpc("get_config")
rexp = {
'body': re.compile(r'(<body.*?\s*>)', re.I),
'w2t': re.compile(r'(https:)?//([a-z0-9]{16}).' + config['basehost'] + '(:443)?', re.I),
't2w': re.compile(r'(http:)?//([a-z0-9]{16}).onion(:80)?', re.I)
}
###############################################################################
# Templates loading
###############################################################################
antanistaticmap = {}
files = FilePath('/usr/share/tor2web/static/').globChildren("*")
for file in files:
file = FilePath(t2w_file_path(config['datadir'], os.path.join('static', file.basename())))
antanistaticmap[file.basename()] = file.getContent()
# we add additional files eventually written in datadir/static
# and not already loaded by previos lines.
if os.path.exists(os.path.join(config['datadir'], "static/")):
for file in files:
if file.basename() not in antanistaticmap:
antanistaticmap[file.basename()] = file.getContent()
###############################################################################
###############################################################################
# Templates loading
###############################################################################
templates = {}
files = FilePath('/usr/share/tor2web/templates/').globChildren("*.tpl")
for file in files:
file = FilePath(t2w_file_path(config['datadir'], os.path.join('templates', file.basename())))
templates[file.basename()] = PageTemplate(XMLString(file.getContent()))
###############################################################################
pool = HTTPConnectionPool(reactor, True,
config['sockmaxpersistentperhost'],
config['sockcachedconnectiontimeout'],
config['sockretryautomatically'])
factory = T2WProxyFactory()
factory.requestCountdown = config['requests_per_process']
context_factory = T2WSSLContextFactory(os.path.join(config['datadir'], "certs/tor2web-key.pem"),
os.path.join(config['datadir'], "certs/tor2web-intermediate.pem"),
os.path.join(config['datadir'], "certs/tor2web-dh.pem"),
config['cipher_list'])
if config['debugmode']:
if config['debugtostdout']:
log.startLogging(sys.stdout)
else:
log.startLogging(log.NullFile)
fds_https = filter(None, args[0].split(","))
fds_https = [int(i) for i in fds_https]
fds_http = filter(None, args[1].split(","))
fds_http = [int(i) for i in fds_http]
reactor.listenTCPonExistingFD = listenTCPonExistingFD
reactor.listenSSLonExistingFD = listenSSLonExistingFD
for fd in fds_https:
ports.append(reactor.listenSSLonExistingFD(reactor,
fd=fd,
factory=factory,
contextFactory=context_factory))
for fd in fds_http:
ports.append(reactor.listenTCPonExistingFD(reactor,
fd=fd,
factory=factory))
# we do not want all workers to die in the same moment
requests_countdown = config['requests_per_process'] / random.randint(3, 5)
sys.excepthook = MailException
def start():
global config
global antanistaticmap
global templates
global pool
global rexp
global fds_https
global fds_http
global ports
config = yield rpc("get_config")
lc = LoopingCall(updateTask)
lc.start(600)
rexp = {
'body': re.compile(r'(<body.*?\s*>)', re.I),
'w2t': re.compile(r'(http.?:)?//([a-z0-9]{16}).' + config['basehost'] + '(?!:\d+)', re.I),
't2w': re.compile(r'(http.?:)?//([a-z0-9]{16}).(?!' + config['basehost'] + ')onion(?!:\d+)', re.I)
}
###############################################################################
# Static Data loading
# Here we make a file caching to not handle I/O
# at run-time and achieve better performance
###############################################################################
antanistaticmap = {}
files = FilePath(static_path).globChildren("*")
for f in files:
f = FilePath(t2w_file_path(config['datadir'], os.path.join('static', f.basename())))
antanistaticmap[f.basename()] = f.getContent()
# we add also user defined data allowing also the user to override tor2web defaults.
userstaticdir = os.path.join(config['datadir'], "static/")
if os.path.exists(userstaticdir):
for root, dirs, files in os.walk(os.path.join(userstaticdir)):
for basename in files:
filename = os.path.join(root, basename)
f = FilePath(filename)
antanistaticmap[filename.replace(userstaticdir, "")] = f.getContent()
###############################################################################
###############################################################################
# Templates loading
# Here we make a templates caching to not handle I/O
# at run-time and achieve better performance
###############################################################################
templates = {}
files = FilePath(templates_path).globChildren("*.tpl")
for f in files:
f = FilePath(t2w_file_path(config['datadir'], os.path.join('templates', f.basename())))
templates[f.basename()] = PageTemplate(XMLString(f.getContent()))
###############################################################################
pool = HTTPConnectionPool(reactor, True,
config['sockmaxpersistentperhost'],
config['sockcachedconnectiontimeout'],
config['sockretryautomatically'])
factory = T2WProxyFactory()
# we do not want all workers to die in the same moment
requests_countdown = config['requests_per_process'] / random.randint(1, 3)
factory = T2WLimitedRequestsFactory(factory, requests_countdown)
context_factory = T2WSSLContextFactory(os.path.join(config['datadir'], "certs/tor2web-key.pem"),
os.path.join(config['datadir'], "certs/tor2web-intermediate.pem"),
os.path.join(config['datadir'], "certs/tor2web-dh.pem"),
config['cipher_list'])
if config['debugmode'] and config['debugtostdout']:
log.startLogging(sys.stdout)
else:
log.startLogging(log.NullFile)
fds_https = filter(None, args[0].split(","))
fds_https = [int(i) for i in fds_https]
fds_http = filter(None, args[1].split(","))
fds_http = [int(i) for i in fds_http]
reactor.listenTCPonExistingFD = listenTCPonExistingFD
reactor.listenSSLonExistingFD = listenSSLonExistingFD
for fd in fds_https:
ports.append(reactor.listenSSLonExistingFD(reactor,
fd=fd,
factory=factory,
contextFactory=context_factory))
for fd in fds_http:
ports.append(reactor.listenTCPonExistingFD(reactor,
fd=fd,
factory=factory))
sys.excepthook = MailException
def start():
global config
global antanistaticmap
global templates
global pool
global rexp
global fds_https
global fds_http
global ports
config = yield rpc("get_config")
lc = LoopingCall(updateTask)
lc.start(600)
rexp = {
'body':
re.compile(r'(<body.*?\s*>)', re.I),
'w2t':
re.compile(
r'(https:)?//([a-z0-9]{16}).' + config['basehost'] + '(:443)?',
re.I),
't2w':
re.compile(r'(http:)?//([a-z0-9]{16}).onion(:80)?', re.I)
}
###############################################################################
# Templates loading
###############################################################################
antanistaticmap = {}
files = FilePath('/usr/share/tor2web/static/').globChildren("*")
for file in files:
file = FilePath(
t2w_file_path(config['datadir'],
os.path.join('static', file.basename())))
antanistaticmap[file.basename()] = file.getContent()
# we add additional files eventually written in datadir/static
# and not already loaded by previos lines.
if os.path.exists(os.path.join(config['datadir'], "static/")):
for file in files:
if file.basename() not in antanistaticmap:
antanistaticmap[file.basename()] = file.getContent()
###############################################################################
###############################################################################
# Templates loading
###############################################################################
templates = {}
files = FilePath('/usr/share/tor2web/templates/').globChildren("*.tpl")
for file in files:
file = FilePath(
t2w_file_path(config['datadir'],
os.path.join('templates', file.basename())))
templates[file.basename()] = PageTemplate(XMLString(file.getContent()))
###############################################################################
pool = HTTPConnectionPool(reactor, True,
config['sockmaxpersistentperhost'],
config['sockcachedconnectiontimeout'],
config['sockretryautomatically'])
factory = T2WProxyFactory()
# we do not want all workers to die in the same moment
requests_countdown = config['requests_per_process'] / random.randint(1, 3)
factory = T2WLimitedRequestsFactory(factory, requests_countdown)
context_factory = T2WSSLContextFactory(
os.path.join(config['datadir'], "certs/tor2web-key.pem"),
os.path.join(config['datadir'], "certs/tor2web-intermediate.pem"),
os.path.join(config['datadir'], "certs/tor2web-dh.pem"),
config['cipher_list'])
if config['debugmode'] and config['debugtostdout']:
log.startLogging(sys.stdout)
else:
log.startLogging(log.NullFile)
fds_https = filter(None, args[0].split(","))
fds_https = [int(i) for i in fds_https]
fds_http = filter(None, args[1].split(","))
fds_http = [int(i) for i in fds_http]
reactor.listenTCPonExistingFD = listenTCPonExistingFD
reactor.listenSSLonExistingFD = listenSSLonExistingFD
for fd in fds_https:
ports.append(
reactor.listenSSLonExistingFD(reactor,
fd=fd,
factory=factory,
contextFactory=context_factory))
for fd in fds_http:
ports.append(
reactor.listenTCPonExistingFD(reactor, fd=fd, factory=factory))
sys.excepthook = MailException
def start():
global config
global antanistaticmap
global templates
global pool
global rexp
global fds_https
global fds_http
global ports
config = yield rpc("get_config")
lc = LoopingCall(updateTask)
lc.start(600)
rexp = {
'body':
re.compile(r'(<body.*?\s*>)', re.I),
'w2t':
re.compile(
r'(http.?:)?//([a-z0-9]{16}).' + config['basehost'] + '(?!:\d+)',
re.I),
't2w':
re.compile(
r'(http.?:)?//([a-z0-9]{16}).(?!' + config['basehost'] +
')onion(?!:\d+)', re.I)
}
###############################################################################
# Static Data loading
# Here we make a file caching to not handle I/O
# at run-time and achieve better performance
###############################################################################
antanistaticmap = {}
files = FilePath('/usr/share/tor2web/static/').globChildren("*")
for f in files:
f = FilePath(
t2w_file_path(config['datadir'],
os.path.join('static', f.basename())))
antanistaticmap[f.basename()] = f.getContent()
# we add also user defined data allowing also the user to override tor2web defaults.
userstaticdir = os.path.join(config['datadir'], "static/")
if os.path.exists(userstaticdir):
for root, dirs, files in os.walk(os.path.join(userstaticdir)):
for basename in files:
filename = os.path.join(root, basename)
f = FilePath(filename)
antanistaticmap[filename.replace(userstaticdir,
"")] = f.getContent()
###############################################################################
###############################################################################
# Templates loading
# Here we make a templates caching to not handle I/O
# at run-time and achieve better performance
###############################################################################
templates = {}
files = FilePath('/usr/share/tor2web/templates/').globChildren("*.tpl")
for f in files:
f = FilePath(
t2w_file_path(config['datadir'],
os.path.join('templates', f.basename())))
templates[f.basename()] = PageTemplate(XMLString(f.getContent()))
###############################################################################
pool = HTTPConnectionPool(reactor, True,
config['sockmaxpersistentperhost'],
config['sockcachedconnectiontimeout'],
config['sockretryautomatically'])
factory = T2WProxyFactory()
# we do not want all workers to die in the same moment
requests_countdown = config['requests_per_process'] / random.randint(1, 3)
factory = T2WLimitedRequestsFactory(factory, requests_countdown)
context_factory = T2WSSLContextFactory(
os.path.join(config['datadir'], "certs/tor2web-key.pem"),
os.path.join(config['datadir'], "certs/tor2web-intermediate.pem"),
os.path.join(config['datadir'], "certs/tor2web-dh.pem"),
config['cipher_list'])
if config['debugmode'] and config['debugtostdout']:
log.startLogging(sys.stdout)
else:
log.startLogging(log.NullFile)
fds_https = filter(None, args[0].split(","))
fds_https = [int(i) for i in fds_https]
fds_http = filter(None, args[1].split(","))
fds_http = [int(i) for i in fds_http]
reactor.listenTCPonExistingFD = listenTCPonExistingFD
reactor.listenSSLonExistingFD = listenSSLonExistingFD
for fd in fds_https:
ports.append(
reactor.listenSSLonExistingFD(reactor,
fd=fd,
factory=factory,
contextFactory=context_factory))
for fd in fds_http:
ports.append(
reactor.listenTCPonExistingFD(reactor, fd=fd, factory=factory))
sys.excepthook = MailException
def start_worker():
global antanistaticmap
global templates
global pool
global rexp
global ports
lc = LoopingCall(updateListsTask)
lc.start(600)
rexp = {
'head': re.compile(r'(<head.*?\s*>)', re.I),
'w2t': re.compile(r'(http.?:)?//([a-z0-9]{16}).' + config.basehost + '(?!:\d+)', re.I),
't2w': re.compile(r'(http.?:)?//([a-z0-9]{16}).(?!' + config.basehost + ')onion(?!:\d+)', re.I)
}
###############################################################################
# Static Data loading
# Here we make a file caching to not handle I/O
# at run-time and achieve better performance
###############################################################################
antanistaticmap = {}
# system default static files
sys_static_dir = os.path.join(config.sysdatadir, "static/")
if os.path.exists(sys_static_dir):
for root, dirs, files in os.walk(os.path.join(sys_static_dir)):
for basename in files:
filename = os.path.join(root, basename)
f = FilePath(filename)
antanistaticmap[filename.replace(sys_static_dir, "")] = f.getContent()
# user defined static files
usr_static_dir = os.path.join(config.datadir, "static/")
if usr_static_dir != sys_static_dir and os.path.exists(usr_static_dir):
for root, dirs, files in os.walk(os.path.join(usr_static_dir)):
for basename in files:
filename = os.path.join(root, basename)
f = FilePath(filename)
antanistaticmap[filename.replace(usr_static_dir, "")] = f.getContent()
###############################################################################
###############################################################################
# Templates loading
# Here we make a templates cache in order to not handle I/O
# at run-time and achieve better performance
###############################################################################
templates = {}
# system default templates
sys_tpl_dir = os.path.join(config.sysdatadir, "templates/")
if os.path.exists(sys_tpl_dir):
files = FilePath(sys_tpl_dir).globChildren("*.tpl")
for f in files:
f = FilePath(config.t2w_file_path(os.path.join('templates', f.basename())))
templates[f.basename()] = PageTemplate(XMLString(f.getContent()))
# user defined templates
usr_tpl_dir = os.path.join(config.datadir, "templates/")
if usr_tpl_dir != sys_tpl_dir and os.path.exists(usr_tpl_dir):
files = FilePath(usr_tpl_dir).globChildren("*.tpl")
for f in files:
f = FilePath(config.t2w_file_path(os.path.join('templates', f.basename())))
templates[f.basename()] = PageTemplate(XMLString(f.getContent()))
###############################################################################
pool = HTTPConnectionPool(reactor, True,
config.sockmaxpersistentperhost,
config.sockcachedconnectiontimeout,
config.sockretryautomatically)
factory = T2WProxyFactory()
# we do not want all workers to die in the same moment
requests_countdown = config.requests_per_process / random.randint(1, 3)
factory = T2WLimitedRequestsFactory(factory, requests_countdown)
context_factory = T2WSSLContextFactory(os.path.join(config.datadir, "certs/tor2web-key.pem"),
os.path.join(config.datadir, "certs/tor2web-intermediate.pem"),
os.path.join(config.datadir, "certs/tor2web-dh.pem"),
config.cipher_list)
fds_https = []
if 'T2W_FDS_HTTPS' in os.environ:
fds_https = filter(None, os.environ['T2W_FDS_HTTPS'].split(","))
fds_https = [int(i) for i in fds_https]
fds_http = []
if 'T2W_FDS_HTTP' in os.environ:
fds_http = filter(None, os.environ['T2W_FDS_HTTP'].split(","))
fds_http = [int(i) for i in fds_http]
reactor.listenTCPonExistingFD = listenTCPonExistingFD
reactor.listenSSLonExistingFD = listenSSLonExistingFD
for fd in fds_https:
ports.append(reactor.listenSSLonExistingFD(reactor,
fd=fd,
factory=factory,
contextFactory=context_factory))
for fd in fds_http:
ports.append(reactor.listenTCPonExistingFD(reactor,
fd=fd,
factory=factory))
def MailException(etype, value, tb):
sendexceptionmail(config, etype, value, tb)
sys.excepthook = MailException
