def load_lists(self):
self.access_list = []
if config.mode == "TRANSLATION":
pass
elif config.mode == "WHITELIST":
self.access_list = List(t2w_file_path(config.datadir, 'lists/whitelist.txt'))
elif config.mode == "BLACKLIST":
self.access_list = List(t2w_file_path(config.datadir, 'lists/blocklist_hashed.txt'),
config.automatic_blocklist_updates_source,
config.automatic_blocklist_updates_refresh)
# clear local cleartext list
# (load -> hash -> clear feature; for security reasons)
self.blocklist_cleartext = List(t2w_file_path(config.datadir, 'lists/blocklist_cleartext.txt'))
for i in self.blocklist_cleartext:
self.access_list.add(hashlib.md5(i).hexdigest())
self.access_list.dump()
self.blocklist_cleartext.clear()
self.blocklist_cleartext.dump()
self.blocked_ua = []
if config.blockcrawl:
tmp = List(t2w_file_path(config.datadir, 'lists/blocked_ua.txt'))
for ua in tmp:
self.blocked_ua.append(ua.lower())
# Load Exit Nodes list with the refresh rate configured in config file
self.TorExitNodes = TorExitNodeList(os.path.join(config.datadir, 'lists', 'exitnodelist.txt'),
"https://onionoo.torproject.org/summary?type=relay",
config.exit_node_list_refresh)
def load_lists(self):
self.access_list = []
if config.mode == "TRANSLATION":
pass
elif config.mode == "WHITELIST":
self.access_list = List(
t2w_file_path(config.datadir, 'lists/whitelist.txt'))
elif config.mode == "BLACKLIST":
self.access_list = List(
t2w_file_path(config.datadir, 'lists/blocklist_hashed.txt'),
config.automatic_blocklist_updates_source,
config.automatic_blocklist_updates_refresh)
# clear local cleartext list
# (load -> hash -> clear feature; for security reasons)
self.blocklist_cleartext = List(
t2w_file_path(config.datadir, 'lists/blocklist_cleartext.txt'))
for i in self.blocklist_cleartext:
self.access_list.add(hashlib.md5(i).hexdigest())
self.access_list.dump()
self.blocklist_cleartext.clear()
self.blocklist_cleartext.dump()
self.blocked_ua = []
if config.blockcrawl:
tmp = List(t2w_file_path(config.datadir, 'lists/blocked_ua.txt'))
for ua in tmp:
self.blocked_ua.append(ua.lower())
# Load Exit Nodes list with the refresh rate configured in config file
self.TorExitNodes = TorExitNodeList(
os.path.join(config.datadir, 'lists', 'exitnodelist.txt'),
"https://onionoo.torproject.org/summary?type=relay",
config.exit_node_list_refresh)
def start():
global config
global antanistaticmap
global templates
global pool
global rexp
global fds_https
global fds_http
global ports
global requests_countdown
config = yield rpc("get_config")
rexp = {
'body': re.compile(r'(<body.*?\s*>)', re.I),
'w2t': re.compile(r'(https:)?//([a-z0-9]{16}).' + config['basehost'] + '(:443)?', re.I),
't2w': re.compile(r'(http:)?//([a-z0-9]{16}).onion(:80)?', re.I)
}
###############################################################################
# Templates loading
###############################################################################
antanistaticmap = {}
files = FilePath('/usr/share/tor2web/static/').globChildren("*")
for file in files:
file = FilePath(t2w_file_path(config['datadir'], os.path.join('static', file.basename())))
antanistaticmap[file.basename()] = file.getContent()
# we add additional files eventually written in datadir/static
# and not already loaded by previos lines.
if os.path.exists(os.path.join(config['datadir'], "static/")):
for file in files:
if file.basename() not in antanistaticmap:
antanistaticmap[file.basename()] = file.getContent()
###############################################################################
###############################################################################
# Templates loading
###############################################################################
templates = {}
files = FilePath('/usr/share/tor2web/templates/').globChildren("*.tpl")
for file in files:
file = FilePath(t2w_file_path(config['datadir'], os.path.join('templates', file.basename())))
templates[file.basename()] = PageTemplate(XMLString(file.getContent()))
###############################################################################
pool = HTTPConnectionPool(reactor, True,
config['sockmaxpersistentperhost'],
config['sockcachedconnectiontimeout'],
config['sockretryautomatically'])
factory = T2WProxyFactory()
factory.requestCountdown = config['requests_per_process']
context_factory = T2WSSLContextFactory(os.path.join(config['datadir'], "certs/tor2web-key.pem"),
os.path.join(config['datadir'], "certs/tor2web-intermediate.pem"),
os.path.join(config['datadir'], "certs/tor2web-dh.pem"),
config['cipher_list'])
if config['debugmode']:
if config['debugtostdout']:
log.startLogging(sys.stdout)
else:
log.startLogging(log.NullFile)
fds_https = filter(None, args[0].split(","))
fds_https = [int(i) for i in fds_https]
fds_http = filter(None, args[1].split(","))
fds_http = [int(i) for i in fds_http]
reactor.listenTCPonExistingFD = listenTCPonExistingFD
reactor.listenSSLonExistingFD = listenSSLonExistingFD
for fd in fds_https:
ports.append(reactor.listenSSLonExistingFD(reactor,
fd=fd,
factory=factory,
contextFactory=context_factory))
for fd in fds_http:
ports.append(reactor.listenTCPonExistingFD(reactor,
fd=fd,
factory=factory))
# we do not want all workers to die in the same moment
requests_countdown = config['requests_per_process'] / random.randint(3, 5)
sys.excepthook = MailException
def start():
global config
global antanistaticmap
global templates
global pool
global rexp
global fds_https
global fds_http
global ports
config = yield rpc("get_config")
lc = LoopingCall(updateTask)
lc.start(600)
rexp = {
'body': re.compile(r'(<body.*?\s*>)', re.I),
'w2t': re.compile(r'(http.?:)?//([a-z0-9]{16}).' + config['basehost'] + '(?!:\d+)', re.I),
't2w': re.compile(r'(http.?:)?//([a-z0-9]{16}).(?!' + config['basehost'] + ')onion(?!:\d+)', re.I)
}
###############################################################################
# Static Data loading
# Here we make a file caching to not handle I/O
# at run-time and achieve better performance
###############################################################################
antanistaticmap = {}
files = FilePath(static_path).globChildren("*")
for f in files:
f = FilePath(t2w_file_path(config['datadir'], os.path.join('static', f.basename())))
antanistaticmap[f.basename()] = f.getContent()
# we add also user defined data allowing also the user to override tor2web defaults.
userstaticdir = os.path.join(config['datadir'], "static/")
if os.path.exists(userstaticdir):
for root, dirs, files in os.walk(os.path.join(userstaticdir)):
for basename in files:
filename = os.path.join(root, basename)
f = FilePath(filename)
antanistaticmap[filename.replace(userstaticdir, "")] = f.getContent()
###############################################################################
###############################################################################
# Templates loading
# Here we make a templates caching to not handle I/O
# at run-time and achieve better performance
###############################################################################
templates = {}
files = FilePath(templates_path).globChildren("*.tpl")
for f in files:
f = FilePath(t2w_file_path(config['datadir'], os.path.join('templates', f.basename())))
templates[f.basename()] = PageTemplate(XMLString(f.getContent()))
###############################################################################
pool = HTTPConnectionPool(reactor, True,
config['sockmaxpersistentperhost'],
config['sockcachedconnectiontimeout'],
config['sockretryautomatically'])
factory = T2WProxyFactory()
# we do not want all workers to die in the same moment
requests_countdown = config['requests_per_process'] / random.randint(1, 3)
factory = T2WLimitedRequestsFactory(factory, requests_countdown)
context_factory = T2WSSLContextFactory(os.path.join(config['datadir'], "certs/tor2web-key.pem"),
os.path.join(config['datadir'], "certs/tor2web-intermediate.pem"),
os.path.join(config['datadir'], "certs/tor2web-dh.pem"),
config['cipher_list'])
if config['debugmode'] and config['debugtostdout']:
log.startLogging(sys.stdout)
else:
log.startLogging(log.NullFile)
fds_https = filter(None, args[0].split(","))
fds_https = [int(i) for i in fds_https]
fds_http = filter(None, args[1].split(","))
fds_http = [int(i) for i in fds_http]
reactor.listenTCPonExistingFD = listenTCPonExistingFD
reactor.listenSSLonExistingFD = listenSSLonExistingFD
for fd in fds_https:
ports.append(reactor.listenSSLonExistingFD(reactor,
fd=fd,
factory=factory,
contextFactory=context_factory))
for fd in fds_http:
ports.append(reactor.listenTCPonExistingFD(reactor,
fd=fd,
factory=factory))
sys.excepthook = MailException
def start():
global config
global antanistaticmap
global templates
global pool
global rexp
global fds_https
global fds_http
global ports
config = yield rpc("get_config")
lc = LoopingCall(updateTask)
lc.start(600)
rexp = {
'body':
re.compile(r'(<body.*?\s*>)', re.I),
'w2t':
re.compile(
r'(https:)?//([a-z0-9]{16}).' + config['basehost'] + '(:443)?',
re.I),
't2w':
re.compile(r'(http:)?//([a-z0-9]{16}).onion(:80)?', re.I)
}
###############################################################################
# Templates loading
###############################################################################
antanistaticmap = {}
files = FilePath('/usr/share/tor2web/static/').globChildren("*")
for file in files:
file = FilePath(
t2w_file_path(config['datadir'],
os.path.join('static', file.basename())))
antanistaticmap[file.basename()] = file.getContent()
# we add additional files eventually written in datadir/static
# and not already loaded by previos lines.
if os.path.exists(os.path.join(config['datadir'], "static/")):
for file in files:
if file.basename() not in antanistaticmap:
antanistaticmap[file.basename()] = file.getContent()
###############################################################################
###############################################################################
# Templates loading
###############################################################################
templates = {}
files = FilePath('/usr/share/tor2web/templates/').globChildren("*.tpl")
for file in files:
file = FilePath(
t2w_file_path(config['datadir'],
os.path.join('templates', file.basename())))
templates[file.basename()] = PageTemplate(XMLString(file.getContent()))
###############################################################################
pool = HTTPConnectionPool(reactor, True,
config['sockmaxpersistentperhost'],
config['sockcachedconnectiontimeout'],
config['sockretryautomatically'])
factory = T2WProxyFactory()
# we do not want all workers to die in the same moment
requests_countdown = config['requests_per_process'] / random.randint(1, 3)
factory = T2WLimitedRequestsFactory(factory, requests_countdown)
context_factory = T2WSSLContextFactory(
os.path.join(config['datadir'], "certs/tor2web-key.pem"),
os.path.join(config['datadir'], "certs/tor2web-intermediate.pem"),
os.path.join(config['datadir'], "certs/tor2web-dh.pem"),
config['cipher_list'])
if config['debugmode'] and config['debugtostdout']:
log.startLogging(sys.stdout)
else:
log.startLogging(log.NullFile)
fds_https = filter(None, args[0].split(","))
fds_https = [int(i) for i in fds_https]
fds_http = filter(None, args[1].split(","))
fds_http = [int(i) for i in fds_http]
reactor.listenTCPonExistingFD = listenTCPonExistingFD
reactor.listenSSLonExistingFD = listenSSLonExistingFD
for fd in fds_https:
ports.append(
reactor.listenSSLonExistingFD(reactor,
fd=fd,
factory=factory,
contextFactory=context_factory))
for fd in fds_http:
ports.append(
reactor.listenTCPonExistingFD(reactor, fd=fd, factory=factory))
sys.excepthook = MailException
def start():
global config
global antanistaticmap
global templates
global pool
global rexp
global fds_https
global fds_http
global ports
config = yield rpc("get_config")
lc = LoopingCall(updateTask)
lc.start(600)
rexp = {
'body':
re.compile(r'(<body.*?\s*>)', re.I),
'w2t':
re.compile(
r'(http.?:)?//([a-z0-9]{16}).' + config['basehost'] + '(?!:\d+)',
re.I),
't2w':
re.compile(
r'(http.?:)?//([a-z0-9]{16}).(?!' + config['basehost'] +
')onion(?!:\d+)', re.I)
}
###############################################################################
# Static Data loading
# Here we make a file caching to not handle I/O
# at run-time and achieve better performance
###############################################################################
antanistaticmap = {}
files = FilePath('/usr/share/tor2web/static/').globChildren("*")
for f in files:
f = FilePath(
t2w_file_path(config['datadir'],
os.path.join('static', f.basename())))
antanistaticmap[f.basename()] = f.getContent()
# we add also user defined data allowing also the user to override tor2web defaults.
userstaticdir = os.path.join(config['datadir'], "static/")
if os.path.exists(userstaticdir):
for root, dirs, files in os.walk(os.path.join(userstaticdir)):
for basename in files:
filename = os.path.join(root, basename)
f = FilePath(filename)
antanistaticmap[filename.replace(userstaticdir,
"")] = f.getContent()
###############################################################################
###############################################################################
# Templates loading
# Here we make a templates caching to not handle I/O
# at run-time and achieve better performance
###############################################################################
templates = {}
files = FilePath('/usr/share/tor2web/templates/').globChildren("*.tpl")
for f in files:
f = FilePath(
t2w_file_path(config['datadir'],
os.path.join('templates', f.basename())))
templates[f.basename()] = PageTemplate(XMLString(f.getContent()))
###############################################################################
pool = HTTPConnectionPool(reactor, True,
config['sockmaxpersistentperhost'],
config['sockcachedconnectiontimeout'],
config['sockretryautomatically'])
factory = T2WProxyFactory()
# we do not want all workers to die in the same moment
requests_countdown = config['requests_per_process'] / random.randint(1, 3)
factory = T2WLimitedRequestsFactory(factory, requests_countdown)
context_factory = T2WSSLContextFactory(
os.path.join(config['datadir'], "certs/tor2web-key.pem"),
os.path.join(config['datadir'], "certs/tor2web-intermediate.pem"),
os.path.join(config['datadir'], "certs/tor2web-dh.pem"),
config['cipher_list'])
if config['debugmode'] and config['debugtostdout']:
log.startLogging(sys.stdout)
else:
log.startLogging(log.NullFile)
fds_https = filter(None, args[0].split(","))
fds_https = [int(i) for i in fds_https]
fds_http = filter(None, args[1].split(","))
fds_http = [int(i) for i in fds_http]
reactor.listenTCPonExistingFD = listenTCPonExistingFD
reactor.listenSSLonExistingFD = listenSSLonExistingFD
for fd in fds_https:
ports.append(
reactor.listenSSLonExistingFD(reactor,
fd=fd,
factory=factory,
contextFactory=context_factory))
for fd in fds_http:
ports.append(
reactor.listenTCPonExistingFD(reactor, fd=fd, factory=factory))
sys.excepthook = MailException