Beispiel #1
0
    def __init__(self):
        self.settings = Settings

        self.tor_exit_set = TorExitSet()

        self.https_socks = []
        self.http_socks = []

        self.snimap = SNIMap()

        self.jobs = []
        self.jobs_monitor = None
        self.services = []
        self.onion_service_job = None

        self.exceptions = {}
        self.exceptions_email_count = 0
        self.stats_collection_start_time = datetime_now()

        self.accept_submissions = True

        self.tenant_state = {}
        self.tenant_cache = {}
        self.tenant_hostname_id_map = {}

        self.set_orm_tp(ThreadPool(4, 16))

        self.TempLogs = []
        self.TempKeys = TempDict(3600 * 72)
        self.TempUploadFiles = TempDict(3600)

        self.shutdown = False
Beispiel #2
0
    def __init__(self):
        from globaleaks.settings import Settings

        self.settings = Settings

        self.process_supervisor = None
        self.tor_exit_set = TorExitSet()

        self.https_socks = []
        self.http_socks = []

        self.jobs = []
        self.jobs_monitor = None
        self.services = []
        self.onion_service_job = None

        self.api_token_session = None

        self.exceptions = {}
        self.exceptions_email_count = 0
        self.mail_counters = {}
        self.stats_collection_start_time = datetime_now()

        self.accept_submissions = True

        self.tenant_state = {}
        self.tenant_cache = {}
        self.tenant_hostname_id_map = {}

        self.set_orm_tp(ThreadPool(4, 16))
        self.TempUploadFiles = TempDict(timeout=3600)
Beispiel #3
0
    def __init__(self):
        self.settings = Settings

        self.process_supervisor = None
        self.tor_exit_set = TorExitSet()

        self.https_socks = []
        self.http_socks = []

        self.jobs = []
        self.jobs_monitor = None
        self.services = []
        self.onion_service_job = None

        self.api_token_session = None

        self.exceptions = {}
        self.exceptions_email_count = 0
        self.mail_counters = {}
        self.stats_collection_start_time = datetime_now()

        self.accept_submissions = True

        self.tenant_state = {}
        self.tenant_cache = {}
        self.tenant_hostname_id_map = {}

        self.set_orm_tp(ThreadPool(4, 16))
        self.TempUploadFiles = TempDict(timeout=3600)

        self.shutdown = False
Beispiel #4
0
    def __init__(self, state):
        self.RecentEventQ = []
        self.EventQ = []
        self.AnomaliesQ = []

        # An ACME challenge will have 5 minutes to resolve
        self.acme_tmp_chall_dict = TempDict(300)

        self.Alarm = getAlarm(state)
Beispiel #5
0
    def test_timeout(self):
        timeout = 1337

        xxx = TempDict(timeout=timeout)

        xxx.expireCallback = expireCallback

        for x in range(1, timeout + 1):
            xxx.set(x, TestObject(x))
            self.assertEqual(len(xxx), x)
            self.test_reactor.advance(1)

        for x in range(1, timeout + 1):
            self.assertEqual(len(xxx), timeout - x)
            self.test_reactor.advance(1)

        self.assertEqual(len(xxx), 0)

        self.assertEqual(TestObject.callbacks_count, timeout)
    def test_timeout(self):
        timeout = 1337
        size_limit = 1000000

        xxx = TempDict(timeout=timeout, size_limit=size_limit)

        xxx.expireCallback = expireCallback

        for x in range(1, timeout + 1):
            xxx.set(x, TestObject(x))
            self.assertEqual(len(xxx), x)
            self.test_reactor.advance(1)

        for x in range(1, timeout + 1):
            self.assertEqual(len(xxx), timeout - x)
            self.test_reactor.advance(1)

        self.assertEqual(len(xxx), 0)

        self.assertEqual(TestObject.callbacks_count, timeout)
Beispiel #7
0
    def test_size_limit(self):
        timeout = 666
        size_limit = 666

        xxx = TempDict(timeout=timeout, size_limit=size_limit)

        for x in range(1, size_limit * 2):
            xxx.set(x, TestObject(x))
            if x < size_limit:
                self.assertEqual(len(xxx), x)
                self.assertEqual(xxx.get(x).id, x)
                self.assertEqual(xxx.get(x + 1), None)
            else:
                self.assertEqual(len(xxx), size_limit)
                self.assertEqual(
                    xxx.get(x - size_limit + 1).id, x - size_limit + 1)
                self.assertEqual(xxx.get(x - size_limit), None)
Beispiel #8
0
    def test_timeout(self):
        timeout = 1337
        size_limit = 1000000

        c = task.Clock() # deterministic clock

        xxx = TempDict(timeout=timeout, size_limit=size_limit)

        xxx.expireCallback = expireCallback

        xxx.reactor = c

        for x in range(1, timeout + 1):
            xxx.set(x, TestObject(x))
            self.assertEqual(len(xxx), x)
            c.advance(1)

        for x in range(1, timeout + 1):
            self.assertEqual(len(xxx), timeout - x)
            c.advance(1)

        self.assertEqual(len(xxx), 0)

        self.assertEqual(TestObject.callbacks_count, timeout)
    def test_size_limit(self):
        timeout = 666
        size_limit = 666

        xxx = TempDict(timeout=timeout, size_limit=size_limit)

        for x in range(1, size_limit * 2):
            xxx.set(x, TestObject(x))
            if x < size_limit:
                self.assertEqual(len(xxx), x)
                self.assertEqual(xxx.get(x).id, x)
                self.assertEqual(xxx.get(x + 1), None)
            else:
                self.assertEqual(len(xxx), size_limit)
                self.assertEqual(xxx.get(x - size_limit + 1).id, x - size_limit + 1)
                self.assertEqual(xxx.get(x - size_limit), None)
Beispiel #10
0
    def test_timeout(self):
        timeout = 1337
        size_limit = 1000000

        c = task.Clock()  # deterministic clock

        xxx = TempDict(timeout=timeout, size_limit=size_limit)

        xxx.expireCallback = expireCallback

        xxx.reactor = c

        for x in range(1, timeout + 1):
            xxx.set(x, TestObject(x))
            self.assertEqual(len(xxx), x)
            c.advance(1)

        for x in range(1, timeout + 1):
            self.assertEqual(len(xxx), timeout - x)
            c.advance(1)

        self.assertEqual(len(xxx), 0)

        self.assertEqual(TestObject.callbacks_count, timeout)
Beispiel #11
0
from cyclone import httputil, web, template
from cyclone.escape import native_str
from cyclone.httpserver import HTTPConnection, HTTPRequest, _BadRequestException
from cyclone.web import RequestHandler, HTTPError, HTTPAuthenticationRequired, RedirectHandler
from globaleaks.event import track_handler
from globaleaks.rest import errors, requests
from globaleaks.security import GLSecureTemporaryFile, directory_traversal_check, generateRandomKey
from globaleaks.settings import GLSettings
from globaleaks.utils.mailutils import mail_exception_handler, send_exception_email
from globaleaks.utils.tempdict import TempDict
from globaleaks.utils.utility import log, datetime_now, deferred_sleep

HANDLER_EXEC_TIME_THRESHOLD = 30

GLUploads = {}
GLSessions = TempDict(timeout=GLSettings.authentication_lifetime)

# https://github.com/globaleaks/GlobaLeaks/issues/1601
mimetypes.add_type('image/svg+xml', '.svg')
mimetypes.add_type('application/vnd.ms-fontobject', '.eot')
mimetypes.add_type('application/x-font-ttf', '.ttf')
mimetypes.add_type('application/woff', '.woff')
mimetypes.add_type('application/woff2', '.woff2')

class GLSession(object):
    def __init__(self, user_id, user_role, user_status):
        self.id = generateRandomKey(42)
        self.user_id = user_id
        self.user_role = user_role
        self.user_status = user_status
Beispiel #12
0
 def __init__(self, state, file_path, *args, **kwds):
     self.state = state
     self.file_path = file_path
     TempDict.__init__(self, *args, **kwds)
Beispiel #13
0
    def get(self, key):
        ret = TempDict.get(self, key)
        if ret is None:
            raise Exception('TokenFailure: Invalid token')

        return ret
Beispiel #14
0
 def __init__(self, file_path, *args, **kwds):
     self.file_path = file_path
     TempDict.__init__(self, *args, **kwds)
Beispiel #15
0
    def get(self, key):
        ret = TempDict.get(self, key)
        if ret is None:
            raise errors.TokenFailure("Not found")

        return ret
Beispiel #16
0
 def __init__(self, *args, **kwds):
     TempDict.__init__(self, *args, **kwds)
Beispiel #17
0
from twisted.internet.error import ConnectError
from twisted.web.client import Agent, readBody

from globaleaks.orm import transact, transact_sync
from globaleaks.settings import GLSettings
from globaleaks.handlers.base import BaseHandler, HANDLER_EXEC_TIME_THRESHOLD
from globaleaks.models.config import PrivateFactory, NodeFactory, load_tls_dict
from globaleaks.rest import errors, requests
from globaleaks.utils import tls, agent
from globaleaks.utils import letsencrypt
from globaleaks.utils.utility import datetime_to_ISO8601, format_cert_expr_date, log
from globaleaks.utils.tempdict import TempDict


# Access auth tokens expire after a 5 minutes
tmp_chall_dict = TempDict(300)


class FileResource(object):
    """
    An interface for interacting with files stored on disk or in the db
    """
    @classmethod
    @transact
    def create_file(store, content):
        raise errors.MethodNotImplemented()

    @staticmethod
    def perform_file_action():
        raise errors.MethodNotImplemented()
Beispiel #18
0
    def get(self, key):
        ret = TempDict.get(self, key)
        if ret == None:
            raise errors.TokenFailure("Not found")

        return ret
Beispiel #19
0
 def __init__(self, *args, **kwds):
     TempDict.__init__(self, *args, **kwds)
Beispiel #20
0
class StateClass(ObjectDict):
    __metaclass__ = Singleton

    def __init__(self):
        from globaleaks.settings import Settings

        self.settings = Settings

        self.process_supervisor = None
        self.tor_exit_set = TorExitSet()

        self.https_socks = []
        self.http_socks = []

        self.jobs = []
        self.jobs_monitor = None
        self.services = []
        self.onion_service_job = None

        self.api_token_session = None

        self.exceptions = {}
        self.exceptions_email_count = 0
        self.mail_counters = {}
        self.stats_collection_start_time = datetime_now()

        self.accept_submissions = True

        self.tenant_state = {}
        self.tenant_cache = {}
        self.tenant_hostname_id_map = {}

        self.set_orm_tp(ThreadPool(4, 16))
        self.TempUploadFiles = TempDict(timeout=3600)

    def init_environment(self):
        os.umask(0o77)
        self.settings.eval_paths()
        self.create_directories()
        self.cleaning_dead_files()

    def set_orm_tp(self, orm_tp):
        self.orm_tp = orm_tp
        orm.set_thread_pool(orm_tp)

    def get_agent(self, tid=1):
        if self.tenant_cache[tid].anonymize_outgoing_connections:
            return get_tor_agent(self.settings.socks_host,
                                 self.settings.socks_port)

        return get_web_agent()

    def create_directory(self, path):
        """
        Create the specified directory;
        Returns True on success, False if the directory was already existing
        """
        if os.path.exists(path):
            return False

        log.debug("Creating directory: %s", path)

        try:
            os.mkdir(path)
        except OSError as excep:
            log.debug("Error in creating directory: %s (%s)", path,
                      excep.strerror)
            raise excep

        return True

    def create_directories(self):
        """
        Execute some consistency checks on command provided Globaleaks paths

        if one of working_path or static path is created we copy
        here the static files (default logs, and in the future pot files for localization)
        because here stay all the files needed by the application except the python scripts
        """
        for dirpath in [
                self.settings.working_path, self.settings.files_path,
                self.settings.attachments_path, self.settings.tmp_path,
                self.settings.log_path, self.settings.backups_path
        ]:
            self.create_directory(dirpath)

    def cleaning_dead_files(self):
        """
        This function is called at the start of GlobaLeaks, in
        bin/globaleaks, and checks if the file is present in
        temporally_encrypted_dir
        """
        # temporary .aes files must be simply deleted
        for f in os.listdir(self.settings.tmp_path):
            path = os.path.join(self.settings.tmp_path, f)
            log.debug("Removing old temporary file: %s", path)

            try:
                os.remove(path)
            except OSError as excep:
                log.debug("Error while evaluating removal for %s: %s", path,
                          excep.strerror)

        # temporary .aes files with lost keys can be deleted
        # while temporary .aes files with valid current key
        # will be automagically handled by delivery sched.
        keypath = os.path.join(self.settings.tmp_path,
                               self.settings.AES_keyfile_prefix)

        for f in os.listdir(self.settings.attachments_path):
            path = os.path.join(self.settings.attachments_path, f)
            try:
                result = self.settings.AES_file_regexp_comp.match(f)
                if result is not None:
                    if not os.path.isfile("%s%s" % (keypath, result.group(1))):
                        log.debug("Removing old encrypted file (lost key): %s",
                                  path)
                        os.remove(path)
            except Exception as excep:
                log.debug("Error while evaluating removal for %s: %s", path,
                          excep)

    def get_mail_counter(self, receiver_id):
        return self.mail_counters.get(receiver_id, 0)

    def increment_mail_counter(self, receiver_id):
        self.mail_counters[receiver_id] = self.mail_counters.get(
            receiver_id, 0) + 1

    def reset_hourly(self):
        for tid in self.tenant_state:
            self.tenant_state[tid] = TenantState(self)

        self.exceptions.clear()
        self.exceptions_email_count = 0
        self.mail_counters.clear()

        self.stats_collection_start_time = datetime_now()

    def sendmail(self, tid, to_address, subject, body):
        if self.settings.testing:
            # during unit testing do not try to send the mail
            return defer.succeed(True)

        return sendmail(
            tid, self.tenant_cache[tid].notification.smtp_server,
            self.tenant_cache[tid].notification.smtp_port,
            self.tenant_cache[tid].notification.smtp_security,
            self.tenant_cache[tid].notification.smtp_authentication,
            self.tenant_cache[tid].notification.smtp_username,
            self.tenant_cache[tid].notification.smtp_password,
            self.tenant_cache[tid].notification.smtp_source_name,
            self.tenant_cache[tid].notification.smtp_source_email, to_address,
            self.tenant_cache[tid].name + ' - ' + subject, body,
            self.tenant_cache[tid].anonymize_outgoing_connections,
            self.settings.socks_host, self.settings.socks_port)

    def schedule_exception_email(self, exception_text, *args):
        if not hasattr(self.tenant_cache[1], 'notification'):
            log.err(
                "Error: Cannot send mail exception before complete initialization."
            )
            return

        if self.exceptions_email_count >= self.settings.exceptions_email_hourly_limit:
            return

        exception_text = (exception_text % args) if args else exception_text

        sha256_hash = sha256(exception_text.encode())

        if sha256_hash not in self.exceptions:
            self.exceptions[sha256_hash] = 0

        self.exceptions[sha256_hash] += 1
        if self.exceptions[sha256_hash] > 5:
            log.err(
                "Exception mail suppressed for (%s) [reason: threshold exceeded]",
                sha256_hash)
            return

        self.exceptions_email_count += 1

        mail_subject = "GlobaLeaks Exception"
        delivery_list = self.tenant_cache[
            1].notification.exception_delivery_list

        if self.settings.devel_mode:
            mail_subject += " [%s]" % self.settings.developer_name
            delivery_list = [("*****@*****.**",
                              '')]

        mail_body = text_type("Platform: %s\nHost: %s (%s)\nVersion: %s\n\n%s" \
                          % (self.tenant_cache[1].name,
                             self.tenant_cache[1].hostname,
                             self.tenant_cache[1].onionservice,
                             __version__,
                             exception_text))

        for mail_address, pgp_key_public in delivery_list:
            # Opportunisticly encrypt the mail body. NOTE that mails will go out
            # unencrypted if one address in the list does not have a public key set.
            if pgp_key_public:
                pgpctx = PGPContext(self.settings.tmp_path)
                fingerprint = pgpctx.load_key(pgp_key_public)['fingerprint']
                mail_body = pgpctx.encrypt_message(fingerprint, mail_body)

            # avoid waiting for the notification to send and instead rely on threads to handle it
            schedule_email(1, mail_address, mail_subject, mail_body)

    def refresh_tenant_statuses(self):
        # Remove selected onion services and add missing services
        if self.onion_service_job is not None:

            def f(*args):
                return self.onion_service_job.add_all_hidden_services()

            self.onion_service_job.remove_unwanted_hidden_services().addBoth(f)  # pylint: disable=no-member

        # Power cycle HTTPS processes
        def g(*args):
            return self.process_supervisor.maybe_launch_https_workers()

        self.process_supervisor.shutdown(friendly=True).addBoth(g)  # pylint: disable=no-member

    def format_and_send_mail(self, session, tid, user_desc, template_vars):
        subject, body = Templating().get_mail_subject_and_body(template_vars)

        if user_desc.get('pgp_key_public', ''):
            pgpctx = PGPContext(self.settings.tmp_path)
            fingerprint = pgpctx.load_key(
                user_desc['pgp_key_public'])['fingerprint']
            body = pgpctx.encrypt_message(fingerprint, body)

        session.add(
            models.Mail({
                'address': user_desc['mail_address'],
                'subject': subject,
                'body': body,
                'tid': tid,
            }))

    def get_tmp_file_by_name(self, filename):
        for k, v in self.TempUploadFiles.items():
            if os.path.basename(v.filepath) == filename:
                return self.TempUploadFiles.pop(k)
Beispiel #21
0
    def get(self, key):
        ret = TempDict.get(self, key)
        if ret is None:
            raise errors.InternalServerError("TokenFailure: Invalid token")

        return ret
Beispiel #22
0
 def __init__(self, state, file_path):
     self.timeout = Token.ttl
     self.state = state
     self.file_path = file_path
     TempDict.__init__(self, self.timeout)
Beispiel #23
0
    def __init__(self):
        # command line parsing utils
        self.parser = OptionParser()
        self.cmdline_options = None

        # version
        self.version_string = __version__

        # testing
        # This variable is to be able to hook/bypass code when unit-tests are runned
        self.testing = False

        # daemon
        self.nodaemon = False

        # thread pool size of 1
        self.orm_tp = ThreadPool(0, 1)

        self.bind_addresses = '127.0.0.1'

        # bind port
        self.bind_port = 8082

        # store name
        self.store_name = 'main_store'

        self.db_type = 'sqlite'

        # debug defaults
        self.orm_debug = False
        self.log_requests_responses = -1
        self.requests_counter = 0
        self.loglevel = "CRITICAL"

        # files and paths
        self.root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
        self.pid_path = '/var/run/globaleaks'
        self.working_path = '/var/globaleaks'
        self.static_source = '/usr/share/globaleaks/data'

        self.client_path = '/usr/share/globaleaks/client'
        for path in possible_client_paths:
            if os.path.exists(path):
                self.client_path = path
                break

        self.set_ramdisk_path()

        self.default_password = '******'

        # some singleton classes: sessions and some event queues
        self.authentication_lifetime = 3600
        self.sessions = TempDict(timeout=self.authentication_lifetime)
        self.RecentEventQ = []
        self.RecentAnomaliesQ = {}

        self.accept_submissions = True

        # statistical, referred to latest period
        # and resetted by session_management sched
        self.failed_login_attempts = 0

        # static file rules
        self.staticfile_regexp = r'(.*)'
        self.staticfile_overwrite = False

        self.reserved_names = OD({
          'logo': 'logo',
          'css': 'custom_stylesheet',
          'html': 'custom_homepage'
        })

        # acceptable 'Host:' header in HTTP request
        self.accepted_hosts = "127.0.0.1, localhost"
        self.configured_hosts = []

        self.receipt_regexp = u'[0-9]{16}'

        # A lot of operations performed massively by globaleaks
        # should avoid to fetch continuously variables from the DB so that
        # it is important to keep this variables in memory
        #
        # The following initialization is needed only for variables that need
        # to be used in the startup queries, after that memory_copy is
        # initialized with the content Node table.
        self.memory_copy = OD({
            'maximum_namesize': 128,
            'maximum_textsize': 4096,
            'maximum_filesize': 30,
            'allow_iframes_inclusion': False,
            'tor2web_access': {
                'admin': True,
                'whistleblower': False,
                'custodian': False,
                'receiver': False,
                'unauth': True
            }
        })

        # Default request time uniform value
        self.side_channels_guard = 0.150

        # unchecked_tor_input contains information that cannot be validated now
        # due to complex inclusions or requirements. Data is used in
        # globaleaks.db.appdata.apply_cli_options()
        self.unchecked_tor_input = {}

        # SOCKS default
        self.socks_host = "127.0.0.1"
        self.socks_port = 9050

        self.notification_limit = 30
        self.jobs_operation_limit = 20

        self.user = getpass.getuser()
        self.group = getpass.getuser()
        self.uid = os.getuid()
        self.gid = os.getgid()
        self.start_clean = False
        self.devel_mode = False
        self.developer_name = ''
        self.skip_wizard = False
        self.log_timing_stats = False

        # Number of failed login enough to generate an alarm
        self.failed_login_alarm = 5

        # Number of minutes in which a user is prevented to login in case of triggered alarm
        self.failed_login_block_time = 5

        # Alarm to be ignored: can be raise with the -A command line switch
        self.disk_alarm_threshold = 0

        # Size in bytes of every log file. Once this size is reached the
        # logfile is rotated.
        # Default: 1M
        self.log_file_size = 1000000
        # Number of log files to conserve.
        self.maximum_rotated_log_files = 100

        # size used while streaming files
        self.file_chunk_size = 8192

        self.AES_key_size = 32
        self.AES_key_id_regexp = u'[A-Za-z0-9]{16}'
        self.AES_counter_nonce = 128 / 8
        self.AES_file_regexp = r'(.*)\.aes'
        self.AES_file_regexp_comp = re.compile(self.AES_file_regexp)
        self.AES_keyfile_prefix = "aeskey-"

        self.exceptions = {}
        self.exceptions_email_count = 0
        self.exceptions_email_hourly_limit = 20

        # Extreme debug options triggered by --XXX, that's are the defaults
        self.debug_option_in_the_future = 0
        self.debug_option_UUID_human = ""
        self.debug_UUID_human_counter = 0
        self.debug_option_mlockall = False

        self.disable_mail_torification = False
        self.disable_mail_notification = False
        self.disable_backend_exception_notification = False
        self.disable_client_exception_notification = False

        self.enable_input_length_checks = True

        self.mail_counters = {}
        self.mail_timeout = 15 # seconds
        self.mail_attempts_limit = 3 # per mail limit

        reactor.addSystemEventTrigger('after', 'shutdown', self.orm_tp.stop)
        self.orm_tp.start()
Beispiel #24
0
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks

from globaleaks.event import track_handler
from globaleaks.rest import errors, requests
from globaleaks.security import SecureTemporaryFile, directory_traversal_check, generateRandomKey, sha512
from globaleaks.settings import Settings
from globaleaks.transactions import schedule_email_for_all_admins
from globaleaks.utils.mailutils import schedule_exception_email
from globaleaks.utils.tempdict import TempDict
from globaleaks.utils.utility import log, deferred_sleep

HANDLER_EXEC_TIME_THRESHOLD = 120

Uploads = TempDict(timeout=60 * HANDLER_EXEC_TIME_THRESHOLD)


class SessionsFactory(TempDict):
    """Extends TempDict to provide session management functions ontop of temp session keys"""
    def revoke_all_sessions(self, user_id):
        for other_session in Sessions.values():
            if other_session.user_id == user_id:
                log.debug("Revoking old session for %s", user_id)
                Sessions.delete(other_session.id)


Sessions = SessionsFactory(timeout=Settings.authentication_lifetime)

# https://github.com/globaleaks/GlobaLeaks/issues/1601
mimetypes.add_type('image/svg+xml', '.svg')
Beispiel #25
0
class StateClass(ObjectDict):
    __metaclass__ = Singleton

    def __init__(self):
        self.settings = Settings

        self.process_supervisor = None
        self.tor_exit_set = TorExitSet()

        self.https_socks = []
        self.http_socks = []

        self.jobs = []
        self.jobs_monitor = None
        self.services = []
        self.onion_service_job = None

        self.api_token_session = None

        self.exceptions = {}
        self.exceptions_email_count = 0
        self.mail_counters = {}
        self.stats_collection_start_time = datetime_now()

        self.accept_submissions = True

        self.tenant_state = {}
        self.tenant_cache = {}
        self.tenant_hostname_id_map = {}

        self.set_orm_tp(ThreadPool(4, 16))
        self.TempUploadFiles = TempDict(timeout=3600)

        self.shutdown = False

    def init_environment(self):
        os.umask(0o77)
        self.settings.eval_paths()
        self.create_directories()
        self.cleaning_dead_files()

        self.tokens = TokenList(self.settings.tmp_path)

    def set_orm_tp(self, orm_tp):
        self.orm_tp = orm_tp
        orm.set_thread_pool(orm_tp)

    def get_agent(self):
        if self.tenant_cache[1].anonymize_outgoing_connections:
            return get_tor_agent(self.settings.socks_host, self.settings.socks_port)

        return get_web_agent()

    def create_directory(self, path):
        """
        Create the specified directory;
        Returns True on success, False if the directory was already existing
        """
        if os.path.exists(path):
            return False

        log.debug("Creating directory: %s", path)

        try:
            os.mkdir(path)
        except OSError as excep:
            log.debug("Error in creating directory: %s (%s)", path, excep.strerror)
            raise excep

        return True

    def create_directories(self):
        """
        Execute some consistency checks on command provided Globaleaks paths

        if one of working_path or static path is created we copy
        here the static files (default logs, and in the future pot files for localization)
        because here stay all the files needed by the application except the python scripts
        """
        for dirpath in [self.settings.working_path,
                        self.settings.files_path,
                        self.settings.attachments_path,
                        self.settings.tmp_path,
                        self.settings.log_path,
                        self.settings.backup_path]:
            self.create_directory(dirpath)

    def cleaning_dead_files(self):
        """
        This function is called at the start of GlobaLeaks, in
        bin/globaleaks, and checks if the file is present in
        temporally_encrypted_dir
        """
        # temporary .aes files must be simply deleted
        for f in os.listdir(self.settings.tmp_path):
            path = os.path.join(self.settings.tmp_path, f)
            log.debug("Removing old temporary file: %s", path)

            try:
                os.remove(path)
            except OSError as excep:
                log.debug("Error while evaluating removal for %s: %s", path, excep.strerror)

        # temporary .aes files with lost keys can be deleted
        # while temporary .aes files with valid current key
        # will be automagically handled by delivery sched.
        keypath = os.path.join(self.settings.tmp_path, self.settings.AES_keyfile_prefix)

        for f in os.listdir(self.settings.attachments_path):
            path = os.path.join(self.settings.attachments_path, f)
            try:
                result = self.settings.AES_file_regexp_comp.match(f)
                if result is not None:
                    if not os.path.isfile("%s%s" % (keypath, result.group(1))):
                        log.debug("Removing old encrypted file (lost key): %s", path)
                        os.remove(path)
            except Exception as excep:
                log.debug("Error while evaluating removal for %s: %s", path, excep)

    def get_mail_counter(self, receiver_id):
        return self.mail_counters.get(receiver_id, 0)

    def increment_mail_counter(self, receiver_id):
        self.mail_counters[receiver_id] = self.mail_counters.get(receiver_id, 0) + 1

    def reset_hourly(self):
        for tid in self.tenant_state:
            self.tenant_state[tid] = TenantState(self)

        self.exceptions.clear()
        self.exceptions_email_count = 0
        self.mail_counters.clear()

        self.stats_collection_start_time = datetime_now()

    def sendmail(self, tid, to_address, subject, body):
        if self.settings.testing:
            # during unit testing do not try to send the mail
            return defer.succeed(True)

        if self.tenant_cache[tid].mode == u'whistleblowing.it':
            tid = 1

        return sendmail(tid,
                        self.tenant_cache[tid].notification.smtp_server,
                        self.tenant_cache[tid].notification.smtp_port,
                        self.tenant_cache[tid].notification.smtp_security,
                        self.tenant_cache[tid].notification.smtp_authentication,
                        self.tenant_cache[tid].notification.smtp_username,
                        self.tenant_cache[tid].notification.smtp_password,
                        self.tenant_cache[tid].notification.smtp_source_name,
                        self.tenant_cache[tid].notification.smtp_source_email,
                        to_address,
                        self.tenant_cache[tid].name + ' - ' + subject,
                        body,
                        self.tenant_cache[1].anonymize_outgoing_connections,
                        self.settings.socks_host,
                        self.settings.socks_port)

    def schedule_exception_email(self, exception_text, *args):
        if not hasattr(self.tenant_cache[1], 'notification'):
            log.err("Error: Cannot send mail exception before complete initialization.")
            return

        if self.exceptions_email_count >= self.settings.exceptions_email_hourly_limit:
            return

        exception_text = (exception_text % args) if args else exception_text

        sha256_hash = sha256(exception_text.encode())

        if sha256_hash not in self.exceptions:
            self.exceptions[sha256_hash] = 0

        self.exceptions[sha256_hash] += 1
        if self.exceptions[sha256_hash] > 5:
            log.err("Exception mail suppressed for (%s) [reason: threshold exceeded]", sha256_hash)
            return

        self.exceptions_email_count += 1

        mail_subject = "GlobaLeaks Exception"
        delivery_list = self.tenant_cache[1].notification.exception_delivery_list

        mail_body = text_type("Platform: %s\nHost: %s (%s)\nVersion: %s\n\n%s"
                          % (self.tenant_cache[1].name,
                             self.tenant_cache[1].hostname,
                             self.tenant_cache[1].onionservice,
                             __version__,
                             exception_text))

        for mail_address, pgp_key_public in delivery_list:
            # Opportunisticly encrypt the mail body. NOTE that mails will go out
            # unencrypted if one address in the list does not have a public key set.
            if pgp_key_public:
                pgpctx = PGPContext(self.settings.tmp_path)
                fingerprint = pgpctx.load_key(pgp_key_public)['fingerprint']
                mail_body = pgpctx.encrypt_message(fingerprint, mail_body)

            # avoid waiting for the notification to send and instead rely on threads to handle it
            tw(db_schedule_email, 1, mail_address, mail_subject, mail_body)

    def refresh_connection_handpoints(self):
        # Remove selected onion services and add missing services
        if self.onion_service_job is not None:
            def f(*args):
                return self.onion_service_job.add_all_hidden_services()

            self.onion_service_job.remove_unwanted_hidden_services().addBoth(f)  # pylint: disable=no-member

        self.process_supervisor.reload()

    def format_and_send_mail(self, session, tid, user_desc, template_vars):
        subject, body = Templating().get_mail_subject_and_body(template_vars)

        if user_desc.get('pgp_key_public', ''):
            pgpctx = PGPContext(self.settings.tmp_path)
            fingerprint = pgpctx.load_key(user_desc['pgp_key_public'])['fingerprint']
            body = pgpctx.encrypt_message(fingerprint, body)

        db_schedule_email(session, tid, user_desc['mail_address'], subject, body)

    def get_tmp_file_by_name(self, filename):
        for k, v in self.TempUploadFiles.items():
            if os.path.basename(v.filepath) == filename:
                return self.TempUploadFiles.pop(k)