Пример #1
0
def update_notification(store, request, language=GLSetting.memory_copy.default_language):

    try:
        notif = store.find(Notification).one()
    except Exception as excep:
        log.err("Database error or application error: %s" % excep )
        raise excep

    v = dict(request)

    for attr in getattr(notif, "localized_strings"):
        v[attr] = getattr(notif, attr)
        v[attr][language] = unicode(request[attr])

    request = v

    if request['security'] in Notification._security_types:
        notif.security = request['security']
    else:
        log.err("Invalid request: Security option not recognized")
        log.debug("Invalid Security value: %s" % request['security'])
        raise errors.InvalidInputFormat("Security selection not recognized")

    notif.update(request)

    if request['disable'] != GLSetting.notification_temporary_disable:
        log.msg("Switching notification mode: was %s and now is %s" %
                ("DISABLE" if GLSetting.notification_temporary_disable else "ENABLE",
                 "DISABLE" if request['disable'] else "ENABLE")
        )
        GLSetting.notification_temporary_disable = request['disable']

    return admin_serialize_notification(notif, language)
Пример #2
0
 def _on_request_body(self, data):
     try:
         self._request.body = data
         content_type = self._request.headers.get("Content-Type", "")
         if self._request.method in ("POST", "PATCH", "PUT"):
             if content_type.startswith(
                     "application/x-www-form-urlencoded"
             ) and self.content_length < GLSetting.www_form_urlencoded_maximum_size:
                 arguments = parse_qs_bytes(native_str(self._request.body))
                 for name, values in arguments.iteritems():
                     values = [v for v in values if v]
                     if values:
                         self._request.arguments.setdefault(
                             name, []).extend(values)
             elif content_type.startswith(
                     "application/x-www-form-urlencoded"):
                 raise errors.InvalidInputFormat(
                     "content type application/x-www-form-urlencoded not supported"
                 )
             elif content_type.startswith("multipart/form-data"):
                 raise errors.InvalidInputFormat(
                     "content type multipart/form-data not supported")
         self.request_callback(self._request)
     except Exception as exception:
         log.msg("Malformed HTTP request from %s: %s" %
                 (self._remote_ip, exception))
         log.exception(exception)
         if self._request:
             self._request.finish()
         if self.transport:
             self.transport.loseConnection()
Пример #3
0
def update_notification(store, request, language=GLSetting.memory_copy.default_language):

    try:
        notif = store.find(Notification).one()
    except Exception as excep:
        log.err("Database error or application error: %s" % excep )
        raise excep

    fill_localized_keys(request, Notification.localized_strings, language)

    if request['security'] in Notification._security_types:
        notif.security = request['security']
    else:
        log.err("Invalid request: Security option not recognized")
        log.debug("Invalid Security value: %s" % request['security'])
        raise errors.InvalidInputFormat("Security selection not recognized")

    try:
        notif.update(request)
    except DatabaseError as dberror:
        log.err("Unable to update Notification: %s" % dberror)
        raise errors.InvalidInputFormat(dberror)

    if request['disable'] != GLSetting.notification_temporary_disable:
        log.msg("Switching notification mode: was %s and now is %s" %
                ("DISABLE" if GLSetting.notification_temporary_disable else "ENABLE",
                 "DISABLE" if request['disable'] else "ENABLE")
        )
        GLSetting.notification_temporary_disable = request['disable']

    return admin_serialize_notification(notif, language)
Пример #4
0
def update_notification(store,
                        request,
                        language=GLSetting.memory_copy.default_language):

    try:
        notif = store.find(Notification).one()
    except Exception as excep:
        log.err("Database error or application error: %s" % excep)
        raise excep

    mo = structures.Rosetta()
    mo.acquire_request(language, request, Notification)
    for attr in mo.get_localized_attrs():
        request[attr] = mo.get_localized_dict(attr)

    if request['security'] in Notification._security_types:
        notif.security = request['security']
    else:
        log.err("Invalid request: Security option not recognized")
        log.debug("Invalid Security value: %s" % request['security'])
        raise errors.InvalidInputFormat("Security selection not recognized")

    try:
        notif.update(request)
    except Exception as dberror:
        log.err("Unable to update Notification: %s" % dberror)
        raise errors.InvalidInputFormat(dberror)

    if request['disable'] != GLSetting.notification_temporary_disable:
        log.msg("Switching notification mode: was %s and now is %s" %
                ("DISABLE" if GLSetting.notification_temporary_disable else
                 "ENABLE", "DISABLE" if request['disable'] else "ENABLE"))
        GLSetting.notification_temporary_disable = request['disable']

    return admin_serialize_notification(notif, language)
Пример #5
0
    def _on_headers(self, data):
        try:
            data = native_str(data.decode("latin1"))
            eol = data.find("\r\n")
            start_line = data[:eol]

            try:
                method, uri, version = start_line.split(" ")
            except ValueError:
                raise _BadRequestException("Malformed HTTP request line")

            if not version.startswith("HTTP/"):
                raise _BadRequestException(
                    "Malformed HTTP version in HTTP Request-Line")

            try:
                headers = httputil.HTTPHeaders.parse(data[eol:])
                content_length = int(headers.get("Content-Length", 0))
            except ValueError:
                raise _BadRequestException("Malformed Content-Length header")

            self._request = HTTPRequest(connection=self,
                                        method=method,
                                        uri=uri,
                                        version=version,
                                        headers=headers,
                                        remote_ip=self._remote_ip)

            if content_length:
                megabytes = int(content_length) / (1024 * 1024)
                if megabytes > GLSettings.memory_copy.maximum_filesize:
                    raise _BadRequestException(
                        "Request exceeded size limit %d" %
                        GLSettings.memory_copy.maximum_filesize)

                if headers.get("Expect") == "100-continue":
                    self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")

                if content_length < 100000:
                    self._contentbuffer = StringIO()
                else:
                    self._contentbuffer = GLSecureTemporaryFile(
                        GLSettings.tmp_upload_path)

                self.content_length = content_length
                self.setRawMode()
                return

            self.request_callback(self._request)
        except _BadRequestException as e:
            log.msg("Exception while handling HTTP request from %s: %s" %
                    (self._remote_ip, e))
            self.transport.loseConnection()
Пример #6
0
    def _on_headers(self, data):
        try:
            data = native_str(data.decode("latin1"))
            eol = data.find("\r\n")
            start_line = data[:eol]

            try:
                method, uri, version = start_line.split(" ")
            except ValueError:
                raise _BadRequestException("Malformed HTTP request line")

            if not version.startswith("HTTP/"):
                raise _BadRequestException(
                    "Malformed HTTP version in HTTP Request-Line")

            try:
                headers = httputil.HTTPHeaders.parse(data[eol:])
                content_length = int(headers.get("Content-Length", 0))
            except ValueError:
                raise _BadRequestException(
                    "Malformed Content-Length header")

            self._request = HTTPRequest(
                connection=self, method=method, uri=uri, version=version,
                headers=headers, remote_ip=self._remote_ip)

            if content_length:
                megabytes = int(content_length) / (1024 * 1024)
                if megabytes > GLSettings.memory_copy.maximum_filesize:
                    raise _BadRequestException("Request exceeded size limit %d" %
                                               GLSettings.memory_copy.maximum_filesize)

                if headers.get("Expect") == "100-continue":
                    self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")

                if content_length < 100000:
                    self._contentbuffer = StringIO()
                else:
                    self._contentbuffer = GLSecureTemporaryFile(GLSettings.tmp_upload_path)

                self.content_length = content_length
                self.setRawMode()
                return

            self.request_callback(self._request)
        except _BadRequestException as e:
            log.msg("Exception while handling HTTP request from %s: %s" % (self._remote_ip, e))
            self.transport.loseConnection()
Пример #7
0
 def _on_request_body(self, data):
     try:
         self._request.body = data
         content_type = self._request.headers.get("Content-Type", "")
         if self._request.method in ("POST", "PATCH", "PUT"):
             if content_type.startswith("application/x-www-form-urlencoded"):
                 raise errors.InvalidInputFormat("content type application/x-www-form-urlencoded not supported")
             elif content_type.startswith("multipart/form-data"):
                 raise errors.InvalidInputFormat("content type multipart/form-data not supported")
         self.request_callback(self._request)
     except Exception as exception:
         log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
         log.exception(exception)
         if self._request:
             self._request.finish()
         if self.transport:
             self.transport.loseConnection()
Пример #8
0
    def _handle_request_exception(self, e):
        if isinstance(e, Failure):
            exc_type = e.type
            exc_value = e.value
            exc_tb = e.getTracebackObject()
            e = e.value
        else:
            exc_type, exc_value, exc_tb = sys.exc_info()

        if isinstance(e, (HTTPError, HTTPAuthenticationRequired)):
            if GLSettings.log_requests_responses and e.log_message:
                string_format = "%d %s: " + e.log_message
                args = [e.status_code, self._request_summary()] + list(e.args)
                msg = lambda *args: string_format % args
                log.msg(msg(*args))
            if e.status_code not in httplib.responses:
                log.msg("Bad HTTP status code: %d" % e.status_code)
                return self.send_error(500, exception=e)
            else:
                return self.send_error(e.status_code, exception=e)
        else:
            log.err("Uncaught exception %s %s %s" %
                    (exc_type, exc_value, exc_tb))
            if GLSettings.log_requests_responses:
                log.msg(e)
            mail_exception_handler(exc_type, exc_value, exc_tb)
            return self.send_error(500, exception=e)
Пример #9
0
    def _handle_request_exception(self, e):
        if isinstance(e, Failure):
            exc_type = e.type
            exc_value = e.value
            exc_tb = e.getTracebackObject()
            e = e.value
        else:
            exc_type, exc_value, exc_tb = sys.exc_info()

        if isinstance(e, (HTTPError, HTTPAuthenticationRequired)):
            if GLSettings.log_requests_responses and e.log_message:
                string_format = "%d %s: " + e.log_message
                args = [e.status_code, self._request_summary()] + list(e.args)
                msg = lambda *args: string_format % args
                log.msg(msg(*args))
            if e.status_code not in httplib.responses:
                log.msg("Bad HTTP status code: %d" % e.status_code)
                return self.send_error(500, exception=e)
            else:
                return self.send_error(e.status_code, exception=e)
        else:
            log.err("Uncaught exception %s %s %s" % (exc_type, exc_value, exc_tb))
            if GLSettings.log_requests_responses:
                log.msg(e)
            mail_exception_handler(exc_type, exc_value, exc_tb)
            return self.send_error(500, exception=e)
Пример #10
0
 def _on_request_body(self, data):
     try:
         self._request.body = data
         content_type = self._request.headers.get("Content-Type", "")
         if self._request.method in ("POST", "PATCH", "PUT"):
             if content_type.startswith("application/x-www-form-urlencoded") and self.content_length < GLSetting.www_form_urlencoded_maximum_size:
                 arguments = parse_qs_bytes(native_str(self._request.body))
                 for name, values in arguments.iteritems():
                     values = [v for v in values if v]
                     if values:
                         self._request.arguments.setdefault(name,
                                                            []).extend(values)
             elif content_type.startswith("application/x-www-form-urlencoded"):
                 raise errors.InvalidInputFormat("content type application/x-www-form-urlencoded not supported")
             elif content_type.startswith("multipart/form-data"):
                 raise errors.InvalidInputFormat("content type multipart/form-data not supported")
         self.request_callback(self._request)
     except Exception as exception:
         log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
         log.exception(exception)
         if self._request:
             self._request.finish()
         if self.transport:
             self.transport.loseConnection()
Пример #11
0
    def cb(res):
        start_asynchronous()
        yield import_memory_variables()
        yield apply_cli_options()

        log.msg("GLBackend is now running")
        for ip in GLSetting.bind_addresses:
            log.msg("Visit http://%s:%d to interact with me" % (ip, GLSetting.bind_port))

        for host in GLSetting.accepted_hosts:
            if host not in GLSetting.bind_addresses:
                log.msg("Visit http://%s:%d to interact with me" % (host, GLSetting.bind_port))
Пример #12
0
def apply_cmdline_options(store):
    """
    Remind: GLSettings.unchecked_tor_input contain data that are not
    checked until this function!
    """
    node = store.find(models.Node).one()

    verb = "Hardwriting"
    if 'hostname_tor_content' in GLSettings.unchecked_tor_input:
        composed_hs_url = 'http://%s' % GLSettings.unchecked_tor_input['hostname_tor_content']
        hs = GLSettings.unchecked_tor_input['hostname_tor_content'].split('.onion')[0]
        composed_t2w_url = 'https://%s.tor2web.org' % hs

        if not (re.match(requests.hidden_service_regexp, composed_hs_url) or
                    re.match(requests.https_url_regexp, composed_t2w_url)):
            log.msg("[!!] Invalid content found in the 'hostname' file specified (%s): Ignored" % \
                    GLSettings.unchecked_tor_input['hostname_tor_content'])
        else:
            node.hidden_service = unicode(composed_hs_url)
            log.msg("[+] %s hidden service in the DB: %s" % (verb, composed_hs_url))

            if node.public_site:
                log.msg("[!!] Public Website (%s) is not automatically overwritten by (%s)" % \
                        (node.public_site, composed_t2w_url))
            else:
                node.public_site = unicode(composed_t2w_url)
                log_msg("[+] %s public site in the DB: %s" % (verb, composed_t2w_url))

            verb = "Overwriting"

    if GLSettings.cmdline_options.public_website:
        if not re.match(requests.https_url_regexp, GLSettings.cmdline_options.public_website):
            log.msg("[!!] Invalid public site: %s: Ignored" % GLSettings.cmdline_options.public_website)
        else:
            log.msg("[+] %s public site in the DB: %s" % (verb, GLSettings.cmdline_options.public_website))
            node.public_site = unicode(GLSettings.cmdline_options.public_website)

    if GLSettings.cmdline_options.hidden_service:
        if not re.match(requests.hidden_service_regexp, GLSettings.cmdline_options.hidden_service):
            log.msg("[!!] Invalid hidden service: %s: Ignored" % GLSettings.cmdline_options.hidden_service)
        else:
            log.msg("[+] %s hidden service in the DB: %s" % (verb, GLSettings.cmdline_options.hidden_service))
            node.hidden_service = unicode(GLSettings.cmdline_options.hidden_service)

    # return configured URL for the log/console output
    if node.hidden_service or node.public_site:
        GLSettings.configured_hosts = [node.hidden_service, node.public_site]
Пример #13
0
    def start_globaleaks(self):
        try:
            GLSettings.fix_file_permissions()
            GLSettings.drop_privileges()
            GLSettings.check_directories()

            # Check presence of an existing database and eventually perform its migration
            check = check_db_files()
            if check == -1:
                 self._reactor.stop()
            elif check == 0:
                yield init_db()
            else:
                yield update_version()
                yield init_appdata()

            yield clean_untracked_files()

            yield refresh_memory_variables()

            if GLSettings.cmdline_options:
                yield apply_cmdline_options()

            self.start_asynchronous_jobs()

            log.msg("GLBackend is now running")
            for ip in GLSettings.bind_addresses:
                log.msg("Visit http://%s:%d to interact with me" % (ip, GLSettings.bind_port))

            for host in GLSettings.accepted_hosts:
                if host not in GLSettings.bind_addresses:
                    log.msg("Visit http://%s:%d to interact with me" % (host, GLSettings.bind_port))

            for other in GLSettings.configured_hosts:
                if other:
                    log.msg("Visit %s to interact with me" % other)

            log.msg("Remind: GlobaLeaks is not accessible from other URLs, this is strictly enforced")
            log.msg("Check documentation in https://github.com/globaleaks/GlobaLeaks/wiki/ for special enhancement")

        except Exception as excep:
            log.err("ERROR: Cannot start GlobaLeaks; please manual check the error.")
            log.err("EXCEPTION: %s" % excep)
            self._reactor.stop()
Пример #14
0
def perform_system_update():
    """
    This function checks the system and database versions and executes migration
    routines based on the system's state. After this function has completed the
    node is either ready for initialization (0), running a version of the DB
    (>1), or broken (-1).
    """
    from globaleaks.db import migration
    db_files = []
    max_version = 0
    min_version = 0
    for filename in os.listdir(GLSettings.db_path):
        if filename.startswith('glbackend'):
            filepath = os.path.join(GLSettings.db_path, filename)
            if filename.endswith('.db'):
                db_files.append(filepath)
                nameindex = filename.rfind('glbackend')
                extensindex = filename.rfind('.db')
                fileversion = int(filename[nameindex + len('glbackend-'):extensindex])
                max_version = fileversion if fileversion > max_version else max_version
                min_version = fileversion if fileversion < min_version else min_version

    db_version = max_version

    if len(db_files) > 1:
        log.msg("Error: Cannot start the application because more than one database file are present in: %s" % GLSettings.db_path)
        log.msg("Manual check needed and is suggested to first make a backup of %s\n" % GLSettings.working_path)
        log.msg("Files found:")

        for f in db_files:
            log.msg("\t%s" % f)

        return -1

    if len(db_files) == 1:
        log.msg("Found an already initialized database version: %d" % db_version)

        if db_version < DATABASE_VERSION:
            log.msg("Performing schema migration from version %d to version %d" % (db_version, DATABASE_VERSION))
            try:
                migration.perform_schema_migration(db_version)
            except Exception as exception:
                log.msg("Migration failure: %s" % exception)
                log.msg("Verbose exception traceback:")
                etype, value, tback = sys.exc_info()
                log.msg('\n'.join(traceback.format_exception(etype, value, tback)))
                return -1

            log.msg("Migration completed with success!")

        else:
            log.msg('Performing data update')
            migration.perform_data_update(os.path.abspath(os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION)))

    return db_version
Пример #15
0
    def _on_headers(self, data):
        try:
            data = native_str(data.decode("latin1"))
            eol = data.find("\r\n")
            start_line = data[:eol]
            try:
                method, uri, version = start_line.split(" ")
            except ValueError:
                raise _BadRequestException("Malformed HTTP request line")
            if not version.startswith("HTTP/"):
                raise _BadRequestException(
                    "Malformed HTTP version in HTTP Request-Line")
            headers = httputil.HTTPHeaders.parse(data[eol:])
            self._request = HTTPRequest(
                connection=self, method=method, uri=uri, version=version,
                headers=headers, remote_ip=self._remote_ip)

            try:
                self.content_length = int(headers.get("Content-Length", 0))
            except ValueError:
                raise _BadRequestException("Malformed Content-Length header")

            # we always use secure temporary files in case of large json or file uploads
            if self.content_length < 100000 and self._request.headers.get("Content-Disposition") is None:
                self._contentbuffer = StringIO('')
            else:
                self._contentbuffer = GLSecureTemporaryFile(GLSetting.tmp_upload_path)

            if headers.get("Expect") == "100-continue":
                self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")

            c_d_header = self._request.headers.get("Content-Disposition")
            if c_d_header is not None:
                key, pdict = parse_header(c_d_header)
                if key != 'attachment' or 'filename' not in pdict:
                    raise _BadRequestException("Malformed Content-Disposition header")

                self.file_upload = True
                self.uploaded_file['filename'] = pdict['filename']
                self.uploaded_file['content_type'] = self._request.headers.get("Content-Type",
                                                                               'application/octet-stream')

                self.uploaded_file['body'] = self._contentbuffer
                self.uploaded_file['body_len'] = int(self.content_length)
                self.uploaded_file['body_filepath'] = self._contentbuffer.filepath

            megabytes = int(self.content_length) / (1024 * 1024)

            if self.file_upload:
                limit_type = "upload"
                limit = GLSetting.memory_copy.maximum_filesize
            else:
                limit_type = "json"
                limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize
                # is 1MB probably too high. probably this variable must be in kB

            # less than 1 megabytes is always accepted
            if megabytes > limit:
                log.err("Tried %s request larger than expected (%dMb > %dMb)" %
                        (limit_type,
                         megabytes,
                         limit))

                # In HTTP Protocol errors need to be managed differently than handlers
                raise errors.HTTPRawLimitReach

            if self.content_length > 0:
                self.setRawMode()
                return
            elif self.file_upload:
                self._on_request_body(self.uploaded_file)
                self.file_upload = False
                self.uploaded_file = {}
                return

            self.request_callback(self._request)
        except Exception as exception:
            log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
            log.exception(exception)
            if self._request:
                self._request.finish()
            if self.transport:
                self.transport.loseConnection()
Пример #16
0
def check_db_files():
    """
    This function checks the database version and executes eventually
    executes migration scripts
    """
    db_files = []
    max_version = 0
    min_version = 0
    for filename in os.listdir(GLSettings.db_path):
        if filename.startswith('glbackend'):
            filepath = os.path.join(GLSettings.db_path, filename)
            if filename.endswith('.db'):
                db_files.append(filepath)
                nameindex = filename.rfind('glbackend')
                extensindex = filename.rfind('.db')
                fileversion = int(filename[nameindex +
                                           len('glbackend-'):extensindex])
                max_version = fileversion if fileversion > max_version else max_version
                min_version = fileversion if fileversion < min_version else min_version

    db_version = max_version

    if len(db_files) == 1 and db_version > 0:
        from globaleaks.db import migration

        log.msg("Found an already initialized database version: %d" %
                db_version)

        if db_version < DATABASE_VERSION:
            log.msg(
                "Performing update of database from version %d to version %d" %
                (db_version, DATABASE_VERSION))
            try:
                migration.perform_version_update(db_version)
                log.msg("Migration completed with success!")
            except Exception as exception:
                log.msg("Migration failure: %s" % exception)
                log.msg("Verbose exception traceback:")
                etype, value, tback = sys.exc_info()
                log.msg('\n'.join(
                    traceback.format_exception(etype, value, tback)))
                return -1

    elif len(db_files) > 1:
        log.msg(
            "Error: Cannot start the application because more than one database file are present in: %s"
            % GLSettings.db_path)
        log.msg(
            "Manual check needed and is suggested to first make a backup of %s\n"
            % GLSettings.working_path)
        log.msg("Files found:")

        for f in db_files:
            log.msg("\t%s" % f)

        return -1

    return db_version
Пример #17
0
    def _on_headers(self, data):
        try:
            data = native_str(data.decode("latin1"))
            eol = data.find("\r\n")
            start_line = data[:eol]
            try:
                method, uri, version = start_line.split(" ")
            except ValueError:
                raise _BadRequestException("Malformed HTTP request line")
            if not version.startswith("HTTP/"):
                raise _BadRequestException("Malformed HTTP version in HTTP Request-Line")
            headers = httputil.HTTPHeaders.parse(data[eol:])
            self._request = HTTPRequest(
                connection=self, method=method, uri=uri, version=version, headers=headers, remote_ip=self._remote_ip
            )

            content_length = int(headers.get("Content-Length", 0))
            self.content_length = content_length

            if content_length:
                if headers.get("Expect") == "100-continue":
                    self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")

                if content_length < 100000:
                    self._contentbuffer = StringIO("")
                else:
                    self._contentbuffer = TemporaryFile()

                c_d_header = self._request.headers.get("Content-Disposition")
                if c_d_header is not None:
                    c_d_header = c_d_header.lower()
                    m = content_disposition_re.match(c_d_header)
                    if m is None:
                        raise Exception
                    self.file_upload = True
                    self.uploaded_file["filename"] = m.group(1)
                    self.uploaded_file["content_type"] = self._request.headers.get(
                        "Content-Type", "application/octet-stream"
                    )
                    self.uploaded_file["body"] = self._contentbuffer
                    self.uploaded_file["body_len"] = int(content_length)

                megabytes = int(content_length) / (1024 * 1024)

                if self.file_upload:
                    limit_type = "upload"
                    limit = GLSetting.memory_copy.maximum_filesize
                else:
                    limit_type = "json"
                    limit = 1000000  # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize
                    # is 1MB probably too high. probably this variable must be
                    # in kB

                # less than 1 megabytes is always accepted
                if megabytes > limit:

                    log.err("Tried %s request larger than expected (%dMb > %dMb)" % (limit_type, megabytes, limit))

                    # In HTTP Protocol errors need to be managed differently than handlers
                    raise errors.HTTPRawLimitReach

                self.setRawMode()
                return

            self.request_callback(self._request)
        except Exception as exception:
            log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
            log.exception(exception)
            if self._request:
                self._request.finish()
            if self.transport:
                self.transport.loseConnection()
Пример #18
0
    def start_globaleaks(self):
        try:
            GLSettings.fix_file_permissions()
            GLSettings.drop_privileges()
            GLSettings.check_directories()

            # Check presence of an existing database and eventually perform its migration
            check = check_db_files()
            if check == -1:
                self._reactor.stop()
            elif check == 0:
                yield init_db()
            else:
                yield update_version()
                yield init_appdata()

            yield clean_untracked_files()

            yield refresh_memory_variables()

            if GLSettings.cmdline_options:
                yield apply_cmdline_options()

            self.start_asynchronous_jobs()

            log.msg("GLBackend is now running")
            for ip in GLSettings.bind_addresses:
                log.msg("Visit http://%s:%d to interact with me" %
                        (ip, GLSettings.bind_port))

            for host in GLSettings.accepted_hosts:
                if host not in GLSettings.bind_addresses:
                    log.msg("Visit http://%s:%d to interact with me" %
                            (host, GLSettings.bind_port))

            for other in GLSettings.configured_hosts:
                if other:
                    log.msg("Visit %s to interact with me" % other)

            log.msg(
                "Remind: GlobaLeaks is not accessible from other URLs, this is strictly enforced"
            )
            log.msg(
                "Check documentation in https://github.com/globaleaks/GlobaLeaks/wiki/ for special enhancement"
            )

        except Exception as excep:
            log.err(
                "ERROR: Cannot start GlobaLeaks; please manual check the error."
            )
            log.err("EXCEPTION: %s" % excep)
            self._reactor.stop()
Пример #19
0
def perform_schema_migration(version):
    """
    @param version:
    @return:
    """
    to_delete_on_fail = []
    to_delete_on_success = []

    if version < FIRST_DATABASE_VERSION_SUPPORTED:
        log.msg(
            "Migrations from DB version lower than %d are no longer supported!"
            % FIRST_DATABASE_VERSION_SUPPORTED)
        quit()

    tmpdir = os.path.abspath(os.path.join(GLSettings.db_path, 'tmp'))
    orig_db_file = os.path.abspath(
        os.path.join(GLSettings.db_path, 'glbackend-%d.db' % version))
    final_db_file = os.path.abspath(
        os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION))

    shutil.rmtree(tmpdir, True)
    os.mkdir(tmpdir)
    shutil.copy2(orig_db_file, tmpdir)

    new_db_file = None

    try:
        while version < DATABASE_VERSION:
            old_db_file = os.path.abspath(
                os.path.join(tmpdir, 'glbackend-%d.db' % version))
            new_db_file = os.path.abspath(
                os.path.join(tmpdir, 'glbackend-%d.db' % (version + 1)))

            GLSettings.db_file = new_db_file
            GLSettings.enable_input_length_checks = False

            to_delete_on_fail.append(new_db_file)
            to_delete_on_success.append(old_db_file)

            log.msg("Updating DB from version %d to version %d" %
                    (version, version + 1))

            store_old = Store(create_database('sqlite:' + old_db_file))
            store_new = Store(create_database('sqlite:' + new_db_file))

            # Here is instanced the migration script
            MigrationModule = importlib.import_module(
                "globaleaks.db.migrations.update_%d" % (version + 1))
            migration_script = MigrationModule.MigrationScript(
                migration_mapping, version, store_old, store_new)

            log.msg("Migrating table:")

            try:
                try:
                    migration_script.prologue()
                except Exception as exception:
                    log.err("Failure while executing migration prologue: %s" %
                            exception)
                    raise exception

                for model_name, _ in migration_mapping.iteritems():
                    if migration_script.model_from[
                            model_name] is not None and migration_script.model_to[
                                model_name] is not None:
                        try:
                            migration_script.migrate_model(model_name)

                            # Commit at every table migration in order to be able to detect
                            # the precise migration that may fail.
                            migration_script.commit()
                        except Exception as exception:
                            log.err("Failure while migrating table %s: %s " %
                                    (model_name, exception))
                            raise exception
                try:
                    migration_script.epilogue()
                    migration_script.commit()
                except Exception as exception:
                    log.err("Failure while executing migration epilogue: %s " %
                            exception)
                    raise exception

            finally:
                # the database should be always closed before leaving the application
                # in order to not keep leaking journal files.
                migration_script.close()

            log.msg("Migration stats:")

            # we open a new db in order to verify integrity of the generated file
            store_verify = Store(
                create_database(GLSettings.make_db_uri(new_db_file)))

            for model_name, _ in migration_mapping.iteritems():
                if model_name == 'ApplicationData':
                    continue

                if migration_script.model_from[
                        model_name] is not None and migration_script.model_to[
                            model_name] is not None:
                    count = store_verify.find(
                        migration_script.model_to[model_name]).count()
                    if migration_script.entries_count[model_name] != count:
                        if migration_script.fail_on_count_mismatch[model_name]:
                            raise AssertionError("Integrity check failed on count equality for table %s: %d != %d" % \
                                                 (model_name, count, migration_script.entries_count[model_name]))
                        else:
                            log.msg(" * %s table migrated (entries count changed from %d to %d)" % \
                                                 (model_name, migration_script.entries_count[model_name], count))
                    else:
                        log.msg(" * %s table migrated (%d entry(s))" % \
                                             (model_name, migration_script.entries_count[model_name]))

            version += 1

            store_verify.close()

        perform_data_update(new_db_file)

    except Exception:
        raise

    else:
        # in case of success first copy the new migrated db, then as last action delete the original db file
        shutil.copy(new_db_file, final_db_file)
        security.overwrite_and_remove(orig_db_file)

    finally:
        # Always cleanup the temporary directory used for the migration
        for f in os.listdir(tmpdir):
            tmp_db_file = os.path.join(tmpdir, f)
            security.overwrite_and_remove(tmp_db_file)

        shutil.rmtree(tmpdir)
Пример #20
0
    def _on_headers(self, data):
        try:
            data = native_str(data.decode("latin1"))
            eol = data.find("\r\n")
            start_line = data[:eol]
            try:
                method, uri, version = start_line.split(" ")
            except ValueError:
                raise _BadRequestException("Malformed HTTP request line")
            if not version.startswith("HTTP/"):
                raise _BadRequestException(
                    "Malformed HTTP version in HTTP Request-Line")
            headers = httputil.HTTPHeaders.parse(data[eol:])
            self._request = HTTPRequest(
                connection=self, method=method, uri=uri, version=version,
                headers=headers, remote_ip=self._remote_ip)

            try:
                self.content_length = int(headers.get("Content-Length", 0))
            except ValueError:
                raise _BadRequestException("Malformed Content-Length header")

            # we always use secure temporary files in case of large json or file uploads
            if self.content_length < 100000 and self._request.headers.get("Content-Disposition") is None:
                self._contentbuffer = StringIO('')
            else:
                self._contentbuffer = GLSecureTemporaryFile(GLSetting.tmp_upload_path)

            if headers.get("Expect") == "100-continue":
                self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")

            c_d_header = self._request.headers.get("Content-Disposition")
            if c_d_header is not None:
                key, pdict = parse_header(c_d_header)
                if key != 'attachment' or 'filename' not in pdict:
                    raise _BadRequestException("Malformed Content-Disposition header")

                self.file_upload = True
                self.uploaded_file['filename'] = pdict['filename']
                self.uploaded_file['content_type'] = self._request.headers.get("Content-Type",
                                                                               'application/octet-stream')

                self.uploaded_file['body'] = self._contentbuffer
                self.uploaded_file['body_len'] = int(self.content_length)
                self.uploaded_file['body_filepath'] = self._contentbuffer.filepath

            megabytes = int(self.content_length) / (1024 * 1024)

            if self.file_upload:
                limit_type = "upload"
                limit = GLSetting.memory_copy.maximum_filesize
            else:
                limit_type = "json"
                limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize
                # is 1MB probably too high. probably this variable must be in kB

            # less than 1 megabytes is always accepted
            if megabytes > limit:
                log.err("Tried %s request larger than expected (%dMb > %dMb)" %
                        (limit_type,
                         megabytes,
                         limit))

                # In HTTP Protocol errors need to be managed differently than handlers
                raise errors.HTTPRawLimitReach

            if self.content_length > 0:
                self.setRawMode()
                return
            elif self.file_upload:
                self._on_request_body(self.uploaded_file)
                self.file_upload = False
                self.uploaded_file = {}
                return

            self.request_callback(self._request)
        except Exception as exception:
            log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
            log.exception(exception)
            if self._request:
                self._request.finish()
            if self.transport:
                self.transport.loseConnection()
Пример #21
0
def apply_cmdline_options(store):
    """
    Remind: GLSettings.unchecked_tor_input contain data that are not
    checked until this function!
    """
    node = store.find(models.Node).one()

    verb = "Hardwriting"
    if 'hostname_tor_content' in GLSettings.unchecked_tor_input:
        composed_hs_url = 'http://%s' % GLSettings.unchecked_tor_input[
            'hostname_tor_content']
        hs = GLSettings.unchecked_tor_input['hostname_tor_content'].split(
            '.onion')[0]
        composed_t2w_url = 'https://%s.tor2web.org' % hs

        if not (re.match(requests.hidden_service_regexp, composed_hs_url)
                or re.match(requests.https_url_regexp, composed_t2w_url)):
            log.msg("[!!] Invalid content found in the 'hostname' file specified (%s): Ignored" % \
                    GLSettings.unchecked_tor_input['hostname_tor_content'])
        else:
            node.hidden_service = unicode(composed_hs_url)
            log.msg("[+] %s hidden service in the DB: %s" %
                    (verb, composed_hs_url))

            if node.public_site:
                log.msg("[!!] Public Website (%s) is not automatically overwritten by (%s)" % \
                        (node.public_site, composed_t2w_url))
            else:
                node.public_site = unicode(composed_t2w_url)
                log.msg("[+] %s public site in the DB: %s" %
                        (verb, composed_t2w_url))

            verb = "Overwriting"

    if GLSettings.cmdline_options.public_website:
        if not re.match(requests.https_url_regexp,
                        GLSettings.cmdline_options.public_website):
            log.msg("[!!] Invalid public site: %s: Ignored" %
                    GLSettings.cmdline_options.public_website)
        else:
            log.msg("[+] %s public site in the DB: %s" %
                    (verb, GLSettings.cmdline_options.public_website))
            node.public_site = unicode(
                GLSettings.cmdline_options.public_website)

    if GLSettings.cmdline_options.hidden_service:
        if not re.match(requests.hidden_service_regexp,
                        GLSettings.cmdline_options.hidden_service):
            log.msg("[!!] Invalid hidden service: %s: Ignored" %
                    GLSettings.cmdline_options.hidden_service)
        else:
            log.msg("[+] %s hidden service in the DB: %s" %
                    (verb, GLSettings.cmdline_options.hidden_service))
            node.hidden_service = unicode(
                GLSettings.cmdline_options.hidden_service)

    # return configured URL for the log/console output
    if node.hidden_service or node.public_site:
        GLSettings.configured_hosts = [node.hidden_service, node.public_site]
Пример #22
0
def check_db_files():
    """
    This function checks the database version and executes eventually
    executes migration scripts
    """
    db_files = []
    max_version = 0
    min_version = 0
    for filename in os.listdir(GLSettings.db_path):
        if filename.startswith('glbackend'):
            filepath = os.path.join(GLSettings.db_path, filename)
            if filename.endswith('.db'):
                db_files.append(filepath)
                nameindex = filename.rfind('glbackend')
                extensindex = filename.rfind('.db')
                fileversion = int(filename[nameindex + len('glbackend-'):extensindex])
                max_version = fileversion if fileversion > max_version else max_version
                min_version = fileversion if fileversion < min_version else min_version

    db_version = max_version

    if len(db_files) == 1 and db_version > 0:
        from globaleaks.db import migration

        log.msg("Found an already initialized database version: %d" % db_version)

        if db_version < DATABASE_VERSION:
            log.msg("Performing update of database from version %d to version %d" % (db_version, DATABASE_VERSION))
            try:
                migration.perform_version_update(db_version)
                log.msg("Migration completed with success!")
            except Exception as exception:
                log.msg("Migration failure: %s" % exception)
                log.msg("Verbose exception traceback:")
                etype, value, tback = sys.exc_info()
                log.msg('\n'.join(traceback.format_exception(etype, value, tback)))
                return -1

    elif len(db_files) > 1:
        log.msg("Error: Cannot start the application because more than one database file are present in: %s" % GLSettings.db_path)
        log.msg("Manual check needed and is suggested to first make a backup of %s\n" % GLSettings.working_path)
        log.msg("Files found:")

        for f in db_files:
            log.msg("\t%s" % f)

        return -1

    return db_version
Пример #23
0
    def cb(res):
        start_asynchronous()
        yield import_memory_variables()
        tor_configured_hosts = yield apply_cli_options()

        log.msg("GLBackend is now running")
        for ip in GLSetting.bind_addresses:
            log.msg("Visit http://%s:%d to interact with me" %
                    (ip, GLSetting.bind_port))

        for host in GLSetting.accepted_hosts:
            if host not in GLSetting.bind_addresses:
                log.msg("Visit http://%s:%d to interact with me" %
                        (host, GLSetting.bind_port))

        if tor_configured_hosts:
            for other in tor_configured_hosts:
                if other:
                    log.msg("Visit %s to interact with me" % other)

        log.msg(
            "Remind: GlobaLeaks is not accessible from other URLs, this is strictly enforced"
        )
        log.msg(
            "Check documentation in https://github.com/globaleaks/GlobaLeaks/wiki/ for special enhancement"
        )
Пример #24
0
def perform_system_update():
    """
    This function checks the system and database versions and executes migration
    routines based on the system's state. After this function has completed the
    node is either ready for initialization (0), running a version of the DB
    (>1), or broken (-1).
    """
    from globaleaks.db import migration
    db_files = []
    max_version = 0
    min_version = 0
    for filename in os.listdir(GLSettings.db_path):
        if filename.startswith('glbackend'):
            filepath = os.path.join(GLSettings.db_path, filename)
            if filename.endswith('.db'):
                db_files.append(filepath)
                nameindex = filename.rfind('glbackend')
                extensindex = filename.rfind('.db')
                fileversion = int(filename[nameindex +
                                           len('glbackend-'):extensindex])
                max_version = fileversion if fileversion > max_version else max_version
                min_version = fileversion if fileversion < min_version else min_version

    db_version = max_version

    if len(db_files) > 1:
        log.msg(
            "Error: Cannot start the application because more than one database file are present in: %s"
            % GLSettings.db_path)
        log.msg(
            "Manual check needed and is suggested to first make a backup of %s\n"
            % GLSettings.working_path)
        log.msg("Files found:")

        for f in db_files:
            log.msg("\t%s" % f)

        return -1

    if len(db_files) == 1:
        log.msg("Found an already initialized database version: %d" %
                db_version)

        if db_version < DATABASE_VERSION:
            log.msg(
                "Performing schema migration from version %d to version %d" %
                (db_version, DATABASE_VERSION))
            try:
                migration.perform_schema_migration(db_version)
            except Exception as exception:
                log.msg("Migration failure: %s" % exception)
                log.msg("Verbose exception traceback:")
                etype, value, tback = sys.exc_info()
                log.msg('\n'.join(
                    traceback.format_exception(etype, value, tback)))
                return -1

            log.msg("Migration completed with success!")

        else:
            log.msg('Performing data update')
            migration.perform_data_update(
                os.path.abspath(
                    os.path.join(GLSettings.db_path,
                                 'glbackend-%d.db' % DATABASE_VERSION)))

    return db_version
Пример #25
0
    def cb(res):
        start_asynchronous()
        yield import_memory_variables()
        tor_configured_hosts = yield apply_cli_options()

        log.msg("GLBackend is now running")
        for ip in GLSetting.bind_addresses:
            log.msg("Visit http://%s:%d to interact with me" % (ip, GLSetting.bind_port))

        for host in GLSetting.accepted_hosts:
            if host not in GLSetting.bind_addresses:
                log.msg("Visit http://%s:%d to interact with me" % (host, GLSetting.bind_port))

        if tor_configured_hosts:
            for other in tor_configured_hosts:
                if other:
                    log.msg("Visit %s to interact with me" % other)

        log.msg("Remind: GlobaLeaks is not accessible from other URLs, this is strictly enforced")
        log.msg("Check documentation in https://github.com/globaleaks/GlobaLeaks/wiki/ for special enhancement")
Пример #26
0
def update_db():
    """
    This function handles update of an existing database
    """
    try:
        db_version, db_file_path = get_db_file(GLSettings.db_path)
        if db_version == 0:
            return 0

        from globaleaks.db import migration

        log.msg("Found an already initialized database version: %d" %
                db_version)

        if db_version >= FIRST_DATABASE_VERSION_SUPPORTED and db_version < DATABASE_VERSION:
            log.msg(
                "Performing schema migration from version %d to version %d" %
                (db_version, DATABASE_VERSION))
            migration.perform_schema_migration(db_version)
            log.msg("Migration completed with success!")

        else:
            log.msg('Performing data update')
            # TODO on normal startup this line is run. We need better control flow here.
            migration.perform_data_update(
                os.path.abspath(
                    os.path.join(GLSettings.db_path,
                                 'glbackend-%d.db' % DATABASE_VERSION)))

    except Exception as exception:
        log.msg("Migration failure: %s" % exception)
        log.msg("Verbose exception traceback:")
        etype, value, tback = sys.exc_info()
        log.msg('\n'.join(traceback.format_exception(etype, value, tback)))
        return -1