def update_notification(store, request, language=GLSetting.memory_copy.default_language): try: notif = store.find(Notification).one() except Exception as excep: log.err("Database error or application error: %s" % excep ) raise excep v = dict(request) for attr in getattr(notif, "localized_strings"): v[attr] = getattr(notif, attr) v[attr][language] = unicode(request[attr]) request = v if request['security'] in Notification._security_types: notif.security = request['security'] else: log.err("Invalid request: Security option not recognized") log.debug("Invalid Security value: %s" % request['security']) raise errors.InvalidInputFormat("Security selection not recognized") notif.update(request) if request['disable'] != GLSetting.notification_temporary_disable: log.msg("Switching notification mode: was %s and now is %s" % ("DISABLE" if GLSetting.notification_temporary_disable else "ENABLE", "DISABLE" if request['disable'] else "ENABLE") ) GLSetting.notification_temporary_disable = request['disable'] return admin_serialize_notification(notif, language)
def _on_request_body(self, data): try: self._request.body = data content_type = self._request.headers.get("Content-Type", "") if self._request.method in ("POST", "PATCH", "PUT"): if content_type.startswith( "application/x-www-form-urlencoded" ) and self.content_length < GLSetting.www_form_urlencoded_maximum_size: arguments = parse_qs_bytes(native_str(self._request.body)) for name, values in arguments.iteritems(): values = [v for v in values if v] if values: self._request.arguments.setdefault( name, []).extend(values) elif content_type.startswith( "application/x-www-form-urlencoded"): raise errors.InvalidInputFormat( "content type application/x-www-form-urlencoded not supported" ) elif content_type.startswith("multipart/form-data"): raise errors.InvalidInputFormat( "content type multipart/form-data not supported") self.request_callback(self._request) except Exception as exception: log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception)) log.exception(exception) if self._request: self._request.finish() if self.transport: self.transport.loseConnection()
def update_notification(store, request, language=GLSetting.memory_copy.default_language): try: notif = store.find(Notification).one() except Exception as excep: log.err("Database error or application error: %s" % excep ) raise excep fill_localized_keys(request, Notification.localized_strings, language) if request['security'] in Notification._security_types: notif.security = request['security'] else: log.err("Invalid request: Security option not recognized") log.debug("Invalid Security value: %s" % request['security']) raise errors.InvalidInputFormat("Security selection not recognized") try: notif.update(request) except DatabaseError as dberror: log.err("Unable to update Notification: %s" % dberror) raise errors.InvalidInputFormat(dberror) if request['disable'] != GLSetting.notification_temporary_disable: log.msg("Switching notification mode: was %s and now is %s" % ("DISABLE" if GLSetting.notification_temporary_disable else "ENABLE", "DISABLE" if request['disable'] else "ENABLE") ) GLSetting.notification_temporary_disable = request['disable'] return admin_serialize_notification(notif, language)
def update_notification(store, request, language=GLSetting.memory_copy.default_language): try: notif = store.find(Notification).one() except Exception as excep: log.err("Database error or application error: %s" % excep) raise excep mo = structures.Rosetta() mo.acquire_request(language, request, Notification) for attr in mo.get_localized_attrs(): request[attr] = mo.get_localized_dict(attr) if request['security'] in Notification._security_types: notif.security = request['security'] else: log.err("Invalid request: Security option not recognized") log.debug("Invalid Security value: %s" % request['security']) raise errors.InvalidInputFormat("Security selection not recognized") try: notif.update(request) except Exception as dberror: log.err("Unable to update Notification: %s" % dberror) raise errors.InvalidInputFormat(dberror) if request['disable'] != GLSetting.notification_temporary_disable: log.msg("Switching notification mode: was %s and now is %s" % ("DISABLE" if GLSetting.notification_temporary_disable else "ENABLE", "DISABLE" if request['disable'] else "ENABLE")) GLSetting.notification_temporary_disable = request['disable'] return admin_serialize_notification(notif, language)
def _on_headers(self, data): try: data = native_str(data.decode("latin1")) eol = data.find("\r\n") start_line = data[:eol] try: method, uri, version = start_line.split(" ") except ValueError: raise _BadRequestException("Malformed HTTP request line") if not version.startswith("HTTP/"): raise _BadRequestException( "Malformed HTTP version in HTTP Request-Line") try: headers = httputil.HTTPHeaders.parse(data[eol:]) content_length = int(headers.get("Content-Length", 0)) except ValueError: raise _BadRequestException("Malformed Content-Length header") self._request = HTTPRequest(connection=self, method=method, uri=uri, version=version, headers=headers, remote_ip=self._remote_ip) if content_length: megabytes = int(content_length) / (1024 * 1024) if megabytes > GLSettings.memory_copy.maximum_filesize: raise _BadRequestException( "Request exceeded size limit %d" % GLSettings.memory_copy.maximum_filesize) if headers.get("Expect") == "100-continue": self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n") if content_length < 100000: self._contentbuffer = StringIO() else: self._contentbuffer = GLSecureTemporaryFile( GLSettings.tmp_upload_path) self.content_length = content_length self.setRawMode() return self.request_callback(self._request) except _BadRequestException as e: log.msg("Exception while handling HTTP request from %s: %s" % (self._remote_ip, e)) self.transport.loseConnection()
def _on_headers(self, data): try: data = native_str(data.decode("latin1")) eol = data.find("\r\n") start_line = data[:eol] try: method, uri, version = start_line.split(" ") except ValueError: raise _BadRequestException("Malformed HTTP request line") if not version.startswith("HTTP/"): raise _BadRequestException( "Malformed HTTP version in HTTP Request-Line") try: headers = httputil.HTTPHeaders.parse(data[eol:]) content_length = int(headers.get("Content-Length", 0)) except ValueError: raise _BadRequestException( "Malformed Content-Length header") self._request = HTTPRequest( connection=self, method=method, uri=uri, version=version, headers=headers, remote_ip=self._remote_ip) if content_length: megabytes = int(content_length) / (1024 * 1024) if megabytes > GLSettings.memory_copy.maximum_filesize: raise _BadRequestException("Request exceeded size limit %d" % GLSettings.memory_copy.maximum_filesize) if headers.get("Expect") == "100-continue": self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n") if content_length < 100000: self._contentbuffer = StringIO() else: self._contentbuffer = GLSecureTemporaryFile(GLSettings.tmp_upload_path) self.content_length = content_length self.setRawMode() return self.request_callback(self._request) except _BadRequestException as e: log.msg("Exception while handling HTTP request from %s: %s" % (self._remote_ip, e)) self.transport.loseConnection()
def _on_request_body(self, data): try: self._request.body = data content_type = self._request.headers.get("Content-Type", "") if self._request.method in ("POST", "PATCH", "PUT"): if content_type.startswith("application/x-www-form-urlencoded"): raise errors.InvalidInputFormat("content type application/x-www-form-urlencoded not supported") elif content_type.startswith("multipart/form-data"): raise errors.InvalidInputFormat("content type multipart/form-data not supported") self.request_callback(self._request) except Exception as exception: log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception)) log.exception(exception) if self._request: self._request.finish() if self.transport: self.transport.loseConnection()
def _handle_request_exception(self, e): if isinstance(e, Failure): exc_type = e.type exc_value = e.value exc_tb = e.getTracebackObject() e = e.value else: exc_type, exc_value, exc_tb = sys.exc_info() if isinstance(e, (HTTPError, HTTPAuthenticationRequired)): if GLSettings.log_requests_responses and e.log_message: string_format = "%d %s: " + e.log_message args = [e.status_code, self._request_summary()] + list(e.args) msg = lambda *args: string_format % args log.msg(msg(*args)) if e.status_code not in httplib.responses: log.msg("Bad HTTP status code: %d" % e.status_code) return self.send_error(500, exception=e) else: return self.send_error(e.status_code, exception=e) else: log.err("Uncaught exception %s %s %s" % (exc_type, exc_value, exc_tb)) if GLSettings.log_requests_responses: log.msg(e) mail_exception_handler(exc_type, exc_value, exc_tb) return self.send_error(500, exception=e)
def _on_request_body(self, data): try: self._request.body = data content_type = self._request.headers.get("Content-Type", "") if self._request.method in ("POST", "PATCH", "PUT"): if content_type.startswith("application/x-www-form-urlencoded") and self.content_length < GLSetting.www_form_urlencoded_maximum_size: arguments = parse_qs_bytes(native_str(self._request.body)) for name, values in arguments.iteritems(): values = [v for v in values if v] if values: self._request.arguments.setdefault(name, []).extend(values) elif content_type.startswith("application/x-www-form-urlencoded"): raise errors.InvalidInputFormat("content type application/x-www-form-urlencoded not supported") elif content_type.startswith("multipart/form-data"): raise errors.InvalidInputFormat("content type multipart/form-data not supported") self.request_callback(self._request) except Exception as exception: log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception)) log.exception(exception) if self._request: self._request.finish() if self.transport: self.transport.loseConnection()
def cb(res): start_asynchronous() yield import_memory_variables() yield apply_cli_options() log.msg("GLBackend is now running") for ip in GLSetting.bind_addresses: log.msg("Visit http://%s:%d to interact with me" % (ip, GLSetting.bind_port)) for host in GLSetting.accepted_hosts: if host not in GLSetting.bind_addresses: log.msg("Visit http://%s:%d to interact with me" % (host, GLSetting.bind_port))
def apply_cmdline_options(store): """ Remind: GLSettings.unchecked_tor_input contain data that are not checked until this function! """ node = store.find(models.Node).one() verb = "Hardwriting" if 'hostname_tor_content' in GLSettings.unchecked_tor_input: composed_hs_url = 'http://%s' % GLSettings.unchecked_tor_input['hostname_tor_content'] hs = GLSettings.unchecked_tor_input['hostname_tor_content'].split('.onion')[0] composed_t2w_url = 'https://%s.tor2web.org' % hs if not (re.match(requests.hidden_service_regexp, composed_hs_url) or re.match(requests.https_url_regexp, composed_t2w_url)): log.msg("[!!] Invalid content found in the 'hostname' file specified (%s): Ignored" % \ GLSettings.unchecked_tor_input['hostname_tor_content']) else: node.hidden_service = unicode(composed_hs_url) log.msg("[+] %s hidden service in the DB: %s" % (verb, composed_hs_url)) if node.public_site: log.msg("[!!] Public Website (%s) is not automatically overwritten by (%s)" % \ (node.public_site, composed_t2w_url)) else: node.public_site = unicode(composed_t2w_url) log_msg("[+] %s public site in the DB: %s" % (verb, composed_t2w_url)) verb = "Overwriting" if GLSettings.cmdline_options.public_website: if not re.match(requests.https_url_regexp, GLSettings.cmdline_options.public_website): log.msg("[!!] Invalid public site: %s: Ignored" % GLSettings.cmdline_options.public_website) else: log.msg("[+] %s public site in the DB: %s" % (verb, GLSettings.cmdline_options.public_website)) node.public_site = unicode(GLSettings.cmdline_options.public_website) if GLSettings.cmdline_options.hidden_service: if not re.match(requests.hidden_service_regexp, GLSettings.cmdline_options.hidden_service): log.msg("[!!] Invalid hidden service: %s: Ignored" % GLSettings.cmdline_options.hidden_service) else: log.msg("[+] %s hidden service in the DB: %s" % (verb, GLSettings.cmdline_options.hidden_service)) node.hidden_service = unicode(GLSettings.cmdline_options.hidden_service) # return configured URL for the log/console output if node.hidden_service or node.public_site: GLSettings.configured_hosts = [node.hidden_service, node.public_site]
def start_globaleaks(self): try: GLSettings.fix_file_permissions() GLSettings.drop_privileges() GLSettings.check_directories() # Check presence of an existing database and eventually perform its migration check = check_db_files() if check == -1: self._reactor.stop() elif check == 0: yield init_db() else: yield update_version() yield init_appdata() yield clean_untracked_files() yield refresh_memory_variables() if GLSettings.cmdline_options: yield apply_cmdline_options() self.start_asynchronous_jobs() log.msg("GLBackend is now running") for ip in GLSettings.bind_addresses: log.msg("Visit http://%s:%d to interact with me" % (ip, GLSettings.bind_port)) for host in GLSettings.accepted_hosts: if host not in GLSettings.bind_addresses: log.msg("Visit http://%s:%d to interact with me" % (host, GLSettings.bind_port)) for other in GLSettings.configured_hosts: if other: log.msg("Visit %s to interact with me" % other) log.msg("Remind: GlobaLeaks is not accessible from other URLs, this is strictly enforced") log.msg("Check documentation in https://github.com/globaleaks/GlobaLeaks/wiki/ for special enhancement") except Exception as excep: log.err("ERROR: Cannot start GlobaLeaks; please manual check the error.") log.err("EXCEPTION: %s" % excep) self._reactor.stop()
def perform_system_update(): """ This function checks the system and database versions and executes migration routines based on the system's state. After this function has completed the node is either ready for initialization (0), running a version of the DB (>1), or broken (-1). """ from globaleaks.db import migration db_files = [] max_version = 0 min_version = 0 for filename in os.listdir(GLSettings.db_path): if filename.startswith('glbackend'): filepath = os.path.join(GLSettings.db_path, filename) if filename.endswith('.db'): db_files.append(filepath) nameindex = filename.rfind('glbackend') extensindex = filename.rfind('.db') fileversion = int(filename[nameindex + len('glbackend-'):extensindex]) max_version = fileversion if fileversion > max_version else max_version min_version = fileversion if fileversion < min_version else min_version db_version = max_version if len(db_files) > 1: log.msg("Error: Cannot start the application because more than one database file are present in: %s" % GLSettings.db_path) log.msg("Manual check needed and is suggested to first make a backup of %s\n" % GLSettings.working_path) log.msg("Files found:") for f in db_files: log.msg("\t%s" % f) return -1 if len(db_files) == 1: log.msg("Found an already initialized database version: %d" % db_version) if db_version < DATABASE_VERSION: log.msg("Performing schema migration from version %d to version %d" % (db_version, DATABASE_VERSION)) try: migration.perform_schema_migration(db_version) except Exception as exception: log.msg("Migration failure: %s" % exception) log.msg("Verbose exception traceback:") etype, value, tback = sys.exc_info() log.msg('\n'.join(traceback.format_exception(etype, value, tback))) return -1 log.msg("Migration completed with success!") else: log.msg('Performing data update') migration.perform_data_update(os.path.abspath(os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION))) return db_version
def _on_headers(self, data): try: data = native_str(data.decode("latin1")) eol = data.find("\r\n") start_line = data[:eol] try: method, uri, version = start_line.split(" ") except ValueError: raise _BadRequestException("Malformed HTTP request line") if not version.startswith("HTTP/"): raise _BadRequestException( "Malformed HTTP version in HTTP Request-Line") headers = httputil.HTTPHeaders.parse(data[eol:]) self._request = HTTPRequest( connection=self, method=method, uri=uri, version=version, headers=headers, remote_ip=self._remote_ip) try: self.content_length = int(headers.get("Content-Length", 0)) except ValueError: raise _BadRequestException("Malformed Content-Length header") # we always use secure temporary files in case of large json or file uploads if self.content_length < 100000 and self._request.headers.get("Content-Disposition") is None: self._contentbuffer = StringIO('') else: self._contentbuffer = GLSecureTemporaryFile(GLSetting.tmp_upload_path) if headers.get("Expect") == "100-continue": self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n") c_d_header = self._request.headers.get("Content-Disposition") if c_d_header is not None: key, pdict = parse_header(c_d_header) if key != 'attachment' or 'filename' not in pdict: raise _BadRequestException("Malformed Content-Disposition header") self.file_upload = True self.uploaded_file['filename'] = pdict['filename'] self.uploaded_file['content_type'] = self._request.headers.get("Content-Type", 'application/octet-stream') self.uploaded_file['body'] = self._contentbuffer self.uploaded_file['body_len'] = int(self.content_length) self.uploaded_file['body_filepath'] = self._contentbuffer.filepath megabytes = int(self.content_length) / (1024 * 1024) if self.file_upload: limit_type = "upload" limit = GLSetting.memory_copy.maximum_filesize else: limit_type = "json" limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize # is 1MB probably too high. probably this variable must be in kB # less than 1 megabytes is always accepted if megabytes > limit: log.err("Tried %s request larger than expected (%dMb > %dMb)" % (limit_type, megabytes, limit)) # In HTTP Protocol errors need to be managed differently than handlers raise errors.HTTPRawLimitReach if self.content_length > 0: self.setRawMode() return elif self.file_upload: self._on_request_body(self.uploaded_file) self.file_upload = False self.uploaded_file = {} return self.request_callback(self._request) except Exception as exception: log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception)) log.exception(exception) if self._request: self._request.finish() if self.transport: self.transport.loseConnection()
def check_db_files(): """ This function checks the database version and executes eventually executes migration scripts """ db_files = [] max_version = 0 min_version = 0 for filename in os.listdir(GLSettings.db_path): if filename.startswith('glbackend'): filepath = os.path.join(GLSettings.db_path, filename) if filename.endswith('.db'): db_files.append(filepath) nameindex = filename.rfind('glbackend') extensindex = filename.rfind('.db') fileversion = int(filename[nameindex + len('glbackend-'):extensindex]) max_version = fileversion if fileversion > max_version else max_version min_version = fileversion if fileversion < min_version else min_version db_version = max_version if len(db_files) == 1 and db_version > 0: from globaleaks.db import migration log.msg("Found an already initialized database version: %d" % db_version) if db_version < DATABASE_VERSION: log.msg( "Performing update of database from version %d to version %d" % (db_version, DATABASE_VERSION)) try: migration.perform_version_update(db_version) log.msg("Migration completed with success!") except Exception as exception: log.msg("Migration failure: %s" % exception) log.msg("Verbose exception traceback:") etype, value, tback = sys.exc_info() log.msg('\n'.join( traceback.format_exception(etype, value, tback))) return -1 elif len(db_files) > 1: log.msg( "Error: Cannot start the application because more than one database file are present in: %s" % GLSettings.db_path) log.msg( "Manual check needed and is suggested to first make a backup of %s\n" % GLSettings.working_path) log.msg("Files found:") for f in db_files: log.msg("\t%s" % f) return -1 return db_version
def _on_headers(self, data): try: data = native_str(data.decode("latin1")) eol = data.find("\r\n") start_line = data[:eol] try: method, uri, version = start_line.split(" ") except ValueError: raise _BadRequestException("Malformed HTTP request line") if not version.startswith("HTTP/"): raise _BadRequestException("Malformed HTTP version in HTTP Request-Line") headers = httputil.HTTPHeaders.parse(data[eol:]) self._request = HTTPRequest( connection=self, method=method, uri=uri, version=version, headers=headers, remote_ip=self._remote_ip ) content_length = int(headers.get("Content-Length", 0)) self.content_length = content_length if content_length: if headers.get("Expect") == "100-continue": self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n") if content_length < 100000: self._contentbuffer = StringIO("") else: self._contentbuffer = TemporaryFile() c_d_header = self._request.headers.get("Content-Disposition") if c_d_header is not None: c_d_header = c_d_header.lower() m = content_disposition_re.match(c_d_header) if m is None: raise Exception self.file_upload = True self.uploaded_file["filename"] = m.group(1) self.uploaded_file["content_type"] = self._request.headers.get( "Content-Type", "application/octet-stream" ) self.uploaded_file["body"] = self._contentbuffer self.uploaded_file["body_len"] = int(content_length) megabytes = int(content_length) / (1024 * 1024) if self.file_upload: limit_type = "upload" limit = GLSetting.memory_copy.maximum_filesize else: limit_type = "json" limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize # is 1MB probably too high. probably this variable must be # in kB # less than 1 megabytes is always accepted if megabytes > limit: log.err("Tried %s request larger than expected (%dMb > %dMb)" % (limit_type, megabytes, limit)) # In HTTP Protocol errors need to be managed differently than handlers raise errors.HTTPRawLimitReach self.setRawMode() return self.request_callback(self._request) except Exception as exception: log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception)) log.exception(exception) if self._request: self._request.finish() if self.transport: self.transport.loseConnection()
def start_globaleaks(self): try: GLSettings.fix_file_permissions() GLSettings.drop_privileges() GLSettings.check_directories() # Check presence of an existing database and eventually perform its migration check = check_db_files() if check == -1: self._reactor.stop() elif check == 0: yield init_db() else: yield update_version() yield init_appdata() yield clean_untracked_files() yield refresh_memory_variables() if GLSettings.cmdline_options: yield apply_cmdline_options() self.start_asynchronous_jobs() log.msg("GLBackend is now running") for ip in GLSettings.bind_addresses: log.msg("Visit http://%s:%d to interact with me" % (ip, GLSettings.bind_port)) for host in GLSettings.accepted_hosts: if host not in GLSettings.bind_addresses: log.msg("Visit http://%s:%d to interact with me" % (host, GLSettings.bind_port)) for other in GLSettings.configured_hosts: if other: log.msg("Visit %s to interact with me" % other) log.msg( "Remind: GlobaLeaks is not accessible from other URLs, this is strictly enforced" ) log.msg( "Check documentation in https://github.com/globaleaks/GlobaLeaks/wiki/ for special enhancement" ) except Exception as excep: log.err( "ERROR: Cannot start GlobaLeaks; please manual check the error." ) log.err("EXCEPTION: %s" % excep) self._reactor.stop()
def perform_schema_migration(version): """ @param version: @return: """ to_delete_on_fail = [] to_delete_on_success = [] if version < FIRST_DATABASE_VERSION_SUPPORTED: log.msg( "Migrations from DB version lower than %d are no longer supported!" % FIRST_DATABASE_VERSION_SUPPORTED) quit() tmpdir = os.path.abspath(os.path.join(GLSettings.db_path, 'tmp')) orig_db_file = os.path.abspath( os.path.join(GLSettings.db_path, 'glbackend-%d.db' % version)) final_db_file = os.path.abspath( os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION)) shutil.rmtree(tmpdir, True) os.mkdir(tmpdir) shutil.copy2(orig_db_file, tmpdir) new_db_file = None try: while version < DATABASE_VERSION: old_db_file = os.path.abspath( os.path.join(tmpdir, 'glbackend-%d.db' % version)) new_db_file = os.path.abspath( os.path.join(tmpdir, 'glbackend-%d.db' % (version + 1))) GLSettings.db_file = new_db_file GLSettings.enable_input_length_checks = False to_delete_on_fail.append(new_db_file) to_delete_on_success.append(old_db_file) log.msg("Updating DB from version %d to version %d" % (version, version + 1)) store_old = Store(create_database('sqlite:' + old_db_file)) store_new = Store(create_database('sqlite:' + new_db_file)) # Here is instanced the migration script MigrationModule = importlib.import_module( "globaleaks.db.migrations.update_%d" % (version + 1)) migration_script = MigrationModule.MigrationScript( migration_mapping, version, store_old, store_new) log.msg("Migrating table:") try: try: migration_script.prologue() except Exception as exception: log.err("Failure while executing migration prologue: %s" % exception) raise exception for model_name, _ in migration_mapping.iteritems(): if migration_script.model_from[ model_name] is not None and migration_script.model_to[ model_name] is not None: try: migration_script.migrate_model(model_name) # Commit at every table migration in order to be able to detect # the precise migration that may fail. migration_script.commit() except Exception as exception: log.err("Failure while migrating table %s: %s " % (model_name, exception)) raise exception try: migration_script.epilogue() migration_script.commit() except Exception as exception: log.err("Failure while executing migration epilogue: %s " % exception) raise exception finally: # the database should be always closed before leaving the application # in order to not keep leaking journal files. migration_script.close() log.msg("Migration stats:") # we open a new db in order to verify integrity of the generated file store_verify = Store( create_database(GLSettings.make_db_uri(new_db_file))) for model_name, _ in migration_mapping.iteritems(): if model_name == 'ApplicationData': continue if migration_script.model_from[ model_name] is not None and migration_script.model_to[ model_name] is not None: count = store_verify.find( migration_script.model_to[model_name]).count() if migration_script.entries_count[model_name] != count: if migration_script.fail_on_count_mismatch[model_name]: raise AssertionError("Integrity check failed on count equality for table %s: %d != %d" % \ (model_name, count, migration_script.entries_count[model_name])) else: log.msg(" * %s table migrated (entries count changed from %d to %d)" % \ (model_name, migration_script.entries_count[model_name], count)) else: log.msg(" * %s table migrated (%d entry(s))" % \ (model_name, migration_script.entries_count[model_name])) version += 1 store_verify.close() perform_data_update(new_db_file) except Exception: raise else: # in case of success first copy the new migrated db, then as last action delete the original db file shutil.copy(new_db_file, final_db_file) security.overwrite_and_remove(orig_db_file) finally: # Always cleanup the temporary directory used for the migration for f in os.listdir(tmpdir): tmp_db_file = os.path.join(tmpdir, f) security.overwrite_and_remove(tmp_db_file) shutil.rmtree(tmpdir)
def apply_cmdline_options(store): """ Remind: GLSettings.unchecked_tor_input contain data that are not checked until this function! """ node = store.find(models.Node).one() verb = "Hardwriting" if 'hostname_tor_content' in GLSettings.unchecked_tor_input: composed_hs_url = 'http://%s' % GLSettings.unchecked_tor_input[ 'hostname_tor_content'] hs = GLSettings.unchecked_tor_input['hostname_tor_content'].split( '.onion')[0] composed_t2w_url = 'https://%s.tor2web.org' % hs if not (re.match(requests.hidden_service_regexp, composed_hs_url) or re.match(requests.https_url_regexp, composed_t2w_url)): log.msg("[!!] Invalid content found in the 'hostname' file specified (%s): Ignored" % \ GLSettings.unchecked_tor_input['hostname_tor_content']) else: node.hidden_service = unicode(composed_hs_url) log.msg("[+] %s hidden service in the DB: %s" % (verb, composed_hs_url)) if node.public_site: log.msg("[!!] Public Website (%s) is not automatically overwritten by (%s)" % \ (node.public_site, composed_t2w_url)) else: node.public_site = unicode(composed_t2w_url) log.msg("[+] %s public site in the DB: %s" % (verb, composed_t2w_url)) verb = "Overwriting" if GLSettings.cmdline_options.public_website: if not re.match(requests.https_url_regexp, GLSettings.cmdline_options.public_website): log.msg("[!!] Invalid public site: %s: Ignored" % GLSettings.cmdline_options.public_website) else: log.msg("[+] %s public site in the DB: %s" % (verb, GLSettings.cmdline_options.public_website)) node.public_site = unicode( GLSettings.cmdline_options.public_website) if GLSettings.cmdline_options.hidden_service: if not re.match(requests.hidden_service_regexp, GLSettings.cmdline_options.hidden_service): log.msg("[!!] Invalid hidden service: %s: Ignored" % GLSettings.cmdline_options.hidden_service) else: log.msg("[+] %s hidden service in the DB: %s" % (verb, GLSettings.cmdline_options.hidden_service)) node.hidden_service = unicode( GLSettings.cmdline_options.hidden_service) # return configured URL for the log/console output if node.hidden_service or node.public_site: GLSettings.configured_hosts = [node.hidden_service, node.public_site]
def check_db_files(): """ This function checks the database version and executes eventually executes migration scripts """ db_files = [] max_version = 0 min_version = 0 for filename in os.listdir(GLSettings.db_path): if filename.startswith('glbackend'): filepath = os.path.join(GLSettings.db_path, filename) if filename.endswith('.db'): db_files.append(filepath) nameindex = filename.rfind('glbackend') extensindex = filename.rfind('.db') fileversion = int(filename[nameindex + len('glbackend-'):extensindex]) max_version = fileversion if fileversion > max_version else max_version min_version = fileversion if fileversion < min_version else min_version db_version = max_version if len(db_files) == 1 and db_version > 0: from globaleaks.db import migration log.msg("Found an already initialized database version: %d" % db_version) if db_version < DATABASE_VERSION: log.msg("Performing update of database from version %d to version %d" % (db_version, DATABASE_VERSION)) try: migration.perform_version_update(db_version) log.msg("Migration completed with success!") except Exception as exception: log.msg("Migration failure: %s" % exception) log.msg("Verbose exception traceback:") etype, value, tback = sys.exc_info() log.msg('\n'.join(traceback.format_exception(etype, value, tback))) return -1 elif len(db_files) > 1: log.msg("Error: Cannot start the application because more than one database file are present in: %s" % GLSettings.db_path) log.msg("Manual check needed and is suggested to first make a backup of %s\n" % GLSettings.working_path) log.msg("Files found:") for f in db_files: log.msg("\t%s" % f) return -1 return db_version
def cb(res): start_asynchronous() yield import_memory_variables() tor_configured_hosts = yield apply_cli_options() log.msg("GLBackend is now running") for ip in GLSetting.bind_addresses: log.msg("Visit http://%s:%d to interact with me" % (ip, GLSetting.bind_port)) for host in GLSetting.accepted_hosts: if host not in GLSetting.bind_addresses: log.msg("Visit http://%s:%d to interact with me" % (host, GLSetting.bind_port)) if tor_configured_hosts: for other in tor_configured_hosts: if other: log.msg("Visit %s to interact with me" % other) log.msg( "Remind: GlobaLeaks is not accessible from other URLs, this is strictly enforced" ) log.msg( "Check documentation in https://github.com/globaleaks/GlobaLeaks/wiki/ for special enhancement" )
def perform_system_update(): """ This function checks the system and database versions and executes migration routines based on the system's state. After this function has completed the node is either ready for initialization (0), running a version of the DB (>1), or broken (-1). """ from globaleaks.db import migration db_files = [] max_version = 0 min_version = 0 for filename in os.listdir(GLSettings.db_path): if filename.startswith('glbackend'): filepath = os.path.join(GLSettings.db_path, filename) if filename.endswith('.db'): db_files.append(filepath) nameindex = filename.rfind('glbackend') extensindex = filename.rfind('.db') fileversion = int(filename[nameindex + len('glbackend-'):extensindex]) max_version = fileversion if fileversion > max_version else max_version min_version = fileversion if fileversion < min_version else min_version db_version = max_version if len(db_files) > 1: log.msg( "Error: Cannot start the application because more than one database file are present in: %s" % GLSettings.db_path) log.msg( "Manual check needed and is suggested to first make a backup of %s\n" % GLSettings.working_path) log.msg("Files found:") for f in db_files: log.msg("\t%s" % f) return -1 if len(db_files) == 1: log.msg("Found an already initialized database version: %d" % db_version) if db_version < DATABASE_VERSION: log.msg( "Performing schema migration from version %d to version %d" % (db_version, DATABASE_VERSION)) try: migration.perform_schema_migration(db_version) except Exception as exception: log.msg("Migration failure: %s" % exception) log.msg("Verbose exception traceback:") etype, value, tback = sys.exc_info() log.msg('\n'.join( traceback.format_exception(etype, value, tback))) return -1 log.msg("Migration completed with success!") else: log.msg('Performing data update') migration.perform_data_update( os.path.abspath( os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION))) return db_version
def cb(res): start_asynchronous() yield import_memory_variables() tor_configured_hosts = yield apply_cli_options() log.msg("GLBackend is now running") for ip in GLSetting.bind_addresses: log.msg("Visit http://%s:%d to interact with me" % (ip, GLSetting.bind_port)) for host in GLSetting.accepted_hosts: if host not in GLSetting.bind_addresses: log.msg("Visit http://%s:%d to interact with me" % (host, GLSetting.bind_port)) if tor_configured_hosts: for other in tor_configured_hosts: if other: log.msg("Visit %s to interact with me" % other) log.msg("Remind: GlobaLeaks is not accessible from other URLs, this is strictly enforced") log.msg("Check documentation in https://github.com/globaleaks/GlobaLeaks/wiki/ for special enhancement")
def update_db(): """ This function handles update of an existing database """ try: db_version, db_file_path = get_db_file(GLSettings.db_path) if db_version == 0: return 0 from globaleaks.db import migration log.msg("Found an already initialized database version: %d" % db_version) if db_version >= FIRST_DATABASE_VERSION_SUPPORTED and db_version < DATABASE_VERSION: log.msg( "Performing schema migration from version %d to version %d" % (db_version, DATABASE_VERSION)) migration.perform_schema_migration(db_version) log.msg("Migration completed with success!") else: log.msg('Performing data update') # TODO on normal startup this line is run. We need better control flow here. migration.perform_data_update( os.path.abspath( os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION))) except Exception as exception: log.msg("Migration failure: %s" % exception) log.msg("Verbose exception traceback:") etype, value, tback = sys.exc_info() log.msg('\n'.join(traceback.format_exception(etype, value, tback))) return -1