def update_notification(store, request, language=GLSetting.memory_copy.default_language):
try:
notif = store.find(Notification).one()
except Exception as excep:
log.err("Database error or application error: %s" % excep )
raise excep
v = dict(request)
for attr in getattr(notif, "localized_strings"):
v[attr] = getattr(notif, attr)
v[attr][language] = unicode(request[attr])
request = v
if request['security'] in Notification._security_types:
notif.security = request['security']
else:
log.err("Invalid request: Security option not recognized")
log.debug("Invalid Security value: %s" % request['security'])
raise errors.InvalidInputFormat("Security selection not recognized")
notif.update(request)
if request['disable'] != GLSetting.notification_temporary_disable:
log.msg("Switching notification mode: was %s and now is %s" %
("DISABLE" if GLSetting.notification_temporary_disable else "ENABLE",
"DISABLE" if request['disable'] else "ENABLE")
)
GLSetting.notification_temporary_disable = request['disable']
return admin_serialize_notification(notif, language)
def _on_request_body(self, data):
try:
self._request.body = data
content_type = self._request.headers.get("Content-Type", "")
if self._request.method in ("POST", "PATCH", "PUT"):
if content_type.startswith(
"application/x-www-form-urlencoded"
) and self.content_length < GLSetting.www_form_urlencoded_maximum_size:
arguments = parse_qs_bytes(native_str(self._request.body))
for name, values in arguments.iteritems():
values = [v for v in values if v]
if values:
self._request.arguments.setdefault(
name, []).extend(values)
elif content_type.startswith(
"application/x-www-form-urlencoded"):
raise errors.InvalidInputFormat(
"content type application/x-www-form-urlencoded not supported"
)
elif content_type.startswith("multipart/form-data"):
raise errors.InvalidInputFormat(
"content type multipart/form-data not supported")
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" %
(self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
def update_notification(store, request, language=GLSetting.memory_copy.default_language):
try:
notif = store.find(Notification).one()
except Exception as excep:
log.err("Database error or application error: %s" % excep )
raise excep
fill_localized_keys(request, Notification.localized_strings, language)
if request['security'] in Notification._security_types:
notif.security = request['security']
else:
log.err("Invalid request: Security option not recognized")
log.debug("Invalid Security value: %s" % request['security'])
raise errors.InvalidInputFormat("Security selection not recognized")
try:
notif.update(request)
except DatabaseError as dberror:
log.err("Unable to update Notification: %s" % dberror)
raise errors.InvalidInputFormat(dberror)
if request['disable'] != GLSetting.notification_temporary_disable:
log.msg("Switching notification mode: was %s and now is %s" %
("DISABLE" if GLSetting.notification_temporary_disable else "ENABLE",
"DISABLE" if request['disable'] else "ENABLE")
)
GLSetting.notification_temporary_disable = request['disable']
return admin_serialize_notification(notif, language)
def update_notification(store,
request,
language=GLSetting.memory_copy.default_language):
try:
notif = store.find(Notification).one()
except Exception as excep:
log.err("Database error or application error: %s" % excep)
raise excep
mo = structures.Rosetta()
mo.acquire_request(language, request, Notification)
for attr in mo.get_localized_attrs():
request[attr] = mo.get_localized_dict(attr)
if request['security'] in Notification._security_types:
notif.security = request['security']
else:
log.err("Invalid request: Security option not recognized")
log.debug("Invalid Security value: %s" % request['security'])
raise errors.InvalidInputFormat("Security selection not recognized")
try:
notif.update(request)
except Exception as dberror:
log.err("Unable to update Notification: %s" % dberror)
raise errors.InvalidInputFormat(dberror)
if request['disable'] != GLSetting.notification_temporary_disable:
log.msg("Switching notification mode: was %s and now is %s" %
("DISABLE" if GLSetting.notification_temporary_disable else
"ENABLE", "DISABLE" if request['disable'] else "ENABLE"))
GLSetting.notification_temporary_disable = request['disable']
return admin_serialize_notification(notif, language)
def _on_headers(self, data):
try:
data = native_str(data.decode("latin1"))
eol = data.find("\r\n")
start_line = data[:eol]
try:
method, uri, version = start_line.split(" ")
except ValueError:
raise _BadRequestException("Malformed HTTP request line")
if not version.startswith("HTTP/"):
raise _BadRequestException(
"Malformed HTTP version in HTTP Request-Line")
try:
headers = httputil.HTTPHeaders.parse(data[eol:])
content_length = int(headers.get("Content-Length", 0))
except ValueError:
raise _BadRequestException("Malformed Content-Length header")
self._request = HTTPRequest(connection=self,
method=method,
uri=uri,
version=version,
headers=headers,
remote_ip=self._remote_ip)
if content_length:
megabytes = int(content_length) / (1024 * 1024)
if megabytes > GLSettings.memory_copy.maximum_filesize:
raise _BadRequestException(
"Request exceeded size limit %d" %
GLSettings.memory_copy.maximum_filesize)
if headers.get("Expect") == "100-continue":
self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")
if content_length < 100000:
self._contentbuffer = StringIO()
else:
self._contentbuffer = GLSecureTemporaryFile(
GLSettings.tmp_upload_path)
self.content_length = content_length
self.setRawMode()
return
self.request_callback(self._request)
except _BadRequestException as e:
log.msg("Exception while handling HTTP request from %s: %s" %
(self._remote_ip, e))
self.transport.loseConnection()
def _on_headers(self, data):
try:
data = native_str(data.decode("latin1"))
eol = data.find("\r\n")
start_line = data[:eol]
try:
method, uri, version = start_line.split(" ")
except ValueError:
raise _BadRequestException("Malformed HTTP request line")
if not version.startswith("HTTP/"):
raise _BadRequestException(
"Malformed HTTP version in HTTP Request-Line")
try:
headers = httputil.HTTPHeaders.parse(data[eol:])
content_length = int(headers.get("Content-Length", 0))
except ValueError:
raise _BadRequestException(
"Malformed Content-Length header")
self._request = HTTPRequest(
connection=self, method=method, uri=uri, version=version,
headers=headers, remote_ip=self._remote_ip)
if content_length:
megabytes = int(content_length) / (1024 * 1024)
if megabytes > GLSettings.memory_copy.maximum_filesize:
raise _BadRequestException("Request exceeded size limit %d" %
GLSettings.memory_copy.maximum_filesize)
if headers.get("Expect") == "100-continue":
self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")
if content_length < 100000:
self._contentbuffer = StringIO()
else:
self._contentbuffer = GLSecureTemporaryFile(GLSettings.tmp_upload_path)
self.content_length = content_length
self.setRawMode()
return
self.request_callback(self._request)
except _BadRequestException as e:
log.msg("Exception while handling HTTP request from %s: %s" % (self._remote_ip, e))
self.transport.loseConnection()
def _on_request_body(self, data):
try:
self._request.body = data
content_type = self._request.headers.get("Content-Type", "")
if self._request.method in ("POST", "PATCH", "PUT"):
if content_type.startswith("application/x-www-form-urlencoded"):
raise errors.InvalidInputFormat("content type application/x-www-form-urlencoded not supported")
elif content_type.startswith("multipart/form-data"):
raise errors.InvalidInputFormat("content type multipart/form-data not supported")
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
def _handle_request_exception(self, e):
if isinstance(e, Failure):
exc_type = e.type
exc_value = e.value
exc_tb = e.getTracebackObject()
e = e.value
else:
exc_type, exc_value, exc_tb = sys.exc_info()
if isinstance(e, (HTTPError, HTTPAuthenticationRequired)):
if GLSettings.log_requests_responses and e.log_message:
string_format = "%d %s: " + e.log_message
args = [e.status_code, self._request_summary()] + list(e.args)
msg = lambda *args: string_format % args
log.msg(msg(*args))
if e.status_code not in httplib.responses:
log.msg("Bad HTTP status code: %d" % e.status_code)
return self.send_error(500, exception=e)
else:
return self.send_error(e.status_code, exception=e)
else:
log.err("Uncaught exception %s %s %s" %
(exc_type, exc_value, exc_tb))
if GLSettings.log_requests_responses:
log.msg(e)
mail_exception_handler(exc_type, exc_value, exc_tb)
return self.send_error(500, exception=e)
def _handle_request_exception(self, e):
if isinstance(e, Failure):
exc_type = e.type
exc_value = e.value
exc_tb = e.getTracebackObject()
e = e.value
else:
exc_type, exc_value, exc_tb = sys.exc_info()
if isinstance(e, (HTTPError, HTTPAuthenticationRequired)):
if GLSettings.log_requests_responses and e.log_message:
string_format = "%d %s: " + e.log_message
args = [e.status_code, self._request_summary()] + list(e.args)
msg = lambda *args: string_format % args
log.msg(msg(*args))
if e.status_code not in httplib.responses:
log.msg("Bad HTTP status code: %d" % e.status_code)
return self.send_error(500, exception=e)
else:
return self.send_error(e.status_code, exception=e)
else:
log.err("Uncaught exception %s %s %s" % (exc_type, exc_value, exc_tb))
if GLSettings.log_requests_responses:
log.msg(e)
mail_exception_handler(exc_type, exc_value, exc_tb)
return self.send_error(500, exception=e)
def _on_request_body(self, data):
try:
self._request.body = data
content_type = self._request.headers.get("Content-Type", "")
if self._request.method in ("POST", "PATCH", "PUT"):
if content_type.startswith("application/x-www-form-urlencoded") and self.content_length < GLSetting.www_form_urlencoded_maximum_size:
arguments = parse_qs_bytes(native_str(self._request.body))
for name, values in arguments.iteritems():
values = [v for v in values if v]
if values:
self._request.arguments.setdefault(name,
[]).extend(values)
elif content_type.startswith("application/x-www-form-urlencoded"):
raise errors.InvalidInputFormat("content type application/x-www-form-urlencoded not supported")
elif content_type.startswith("multipart/form-data"):
raise errors.InvalidInputFormat("content type multipart/form-data not supported")
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
def cb(res):
start_asynchronous()
yield import_memory_variables()
yield apply_cli_options()
log.msg("GLBackend is now running")
for ip in GLSetting.bind_addresses:
log.msg("Visit http://%s:%d to interact with me" % (ip, GLSetting.bind_port))
for host in GLSetting.accepted_hosts:
if host not in GLSetting.bind_addresses:
log.msg("Visit http://%s:%d to interact with me" % (host, GLSetting.bind_port))
def apply_cmdline_options(store):
"""
Remind: GLSettings.unchecked_tor_input contain data that are not
checked until this function!
"""
node = store.find(models.Node).one()
verb = "Hardwriting"
if 'hostname_tor_content' in GLSettings.unchecked_tor_input:
composed_hs_url = 'http://%s' % GLSettings.unchecked_tor_input['hostname_tor_content']
hs = GLSettings.unchecked_tor_input['hostname_tor_content'].split('.onion')[0]
composed_t2w_url = 'https://%s.tor2web.org' % hs
if not (re.match(requests.hidden_service_regexp, composed_hs_url) or
re.match(requests.https_url_regexp, composed_t2w_url)):
log.msg("[!!] Invalid content found in the 'hostname' file specified (%s): Ignored" % \
GLSettings.unchecked_tor_input['hostname_tor_content'])
else:
node.hidden_service = unicode(composed_hs_url)
log.msg("[+] %s hidden service in the DB: %s" % (verb, composed_hs_url))
if node.public_site:
log.msg("[!!] Public Website (%s) is not automatically overwritten by (%s)" % \
(node.public_site, composed_t2w_url))
else:
node.public_site = unicode(composed_t2w_url)
log_msg("[+] %s public site in the DB: %s" % (verb, composed_t2w_url))
verb = "Overwriting"
if GLSettings.cmdline_options.public_website:
if not re.match(requests.https_url_regexp, GLSettings.cmdline_options.public_website):
log.msg("[!!] Invalid public site: %s: Ignored" % GLSettings.cmdline_options.public_website)
else:
log.msg("[+] %s public site in the DB: %s" % (verb, GLSettings.cmdline_options.public_website))
node.public_site = unicode(GLSettings.cmdline_options.public_website)
if GLSettings.cmdline_options.hidden_service:
if not re.match(requests.hidden_service_regexp, GLSettings.cmdline_options.hidden_service):
log.msg("[!!] Invalid hidden service: %s: Ignored" % GLSettings.cmdline_options.hidden_service)
else:
log.msg("[+] %s hidden service in the DB: %s" % (verb, GLSettings.cmdline_options.hidden_service))
node.hidden_service = unicode(GLSettings.cmdline_options.hidden_service)
# return configured URL for the log/console output
if node.hidden_service or node.public_site:
GLSettings.configured_hosts = [node.hidden_service, node.public_site]
def start_globaleaks(self):
try:
GLSettings.fix_file_permissions()
GLSettings.drop_privileges()
GLSettings.check_directories()
# Check presence of an existing database and eventually perform its migration
check = check_db_files()
if check == -1:
self._reactor.stop()
elif check == 0:
yield init_db()
else:
yield update_version()
yield init_appdata()
yield clean_untracked_files()
yield refresh_memory_variables()
if GLSettings.cmdline_options:
yield apply_cmdline_options()
self.start_asynchronous_jobs()
log.msg("GLBackend is now running")
for ip in GLSettings.bind_addresses:
log.msg("Visit http://%s:%d to interact with me" % (ip, GLSettings.bind_port))
for host in GLSettings.accepted_hosts:
if host not in GLSettings.bind_addresses:
log.msg("Visit http://%s:%d to interact with me" % (host, GLSettings.bind_port))
for other in GLSettings.configured_hosts:
if other:
log.msg("Visit %s to interact with me" % other)
log.msg("Remind: GlobaLeaks is not accessible from other URLs, this is strictly enforced")
log.msg("Check documentation in https://github.com/globaleaks/GlobaLeaks/wiki/ for special enhancement")
except Exception as excep:
log.err("ERROR: Cannot start GlobaLeaks; please manual check the error.")
log.err("EXCEPTION: %s" % excep)
self._reactor.stop()
def perform_system_update():
"""
This function checks the system and database versions and executes migration
routines based on the system's state. After this function has completed the
node is either ready for initialization (0), running a version of the DB
(>1), or broken (-1).
"""
from globaleaks.db import migration
db_files = []
max_version = 0
min_version = 0
for filename in os.listdir(GLSettings.db_path):
if filename.startswith('glbackend'):
filepath = os.path.join(GLSettings.db_path, filename)
if filename.endswith('.db'):
db_files.append(filepath)
nameindex = filename.rfind('glbackend')
extensindex = filename.rfind('.db')
fileversion = int(filename[nameindex + len('glbackend-'):extensindex])
max_version = fileversion if fileversion > max_version else max_version
min_version = fileversion if fileversion < min_version else min_version
db_version = max_version
if len(db_files) > 1:
log.msg("Error: Cannot start the application because more than one database file are present in: %s" % GLSettings.db_path)
log.msg("Manual check needed and is suggested to first make a backup of %s\n" % GLSettings.working_path)
log.msg("Files found:")
for f in db_files:
log.msg("\t%s" % f)
return -1
if len(db_files) == 1:
log.msg("Found an already initialized database version: %d" % db_version)
if db_version < DATABASE_VERSION:
log.msg("Performing schema migration from version %d to version %d" % (db_version, DATABASE_VERSION))
try:
migration.perform_schema_migration(db_version)
except Exception as exception:
log.msg("Migration failure: %s" % exception)
log.msg("Verbose exception traceback:")
etype, value, tback = sys.exc_info()
log.msg('\n'.join(traceback.format_exception(etype, value, tback)))
return -1
log.msg("Migration completed with success!")
else:
log.msg('Performing data update')
migration.perform_data_update(os.path.abspath(os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION)))
return db_version
def _on_headers(self, data):
try:
data = native_str(data.decode("latin1"))
eol = data.find("\r\n")
start_line = data[:eol]
try:
method, uri, version = start_line.split(" ")
except ValueError:
raise _BadRequestException("Malformed HTTP request line")
if not version.startswith("HTTP/"):
raise _BadRequestException(
"Malformed HTTP version in HTTP Request-Line")
headers = httputil.HTTPHeaders.parse(data[eol:])
self._request = HTTPRequest(
connection=self, method=method, uri=uri, version=version,
headers=headers, remote_ip=self._remote_ip)
try:
self.content_length = int(headers.get("Content-Length", 0))
except ValueError:
raise _BadRequestException("Malformed Content-Length header")
# we always use secure temporary files in case of large json or file uploads
if self.content_length < 100000 and self._request.headers.get("Content-Disposition") is None:
self._contentbuffer = StringIO('')
else:
self._contentbuffer = GLSecureTemporaryFile(GLSetting.tmp_upload_path)
if headers.get("Expect") == "100-continue":
self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")
c_d_header = self._request.headers.get("Content-Disposition")
if c_d_header is not None:
key, pdict = parse_header(c_d_header)
if key != 'attachment' or 'filename' not in pdict:
raise _BadRequestException("Malformed Content-Disposition header")
self.file_upload = True
self.uploaded_file['filename'] = pdict['filename']
self.uploaded_file['content_type'] = self._request.headers.get("Content-Type",
'application/octet-stream')
self.uploaded_file['body'] = self._contentbuffer
self.uploaded_file['body_len'] = int(self.content_length)
self.uploaded_file['body_filepath'] = self._contentbuffer.filepath
megabytes = int(self.content_length) / (1024 * 1024)
if self.file_upload:
limit_type = "upload"
limit = GLSetting.memory_copy.maximum_filesize
else:
limit_type = "json"
limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize
# is 1MB probably too high. probably this variable must be in kB
# less than 1 megabytes is always accepted
if megabytes > limit:
log.err("Tried %s request larger than expected (%dMb > %dMb)" %
(limit_type,
megabytes,
limit))
# In HTTP Protocol errors need to be managed differently than handlers
raise errors.HTTPRawLimitReach
if self.content_length > 0:
self.setRawMode()
return
elif self.file_upload:
self._on_request_body(self.uploaded_file)
self.file_upload = False
self.uploaded_file = {}
return
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
def check_db_files():
"""
This function checks the database version and executes eventually
executes migration scripts
"""
db_files = []
max_version = 0
min_version = 0
for filename in os.listdir(GLSettings.db_path):
if filename.startswith('glbackend'):
filepath = os.path.join(GLSettings.db_path, filename)
if filename.endswith('.db'):
db_files.append(filepath)
nameindex = filename.rfind('glbackend')
extensindex = filename.rfind('.db')
fileversion = int(filename[nameindex +
len('glbackend-'):extensindex])
max_version = fileversion if fileversion > max_version else max_version
min_version = fileversion if fileversion < min_version else min_version
db_version = max_version
if len(db_files) == 1 and db_version > 0:
from globaleaks.db import migration
log.msg("Found an already initialized database version: %d" %
db_version)
if db_version < DATABASE_VERSION:
log.msg(
"Performing update of database from version %d to version %d" %
(db_version, DATABASE_VERSION))
try:
migration.perform_version_update(db_version)
log.msg("Migration completed with success!")
except Exception as exception:
log.msg("Migration failure: %s" % exception)
log.msg("Verbose exception traceback:")
etype, value, tback = sys.exc_info()
log.msg('\n'.join(
traceback.format_exception(etype, value, tback)))
return -1
elif len(db_files) > 1:
log.msg(
"Error: Cannot start the application because more than one database file are present in: %s"
% GLSettings.db_path)
log.msg(
"Manual check needed and is suggested to first make a backup of %s\n"
% GLSettings.working_path)
log.msg("Files found:")
for f in db_files:
log.msg("\t%s" % f)
return -1
return db_version
def _on_headers(self, data):
try:
data = native_str(data.decode("latin1"))
eol = data.find("\r\n")
start_line = data[:eol]
try:
method, uri, version = start_line.split(" ")
except ValueError:
raise _BadRequestException("Malformed HTTP request line")
if not version.startswith("HTTP/"):
raise _BadRequestException("Malformed HTTP version in HTTP Request-Line")
headers = httputil.HTTPHeaders.parse(data[eol:])
self._request = HTTPRequest(
connection=self, method=method, uri=uri, version=version, headers=headers, remote_ip=self._remote_ip
)
content_length = int(headers.get("Content-Length", 0))
self.content_length = content_length
if content_length:
if headers.get("Expect") == "100-continue":
self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")
if content_length < 100000:
self._contentbuffer = StringIO("")
else:
self._contentbuffer = TemporaryFile()
c_d_header = self._request.headers.get("Content-Disposition")
if c_d_header is not None:
c_d_header = c_d_header.lower()
m = content_disposition_re.match(c_d_header)
if m is None:
raise Exception
self.file_upload = True
self.uploaded_file["filename"] = m.group(1)
self.uploaded_file["content_type"] = self._request.headers.get(
"Content-Type", "application/octet-stream"
)
self.uploaded_file["body"] = self._contentbuffer
self.uploaded_file["body_len"] = int(content_length)
megabytes = int(content_length) / (1024 * 1024)
if self.file_upload:
limit_type = "upload"
limit = GLSetting.memory_copy.maximum_filesize
else:
limit_type = "json"
limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize
# is 1MB probably too high. probably this variable must be
# in kB
# less than 1 megabytes is always accepted
if megabytes > limit:
log.err("Tried %s request larger than expected (%dMb > %dMb)" % (limit_type, megabytes, limit))
# In HTTP Protocol errors need to be managed differently than handlers
raise errors.HTTPRawLimitReach
self.setRawMode()
return
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
def start_globaleaks(self):
try:
GLSettings.fix_file_permissions()
GLSettings.drop_privileges()
GLSettings.check_directories()
# Check presence of an existing database and eventually perform its migration
check = check_db_files()
if check == -1:
self._reactor.stop()
elif check == 0:
yield init_db()
else:
yield update_version()
yield init_appdata()
yield clean_untracked_files()
yield refresh_memory_variables()
if GLSettings.cmdline_options:
yield apply_cmdline_options()
self.start_asynchronous_jobs()
log.msg("GLBackend is now running")
for ip in GLSettings.bind_addresses:
log.msg("Visit http://%s:%d to interact with me" %
(ip, GLSettings.bind_port))
for host in GLSettings.accepted_hosts:
if host not in GLSettings.bind_addresses:
log.msg("Visit http://%s:%d to interact with me" %
(host, GLSettings.bind_port))
for other in GLSettings.configured_hosts:
if other:
log.msg("Visit %s to interact with me" % other)
log.msg(
"Remind: GlobaLeaks is not accessible from other URLs, this is strictly enforced"
)
log.msg(
"Check documentation in https://github.com/globaleaks/GlobaLeaks/wiki/ for special enhancement"
)
except Exception as excep:
log.err(
"ERROR: Cannot start GlobaLeaks; please manual check the error."
)
log.err("EXCEPTION: %s" % excep)
self._reactor.stop()
def perform_schema_migration(version):
"""
@param version:
@return:
"""
to_delete_on_fail = []
to_delete_on_success = []
if version < FIRST_DATABASE_VERSION_SUPPORTED:
log.msg(
"Migrations from DB version lower than %d are no longer supported!"
% FIRST_DATABASE_VERSION_SUPPORTED)
quit()
tmpdir = os.path.abspath(os.path.join(GLSettings.db_path, 'tmp'))
orig_db_file = os.path.abspath(
os.path.join(GLSettings.db_path, 'glbackend-%d.db' % version))
final_db_file = os.path.abspath(
os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION))
shutil.rmtree(tmpdir, True)
os.mkdir(tmpdir)
shutil.copy2(orig_db_file, tmpdir)
new_db_file = None
try:
while version < DATABASE_VERSION:
old_db_file = os.path.abspath(
os.path.join(tmpdir, 'glbackend-%d.db' % version))
new_db_file = os.path.abspath(
os.path.join(tmpdir, 'glbackend-%d.db' % (version + 1)))
GLSettings.db_file = new_db_file
GLSettings.enable_input_length_checks = False
to_delete_on_fail.append(new_db_file)
to_delete_on_success.append(old_db_file)
log.msg("Updating DB from version %d to version %d" %
(version, version + 1))
store_old = Store(create_database('sqlite:' + old_db_file))
store_new = Store(create_database('sqlite:' + new_db_file))
# Here is instanced the migration script
MigrationModule = importlib.import_module(
"globaleaks.db.migrations.update_%d" % (version + 1))
migration_script = MigrationModule.MigrationScript(
migration_mapping, version, store_old, store_new)
log.msg("Migrating table:")
try:
try:
migration_script.prologue()
except Exception as exception:
log.err("Failure while executing migration prologue: %s" %
exception)
raise exception
for model_name, _ in migration_mapping.iteritems():
if migration_script.model_from[
model_name] is not None and migration_script.model_to[
model_name] is not None:
try:
migration_script.migrate_model(model_name)
# Commit at every table migration in order to be able to detect
# the precise migration that may fail.
migration_script.commit()
except Exception as exception:
log.err("Failure while migrating table %s: %s " %
(model_name, exception))
raise exception
try:
migration_script.epilogue()
migration_script.commit()
except Exception as exception:
log.err("Failure while executing migration epilogue: %s " %
exception)
raise exception
finally:
# the database should be always closed before leaving the application
# in order to not keep leaking journal files.
migration_script.close()
log.msg("Migration stats:")
# we open a new db in order to verify integrity of the generated file
store_verify = Store(
create_database(GLSettings.make_db_uri(new_db_file)))
for model_name, _ in migration_mapping.iteritems():
if model_name == 'ApplicationData':
continue
if migration_script.model_from[
model_name] is not None and migration_script.model_to[
model_name] is not None:
count = store_verify.find(
migration_script.model_to[model_name]).count()
if migration_script.entries_count[model_name] != count:
if migration_script.fail_on_count_mismatch[model_name]:
raise AssertionError("Integrity check failed on count equality for table %s: %d != %d" % \
(model_name, count, migration_script.entries_count[model_name]))
else:
log.msg(" * %s table migrated (entries count changed from %d to %d)" % \
(model_name, migration_script.entries_count[model_name], count))
else:
log.msg(" * %s table migrated (%d entry(s))" % \
(model_name, migration_script.entries_count[model_name]))
version += 1
store_verify.close()
perform_data_update(new_db_file)
except Exception:
raise
else:
# in case of success first copy the new migrated db, then as last action delete the original db file
shutil.copy(new_db_file, final_db_file)
security.overwrite_and_remove(orig_db_file)
finally:
# Always cleanup the temporary directory used for the migration
for f in os.listdir(tmpdir):
tmp_db_file = os.path.join(tmpdir, f)
security.overwrite_and_remove(tmp_db_file)
shutil.rmtree(tmpdir)
def _on_headers(self, data):
try:
data = native_str(data.decode("latin1"))
eol = data.find("\r\n")
start_line = data[:eol]
try:
method, uri, version = start_line.split(" ")
except ValueError:
raise _BadRequestException("Malformed HTTP request line")
if not version.startswith("HTTP/"):
raise _BadRequestException(
"Malformed HTTP version in HTTP Request-Line")
headers = httputil.HTTPHeaders.parse(data[eol:])
self._request = HTTPRequest(
connection=self, method=method, uri=uri, version=version,
headers=headers, remote_ip=self._remote_ip)
try:
self.content_length = int(headers.get("Content-Length", 0))
except ValueError:
raise _BadRequestException("Malformed Content-Length header")
# we always use secure temporary files in case of large json or file uploads
if self.content_length < 100000 and self._request.headers.get("Content-Disposition") is None:
self._contentbuffer = StringIO('')
else:
self._contentbuffer = GLSecureTemporaryFile(GLSetting.tmp_upload_path)
if headers.get("Expect") == "100-continue":
self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")
c_d_header = self._request.headers.get("Content-Disposition")
if c_d_header is not None:
key, pdict = parse_header(c_d_header)
if key != 'attachment' or 'filename' not in pdict:
raise _BadRequestException("Malformed Content-Disposition header")
self.file_upload = True
self.uploaded_file['filename'] = pdict['filename']
self.uploaded_file['content_type'] = self._request.headers.get("Content-Type",
'application/octet-stream')
self.uploaded_file['body'] = self._contentbuffer
self.uploaded_file['body_len'] = int(self.content_length)
self.uploaded_file['body_filepath'] = self._contentbuffer.filepath
megabytes = int(self.content_length) / (1024 * 1024)
if self.file_upload:
limit_type = "upload"
limit = GLSetting.memory_copy.maximum_filesize
else:
limit_type = "json"
limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize
# is 1MB probably too high. probably this variable must be in kB
# less than 1 megabytes is always accepted
if megabytes > limit:
log.err("Tried %s request larger than expected (%dMb > %dMb)" %
(limit_type,
megabytes,
limit))
# In HTTP Protocol errors need to be managed differently than handlers
raise errors.HTTPRawLimitReach
if self.content_length > 0:
self.setRawMode()
return
elif self.file_upload:
self._on_request_body(self.uploaded_file)
self.file_upload = False
self.uploaded_file = {}
return
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
def apply_cmdline_options(store):
"""
Remind: GLSettings.unchecked_tor_input contain data that are not
checked until this function!
"""
node = store.find(models.Node).one()
verb = "Hardwriting"
if 'hostname_tor_content' in GLSettings.unchecked_tor_input:
composed_hs_url = 'http://%s' % GLSettings.unchecked_tor_input[
'hostname_tor_content']
hs = GLSettings.unchecked_tor_input['hostname_tor_content'].split(
'.onion')[0]
composed_t2w_url = 'https://%s.tor2web.org' % hs
if not (re.match(requests.hidden_service_regexp, composed_hs_url)
or re.match(requests.https_url_regexp, composed_t2w_url)):
log.msg("[!!] Invalid content found in the 'hostname' file specified (%s): Ignored" % \
GLSettings.unchecked_tor_input['hostname_tor_content'])
else:
node.hidden_service = unicode(composed_hs_url)
log.msg("[+] %s hidden service in the DB: %s" %
(verb, composed_hs_url))
if node.public_site:
log.msg("[!!] Public Website (%s) is not automatically overwritten by (%s)" % \
(node.public_site, composed_t2w_url))
else:
node.public_site = unicode(composed_t2w_url)
log.msg("[+] %s public site in the DB: %s" %
(verb, composed_t2w_url))
verb = "Overwriting"
if GLSettings.cmdline_options.public_website:
if not re.match(requests.https_url_regexp,
GLSettings.cmdline_options.public_website):
log.msg("[!!] Invalid public site: %s: Ignored" %
GLSettings.cmdline_options.public_website)
else:
log.msg("[+] %s public site in the DB: %s" %
(verb, GLSettings.cmdline_options.public_website))
node.public_site = unicode(
GLSettings.cmdline_options.public_website)
if GLSettings.cmdline_options.hidden_service:
if not re.match(requests.hidden_service_regexp,
GLSettings.cmdline_options.hidden_service):
log.msg("[!!] Invalid hidden service: %s: Ignored" %
GLSettings.cmdline_options.hidden_service)
else:
log.msg("[+] %s hidden service in the DB: %s" %
(verb, GLSettings.cmdline_options.hidden_service))
node.hidden_service = unicode(
GLSettings.cmdline_options.hidden_service)
# return configured URL for the log/console output
if node.hidden_service or node.public_site:
GLSettings.configured_hosts = [node.hidden_service, node.public_site]
def check_db_files():
"""
This function checks the database version and executes eventually
executes migration scripts
"""
db_files = []
max_version = 0
min_version = 0
for filename in os.listdir(GLSettings.db_path):
if filename.startswith('glbackend'):
filepath = os.path.join(GLSettings.db_path, filename)
if filename.endswith('.db'):
db_files.append(filepath)
nameindex = filename.rfind('glbackend')
extensindex = filename.rfind('.db')
fileversion = int(filename[nameindex + len('glbackend-'):extensindex])
max_version = fileversion if fileversion > max_version else max_version
min_version = fileversion if fileversion < min_version else min_version
db_version = max_version
if len(db_files) == 1 and db_version > 0:
from globaleaks.db import migration
log.msg("Found an already initialized database version: %d" % db_version)
if db_version < DATABASE_VERSION:
log.msg("Performing update of database from version %d to version %d" % (db_version, DATABASE_VERSION))
try:
migration.perform_version_update(db_version)
log.msg("Migration completed with success!")
except Exception as exception:
log.msg("Migration failure: %s" % exception)
log.msg("Verbose exception traceback:")
etype, value, tback = sys.exc_info()
log.msg('\n'.join(traceback.format_exception(etype, value, tback)))
return -1
elif len(db_files) > 1:
log.msg("Error: Cannot start the application because more than one database file are present in: %s" % GLSettings.db_path)
log.msg("Manual check needed and is suggested to first make a backup of %s\n" % GLSettings.working_path)
log.msg("Files found:")
for f in db_files:
log.msg("\t%s" % f)
return -1
return db_version
def cb(res):
start_asynchronous()
yield import_memory_variables()
tor_configured_hosts = yield apply_cli_options()
log.msg("GLBackend is now running")
for ip in GLSetting.bind_addresses:
log.msg("Visit http://%s:%d to interact with me" %
(ip, GLSetting.bind_port))
for host in GLSetting.accepted_hosts:
if host not in GLSetting.bind_addresses:
log.msg("Visit http://%s:%d to interact with me" %
(host, GLSetting.bind_port))
if tor_configured_hosts:
for other in tor_configured_hosts:
if other:
log.msg("Visit %s to interact with me" % other)
log.msg(
"Remind: GlobaLeaks is not accessible from other URLs, this is strictly enforced"
)
log.msg(
"Check documentation in https://github.com/globaleaks/GlobaLeaks/wiki/ for special enhancement"
)
def perform_system_update():
"""
This function checks the system and database versions and executes migration
routines based on the system's state. After this function has completed the
node is either ready for initialization (0), running a version of the DB
(>1), or broken (-1).
"""
from globaleaks.db import migration
db_files = []
max_version = 0
min_version = 0
for filename in os.listdir(GLSettings.db_path):
if filename.startswith('glbackend'):
filepath = os.path.join(GLSettings.db_path, filename)
if filename.endswith('.db'):
db_files.append(filepath)
nameindex = filename.rfind('glbackend')
extensindex = filename.rfind('.db')
fileversion = int(filename[nameindex +
len('glbackend-'):extensindex])
max_version = fileversion if fileversion > max_version else max_version
min_version = fileversion if fileversion < min_version else min_version
db_version = max_version
if len(db_files) > 1:
log.msg(
"Error: Cannot start the application because more than one database file are present in: %s"
% GLSettings.db_path)
log.msg(
"Manual check needed and is suggested to first make a backup of %s\n"
% GLSettings.working_path)
log.msg("Files found:")
for f in db_files:
log.msg("\t%s" % f)
return -1
if len(db_files) == 1:
log.msg("Found an already initialized database version: %d" %
db_version)
if db_version < DATABASE_VERSION:
log.msg(
"Performing schema migration from version %d to version %d" %
(db_version, DATABASE_VERSION))
try:
migration.perform_schema_migration(db_version)
except Exception as exception:
log.msg("Migration failure: %s" % exception)
log.msg("Verbose exception traceback:")
etype, value, tback = sys.exc_info()
log.msg('\n'.join(
traceback.format_exception(etype, value, tback)))
return -1
log.msg("Migration completed with success!")
else:
log.msg('Performing data update')
migration.perform_data_update(
os.path.abspath(
os.path.join(GLSettings.db_path,
'glbackend-%d.db' % DATABASE_VERSION)))
return db_version
def cb(res):
start_asynchronous()
yield import_memory_variables()
tor_configured_hosts = yield apply_cli_options()
log.msg("GLBackend is now running")
for ip in GLSetting.bind_addresses:
log.msg("Visit http://%s:%d to interact with me" % (ip, GLSetting.bind_port))
for host in GLSetting.accepted_hosts:
if host not in GLSetting.bind_addresses:
log.msg("Visit http://%s:%d to interact with me" % (host, GLSetting.bind_port))
if tor_configured_hosts:
for other in tor_configured_hosts:
if other:
log.msg("Visit %s to interact with me" % other)
log.msg("Remind: GlobaLeaks is not accessible from other URLs, this is strictly enforced")
log.msg("Check documentation in https://github.com/globaleaks/GlobaLeaks/wiki/ for special enhancement")
def update_db():
"""
This function handles update of an existing database
"""
try:
db_version, db_file_path = get_db_file(GLSettings.db_path)
if db_version == 0:
return 0
from globaleaks.db import migration
log.msg("Found an already initialized database version: %d" %
db_version)
if db_version >= FIRST_DATABASE_VERSION_SUPPORTED and db_version < DATABASE_VERSION:
log.msg(
"Performing schema migration from version %d to version %d" %
(db_version, DATABASE_VERSION))
migration.perform_schema_migration(db_version)
log.msg("Migration completed with success!")
else:
log.msg('Performing data update')
# TODO on normal startup this line is run. We need better control flow here.
migration.perform_data_update(
os.path.abspath(
os.path.join(GLSettings.db_path,
'glbackend-%d.db' % DATABASE_VERSION)))
except Exception as exception:
log.msg("Migration failure: %s" % exception)
log.msg("Verbose exception traceback:")
etype, value, tback = sys.exc_info()
log.msg('\n'.join(traceback.format_exception(etype, value, tback)))
return -1
