def allocate_application(arguments):
application = Storage(
bootstrap=arguments.bootstrap,
miner=arguments.miner,
)
return application
class MainHTTPHandler(BaseHTTPRequestHandler):
router = {
"method": method_handler
}
store = Storage()
def get_request_id(self, headers):
return headers.get('HTTP_X_REQUEST_ID', uuid.uuid4().hex)
def process_request_(self, request, response, data_string, context):
"""Process user's request, return response and code of response"""
path = self.path.strip("/")
logging.info("%s: %s %s" % (self.path, data_string, context["request_id"]))
if path in self.router:
try:
response, code = self.router[path]({"body": request, "headers": self.headers},
context, self.store)
except (UserRequestError, StorageError) as e:
if isinstance(e, StorageIsDeadError):
response, code = e.args[0], INTERNAL_ERROR
else:
response, code = e.args[0], INVALID_REQUEST
except Exception as e:
logging.exception("Unexpected error: %s" % e)
code = INTERNAL_ERROR
else:
code = NOT_FOUND
return response, code
def do_POST(self):
"""Only posts requests allowed"""
response, code = {}, OK
context = {"request_id": self.get_request_id(self.headers)}
request = None
try:
data_string = self.rfile.read(int(self.headers['Content-Length']))
request = json.loads(data_string.decode("utf-8"))
except Exception as e:
code = BAD_REQUEST
if request:
response, code = self.process_request_(request, response,
data_string, context)
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.end_headers()
if code not in ERRORS:
r = {"response": response, "code": code}
else:
r = {"error": response or ERRORS.get(code, "Unknown Error"), "code": code}
context.update(r)
logging.info(context)
self.wfile.write(json.dumps(r).encode("utf-8"))
return
def __init__(self, config):
Helper.__init__(self)
self.config = config
self.app = create_app(config)
# as this class requires access to the Storage object, which is no longer
# attached to app, we create it here and mock the call to return it below.
self.storage = Storage(config.STORE_DIR, config.TEMP_DIR)
def test_storage() -> Generator[Storage, None, None]:
# Setup the filesystem for the storage object
with TemporaryDirectory() as data_dir_name:
data_dir = Path(data_dir_name)
store_dir = data_dir / "store"
store_dir.mkdir()
tmp_dir = data_dir / "tmp"
tmp_dir.mkdir()
storage = Storage(str(store_dir), str(tmp_dir))
yield storage
class MainHTTPHandler(BaseHTTPRequestHandler):
router = {"method": method_handler}
store = Storage(RedisConnection, STORE_CONFIG)
def get_request_id(self, headers):
return headers.get('HTTP_X_REQUEST_ID', uuid.uuid4().hex)
def do_POST(self):
response, code = {}, OK
context = {"request_id": self.get_request_id(self.headers)}
request = None
try:
data_string = self.rfile.read(int(self.headers['Content-Length']))
print data_string
request = json.loads(data_string) # in Unicode
except:
code = BAD_REQUEST
if request:
path = self.path.strip("/")
logging.info("%s: %s %s" %
(self.path, data_string, context["request_id"]))
if path in self.router:
try:
response, code = self.router[path]({
"body": request,
"headers": self.headers
}, context, self.store)
except Exception, e:
logging.exception("Unexpected error: %s" % e)
code = INTERNAL_ERROR
else:
code = NOT_FOUND
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.end_headers()
if code not in ERRORS:
r = {"response": response, "code": code}
else:
r = {
"error": response or ERRORS.get(code, "Unknown Error"),
"code": code
}
context.update(r)
logging.info(context)
# save correct unicode in response
response_data = json.dumps(r, sort_keys=True,
ensure_ascii=False).encode('utf8')
self.wfile.write(response_data)
return
class UsersView():
storage = Storage()
def __init__(self):
pass
# === Sing <BEGIN> =======================
# initial key = the HAS256 result of "userid+pw+time"
def token_to_global(self, old_token) -> (str, str, str):
"""
return
---
old_token, new_token, the_global
"""
new_token = hashlib.sha384(old_token.encode())
new_token = new_token.hexdigest()
the_global = hashlib.sha256(new_token.encode())
the_global = the_global.hexdigest()
return old_token, new_token, the_global
def logout(self, the_global):
self.storage.delete_one(the_global)
def refresh_global(self, received_global: str, received_time):
"""
process
---
(1) check match (2) check timeout
==> if matched & no timeout
(3) regennrate & update the global, and (4) send 'OK' http request
"""
# received_time = datetime.now() + timedelta(minutes=30)
# regenerate & update the token and global
_, new_token, new_global = self.token_to_global(received_global)
self.storage.update_one(received_global, new_global, new_token,
received_time)
print("> Global refreshed")
def create_app(config):
# type: (SDConfig) -> Flask
app = Flask(__name__,
template_folder=config.SOURCE_TEMPLATES_DIR,
static_folder=path.join(config.SECUREDROP_ROOT, 'static'))
app.request_class = RequestThatSecuresFileUploads
app.config.from_object(config.SourceInterfaceFlaskConfig) # type: ignore
app.sdconfig = config
# The default CSRF token expiration is 1 hour. Since large uploads can
# take longer than an hour over Tor, we increase the valid window to 24h.
app.config['WTF_CSRF_TIME_LIMIT'] = 60 * 60 * 24
CSRFProtect(app)
if config.DATABASE_ENGINE == "sqlite":
db_uri = (config.DATABASE_ENGINE + ":///" + config.DATABASE_FILE)
else:
db_uri = (config.DATABASE_ENGINE + '://' + config.DATABASE_USERNAME +
':' + config.DATABASE_PASSWORD + '@' + config.DATABASE_HOST +
'/' + config.DATABASE_NAME)
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SQLALCHEMY_DATABASE_URI'] = db_uri
db.init_app(app)
app.storage = Storage(config.STORE_DIR, config.TEMP_DIR,
config.JOURNALIST_KEY)
app.crypto_util = CryptoUtil(
scrypt_params=config.SCRYPT_PARAMS,
scrypt_id_pepper=config.SCRYPT_ID_PEPPER,
scrypt_gpg_pepper=config.SCRYPT_GPG_PEPPER,
securedrop_root=config.SECUREDROP_ROOT,
word_list=config.WORD_LIST,
nouns_file=config.NOUNS,
adjectives_file=config.ADJECTIVES,
gpg_key_dir=config.GPG_KEY_DIR,
)
@app.errorhandler(CSRFError)
def handle_csrf_error(e):
msg = render_template('session_timeout.html')
session.clear()
flash(Markup(msg), "important")
return redirect(url_for('main.index'))
assets = Environment(app)
app.config['assets'] = assets
i18n.setup_app(config, app)
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
app.jinja_env.globals['version'] = version.__version__
if getattr(config, 'CUSTOM_HEADER_IMAGE', None):
app.jinja_env.globals['header_image'] = \
config.CUSTOM_HEADER_IMAGE # type: ignore
app.jinja_env.globals['use_custom_header_image'] = True
else:
app.jinja_env.globals['header_image'] = 'logo.png'
app.jinja_env.globals['use_custom_header_image'] = False
app.jinja_env.filters['rel_datetime_format'] = \
template_filters.rel_datetime_format
app.jinja_env.filters['nl2br'] = evalcontextfilter(template_filters.nl2br)
app.jinja_env.filters['filesizeformat'] = template_filters.filesizeformat
for module in [main, info, api]:
app.register_blueprint(module.make_blueprint(config)) # type: ignore
@app.before_request
@ignore_static
def setup_i18n():
"""Store i18n-related values in Flask's special g object"""
g.locale = i18n.get_locale(config)
g.text_direction = i18n.get_text_direction(g.locale)
g.html_lang = i18n.locale_to_rfc_5646(g.locale)
g.locales = i18n.get_locale2name()
@app.before_request
@ignore_static
def check_tor2web():
# ignore_static here so we only flash a single message warning
# about Tor2Web, corresponding to the initial page load.
if 'X-tor2web' in request.headers:
flash(
Markup(
gettext(
'<strong>WARNING: </strong> '
'You appear to be using Tor2Web. '
'This <strong> does not </strong> '
'provide anonymity. '
'<a href="{url}">Why is this dangerous?</a>').format(
url=url_for('info.tor2web_warning'))),
"banner-warning")
@app.before_request
@ignore_static
def load_instance_config():
app.instance_config = InstanceConfig.get_current()
@app.before_request
@ignore_static
def setup_g():
"""Store commonly used values in Flask's special g object"""
if 'expires' in session and datetime.utcnow() >= session['expires']:
msg = render_template('session_timeout.html')
# clear the session after we render the message so it's localized
session.clear()
# Redirect to index with flashed message
flash(Markup(msg), "important")
return redirect(url_for('main.index'))
session['expires'] = datetime.utcnow() + \
timedelta(minutes=getattr(config,
'SESSION_EXPIRATION_MINUTES',
120))
# ignore_static here because `crypto_util.hash_codename` is scrypt
# (very time consuming), and we don't need to waste time running if
# we're just serving a static resource that won't need to access
# these common values.
if logged_in():
g.codename = session['codename']
g.filesystem_id = app.crypto_util.hash_codename(g.codename)
try:
g.source = Source.query \
.filter(Source.filesystem_id == g.filesystem_id) \
.one()
except NoResultFound as e:
app.logger.error("Found no Sources when one was expected: %s" %
(e, ))
del session['logged_in']
del session['codename']
return redirect(url_for('main.index'))
g.loc = app.storage.path(g.filesystem_id)
@app.errorhandler(404)
def page_not_found(error):
return render_template('notfound.html'), 404
@app.errorhandler(500)
def internal_error(error):
return render_template('error.html'), 500
return app
def create_app(config: 'SDConfig') -> Flask:
app = Flask(__name__,
template_folder=config.JOURNALIST_TEMPLATES_DIR,
static_folder=path.join(config.SECUREDROP_ROOT, 'static'))
app.config.from_object(config.JOURNALIST_APP_FLASK_CONFIG_CLS)
app.session_interface = JournalistInterfaceSessionInterface()
csrf = CSRFProtect(app)
Environment(app)
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SQLALCHEMY_DATABASE_URI'] = config.DATABASE_URI
db.init_app(app)
v2_enabled = path.exists(
path.join(config.SECUREDROP_DATA_ROOT, 'source_v2_url'))
v3_enabled = path.exists(
path.join(config.SECUREDROP_DATA_ROOT, 'source_v3_url'))
app.config.update(V2_ONION_ENABLED=v2_enabled, V3_ONION_ENABLED=v3_enabled)
# TODO: Attaching a Storage dynamically like this disables all type checking (and
# breaks code analysis tools) for code that uses current_app.storage; it should be refactored
app.storage = Storage(config.STORE_DIR, config.TEMP_DIR,
config.JOURNALIST_KEY)
# TODO: Attaching a CryptoUtil dynamically like this disables all type checking (and
# breaks code analysis tools) for code that uses current_app.storage; it should be refactored
app.crypto_util = CryptoUtil(
scrypt_params=config.SCRYPT_PARAMS,
scrypt_id_pepper=config.SCRYPT_ID_PEPPER,
scrypt_gpg_pepper=config.SCRYPT_GPG_PEPPER,
securedrop_root=config.SECUREDROP_ROOT,
word_list=config.WORD_LIST,
nouns_file=config.NOUNS,
adjectives_file=config.ADJECTIVES,
gpg_key_dir=config.GPG_KEY_DIR,
)
@app.errorhandler(CSRFError)
def handle_csrf_error(e: CSRFError) -> 'Response':
# render the message first to ensure it's localized.
msg = gettext('You have been logged out due to inactivity.')
session.clear()
flash(msg, 'error')
return redirect(url_for('main.login'))
def _handle_http_exception(
error: 'HTTPException'
) -> 'Tuple[Union[Response, str], Optional[int]]':
# Workaround for no blueprint-level 404/5 error handlers, see:
# https://github.com/pallets/flask/issues/503#issuecomment-71383286
handler = list(app.error_handler_spec['api'][error.code].values())[0]
if request.path.startswith('/api/') and handler:
return handler(error)
return render_template('error.html', error=error), error.code
for code in default_exceptions:
app.errorhandler(code)(_handle_http_exception)
i18n.setup_app(config, app)
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
app.jinja_env.globals['version'] = version.__version__
app.jinja_env.filters['rel_datetime_format'] = \
template_filters.rel_datetime_format
app.jinja_env.filters['filesizeformat'] = template_filters.filesizeformat
@app.before_first_request
def expire_blacklisted_tokens() -> None:
cleanup_expired_revoked_tokens()
@app.before_request
def load_instance_config() -> None:
app.instance_config = InstanceConfig.get_current()
@app.before_request
def setup_g() -> 'Optional[Response]':
"""Store commonly used values in Flask's special g object"""
if 'expires' in session and datetime.utcnow() >= session['expires']:
session.clear()
flash(gettext('You have been logged out due to inactivity.'),
'error')
uid = session.get('uid', None)
if uid:
user = Journalist.query.get(uid)
if user and 'nonce' in session and \
session['nonce'] != user.session_nonce:
session.clear()
flash(
gettext('You have been logged out due to password change'),
'error')
session['expires'] = datetime.utcnow() + \
timedelta(minutes=getattr(config,
'SESSION_EXPIRATION_MINUTES',
120))
# Work around https://github.com/lepture/flask-wtf/issues/275
# -- after upgrading from Python 2 to Python 3, any existing
# session's csrf_token value will be retrieved as bytes,
# causing a TypeError. This simple fix, deleting the existing
# token, was suggested in the issue comments. This code will
# be safe to remove after Python 2 reaches EOL in 2020, and no
# supported SecureDrop installations can still have this
# problem.
if sys.version_info.major > 2 and type(
session.get('csrf_token')) is bytes:
del session['csrf_token']
uid = session.get('uid', None)
if uid:
g.user = Journalist.query.get(uid)
g.locale = i18n.get_locale(config)
g.text_direction = i18n.get_text_direction(g.locale)
g.html_lang = i18n.locale_to_rfc_5646(g.locale)
g.locales = i18n.get_locale2name()
if not app.config['V3_ONION_ENABLED'] or app.config['V2_ONION_ENABLED']:
g.show_v2_onion_eol_warning = True
if request.path.split('/')[1] == 'api':
pass # We use the @token_required decorator for the API endpoints
else: # We are not using the API
if request.endpoint not in _insecure_views and not logged_in():
return redirect(url_for('main.login'))
if request.method == 'POST':
filesystem_id = request.form.get('filesystem_id')
if filesystem_id:
g.filesystem_id = filesystem_id
g.source = get_source(filesystem_id)
return None
app.register_blueprint(main.make_blueprint(config))
app.register_blueprint(account.make_blueprint(config),
url_prefix='/account')
app.register_blueprint(admin.make_blueprint(config), url_prefix='/admin')
app.register_blueprint(col.make_blueprint(config), url_prefix='/col')
api_blueprint = api.make_blueprint(config)
app.register_blueprint(api_blueprint, url_prefix='/api/v1')
csrf.exempt(api_blueprint)
return app
def setUp(self):
self.context = {}
self.store = Storage()
self.backup = {}
self.fulfill()
self.key_hash = set()
def create_app(config: SDConfig) -> Flask:
app = Flask(__name__,
template_folder=config.SOURCE_TEMPLATES_DIR,
static_folder=path.join(config.SECUREDROP_ROOT, 'static'))
app.request_class = RequestThatSecuresFileUploads
app.config.from_object(config.SOURCE_APP_FLASK_CONFIG_CLS)
# The default CSRF token expiration is 1 hour. Since large uploads can
# take longer than an hour over Tor, we increase the valid window to 24h.
app.config['WTF_CSRF_TIME_LIMIT'] = 60 * 60 * 24
CSRFProtect(app)
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SQLALCHEMY_DATABASE_URI'] = config.DATABASE_URI
db.init_app(app)
# TODO: Attaching a Storage dynamically like this disables all type checking (and
# breaks code analysis tools) for code that uses current_app.storage; it should be refactored
app.storage = Storage(config.STORE_DIR,
config.TEMP_DIR,
config.JOURNALIST_KEY)
# TODO: Attaching a CryptoUtil dynamically like this disables all type checking (and
# breaks code analysis tools) for code that uses current_app.storage; it should be refactored
app.crypto_util = CryptoUtil(
scrypt_params=config.SCRYPT_PARAMS,
scrypt_id_pepper=config.SCRYPT_ID_PEPPER,
scrypt_gpg_pepper=config.SCRYPT_GPG_PEPPER,
securedrop_root=config.SECUREDROP_ROOT,
word_list=config.WORD_LIST,
nouns_file=config.NOUNS,
adjectives_file=config.ADJECTIVES,
gpg_key_dir=config.GPG_KEY_DIR,
)
@app.errorhandler(CSRFError)
def handle_csrf_error(e: CSRFError) -> werkzeug.Response:
msg = render_template('session_timeout.html')
session.clear()
flash(Markup(msg), "important")
return redirect(url_for('main.index'))
assets = Environment(app)
app.config['assets'] = assets
i18n.setup_app(config, app)
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
app.jinja_env.globals['version'] = version.__version__
# Exported to source templates for being included in instructions
app.jinja_env.globals['submission_key_fpr'] = config.JOURNALIST_KEY
app.jinja_env.filters['rel_datetime_format'] = \
template_filters.rel_datetime_format
app.jinja_env.filters['nl2br'] = evalcontextfilter(template_filters.nl2br)
app.jinja_env.filters['filesizeformat'] = template_filters.filesizeformat
for module in [main, info, api]:
app.register_blueprint(module.make_blueprint(config)) # type: ignore
@app.before_request
@ignore_static
def setup_i18n() -> None:
"""Store i18n-related values in Flask's special g object"""
g.locale = i18n.get_locale(config)
g.text_direction = i18n.get_text_direction(g.locale)
g.html_lang = i18n.locale_to_rfc_5646(g.locale)
g.locales = i18n.get_locale2name()
@app.before_request
@ignore_static
def check_tor2web() -> None:
# ignore_static here so we only flash a single message warning
# about Tor2Web, corresponding to the initial page load.
if 'X-tor2web' in request.headers:
flash(Markup(gettext(
'<strong>WARNING: </strong> '
'You appear to be using Tor2Web. '
'This <strong> does not </strong> '
'provide anonymity. '
'<a href="{url}">Why is this dangerous?</a>')
.format(url=url_for('info.tor2web_warning'))),
"banner-warning")
@app.before_request
@ignore_static
def load_instance_config() -> None:
app.instance_config = InstanceConfig.get_current()
@app.before_request
@ignore_static
def setup_g() -> Optional[werkzeug.Response]:
"""Store commonly used values in Flask's special g object"""
if 'expires' in session and datetime.utcnow() >= session['expires']:
msg = render_template('session_timeout.html')
# Show expiration message only if the user was
# either in the codename generation flow or logged in
show_expiration_message = any([
session.get('show_expiration_message'),
logged_in(),
was_in_generate_flow(),
])
# clear the session after we render the message so it's localized
session.clear()
# Persist this properety across sessions to distinguish users whose sessions expired
# from users who never logged in or generated a codename
session['show_expiration_message'] = show_expiration_message
# Redirect to index with flashed message
if session['show_expiration_message']:
flash(Markup(msg), "important")
return redirect(url_for('main.index'))
session['expires'] = datetime.utcnow() + \
timedelta(minutes=getattr(config,
'SESSION_EXPIRATION_MINUTES',
120))
# ignore_static here because `crypto_util.hash_codename` is scrypt
# (very time consuming), and we don't need to waste time running if
# we're just serving a static resource that won't need to access
# these common values.
if logged_in():
g.codename = session['codename']
g.filesystem_id = app.crypto_util.hash_codename(g.codename)
try:
g.source = Source.query \
.filter(Source.filesystem_id == g.filesystem_id) \
.filter_by(deleted_at=None) \
.one()
except NoResultFound as e:
app.logger.error(
"Found no Sources when one was expected: %s" %
(e,))
del session['logged_in']
del session['codename']
return redirect(url_for('main.index'))
g.loc = app.storage.path(g.filesystem_id)
return None
@app.errorhandler(404)
def page_not_found(error: werkzeug.exceptions.HTTPException) -> Tuple[str, int]:
return render_template('notfound.html'), 404
@app.errorhandler(500)
def internal_error(error: werkzeug.exceptions.HTTPException) -> Tuple[str, int]:
return render_template('error.html'), 500
return app
def app_storage(config: SDConfig) -> "Storage":
return Storage(config.STORE_DIR, config.TEMP_DIR)
def __init__(self):
self.session = requests.Session()
self.session.headers = {'user-agent': 'shr-podcasts-bot'}
self.scraper = Scraper(self.session)
self.parser = Parser(self.session)
self.storage = Storage()
return
if __name__ == "__main__":
parser_config = argparse.ArgumentParser()
parser_config.add_argument('-c',
'--config',
default="{}/config_api".format(
os.path.dirname(os.path.abspath(__file__))))
path_config = parser_config.parse_args(
) # path_config.config -> path to config file
try:
config = parse_config(config, path_config.config)
except Exception:
raise Exception("Bad config!")
logging.basicConfig(filename=config['LOGGING_TO_FILE'],
level=config['LOGGING_LEVEL'],
format='[%(asctime)s] %(levelname).1s %(message)s',
datefmt='%Y.%m.%d %H:%M:%S')
MainHTTPHandler.store = Storage(host=config['STORE_URL'],
port=config['STORE_PORT'],
db=config['NUMBER_DB'],
timeout=config['TIMEOUT'])
server = HTTPServer(("localhost", config['PORT']), MainHTTPHandler)
logging.info("Starting server at %s" % config['PORT'])
try:
server.serve_forever()
except KeyboardInterrupt:
pass
server.server_close()
def test_verify_store_temp_dir_not_absolute():
with pytest.raises(store.PathException) as exc_info:
Storage("/", "..")
msg = str(exc_info.value)
assert re.compile("temp_dir.*is not absolute").match(msg)
self.end_headers()
if code not in ERRORS:
r = {"response": response, "code": code}
else:
r = {"error": response or ERRORS.get(code, "Unknown Error"), "code": code}
context.update(r)
logging.info(context)
self.wfile.write(json.dumps(r).encode("utf-8"))
return
# -------------------------- main ------------------------- #
if __name__ == "__main__":
op = OptionParser()
op.add_option("-p", "--port", action="store", type=int, default=8080)
op.add_option("-m", "--memcached", action="store", type=int, default=11211)
op.add_option("-l", "--log", action="store", default=None)
(opts, args) = op.parse_args()
logging.basicConfig(filename=opts.log, level=logging.INFO,
format='[%(asctime)s] %(levelname).1s %(message)s',
datefmt='%Y.%m.%d %H:%M:%S')
setattr(MainHTTPHandler, "store", Storage(port=opts.memcached))
server = HTTPServer(("localhost", opts.port), MainHTTPHandler)
logging.info("Starting server at %s" % opts.port)
try:
server.serve_forever()
except KeyboardInterrupt:
pass
server.server_close()
def test_verify_store_temp_dir_not_absolute():
with pytest.raises(store.PathException) as exc_info:
Storage('/', '..', '<not a gpg key>')
msg = str(exc_info.value)
assert re.compile('temp_dir.*is not absolute').match(msg)