def add_reply(source: Source, journalist: Journalist,
journalist_who_saw: Optional[Journalist]) -> None:
"""
Adds a single reply to a source.
"""
record_source_interaction(source)
fname = "{}-{}-reply.gpg".format(source.interaction_count,
source.journalist_filename)
EncryptionManager.get_default().encrypt_journalist_reply(
for_source_with_filesystem_id=source.filesystem_id,
reply_in=next(replies),
encrypted_reply_path_out=Path(Storage.get_default().path(
source.filesystem_id, fname)),
)
reply = Reply(journalist, source, fname, Storage.get_default())
db.session.add(reply)
# Journalist who replied has seen the reply
author_seen_reply = SeenReply(reply=reply, journalist=journalist)
db.session.add(author_seen_reply)
if journalist_who_saw:
other_seen_reply = SeenReply(reply=reply,
journalist=journalist_who_saw)
db.session.add(other_seen_reply)
db.session.commit()
def upgrade() -> None:
with op.batch_alter_table("replies", schema=None) as batch_op:
batch_op.add_column(
sa.Column("checksum", sa.String(length=255), nullable=True))
with op.batch_alter_table("submissions", schema=None) as batch_op:
batch_op.add_column(
sa.Column("checksum", sa.String(length=255), nullable=True))
op.create_table(
"revoked_tokens",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("journalist_id", sa.Integer(), nullable=True),
sa.Column("token", sa.Text(), nullable=False),
sa.ForeignKeyConstraint(["journalist_id"], ["journalists.id"]),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("token"),
)
try:
app = create_app(config)
# we need an app context for the rq worker extension to work properly
with app.app_context():
conn = op.get_bind()
query = sa.text(
"""SELECT submissions.id, sources.filesystem_id, submissions.filename
FROM submissions
INNER JOIN sources
ON submissions.source_id = sources.id
""")
for (sub_id, filesystem_id, filename) in conn.execute(query):
full_path = Storage.get_default().path(filesystem_id, filename)
create_queue().enqueue(
queued_add_checksum_for_file,
Submission,
int(sub_id),
full_path,
app.config["SQLALCHEMY_DATABASE_URI"],
)
query = sa.text(
"""SELECT replies.id, sources.filesystem_id, replies.filename
FROM replies
INNER JOIN sources
ON replies.source_id = sources.id
""")
for (rep_id, filesystem_id, filename) in conn.execute(query):
full_path = Storage.get_default().path(filesystem_id, filename)
create_queue().enqueue(
queued_add_checksum_for_file,
Reply,
int(rep_id),
full_path,
app.config["SQLALCHEMY_DATABASE_URI"],
)
except: # noqa
if raise_errors:
raise
def _create_source_and_submission(config_in_use: SecureDropConfig) -> Path:
"""Directly create a source and a submission within the app.
Some tests for the journalist app require a submission to already be present, and this
function is used to create the source user and submission when the journalist app starts.
This implementation is much faster than using Selenium to navigate the source app in order
to create a submission: it takes 0.2s to run, while the Selenium implementation takes 7s.
"""
# This function will be called in a separate Process that runs the app
# Hence the late imports
from encryption import EncryptionManager
from models import Submission
from passphrases import PassphraseGenerator
from source_user import create_source_user
from store import Storage, add_checksum_for_file
from tests.functional.db_session import get_database_session
# Create a source
passphrase = PassphraseGenerator.get_default().generate_passphrase()
with get_database_session(
database_uri=config_in_use.DATABASE_URI) as db_session:
source_user = create_source_user(
db_session=db_session,
source_passphrase=passphrase,
source_app_storage=Storage.get_default(),
)
source_db_record = source_user.get_db_record()
EncryptionManager.get_default().generate_source_key_pair(source_user)
# Create a file submission from this source
source_db_record.interaction_count += 1
app_storage = Storage.get_default()
encrypted_file_name = app_storage.save_file_submission(
filesystem_id=source_user.filesystem_id,
count=source_db_record.interaction_count,
journalist_filename=source_db_record.journalist_filename,
filename="filename.txt",
stream=BytesIO(b"File with S3cr3t content"),
)
submission = Submission(source_db_record, encrypted_file_name,
app_storage)
db_session.add(submission)
source_db_record.pending = False
source_db_record.last_updated = datetime.now(timezone.utc)
db_session.commit()
submission_file_path = app_storage.path(source_user.filesystem_id,
submission.filename)
add_checksum_for_file(
session=db_session,
db_obj=submission,
file_path=submission_file_path,
)
return Path(submission_file_path)
def delete_file_object(file_object: Union[Submission, Reply]) -> None:
path = Storage.get_default().path(file_object.source.filesystem_id,
file_object.filename)
try:
Storage.get_default().move_to_shredder(path)
except ValueError as e:
current_app.logger.error("could not queue file for deletion: %s", e)
raise
finally:
db.session.delete(file_object)
db.session.commit()
def delete_collection(filesystem_id: str) -> None:
"""deletes source account including files and reply key"""
# Delete the source's collection of submissions
path = Storage.get_default().path(filesystem_id)
if os.path.exists(path):
Storage.get_default().move_to_shredder(path)
# Delete the source's reply keypair
EncryptionManager.get_default().delete_source_key_pair(filesystem_id)
# Delete their entry in the db
source = get_source(filesystem_id, include_deleted=True)
db.session.delete(source)
db.session.commit()
def submit_message(source: Source,
journalist_who_saw: Optional[Journalist]) -> None:
"""
Adds a single message submitted by a source.
"""
record_source_interaction(source)
fpath = Storage.get_default().save_message_submission(
source.filesystem_id,
source.interaction_count,
source.journalist_filename,
next(messages),
)
submission = Submission(source, fpath, Storage.get_default())
db.session.add(submission)
if journalist_who_saw:
seen_message = SeenMessage(message=submission,
journalist=journalist_who_saw)
db.session.add(seen_message)
def submit_file(source: Source,
journalist_who_saw: Optional[Journalist]) -> None:
"""
Adds a single file submitted by a source.
"""
record_source_interaction(source)
fpath = Storage.get_default().save_file_submission(
source.filesystem_id,
source.interaction_count,
source.journalist_filename,
"memo.txt",
io.BytesIO(b"This is an example of a plain text file upload."),
)
submission = Submission(source, fpath, Storage.get_default())
db.session.add(submission)
if journalist_who_saw:
seen_file = SeenFile(file=submission, journalist=journalist_who_saw)
db.session.add(seen_file)
def lookup(logged_in_source: SourceUser) -> str:
replies = []
logged_in_source_in_db = logged_in_source.get_db_record()
source_inbox = Reply.query.filter_by(
source_id=logged_in_source_in_db.id, deleted_by_source=False
).all()
first_submission = logged_in_source_in_db.interaction_count == 0
if first_submission:
min_message_length = InstanceConfig.get_default().initial_message_min_len
else:
min_message_length = 0
for reply in source_inbox:
reply_path = Storage.get_default().path(
logged_in_source.filesystem_id,
reply.filename,
)
try:
with io.open(reply_path, "rb") as f:
contents = f.read()
decrypted_reply = EncryptionManager.get_default().decrypt_journalist_reply(
for_source_user=logged_in_source, ciphertext_in=contents
)
reply.decrypted = decrypted_reply
except UnicodeDecodeError:
current_app.logger.error("Could not decode reply %s" % reply.filename)
except FileNotFoundError:
current_app.logger.error("Reply file missing: %s" % reply.filename)
else:
reply.date = datetime.utcfromtimestamp(os.stat(reply_path).st_mtime)
replies.append(reply)
# Sort the replies by date
replies.sort(key=operator.attrgetter("date"), reverse=True)
# If not done yet, generate a keypair to encrypt replies from the journalist
encryption_mgr = EncryptionManager.get_default()
try:
encryption_mgr.get_source_public_key(logged_in_source.filesystem_id)
except GpgKeyNotFoundError:
encryption_mgr.generate_source_key_pair(logged_in_source)
return render_template(
"lookup.html",
is_user_logged_in=True,
allow_document_uploads=InstanceConfig.get_default().allow_document_uploads,
replies=replies,
min_len=min_message_length,
new_user_codename=session.get("new_user_codename", None),
form=SubmissionForm(),
)
def download_single_file(filesystem_id: str, fn: str) -> werkzeug.Response:
"""
Marks the file being download (the file being downloaded is either a submission message,
submission file attachement, or journalist reply) as seen by the current logged-in user and
send the file to a client to be saved or opened.
"""
if ".." in fn or fn.startswith("/"):
abort(404)
file = Storage.get_default().path(filesystem_id, fn)
if not Path(file).is_file():
flash(
gettext(
"Your download failed because the file could not be found. An admin can find "
+ "more information in the system and monitoring logs."),
"error",
)
current_app.logger.error("File {} not found".format(file))
return redirect(url_for("col.col", filesystem_id=filesystem_id))
# mark as seen by the current user
try:
journalist = g.get("user")
if fn.endswith("reply.gpg"):
reply = Reply.query.filter(Reply.filename == fn).one()
mark_seen([reply], journalist)
elif fn.endswith("-doc.gz.gpg") or fn.endswith("doc.zip.gpg"):
submitted_file = Submission.query.filter(
Submission.filename == fn).one()
mark_seen([submitted_file], journalist)
else:
message = Submission.query.filter(
Submission.filename == fn).one()
mark_seen([message], journalist)
except NoResultFound as e:
current_app.logger.error("Could not mark {} as seen: {}".format(
fn, e))
return send_file(Storage.get_default().path(filesystem_id, fn),
mimetype="application/pgp-encrypted")
def serve_file_with_etag(db_obj: Union[Reply, Submission]) -> flask.Response:
file_path = Storage.get_default().path(db_obj.source.filesystem_id,
db_obj.filename)
response = send_file(file_path,
mimetype="application/pgp-encrypted",
as_attachment=True,
etag=False) # Disable Flask default ETag
if not db_obj.checksum:
add_checksum_for_file(db.session, db_obj, file_path)
response.direct_passthrough = False
response.headers["Etag"] = db_obj.checksum
return response
def upgrade() -> None:
conn = op.get_bind()
submissions = conn.execute(
sa.text(raw_sql_grab_orphaned_objects("submissions"))).fetchall()
replies = conn.execute(sa.text(
raw_sql_grab_orphaned_objects("replies"))).fetchall()
try:
app = create_app(config)
with app.app_context():
for submission in submissions:
try:
conn.execute(
sa.text("""
DELETE FROM submissions
WHERE id=:id
""").bindparams(id=submission.id))
path = Storage.get_default().path_without_filesystem_id(
submission.filename)
Storage.get_default().move_to_shredder(path)
except NoFileFoundException:
# The file must have been deleted by the admin, remove the row
conn.execute(
sa.text("""
DELETE FROM submissions
WHERE id=:id
""").bindparams(id=submission.id))
except TooManyFilesException:
pass
for reply in replies:
try:
conn.execute(
sa.text("""
DELETE FROM replies
WHERE id=:id
""").bindparams(id=reply.id))
path = Storage.get_default().path_without_filesystem_id(
reply.filename)
Storage.get_default().move_to_shredder(path)
except NoFileFoundException:
# The file must have been deleted by the admin, remove the row
conn.execute(
sa.text("""
DELETE FROM replies
WHERE id=:id
""").bindparams(id=reply.id))
except TooManyFilesException:
pass
except: # noqa
if raise_errors:
raise
def create() -> werkzeug.Response:
if SessionManager.is_user_logged_in(db_session=db.session):
flash_msg(
"notification",
None,
gettext(
"You are already logged in. Please verify your codename as it "
"may differ from the one displayed on the previous page."
),
)
else:
# Ensure the codenames have not expired
date_codenames_expire = session.get("codenames_expire")
if not date_codenames_expire or datetime.now(timezone.utc) >= date_codenames_expire:
return clear_session_and_redirect_to_logged_out_page(flask_session=session)
tab_id = request.form["tab_id"]
codename = session["codenames"][tab_id]
del session["codenames"]
try:
current_app.logger.info("Creating new source user...")
create_source_user(
db_session=db.session,
source_passphrase=codename,
source_app_storage=Storage.get_default(),
)
except (SourcePassphraseCollisionError, SourceDesignationCollisionError) as e:
current_app.logger.error("Could not create a source: {}".format(e))
flash_msg(
"error",
None,
gettext(
"There was a temporary problem creating your account. Please try again."
),
)
return redirect(url_for(".index"))
# All done - source user was successfully created
current_app.logger.info("New source user created")
session["new_user_codename"] = codename
SessionManager.log_user_in(
db_session=db.session, supplied_passphrase=DicewarePassphrase(codename)
)
return redirect(url_for(".lookup"))
def download(
zip_basename: str,
submissions: List[Union[Submission, Reply]],
on_error_redirect: Optional[str] = None,
) -> werkzeug.Response:
"""Send client contents of ZIP-file *zip_basename*-<timestamp>.zip
containing *submissions*. The ZIP-file, being a
:class:`tempfile.NamedTemporaryFile`, is stored on disk only
temporarily.
:param str zip_basename: The basename of the ZIP-file download.
:param list submissions: A list of :class:`models.Submission`s to
include in the ZIP-file.
"""
try:
zf = Storage.get_default().get_bulk_archive(submissions,
zip_directory=zip_basename)
except FileNotFoundError:
flash(
ngettext(
"Your download failed because the file could not be found. An admin can find "
+ "more information in the system and monitoring logs.",
"Your download failed because a file could not be found. An admin can find "
+ "more information in the system and monitoring logs.",
len(submissions),
),
"error",
)
if on_error_redirect is None:
on_error_redirect = url_for("main.index")
return redirect(on_error_redirect)
attachment_filename = "{}--{}.zip".format(
zip_basename,
datetime.now(timezone.utc).strftime("%Y-%m-%d--%H-%M-%S"))
mark_seen(submissions, g.user)
return send_file(
zf.name,
mimetype="application/zip",
download_name=attachment_filename,
as_attachment=True,
)
def add_source() -> Tuple[Source, str]:
"""
Adds a single source.
"""
codename = PassphraseGenerator.get_default().generate_passphrase()
source_user = create_source_user(
db_session=db.session,
source_passphrase=codename,
source_app_storage=Storage.get_default(),
)
source = source_user.get_db_record()
source.pending = False
db.session.commit()
# Generate source key
EncryptionManager.get_default().generate_source_key_pair(source_user)
return source, codename
def normalize_timestamps(logged_in_source: SourceUser) -> None:
"""
Update the timestamps on all of the source's submissions. This
minimizes metadata that could be useful to investigators. See
#301.
"""
source_in_db = logged_in_source.get_db_record()
sub_paths = [
Storage.get_default().path(logged_in_source.filesystem_id,
submission.filename)
for submission in source_in_db.submissions
]
if len(sub_paths) > 1:
args = ["touch", "--no-create"]
args.extend(sub_paths)
rc = subprocess.call(args)
if rc != 0:
current_app.logger.warning("Couldn't normalize submission "
"timestamps (touch exited with %d)" %
rc)
def load_data(self):
global DATA
with mock.patch("store.Storage.get_default") as mock_storage_global:
mock_storage_global.return_value = self.storage
with self.app.app_context():
self.create_journalist()
self.create_source()
submission_id, submission_filename = self.create_submission()
reply_id, reply_filename = self.create_reply()
# we need to actually create files and write data to them so the
# RQ worker can hash them
for fn in [submission_filename, reply_filename]:
full_path = Storage.get_default().path(
self.source_filesystem_id, fn)
dirname = path.dirname(full_path)
if not path.exists(dirname):
os.mkdir(dirname)
with io.open(full_path, "wb") as f:
f.write(DATA)
def assertion():
assert not any([
os.path.exists(Storage.get_default().path(filesystem_id, doc_name))
for doc_name in checkbox_values
])
def assertion():
assert not os.path.exists(
Storage.get_default().path(filesystem_id))
def submit(logged_in_source: SourceUser) -> werkzeug.Response:
allow_document_uploads = InstanceConfig.get_default().allow_document_uploads
form = SubmissionForm()
if not form.validate():
for field, errors in form.errors.items():
for error in errors:
flash_msg("error", None, error)
return redirect(url_for("main.lookup"))
msg = request.form["msg"]
fh = None
if allow_document_uploads and "fh" in request.files:
fh = request.files["fh"]
# Don't submit anything if it was an "empty" submission. #878
if not (msg or fh):
if allow_document_uploads:
html_contents = gettext("You must enter a message or choose a file to submit.")
else:
html_contents = gettext("You must enter a message.")
flash_msg("error", None, html_contents)
return redirect(url_for("main.lookup"))
fnames = []
logged_in_source_in_db = logged_in_source.get_db_record()
first_submission = logged_in_source_in_db.interaction_count == 0
if first_submission:
min_len = InstanceConfig.get_default().initial_message_min_len
if (min_len > 0) and (msg and not fh) and (len(msg) < min_len):
flash_msg(
"error",
None,
gettext("Your first message must be at least {} characters long.").format(
min_len
),
)
return redirect(url_for("main.lookup"))
# if the new_user_codename key is not present in the session, this is
# not a first session
new_codename = session.get("new_user_codename", None)
codenames_rejected = InstanceConfig.get_default().reject_message_with_codename
if new_codename is not None:
if codenames_rejected and codename_detected(msg, new_codename):
flash_msg(
"error",
None,
gettext("Please do not submit your codename!"),
gettext(
"Keep your codename secret, and use it to log in later to "
"check for replies."
),
)
return redirect(url_for("main.lookup"))
if not os.path.exists(Storage.get_default().path(logged_in_source.filesystem_id)):
current_app.logger.debug(
"Store directory not found for source '{}', creating one.".format(
logged_in_source_in_db.journalist_designation
)
)
os.mkdir(Storage.get_default().path(logged_in_source.filesystem_id))
if msg:
logged_in_source_in_db.interaction_count += 1
fnames.append(
Storage.get_default().save_message_submission(
logged_in_source_in_db.filesystem_id,
logged_in_source_in_db.interaction_count,
logged_in_source_in_db.journalist_filename,
msg,
)
)
if fh:
logged_in_source_in_db.interaction_count += 1
fnames.append(
Storage.get_default().save_file_submission(
logged_in_source_in_db.filesystem_id,
logged_in_source_in_db.interaction_count,
logged_in_source_in_db.journalist_filename,
fh.filename,
fh.stream,
)
)
if first_submission or msg or fh:
if first_submission:
html_contents = gettext(
"Thank you for sending this information to us. Please "
"check back later for replies."
)
elif msg and not fh:
html_contents = gettext("Thanks! We received your message.")
elif fh and not msg:
html_contents = gettext("Thanks! We received your document.")
else:
html_contents = gettext("Thanks! We received your message and document.")
flash_msg("success", gettext("Success!"), html_contents)
new_submissions = []
for fname in fnames:
submission = Submission(logged_in_source_in_db, fname, Storage.get_default())
db.session.add(submission)
new_submissions.append(submission)
logged_in_source_in_db.pending = False
logged_in_source_in_db.last_updated = datetime.now(timezone.utc)
db.session.commit()
for sub in new_submissions:
store.async_add_checksum_for_file(sub, Storage.get_default())
normalize_timestamps(logged_in_source)
return redirect(url_for("main.lookup"))
def assertion():
assert not (any([
os.path.exists(Storage.get_default().path(filesystem_id))
for filesystem_id in checkbox_values
]))
def all_source_replies(source_uuid: str) -> Tuple[flask.Response, int]:
if request.method == "GET":
source = get_or_404(Source, source_uuid, column=Source.uuid)
return jsonify(
{"replies":
[reply.to_json() for reply in source.replies]}), 200
elif request.method == "POST":
source = get_or_404(Source, source_uuid, column=Source.uuid)
if request.json is None:
abort(400, "please send requests in valid JSON")
if "reply" not in request.json:
abort(400, "reply not found in request body")
user = _authenticate_user_from_auth_header(request)
data = request.json
if not data["reply"]:
abort(400, "reply should not be empty")
source.interaction_count += 1
try:
filename = Storage.get_default().save_pre_encrypted_reply(
source.filesystem_id,
source.interaction_count,
source.journalist_filename,
data["reply"],
)
except NotEncrypted:
return jsonify(
{"message": "You must encrypt replies client side"}), 400
# issue #3918
filename = path.basename(filename)
reply = Reply(user, source, filename, Storage.get_default())
reply_uuid = data.get("uuid", None)
if reply_uuid is not None:
# check that is is parseable
try:
UUID(reply_uuid)
except ValueError:
abort(400, "'uuid' was not a valid UUID")
reply.uuid = reply_uuid
try:
db.session.add(reply)
seen_reply = SeenReply(reply=reply, journalist=user)
db.session.add(seen_reply)
db.session.add(source)
db.session.commit()
except IntegrityError as e:
db.session.rollback()
if "UNIQUE constraint failed: replies.uuid" in str(e):
abort(409, "That UUID is already in use.")
else:
raise e
return (
jsonify({
"message": "Your reply has been stored",
"uuid": reply.uuid,
"filename": reply.filename,
}),
201,
)
else:
abort(405)
def reply() -> werkzeug.Response:
"""Attempt to send a Reply from a Journalist to a Source. Empty
messages are rejected, and an informative error message is flashed
on the client. In the case of unexpected errors involving database
transactions (potentially caused by racing request threads that
modify the same the database object) logging is done in such a way
so as not to write potentially sensitive information to disk, and a
generic error message is flashed on the client.
Returns:
flask.Response: The user is redirected to the same Source
collection view, regardless if the Reply is created
successfully.
"""
form = ReplyForm()
if not form.validate_on_submit():
for error in form.message.errors:
flash(error, "error")
return redirect(url_for("col.col", filesystem_id=g.filesystem_id))
g.source.interaction_count += 1
filename = "{0}-{1}-reply.gpg".format(g.source.interaction_count,
g.source.journalist_filename)
EncryptionManager.get_default().encrypt_journalist_reply(
for_source_with_filesystem_id=g.filesystem_id,
reply_in=form.message.data,
encrypted_reply_path_out=Path(Storage.get_default().path(
g.filesystem_id, filename)),
)
try:
reply = Reply(g.user, g.source, filename, Storage.get_default())
db.session.add(reply)
seen_reply = SeenReply(reply=reply, journalist=g.user)
db.session.add(seen_reply)
db.session.commit()
store.async_add_checksum_for_file(reply, Storage.get_default())
except Exception as exc:
flash(
gettext("An unexpected error occurred! Please "
"inform your admin."), "error")
# We take a cautious approach to logging here because we're dealing
# with responses to sources. It's possible the exception message
# could contain information we don't want to write to disk.
current_app.logger.error(
"Reply from '{}' (ID {}) failed: {}!".format(
g.user.username, g.user.id, exc.__class__))
else:
flash(
Markup("<b>{}</b> {}".format(
# Translators: Precedes a message confirming the success of an operation.
escape(gettext("Success!")),
escape(
gettext("The source will receive your reply "
"next time they log in.")),
)),
"success",
)
finally:
return redirect(url_for("col.col", filesystem_id=g.filesystem_id))