def make_error_sentry(sentry_type, tags=None):
dsn = get_dsn_from_string(sentry_type)
tags = tags or {}
try:
return ErrorSentry(dsn,
sentry_client_kwargs={
'tags': tags,
'transport': HTTPTransport
})
except InvalidDsn as e:
log_error(e)
return ErrorHandler()
def make_error_sentry(sentry_type, tags=None):
""" Creates an ErrorSentry, defaults to error limit 10. """
dsn = get_dsn_from_string(sentry_type)
tags = tags or {}
try:
return ErrorSentry(dsn,
sentry_client_kwargs={
'tags': tags,
'transport': HTTPTransport
},
sentry_report_limit=10)
except InvalidDsn as e:
log_error(e)
return ErrorHandler()
def make_error_sentry(sentry_type, tags=None, null=False):
""" Creates an ErrorSentry, defaults to error limit 10.
If the applicable sentry DSN is missing will return an ErrorSentry,
but if null truthy a NullErrorHandler will be returned instead. """
dsn = get_dsn_from_string(sentry_type)
tags = tags or {}
try:
return ErrorSentry(
dsn,
sentry_client_kwargs={'tags': tags, 'transport': HTTPTransport},
sentry_report_limit=10
)
except InvalidDsn as e:
log_error(e)
if null:
return NullErrorHandler()
else:
return ErrorHandler()
def upload(OS_API=""):
""" Entry point to upload GPS, Accelerometer, Audio, PowerState, Calls Log, Texts Log,
Survey Response, and debugging files to s3.
Behavior:
The Beiwe app is supposed to delete the uploaded file if it receives an html 200 response.
The API returns a 200 response when the file has A) been successfully handled, B) the file it
has been sent is empty, C) the file did not decrypt properly. We encountered problems in
production with incorrectly encrypted files (as well as Android generating "rList" files
under unknown circumstances) and the app then uploads them. The source of encryption errors
is not well understood and could not be tracked down. In order to salvage partial data the
server decrypts files to the best of its ability and uploads it to S3. In order to delete
these files we still send a 200 response.
(The above about encryption is awful, in a theoretical version 2.0 the 200 response would be
replaced with a difference response code to allow for better debugging and less/fewer ... hax.)
A 400 error means there is something is wrong with the uploaded file or its parameters,
administrators will be emailed regarding this upload, the event will be logged to the apache
log. The app should not delete the file, it should try to upload it again at some point.
If a 500 error occurs that means there is something wrong server side, administrators will be
emailed and the event will be logged. The app should not delete the file, it should try to
upload it again at some point.
Request format:
send an http post request to [domain name]/upload, remember to include security
parameters (see user_authentication for documentation). Provide the contents of the file,
encrypted (see encryption specification) and properly converted to Base64 encoded text,
as a request parameter entitled "file".
Provide the file name in a request parameter entitled "file_name". """
patient_id = request.values['patient_id']
user = Participant.objects.get(patient_id=patient_id)
# Slightly different values for iOS vs Android behavior.
# Android sends the file data as standard form post parameter (request.values)
# iOS sends the file as a multipart upload (so ends up in request.files)
# if neither is found, consider the "body" of the post the file
# ("body" post is not currently used by any client, only here for completeness)
if "file" in request.files:
uploaded_file = request.files['file']
elif "file" in request.values:
uploaded_file = request.values['file']
else:
uploaded_file = request.data
if isinstance(uploaded_file, FileStorage):
uploaded_file = uploaded_file.read()
file_name = request.values['file_name']
# print "uploaded file name:", file_name, len(uploaded_file)
if "crashlog" in file_name.lower():
send_android_error_report(patient_id, uploaded_file)
return render_template('blank.html'), 200
if file_name[:6] == "rList-":
return render_template('blank.html'), 200
client_private_key = get_client_private_key(patient_id,
user.study.object_id)
try:
uploaded_file = decrypt_device_file(patient_id, uploaded_file,
client_private_key, user)
except HandledError as e:
# when decrypting fails, regardless of why, we rely on the decryption code
# to log it correctly and return 200 OK to get the device to delete the file.
# We do not want emails on these types of errors, so we use log_error explicitly.
print("the following error was handled:")
log_error(e, "%s; %s; %s" % (patient_id, file_name, e.message))
return render_template('blank.html'), 200
except OurBase64Error:
if IS_STAGING:
print("decryption problems" + "#" * 200)
print(patient_id)
print(file_name)
print(uploaded_file)
raise
# This is what the decryption failure mode SHOULD be, but we are still identifying the decryption bug
# except DecryptionKeyInvalidError:
# return render_template('blank.html'), 200
# print "decryption success:", file_name
# if uploaded data a) actually exists, B) is validly named and typed...
if uploaded_file and file_name and contains_valid_extension(file_name):
s3_upload(file_name.replace("_", "/"), uploaded_file,
user.study.object_id)
FileToProcess.append_file_for_processing(file_name.replace("_", "/"),
user.study.object_id,
participant=user)
UploadTracking.objects.create(
file_path=file_name.replace("_", "/"),
file_size=len(uploaded_file),
timestamp=timezone.now(),
participant=user,
)
return render_template('blank.html'), 200
else:
error_message = "an upload has failed " + patient_id + ", " + file_name + ", "
if not uploaded_file:
# it appears that occasionally the app creates some spurious files
# with a name like "rList-org.beiwe.app.LoadingActivity"
error_message += "there was no/an empty file, returning 200 OK so device deletes bad file."
log_error(Exception("upload error"), error_message)
return render_template('blank.html'), 200
elif not file_name:
error_message += "there was no provided file name, this is an app error."
elif file_name and not contains_valid_extension(file_name):
error_message += "contains an invalid extension, it was interpretted as "
error_message += grab_file_extension(file_name)
else:
error_message += "AN UNKNOWN ERROR OCCURRED."
tags = {"upload_error": "upload error", "user_id": patient_id}
sentry_client = make_sentry_client('eb', tags)
sentry_client.captureMessage(error_message)
# log_and_email_500_error(Exception("upload error"), error_message)
return abort(400)
def decrypt_device_file(patient_id, original_data, private_key, user):
""" Runs the line-by-line decryption of a file encrypted by a device.
This function is a special handler for iOS file uploads. """
def create_line_error_db_entry(error_type):
# declaring this inside decrypt device file to access its function-global variables
if IS_STAGING:
LineEncryptionError.create( {
"type": error_type,
"line": encode_base64(line),
"base64_decryption_key": encode_base64(private_key.decrypt(decoded_key)),
"prev_line": encode_base64(file_data[i - 1]) if i > 0 else None,
"next_line": encode_base64(file_data[i + 1]) if i < len(file_data) - 1 else None },
random_id=True
)
bad_lines = []
error_types = []
error_count = 0
return_data = ""
file_data = [line for line in original_data.split('\n') if line != ""]
if not file_data:
raise HandledError("The file had no data in it. Return 200 to delete file from device.")
try: #get the decryption key from the file.
decoded_key = decode_base64(file_data[0].encode("utf-8"))
decrypted_key = decode_base64(private_key.decrypt( decoded_key ) )
except (TypeError, IndexError) as e:
DecryptionKeyError.create( {
"file_path": request.values['file_name'],
"contents": original_data,
"user_id": user._id },
random_id=True
)
raise DecryptionKeyInvalidError("invalid decryption key. %s" % e.message)
# (we have an inefficiency in this encryption process, this might not need to be doubly
# encoded in base64. This is probably never going to be changed.)
# The following is all error catching code for bugs we encountered (and solved) in development.
# print "length decrypted key", len(decrypted_key)
for i, line in enumerate(file_data):
#we need to skip the first line (the decryption key), but need real index values in i
if i==0: continue
if line is None:
#this case causes weird behavior inside decrypt_device_line, so we test for it instead.
error_count += 1
create_line_error_db_entry(LINE_IS_NONE)
error_types.append(LINE_IS_NONE)
bad_lines.append(line)
print "encountered empty line of data, ignoring."
continue
try:
return_data += decrypt_device_line(patient_id, decrypted_key, line) + "\n"
except Exception as e:
error_count += 1
error_message = "There was an error in user decryption: "
if isinstance(e, IndexError):
error_message += "Something is wrong with data padding:\n\tline: %s" % line
log_error(e, error_message)
create_line_error_db_entry(PADDING_ERROR)
error_types.append(PADDING_ERROR)
bad_lines.append(line)
continue
if isinstance(e, TypeError) and decrypted_key is None:
error_message += "The key was empty:\n\tline: %s" % line
log_error(e, error_message)
create_line_error_db_entry(EMPTY_KEY)
error_types.append(EMPTY_KEY)
bad_lines.append(line)
continue
################### skip these errors ##############################
if "unpack" in e.message:
error_message += "malformed line of config, dropping it and continuing."
log_error(e, error_message)
create_line_error_db_entry(MALFORMED_CONFIG)
error_types.append(MALFORMED_CONFIG)
bad_lines.append(line)
#the config is not colon separated correctly, this is a single
# line error, we can just drop it.
# implies an interrupted write operation (or read)
continue
if "Input strings must be a multiple of 16 in length" in e.message:
error_message += "Line was of incorrect length, dropping it and continuing."
log_error(e, error_message)
create_line_error_db_entry(INVALID_LENGTH)
error_types.append(INVALID_LENGTH)
bad_lines.append(line)
continue
if isinstance(e, InvalidData):
error_message += "Line contained no data, skipping: " + str(line)
log_error(e, error_message)
create_line_error_db_entry(LINE_EMPTY)
error_types.append(LINE_EMPTY)
bad_lines.append(line)
continue
if isinstance(e, InvalidIV):
error_message += "Line contained no iv, skipping: " + str(line)
log_error(e, error_message)
create_line_error_db_entry(IV_MISSING)
error_types.append(IV_MISSING)
bad_lines.append(line)
continue
##################### flip out on these errors #####################
if 'AES key' in e.message:
error_message += "AES key has bad length."
create_line_error_db_entry(AES_KEY_BAD_LENGTH)
error_types.append(AES_KEY_BAD_LENGTH)
bad_lines.append(line)
elif 'IV must be' in e.message:
error_message += "iv has bad length."
create_line_error_db_entry(IV_BAD_LENGTH)
error_types.append(IV_BAD_LENGTH)
bad_lines.append(line)
elif 'Incorrect padding' in e.message:
error_message += "base64 padding error, config is truncated."
create_line_error_db_entry(MP4_PADDING)
error_types.append(MP4_PADDING)
bad_lines.append(line)
# this is only seen in mp4 files. possibilities:
# upload during write operation.
# broken base64 conversion in the app
# some unanticipated error in the file upload
else:
raise #If none of the above errors happened, raise the error.
raise HandledError(error_message)
# if any of them did happen, raise a HandledError to cease execution.
if error_count:
EncryptionErrorMetadata.create( {
"file_name": request.values['file_name'],
"total_lines": len(file_data),
"number_errors": error_count,
"errors_lines": bad_lines,
"error_types": error_types},
random_id=True
)
return return_data
def decrypt_device_file(patient_id, original_data: bytes, private_key_cipher,
user) -> bytes:
""" Runs the line-by-line decryption of a file encrypted by a device.
This function is a special handler for iOS file uploads. """
def create_line_error_db_entry(error_type):
# declaring this inside decrypt device file to access its function-global variables
if IS_STAGING:
LineEncryptionError.objects.create(
type=error_type,
base64_decryption_key=private_key_cipher.decrypt(decoded_key),
line=encode_base64(line),
prev_line=encode_base64(file_data[i - 1] if i > 0 else ''),
next_line=encode_base64(file_data[i +
1] if i < len(file_data) -
1 else ''),
participant=user,
)
def create_decryption_key_error(an_traceback):
DecryptionKeyError.objects.create(
file_path=request.values['file_name'],
contents=original_data,
traceback=an_traceback,
participant=user,
)
bad_lines = []
error_types = []
error_count = 0
good_lines = []
file_data = [line for line in original_data.split(b'\n') if line != b""]
if not file_data:
raise HandledError(
"The file had no data in it. Return 200 to delete file from device."
)
# The following code is strange because of an unfortunate design design decision made quite
# some time ago: the decryption key is encoded as base64 twice, once wrapping the output of the
# RSA encryption, and once wrapping the AES decryption key.
# The second of the two except blocks likely means that the device failed to write the encryption
# key as the first line of the file, but it may be a valid (but undecryptable) line of the file.
try:
decoded_key = decode_base64(file_data[0])
except (TypeError, IndexError, PaddingException) as decode_error:
create_decryption_key_error(traceback.format_exc())
raise DecryptionKeyInvalidError("invalid decryption key. %s" %
decode_error)
try:
base64_key = private_key_cipher.decrypt(decoded_key)
decrypted_key = decode_base64(base64_key)
except (TypeError, IndexError, PaddingException) as decr_error:
create_decryption_key_error(traceback.format_exc())
raise DecryptionKeyInvalidError("invalid decryption key. %s" %
decr_error)
for i, line in enumerate(file_data):
# we need to skip the first line (the decryption key), but need real index values in i
if i == 0:
continue
if line is None:
# this case causes weird behavior inside decrypt_device_line, so we test for it instead.
error_count += 1
create_line_error_db_entry(LineEncryptionError.LINE_IS_NONE)
error_types.append(LineEncryptionError.LINE_IS_NONE)
bad_lines.append(line)
print("encountered empty line of data, ignoring.")
continue
try:
good_lines.append(
decrypt_device_line(patient_id, decrypted_key, line))
except Exception as error_orig:
error_string = str(error_orig)
error_count += 1
error_message = "There was an error in user decryption: "
if isinstance(error_string, IndexError):
error_message += "Something is wrong with data padding:\n\tline: %s" % line
log_error(error_string, error_message)
create_line_error_db_entry(LineEncryptionError.PADDING_ERROR)
error_types.append(LineEncryptionError.PADDING_ERROR)
bad_lines.append(line)
continue
if isinstance(error_string, TypeError) and decrypted_key is None:
error_message += "The key was empty:\n\tline: %s" % line
log_error(error_string, error_message)
create_line_error_db_entry(LineEncryptionError.EMPTY_KEY)
error_types.append(LineEncryptionError.EMPTY_KEY)
bad_lines.append(line)
continue
################### skip these errors ##############################
if "unpack" in error_string:
error_message += "malformed line of config, dropping it and continuing."
log_error(error_string, error_message)
create_line_error_db_entry(
LineEncryptionError.MALFORMED_CONFIG)
error_types.append(LineEncryptionError.MALFORMED_CONFIG)
bad_lines.append(line)
#the config is not colon separated correctly, this is a single
# line error, we can just drop it.
# implies an interrupted write operation (or read)
continue
if "Input strings must be a multiple of 16 in length" in error_string:
error_message += "Line was of incorrect length, dropping it and continuing."
log_error(error_string, error_message)
create_line_error_db_entry(LineEncryptionError.INVALID_LENGTH)
error_types.append(LineEncryptionError.INVALID_LENGTH)
bad_lines.append(line)
continue
if isinstance(error_string, InvalidData):
error_message += "Line contained no data, skipping: " + str(
line)
log_error(error_string, error_message)
create_line_error_db_entry(LineEncryptionError.LINE_EMPTY)
error_types.append(LineEncryptionError.LINE_EMPTY)
bad_lines.append(line)
continue
if isinstance(error_string, InvalidIV):
error_message += "Line contained no iv, skipping: " + str(line)
log_error(error_string, error_message)
create_line_error_db_entry(LineEncryptionError.IV_MISSING)
error_types.append(LineEncryptionError.IV_MISSING)
bad_lines.append(line)
continue
##################### flip out on these errors #####################
if 'AES key' in error_string:
error_message += "AES key has bad length."
create_line_error_db_entry(
LineEncryptionError.AES_KEY_BAD_LENGTH)
error_types.append(LineEncryptionError.AES_KEY_BAD_LENGTH)
bad_lines.append(line)
elif 'IV must be' in error_string:
error_message += "iv has bad length."
create_line_error_db_entry(LineEncryptionError.IV_BAD_LENGTH)
error_types.append(LineEncryptionError.IV_BAD_LENGTH)
bad_lines.append(line)
elif 'Incorrect padding' in error_string:
error_message += "base64 padding error, config is truncated."
create_line_error_db_entry(LineEncryptionError.MP4_PADDING)
error_types.append(LineEncryptionError.MP4_PADDING)
bad_lines.append(line)
# this is only seen in mp4 files. possibilities:
# upload during write operation.
# broken base64 conversion in the app
# some unanticipated error in the file upload
else:
# If none of the above errors happened, raise the error raw
raise
raise HandledError(error_message)
# if any of them did happen, raise a HandledError to cease execution.
if error_count:
EncryptionErrorMetadata.objects.create(
file_name=request.values['file_name'],
total_lines=len(file_data),
number_errors=error_count,
# generator comprehension:
error_lines=json.dumps((str(line for line in bad_lines))),
error_types=json.dumps(error_types),
participant=user,
)
# join should be rather well optimized and not cause O(n^2) total memory copies
return b"\n".join(good_lines)
def upload(OS_API=""):
""" Entry point to upload GPS, Accelerometer, Audio, PowerState, Calls Log, Texts Log,
Survey Response, and debugging files to s3.
Behavior:
The Beiwe app is supposed to delete the uploaded file if it receives an html 200 response.
The API returns a 200 response when the file has A) been successfully handled, B) the file it
has been sent is empty, C) the file did not decrypt properly. We encountered problems in
production with incorrectly encrypted files (as well as Android generating "rList" files
under unknown circumstances) and the app then uploads them. The source of encryption errors
is not well understood and could not be tracked down. In order to salvage partial data the
server decrypts files to the best of its ability and uploads it to S3. In order to delete
these files we still send a 200 response.
(The above about encryption is awful, in a theoretical version 2.0 the 200 response would be
replaced with a difference response code to allow for better debugging and less/fewer ... hax.)
A 400 error means there is something is wrong with the uploaded file or its parameters,
administrators will be emailed regarding this upload, the event will be logged to the apache
log. The app should not delete the file, it should try to upload it again at some point.
If a 500 error occurs that means there is something wrong server side, administrators will be
emailed and the event will be logged. The app should not delete the file, it should try to
upload it again at some point.
Request format:
send an http post request to [domain name]/upload, remember to include security
parameters (see user_authentication for documentation). Provide the contents of the file,
encrypted (see encryption specification) and properly converted to Base64 encoded text,
as a request parameter entitled "file".
Provide the file name in a request parameter entitled "file_name". """
patient_id = request.values['patient_id']
user = Participant.objects.get(patient_id=patient_id)
# Slightly different values for iOS vs Android behavior.
# Android sends the file data as standard form post parameter (request.values)
# iOS sends the file as a multipart upload (so ends up in request.files)
# if neither is found, consider the "body" of the post the file
# ("body" post is not currently used by any client, only here for completeness)
if "file" in request.files:
uploaded_file = request.files['file']
elif "file" in request.values:
uploaded_file = request.values['file']
else:
uploaded_file = request.data
if isinstance(uploaded_file, FileStorage):
uploaded_file = uploaded_file.read()
file_name = request.values['file_name']
def save(file_name, uploaded_file):
uploaded_file0 = uploaded_file
error_count = 0
if "crashlog" in file_name.lower():
send_android_error_report(patient_id, uploaded_file)
return render_template('blank.html'), 200
# it appears that occasionally the app creates some spurious files with a name like "rList-org.beiwe.app.LoadingActivity"
if file_name[:6] == "rList-":
return render_template('blank.html'), 200
# test whether can decrypt successfully
# if cannot decrypt, save the raw file, return OK:200 to free up phone storage
# if cannot save to S3 bucket, return Error:500 to postpone upload & keep the file on the phone
client_private_key = get_client_private_key(patient_id,
user.study.object_id)
try:
uploaded_file, error_count = decrypt_device_file(
patient_id, uploaded_file, client_private_key, user)
except HandledError as e:
canUpload = s3_upload(file_name.replace("_", "/"),
uploaded_file,
user.study.object_id,
encrypt=False)
print("The following upload error was handled:")
log_error(e, "%s; %s; %s" % (patient_id, file_name, e.message))
return render_template('blank.html'), 200 if canUpload else 500
except OurBase64Error:
canUpload = s3_upload(file_name.replace("_", "/"),
uploaded_file,
user.study.object_id,
encrypt=False)
print(
"### decryption error: patient_id=%s, file_name=%s, file_size=%s"
% (patient_id, file_name, len(uploaded_file)))
return render_template('blank.html'), 200 if canUpload else 500
except:
canUpload = s3_upload(file_name.replace("_", "/"),
uploaded_file,
user.study.object_id,
encrypt=False)
return render_template('blank.html'), 200 if canUpload else 500
# set upload info
file_basename = file_name.split('_')[-2]
if file_basename in CHECKABLE_FILES:
try:
upload_info = user.get_upload_info()
update_upload_info(file_basename, upload_info,
uploaded_file.strip().splitlines()[1:],
2 if file_basename == 'callLog' else 0)
user.set_upload_info(upload_info)
except Exception as e:
log_error(
e,
"Failed to update upload info: patient_id=%s; file_name=%s; msg=%s"
% (patient_id, file_name, e.message))
# if uploaded data a) actually exists, B) is validly named and typed...
if uploaded_file and file_name and contains_valid_extension(file_name):
canUpload = s3_upload(file_name.replace("_", "/"), uploaded_file,
user.study.object_id)
user.set_upload_time()
# for files with non-fatal decryption errors, save another raw copy
if canUpload and error_count > 0:
canUpload = s3_upload(file_name.replace("_", "/"),
uploaded_file0,
user.study.object_id,
encrypt=False)
return render_template('blank.html'), 200 if canUpload else 500
else:
error_message = "an upload has failed " + patient_id + ", " + file_name + ", "
canUpload = s3_upload(file_name.replace("_", "/"),
uploaded_file,
user.study.object_id,
encrypt=False)
user.set_upload_time()
if not uploaded_file:
error_message += "there was an empty file, returning 200 OK so device deletes bad file."
log_error(Exception("upload error"), error_message)
return render_template('blank.html'), 200 if canUpload else 500
elif not file_name:
error_message += "there was no provided file name, this is an app error."
elif not contains_valid_extension(file_name):
error_message += "contains an invalid extension, it was interpretted as "
error_message += grab_file_extension(file_name)
else:
error_message += "AN UNKNOWN ERROR OCCURRED."
tags = {"upload_error": "upload error", "user_id": patient_id}
sentry_client = make_sentry_client('eb', tags)
sentry_client.captureMessage(error_message)
# log_and_email_500_error(Exception("upload error"), error_message)
return render_template('blank.html'), 200 if canUpload else 500
# save file directly or unzip and store each extracted file
if file_name.lower().endswith('.zip'):
try:
zip_obj = ZipFile(StringIO(uploaded_file))
except Exception as e:
log_error(
e,
"Failed to unzip file: patient_id=%s; file_name=%s; msg=%s" %
(patient_id, file_name, e.message))
for fn in zip_obj.namelist():
try:
save(fn, zip_obj.read(fn))
except Exception as e:
log_error(
e,
"Failed to save file: patient_id=%s; zip_file=%s; inner_file=%s; msg=%s"
% (patient_id, file_name, fn, e.message))
return render_template('blank.html'), 200
else:
return save(file_name, uploaded_file)
def save(file_name, uploaded_file):
uploaded_file0 = uploaded_file
error_count = 0
if "crashlog" in file_name.lower():
send_android_error_report(patient_id, uploaded_file)
return render_template('blank.html'), 200
# it appears that occasionally the app creates some spurious files with a name like "rList-org.beiwe.app.LoadingActivity"
if file_name[:6] == "rList-":
return render_template('blank.html'), 200
# test whether can decrypt successfully
# if cannot decrypt, save the raw file, return OK:200 to free up phone storage
# if cannot save to S3 bucket, return Error:500 to postpone upload & keep the file on the phone
client_private_key = get_client_private_key(patient_id,
user.study.object_id)
try:
uploaded_file, error_count = decrypt_device_file(
patient_id, uploaded_file, client_private_key, user)
except HandledError as e:
canUpload = s3_upload(file_name.replace("_", "/"),
uploaded_file,
user.study.object_id,
encrypt=False)
print("The following upload error was handled:")
log_error(e, "%s; %s; %s" % (patient_id, file_name, e.message))
return render_template('blank.html'), 200 if canUpload else 500
except OurBase64Error:
canUpload = s3_upload(file_name.replace("_", "/"),
uploaded_file,
user.study.object_id,
encrypt=False)
print(
"### decryption error: patient_id=%s, file_name=%s, file_size=%s"
% (patient_id, file_name, len(uploaded_file)))
return render_template('blank.html'), 200 if canUpload else 500
except:
canUpload = s3_upload(file_name.replace("_", "/"),
uploaded_file,
user.study.object_id,
encrypt=False)
return render_template('blank.html'), 200 if canUpload else 500
# set upload info
file_basename = file_name.split('_')[-2]
if file_basename in CHECKABLE_FILES:
try:
upload_info = user.get_upload_info()
update_upload_info(file_basename, upload_info,
uploaded_file.strip().splitlines()[1:],
2 if file_basename == 'callLog' else 0)
user.set_upload_info(upload_info)
except Exception as e:
log_error(
e,
"Failed to update upload info: patient_id=%s; file_name=%s; msg=%s"
% (patient_id, file_name, e.message))
# if uploaded data a) actually exists, B) is validly named and typed...
if uploaded_file and file_name and contains_valid_extension(file_name):
canUpload = s3_upload(file_name.replace("_", "/"), uploaded_file,
user.study.object_id)
user.set_upload_time()
# for files with non-fatal decryption errors, save another raw copy
if canUpload and error_count > 0:
canUpload = s3_upload(file_name.replace("_", "/"),
uploaded_file0,
user.study.object_id,
encrypt=False)
return render_template('blank.html'), 200 if canUpload else 500
else:
error_message = "an upload has failed " + patient_id + ", " + file_name + ", "
canUpload = s3_upload(file_name.replace("_", "/"),
uploaded_file,
user.study.object_id,
encrypt=False)
user.set_upload_time()
if not uploaded_file:
error_message += "there was an empty file, returning 200 OK so device deletes bad file."
log_error(Exception("upload error"), error_message)
return render_template('blank.html'), 200 if canUpload else 500
elif not file_name:
error_message += "there was no provided file name, this is an app error."
elif not contains_valid_extension(file_name):
error_message += "contains an invalid extension, it was interpretted as "
error_message += grab_file_extension(file_name)
else:
error_message += "AN UNKNOWN ERROR OCCURRED."
tags = {"upload_error": "upload error", "user_id": patient_id}
sentry_client = make_sentry_client('eb', tags)
sentry_client.captureMessage(error_message)
# log_and_email_500_error(Exception("upload error"), error_message)
return render_template('blank.html'), 200 if canUpload else 500
def upload(OS_API=""):
""" Entry point to upload GPS, Accelerometer, Audio, PowerState, Calls Log, Texts Log,
Survey Response, and debugging files to s3.
Behavior:
The Beiwe app is supposed to delete the uploaded file if it receives an html 200 response.
The API returns a 200 response when the file has A) been successfully handled, B) the file it
has been sent is empty, C) the file did not decrypt properly. We encountered problems in
production with incorrectly encrypted files (as well as Android generating "rList" files
under unknown circumstances) and the app then uploads them. When the device receives a 200
that is its signal to delete the file.
When a file is undecryptable (this was tracked to a scenario where the device could not
create/write an AES encryption key) we send a 200 response to stop that device attempting to
re-upload the data.
In the event of a single line being undecryptable (can happen due to io errors on the device)
we drop only that line (and store the erroring line in an attempt to track it down.
A 400 error means there is something is wrong with the uploaded file or its parameters,
administrators will be emailed regarding this upload, the event will be logged to the apache
log. The app should not delete the file, it should try to upload it again at some point.
If a 500 error occurs that means there is something wrong server side, administrators will be
emailed and the event will be logged. The app should not delete the file, it should try to
upload it again at some point.
Request format:
send an http post request to [domain name]/upload, remember to include security
parameters (see user_authentication for documentation). Provide the contents of the file,
encrypted (see encryption specification) and properly converted to Base64 encoded text,
as a request parameter entitled "file".
Provide the file name in a request parameter entitled "file_name". """
# Handle these corner cases first because they requires no database input.
# Crash logs are from truly ancient versions of the android codebase
file_name = request.values['file_name']
if file_name[:6] == "rList-" or "crashlog" in file_name.lower():
return render_template('blank.html'), 200
patient_id = request.values['patient_id']
user = Participant.objects.get(patient_id=patient_id)
# Slightly different values for iOS vs Android behavior.
# Android sends the file data as standard form post parameter (request.values)
# iOS sends the file as a multipart upload (so ends up in request.files)
# if neither is found, consider the "body" of the post the file
# ("body" post is not currently used by any client, only here for completeness)
if "file" in request.files:
uploaded_file = request.files['file']
elif "file" in request.values:
uploaded_file = request.values['file']
else:
uploaded_file = request.data
if isinstance(uploaded_file, FileStorage):
uploaded_file = uploaded_file.read()
elif isinstance(uploaded_file, str):
uploaded_file = uploaded_file.encode()
elif isinstance(uploaded_file, bytes):
# not current behavior on any app
pass
else:
raise TypeError("uploaded_file was a %s" % type(uploaded_file))
# print("uploaded file name:", file_name, len(uploaded_file))
client_private_key = get_client_private_key(patient_id,
user.study.object_id)
try:
uploaded_file = decrypt_device_file(patient_id, uploaded_file,
client_private_key, user)
except HandledError as e:
# when decrypting fails, regardless of why, we rely on the decryption code
# to log it correctly and return 200 OK to get the device to delete the file.
# We do not want emails on these types of errors, so we use log_error explicitly.
print("the following error was handled:")
log_error(e, "%s; %s; %s" % (patient_id, file_name, e))
return render_template('blank.html'), 200
except DecryptionKeyInvalidError:
# when the decryption key is invalid the file is lost. Nothing we can do.
# record the event, send the device a 200 so it can clear out the file.
tags = {
"participant": patient_id,
"operating system":
"ios" if "ios" in request.path.lower() else "android",
"DecryptionKeyError id": str(DecryptionKeyError.objects.last().id),
"file_name": file_name,
}
make_sentry_client('eb',
tags).captureMessage("DecryptionKeyInvalidError")
return render_template('blank.html'), 200
# print "decryption success:", file_name
# if uploaded data a) actually exists, B) is validly named and typed...
if uploaded_file and file_name and contains_valid_extension(file_name):
s3_upload(file_name.replace("_", "/"), uploaded_file,
user.study.object_id)
FileToProcess.append_file_for_processing(file_name.replace("_", "/"),
user.study.object_id,
participant=user)
UploadTracking.objects.create(
file_path=file_name.replace("_", "/"),
file_size=len(uploaded_file),
timestamp=timezone.now(),
participant=user,
)
return render_template('blank.html'), 200
else:
error_message = "an upload has failed " + patient_id + ", " + file_name + ", "
if not uploaded_file:
# it appears that occasionally the app creates some spurious files
# with a name like "rList-org.beiwe.app.LoadingActivity"
error_message += "there was no/an empty file, returning 200 OK so device deletes bad file."
log_error(Exception("upload error"), error_message)
return render_template('blank.html'), 200
elif not file_name:
error_message += "there was no provided file name, this is an app error."
elif file_name and not contains_valid_extension(file_name):
error_message += "contains an invalid extension, it was interpretted as "
error_message += grab_file_extension(file_name)
else:
error_message += "AN UNKNOWN ERROR OCCURRED."
tags = {"upload_error": "upload error", "user_id": patient_id}
sentry_client = make_sentry_client('eb', tags)
sentry_client.captureMessage(error_message)
return abort(400)