def make_error_sentry(sentry_type, tags=None):
dsn = get_dsn_from_string(sentry_type)
tags = tags or {}
return ErrorSentry(dsn,
sentry_client_kwargs={
'tags': tags,
'transport': HTTPTransport
})
def make_error_sentry(sentry_type, tags=None):
dsn = get_dsn_from_string(sentry_type)
tags = tags or {}
try:
return ErrorSentry(dsn, sentry_client_kwargs={'tags': tags, 'transport': HTTPTransport})
except InvalidDsn as e:
log_error(e)
return ErrorHandler()
def create_file_processing_tasks():
#literally wrapping the entire thing in an ErrorSentry...
with ErrorSentry(SENTRY_DSN,
sentry_client_kwargs={'transport':
HTTPTransport}) as error_sentry:
print error_sentry.sentry_client.is_enabled()
if FileProcessLock.islocked():
# This is really a safety check to ensure that no code executes if this runs
report_file_processing_locked_and_exit()
exit(0)
else:
FileProcessLock.lock()
print "starting."
now = datetime.now()
expiry = now + timedelta(minutes=CELERY_EXPIRY_MINUTES)
user_ids = get_user_list_safely()
running = []
for user_id in user_ids:
# queue all users, get list of futures to check
running.append(
safe_queue_user(args=[user_id],
max_retries=0,
expires=expiry,
task_track_started=True,
task_publish_retry=False,
retry=False))
print "tasks:", running
while running:
new_running = []
failed = []
successful = []
for future in running:
####################################################################################
# This variable can mutate on a separate thread. We need the value as it was at
# this snapshot in time, so we store it. (The object is a string, passed by value.)
####################################################################################
state = future.state
if state == SUCCESS:
successful.append(future)
if state in FAILED:
failed.append(future)
if state in STARTED_OR_WAITING:
new_running.append(future)
running = new_running
print "tasks:", running
if running:
sleep(5)
print "Finished, unlocking."
FileProcessLock.unlock(
) # This MUST be ___inside___ the with statement.
def make_error_sentry(sentry_type, tags=None):
""" Creates an ErrorSentry, defaults to error limit 10. """
dsn = get_dsn_from_string(sentry_type)
tags = tags or {}
try:
return ErrorSentry(dsn,
sentry_client_kwargs={
'tags': tags,
'transport': HTTPTransport
},
sentry_report_limit=10)
except InvalidDsn as e:
log_error(e)
return ErrorHandler()
def send_android_error_report(user_id, error_report):
# Encountered a corrupted (write error) error report upload on Apr 30 2017, adding error sentry
# so that we get *some* report of the error occuring but also delete that file from the device.
with ErrorSentry(SENTRY_DSN, sentry_client_kwargs={'transport': HTTPTransport}):
#get all non-empty lines in the error report
contents = [line for line in error_report.splitlines() if line.strip() ]
# the first line contains a unix millisecond timestamp, construct a datetime
# The printed value in the crash report is in UTC
try: #Beiwe version greater than 4
timestamp = datetime.fromtimestamp(float(contents[0]) / 1000)
contents.pop(0) #remove timestamp from message text
except ValueError: #Beiwe version 4
timestamp = datetime.fromtimestamp(float(request.values['file_name'].split("_")[1]) / 1000)
device_identifiers = contents[0].split(',')
contents.pop(0) # remove device identifiers from message text
# Insert the actual error message as the first line
report_title = contents[0].split(":", 1)[1].strip()
if "}" in report_title: #cut title at end of file name
report_title = report_title.split("}", 1)[0] + "}"
contents.insert(0, "Android Error: %s" % report_title)
# the second line contains all the identifiers. Clean it up and parse into a dictionary.
device_identifiers = {ID.strip().split(":",1)[0] : ID.strip().split(":",1)[1]
for ID in device_identifiers}
#get a useful timestamp...
eastern_time = timestamp.replace(tzinfo=tz.gettz('UTC')).astimezone(tz.gettz('America/New_York'))
#construct some useful tags for this error report, add all identifiers as tags.
tags = {"Android_Error": "android error",
"user_id":user_id,
"date": str(timestamp.date()),
"time": str(timestamp).split(" ")[1].split(".")[0],
"eastern_date": str(eastern_time.date()),
"eastern_time": str(eastern_time).split(" ")[1].split(".")[0]
}
tags.update(device_identifiers)
sentry_client = SentryClient(dsn=SENTRY_DSN,
tags=tags,
transport=HTTPTransport )
sentry_client.captureMessage("\n".join(contents))
def celery_process_file_chunks(user_id):
""" This is the function that is called from cron. It runs through all new files that have
been uploaded and 'chunks' them. Handles logic for skipping bad files, raising errors
appropriately. """
log = LogList()
number_bad_files = 0
error_sentry = ErrorSentry(SENTRY_DSN,
sentry_client_kwargs={
"tags": {
"user_id": user_id
},
'transport': HTTPTransport
})
log.append("processing files for %s" % user_id)
while True:
previous_number_bad_files = number_bad_files
starting_length = FilesToProcess.count(user_id=user_id)
log.append(
str(datetime.now()) + " processing %s, %s files remaining" %
(user_id, starting_length))
number_bad_files += do_process_user_file_chunks(
count=FILE_PROCESS_PAGE_SIZE,
error_handler=error_sentry,
skip_count=number_bad_files,
user_id=user_id)
if starting_length == FilesToProcess.count(
user_id=user_id): # zero files processed
if previous_number_bad_files == number_bad_files:
# Cases:
# every file broke, blow up. (would cause infinite loop otherwise)
# no new files.
break
else:
continue
def make_synchronous_error_sentry(tags=None):
"""Uses synchronous transport HTTPTransport in order to make error sentry reports"""
if settings.TESTING:
return NullErrorHandler()
tags = tags or {}
return ErrorSentry(SENTRY_DSN, sentry_client_kwargs={'tags': tags, 'transport': HTTPTransport})