def unsubscribe_view(request,
default_active='',
template='templates/master.pt'):
renderer = get_renderer(template)
message = "We are so sorry your request could not be completed. Please contact [email protected] if you need assistance."
email = '*****@*****.**'
url = ''
substatus = 500
rc_id = ''
try:
if 'unsubscribe.submitted' in request.params:
email = request.params['email']
url = request.route_path("unsubscribe")
rc_id = request.params['rc_id']
if rc_id == None and email == None:
raise
site = rc_id[:-5].upper()
redcap = RCProject([site], rcs)
fields = ['rc_id', 'testing_reminder']
records = redcap.project[site].export_records(records=[rc_id],
fields=fields)
if len(records) == 1:
if records[0]['testing_reminder'] == u'1':
records[0]['testing_reminder'] = u'0'
if redcap.project[site].import_records(records) == 0:
raise
message = "Your email address " + email + " has been in thennehwin thehennhd removed from our mailing list."
substatus = 200
else:
message = "Our records indicate " + email + " is not subscribed to receive any emails. Please contact [email protected] if you need assistance."
substatus = 200
else:
message = "Test ID Unknown. Please contact [email protected] if you need assistance."
except:
log.debug(traceback.format_exc())
return \
{
'project' : 'aeh.earlytest',
'language' : request._LOCALE_,
'main_template' : renderer.implementation().macros['main_template'],
'default_active' : default_active,
'logged_in' : '',
'message' : message,
'email' : email,
'url' : url,
'rc_id' : rc_id,
'substatus' : substatus
}
def get_patient(rc_id):
try:
patient = None
patient_sites = Config.get('aeh:results','pat.codes').split()
#log.info(patient_sites)
redcap = RCProject(patient_sites, rcs)
for key in patient_sites:
try:
patients = redcap.project[key].export_records(records=[str(rc_id)])
patient = patients[0]
#log.info(patient)
return patient
except:
pass
#log.info(traceback.format_exc())
except:
pass
def get_patient(rc_id, settings):
try:
log.info("Enter Get_Patient")
patient = None
rcs = json.loads(open(settings['redcap_json'], 'r').read())
patient_sites = settings['pat.codes'].split()
redcap = RCProject(patient_sites, rcs)
for key in patient_sites:
try:
patients = redcap.project[key].export_records(records=[rc_id])
patient = patients[0]
break
except IndexError:
pass
except:
log.critical(traceback.format_exc())
log.info("Exit Get_Patient")
return patient
def send_reminder(settings):
"""
Parameters:
buckets: Dict of site codes and list of Results for that site as key, value pairs
settings: A dictionary of settings specified in the configuration file
Returns None
"""
try:
log.info("Early Test Email Alerts are about to be sent")
ses_key_id = settings['ses_key_id']
ses_key = settings['ses_key']
rcs = json.loads(open(settings['redcap_json'], 'r').read())
# Read from the configuration files
rem_keys = settings['rem.codes'].split()
pat_keys = settings['pat.codes'].split()
keys = rem_keys + pat_keys
months_to_notify = int(settings['months.tonotify'])
# Connect to all the required projects in RedCAP
redcap = RCProject(keys, rcs)
staff_emails = get_receipients(redcap, '1')
staff_emails_other = get_receipients(redcap, '2')
#send other email reminders
sdun_patient_count, sdun_emails_sent, sdun_invalid_emails_count = send_reminder_single(
ses_key_id, ses_key, settings, staff_emails_other,
months_to_notify)
er_patient_count, er_emails_sent, er_invalid_emails_count, asr_patient_count, asr_emails_sent, asr_invalid_emails_count = send_reminder_etc(
ses_key_id, ses_key, settings, staff_emails_other,
months_to_notify)
# Required Patient Fields
pfields = [
'et_pid', 'rc_id', 'phone1', 'phone2', 'email1', 'email2',
'first_name', 'last_name'
]
precords = {}
for key in pat_keys:
precords[key] = redcap.project[key].export_records(fields=pfields)
# This dictionary holds a map of Patient email to a list of
# rc_ids of that specific patient
# NOTE: The rc_ids may also belong to multiple sites.
hash_email = {}
# This is a dictionary to the email ids of patients to their names
hash_names = {}
# This dictionary will hold the list of rcid to specific
# sites, and therefore prevent multiple calls to the
# RedCAP rest APIs.
site_rcid = {}
for site in rem_keys:
site_rcid[site] = []
# Today we have a single redcap project for patients. In the future
# there can be multiple projects for patients. The below code will
# still work and have a better performance.
import sys
for pat_key, pat_records in precords.iteritems():
log.debug("Total of patient records in %s = %d", pat_key,
len(pat_records))
log.debug("The size of the records: %d Bytes",
sys.getsizeof(pat_records))
# Patient Records that need to be updated
# with working primary email for this site
to_update = []
invalid_emails = {}
for patient in pat_records:
try:
# Extract Site information
#site = patient['rc_id'].upper()
# The testing site visited by this patient is not configured
# to send email reminders. So continue to the next patient
#if site.startswith(tuple(rem_keys)) == False:
#continue
if patient['email1'] == '':
# ignore patients for whom we don't have email IDs
if patient['email2'] == '':
continue
else:
# Set the primary email and delete the secondary
patient['email1'] = patient['email2']
patient['email2'] = ''
to_update.append(patient)
if patient['et_pid'] in hash_email:
continue
else:
hash_email[patient['et_pid']] = [patient['email1']]
hash_names[patient['et_pid']] = {
'first_name': patient['first_name'],
'last_name': patient['last_name']
}
#site_rcid[patient['rc_id'][:-5]].append(patient['rc_id'])
except KeyError as e:
log.critical(traceback.format_exc())
pass
"""if len(to_update) > 0:
try:
#updated_count = redcap.project[pat_key].import_records(to_update)
log.debug(" %d Emails needed to be moved from email2 to email1 for %s", len(to_update), pat_key)
except:
log.critical(traceback.format_exc())
pass"""
# Some stats to log
log.debug("Unique Patient with email Ids: %d",
len(hash_email.keys()))
#for email, visits in hash_email.iteritems():
# log.debug("Patient Email: %s, Number of Visits: %d", email, len(visits))
#History of the patients as a list returned by redcap
patient_history = []
# We need the following fields to decide if we need to send emails for this patient
fields = [
'et_pid', 'rc_id', 'visit_date', 'rapid1', 'rapid2',
'testing_reminder', 'dhiv', 'lstremndr_dt'
]
#for site, records in hash_email.iteritems():
patient_history.extend(
redcap.project['SDET'].export_records(fields=fields))
patient_history.extend(
redcap.project['76C'].export_records(fields=fields))
log.debug("Total Patient history to Process:%d",
len(patient_history))
# Patient history modified data structure for convenience
hist_map = {}
for history in patient_history:
hist_map[history['rc_id']] = history
count = 0
for key, val in hash_email.iteritems():
print 'key'
print key
print 'value'
print val
latest_record = None
# At the end of the following loop, record will consists of the latest visit
# information for this patient.(Indicated by the email string 'key')
skip = False
for rc_id, visit_val in hist_map.iteritems():
try:
if visit_val['et_pid'] != key:
continue
if visit_val['visit_date'] == '':
continue
print 'visit date exists'
print 'rcid'
print rc_id
print 'visit'
print visit_val
visit = datetime.strptime(visit_val['visit_date'],
"%Y-%m-%d")
if latest_record == None:
latest_record = visit_val
else:
latest_date = datetime.strptime(
latest_record['visit_date'], "%Y-%m-%d")
if latest_date <= visit:
print 'compare dates'
print visit
print latest_date
latest_record = hist_map[rc_id]
except KeyError:
log.critical(traceback.format_exc())
print 'latest record'
print latest_record
if latest_record == None:
continue
notify, months = is_reminder_required(latest_record,
months_to_notify)
if notify == True:
print 'patient email process initiated ' + hash_names[key][
'first_name']
# We could send messages in bulk if it had a common body. But since the body is
# unique for every recipient we have to go through the non-performant process
# of sending one email at a time
template_input = {
'username':
hash_names[key]['first_name'],
'months':
months,
'visit_date':
latest_record['visit_date'],
'ltw_url':
settings['ltw_url'],
'unsubscribe_url':
settings['unsubscribe_url'] + '?email=' + val[0] +
"&rc_id=" + latest_record['rc_id'] +
"&unsubscribe.submitted=Unsubscribe",
'phone':
settings['phone'],
'email':
val[0]
}
try:
#turbomail.send(turbomail.Message(
# author="UCSD - Good to Go<" + settings["remind.email"] + ">",
# organization=["UCSD - Good to Go"],
# bcc=[key],
# reply_to=settings["remind.email"],
# subject="UCSD Early Test - Good to Go reminders",
# rich = lookup.get_template('email/reminder.mako').render(**template_input),
# headers = {'list-unsubscribe': "<" + template_input['unsubscribe_url']
# + ">,<mailto:" + settings["remind.email"] +">"},
# plain = "this is a reminder email to complete early test at " + settings['ltw_url']))
p_email = []
p_email.append(val[0])
text = lookup.get_template(
'email/reminder.mako').render(**template_input)
send_email(
text, "UCSD Early Test - Good to Go reminders",
"UCSD - Good to Go<" + settings["remind.email"] +
">", p_email, ses_key_id, ses_key, "html")
count = count + 1
match = next(d for d in patient_history
if d['rc_id'] == latest_record['rc_id'])
print 'patient email date record '
print match
if match != '':
match['lstremndr_dt'] = datetime.today().date(
).strftime('%Y/%m/%d')
except:
invalid_emails[latest_record['et_pid']] = val[0]
log.critical(traceback.format_exc())
pass
try:
redcap.project[site].import_records(patient_history)
print 'patient email date updated'
except:
pass
# Delete invalid email Ids from redcap. A hashSet is handy for this operation
# invalid_emails = set(invalid_emails)
invalid_emails_count = len(invalid_emails.keys())
if invalid_emails_count > 0:
with open(
'/var/tmp/invalid_emails_' + pat_key +
str(datetime.today().date()), 'wr+') as f:
json.dump(invalid_emails, f)
""" This logic will be commented out, until the script is mature
and all other features are well tested
# Remove wrong email ids from redcap
for pat_key, pat_records in precords.iteritems():
try:
to_update = []
for patient in pat_records:
if patient['email1'] in invalid_emails:
patient['email1'] = ''
to_update.append(patient)
log.debug("Rcid: %s, Email: %s is removed as invalid from redcap", patient['rc_id'], patient['email1'])
updated_count = redcap.project[pat_key].import_records(to_update)
except Exception as e:
log.critical
pass
"""
log.debug("Total Emails sent out today:%d", count)
print 'data'
print len(hash_email.keys())
print count
print invalid_emails_count
print staff_emails
stats = {
'date': datetime.today().date(),
'sdet_patient_count': len(hash_email.keys()),
'sdet_emails_sent': count,
'sdet_invalid_emails_count': invalid_emails_count,
'sdun_patient_count': sdun_patient_count,
'sdun_emails_sent': sdun_emails_sent,
'sdun_invalid_emails_count': sdun_invalid_emails_count,
'asr_patient_count': asr_patient_count,
'asr_emails_sent': asr_emails_sent,
'asr_invalid_emails_count': asr_invalid_emails_count,
'er_patient_count': er_patient_count,
'er_emails_sent': er_emails_sent,
'er_invalid_emails_count': er_invalid_emails_count
}
try:
text = lookup.get_template('email/stats.mako').render(**stats)
#turbomail.send(turbomail.Message(
# author = "UCSD - Good to Go<" + settings["remind.email"] + ">",
# organization = ["UCSD - Good to Go"],
# to = staff_emails,
# reply_to = settings["remind.email"],
# subject = "Good to Go Email Reminders Statistics",
# plain = text))
#send_email(text, "SDET: Good to Go Email Reminders Statistics", "UCSD - Good to Go<" + settings["remind.email"] + ">", staff_emails, ses_key_id, ses_key, "plain")
send_email(
text, "Good to Go Email Reminders Statistics",
"UCSD - Good to Go<" + settings["remind.email"] + ">",
staff_emails, ses_key_id, ses_key, "plain")
except:
log.debug(
lookup.get_template('email/stats.mako').render(**stats))
log.critical(traceback.format_exc())
except:
log.critical(traceback.format_exc())
turbomail.send(
turbomail.Message(
to=settings['notify.error'].split(),
subject=
'[Early Test Reminders]:There is an exception in sending out monthly reminders!',
plain=traceback.format_exc()))
def send_reminder_etc(ses_key_id, ses_key, settings, staff_emails,
months_to_notify):
try:
log.info("Early Test ETC Email Alerts are about to be sent")
er_patient_count = er_emails_sent = er_invalid_emails_count = asr_patient_count = asr_emails_sent = asr_invalid_emails_count = 0
rcs = json.loads(open(settings['redcap_json'], 'r').read())
a_keys = settings['avrc_email.code'].split()
e_keys = settings['etc_email.code'].split()
# Connect to all the required etc projects in RedCAP
a_redcap = RCProject(a_keys, rcs)
e_redcap = RCProject(e_keys, rcs)
# Required Patient Fields
a_fields = [
'consent_date', 'other_phone', 'email', 'current_hiv', 'fname',
'lname', 'testing_reminder'
]
e_fields = [
'collect_date', 'phone', 'email', 'first_name', 'last_name'
]
arecords = {}
erecords = {}
#AVRC Screening Registry
for key in a_keys:
invalid_emails_count = 0
emails_sent = 0
patient_count = 0
arecords = a_redcap.project[key].export_records(fields=a_fields)
print(arecords)
for record in arecords:
patient_count = patient_count + 1
notify, months = is_reminder_required_screening(
record, months_to_notify)
if notify == True and record['email'] != '':
print 'patient email process avrc screening registry initiated '
template_input = {
'username': record['fname'],
'visit_date': record['consent_date'],
'ltw_url': settings['ltw_url'],
'phone': settings['phone'],
'email': record['email']
}
try:
p_email = []
p_email.append(record['email'])
text = lookup.get_template(
'email/reminder_etc.mako').render(**template_input)
send_email(
text, "UCSD Early Test - Good to Go reminders",
"UCSD - Good to Go<" + settings["remind.email"] +
">", p_email, ses_key_id, ses_key, "html")
emails_sent = emails_sent + 1
except:
invalid_emails_count = invalid_emails_count + 1
log.critical(traceback.format_exc())
pass
#send_reminder_statistics(settings, key, patient_count, emails_sent, invalid_emails_count, staff_emails, ses_key_id, ses_key)
asr_patient_count = patient_count
asr_emails_sent = emails_sent
asr_invalid_emails_count = invalid_emails_count
print 'ASR'
print patient_count
print emails_sent
print invalid_emails_count
#Email Registry
for key in e_keys:
invalid_emails_count = 0
emails_sent = 0
patient_count = 0
erecords = e_redcap.project[key].export_records(fields=e_fields)
for record in erecords:
patient_count = patient_count + 1
notify, months = is_reminder_required_etc(
record, months_to_notify)
print(record)
if notify == True and record['email'] != '':
print 'patient email process email registry initiated '
template_input = {
'username': record['first_name'],
'visit_date': record['collect_date'],
'ltw_url': settings['ltw_url'],
'phone': settings['phone'],
'email': record['email']
}
try:
p_email = []
p_email.append(record['email'])
text = lookup.get_template(
'email/reminder_etc.mako').render(**template_input)
send_email(
text, "UCSD Early Test - Good to Go reminders",
"UCSD - Good to Go<" + settings["remind.email"] +
">", p_email, ses_key_id, ses_key, "html")
emails_sent = emails_sent + 1
except:
invalid_emails_count = invalid_emails_count + 1
log.critical(traceback.format_exc())
pass
#send_reminder_statistics(settings, key, patient_count, emails_sent, invalid_emails_count, staff_emails, ses_key_id, ses_key)
er_patient_count = patient_count
er_emails_sent = emails_sent
er_invalid_emails_count = invalid_emails_count
print 'ER'
print patient_count
print emails_sent
print invalid_emails_count
return er_patient_count, er_emails_sent, er_invalid_emails_count, asr_patient_count, asr_emails_sent, asr_invalid_emails_count
except:
#Ignore if we don't have the user information for the rcid
log.critical(traceback.format_exc())
return 0, 0, 0, 0, 0, 0
def send_reminder_single(ses_key_id, ses_key, settings, staff_emails,
months_to_notify):
try:
log.info("Early Test Single Email Alerts are about to be sent")
sdun_patient_count = sdun_emails_sent = sdun_invalid_emails_count = 0
rcs = json.loads(open(settings['redcap_json'], 'r').read())
keys = settings['single_email.code'].split()
redcap = RCProject(keys, rcs)
fields = [
'rc_id', 'visit_date', 'rapid1', 'rapid2', 'testing_reminder',
'dhiv', 'first_name', 'last_name', 'email1', 'phone1'
]
for key in keys:
invalid_emails_count = 0
emails_sent = 0
patient_count = 0
arecords = redcap.project[key].export_records()
for record in arecords:
patient_count = patient_count + 1
notify, months = is_reminder_required(record, months_to_notify)
if notify == True and record['email1'] != '':
print 'patient email process single email initiated ' + record[
'first_name']
template_input = {
'username': record['first_name'],
'visit_date': record['visit_date'],
'ltw_url': settings['ltw_url'],
'phone': settings['phone'],
'email': record['email1']
}
try:
p_email = []
p_email.append(record['email1'])
text = lookup.get_template(
'email/reminder_etc.mako').render(**template_input)
send_email(
text, "UCSD Early Test - Good to Go reminders",
"UCSD - Good to Go<" + settings["remind.email"] +
">", p_email, ses_key_id, ses_key, "html")
emails_sent = emails_sent + 1
except:
invalid_emails_count = invalid_emails_count + 1
log.critical(traceback.format_exc())
pass
if key == 'SDUN':
sdun_patient_count = patient_count
sdun_emails_sent = emails_sent
sdun_invalid_emails_count = invalid_emails_count
print 'SDUN'
print patient_count
print emails_sent
print invalid_emails_count
#send_reminder_statistics(settings, key, patient_count, emails_sent, invalid_emails_count, staff_emails, ses_key_id, ses_key)
return sdun_patient_count, sdun_emails_sent, sdun_invalid_emails_count
except:
#Ignore if we don't have the user information for the rcid
log.critical(traceback.format_exc())
return 0, 0, 0
def main():
args = cli.parse_args()
settings = args.settings
days_till_expiration = int(settings['days.tillexpiration'])
days_till_notify = int(settings['days.tillnotify'])
try:
results = sync_redcap.get_cts_results(settings)
if not args.dry:
Session.add_all(results)
Session.commit()
else:
log.info('Dry run, not commiting changes')
sync_site_codes = settings.get('site.codes').split()
ucsd_site_codes = settings.get('ucsd.site.codes').split()
emory_site_codes = settings.get('emory.site.codes').split()
gwu_site_codes = settings.get('gwu.site.codes').split()
rcs = json.loads(open(settings['redcap_json'], 'r').read())
redcap = RCProject(sync_site_codes, rcs)
# Refresh results
for site_code in sync_site_codes:
print 'site code'
print site_code
for type_ in models.TYPES:
#notify = settings.get('notify.%s.%s' % (site_code.lower(), type_.lower()), '').split()
notify = []
print type_
pnt = list(
r for r in results
if r.check(type_) is True and r.site_code == site_code)
neg = [
r for r in results
if r.check(type_) is False and r.site_code == site_code
]
odd = [
r for r in results
if r.check(type_) is None and r.site_code == site_code
]
if not (pnt):
continue
'clear'
if type_ == 'dhiv':
notify = get_receipients(redcap, 'hiv_pos', site_code,
ucsd_site_codes, emory_site_codes,
gwu_site_codes)
t_type = 'HIV'
elif type_ == 'dhcv':
notify = get_receipients(redcap, 'hcv_pos', site_code,
ucsd_site_codes, emory_site_codes,
gwu_site_codes)
t_type = 'HCV'
elif type_ == 'dhbv':
notify = get_receipients(redcap, 'hbv_pos', site_code,
ucsd_site_codes, emory_site_codes,
gwu_site_codes)
t_type = 'HBV'
print notify
if not notify:
continue
turbomail.send(
turbomail.Message(
to=notify,
subject='New %s+ NAT' % t_type,
plain=lookup.get_template('email/parse.mako').render(
**{
'timestamp': datetime.datetime.now(),
'type_': t_type,
'site_code': site_code,
'pnt': pnt,
'neg': neg,
'odd': odd
})))
log.info('Notified %s mailing lists of results for "%s"' %
(site_code, type_))
for code in sync_site_codes:
results_count = 0
shouldNotify = False
# Get number of site specific results in this upload
for r in results:
if r.site_code == code.upper():
results_count += 1
# Get list of results with missing draw dates
missing_draw = find_missing_draw(days_till_expiration, code)
missing_draw_count = len(missing_draw)
# Get list of draw dates with missing Red Cross results that are more than 7 days old
missing_results = find_missing_results(days_till_notify,
days_till_expiration,
redcap, code)
missing_results_count = len(missing_results)
# Notify recipients if there is anything to notify about
if results_count > 0 or missing_results_count > 0 or missing_draw_count > 0:
shouldNotify = True
if shouldNotify:
#notify = settings.get('notify.%s.sync' % code.lower()).split()
notify = get_receipients(redcap, 'date_missing', code,
ucsd_site_codes, emory_site_codes,
gwu_site_codes)
# Notify appropriate people about missing draw dates and Red Cross results
turbomail.send(
turbomail.Message(
to=notify,
subject=
'[The Early Test]: Red Cross Synchronize Report (%s)' %
code,
plain=lookup.get_template('email/sync.mako').render(
**{
'timestamp': datetime.datetime.now(),
'results_count': results_count,
'missing_draw_count': missing_draw_count,
'missing_draw': missing_draw,
'missing_results_count': missing_results_count,
'missing_results': missing_results,
'days_till_notify': days_till_notify,
'code': code
})))
log.info(
'Notified mailing lists of %s for missing draw dates' %
(code))
log.info(
'Notified mailing lists of %s diagnostic for missing Red Cross results'
% (code))
except:
# If an unexpected error occurs, let the developers know
turbomail.send(
turbomail.Message(
to=settings['notify.error'].split(),
subject='[The Early Test]: Parser failed execution',
plain=traceback.format_exc()))
raise
def sync_sql_result(buckets, settings):
"""
Parameters:
buckets: Dict of site codes and list of Results for that site as key, value pairs
settings: A dictionary of settings specified in the configuration file
Returns None
"""
try:
log.info("Synchronization Routine Started")
#only two attributes could have been updated Draw Date or Location
#Result should not be modified in the RedCAP directly
rcs = json.loads(open(settings['redcap_json'], 'r').read())
# Days after which results become invalid
days_till_expiration = int(settings['days.tillexpiration'])
# Days after which results can be published to patients
days_till_notify = int(settings['days.tillnotify'])
# Days after which tester should definitely act ASAP on CRF's or redcap entries
days_till_urgent = int(settings['days.tillurgent'])
pk_sites = settings.get('pk_site').split()
redcap = RCProject(buckets.keys(), rcs)
for key, value in buckets.iteritems():
ctsrecords = []
# These malformed draw dates need to be handled offline. This script will only be
# reporting such entries
malformed_draw = []
# Unmindful check. This line is mostly a deadcode, I guess.
if key not in redcap.project.keys():
continue
print 'key'
print key
#fields = ['visit_date','test_site']
# we pass the 'label' flag to get the location value as string instead numeric values
records = []
if redcap.project[key].is_longitudinal(
) == True or key in pk_sites:
print 'is logitudinal'
print key
l_records = redcap.project[key].export_records()
records = list(x for x in l_records if x['rc_id'] in value)
else:
records = redcap.project[key].export_records(
records=value,
fields=redcap.nat_fields,
raw_or_label='label')
ctsrecord = redcap.project['CTS'].export_records(
records=value, fields='rec_status')
for each in ctsrecord:
new_ctsrecord = {}
new_ctsrecord['rc_id'] = each['rc_id']
new_ctsrecord['rec_status'] = 1
ctsrecords.append(new_ctsrecord)
# RCIDs for which we have new results will be in push records
push_records = []
for record in records:
sql_row = Session.query(models.Result)\
.filter(models.Result.site_code == key)\
.filter(models.Result.reference_number == record['rc_id'][-5:]).first()
# Visit/Draw date update in SQL DB
if 'visit_date' in record.keys(
) and record['visit_date'] != '':
visit_date = dt.strptime(record['visit_date'],
"%Y-%m-%d").date()
if sql_row.site_code == 'SDPT':
print 'dates not equal'
print sql_row.draw_date
print visit_date
if sql_row.draw_date != visit_date:
if sql_row.test_date >= visit_date:
print 'update visit date'
sql_row.draw_date = visit_date
min_visit_date = sql_row.test_date - datetime.timedelta(
days=int(settings['result_buffer']))
if visit_date < min_visit_date:
malformed_draw.append(record['rc_id'])
# The malformed draw dates are the ones that don't fall in to the
# accepted window for results. Just report them, nothin more.
else:
malformed_draw.append(record['rc_id'])
# Location update in SQL DB
if 'test_site' in record.keys() and record['test_site'] != '':
rc_location = ''
if redcap.project[key].is_longitudinal() == True:
labeled_recs = redcap.project[key].export_records(
raw_or_label='label')
fil_rec = list(x for x in labeled_recs
if x['rc_id'] == record['rc_id'])[0]
rc_location = fil_rec['test_site']
else:
rc_location = record['test_site']
if sql_row.location != rc_location:
sql_row.location = rc_location
# Keep track for bulk update in RedCAP later
if 'nat_result_date' in record.keys():
if record['nat_result_date'] == '':
push_records.append(
update_result_redcap(record, sql_row))
else:
if 'nat_test_complete' in record.keys() and \
record['nat_test_complete'] == "Incomplete":
new_record = {}
new_record['rc_id'] = record['rc_id']
new_record['nat_test_complete'] = 2
push_records.append(new_record)
Session.commit()
# Make the bulk update for every 'key' and 'site'
value = redcap.project[key].import_records(push_records)
redcap.project['CTS'].import_records(ctsrecords)
# The following lines form the sophisticated email system ;-P. Well we again
# ask the human to help.
malformed_draw_count = len(malformed_draw)
# Get list of results with missing draw dates
missing_draw, warn_draw, critical_draw = find_missing_draw(
days_till_expiration, days_till_urgent, key)
missing_draw_count = len(missing_draw)
warn_draw_count = len(warn_draw)
critical_draw_count = len(critical_draw)
weekday = datetime.date.today().weekday()
if warn_draw_count != 0 and time_in_range(
datetime.datetime.now().time()) and weekday == 5:
#Special notifications when draw dates are missing for days_till_urgent
# This notification is sent on Fridays(5)
report_date = warn_draw
report_date_count = warn_draw_count
level = 1
notify = settings.get('notify.%s.action' % key.lower()).split()
turbomail.send(
turbomail.Message(
to=notify,
subject=
'[RedCAP Sync Update]: Prolonged Missing RedCAP Entries for (%s)'
% key,
plain=lookup.get_template('email/date.mako').render(
**{
'timestamp': datetime.datetime.now(),
'report_date': report_date,
'report_date_count': report_date_count,
'level': level,
'days_till_urgent': days_till_urgent,
'code': key
})))
if critical_draw_count != 0:
# Very critical draw date events emailed everyday
# Between 8-9am
time_now = datetime.datetime.now().time()
if time_in_range(time_now):
log.info(
"Some of the draw dates are missing for over %d days" %
(int(days_till_urgent) + 2))
report_date = critical_draw
report_date_count = critical_draw_count
level = 2
notify = settings.get('notify.%s.action' %
key.lower()).split()
turbomail.send(
turbomail.Message(
to=notify,
subject=
'[RedCAP Sync]: Action Required in RedCAP Entries for (%s)!'
% key,
plain=lookup.get_template(
'email/date.mako').render(
**{
'timestamp': datetime.datetime.now(),
'report_date': report_date,
'report_date_count': report_date_count,
'level': level,
'days_till_urgent': days_till_urgent,
'code': key
})))
# Get list of draw dates with missing Red Cross results that are more than 7 days old
missing_results = find_missing_results(days_till_notify,
days_till_expiration,
redcap, key)
missing_results_count = len(missing_results)
shouldNotify = False
# Notify recipients if there is anything to notify about only if its a Monday
if missing_results_count > 0 or missing_draw_count > 0 and datetime.date.today(
).weekday() == 1:
shouldNotify = True
if shouldNotify and time_in_range(datetime.datetime.now().time()):
notify = settings.get('notify.%s.action' % key.lower()).split()
# Notify appropriate people if notify is set to true
turbomail.send(
turbomail.Message(
to=notify,
subject=
'[RedCAP Sync]: Synchronization Status Check (%s)' %
key,
plain=lookup.get_template('email/rcap.mako').render(
**{
'timestamp': datetime.datetime.now(),
'results_count': 0,
'missing_draw_count': missing_draw_count,
'missing_draw': missing_draw,
'missing_results_count': missing_results_count,
'missing_results': missing_results,
'days_till_notify': days_till_notify,
'days_till_urgent': days_till_urgent,
'code': key,
'malformed_draw_count': malformed_draw_count,
'malformed_draw': malformed_draw
})))
except:
turbomail.send(
turbomail.Message(
to=settings['notify.error'].split(),
subject='[The Early Test]:Exception in matching visit dates!',
plain=traceback.format_exc()))
def main():
args = cli.parse_args()
settings = args.settings
days_till_expiration = int(settings['days.tillexpiration'])
days_till_notify = int(settings['days.tillnotify'])
try:
log.info('Called on %s' % args.srcfile)
results, duplicates = parser.parse(args.srcfile, settings)
if not args.dry:
if duplicates:
raise Exception('\n'.join(
['Already exists: %s%s' % (r.site_code, r.reference_number)
for r in duplicates]))
# Archive processed file
shutil.move(args.srcfile, settings['dir.raw'])
log.info('Moved encrypted file to %s' % settings['dir.raw'])
# Commit all changes now that we've successfully processed the file
map(lambda r: setattr(r, 'file', os.path.basename(args.srcfile)), results)
Session.add_all(results)
Session.commit()
else:
log.info('Dry run, not commiting changes')
sync_site_codes = settings.get('site.codes').split()
rcs = json.loads(open(settings['redcap_json'], 'r').read())
redcap = RCProject(sync_site_codes, rcs)
# Refresh results
for site_code in sync_site_codes:
for type_ in models.TYPES:
notify = settings.get('notify.%s.%s' % (site_code.lower(), type_.lower()), '').split()
if not notify:
continue
pnt = [r for r in results if r.check(type_) is True and r.site_code == site_code]
neg = [r for r in results if r.check(type_) is False and r.site_code == site_code]
odd = [r for r in results if r.check(type_) is None and r.site_code == site_code]
if not (pnt or odd):
continue
turbomail.send(turbomail.Message(
to=notify,
subject='[The Early Test]: New Records Notification (%s)' % type_,
plain=lookup.get_template('email/parse.mako').render(**{
'timestamp': datetime.datetime.now(),
'type_': type_,
'site_code': site_code,
'pnt': pnt,
'neg': neg,
'odd': odd})))
log.info('Notified %s mailing lists of results for "%s"' % (site_code, type_))
for code in sync_site_codes:
results_count = 0
shouldNotify = False
# Get number of site specific results in this upload
for r in results:
if r.site_code == code.upper():
results_count += 1
# Get list of results with missing draw dates
missing_draw = find_missing_draw(days_till_expiration, code)
missing_draw_count = len(missing_draw)
# Get list of draw dates with missing Red Cross results that are more than 7 days old
missing_results = find_missing_results(days_till_notify, days_till_expiration, redcap, code)
missing_results_count = len(missing_results)
# Notify recipients if there is anything to notify about
if results_count > 0 or missing_results_count > 0 or missing_draw_count > 0:
shouldNotify = True
if shouldNotify:
notify = settings.get('notify.%s.sync' % code.lower()).split()
# Notify appropriate people about missing draw dates and Red Cross results
turbomail.send(turbomail.Message(
to=notify,
subject='[The Early Test]: Red Cross Synchronize Report (%s)' % code,
plain=lookup.get_template('email/sync.mako').render(**{
'timestamp': datetime.datetime.now(),
'results_count': results_count,
'missing_draw_count': missing_draw_count,
'missing_draw': missing_draw,
'missing_results_count': missing_results_count,
'missing_results': missing_results,
'days_till_notify': days_till_notify,
'code': code})))
log.info('Notified mailing lists of %s for missing draw dates' % (code))
log.info('Notified mailing lists of %s diagnostic for missing Red Cross results' % (code))
except:
# If an unexpected error occurs, let the developers know
turbomail.send(turbomail.Message(
to=settings['notify.error'].split(),
subject='[The Early Test]: Parser failed execution',
plain=traceback.format_exc()))
raise