def put_records(self, records):
try:
error = {}
# Create sublist to update different projects
spare_records = copy.deepcopy(records)
expected = len(records)
for site, proj in self.project.iteritems():
site_records = []
for record in records:
if record['rc_id'][:-5] == site:
#Take only the required fields
site_records.append(record)
response = proj.import_records(site_records, date_format='YMD')
print(response)
if response['count'] != len(site_records):
log.critical("Response not matched for site:%s", site)
error[site] = {
'expected': len(site_records),
'updated': response['count'],
'values': site_records
}
except:
raise
return error
def main():
try:
args = cli.parse_args()
settings = args.settings
if settings['otp_mock'] == 'true':
patient = get_patient(settings['otp_mock_rcid'], settings)
patient['phone1'] = settings['otp_mock_phone']
log.info("Patient Info: %s", patient)
sent_otp, status = is_otp_sent(patient, settings)
if sent_otp == True:
log.info("Intentional %d sec delay", 15)
time.sleep(15)
if is_otp_valid(str(status['otp']), patient, settings) == True:
log.info("Successful OTP Verfication. OTP: %s",
status['otp'])
else:
raise Exception("Invalid OTP")
else:
raise Exception("OTP not sent!")
except Exception as e:
log.critical(traceback.format_exc())
turbomail.send(
turbomail.Message(to=settings['notify.error'].split(),
subject='[Early Test]: OTP Exception',
plain=traceback.format_exc()))
def main():
try:
args = cli.parse_args()
settings = args.settings
send_reminder(settings)
except Exception as e:
log.critical(traceback.format_exc())
turbomail.send(
turbomail.Message(
to=settings['notify.error'].split(),
subject='[The Early Test]: Reminder Email Exception!',
plain=traceback.format_exc()))
def main():
try:
args = cli.parse_args()
settings = args.settings
buckets = bucketize_results()
sync_sql_result(buckets, settings)
except Exception as e:
log.critical(e)
turbomail.send(
turbomail.Message(
to=settings['notify.error'].split(),
subject=
'[The Early Test]: RedCAP - SQLite DB Synchronization Error!',
plain=traceback.format_exc()))
def is_reminder_required(record, months_to_notify):
"""
Input
record: A dictionary of key rcid to value of a subset of fields from the redcap db
Output
boolean: True/False based on satisfying the condition to email
"""
try:
if record['lstremndr_dt'] == datetime.today().date().strftime(
'%Y/%m/%d'):
print 'already sent'
return False, 0
if record['testing_reminder'] != u'1':
# if the patient had opted out of reminders Don't bother further condtions
return False, 0
elif (record['rapid1'] is u'1'
and record['rapid2'] is not u'0') or record['dhiv'] is u'P':
# The above condition is true for positive patients
# Since positive patients will be going through
# a different treatment plan, they dont need to
# be reminded for testing. So return false
# The values meani, 0-Negative 1-Positive 2-Indeterminate
return False, 0
else:
current_date = datetime.today()
visit_date = datetime.strptime(record['visit_date'], "%Y-%m-%d")
year = visit_date.year
current_year = current_date.year
limit_year = current_year - 2
print(limit_year)
if year < limit_year:
return False, 0
month, days = monthmod(visit_date, current_date)
# Now, if record has a visit date that falls in to you notification slab,
# Then this email Id should be notified
if month.months >= months_to_notify:
if visit_date.month in companion_months(current_date.month):
return True, month.months
except:
#Ignore if we don't have the user information for the rcid
log.critical(traceback.format_exc())
return False, 0
def is_reminder_required_etc(record, months_to_notify):
try:
current_date = datetime.today()
visit_date = datetime.strptime(record['collect_date'], "%Y-%m-%d")
month, days = monthmod(visit_date, current_date)
# Now, if record has a visit date that falls in to you notification slab,
# Then this email Id should be notified
if month.months >= months_to_notify:
if visit_date.month in companion_months(current_date.month):
return True, month.months
except:
#Ignore if we don't have the user information for the rcid
log.critical(traceback.format_exc())
return False, 0
def is_otp_valid(otp, patient, settings):
try:
email = patient['email1'] if patient['email1'] != None else patient[
'email2']
Response = requests.get(settings['otp_verify_url'], {
'otp': otp,
'userid': email,
'key': settings['otp_key']
})
status = json.loads(Response.text)
log.info("OTP Verification Status: %s", status)
if status['success'] == True:
return status['isValidOtp']
except:
log.critical(traceback.format_exc())
return False
def is_otp_sent(patient, settings):
try:
phone = patient['phone1'] if patient['phone1'] != None else patient[
'phone2']
email = patient['email1'] if patient['email1'] != None else patient[
'email2']
Response = requests.post(settings['otp_gen_url'], {
'phone': phone,
'userid': email,
'key': settings['otp_key']
})
status = json.loads(Response.text)
log.info("OTP request status: %s", status)
return status['success'], status
except:
log.critical(traceback.format_exc())
return False, None
def connect_project(settings):
url = settings['cap_url']
key = settings['cap_key']
exp_date = settings['exp_date']
try:
if datetime.datetime.strptime(
exp_date, "%Y-%m-%d").date() <= datetime.date.today():
raise exc.RedCAPTokenExpired(exp_date)
project = Project(url, key, verify_ssl=True)
log.info('Project connected: %s' % project)
except:
log.critical('Exception on RedCap Project connect')
turbomail.send(
turbomail.Message(
to=settings['notify.error'].split(),
subject='[The Early Test]: RedCap Connection failure',
plain=traceback.format_exc()))
raise
return project
def bucketize_results():
""" Returns a dictonary of buckets which has site_codes as keys
and list of results as values"""
buckets = {}
try:
all_results = Session.query(models.Result)
site_codes = []
for result in all_results:
try:
if result.site_code not in site_codes:
site_codes.append(result.site_code)
buckets[result.site_code] = []
buckets[result.site_code].append(
str(result.site_code) + str(result.reference_number))
except:
pass
except Exception as e:
log.critical(traceback.format_exc())
return buckets
def get_patient(rc_id, settings):
try:
log.info("Enter Get_Patient")
patient = None
rcs = json.loads(open(settings['redcap_json'], 'r').read())
patient_sites = settings['pat.codes'].split()
redcap = RCProject(patient_sites, rcs)
for key in patient_sites:
try:
patients = redcap.project[key].export_records(records=[rc_id])
patient = patients[0]
break
except IndexError:
pass
except:
log.critical(traceback.format_exc())
log.info("Exit Get_Patient")
return patient
def send_reminder_statistics(settings, key, patient_count, emails_sent,
invalid_emails_count, staff_emails, ses_key_id,
ses_key):
if key == 'ASR':
key = "AVRC Screening Registry"
elif key == 'ER':
key = "Email Registry"
log.debug("Total Emails sent out today: " + key)
stats = {
'date': datetime.today().date(),
'patient_count': patient_count,
'emails_sent': emails_sent,
'invalid_emails_count': invalid_emails_count
}
try:
text = lookup.get_template('email/stats.mako').render(**stats)
#send_email(text, key + ": Good to Go Email Reminders Statistics", "UCSD - Good to Go<" + settings["remind.email"] + ">", staff_emails, ses_key_id, ses_key, "plain")
send_email(text, ": Good to Go Email Reminders Statistics",
"UCSD - Good to Go<" + settings["remind.email"] + ">",
staff_emails, ses_key_id, ses_key, "plain")
except:
log.debug(lookup.get_template('email/stats.mako').render(**stats))
log.critical(traceback.format_exc())
def send_reminder(settings):
"""
Parameters:
buckets: Dict of site codes and list of Results for that site as key, value pairs
settings: A dictionary of settings specified in the configuration file
Returns None
"""
try:
log.info("Early Test Email Alerts are about to be sent")
ses_key_id = settings['ses_key_id']
ses_key = settings['ses_key']
rcs = json.loads(open(settings['redcap_json'], 'r').read())
# Read from the configuration files
rem_keys = settings['rem.codes'].split()
pat_keys = settings['pat.codes'].split()
keys = rem_keys + pat_keys
months_to_notify = int(settings['months.tonotify'])
# Connect to all the required projects in RedCAP
redcap = RCProject(keys, rcs)
staff_emails = get_receipients(redcap, '1')
staff_emails_other = get_receipients(redcap, '2')
#send other email reminders
sdun_patient_count, sdun_emails_sent, sdun_invalid_emails_count = send_reminder_single(
ses_key_id, ses_key, settings, staff_emails_other,
months_to_notify)
er_patient_count, er_emails_sent, er_invalid_emails_count, asr_patient_count, asr_emails_sent, asr_invalid_emails_count = send_reminder_etc(
ses_key_id, ses_key, settings, staff_emails_other,
months_to_notify)
# Required Patient Fields
pfields = [
'et_pid', 'rc_id', 'phone1', 'phone2', 'email1', 'email2',
'first_name', 'last_name'
]
precords = {}
for key in pat_keys:
precords[key] = redcap.project[key].export_records(fields=pfields)
# This dictionary holds a map of Patient email to a list of
# rc_ids of that specific patient
# NOTE: The rc_ids may also belong to multiple sites.
hash_email = {}
# This is a dictionary to the email ids of patients to their names
hash_names = {}
# This dictionary will hold the list of rcid to specific
# sites, and therefore prevent multiple calls to the
# RedCAP rest APIs.
site_rcid = {}
for site in rem_keys:
site_rcid[site] = []
# Today we have a single redcap project for patients. In the future
# there can be multiple projects for patients. The below code will
# still work and have a better performance.
import sys
for pat_key, pat_records in precords.iteritems():
log.debug("Total of patient records in %s = %d", pat_key,
len(pat_records))
log.debug("The size of the records: %d Bytes",
sys.getsizeof(pat_records))
# Patient Records that need to be updated
# with working primary email for this site
to_update = []
invalid_emails = {}
for patient in pat_records:
try:
# Extract Site information
#site = patient['rc_id'].upper()
# The testing site visited by this patient is not configured
# to send email reminders. So continue to the next patient
#if site.startswith(tuple(rem_keys)) == False:
#continue
if patient['email1'] == '':
# ignore patients for whom we don't have email IDs
if patient['email2'] == '':
continue
else:
# Set the primary email and delete the secondary
patient['email1'] = patient['email2']
patient['email2'] = ''
to_update.append(patient)
if patient['et_pid'] in hash_email:
continue
else:
hash_email[patient['et_pid']] = [patient['email1']]
hash_names[patient['et_pid']] = {
'first_name': patient['first_name'],
'last_name': patient['last_name']
}
#site_rcid[patient['rc_id'][:-5]].append(patient['rc_id'])
except KeyError as e:
log.critical(traceback.format_exc())
pass
"""if len(to_update) > 0:
try:
#updated_count = redcap.project[pat_key].import_records(to_update)
log.debug(" %d Emails needed to be moved from email2 to email1 for %s", len(to_update), pat_key)
except:
log.critical(traceback.format_exc())
pass"""
# Some stats to log
log.debug("Unique Patient with email Ids: %d",
len(hash_email.keys()))
#for email, visits in hash_email.iteritems():
# log.debug("Patient Email: %s, Number of Visits: %d", email, len(visits))
#History of the patients as a list returned by redcap
patient_history = []
# We need the following fields to decide if we need to send emails for this patient
fields = [
'et_pid', 'rc_id', 'visit_date', 'rapid1', 'rapid2',
'testing_reminder', 'dhiv', 'lstremndr_dt'
]
#for site, records in hash_email.iteritems():
patient_history.extend(
redcap.project['SDET'].export_records(fields=fields))
patient_history.extend(
redcap.project['76C'].export_records(fields=fields))
log.debug("Total Patient history to Process:%d",
len(patient_history))
# Patient history modified data structure for convenience
hist_map = {}
for history in patient_history:
hist_map[history['rc_id']] = history
count = 0
for key, val in hash_email.iteritems():
print 'key'
print key
print 'value'
print val
latest_record = None
# At the end of the following loop, record will consists of the latest visit
# information for this patient.(Indicated by the email string 'key')
skip = False
for rc_id, visit_val in hist_map.iteritems():
try:
if visit_val['et_pid'] != key:
continue
if visit_val['visit_date'] == '':
continue
print 'visit date exists'
print 'rcid'
print rc_id
print 'visit'
print visit_val
visit = datetime.strptime(visit_val['visit_date'],
"%Y-%m-%d")
if latest_record == None:
latest_record = visit_val
else:
latest_date = datetime.strptime(
latest_record['visit_date'], "%Y-%m-%d")
if latest_date <= visit:
print 'compare dates'
print visit
print latest_date
latest_record = hist_map[rc_id]
except KeyError:
log.critical(traceback.format_exc())
print 'latest record'
print latest_record
if latest_record == None:
continue
notify, months = is_reminder_required(latest_record,
months_to_notify)
if notify == True:
print 'patient email process initiated ' + hash_names[key][
'first_name']
# We could send messages in bulk if it had a common body. But since the body is
# unique for every recipient we have to go through the non-performant process
# of sending one email at a time
template_input = {
'username':
hash_names[key]['first_name'],
'months':
months,
'visit_date':
latest_record['visit_date'],
'ltw_url':
settings['ltw_url'],
'unsubscribe_url':
settings['unsubscribe_url'] + '?email=' + val[0] +
"&rc_id=" + latest_record['rc_id'] +
"&unsubscribe.submitted=Unsubscribe",
'phone':
settings['phone'],
'email':
val[0]
}
try:
#turbomail.send(turbomail.Message(
# author="UCSD - Good to Go<" + settings["remind.email"] + ">",
# organization=["UCSD - Good to Go"],
# bcc=[key],
# reply_to=settings["remind.email"],
# subject="UCSD Early Test - Good to Go reminders",
# rich = lookup.get_template('email/reminder.mako').render(**template_input),
# headers = {'list-unsubscribe': "<" + template_input['unsubscribe_url']
# + ">,<mailto:" + settings["remind.email"] +">"},
# plain = "this is a reminder email to complete early test at " + settings['ltw_url']))
p_email = []
p_email.append(val[0])
text = lookup.get_template(
'email/reminder.mako').render(**template_input)
send_email(
text, "UCSD Early Test - Good to Go reminders",
"UCSD - Good to Go<" + settings["remind.email"] +
">", p_email, ses_key_id, ses_key, "html")
count = count + 1
match = next(d for d in patient_history
if d['rc_id'] == latest_record['rc_id'])
print 'patient email date record '
print match
if match != '':
match['lstremndr_dt'] = datetime.today().date(
).strftime('%Y/%m/%d')
except:
invalid_emails[latest_record['et_pid']] = val[0]
log.critical(traceback.format_exc())
pass
try:
redcap.project[site].import_records(patient_history)
print 'patient email date updated'
except:
pass
# Delete invalid email Ids from redcap. A hashSet is handy for this operation
# invalid_emails = set(invalid_emails)
invalid_emails_count = len(invalid_emails.keys())
if invalid_emails_count > 0:
with open(
'/var/tmp/invalid_emails_' + pat_key +
str(datetime.today().date()), 'wr+') as f:
json.dump(invalid_emails, f)
""" This logic will be commented out, until the script is mature
and all other features are well tested
# Remove wrong email ids from redcap
for pat_key, pat_records in precords.iteritems():
try:
to_update = []
for patient in pat_records:
if patient['email1'] in invalid_emails:
patient['email1'] = ''
to_update.append(patient)
log.debug("Rcid: %s, Email: %s is removed as invalid from redcap", patient['rc_id'], patient['email1'])
updated_count = redcap.project[pat_key].import_records(to_update)
except Exception as e:
log.critical
pass
"""
log.debug("Total Emails sent out today:%d", count)
print 'data'
print len(hash_email.keys())
print count
print invalid_emails_count
print staff_emails
stats = {
'date': datetime.today().date(),
'sdet_patient_count': len(hash_email.keys()),
'sdet_emails_sent': count,
'sdet_invalid_emails_count': invalid_emails_count,
'sdun_patient_count': sdun_patient_count,
'sdun_emails_sent': sdun_emails_sent,
'sdun_invalid_emails_count': sdun_invalid_emails_count,
'asr_patient_count': asr_patient_count,
'asr_emails_sent': asr_emails_sent,
'asr_invalid_emails_count': asr_invalid_emails_count,
'er_patient_count': er_patient_count,
'er_emails_sent': er_emails_sent,
'er_invalid_emails_count': er_invalid_emails_count
}
try:
text = lookup.get_template('email/stats.mako').render(**stats)
#turbomail.send(turbomail.Message(
# author = "UCSD - Good to Go<" + settings["remind.email"] + ">",
# organization = ["UCSD - Good to Go"],
# to = staff_emails,
# reply_to = settings["remind.email"],
# subject = "Good to Go Email Reminders Statistics",
# plain = text))
#send_email(text, "SDET: Good to Go Email Reminders Statistics", "UCSD - Good to Go<" + settings["remind.email"] + ">", staff_emails, ses_key_id, ses_key, "plain")
send_email(
text, "Good to Go Email Reminders Statistics",
"UCSD - Good to Go<" + settings["remind.email"] + ">",
staff_emails, ses_key_id, ses_key, "plain")
except:
log.debug(
lookup.get_template('email/stats.mako').render(**stats))
log.critical(traceback.format_exc())
except:
log.critical(traceback.format_exc())
turbomail.send(
turbomail.Message(
to=settings['notify.error'].split(),
subject=
'[Early Test Reminders]:There is an exception in sending out monthly reminders!',
plain=traceback.format_exc()))
def send_reminder_etc(ses_key_id, ses_key, settings, staff_emails,
months_to_notify):
try:
log.info("Early Test ETC Email Alerts are about to be sent")
er_patient_count = er_emails_sent = er_invalid_emails_count = asr_patient_count = asr_emails_sent = asr_invalid_emails_count = 0
rcs = json.loads(open(settings['redcap_json'], 'r').read())
a_keys = settings['avrc_email.code'].split()
e_keys = settings['etc_email.code'].split()
# Connect to all the required etc projects in RedCAP
a_redcap = RCProject(a_keys, rcs)
e_redcap = RCProject(e_keys, rcs)
# Required Patient Fields
a_fields = [
'consent_date', 'other_phone', 'email', 'current_hiv', 'fname',
'lname', 'testing_reminder'
]
e_fields = [
'collect_date', 'phone', 'email', 'first_name', 'last_name'
]
arecords = {}
erecords = {}
#AVRC Screening Registry
for key in a_keys:
invalid_emails_count = 0
emails_sent = 0
patient_count = 0
arecords = a_redcap.project[key].export_records(fields=a_fields)
print(arecords)
for record in arecords:
patient_count = patient_count + 1
notify, months = is_reminder_required_screening(
record, months_to_notify)
if notify == True and record['email'] != '':
print 'patient email process avrc screening registry initiated '
template_input = {
'username': record['fname'],
'visit_date': record['consent_date'],
'ltw_url': settings['ltw_url'],
'phone': settings['phone'],
'email': record['email']
}
try:
p_email = []
p_email.append(record['email'])
text = lookup.get_template(
'email/reminder_etc.mako').render(**template_input)
send_email(
text, "UCSD Early Test - Good to Go reminders",
"UCSD - Good to Go<" + settings["remind.email"] +
">", p_email, ses_key_id, ses_key, "html")
emails_sent = emails_sent + 1
except:
invalid_emails_count = invalid_emails_count + 1
log.critical(traceback.format_exc())
pass
#send_reminder_statistics(settings, key, patient_count, emails_sent, invalid_emails_count, staff_emails, ses_key_id, ses_key)
asr_patient_count = patient_count
asr_emails_sent = emails_sent
asr_invalid_emails_count = invalid_emails_count
print 'ASR'
print patient_count
print emails_sent
print invalid_emails_count
#Email Registry
for key in e_keys:
invalid_emails_count = 0
emails_sent = 0
patient_count = 0
erecords = e_redcap.project[key].export_records(fields=e_fields)
for record in erecords:
patient_count = patient_count + 1
notify, months = is_reminder_required_etc(
record, months_to_notify)
print(record)
if notify == True and record['email'] != '':
print 'patient email process email registry initiated '
template_input = {
'username': record['first_name'],
'visit_date': record['collect_date'],
'ltw_url': settings['ltw_url'],
'phone': settings['phone'],
'email': record['email']
}
try:
p_email = []
p_email.append(record['email'])
text = lookup.get_template(
'email/reminder_etc.mako').render(**template_input)
send_email(
text, "UCSD Early Test - Good to Go reminders",
"UCSD - Good to Go<" + settings["remind.email"] +
">", p_email, ses_key_id, ses_key, "html")
emails_sent = emails_sent + 1
except:
invalid_emails_count = invalid_emails_count + 1
log.critical(traceback.format_exc())
pass
#send_reminder_statistics(settings, key, patient_count, emails_sent, invalid_emails_count, staff_emails, ses_key_id, ses_key)
er_patient_count = patient_count
er_emails_sent = emails_sent
er_invalid_emails_count = invalid_emails_count
print 'ER'
print patient_count
print emails_sent
print invalid_emails_count
return er_patient_count, er_emails_sent, er_invalid_emails_count, asr_patient_count, asr_emails_sent, asr_invalid_emails_count
except:
#Ignore if we don't have the user information for the rcid
log.critical(traceback.format_exc())
return 0, 0, 0, 0, 0, 0
def send_reminder_single(ses_key_id, ses_key, settings, staff_emails,
months_to_notify):
try:
log.info("Early Test Single Email Alerts are about to be sent")
sdun_patient_count = sdun_emails_sent = sdun_invalid_emails_count = 0
rcs = json.loads(open(settings['redcap_json'], 'r').read())
keys = settings['single_email.code'].split()
redcap = RCProject(keys, rcs)
fields = [
'rc_id', 'visit_date', 'rapid1', 'rapid2', 'testing_reminder',
'dhiv', 'first_name', 'last_name', 'email1', 'phone1'
]
for key in keys:
invalid_emails_count = 0
emails_sent = 0
patient_count = 0
arecords = redcap.project[key].export_records()
for record in arecords:
patient_count = patient_count + 1
notify, months = is_reminder_required(record, months_to_notify)
if notify == True and record['email1'] != '':
print 'patient email process single email initiated ' + record[
'first_name']
template_input = {
'username': record['first_name'],
'visit_date': record['visit_date'],
'ltw_url': settings['ltw_url'],
'phone': settings['phone'],
'email': record['email1']
}
try:
p_email = []
p_email.append(record['email1'])
text = lookup.get_template(
'email/reminder_etc.mako').render(**template_input)
send_email(
text, "UCSD Early Test - Good to Go reminders",
"UCSD - Good to Go<" + settings["remind.email"] +
">", p_email, ses_key_id, ses_key, "html")
emails_sent = emails_sent + 1
except:
invalid_emails_count = invalid_emails_count + 1
log.critical(traceback.format_exc())
pass
if key == 'SDUN':
sdun_patient_count = patient_count
sdun_emails_sent = emails_sent
sdun_invalid_emails_count = invalid_emails_count
print 'SDUN'
print patient_count
print emails_sent
print invalid_emails_count
#send_reminder_statistics(settings, key, patient_count, emails_sent, invalid_emails_count, staff_emails, ses_key_id, ses_key)
return sdun_patient_count, sdun_emails_sent, sdun_invalid_emails_count
except:
#Ignore if we don't have the user information for the rcid
log.critical(traceback.format_exc())
return 0, 0, 0