def sync_sql_result(buckets, settings):
"""
Parameters:
buckets: Dict of site codes and list of Results for that site as key, value pairs
settings: A dictionary of settings specified in the configuration file
Returns None
"""
try:
log.info("Synchronization Routine Started")
#only two attributes could have been updated Draw Date or Location
#Result should not be modified in the RedCAP directly
rcs = json.loads(open(settings['redcap_json'], 'r').read())
# Days after which results become invalid
days_till_expiration = int(settings['days.tillexpiration'])
# Days after which results can be published to patients
days_till_notify = int(settings['days.tillnotify'])
# Days after which tester should definitely act ASAP on CRF's or redcap entries
days_till_urgent = int(settings['days.tillurgent'])
pk_sites = settings.get('pk_site').split()
redcap = RCProject(buckets.keys(), rcs)
for key, value in buckets.iteritems():
ctsrecords = []
# These malformed draw dates need to be handled offline. This script will only be
# reporting such entries
malformed_draw = []
# Unmindful check. This line is mostly a deadcode, I guess.
if key not in redcap.project.keys():
continue
print 'key'
print key
#fields = ['visit_date','test_site']
# we pass the 'label' flag to get the location value as string instead numeric values
records = []
if redcap.project[key].is_longitudinal(
) == True or key in pk_sites:
print 'is logitudinal'
print key
l_records = redcap.project[key].export_records()
records = list(x for x in l_records if x['rc_id'] in value)
else:
records = redcap.project[key].export_records(
records=value,
fields=redcap.nat_fields,
raw_or_label='label')
ctsrecord = redcap.project['CTS'].export_records(
records=value, fields='rec_status')
for each in ctsrecord:
new_ctsrecord = {}
new_ctsrecord['rc_id'] = each['rc_id']
new_ctsrecord['rec_status'] = 1
ctsrecords.append(new_ctsrecord)
# RCIDs for which we have new results will be in push records
push_records = []
for record in records:
sql_row = Session.query(models.Result)\
.filter(models.Result.site_code == key)\
.filter(models.Result.reference_number == record['rc_id'][-5:]).first()
# Visit/Draw date update in SQL DB
if 'visit_date' in record.keys(
) and record['visit_date'] != '':
visit_date = dt.strptime(record['visit_date'],
"%Y-%m-%d").date()
if sql_row.site_code == 'SDPT':
print 'dates not equal'
print sql_row.draw_date
print visit_date
if sql_row.draw_date != visit_date:
if sql_row.test_date >= visit_date:
print 'update visit date'
sql_row.draw_date = visit_date
min_visit_date = sql_row.test_date - datetime.timedelta(
days=int(settings['result_buffer']))
if visit_date < min_visit_date:
malformed_draw.append(record['rc_id'])
# The malformed draw dates are the ones that don't fall in to the
# accepted window for results. Just report them, nothin more.
else:
malformed_draw.append(record['rc_id'])
# Location update in SQL DB
if 'test_site' in record.keys() and record['test_site'] != '':
rc_location = ''
if redcap.project[key].is_longitudinal() == True:
labeled_recs = redcap.project[key].export_records(
raw_or_label='label')
fil_rec = list(x for x in labeled_recs
if x['rc_id'] == record['rc_id'])[0]
rc_location = fil_rec['test_site']
else:
rc_location = record['test_site']
if sql_row.location != rc_location:
sql_row.location = rc_location
# Keep track for bulk update in RedCAP later
if 'nat_result_date' in record.keys():
if record['nat_result_date'] == '':
push_records.append(
update_result_redcap(record, sql_row))
else:
if 'nat_test_complete' in record.keys() and \
record['nat_test_complete'] == "Incomplete":
new_record = {}
new_record['rc_id'] = record['rc_id']
new_record['nat_test_complete'] = 2
push_records.append(new_record)
Session.commit()
# Make the bulk update for every 'key' and 'site'
value = redcap.project[key].import_records(push_records)
redcap.project['CTS'].import_records(ctsrecords)
# The following lines form the sophisticated email system ;-P. Well we again
# ask the human to help.
malformed_draw_count = len(malformed_draw)
# Get list of results with missing draw dates
missing_draw, warn_draw, critical_draw = find_missing_draw(
days_till_expiration, days_till_urgent, key)
missing_draw_count = len(missing_draw)
warn_draw_count = len(warn_draw)
critical_draw_count = len(critical_draw)
weekday = datetime.date.today().weekday()
if warn_draw_count != 0 and time_in_range(
datetime.datetime.now().time()) and weekday == 5:
#Special notifications when draw dates are missing for days_till_urgent
# This notification is sent on Fridays(5)
report_date = warn_draw
report_date_count = warn_draw_count
level = 1
notify = settings.get('notify.%s.action' % key.lower()).split()
turbomail.send(
turbomail.Message(
to=notify,
subject=
'[RedCAP Sync Update]: Prolonged Missing RedCAP Entries for (%s)'
% key,
plain=lookup.get_template('email/date.mako').render(
**{
'timestamp': datetime.datetime.now(),
'report_date': report_date,
'report_date_count': report_date_count,
'level': level,
'days_till_urgent': days_till_urgent,
'code': key
})))
if critical_draw_count != 0:
# Very critical draw date events emailed everyday
# Between 8-9am
time_now = datetime.datetime.now().time()
if time_in_range(time_now):
log.info(
"Some of the draw dates are missing for over %d days" %
(int(days_till_urgent) + 2))
report_date = critical_draw
report_date_count = critical_draw_count
level = 2
notify = settings.get('notify.%s.action' %
key.lower()).split()
turbomail.send(
turbomail.Message(
to=notify,
subject=
'[RedCAP Sync]: Action Required in RedCAP Entries for (%s)!'
% key,
plain=lookup.get_template(
'email/date.mako').render(
**{
'timestamp': datetime.datetime.now(),
'report_date': report_date,
'report_date_count': report_date_count,
'level': level,
'days_till_urgent': days_till_urgent,
'code': key
})))
# Get list of draw dates with missing Red Cross results that are more than 7 days old
missing_results = find_missing_results(days_till_notify,
days_till_expiration,
redcap, key)
missing_results_count = len(missing_results)
shouldNotify = False
# Notify recipients if there is anything to notify about only if its a Monday
if missing_results_count > 0 or missing_draw_count > 0 and datetime.date.today(
).weekday() == 1:
shouldNotify = True
if shouldNotify and time_in_range(datetime.datetime.now().time()):
notify = settings.get('notify.%s.action' % key.lower()).split()
# Notify appropriate people if notify is set to true
turbomail.send(
turbomail.Message(
to=notify,
subject=
'[RedCAP Sync]: Synchronization Status Check (%s)' %
key,
plain=lookup.get_template('email/rcap.mako').render(
**{
'timestamp': datetime.datetime.now(),
'results_count': 0,
'missing_draw_count': missing_draw_count,
'missing_draw': missing_draw,
'missing_results_count': missing_results_count,
'missing_results': missing_results,
'days_till_notify': days_till_notify,
'days_till_urgent': days_till_urgent,
'code': key,
'malformed_draw_count': malformed_draw_count,
'malformed_draw': malformed_draw
})))
except:
turbomail.send(
turbomail.Message(
to=settings['notify.error'].split(),
subject='[The Early Test]:Exception in matching visit dates!',
plain=traceback.format_exc()))
def main():
args = cli.parse_args()
settings = args.settings
days_till_expiration = int(settings['days.tillexpiration'])
days_till_notify = int(settings['days.tillnotify'])
try:
results = sync_redcap.get_cts_results(settings)
if not args.dry:
Session.add_all(results)
Session.commit()
else:
log.info('Dry run, not commiting changes')
sync_site_codes = settings.get('site.codes').split()
ucsd_site_codes = settings.get('ucsd.site.codes').split()
emory_site_codes = settings.get('emory.site.codes').split()
gwu_site_codes = settings.get('gwu.site.codes').split()
rcs = json.loads(open(settings['redcap_json'], 'r').read())
redcap = RCProject(sync_site_codes, rcs)
# Refresh results
for site_code in sync_site_codes:
print 'site code'
print site_code
for type_ in models.TYPES:
#notify = settings.get('notify.%s.%s' % (site_code.lower(), type_.lower()), '').split()
notify = []
print type_
pnt = list(
r for r in results
if r.check(type_) is True and r.site_code == site_code)
neg = [
r for r in results
if r.check(type_) is False and r.site_code == site_code
]
odd = [
r for r in results
if r.check(type_) is None and r.site_code == site_code
]
if not (pnt):
continue
'clear'
if type_ == 'dhiv':
notify = get_receipients(redcap, 'hiv_pos', site_code,
ucsd_site_codes, emory_site_codes,
gwu_site_codes)
t_type = 'HIV'
elif type_ == 'dhcv':
notify = get_receipients(redcap, 'hcv_pos', site_code,
ucsd_site_codes, emory_site_codes,
gwu_site_codes)
t_type = 'HCV'
elif type_ == 'dhbv':
notify = get_receipients(redcap, 'hbv_pos', site_code,
ucsd_site_codes, emory_site_codes,
gwu_site_codes)
t_type = 'HBV'
print notify
if not notify:
continue
turbomail.send(
turbomail.Message(
to=notify,
subject='New %s+ NAT' % t_type,
plain=lookup.get_template('email/parse.mako').render(
**{
'timestamp': datetime.datetime.now(),
'type_': t_type,
'site_code': site_code,
'pnt': pnt,
'neg': neg,
'odd': odd
})))
log.info('Notified %s mailing lists of results for "%s"' %
(site_code, type_))
for code in sync_site_codes:
results_count = 0
shouldNotify = False
# Get number of site specific results in this upload
for r in results:
if r.site_code == code.upper():
results_count += 1
# Get list of results with missing draw dates
missing_draw = find_missing_draw(days_till_expiration, code)
missing_draw_count = len(missing_draw)
# Get list of draw dates with missing Red Cross results that are more than 7 days old
missing_results = find_missing_results(days_till_notify,
days_till_expiration,
redcap, code)
missing_results_count = len(missing_results)
# Notify recipients if there is anything to notify about
if results_count > 0 or missing_results_count > 0 or missing_draw_count > 0:
shouldNotify = True
if shouldNotify:
#notify = settings.get('notify.%s.sync' % code.lower()).split()
notify = get_receipients(redcap, 'date_missing', code,
ucsd_site_codes, emory_site_codes,
gwu_site_codes)
# Notify appropriate people about missing draw dates and Red Cross results
turbomail.send(
turbomail.Message(
to=notify,
subject=
'[The Early Test]: Red Cross Synchronize Report (%s)' %
code,
plain=lookup.get_template('email/sync.mako').render(
**{
'timestamp': datetime.datetime.now(),
'results_count': results_count,
'missing_draw_count': missing_draw_count,
'missing_draw': missing_draw,
'missing_results_count': missing_results_count,
'missing_results': missing_results,
'days_till_notify': days_till_notify,
'code': code
})))
log.info(
'Notified mailing lists of %s for missing draw dates' %
(code))
log.info(
'Notified mailing lists of %s diagnostic for missing Red Cross results'
% (code))
except:
# If an unexpected error occurs, let the developers know
turbomail.send(
turbomail.Message(
to=settings['notify.error'].split(),
subject='[The Early Test]: Parser failed execution',
plain=traceback.format_exc()))
raise
def main():
args = cli.parse_args()
settings = args.settings
days_till_expiration = int(settings['days.tillexpiration'])
days_till_notify = int(settings['days.tillnotify'])
try:
log.info('Called on %s' % args.srcfile)
results, duplicates = parser.parse(args.srcfile, settings)
if not args.dry:
if duplicates:
raise Exception('\n'.join(
['Already exists: %s%s' % (r.site_code, r.reference_number)
for r in duplicates]))
# Archive processed file
shutil.move(args.srcfile, settings['dir.raw'])
log.info('Moved encrypted file to %s' % settings['dir.raw'])
# Commit all changes now that we've successfully processed the file
map(lambda r: setattr(r, 'file', os.path.basename(args.srcfile)), results)
Session.add_all(results)
Session.commit()
else:
log.info('Dry run, not commiting changes')
sync_site_codes = settings.get('site.codes').split()
rcs = json.loads(open(settings['redcap_json'], 'r').read())
redcap = RCProject(sync_site_codes, rcs)
# Refresh results
for site_code in sync_site_codes:
for type_ in models.TYPES:
notify = settings.get('notify.%s.%s' % (site_code.lower(), type_.lower()), '').split()
if not notify:
continue
pnt = [r for r in results if r.check(type_) is True and r.site_code == site_code]
neg = [r for r in results if r.check(type_) is False and r.site_code == site_code]
odd = [r for r in results if r.check(type_) is None and r.site_code == site_code]
if not (pnt or odd):
continue
turbomail.send(turbomail.Message(
to=notify,
subject='[The Early Test]: New Records Notification (%s)' % type_,
plain=lookup.get_template('email/parse.mako').render(**{
'timestamp': datetime.datetime.now(),
'type_': type_,
'site_code': site_code,
'pnt': pnt,
'neg': neg,
'odd': odd})))
log.info('Notified %s mailing lists of results for "%s"' % (site_code, type_))
for code in sync_site_codes:
results_count = 0
shouldNotify = False
# Get number of site specific results in this upload
for r in results:
if r.site_code == code.upper():
results_count += 1
# Get list of results with missing draw dates
missing_draw = find_missing_draw(days_till_expiration, code)
missing_draw_count = len(missing_draw)
# Get list of draw dates with missing Red Cross results that are more than 7 days old
missing_results = find_missing_results(days_till_notify, days_till_expiration, redcap, code)
missing_results_count = len(missing_results)
# Notify recipients if there is anything to notify about
if results_count > 0 or missing_results_count > 0 or missing_draw_count > 0:
shouldNotify = True
if shouldNotify:
notify = settings.get('notify.%s.sync' % code.lower()).split()
# Notify appropriate people about missing draw dates and Red Cross results
turbomail.send(turbomail.Message(
to=notify,
subject='[The Early Test]: Red Cross Synchronize Report (%s)' % code,
plain=lookup.get_template('email/sync.mako').render(**{
'timestamp': datetime.datetime.now(),
'results_count': results_count,
'missing_draw_count': missing_draw_count,
'missing_draw': missing_draw,
'missing_results_count': missing_results_count,
'missing_results': missing_results,
'days_till_notify': days_till_notify,
'code': code})))
log.info('Notified mailing lists of %s for missing draw dates' % (code))
log.info('Notified mailing lists of %s diagnostic for missing Red Cross results' % (code))
except:
# If an unexpected error occurs, let the developers know
turbomail.send(turbomail.Message(
to=settings['notify.error'].split(),
subject='[The Early Test]: Parser failed execution',
plain=traceback.format_exc()))
raise
