def sync_sql_result(buckets, settings): """ Parameters: buckets: Dict of site codes and list of Results for that site as key, value pairs settings: A dictionary of settings specified in the configuration file Returns None """ try: log.info("Synchronization Routine Started") #only two attributes could have been updated Draw Date or Location #Result should not be modified in the RedCAP directly rcs = json.loads(open(settings['redcap_json'], 'r').read()) # Days after which results become invalid days_till_expiration = int(settings['days.tillexpiration']) # Days after which results can be published to patients days_till_notify = int(settings['days.tillnotify']) # Days after which tester should definitely act ASAP on CRF's or redcap entries days_till_urgent = int(settings['days.tillurgent']) pk_sites = settings.get('pk_site').split() redcap = RCProject(buckets.keys(), rcs) for key, value in buckets.iteritems(): ctsrecords = [] # These malformed draw dates need to be handled offline. This script will only be # reporting such entries malformed_draw = [] # Unmindful check. This line is mostly a deadcode, I guess. if key not in redcap.project.keys(): continue print 'key' print key #fields = ['visit_date','test_site'] # we pass the 'label' flag to get the location value as string instead numeric values records = [] if redcap.project[key].is_longitudinal( ) == True or key in pk_sites: print 'is logitudinal' print key l_records = redcap.project[key].export_records() records = list(x for x in l_records if x['rc_id'] in value) else: records = redcap.project[key].export_records( records=value, fields=redcap.nat_fields, raw_or_label='label') ctsrecord = redcap.project['CTS'].export_records( records=value, fields='rec_status') for each in ctsrecord: new_ctsrecord = {} new_ctsrecord['rc_id'] = each['rc_id'] new_ctsrecord['rec_status'] = 1 ctsrecords.append(new_ctsrecord) # RCIDs for which we have new results will be in push records push_records = [] for record in records: sql_row = Session.query(models.Result)\ .filter(models.Result.site_code == key)\ .filter(models.Result.reference_number == record['rc_id'][-5:]).first() # Visit/Draw date update in SQL DB if 'visit_date' in record.keys( ) and record['visit_date'] != '': visit_date = dt.strptime(record['visit_date'], "%Y-%m-%d").date() if sql_row.site_code == 'SDPT': print 'dates not equal' print sql_row.draw_date print visit_date if sql_row.draw_date != visit_date: if sql_row.test_date >= visit_date: print 'update visit date' sql_row.draw_date = visit_date min_visit_date = sql_row.test_date - datetime.timedelta( days=int(settings['result_buffer'])) if visit_date < min_visit_date: malformed_draw.append(record['rc_id']) # The malformed draw dates are the ones that don't fall in to the # accepted window for results. Just report them, nothin more. else: malformed_draw.append(record['rc_id']) # Location update in SQL DB if 'test_site' in record.keys() and record['test_site'] != '': rc_location = '' if redcap.project[key].is_longitudinal() == True: labeled_recs = redcap.project[key].export_records( raw_or_label='label') fil_rec = list(x for x in labeled_recs if x['rc_id'] == record['rc_id'])[0] rc_location = fil_rec['test_site'] else: rc_location = record['test_site'] if sql_row.location != rc_location: sql_row.location = rc_location # Keep track for bulk update in RedCAP later if 'nat_result_date' in record.keys(): if record['nat_result_date'] == '': push_records.append( update_result_redcap(record, sql_row)) else: if 'nat_test_complete' in record.keys() and \ record['nat_test_complete'] == "Incomplete": new_record = {} new_record['rc_id'] = record['rc_id'] new_record['nat_test_complete'] = 2 push_records.append(new_record) Session.commit() # Make the bulk update for every 'key' and 'site' value = redcap.project[key].import_records(push_records) redcap.project['CTS'].import_records(ctsrecords) # The following lines form the sophisticated email system ;-P. Well we again # ask the human to help. malformed_draw_count = len(malformed_draw) # Get list of results with missing draw dates missing_draw, warn_draw, critical_draw = find_missing_draw( days_till_expiration, days_till_urgent, key) missing_draw_count = len(missing_draw) warn_draw_count = len(warn_draw) critical_draw_count = len(critical_draw) weekday = datetime.date.today().weekday() if warn_draw_count != 0 and time_in_range( datetime.datetime.now().time()) and weekday == 5: #Special notifications when draw dates are missing for days_till_urgent # This notification is sent on Fridays(5) report_date = warn_draw report_date_count = warn_draw_count level = 1 notify = settings.get('notify.%s.action' % key.lower()).split() turbomail.send( turbomail.Message( to=notify, subject= '[RedCAP Sync Update]: Prolonged Missing RedCAP Entries for (%s)' % key, plain=lookup.get_template('email/date.mako').render( **{ 'timestamp': datetime.datetime.now(), 'report_date': report_date, 'report_date_count': report_date_count, 'level': level, 'days_till_urgent': days_till_urgent, 'code': key }))) if critical_draw_count != 0: # Very critical draw date events emailed everyday # Between 8-9am time_now = datetime.datetime.now().time() if time_in_range(time_now): log.info( "Some of the draw dates are missing for over %d days" % (int(days_till_urgent) + 2)) report_date = critical_draw report_date_count = critical_draw_count level = 2 notify = settings.get('notify.%s.action' % key.lower()).split() turbomail.send( turbomail.Message( to=notify, subject= '[RedCAP Sync]: Action Required in RedCAP Entries for (%s)!' % key, plain=lookup.get_template( 'email/date.mako').render( **{ 'timestamp': datetime.datetime.now(), 'report_date': report_date, 'report_date_count': report_date_count, 'level': level, 'days_till_urgent': days_till_urgent, 'code': key }))) # Get list of draw dates with missing Red Cross results that are more than 7 days old missing_results = find_missing_results(days_till_notify, days_till_expiration, redcap, key) missing_results_count = len(missing_results) shouldNotify = False # Notify recipients if there is anything to notify about only if its a Monday if missing_results_count > 0 or missing_draw_count > 0 and datetime.date.today( ).weekday() == 1: shouldNotify = True if shouldNotify and time_in_range(datetime.datetime.now().time()): notify = settings.get('notify.%s.action' % key.lower()).split() # Notify appropriate people if notify is set to true turbomail.send( turbomail.Message( to=notify, subject= '[RedCAP Sync]: Synchronization Status Check (%s)' % key, plain=lookup.get_template('email/rcap.mako').render( **{ 'timestamp': datetime.datetime.now(), 'results_count': 0, 'missing_draw_count': missing_draw_count, 'missing_draw': missing_draw, 'missing_results_count': missing_results_count, 'missing_results': missing_results, 'days_till_notify': days_till_notify, 'days_till_urgent': days_till_urgent, 'code': key, 'malformed_draw_count': malformed_draw_count, 'malformed_draw': malformed_draw }))) except: turbomail.send( turbomail.Message( to=settings['notify.error'].split(), subject='[The Early Test]:Exception in matching visit dates!', plain=traceback.format_exc()))
def main(): args = cli.parse_args() settings = args.settings days_till_expiration = int(settings['days.tillexpiration']) days_till_notify = int(settings['days.tillnotify']) try: results = sync_redcap.get_cts_results(settings) if not args.dry: Session.add_all(results) Session.commit() else: log.info('Dry run, not commiting changes') sync_site_codes = settings.get('site.codes').split() ucsd_site_codes = settings.get('ucsd.site.codes').split() emory_site_codes = settings.get('emory.site.codes').split() gwu_site_codes = settings.get('gwu.site.codes').split() rcs = json.loads(open(settings['redcap_json'], 'r').read()) redcap = RCProject(sync_site_codes, rcs) # Refresh results for site_code in sync_site_codes: print 'site code' print site_code for type_ in models.TYPES: #notify = settings.get('notify.%s.%s' % (site_code.lower(), type_.lower()), '').split() notify = [] print type_ pnt = list( r for r in results if r.check(type_) is True and r.site_code == site_code) neg = [ r for r in results if r.check(type_) is False and r.site_code == site_code ] odd = [ r for r in results if r.check(type_) is None and r.site_code == site_code ] if not (pnt): continue 'clear' if type_ == 'dhiv': notify = get_receipients(redcap, 'hiv_pos', site_code, ucsd_site_codes, emory_site_codes, gwu_site_codes) t_type = 'HIV' elif type_ == 'dhcv': notify = get_receipients(redcap, 'hcv_pos', site_code, ucsd_site_codes, emory_site_codes, gwu_site_codes) t_type = 'HCV' elif type_ == 'dhbv': notify = get_receipients(redcap, 'hbv_pos', site_code, ucsd_site_codes, emory_site_codes, gwu_site_codes) t_type = 'HBV' print notify if not notify: continue turbomail.send( turbomail.Message( to=notify, subject='New %s+ NAT' % t_type, plain=lookup.get_template('email/parse.mako').render( **{ 'timestamp': datetime.datetime.now(), 'type_': t_type, 'site_code': site_code, 'pnt': pnt, 'neg': neg, 'odd': odd }))) log.info('Notified %s mailing lists of results for "%s"' % (site_code, type_)) for code in sync_site_codes: results_count = 0 shouldNotify = False # Get number of site specific results in this upload for r in results: if r.site_code == code.upper(): results_count += 1 # Get list of results with missing draw dates missing_draw = find_missing_draw(days_till_expiration, code) missing_draw_count = len(missing_draw) # Get list of draw dates with missing Red Cross results that are more than 7 days old missing_results = find_missing_results(days_till_notify, days_till_expiration, redcap, code) missing_results_count = len(missing_results) # Notify recipients if there is anything to notify about if results_count > 0 or missing_results_count > 0 or missing_draw_count > 0: shouldNotify = True if shouldNotify: #notify = settings.get('notify.%s.sync' % code.lower()).split() notify = get_receipients(redcap, 'date_missing', code, ucsd_site_codes, emory_site_codes, gwu_site_codes) # Notify appropriate people about missing draw dates and Red Cross results turbomail.send( turbomail.Message( to=notify, subject= '[The Early Test]: Red Cross Synchronize Report (%s)' % code, plain=lookup.get_template('email/sync.mako').render( **{ 'timestamp': datetime.datetime.now(), 'results_count': results_count, 'missing_draw_count': missing_draw_count, 'missing_draw': missing_draw, 'missing_results_count': missing_results_count, 'missing_results': missing_results, 'days_till_notify': days_till_notify, 'code': code }))) log.info( 'Notified mailing lists of %s for missing draw dates' % (code)) log.info( 'Notified mailing lists of %s diagnostic for missing Red Cross results' % (code)) except: # If an unexpected error occurs, let the developers know turbomail.send( turbomail.Message( to=settings['notify.error'].split(), subject='[The Early Test]: Parser failed execution', plain=traceback.format_exc())) raise
def main(): args = cli.parse_args() settings = args.settings days_till_expiration = int(settings['days.tillexpiration']) days_till_notify = int(settings['days.tillnotify']) try: log.info('Called on %s' % args.srcfile) results, duplicates = parser.parse(args.srcfile, settings) if not args.dry: if duplicates: raise Exception('\n'.join( ['Already exists: %s%s' % (r.site_code, r.reference_number) for r in duplicates])) # Archive processed file shutil.move(args.srcfile, settings['dir.raw']) log.info('Moved encrypted file to %s' % settings['dir.raw']) # Commit all changes now that we've successfully processed the file map(lambda r: setattr(r, 'file', os.path.basename(args.srcfile)), results) Session.add_all(results) Session.commit() else: log.info('Dry run, not commiting changes') sync_site_codes = settings.get('site.codes').split() rcs = json.loads(open(settings['redcap_json'], 'r').read()) redcap = RCProject(sync_site_codes, rcs) # Refresh results for site_code in sync_site_codes: for type_ in models.TYPES: notify = settings.get('notify.%s.%s' % (site_code.lower(), type_.lower()), '').split() if not notify: continue pnt = [r for r in results if r.check(type_) is True and r.site_code == site_code] neg = [r for r in results if r.check(type_) is False and r.site_code == site_code] odd = [r for r in results if r.check(type_) is None and r.site_code == site_code] if not (pnt or odd): continue turbomail.send(turbomail.Message( to=notify, subject='[The Early Test]: New Records Notification (%s)' % type_, plain=lookup.get_template('email/parse.mako').render(**{ 'timestamp': datetime.datetime.now(), 'type_': type_, 'site_code': site_code, 'pnt': pnt, 'neg': neg, 'odd': odd}))) log.info('Notified %s mailing lists of results for "%s"' % (site_code, type_)) for code in sync_site_codes: results_count = 0 shouldNotify = False # Get number of site specific results in this upload for r in results: if r.site_code == code.upper(): results_count += 1 # Get list of results with missing draw dates missing_draw = find_missing_draw(days_till_expiration, code) missing_draw_count = len(missing_draw) # Get list of draw dates with missing Red Cross results that are more than 7 days old missing_results = find_missing_results(days_till_notify, days_till_expiration, redcap, code) missing_results_count = len(missing_results) # Notify recipients if there is anything to notify about if results_count > 0 or missing_results_count > 0 or missing_draw_count > 0: shouldNotify = True if shouldNotify: notify = settings.get('notify.%s.sync' % code.lower()).split() # Notify appropriate people about missing draw dates and Red Cross results turbomail.send(turbomail.Message( to=notify, subject='[The Early Test]: Red Cross Synchronize Report (%s)' % code, plain=lookup.get_template('email/sync.mako').render(**{ 'timestamp': datetime.datetime.now(), 'results_count': results_count, 'missing_draw_count': missing_draw_count, 'missing_draw': missing_draw, 'missing_results_count': missing_results_count, 'missing_results': missing_results, 'days_till_notify': days_till_notify, 'code': code}))) log.info('Notified mailing lists of %s for missing draw dates' % (code)) log.info('Notified mailing lists of %s diagnostic for missing Red Cross results' % (code)) except: # If an unexpected error occurs, let the developers know turbomail.send(turbomail.Message( to=settings['notify.error'].split(), subject='[The Early Test]: Parser failed execution', plain=traceback.format_exc())) raise