def find_missing_draw(days_till_expiration, days_till_urgent, code):
current_time = datetime.date.today()
results_expiration = current_time - datetime.timedelta(
days=days_till_expiration)
warn_missing = current_time - datetime.timedelta(days=days_till_urgent)
critically_missing = warn_missing - datetime.timedelta(days=2)
missing_draw = Session.query(
models.Result).filter(models.Result.site_code == code).filter(
models.Result.draw_date == None).filter(
models.Result.test_date > warn_missing).all()
warn_missing_draw = Session.query(
models.Result).filter(models.Result.site_code == code).filter(
models.Result.draw_date == None).filter(
models.Result.test_date > critically_missing).filter(
models.Result.test_date <= warn_missing).all()
critically_missing_draw = Session.query(
models.Result).filter(models.Result.site_code == code).filter(
models.Result.draw_date == None).filter(
models.Result.test_date <= critically_missing).filter(
models.Result.test_date > results_expiration).all()
return missing_draw, warn_missing_draw, critically_missing_draw
def test_DrawDateSync_ResultHasNoCorrespondingDrawDateInDrawTable_ResultDrawDateIsNone(
self):
"""
Result draw dates should be None if no available corresponding draw date in Draw table
Sync code is in parser.py
"""
site_code = '999X'
reference_number = '12345'
gpg = self.gpg
datafile = os.path.join(self.tempdir, 'data')
io = StringIO()
print >> io, 'Has 1 result'
print >> io, '999X 12345 P N N N' # NOQA
io.flush()
io.seek(0)
gpg.encrypt_file(io,
str(self.key),
passphrase=self.passphrase,
output=datafile)
io.close()
sys.argv = ['', '-c', self.configfile, datafile]
scripts.parse.main()
result = Session.query(models.Result)\
.filter(models.Result.site_code == site_code)\
.filter(models.Result.reference_number == reference_number).first()
# Is there a draw date?
assert not result.draw_date
def test_dry(self):
"""
Ensure --dry option doesn't affect the file system and database
"""
config = self.config
gpg = self.gpg
datafile = os.path.join(self.tempdir, 'data')
io = StringIO()
print >> io, 'Has 1 result'
print >> io, '076C 12345 P'
io.flush()
io.seek(0)
gpg.encrypt_file(io,
str(self.key),
passphrase=self.passphrase,
output=datafile)
io.close()
sys.argv = ['', '-c', self.configfile, '--dry', datafile]
scripts.parse.main()
eq_(0, Session.query(models.Result).count())
assert not os.path.exists(
os.path.join(config.get('settings', 'dir.web'), '76C.html'))
assert not os.path.exists(
os.path.join(config.get('settings', 'dir.web'), '76C.xls'))
def find_missing_draw(days_till_expiration, code):
current_time = datetime.date.today()
results_expiration = current_time - datetime.timedelta(
days=days_till_expiration)
missing_draw = Session.query(
models.Result).filter(models.Result.site_code == code).filter(
models.Result.draw_date == None).filter(
models.Result.test_date > results_expiration).all()
return missing_draw
def test_main(self):
gpg = self.gpg
datafile = os.path.join(self.tempdir, 'data')
io = StringIO()
print >> io, 'Has 1 result'
print >> io, '076CX 12345 P'
io.flush()
io.seek(0)
gpg.encrypt_file(io,
str(self.key),
passphrase=self.passphrase,
output=datafile)
io.close()
sys.argv = ['', '-c', self.configfile, datafile]
scripts.parse.main()
eq_(1, Session.query(models.Result).count())
def test_DrawDateSync_ResultHasCorrespondingDrawDateInDrawTable_ResultDrawDateIsSynced(
self):
"""
Should sync draw dates from Draw table to results in Result table
"""
# Add draw date to draw table first
site_code = '999X'
reference_number = '12345'
# Add draw date to Draw table
Session.add(
models.Draw(site_code=site_code,
reference_number=reference_number,
draw_date=datetime.date.today()))
transaction.commit()
gpg = self.gpg
datafile = os.path.join(self.tempdir, 'data')
io = StringIO()
print >> io, 'Has 1 result'
print >> io, '999X 12345 P N N N' # NOQA
io.flush()
io.seek(0)
gpg.encrypt_file(io,
str(self.key),
passphrase=self.passphrase,
output=datafile)
io.close()
sys.argv = ['', '-c', self.configfile, datafile]
scripts.parse.main()
result = Session.query(models.Result)\
.filter(models.Result.site_code == site_code)\
.filter(models.Result.reference_number == reference_number).first()
# Is there a draw date?
assert result.draw_date
def bucketize_results():
""" Returns a dictonary of buckets which has site_codes as keys
and list of results as values"""
buckets = {}
try:
all_results = Session.query(models.Result)
site_codes = []
for result in all_results:
try:
if result.site_code not in site_codes:
site_codes.append(result.site_code)
buckets[result.site_code] = []
buckets[result.site_code].append(
str(result.site_code) + str(result.reference_number))
except:
pass
except Exception as e:
log.critical(traceback.format_exc())
return buckets
def sync_sql_result(buckets, settings):
"""
Parameters:
buckets: Dict of site codes and list of Results for that site as key, value pairs
settings: A dictionary of settings specified in the configuration file
Returns None
"""
try:
log.info("Synchronization Routine Started")
#only two attributes could have been updated Draw Date or Location
#Result should not be modified in the RedCAP directly
rcs = json.loads(open(settings['redcap_json'], 'r').read())
# Days after which results become invalid
days_till_expiration = int(settings['days.tillexpiration'])
# Days after which results can be published to patients
days_till_notify = int(settings['days.tillnotify'])
# Days after which tester should definitely act ASAP on CRF's or redcap entries
days_till_urgent = int(settings['days.tillurgent'])
pk_sites = settings.get('pk_site').split()
redcap = RCProject(buckets.keys(), rcs)
for key, value in buckets.iteritems():
ctsrecords = []
# These malformed draw dates need to be handled offline. This script will only be
# reporting such entries
malformed_draw = []
# Unmindful check. This line is mostly a deadcode, I guess.
if key not in redcap.project.keys():
continue
print 'key'
print key
#fields = ['visit_date','test_site']
# we pass the 'label' flag to get the location value as string instead numeric values
records = []
if redcap.project[key].is_longitudinal(
) == True or key in pk_sites:
print 'is logitudinal'
print key
l_records = redcap.project[key].export_records()
records = list(x for x in l_records if x['rc_id'] in value)
else:
records = redcap.project[key].export_records(
records=value,
fields=redcap.nat_fields,
raw_or_label='label')
ctsrecord = redcap.project['CTS'].export_records(
records=value, fields='rec_status')
for each in ctsrecord:
new_ctsrecord = {}
new_ctsrecord['rc_id'] = each['rc_id']
new_ctsrecord['rec_status'] = 1
ctsrecords.append(new_ctsrecord)
# RCIDs for which we have new results will be in push records
push_records = []
for record in records:
sql_row = Session.query(models.Result)\
.filter(models.Result.site_code == key)\
.filter(models.Result.reference_number == record['rc_id'][-5:]).first()
# Visit/Draw date update in SQL DB
if 'visit_date' in record.keys(
) and record['visit_date'] != '':
visit_date = dt.strptime(record['visit_date'],
"%Y-%m-%d").date()
if sql_row.site_code == 'SDPT':
print 'dates not equal'
print sql_row.draw_date
print visit_date
if sql_row.draw_date != visit_date:
if sql_row.test_date >= visit_date:
print 'update visit date'
sql_row.draw_date = visit_date
min_visit_date = sql_row.test_date - datetime.timedelta(
days=int(settings['result_buffer']))
if visit_date < min_visit_date:
malformed_draw.append(record['rc_id'])
# The malformed draw dates are the ones that don't fall in to the
# accepted window for results. Just report them, nothin more.
else:
malformed_draw.append(record['rc_id'])
# Location update in SQL DB
if 'test_site' in record.keys() and record['test_site'] != '':
rc_location = ''
if redcap.project[key].is_longitudinal() == True:
labeled_recs = redcap.project[key].export_records(
raw_or_label='label')
fil_rec = list(x for x in labeled_recs
if x['rc_id'] == record['rc_id'])[0]
rc_location = fil_rec['test_site']
else:
rc_location = record['test_site']
if sql_row.location != rc_location:
sql_row.location = rc_location
# Keep track for bulk update in RedCAP later
if 'nat_result_date' in record.keys():
if record['nat_result_date'] == '':
push_records.append(
update_result_redcap(record, sql_row))
else:
if 'nat_test_complete' in record.keys() and \
record['nat_test_complete'] == "Incomplete":
new_record = {}
new_record['rc_id'] = record['rc_id']
new_record['nat_test_complete'] = 2
push_records.append(new_record)
Session.commit()
# Make the bulk update for every 'key' and 'site'
value = redcap.project[key].import_records(push_records)
redcap.project['CTS'].import_records(ctsrecords)
# The following lines form the sophisticated email system ;-P. Well we again
# ask the human to help.
malformed_draw_count = len(malformed_draw)
# Get list of results with missing draw dates
missing_draw, warn_draw, critical_draw = find_missing_draw(
days_till_expiration, days_till_urgent, key)
missing_draw_count = len(missing_draw)
warn_draw_count = len(warn_draw)
critical_draw_count = len(critical_draw)
weekday = datetime.date.today().weekday()
if warn_draw_count != 0 and time_in_range(
datetime.datetime.now().time()) and weekday == 5:
#Special notifications when draw dates are missing for days_till_urgent
# This notification is sent on Fridays(5)
report_date = warn_draw
report_date_count = warn_draw_count
level = 1
notify = settings.get('notify.%s.action' % key.lower()).split()
turbomail.send(
turbomail.Message(
to=notify,
subject=
'[RedCAP Sync Update]: Prolonged Missing RedCAP Entries for (%s)'
% key,
plain=lookup.get_template('email/date.mako').render(
**{
'timestamp': datetime.datetime.now(),
'report_date': report_date,
'report_date_count': report_date_count,
'level': level,
'days_till_urgent': days_till_urgent,
'code': key
})))
if critical_draw_count != 0:
# Very critical draw date events emailed everyday
# Between 8-9am
time_now = datetime.datetime.now().time()
if time_in_range(time_now):
log.info(
"Some of the draw dates are missing for over %d days" %
(int(days_till_urgent) + 2))
report_date = critical_draw
report_date_count = critical_draw_count
level = 2
notify = settings.get('notify.%s.action' %
key.lower()).split()
turbomail.send(
turbomail.Message(
to=notify,
subject=
'[RedCAP Sync]: Action Required in RedCAP Entries for (%s)!'
% key,
plain=lookup.get_template(
'email/date.mako').render(
**{
'timestamp': datetime.datetime.now(),
'report_date': report_date,
'report_date_count': report_date_count,
'level': level,
'days_till_urgent': days_till_urgent,
'code': key
})))
# Get list of draw dates with missing Red Cross results that are more than 7 days old
missing_results = find_missing_results(days_till_notify,
days_till_expiration,
redcap, key)
missing_results_count = len(missing_results)
shouldNotify = False
# Notify recipients if there is anything to notify about only if its a Monday
if missing_results_count > 0 or missing_draw_count > 0 and datetime.date.today(
).weekday() == 1:
shouldNotify = True
if shouldNotify and time_in_range(datetime.datetime.now().time()):
notify = settings.get('notify.%s.action' % key.lower()).split()
# Notify appropriate people if notify is set to true
turbomail.send(
turbomail.Message(
to=notify,
subject=
'[RedCAP Sync]: Synchronization Status Check (%s)' %
key,
plain=lookup.get_template('email/rcap.mako').render(
**{
'timestamp': datetime.datetime.now(),
'results_count': 0,
'missing_draw_count': missing_draw_count,
'missing_draw': missing_draw,
'missing_results_count': missing_results_count,
'missing_results': missing_results,
'days_till_notify': days_till_notify,
'days_till_urgent': days_till_urgent,
'code': key,
'malformed_draw_count': malformed_draw_count,
'malformed_draw': malformed_draw
})))
except:
turbomail.send(
turbomail.Message(
to=settings['notify.error'].split(),
subject='[The Early Test]:Exception in matching visit dates!',
plain=traceback.format_exc()))