def setup_module():
"""Create a fresh db (once) for all tests in this module"""
configure_logging(verbosity=2, logfile='unittest.log')
c = Config()
if c.get('general', 'in_production'): # pragma: no cover
raise RuntimeError("DO NOT run destructive test on production system")
create_tables(enable_delete=True, **db_params(CONFIG_SECTION))
# create a "test_region" and a couple bogus facilities
f1 = Facility(county='KING',
npi=10987,
zip='12345',
organization_name='Reason Medical Center',
local_code='RMC')
f2 = Facility(county='POND',
npi=65432,
zip='67890',
organization_name='No-Reason Medical Center',
local_code='NMC')
conn = db_connection(CONFIG_SECTION)
conn.session.add(f1)
conn.session.add(f2)
conn.session.commit()
rr1 = ReportableRegion(region_name='test_region', dim_facility_pk=10987)
conn.session.add(rr1)
conn.session.commit()
conn.disconnect()
def setup_module():
"""Create a fresh db (once) for all tests in this module"""
configure_logging(verbosity=2, logfile='unittest.log')
c = Config()
if c.get('general', 'in_production'): # pragma: no cover
raise RuntimeError("DO NOT run destructive test on production system")
create_tables(enable_delete=True, **db_params(CONFIG_SECTION))
def setUp(self):
c = Config()
cfg_value = lambda v: c.get('warehouse', v)
self.alchemy = AlchemyAccess(database=cfg_value('database'),
host='localhost',
user=cfg_value('database_user'),
password=cfg_value('database_password'))
self.session = self.alchemy.session
def test_configure_logging():
logfile = configure_logging(verbosity=2,
logfile='unittest.log',
append=False)
logging.debug("just testing")
# Thanks to nose working so hard to capture logging, it's quite
# difficult to test - hand verified.
assert (logfile == os.path.join(Config().get('general', 'log_dir'),
'unittest.log'))
def __init__(self):
self.config = Config()
# obtain list of files to process
path = os.path.abspath(
os.path.join(os.path.dirname(__file__),
"../../../test_hl7_batchfiles"))
self.filenames = [
os.path.join(path, file) for file in os.listdir(path)
]
def setup_module():
"""Create a fresh db (once) for all tests in this module"""
c = Config()
if c.get('general', 'in_production'): # pragma: no cover
raise RuntimeError("DO NOT run destructive test on production system")
cfg_value = lambda v: c.get('warehouse', v)
create_tables(cfg_value('create_table_user'),
cfg_value('create_table_password'),
cfg_value('database'),
enable_delete=True)
def inProduction():
"""Simple state check to avoid uploading files to thrid party
servers and what not when not 'in production'.
"""
config = Config()
try:
return config.get('general', 'in_production')
except (ConfigParser.NoSectionError,
ConfigParser.NoOptionError): # pragma: no cover
raise ValueError("Config file doesn't specify "
"`[general]in_production` "
"unsafe to guess, can't continue")
def test_override(self):
"same key in second file should override"
section = 'SECTION'
key = 'unittest'
for i in range(2):
cp = ConfigParser.RawConfigParser()
cp.add_section(section)
cp.set(section, key, i)
with open(self.config_files[i], 'w') as f:
cp.write(f)
c = Config(self.config_files)
self.assertEquals(1, c.get(section, key))
def setup_module():
"""Create a fresh db (once) for all tests in this module"""
configure_logging(verbosity=2, logfile='unittest.log')
c = Config()
if c.get('general', 'in_production'): # pragma: no cover
raise RuntimeError("DO NOT run destructive test on production system")
create_tables(enable_delete=True, **db_params(CONFIG_SECTION))
# Load in all the static data in anonymized form
static_data_file = open(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'anon_static_db_data.yaml'), 'r')
load_file(static_data_file)
def test_tilde(self):
"support tilde in directory paths"
section = 'SECTION'
key = 'unittest'
value = "~/tempfile"
cp = ConfigParser.RawConfigParser()
cp.add_section(section)
cp.set(section, key, value)
with open(self.config_files[0], 'w') as f:
cp.write(f)
c = Config(self.config_files)
self.assertEquals(os.path.expanduser("~/tempfile"),
c.get(section, key))
def db_params(section):
"""Return dict of database connection values from named config section
Returned dict includes:
- user
- password
- database (name of database)
"""
config = Config()
database = config.get(section, 'database')
user = config.get(section, 'database_user')
password = config.get(section, 'database_password')
return {'user': user, 'password': password, 'database': database}
def test_float(self):
"Looks like a float, should be one"
section = 'SECTION'
key = 'unittest'
values = ['0.01', '42.0', '-67.3']
cp = ConfigParser.RawConfigParser()
cp.add_section(section)
for i in range(len(values)):
cp.set(section, key + str(i), values[i])
with open(self.config_files[0], 'w') as f:
cp.write(f)
c = Config(self.config_files)
for i in range(len(values)):
self.assertEquals(float(values[i]), c.get(section, key + str(i)))
def test_falseness(self):
"false values should be case insensitive"
section = 'SECTION'
key = 'unittest'
values = ['FALSE', 'false', 'False', ' f ']
cp = ConfigParser.RawConfigParser()
cp.add_section(section)
for i in range(len(values)):
cp.set(section, key + str(i), values[i])
with open(self.config_files[0], 'w') as f:
cp.write(f)
c = Config(self.config_files)
for i in range(len(values)):
self.assertEquals(False, c.get(section, key + str(i)))
def test_truthiness(self):
"truth value should be case insensitive"
section = 'SECTION'
key = 'unittest'
values = ['TRUE', 'true', 'True', ' t ']
cp = ConfigParser.RawConfigParser()
cp.add_section(section)
for i in range(len(values)):
cp.set(section, key + str(i), values[i])
with open(self.config_files[0], 'w') as f:
cp.write(f)
c = Config(self.config_files)
for i in range(len(values)):
self.assertEquals(True, c.get(section, key + str(i)))
def main(): # pragma: no cover
"""Entry point to (re)create the table using config settings"""
config = Config()
database = config.get('longitudinal', 'database')
print "destroy and recreate database %s ? "\
"('destroy' to continue): " % database,
answer = sys.stdin.readline().rstrip()
if answer != 'destroy':
print "aborting..."
sys.exit(1)
user = config.get('longitudinal', 'database_user')
password = config.get('longitudinal', 'database_password')
create_tables(user, password, database)
def test_int(self):
"Looks like an int, should be one (including 0,1)"
section = 'SECTION'
key = 'unittest'
values = ['0', '1', '-67', 42, -1]
cp = ConfigParser.RawConfigParser()
cp.add_section(section)
for i in range(len(values)):
cp.set(section, key + str(i), values[i])
with open(self.config_files[0], 'w') as f:
cp.write(f)
c = Config(self.config_files)
for i in range(len(values)):
self.assertEquals(int(values[i]), c.get(section, key + str(i)))
def main(): # pragma: no cover
"""Entry point to (re)create the table using config settings"""
config = Config()
dbname = config.get('warehouse', 'database')
print "destroy and recreate database %s ? "\
"('destroy' to continue): " % dbname,
answer = sys.stdin.readline().rstrip()
if answer != 'destroy':
print "aborting..."
sys.exit(1)
user = config.get('warehouse', 'create_table_user')
print "password for PostgreSQL user:", user
password = getpass.getpass()
create_tables(user, password, dbname)
def create_tables(user, password, dbname, enable_delete=False):
"""Create the warehouse database tables.
NB the config [warehouse]database_user is granted SELECT and
INSERT permissions (plus DELET if enable_delete is set).
:param user: database user with table creation grants
:param password: the database password
:param dbname: the database name to populate
:param enable_delete: testing hook, override for testing needs
"""
engine = create_engine("postgresql://%s:%s@localhost/%s" %\
(user, password, dbname))
metadata.drop_all(bind=engine)
metadata.create_all(bind=engine)
def bless_user(user):
engine.execute("""BEGIN; GRANT SELECT, INSERT, UPDATE %(delete)s ON
hl7_raw_message,
hl7_msh,
hl7_visit,
hl7_dx,
hl7_obr,
hl7_obx,
hl7_nte,
hl7_spm TO %(user)s;
COMMIT;""" % {
'delete': ", DELETE" if enable_delete else '',
'user': user
})
# Sequences also require UPDATE
engine.execute("""BEGIN; GRANT SELECT, UPDATE ON
hl7_dx_hl7_dx_id_seq,
hl7_msh_hl7_msh_id_seq,
hl7_obr_hl7_obr_id_seq,
hl7_obx_hl7_obx_id_seq,
hl7_nte_hl7_nte_id_seq,
hl7_spm_hl7_spm_id_seq,
hl7_raw_message_hl7_raw_message_id_seq,
hl7_visit_hl7_visit_id_seq TO %(user)s; COMMIT;""" %
{'user': user})
# Bless the mirth user with the minimal set of privileges
# Mirth only SELECTs and INSERTs at this time
bless_user(Config().get('warehouse', 'database_user'))
def _generate_output_filename(self, start_date=None, end_date=None):
start_date = self.criteria.start_date if start_date is None\
else start_date
end_date = self.criteria.end_date if end_date is None else end_date
datestr = end_date.strftime('%Y%m%d')
if start_date != end_date:
datestr = '-'.join(
(start_date.strftime('%Y%m%d'), end_date.strftime('%Y%m%d')))
filename = self.criteria.report_method + '-' + datestr + '.txt'
config = Config()
tmp_dir = config.get('general', 'tmp_dir', default='/tmp')
filepath = os.path.join(tmp_dir, filename)
return filepath
def setup_module():
"""Populate database with test data for module tests"""
c = Config()
if c.get('general', 'in_production'): # pragma: no cover
raise RuntimeError("DO NOT run destructive test on production system")
"Pull in the filesystem dump from a previous mirth run"
mi = MirthInteraction()
mi.restore_database()
"Run a quick sanity check, whole module requires a populated db"
connection = db_connection('warehouse')
count = connection.session.query(HL7_Msh).count()
connection.disconnect()
if count < 4000:
err = "Minimal expected count of records not present. "\
"Be sure to run 'process_testfiles_via_mirth' as a prerequisite"
raise RuntimeError(err)
def __init__(self, verbosity=0, source_db=None):
self.verbosity = verbosity
self.source_db = source_db
config = Config()
self.phinms_receiving_dir = config.get('phinms', 'receiving_dir')
self.phinms_archive_dir = config.get('phinms', 'archive_dir')
self.source_dir = self.phinms_receiving_dir
# Confirm the required directories are present
if not os.path.isdir(self.phinms_receiving_dir):
raise ValueError("Can't find required directory %s" %
self.phinms_receiving_dir)
UPLOAD_PORT = config.get('pheme_http_receiver', 'port')
UPLOAD_HOST = config.get('pheme_http_receiver', 'host')
self.http_pool = HTTPConnectionPool(host=UPLOAD_HOST,
port=UPLOAD_PORT,
timeout=20)
self._copy_tempdir = None
def test_missing(self):
"Asking for missing value without a default should raise"
c = Config(self.config_files)
self.assertRaises(RuntimeError, c.get, 'section', 'value')
def test_absent(self):
"no config files present shouldn't raise"
c = Config(self.config_files)
self.assertTrue(c)
def test_default(self):
"Asking for missing value with a default"
c = Config(self.config_files)
self.assertEquals(42, c.get('Lifes', 'Answer', 42))
class GenerateReport(object):
""" Process options and generate the requested report. Optionally
persists the file to the filesystem, and uploads to the DOH sftp
server.
"""
__version__ = '0.2'
config = Config()
IGNORE_SITE = config.get('longitudinal', 'ignore_site', default='')
# Order matters, create a tuple of paired values (reportColumn,
# essenceColumn) - NB, the Diagnosis column is being bastardized.
# Previously there was an SQL function to do the subselect, but it
# ran way too slow. Now contains the foreign key to join w/ the
# diagnosis for the respective visit.
diagnosis_column_index = 7
patient_class_column_index = 11
columns = (
('Hosp', 'hospital'),
('Reg Date', 'visit_date'),
('Time', 'visit_time'),
('Sex', 'gender'),
('Age', 'age'),
('Reason For Visit', 'chief_complaint'),
('Zip Code', 'zip'),
('Diagnosis', 'visit_pk'),
('Admit Status', 'gipse_disposition'),
('Medical Record No.', 'patient_id'),
('Visit Record No.', 'visit_id'),
('Service Area', 'patient_class'),
)
assert (columns[diagnosis_column_index][1] == 'visit_pk')
assert (columns[patient_class_column_index][1] == 'patient_class')
def __init__(self,
user=None,
password=None,
report_criteria=None,
datefile=None):
"""Initialize report generation.
:param user: database user
:param password: database password
:param report_criteria: ReportCriteria defining specifics
:param datefile: useful for persistent walks through time
"""
self.user = user
self.password = password
self.criteria = report_criteria
self.database = self.criteria.database
if datefile:
assert ((self.criteria.start_date,
self.criteria.end_date) == datefile.get_date_range())
self.datePersistence = datefile
self._diags = {}
self._prepare_output_file()
self._prepare_columns()
self._set_transport()
def _prepare_columns(self):
# Don't include the patient_class column if splitting out by
# patient_class
if self.criteria.patient_class:
len_b4 = len(self.columns)
self.columns =\
self.columns[:self.patient_class_column_index] \
+ self.columns[self.patient_class_column_index + 1:]
assert (len(self.columns) + 1 == len_b4)
def _set_transport(self):
"""Plug in the appropriate transport mechanism"""
# Transport strategies differ for the different reports
if self.criteria.reportable_region:
self._transport = Distribute_client(zip_first=True)
else:
self._transport = PHINMS_client(zip_first=True)
def _generate_output_filename(self, start_date=None, end_date=None):
start_date = self.criteria.start_date if start_date is None\
else start_date
end_date = self.criteria.end_date if end_date is None else end_date
datestr = end_date.strftime('%Y%m%d')
if start_date != end_date:
datestr = '-'.join(
(start_date.strftime('%Y%m%d'), end_date.strftime('%Y%m%d')))
filename = self.criteria.report_method + '-' + datestr + '.txt'
config = Config()
tmp_dir = config.get('general', 'tmp_dir', default='/tmp')
filepath = os.path.join(tmp_dir, filename)
return filepath
def _prepare_output_file(self):
"""Open the local filesystem file for output"""
filepath = self.\
_generate_output_filename(start_date=self.criteria.start_date,
end_date=self.criteria.end_date)
# watch for oversight errors; notify if like report exists -
# unless it's size zero (from a previous failed run)
if os.path.exists(filepath) and os.path.getsize(filepath):
logging.warning("Found requested report file already "\
"exists - overwriting: '%s'"\
% filepath)
self.output = open(filepath, 'w')
self._output_filename = self.output.name
@property
def output_filename(self):
if not hasattr(self, '_output_filename'):
raise RuntimeError("prerequisite call to "\
"_prepare_output_file() "\
"didn't happen!")
return self._output_filename
def _header(self):
if self.criteria.include_vitals:
columns = [c[0] for c in self.columns]
columns += ('Measured Temperature', 'O2 Saturation',
'Self-Reported Influenza Vaccine',
'Self-Reported H1N1 Vaccine')
return '|'.join(columns)
else:
return '|'.join([c[0] for c in self.columns])
def _build_join_tables(self):
""" Scope continues to grow, build all join tables necessary
for the query. Some are only necessary with certain features
on.
"""
# Always need the list of reportable visits
self._build_visit_join_table()
if self.criteria.include_vitals:
self._build_vitals_join_table()
def _build_visit_join_table(self):
""" Helper in selection of visits for the report - this method
builds a temporary table and populates it with the visit_pks
that belong in the report. This should include all visit_pks
with the matching admit_datetime as well as any that have
received updates since the last like report was produced.
"""
# If include_vitals is on, we also need the visit_id to keep
# the joins managable. vitals don't have a patient class, so
# you can't join on the same values.
sql = "CREATE TEMPORARY TABLE reportable_pks (pk "\
"integer not null unique)"
selectCols = "fact_visit.pk"
self._getConn()
self.access.raw_query(sql)
# If we're only selecting those facilites in a region, the SQL
# is more complicated - build up the respective clauses.
joinClause = regionClause = ""
if self.criteria.reportable_region:
joinClause = "JOIN internal_reportable_region ON "\
"internal_reportable_region.dim_facility_pk = "\
"fact_visit.dim_facility_pk"
regionClause = "AND region_name = '%s'" %\
self.criteria.reportable_region
# Another HACK! One site is not even wanted by the state DOH,
# as it's being duplicated from another source, and ESSENCE
# can't help but count them twice. Remove this one site
# regardless
else:
joinClause = "JOIN internal_reportable_region ON "\
"internal_reportable_region.dim_facility_pk = "\
"fact_visit.dim_facility_pk"
regionClause = "AND region_name = '%s'" % self.IGNORE_SITE
# Limit by patient_class if requested. Note we may still end
# up with visit ids that have changed patient classes, so more
# pruning later is necessary.
pc_limit = ""
if self.criteria.patient_class:
pc_limit = "AND patient_class = '%c'" %\
self.criteria.patient_class
# Start with all visits for the requested date range
sql = "INSERT INTO reportable_pks SELECT %s FROM "\
"fact_visit %s WHERE admit_datetime BETWEEN '%s' AND "\
"'%s' %s %s" %\
(selectCols, joinClause, self.criteria.start_date,
self.criteria.end_date + timedelta(days=1),
pc_limit, regionClause)
self.access.raw_query(sql)
if self.criteria.include_updates:
# In this case, add all visits with updates since the
# last run, but no newer than the requested date (in case
# we're building reports forward from historical data)
sql = "SELECT max(processed_datetime) FROM internal_report "\
"WHERE report_method = '%s'" % self.criteria.report_method
cursor = self.access.raw_query(sql)
last_report_generated = cursor.fetchall()[0][0]
if last_report_generated is None:
last_report_generated = '2009-01-01' # our epoch
logging.debug("including updates, last_report_generated: "\
"%s", last_report_generated)
sql = "INSERT INTO reportable_pks SELECT %(sel_cols)s FROM "\
"fact_visit %(join_clause)s LEFT JOIN reportable_pks ON "\
"reportable_pks.pk = fact_visit.pk WHERE "\
"last_updated > '%(last_report)s' AND admit_datetime "\
"< '%(date)s' AND reportable_pks.pk IS NULL "\
"%(pc_limit)s %(region_clause)s" %\
{'sel_cols': selectCols,
'last_report': last_report_generated,
'date': self.criteria.end_date + timedelta(days=1),
'pc_limit': pc_limit,
'join_clause': joinClause,
'region_clause': regionClause}
self.access.raw_query(sql)
cursor = self.access.raw_query("SELECT COUNT(*) FROM "\
"reportable_pks")
logging.debug("%d visits to report on", cursor.fetchall()[0][0])
def _build_vitals_join_table(self):
"""When report is to include vitals - we use an additional
temporary table (visit_loinc_data) to hold the data for more
timely queries.
Like the rest of the report, the list of interesting visits is
limited to the rows in the reportable_pks - see
_build_join_table() for details.
"""
raise ValueError('not ported yet')
sql = """
CREATE TEMPORARY TABLE visit_loinc_data (
visit_id VARCHAR(255) not null,
patient_class CHAR(1) default null,
observation_id VARCHAR(255) not null,
observation_result VARCHAR(255) not null)
"""
self._getConn()
self.access.raw_query(sql)
sql = """
INSERT INTO visit_loinc_data (visit_id, patient_class,
observation_id, observation_result) SELECT visit.visit_id,
visit.patient_class, observation_id,
observation_result FROM visit JOIN hl7_visit ON
visit.visit_id = hl7_visit.visit_id JOIN hl7_obx ON
hl7_visit.hl7_msh_id = hl7_obx.hl7_msh_id JOIN
reportable_pks ON reportable_pks.visit_id = visit.visit_id
AND reportable_pks.patient_class = visit.patient_class
WHERE
observation_id in ('8310-5', '20564-1', '46077-4',
'29544-4')
"""
self.access.raw_query(sql)
def _select_from_essence_view(self):
"""Build up the SQL select statement to be used in gathering
the data for this report.
"""
stmt = """SELECT %s FROM essence e JOIN reportable_pks ri
ON e.visit_pk = ri.pk""" %\
(','.join(['e.' + c[1] for c in self.columns]))
return stmt
def _select_diagnosis(self):
""" Need to pull in all the diagnosis data for this report.
This is saved in an instance dictionary for use in
self._diagnosis to generate the list of diagnoses for each
respective visit.
A list of unique diagnoses ordered by rank is required.
"""
# We order descending on dx_datetime as the most recent should
# be best. Add any others as the persistence mechanism only
# saves a unique icd9 dx that has changed status.
stmt = "SELECT fact_visit_pk, rank, icd9 "\
"FROM assoc_visit_dx JOIN "\
"dim_dx ON dim_dx_pk = dim_dx.pk JOIN "\
"reportable_pks ON "\
"assoc_visit_dx.fact_visit_pk = reportable_pks.pk "\
"ORDER BY dx_datetime DESC"
cursor = self.access.raw_query(stmt)
for row in cursor.fetchall():
visit_pk = row[0]
if visit_pk in self._diags:
self._diags[visit_pk].add(row[0], row[1], row[2])
else:
self._diags[visit_pk] = \
SortedDiagnosis(row[0], row[1], row[2])
def _diagnosis(self, visit_pk):
if visit_pk in self._diags:
return [
self._diags[visit_pk].__repr__(),
]
else:
return [
'',
]
def _select_vitals(self):
""" Need to pull in all the vitals data for this report.
This is saved in an instance dictionary for use in
self._vitals_for_visit to generate the list of vitals for each
respective visit.
This is an effective NOP when self.criteria.include_vitals = False
"""
if not self.criteria.include_vitals:
return None
self._vitals = {}
stmt = """SELECT reportable_pks.visit_pk,
observation_id, observation_result
FROM visit_loinc_data JOIN reportable_pks ON
reportable_pks.visit_id = visit_loinc_data.visit_id"""
cursor = self.access.raw_query(stmt)
for row in cursor.fetchall():
visit_pk = row[0]
if visit_pk in self._vitals:
self._vitals[visit_pk].add(row[1], row[2])
else:
self._vitals[visit_pk] = \
Vitals(row[1], row[2])
def _vitals_for_visit(self, visit_pk):
"""Returns the list of vitals for the visit in question.
This is an effective NOP when self.criteria.include_vitals = False
"""
if not self.criteria.include_vitals:
return []
if visit_pk in self._vitals:
return self._vitals[visit_pk].__repr__()
else:
return Vitals().__repr__()
def _write_report(self, save_report=False):
""" Write out and potentially store the results.
Generate results via database queries and write the results to
self.output.
:param save_report: If set, persist the document and related
metadata to the mbds archive.
returns the document ID, the mbds archive key, if saved
"""
out = self.output
print >> out, self._header()
self._build_join_tables()
self._select_diagnosis()
self._select_vitals()
cursor = self.access.raw_query(self._select_from_essence_view())
for row in cursor.fetchall():
# Each row is the colums up to the diagnosis + the
# comma separated diagnosis + the rest of the columns
# and finally with vitals if configured for such
visit_pk = row[self.diagnosis_column_index] # yuck, but true
print >> out,\
'|'.join([strSansNone(column) for column in
row[:self.diagnosis_column_index]] +
self._diagnosis(visit_pk) +
[strSansNone(column) for column in
row[self.diagnosis_column_index + 1:]] +
self._vitals_for_visit(visit_pk))
# Close the file and persist to the document archive if
# requested
self.output.close()
if save_report:
metadata = {
k: v
for k, v in self.criteria._crit.items() if v is not None
}
# At this point, all documents are of 'essence' type
return document_store(document=self.output.name,
allow_duplicate_filename=True,
document_type='essence',
**metadata)
def _record_report(self, report_oid):
"""Record the details from this report generation in the db"""
if not report_oid:
return
report = Report(processed_datetime=datetime.now(),
file_path=report_oid,
report_method=self.criteria.report_method)
alchemy = AlchemyAccess(database=self.database)
alchemy.session.add(report)
alchemy.session.commit()
alchemy.disconnect()
def _transmit_report(self, report):
"""Transmit report using self._transport()"""
logging.info("initiate upload of %s", report)
self._transport.transfer_file(report)
def _transmit_differences(self, report):
"""Compute differences from yesterday's like report; transport"""
# This option really only makes sense on date range reports,
# as updates hit older data than just 'yesterday'.
if self.criteria.start_date == self.criteria.end_date:
raise ValueError("difference calculation not supported on "\
"single day reports")
# See if we can find a similar report in the archive from
# yesterday
search_criteria = {
'report_method': self.criteria.report_method,
'start_date': self.criteria.start_date - timedelta(days=1),
'end_date': self.criteria.end_date - timedelta(days=1)
}
old_doc = document_find(search_criteria, limit=1)
if old_doc is None:
logging.info("No comparable report found for difference "\
"generation")
self._transmit_report(report)
else:
target_filename = self.\
_generate_output_filename(start_date=self.criteria.start_date,
end_date=self.criteria.end_date)
# RemoveDuplicates not yet ported!!
raise ValueError("RemoveDuplicates not ported")
#from pheme.essence.remove_duplicates import RemoveDuplicates
#rd = RemoveDuplicates(new_report=report,
# old_report=old_doc,
# out=target_filename)
#rd.generate_report()
#logging.info("initiate upload of difference %s", target_filename)
#self._transport.transfer_file(target_filename)
def _getConn(self):
""" Local wrapper to get database connection
"""
if hasattr(self, 'access'):
return
self.access = DirectAccess(database=self.database,
user=self.user,
password=self.password)
def _closeConn(self):
""" Local wrapper to close database connection
"""
if hasattr(self, 'access'):
self.access.close()
def tearDown(self):
"Public interface to clean up internals"
self._closeConn()
def execute(self,
save_report=False,
transmit_report=False,
transmit_differences=False):
"""Execute the report generation
"""
logging.info("Initiate ESSENCE report generation [%s-%s] for %s",
self.criteria.start_date, self.criteria.end_date,
self.criteria.report_method)
self._getConn()
report_oid = self._write_report(save_report)
self._record_report(report_oid)
if transmit_report:
self._transmit_report(report_oid)
if transmit_differences:
self._transmit_differences(report_oid)
self._closeConn()
if hasattr(self, 'datePersistence'):
self.datePersistence.bump_date()
logging.info("Completed ESSENCE report generation [%s-%s] for %s",
self.criteria.start_date, self.criteria.end_date,
self.criteria.report_method)
def process_args(self):
"""Process any optional arguments and possitional parameters
Using the values provided, assemble ReportCriteria and
Datefile instances to control report generation.
"""
parser = OptionParser(usage=usage)
# Provide the ReportCriteria instance an error callback so any
# command line errors provoke the standard graceful exit with
# warning text.
self.criteria.error_callback = parser.error
parser.add_option("-u",
"--user",
dest="user",
default=self.user,
help="database user")
parser.add_option("-p", "--password", dest="password",
default=self.password,
help="database password, or file containing "\
"just the password")
parser.add_option("-c", "--countdown", dest="countdown",
default=None,
help="count {down,up} the start and end dates "\
"set to 'forwards' or 'backwards' "\
"if desired")
parser.add_option("-i", "--include-updates",
action='store_true', dest="includeUpdates",
default=False, help="include "\
"visits updated since last similar report")
parser.add_option("--include-vitals",
action='store_true', dest="includeVitals",
default=False, help="include "\
"vitals (measured temperature, O2 "\
"saturation, influenza and H1N1 vaccine "\
"data) as additional columns in the "\
"report")
parser.add_option("-k", "--patient-class",
dest="patient_class",
default=None, help="use "\
"to filter report on a specific patient "\
"class [E,I,O]")
parser.add_option("-r", "--region", dest="region",
default=None,
help="reportable region defining limited set "\
"of facilities to include, by default "\
"all facilities are included")
parser.add_option("-s", "--save-and-upload",
action='store_true', dest="save_upload",
default=False, help="save file and upload to "\
"DOH")
parser.add_option("-x",
"--save-without-upload",
action='store_true',
dest="save_only",
default=False,
help="save file but don't upload")
parser.add_option("-d", "--upload-diff",
action='store_true', dest="upload_diff",
default=False, help="upload differences only "\
"(from yesterdays like report) to DOH")
parser.add_option("-t", "--thirty-days",
action='store_true', dest="thirty_days",
default=False, help="include 30 days up to "\
"requested date ")
parser.add_option("-v",
"--verbose",
dest="verbosity",
action="count",
default=self.verbosity,
help="increase output verbosity")
(options, args) = parser.parse_args()
if len(args) != 2:
parser.error("incorrect number of arguments")
# Database to query
self.criteria.database = args[0]
self.user = options.user
self.password = options.password
self.criteria.credentials(user=self.user, password=self.password)
# Potential region restriction
self.criteria.reportable_region = options.region
# Potential patient class restriction
self.criteria.patient_class = options.patient_class
# Potential to include vitals (not tied to gipse format)
self.criteria.include_vitals = options.includeVitals
# Potential inclusion of updates
self.criteria.include_updates = options.includeUpdates
# Report date(s) and potential step direction.
# NB - several options affect report_method and must be set
# first!
initial_date = parseDate(args[1])
config = Config()
ps_file = os.path.join(
config.get('general', 'tmp_dir', default='/tmp'),
self.criteria.report_method)
step = options.thirty_days and 30 or None
direction = options.countdown
self.datefile = Datefile(initial_date=initial_date,
persistence_file=ps_file,
direction=direction,
step=step)
self.criteria.start_date, self.criteria.end_date =\
self.datefile.get_date_range()
# What to do once report is completed. Complicated, protect
# user from themselves!
self.save_report = options.save_upload or \
options.save_only or options.upload_diff
self.transmit_report = options.save_upload
self.transmit_differences = options.upload_diff
if options.save_only and options.save_upload:
parser.error("save-without-upload and save-and-upload "\
"are mutually exclusive")
if options.save_only and options.upload_diff:
parser.error("save-without-upload and upload-diff "\
"are mutually exclusive")
if options.upload_diff and options.save_upload:
parser.error("upload-diff and save-and-upload"\
"are mutually exclusive")
# Can't transmit w/o saving
if options.save_upload or options.upload_diff:
assert (self.save_report)
# Sanity check
if options.save_only:
assert (self.save_report and not self.transmit_report
and not self.transmit_differences)
# How verbosely to log
self.verbosity = options.verbosity
def create_tables(user=None, password=None, database=None,
enable_delete=False):
"""Create the longitudinal database tables.
NB - the config [longitudinal]database_user is granted SELECT
INSERT and UPDATE permissions (plus DELETE if enable_delete is set).
:param user: database user with table creation grants
:param password: the database password
:param database: the database name to populate
:param enable_delete: testing hook, override for testing needs
"""
engine = create_engine("postgresql://%s:%s@localhost/%s" %
(user, password, database))
# Can't find a way to code the cascade to a view
# drop manually as the createTables will otherwise fail.
try:
engine.execute(text("DROP VIEW IF EXISTS essence"))
except OperationalError: # pragma: no cover
# Assume it's a new database. Otherwise subsequent failures
# will catch any real trouble.
pass
metadata.drop_all(bind=engine)
metadata.create_all(bind=engine)
def bless_user(user):
engine.execute("""BEGIN; GRANT SELECT, INSERT, UPDATE %(delete)s ON
assoc_visit_dx,
assoc_visit_lab,
dim_admission_o2sat,
dim_admission_source,
dim_admission_temp,
dim_assigned_location,
dim_cc,
dim_disposition,
dim_dx,
dim_facility,
dim_flu_vaccine,
dim_h1n1_vaccine,
dim_lab_flag,
dim_lab_result,
dim_location,
dim_note,
dim_order_number,
dim_performing_lab,
dim_pregnancy,
dim_race,
dim_ref_range,
dim_service_area,
dim_specimen_source,
fact_visit,
internal_export_delta,
internal_message_processed,
internal_report,
internal_reportable_region
TO %(user)s; COMMIT;""" %
{'delete': ", DELETE" if enable_delete else '',
'user': user})
# Sequences also require UPDATE
engine.execute("""BEGIN; GRANT SELECT, UPDATE ON
dim_admission_o2sat_pk_seq,
dim_admission_temp_pk_seq,
dim_assigned_location_pk_seq,
dim_cc_pk_seq,
dim_dx_pk_seq,
dim_flu_vaccine_pk_seq,
dim_h1n1_vaccine_pk_seq,
dim_lab_flag_pk_seq,
dim_lab_result_pk_seq,
dim_location_pk_seq,
dim_note_pk_seq,
dim_order_number_pk_seq,
dim_performing_lab_pk_seq,
dim_pregnancy_pk_seq,
dim_race_pk_seq,
dim_ref_range_pk_seq,
dim_service_area_pk_seq,
dim_specimen_source_pk_seq,
fact_visit_pk_seq,
internal_export_delta_pk_seq,
internal_report_pk_seq
TO %(user)s; COMMIT;""" % {'user': user})
# Provide configured user necessary privileges
bless_user(Config().get('longitudinal', 'database_user'))
# Add any views
create_essence_view(engine)
def __init__(self):
self.config = Config()
self.mirth_home = self.config.get('mirth', 'mirth_home')
self.mirth_system_user = self.config.get('mirth', 'mirth_system_user')
def transform_channels():
"""Apply default transform to PHEME channels"""
doc = """
Mirth channels can be easily exported in XML format. This utility
provides a mechanims to alter an export for subsequent import.
Useful for altering details such as database name and user
authentication.
NB - values defined in the project configuration file will be used
unless provided as optional arguments. See
`pheme.util.config.Config`
"""
config = Config()
ap = argparse.ArgumentParser(description=doc)
ap.add_argument("-d",
"--database",
dest="db",
default=config.get('warehouse', 'database'),
help="name of database (overrides "
"[warehouse]database)")
ap.add_argument("-u",
"--user",
dest="user",
default=config.get('warehouse', 'database_user'),
help="database user (overrides "
"[warehouse]database_user)")
ap.add_argument("-p",
"--password",
dest="password",
default=config.get('warehouse', 'database_password'),
help="database password (overrides [warehouse]"
"database_password)")
ap.add_argument("--input_dir",
dest="input_dir",
default=config.get('warehouse', 'input_dir'),
help="filesystem directory for channel to poll "
"(overrides [warehouse]input_dir)")
ap.add_argument("--output_dir",
dest="output_dir",
default=config.get('warehouse', 'output_dir'),
help="filesystem directory for channel output "
"(overrides [warehouse]output_dir)")
ap.add_argument("--error_dir",
dest="error_dir",
default=config.get('warehouse', 'error_dir'),
help="filesystem directory for channel errors "
"(overrides [warehouse]error_dir)")
ap.add_argument("source_directory",
help="directory containing source channel "
"definition files")
ap.add_argument("target_directory",
help="directory to write transformed channel "
"definition files")
args = ap.parse_args()
source_dir = os.path.realpath(args.source_directory)
target_dir = os.path.realpath(args.target_directory)
transformer = TransformManager(src=None,
target_dir=target_dir,
options=args)
for c in CHANNELS:
transformer.src = os.path.join(source_dir, '%s.xml' % c)
transformer()
# no transformation on codetemplates at this time - but the
# importer expects the codetemplates.xml file to be in the same
# directory, so copy it over.
shutil.copy(os.path.join(source_dir, 'codetemplates.xml'), target_dir)
