def __init__(self):
config = Config()
self.db = config.get('phinms', 'database')
self.username = config.get('phinms', 'user')
self.passwd = config.get('phinms', 'password')
self.workerqueue = config.get('phinms', 'workerqueue')
self.feedertable = self.workerqueue + '_feeder'
def transfer_file(self, compress_with=None):
"""Initiate transfer via HTTPS
:param compress_with: if document isn't already compressed and
this is set, compress the file before transfering.
"""
filename = self.document['filename']
config = Config()
upload_url = config.get('distribute', 'upload_url')
user = config.get('distribute', 'username')
pw = config.get('distribute', 'password')
payload = {'siteShortName': self.document['reportable_region']}
content = self.extract_content(compress_with)
files = {'userfile': content}
if inProduction():
logging.info("POST %s to %s" % (filename, upload_url))
r = requests.post(upload_url, auth=(user, pw), files=files,
data=payload)
# We only get redirected if successful!
if r.status_code != 302:
logging.error("failed distrbute POST")
#Can't rely on status_code, as distribute returns 200
#with problems in the body.
logging.error("".join(list(r.iter_content())))
r.raise_for_status()
else:
self.record_transfer()
else:
logging.warn("inProduction() check failed, not POSTing "
"file '%s' to '%s'", filename, upload_url)
def setUp(self):
c = Config()
cfg_value = lambda v: c.get('warehouse', v)
self.alchemy = AlchemyAccess(database=cfg_value('database'),
host='localhost',
user=cfg_value('database_user'),
password=cfg_value('database_password'))
self.session = self.alchemy.session
def setup_module():
"""Create a fresh db (once) for all tests in this module"""
configure_logging(verbosity=2, logfile='unittest.log')
c = Config()
if c.get('general', 'in_production'): # pragma: no cover
raise RuntimeError("DO NOT run destructive test on production system")
create_tables(enable_delete=True, **db_params(CONFIG_SECTION))
def setUp(self):
c = Config()
cfg_value = lambda v: c.get("longitudinal", v)
self.alchemy = AlchemyAccess(
database=cfg_value("database"),
host="localhost",
user=cfg_value("database_user"),
password=cfg_value("database_password"),
)
self.session = self.alchemy.session
self.remove_after_test = []
def inProduction():
"""Simple state check to avoid uploading files to thrid party
servers and what not when not 'in production'.
"""
config = Config()
try:
return config.get('general', 'in_production')
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): # pragma: no cover
raise ValueError("Config file doesn't specify "
"`[general]in_production` "
"unsafe to guess, can't continue")
def setup_module():
"""Create a fresh db (once) for all tests in this module"""
c = Config()
if c.get('general', 'in_production'): # pragma: no cover
raise RuntimeError("DO NOT run destructive test on production system")
cfg_value = lambda v: c.get('warehouse', v)
create_tables(cfg_value('create_table_user'),
cfg_value('create_table_password'),
cfg_value('database'),
enable_delete=True)
def setup_module():
"""Create a fresh db (once) for all tests in this module"""
configure_logging(verbosity=2, logfile='unittest.log')
c = Config()
if c.get('general', 'in_production'): # pragma: no cover
raise RuntimeError("DO NOT run destructive test on production system")
create_tables(enable_delete=True, **db_params(CONFIG_SECTION))
# Load in all the static data in anonymized form
static_data_file = open(os.path.join(os.path.dirname(
os.path.abspath(__file__)), 'anon_static_db_data.yaml'), 'r')
load_file(static_data_file)
def test_override(self):
"same key in second file should override"
section = 'SECTION'
key = 'unittest'
for i in range(2):
cp = ConfigParser.RawConfigParser()
cp.add_section(section)
cp.set(section, key, i)
with open(self.config_files[i], 'w') as f:
cp.write(f)
c = Config(self.config_files)
self.assertEquals(1, c.get(section, key))
def test_tilde(self):
"support tilde in directory paths"
section = 'SECTION'
key = 'unittest'
value = "~/tempfile"
cp = ConfigParser.RawConfigParser()
cp.add_section(section)
cp.set(section, key, value)
with open(self.config_files[0], 'w') as f:
cp.write(f)
c = Config(self.config_files)
self.assertEquals(os.path.expanduser("~/tempfile"),
c.get(section, key))
def test_falseness(self):
"false values should be case insensitive"
section = 'SECTION'
key = 'unittest'
values = ['FALSE', 'false', 'False', ' f ']
cp = ConfigParser.RawConfigParser()
cp.add_section(section)
for i in range(len(values)):
cp.set(section, key+str(i), values[i])
with open(self.config_files[0], 'w') as f:
cp.write(f)
c = Config(self.config_files)
for i in range(len(values)):
self.assertEquals(False, c.get(section, key+str(i)))
def db_params(section):
"""Return dict of database connection values from named config section
Returned dict includes:
- user
- password
- database (name of database)
"""
config = Config()
database = config.get(section, 'database')
user = config.get(section, 'database_user')
password = config.get(section, 'database_password')
return {'user': user, 'password': password, 'database': database}
def main(): # pragma: no cover
"""Entry point to (re)create the table using config settings"""
config = Config()
database = config.get('longitudinal', 'database')
print "destroy and recreate database %s ? "\
"('destroy' to continue): " % database,
answer = sys.stdin.readline().rstrip()
if answer != 'destroy':
print "aborting..."
sys.exit(1)
user = config.get('longitudinal', 'database_user')
password = config.get('longitudinal', 'database_password')
create_tables(user, password, database)
def test_truthiness(self):
"truth value should be case insensitive"
section = 'SECTION'
key = 'unittest'
values = ['TRUE', 'true', 'True', ' t ']
cp = ConfigParser.RawConfigParser()
cp.add_section(section)
for i in range(len(values)):
cp.set(section, key+str(i), values[i])
with open(self.config_files[0], 'w') as f:
cp.write(f)
c = Config(self.config_files)
for i in range(len(values)):
self.assertEquals(True, c.get(section, key+str(i)))
def test_float(self):
"Looks like a float, should be one"
section = 'SECTION'
key = 'unittest'
values = ['0.01', '42.0', '-67.3']
cp = ConfigParser.RawConfigParser()
cp.add_section(section)
for i in range(len(values)):
cp.set(section, key+str(i), values[i])
with open(self.config_files[0], 'w') as f:
cp.write(f)
c = Config(self.config_files)
for i in range(len(values)):
self.assertEquals(float(values[i]),
c.get(section, key+str(i)))
def main(): # pragma: no cover
"""Entry point to (re)create the table using config settings"""
config = Config()
dbname = config.get('warehouse', 'database')
print "destroy and recreate database %s ? "\
"('destroy' to continue): " % dbname,
answer = sys.stdin.readline().rstrip()
if answer != 'destroy':
print "aborting..."
sys.exit(1)
user = config.get('warehouse', 'create_table_user')
print "password for PostgreSQL user:", user
password = getpass.getpass()
create_tables(user, password, dbname)
def test_int(self):
"Looks like an int, should be one (including 0,1)"
section = 'SECTION'
key = 'unittest'
values = ['0', '1', '-67', 42, -1]
cp = ConfigParser.RawConfigParser()
cp.add_section(section)
for i in range(len(values)):
cp.set(section, key+str(i), values[i])
with open(self.config_files[0], 'w') as f:
cp.write(f)
c = Config(self.config_files)
for i in range(len(values)):
self.assertEquals(int(values[i]),
c.get(section, key+str(i)))
def testPhinmsTransfer(self):
# need a document in the db
self.create_test_file(compression='gzip',
report_type='longitudinal')
# fake a transfer of this object
context = PHINMS_Transfer(testing.DummyRequest())
context.request.fs = self.fs
context.request.document_store = self.document_store
context = context[str(self.oid)]
self.assertFalse(inProduction()) # avoid accidental transfers!
context.transfer_file()
self.assertEqual(self.report_type, 'longitudinal')
config = Config()
path = config.get('phinms', self.report_type)
self.assertEqual(context.outbound_dir, path)
def _generate_output_filename(self, start_date=None,
end_date=None):
start_date = self.criteria.start_date if start_date is None\
else start_date
end_date = self.criteria.end_date if end_date is None else end_date
datestr = end_date.strftime('%Y%m%d')
if start_date != end_date:
datestr = '-'.join((start_date.strftime('%Y%m%d'),
end_date.strftime('%Y%m%d')))
filename = self.criteria.report_method + '-' + datestr + '.txt'
config = Config()
tmp_dir = config.get('general', 'tmp_dir', default='/tmp')
filepath = os.path.join(tmp_dir, filename)
return filepath
def __init__(self):
self.config = Config()
# obtain list of files to process
path = os.path.abspath(
os.path.join(os.path.dirname(__file__),
"../../../test_hl7_batchfiles"))
self.filenames = [os.path.join(path, file) for file in
os.listdir(path)]
def transform_channels():
"""Apply default transform to PHEME channels"""
doc = """
Mirth channels can be easily exported in XML format. This utility
provides a mechanims to alter an export for subsequent import.
Useful for altering details such as database name and user
authentication.
NB - values defined in the project configuration file will be used
unless provided as optional arguments. See
`pheme.util.config.Config`
"""
config = Config()
ap = argparse.ArgumentParser(description=doc)
ap.add_argument("-d", "--database", dest="db",
default=config.get('warehouse', 'database'),
help="name of database (overrides "
"[warehouse]database)")
ap.add_argument("-u", "--user", dest="user",
default=config.get('warehouse', 'database_user'),
help="database user (overrides "
"[warehouse]database_user)")
ap.add_argument("-p", "--password", dest="password",
default=config.get('warehouse', 'database_password'),
help="database password (overrides [warehouse]"
"database_password)")
ap.add_argument("--input_dir", dest="input_dir",
default=config.get('warehouse', 'input_dir'),
help="filesystem directory for channel to poll "
"(overrides [warehouse]input_dir)")
ap.add_argument("--output_dir", dest="output_dir",
default=config.get('warehouse', 'output_dir'),
help="filesystem directory for channel output "
"(overrides [warehouse]output_dir)")
ap.add_argument("--error_dir", dest="error_dir",
default=config.get('warehouse', 'error_dir'),
help="filesystem directory for channel errors "
"(overrides [warehouse]error_dir)")
ap.add_argument("source_directory",
help="directory containing source channel "
"definition files")
ap.add_argument("target_directory",
help="directory to write transformed channel "
"definition files")
args = ap.parse_args()
source_dir = os.path.realpath(args.source_directory)
target_dir = os.path.realpath(args.target_directory)
transformer = TransformManager(src=None,
target_dir=target_dir,
options=args)
for c in CHANNELS:
transformer.src = os.path.join(source_dir, '%s.xml' % c)
transformer()
# no transformation on codetemplates at this time - but the
# importer expects the codetemplates.xml file to be in the same
# directory, so copy it over.
shutil.copy(os.path.join(source_dir, 'codetemplates.xml'), target_dir)
def setup_module():
"""Populate database with test data for module tests"""
c = Config()
if c.get('general', 'in_production'): # pragma: no cover
raise RuntimeError("DO NOT run destructive test on production system")
"Pull in the filesystem dump from a previous mirth run"
mi = MirthInteraction()
mi.restore_database()
"Run a quick sanity check, whole module requires a populated db"
connection = db_connection('warehouse')
count = connection.session.query(HL7_Msh).count()
connection.disconnect()
if count < 4000:
err = "Minimal expected count of records not present. "\
"Be sure to run 'process_testfiles_via_mirth' as a prerequisite"
raise RuntimeError(err)
def __init__(self, verbosity=0, source_db=None):
self.verbosity = verbosity
self.source_db = source_db
config = Config()
self.phinms_receiving_dir = config.get('phinms', 'receiving_dir')
self.phinms_archive_dir = config.get('phinms', 'archive_dir')
self.source_dir = self.phinms_receiving_dir
# Confirm the required directories are present
if not os.path.isdir(self.phinms_receiving_dir):
raise ValueError("Can't find required directory %s" %
self.phinms_receiving_dir)
UPLOAD_PORT = config.get('pheme_http_receiver', 'port')
UPLOAD_HOST = config.get('pheme_http_receiver', 'host')
self.http_pool = HTTPConnectionPool(host=UPLOAD_HOST,
port=UPLOAD_PORT,
timeout=20)
self._copy_tempdir = None
def transfer_document(document_id, transfer_agent,
compress_with=None):
"""Web API client call to request transfer of given document ID
:param document_id: the document ID to transfer, likely returned
from a document_store() call on the same Web API
:param transfer_agent: such as 'phin-ms' or 'distribute'.
:param compress_with: if additional compression is desired, this
may be set to 'zip' or 'gzip', to be performed before sending.
"""
query_params = dict()
if compress_with is not None:
query_params['compress_with'] = compress_with
config = Config()
parts = dict()
parts['doc'] = document_id
parts['host'] = config.get("WebAPI", "host")
parts['port'] = config.get("WebAPI", "port")
parts['agent'] = transfer_agent
url = 'http://%(host)s:%(port)s/%(agent)s/%(doc)s' % parts
if query_params:
url = url + '?' +\
'&'.join([k+'='+v for k, v in query_params.items()])
# Initiate request, wait on response
r = requests.post(url)
if r.status_code != 200: # pragma no cover
failure = "Failed POST (%d) for transfer request: %s" %\
(r.status_code, url)
logging.error(failure)
logging.error(r.text)
raise RuntimeError(failure)
def setup_module():
"""Create a fresh db (once) for all tests in this module"""
configure_logging(verbosity=2, logfile='unittest.log')
c = Config()
if c.get('general', 'in_production'): # pragma: no cover
raise RuntimeError("DO NOT run destructive test on production system")
create_tables(enable_delete=True, **db_params(CONFIG_SECTION))
# create a "test_region" and a couple bogus facilities
f1 = Facility(county='KING', npi=10987, zip='12345',
organization_name='Reason Medical Center',
local_code='RMC')
f2 = Facility(county='POND', npi=65432, zip='67890',
organization_name='No-Reason Medical Center',
local_code='NMC')
conn = db_connection(CONFIG_SECTION)
conn.session.add(f1)
conn.session.add(f2)
conn.session.commit()
rr1 = ReportableRegion(region_name='test_region',
dim_facility_pk=10987)
conn.session.add(rr1)
conn.session.commit()
conn.disconnect()
def url_builder(predicate=None, resource=None, view=None, query_params={}):
"""Build webAPI url from config and passed values
:param predicate: desired action or type of document
:param resource: filename or object identifier
:param view: specilized view, such as metadata
:param query_params: dictionary of key, values to append
returns URL ready for request, post, etc.
"""
config = Config()
url = 'http://%s:%s' % (config.get("WebAPI", "host"),
config.get("WebAPI", "port"))
if predicate:
url = '/'.join((url, predicate))
if resource:
url = '/'.join((url, resource))
if view:
url = '/'.join((url, '@@' + view))
if query_params:
url = '?'.join((url,
'&'.join([k+'='+v for k, v in query_params.items()])))
return url
class MirthInteraction(object):
"""Abstraction to interact with Mirth Connect for testing"""
WAIT_INTERVAL = 15
TIMEOUT = 300
def __init__(self):
self.config = Config()
# obtain list of files to process
path = os.path.abspath(
os.path.join(os.path.dirname(__file__),
"../../../test_hl7_batchfiles"))
self.filenames = [os.path.join(path, file) for file in
os.listdir(path)]
def prepare_filesystem(self):
# create clean database (includes non produciton sanity check)
setup_module()
# wipe previous run(s) files
mirth_user = self.config.get('mirth', 'mirth_system_user')
for dir in ('input_dir', 'output_dir', 'error_dir'):
wipe_dir_contents(self.config.get('warehouse', dir),
mirth_user)
def process_batchfiles(self):
"""Feed the testfiles to mirth - block till done"""
self.prepare_filesystem()
require_mirth()
for batchfile in self.filenames:
copy_file_to_dir(batchfile,
self.config.get('warehouse', 'input_dir'),
self.config.get('mirth', 'mirth_system_user'))
# wait for all files to appear in error our output dirs
# providing occasional output and raising if we appear hung
last_count = 0
last_time = time.time()
output_dir = self.config.get('warehouse', 'output_dir')
error_dir = self.config.get('warehouse', 'error_dir')
while last_count < len(self.filenames):
time.sleep(self.WAIT_INTERVAL)
count = len(os.listdir(output_dir)) + len(os.listdir(error_dir))
if count > last_count:
last_count = count
last_time = time.time()
if time.time() - self.TIMEOUT > last_time:
raise RuntimeError("TIMEOUT exceeded waiting on Mirth")
print "Waiting on mirth to process files...(%d of %d)" %\
(last_count, len(self.filenames))
def persist_database(self):
"""Write the database contents to disk"""
fsp = FilesystemPersistence(\
database=self.config.get('warehouse', 'database'),
user=self.config.get('warehouse', 'database_user'),
password=self.config.get('warehouse', 'database_password'))
fsp.persist()
def restore_database(self):
"""Pull previously persisted data into database"""
fsp = FilesystemPersistence(\
database=self.config.get('warehouse', 'database'),
user=self.config.get('warehouse', 'database_user'),
password=self.config.get('warehouse', 'database_password'))
fsp.restore()
def _set_report_type(self, report_type, patient_class=None):
if report_type == 'essence' and patient_class:
report_type += '_pc' + patient_class
config = Config()
self._outbound_dir = config.get('phinms', report_type)
def test_default(self):
"Asking for missing value with a default"
c = Config(self.config_files)
self.assertEquals(42, c.get('Lifes', 'Answer', 42))
def process_args(self):
"""Process any optional arguments and possitional parameters
Using the values provided, assemble ReportCriteria and
Datefile instances to control report generation.
"""
parser = OptionParser(usage=usage)
# Provide the ReportCriteria instance an error callback so any
# command line errors provoke the standard graceful exit with
# warning text.
self.criteria.error_callback = parser.error
parser.add_option("-u", "--user", dest="user",
default=self.user, help="database user")
parser.add_option("-p", "--password", dest="password",
default=self.password,
help="database password, or file containing "\
"just the password")
parser.add_option("-c", "--countdown", dest="countdown",
default=None,
help="count {down,up} the start and end dates "\
"set to 'forwards' or 'backwards' "\
"if desired")
parser.add_option("-i", "--include-updates",
action='store_true', dest="includeUpdates",
default=False, help="include "\
"visits updated since last similar report")
parser.add_option("--include-vitals",
action='store_true', dest="includeVitals",
default=False, help="include "\
"vitals (measured temperature, O2 "\
"saturation, influenza and H1N1 vaccine "\
"data) as additional columns in the "\
"report")
parser.add_option("-k", "--patient-class",
dest="patient_class",
default=None, help="use "\
"to filter report on a specific patient "\
"class [E,I,O]")
parser.add_option("-r", "--region", dest="region",
default=None,
help="reportable region defining limited set "\
"of facilities to include, by default "\
"all facilities are included")
parser.add_option("-s", "--save-and-upload",
action='store_true', dest="save_upload",
default=False, help="save file and upload to "\
"DOH")
parser.add_option("-x", "--save-without-upload",
action='store_true', dest="save_only",
default=False, help="save file but don't upload")
parser.add_option("-d", "--upload-diff",
action='store_true', dest="upload_diff",
default=False, help="upload differences only "\
"(from yesterdays like report) to DOH")
parser.add_option("-t", "--thirty-days",
action='store_true', dest="thirty_days",
default=False, help="include 30 days up to "\
"requested date ")
parser.add_option("-v", "--verbose", dest="verbosity",
action="count", default=self.verbosity,
help="increase output verbosity")
(options, args) = parser.parse_args()
if len(args) != 2:
parser.error("incorrect number of arguments")
# Database to query
self.criteria.database = args[0]
self.user = options.user
self.password = options.password
self.criteria.credentials(user=self.user,
password=self.password)
# Potential region restriction
self.criteria.reportable_region = options.region
# Potential patient class restriction
self.criteria.patient_class = options.patient_class
# Potential to include vitals (not tied to gipse format)
self.criteria.include_vitals = options.includeVitals
# Potential inclusion of updates
self.criteria.include_updates = options.includeUpdates
# Report date(s) and potential step direction.
# NB - several options affect report_method and must be set
# first!
initial_date = parseDate(args[1])
config = Config()
ps_file = os.path.join(config.get('general', 'tmp_dir',
default='/tmp'),
self.criteria.report_method)
step = options.thirty_days and 30 or None
direction = options.countdown
self.datefile = Datefile(initial_date=initial_date,
persistence_file=ps_file,
direction=direction,
step=step)
self.criteria.start_date, self.criteria.end_date =\
self.datefile.get_date_range()
# What to do once report is completed. Complicated, protect
# user from themselves!
self.save_report = options.save_upload or \
options.save_only or options.upload_diff
self.transmit_report = options.save_upload
self.transmit_differences = options.upload_diff
if options.save_only and options.save_upload:
parser.error("save-without-upload and save-and-upload "\
"are mutually exclusive")
if options.save_only and options.upload_diff:
parser.error("save-without-upload and upload-diff "\
"are mutually exclusive")
if options.upload_diff and options.save_upload:
parser.error("upload-diff and save-and-upload"\
"are mutually exclusive")
# Can't transmit w/o saving
if options.save_upload or options.upload_diff:
assert(self.save_report)
# Sanity check
if options.save_only:
assert(self.save_report and not self.transmit_report and
not self.transmit_differences)
# How verbosely to log
self.verbosity = options.verbosity
