def test_get_sets_value(self, _load_config):
"""Test that if global config value is not set initially, it's set after get is called."""
test_config = {'key': 'value'}
config._CONFIG = None
_load_config.return_value = test_config
config.get('key')
assert config._CONFIG == test_config
def get_access_layer_batch_query(tins, npis, start_date, end_date):
"""Populate the Teradata SQL statement to query the IDR for provider information in batches.
Args:
npis ([str]): National provider identifiers to load.
tins ([str]): Provider tax identification numbers to load.
start_date (datetime): Start date of data to load.
end_date (datetime): End date of data to load.
Returns:
SQL query to retrieve data from the IDR.
"""
if len(tins) != len(npis):
raise InputError('The TINs and NPIs list must be of the same size.')
npi_tins = ['{}{}'.format(npi, tin) for npi, tin in zip(npis, tins)]
return ACCESS_LAYER_BASE_QUERY_BATCH.format(
npis=sql_formatting.to_sql_list(npis),
tins=sql_formatting.to_sql_list(tins),
npi_tins=sql_formatting.to_sql_list(npi_tins),
start_date=datetime.strftime(start_date, '%Y-%m-%d'),
end_date=datetime.strftime(end_date, '%Y-%m-%d'),
as_was_date=datetime.strftime(
config.get('calculation.as_was_date'), '%Y-%m-%d'
),
access_layer_name=config.get('teradata.access_layer_name'),
medicare_vdm_name=config.get('teradata.medicare_vdm_name')
)
def _initialize(newrelic_agent=newrelic.agent):
"""Initialize the New Relic Application."""
if config.get('environment') not in ['DEV', 'IMPL', 'PRD']:
return None
newrelic_agent.initialize('newrelic.ini',
environment=config.get('environment'))
return newrelic_agent.register_application(timeout=10.0)
def _get_mssa_date_ranges(self, claims):
"""
Get mssa_date ranges by querying the IDR.
Returns a dict of {bene_sk: [date_ranges]} that will need to be merged
to keep only non-overlapping intervals.
"""
bene_sks = {claim.bene_sk for claim in claims}
start_date = config.get('calculation.start_date')
end_date = config.get('calculation.end_date')
mssa_query = idr_queries.get_mssa_query(
bene_sks=bene_sks,
encounter_codes=self.procedure_codes,
start_date=start_date,
end_date=end_date)
rows = execute.execute(mssa_query)
if not rows:
logger.error('No MSSA date found despite provider '
'having submitted quality codes for Measure 407.')
return {}
mssa_date_ranges = collections.defaultdict(list)
for row in rows:
mssa_date_ranges[row['bene_sk']].append(
DateRange(row['min_date'], row['max_date']))
return mssa_date_ranges
def get_measure_calculators(measures,
year=config.get('calculation.measures_year')):
"""Generate a dictionary of measure calculator object with correct definition and type."""
json_path = config.get('assets.qpp_single_source_json')[year]
single_source_json = measure_reader.load_single_source(json_path)
return {
measure: get_measure_calculator(
measure_number=measure,
year=year,
single_source_json=single_source_json,
)
for measure in measures
}
def load_single_source(json_path=config.get('assets.qpp_single_source_json')[
config.get('calculation.measures_year')]):
"""
Load the single source file as JSON keyed by measure number.
Only load claims-based measures by using the 'performanceOptions' key.
"""
with open(json_path, encoding='utf-8') as file:
single_source_as_list = json.load(file)
return {
measure_json['measureId']: measure_json
for measure_json in single_source_as_list
if 'performanceOptions' in measure_json
}
def get_headers():
"""Return base headers for Nava's APIs."""
headers = {
'Content-Type':
'application/json',
'Accept':
'application/json',
'Authorization':
'Bearer {api_token}'.format(
api_token=config.get('submission.api_token'))
}
if config.get('submission.cookie'):
headers['Cookie'] = config.get('submission.cookie')
return headers
def get_submissions(npi=None, tin=None, performance_year=None, start_index=0):
"""
Simple GET request to check if submissions have been made.
If NPI is provided, return submissions for this NPI.
If start_index is provided, return submissions after the start_index.
Else, simply return the first 10 submissions starting at start_index=0.
FIXME: Move this into a different file, this is not part of the api submitter.
"""
logger.debug('Making a simple GET request to verify submissions.')
endpoint_url = urllib.parse.urljoin(config.get('submission.endpoint'),
'submissions')
params = {
'startIndex': start_index,
}
if npi:
params['nationalProviderIdentifier'] = npi
headers = get_headers()
if tin:
headers.update({'qpp-taxpayer-identification-number': tin})
if performance_year:
params['performanceYear'] = str(performance_year)
response = requests.get(endpoint_url, params=params, headers=headers)
# If the request failed, raise an error.
_handle_http_error(response, 'get_submissions')
return response.json()
def __init__(self, tin, npi, performance_start, performance_end):
"""Create a MeasurementSet object."""
if config.get('submission.obscure_providers'):
self.tin, self.npi = self._obscure_provider_identifers(tin, npi)
else:
self.tin = self._validate_tin(tin)
self.npi = self._validate_npi(npi)
self.performance_start = performance_start
self.performance_end = performance_end
# TODO: Read this template from a json file.
self.data = {
'submission': {
'programName': 'mips',
'entityType': 'individual',
'taxpayerIdentificationNumber': self.tin,
'nationalProviderIdentifier': self.npi,
'performanceYear': performance_end.year,
},
'category': 'quality',
'submissionMethod': 'claims',
'performanceStart': performance_start,
'performanceEnd': performance_end,
'measurements': [],
}
def test_teradata_connection(
connection_parameters=config.get('teradata.connection.parameters'),
continue_prompt=True):
"""Test Teradata connection and ask user for input if fails."""
try:
# Test access to the IDR instance.
_test_access_to_idr_instance(connection_parameters)
# Test access to the Teradata database and session creation.
_teradata_connection(connection_parameters)
logger.info('Teradata connection confirmed.')
return True
# TODO: avoid catch all.
except Exception as error:
error_message = error.message if hasattr(error,
'message') else repr(error)
if continue_prompt:
print(
'Would you like to continue without connection to IDR? (y/n) ',
'You have 10 seconds to reply.')
# select takes no keyword args. Args are rlist, wlist, xlist, timeout.
read, write, exception = select.select([sys.stdin], [], [], 10)
if read and sys.stdin.readline().strip() in ['y', 'yes']:
logger.warn('Continuing without Teradata connection. '
'Error - {}'.format(error_message))
return False
raise teradata_errors.TeradataError(error_message)
def post_to_new_relic(payload):
"""
Function to post event to New Relic.
Payload should be a string representation of a valid event.
"""
rest_key = config.get(
'new_relic_insights.event_key') # Key for NewRelic Insights POST API
url = config.get(
'new_relic_insights.url') # URL for NewRelic Insights POST API
post = (
'echo \'{payload}\' | curl -d @- -X POST -H "Content-Type: application/json" '
'-H "X-Insert-Key: {rest_key}" '
'{url}')
post = post.format(payload=payload, rest_key=rest_key, url=url)
subprocess.call(args=[post], shell=True)
def query_claims_from_teradata_batch_provider(provider_tins,
provider_npis,
start_date,
end_date,
session=None):
"""
Query claims table for the analyzer for a batch of providers.
Args:
provider_tin_list ([str]): List of tax identification numbers to query for.
provider_npi_list ([str]): List of national provider identification numbers to query for.
start_date (date): Start date of data to load.
end_date (date): End date of data to load.
session (session): Teradata session to use to access IDR.
Returns:
(column_names, rows) (list(str), list(tuple)): Tuple of list of headers, and
list of tuples containing claim line values.
"""
logger.debug('Query claims from TERADATA in env - {}.'.format(
config.get('environment')))
query = idr_queries.get_access_layer_batch_query(tins=provider_tins,
npis=provider_npis,
start_date=start_date,
end_date=end_date)
rows = execute.execute(query, session)
if rows:
columns = rows[0].columns
return (columns, rows)
return ([], rows)
def post_to_slack(message):
"""Function to post message to Slack."""
url = config.get('slack.url')
post = ('curl -X POST -H \'Content-type: application/json\' '
'--data \'{{"text":"{message}"}}\' '
'{url}')
post = post.format(url=url, message=message)
subprocess.call(args=[post], shell=True)
def test_get_uda_exec():
"""
Test _get_uda_exec.
Note: This test only works on an instance or a container
with properly installed Teradata uda_exec drivers.
"""
uda_exec = teradata_connector._get_uda_exec(
params=config.get('teradata.config'))
assert isinstance(uda_exec, teradata.udaexec.UdaExec)
def _post_to_measurement_sets_api(measurement_set):
logger.debug('Making POST request to the measurement-sets API.')
endpoint_url = urllib.parse.urljoin(config.get('submission.endpoint'),
'measurement-sets/')
return requests.post(url=endpoint_url,
data=measurement_set.to_json(),
headers=get_headers())
def add_measure_with_multiple_strata(self, measure_number,
measure_results):
"""
Add a measure score to the measurement set object for a measure with multiple strata.
measure_results should be a list of dictionaries, each with the keys
- name: stratum_name
- results: measure results dictionary with the following keys:
- performance_met
- performance_not_met
- eligible_population_exclusion
- eligible_population_exception
- eligible_population
- returns True if a measure was added to the measurement set.
"""
# If none of the strata contain eligible population, do not add the measure.
if not any([
stratum_dict['results']['eligible_population']
for stratum_dict in measure_results
]):
return False
# If none of the strata contains relevant data, do not add the measure.
if config.get('submission.filter_out_zero_reporting') and not any([
self.has_non_zero_reporting(measure_number,
stratum_dict['results'])
for stratum_dict in measure_results
]):
return False
measurement = {
'measureId': '{:03.0f}'.format(float(measure_number)),
'value': {
'isEndToEndReported': False,
'strata': [],
}
}
for stratum_dict in measure_results:
measurement['value']['strata'].append({
'stratum':
stratum_dict['name'],
'performanceMet':
stratum_dict['results']['performance_met'],
'eligiblePopulationExclusion':
stratum_dict['results']['eligible_population_exclusion'],
'eligiblePopulationException':
stratum_dict['results']['eligible_population_exception'],
'performanceNotMet':
stratum_dict['results']['performance_not_met'],
'eligiblePopulation':
stratum_dict['results']['eligible_population']
})
self.data['measurements'].append(measurement)
return True
def test_get_uda_exec_universal(UdaExec):
"""
Note: This will work across all devices but simply tests
that provided Teradata can create a uda_exec, it is forwarded
properly by _get_uda_exec.
"""
UdaExec.return_value = 'uda_exec'
uda_exec = teradata_connector._get_uda_exec(
params=config.get('teradata.config'))
assert uda_exec == 'uda_exec'
def get_logger(logger_name):
"""Configure and return default logger."""
logger = logging.getLogger(logger_name)
logger.setLevel(config.get('logging.log_level'))
# Set log handling to JSON.
handler = logging.StreamHandler()
handler.setFormatter(
formatter.JsonFormatter(
extra={
'hostname': socket.gethostname(),
'app': 'qpp-claims-to-quality',
'environment': config.get('environment'),
'team': config.get('logging.team'),
'contact': config.get('logging.contact')
}))
logger.addHandler(handler)
return logger
def get_queue(queue_name=None):
"""Create and return an SQS Queue."""
if not queue_name:
queue_name = config.get('aws.sqs.queue_name')
logger.info('Getting SQS queue - {queue}.'.format(queue=queue_name))
access_key_id = config.get('aws.sqs.access_key_id')
secret_access_key = config.get('aws.sqs.secret_access_key')
session = boto3.session.Session(
aws_access_key_id=access_key_id,
aws_secret_access_key=secret_access_key,
region_name='us-east-1')
sqs = session.resource('sqs')
queue = sqs.get_queue_by_name(QueueName=queue_name)
logger.info('Returning SQS queue - {queue}.'.format(queue=queue_name))
return queue
def get_measure_calculator(
measure_number,
year=config.get('calculation.measures_year'),
single_source_json=measure_reader.load_single_source(),
):
"""Generate a measure calculator object with correct definition and type."""
try:
measure_definition = measure_reader.load_measure_definition(
measure_number=measure_number,
single_source_json=single_source_json)
calculator_class = _get_measure_class(measure_number=measure_number,
year=year)
kwargs = _get_measure_args(measure_number=measure_number, year=year)
return calculator_class(measure_definition=measure_definition,
**kwargs)
except KeyError:
raise KeyError(
'Measure number {measure_number} is not yet supported for {year}.'.
format(measure_number=measure_number,
year=config.get('calculation.measures_year')))
def _test_access_to_idr_instance(
connection_parameters=config.get('teradata.connection.parameters')):
idr_endpoint = 'http://' + connection_parameters['system'] + ':1025'
try:
requests.get(idr_endpoint)
except requests.ConnectionError as e:
# If the system is able to connect to the IDR instance, the message will be:
# "414K(The LAN message Format field is invalid."
if 'LAN message Format field is invalid.' in str(e):
logger.debug(
"You can ping the IDR instance! Let's check access to the database."
)
return True
else:
error_message = 'Unable to ping the IDR instance.'
if config.get('teradata.connection.method').upper() == 'JUMP':
error_message += ' An error likely occured during SSH tunneling.'
else:
error_message += ' Likely a network or network-access issue.'
raise teradata_errors.TeradataError(error_message)
def test_send_submissions_no_pop_to_submit(self, mock_api_submitter):
"""Test submission not sent with 0 reporting."""
self.submitter.send_submissions = True
measurement_set = get_measurement_set_no_reporting()
self.submitter._send_submissions('tin', 'npi', measurement_set)
if config.get('submission.filter_out_zero_reporting'):
mock_api_submitter.submit_to_measurement_sets_api.assert_not_called(
)
else:
mock_api_submitter.submit_to_measurement_sets_api.assert_called_once(
)
def load_quality_codes(
measures=None,
json_path=config.get('assets.qpp_single_source_json')[config.get('calculation.measures_year')]
):
"""
Load quality codes from the single source JSON file and return them as a set of strings.
If the measures parameter is provided, restrict to quality codes on those measures.
"""
with open(json_path, encoding='utf-8') as file:
single_source = json.load(file)
# Filter out measures that are not claims-based quality measures with performanceOptions.
single_source = filter(lambda measure: 'performanceOptions' in measure, single_source)
return {
code['code'] for measure in single_source
for option in measure['performanceOptions']
for code in option['qualityCodes']
if (measures is None or measure['measureId'] in measures)
}
def get_discharge_date_query(tins, npis, discharge_period, hidden_codes):
"""
Query the IDR to determine discharge eligibility for a list of providers.
Return query to filter list of eligible (bene, discharge_date).
"""
start_date = config.get('calculation.start_date') - timedelta(days=discharge_period)
end_date = config.get('calculation.end_date')
# Note - hidden_codes are quoted in the query by to_sql_list.
return DISCHARGE_QUERY.format(
start_date=datetime.strftime(start_date, '%Y-%m-%d'),
end_date=datetime.strftime(end_date, '%Y-%m-%d'),
as_was_date=datetime.strftime(
config.get('calculation.as_was_date'), '%Y-%m-%d'
),
tins=sql_formatting.to_sql_list(tins),
npis=sql_formatting.to_sql_list(npis),
access_layer_name=config.get('teradata.access_layer_name'),
hidden_codes=sql_formatting.to_sql_list(hidden_codes)
)
def _put_to_measurement_sets_api(measurement_set, existing_measurement_set_id):
logger.debug('Making PUT request to the measurement-sets API.')
endpoint_url = urllib.parse.urljoin(config.get('submission.endpoint'),
'measurement-sets/')
url = urllib.parse.urljoin(endpoint_url, str(existing_measurement_set_id))
return requests.put(
url=url,
data=measurement_set.to_json(),
headers=get_headers(),
)
def get_mssa_query(bene_sks, encounter_codes, start_date, end_date):
"""
Query the IDR to find all claim lines related to hospitalization due to MSSA.
Return query to get all MSSA claim lines for the beneficiaries.
Note - The list returned needs to be grouped into episodes.
"""
access_layer_name = config.get('teradata.access_layer_name')
medicare_vdm_name = config.get('teradata.medicare_vdm_name')
return MSSA_QUERY.format(
access_layer_name=access_layer_name,
medicare_vdm_name=medicare_vdm_name,
start_date=datetime.strftime(start_date, '%Y-%m-%d'),
end_date=datetime.strftime(end_date, '%Y-%m-%d'),
as_was_date=datetime.strftime(
config.get('calculation.as_was_date'), '%Y-%m-%d'
),
bene_sks=sql_formatting.to_sql_list(bene_sks),
encounter_codes=sql_formatting.to_sql_list(encounter_codes)
)
def delete_measurement_set_api(measurement_set_id):
"""Delete a measuremet set by id."""
logger.warning(
'DELETING measurement {} set using the measurement-sets API.'.format(
measurement_set_id))
endpoint_url = urllib.parse.urljoin(config.get('submission.endpoint'),
'measurement-sets/')
url = urllib.parse.urljoin(endpoint_url, str(measurement_set_id))
response = requests.delete(url=url, headers=get_headers())
_handle_http_error(response, 'delete_measurement_set')
def get_scoring_preview(measurement_set):
"""Send the submission object to the appropriate API endpoint and get scoring preview."""
logger.debug('Sending measurement set to the score-preview endpoint.')
endpoint_url = urllib.parse.urljoin(config.get('submission.endpoint'),
'submissions/score-preview')
response = requests.post(url=endpoint_url,
data=measurement_set.prepare_for_scoring(),
headers=get_headers())
_handle_http_error(response, 'scoring_preview')
return response.json()
def _teradata_connection(
connection_parameters=config.get('teradata.connection.parameters')):
"""Return a connection to the IDR in the given environment."""
teradata_config = config.get('teradata.config')
uda_exec = _get_uda_exec(teradata_config)
session = None
# Create a Teradata session.
try:
session = uda_exec.connect(**connection_parameters)
logger.debug('Session created successfully.')
except teradata_errors.TeradataError:
raise teradata_errors.TeradataError(
'Could not create a new session. You may have a Teradata Driver issue.'
)
except (teradata.DatabaseError, teradata.InterfaceError):
raise teradata_errors.TeradataError(
'Could not create a new session. You may need to check your credentials.'
)
return session
def get_ct_scan_query(bene_date_set):
"""
Query the IDR to determine when beneficiaries had a CT scan.
Return query to get CT scan dates for the beneficiaries.
Note - The list returned then needs to be filtered to only keep
CT scans which happened on the specified claim date.
"""
bene_sks, from_dates = zip(*bene_date_set)
start_date = min(from_dates)
# TODO - Pass in thru_dates as well and use max instead of config.
end_date = config.get('calculation.end_date')
return CT_SCAN_QUERY.format(
access_layer_name=config.get('teradata.access_layer_name'),
start_date=datetime.strftime(start_date, '%Y-%m-%d'),
end_date=datetime.strftime(end_date, '%Y-%m-%d'),
as_was_date=datetime.strftime(
config.get('calculation.as_was_date'), '%Y-%m-%d'
),
bene_sks=sql_formatting.to_sql_list(bene_sks)
)
