def test_get_sets_value(self, _load_config): """Test that if global config value is not set initially, it's set after get is called.""" test_config = {'key': 'value'} config._CONFIG = None _load_config.return_value = test_config config.get('key') assert config._CONFIG == test_config
def get_access_layer_batch_query(tins, npis, start_date, end_date): """Populate the Teradata SQL statement to query the IDR for provider information in batches. Args: npis ([str]): National provider identifiers to load. tins ([str]): Provider tax identification numbers to load. start_date (datetime): Start date of data to load. end_date (datetime): End date of data to load. Returns: SQL query to retrieve data from the IDR. """ if len(tins) != len(npis): raise InputError('The TINs and NPIs list must be of the same size.') npi_tins = ['{}{}'.format(npi, tin) for npi, tin in zip(npis, tins)] return ACCESS_LAYER_BASE_QUERY_BATCH.format( npis=sql_formatting.to_sql_list(npis), tins=sql_formatting.to_sql_list(tins), npi_tins=sql_formatting.to_sql_list(npi_tins), start_date=datetime.strftime(start_date, '%Y-%m-%d'), end_date=datetime.strftime(end_date, '%Y-%m-%d'), as_was_date=datetime.strftime( config.get('calculation.as_was_date'), '%Y-%m-%d' ), access_layer_name=config.get('teradata.access_layer_name'), medicare_vdm_name=config.get('teradata.medicare_vdm_name') )
def _initialize(newrelic_agent=newrelic.agent): """Initialize the New Relic Application.""" if config.get('environment') not in ['DEV', 'IMPL', 'PRD']: return None newrelic_agent.initialize('newrelic.ini', environment=config.get('environment')) return newrelic_agent.register_application(timeout=10.0)
def _get_mssa_date_ranges(self, claims): """ Get mssa_date ranges by querying the IDR. Returns a dict of {bene_sk: [date_ranges]} that will need to be merged to keep only non-overlapping intervals. """ bene_sks = {claim.bene_sk for claim in claims} start_date = config.get('calculation.start_date') end_date = config.get('calculation.end_date') mssa_query = idr_queries.get_mssa_query( bene_sks=bene_sks, encounter_codes=self.procedure_codes, start_date=start_date, end_date=end_date) rows = execute.execute(mssa_query) if not rows: logger.error('No MSSA date found despite provider ' 'having submitted quality codes for Measure 407.') return {} mssa_date_ranges = collections.defaultdict(list) for row in rows: mssa_date_ranges[row['bene_sk']].append( DateRange(row['min_date'], row['max_date'])) return mssa_date_ranges
def get_measure_calculators(measures, year=config.get('calculation.measures_year')): """Generate a dictionary of measure calculator object with correct definition and type.""" json_path = config.get('assets.qpp_single_source_json')[year] single_source_json = measure_reader.load_single_source(json_path) return { measure: get_measure_calculator( measure_number=measure, year=year, single_source_json=single_source_json, ) for measure in measures }
def load_single_source(json_path=config.get('assets.qpp_single_source_json')[ config.get('calculation.measures_year')]): """ Load the single source file as JSON keyed by measure number. Only load claims-based measures by using the 'performanceOptions' key. """ with open(json_path, encoding='utf-8') as file: single_source_as_list = json.load(file) return { measure_json['measureId']: measure_json for measure_json in single_source_as_list if 'performanceOptions' in measure_json }
def get_headers(): """Return base headers for Nava's APIs.""" headers = { 'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': 'Bearer {api_token}'.format( api_token=config.get('submission.api_token')) } if config.get('submission.cookie'): headers['Cookie'] = config.get('submission.cookie') return headers
def get_submissions(npi=None, tin=None, performance_year=None, start_index=0): """ Simple GET request to check if submissions have been made. If NPI is provided, return submissions for this NPI. If start_index is provided, return submissions after the start_index. Else, simply return the first 10 submissions starting at start_index=0. FIXME: Move this into a different file, this is not part of the api submitter. """ logger.debug('Making a simple GET request to verify submissions.') endpoint_url = urllib.parse.urljoin(config.get('submission.endpoint'), 'submissions') params = { 'startIndex': start_index, } if npi: params['nationalProviderIdentifier'] = npi headers = get_headers() if tin: headers.update({'qpp-taxpayer-identification-number': tin}) if performance_year: params['performanceYear'] = str(performance_year) response = requests.get(endpoint_url, params=params, headers=headers) # If the request failed, raise an error. _handle_http_error(response, 'get_submissions') return response.json()
def __init__(self, tin, npi, performance_start, performance_end): """Create a MeasurementSet object.""" if config.get('submission.obscure_providers'): self.tin, self.npi = self._obscure_provider_identifers(tin, npi) else: self.tin = self._validate_tin(tin) self.npi = self._validate_npi(npi) self.performance_start = performance_start self.performance_end = performance_end # TODO: Read this template from a json file. self.data = { 'submission': { 'programName': 'mips', 'entityType': 'individual', 'taxpayerIdentificationNumber': self.tin, 'nationalProviderIdentifier': self.npi, 'performanceYear': performance_end.year, }, 'category': 'quality', 'submissionMethod': 'claims', 'performanceStart': performance_start, 'performanceEnd': performance_end, 'measurements': [], }
def test_teradata_connection( connection_parameters=config.get('teradata.connection.parameters'), continue_prompt=True): """Test Teradata connection and ask user for input if fails.""" try: # Test access to the IDR instance. _test_access_to_idr_instance(connection_parameters) # Test access to the Teradata database and session creation. _teradata_connection(connection_parameters) logger.info('Teradata connection confirmed.') return True # TODO: avoid catch all. except Exception as error: error_message = error.message if hasattr(error, 'message') else repr(error) if continue_prompt: print( 'Would you like to continue without connection to IDR? (y/n) ', 'You have 10 seconds to reply.') # select takes no keyword args. Args are rlist, wlist, xlist, timeout. read, write, exception = select.select([sys.stdin], [], [], 10) if read and sys.stdin.readline().strip() in ['y', 'yes']: logger.warn('Continuing without Teradata connection. ' 'Error - {}'.format(error_message)) return False raise teradata_errors.TeradataError(error_message)
def post_to_new_relic(payload): """ Function to post event to New Relic. Payload should be a string representation of a valid event. """ rest_key = config.get( 'new_relic_insights.event_key') # Key for NewRelic Insights POST API url = config.get( 'new_relic_insights.url') # URL for NewRelic Insights POST API post = ( 'echo \'{payload}\' | curl -d @- -X POST -H "Content-Type: application/json" ' '-H "X-Insert-Key: {rest_key}" ' '{url}') post = post.format(payload=payload, rest_key=rest_key, url=url) subprocess.call(args=[post], shell=True)
def query_claims_from_teradata_batch_provider(provider_tins, provider_npis, start_date, end_date, session=None): """ Query claims table for the analyzer for a batch of providers. Args: provider_tin_list ([str]): List of tax identification numbers to query for. provider_npi_list ([str]): List of national provider identification numbers to query for. start_date (date): Start date of data to load. end_date (date): End date of data to load. session (session): Teradata session to use to access IDR. Returns: (column_names, rows) (list(str), list(tuple)): Tuple of list of headers, and list of tuples containing claim line values. """ logger.debug('Query claims from TERADATA in env - {}.'.format( config.get('environment'))) query = idr_queries.get_access_layer_batch_query(tins=provider_tins, npis=provider_npis, start_date=start_date, end_date=end_date) rows = execute.execute(query, session) if rows: columns = rows[0].columns return (columns, rows) return ([], rows)
def post_to_slack(message): """Function to post message to Slack.""" url = config.get('slack.url') post = ('curl -X POST -H \'Content-type: application/json\' ' '--data \'{{"text":"{message}"}}\' ' '{url}') post = post.format(url=url, message=message) subprocess.call(args=[post], shell=True)
def test_get_uda_exec(): """ Test _get_uda_exec. Note: This test only works on an instance or a container with properly installed Teradata uda_exec drivers. """ uda_exec = teradata_connector._get_uda_exec( params=config.get('teradata.config')) assert isinstance(uda_exec, teradata.udaexec.UdaExec)
def _post_to_measurement_sets_api(measurement_set): logger.debug('Making POST request to the measurement-sets API.') endpoint_url = urllib.parse.urljoin(config.get('submission.endpoint'), 'measurement-sets/') return requests.post(url=endpoint_url, data=measurement_set.to_json(), headers=get_headers())
def add_measure_with_multiple_strata(self, measure_number, measure_results): """ Add a measure score to the measurement set object for a measure with multiple strata. measure_results should be a list of dictionaries, each with the keys - name: stratum_name - results: measure results dictionary with the following keys: - performance_met - performance_not_met - eligible_population_exclusion - eligible_population_exception - eligible_population - returns True if a measure was added to the measurement set. """ # If none of the strata contain eligible population, do not add the measure. if not any([ stratum_dict['results']['eligible_population'] for stratum_dict in measure_results ]): return False # If none of the strata contains relevant data, do not add the measure. if config.get('submission.filter_out_zero_reporting') and not any([ self.has_non_zero_reporting(measure_number, stratum_dict['results']) for stratum_dict in measure_results ]): return False measurement = { 'measureId': '{:03.0f}'.format(float(measure_number)), 'value': { 'isEndToEndReported': False, 'strata': [], } } for stratum_dict in measure_results: measurement['value']['strata'].append({ 'stratum': stratum_dict['name'], 'performanceMet': stratum_dict['results']['performance_met'], 'eligiblePopulationExclusion': stratum_dict['results']['eligible_population_exclusion'], 'eligiblePopulationException': stratum_dict['results']['eligible_population_exception'], 'performanceNotMet': stratum_dict['results']['performance_not_met'], 'eligiblePopulation': stratum_dict['results']['eligible_population'] }) self.data['measurements'].append(measurement) return True
def test_get_uda_exec_universal(UdaExec): """ Note: This will work across all devices but simply tests that provided Teradata can create a uda_exec, it is forwarded properly by _get_uda_exec. """ UdaExec.return_value = 'uda_exec' uda_exec = teradata_connector._get_uda_exec( params=config.get('teradata.config')) assert uda_exec == 'uda_exec'
def get_logger(logger_name): """Configure and return default logger.""" logger = logging.getLogger(logger_name) logger.setLevel(config.get('logging.log_level')) # Set log handling to JSON. handler = logging.StreamHandler() handler.setFormatter( formatter.JsonFormatter( extra={ 'hostname': socket.gethostname(), 'app': 'qpp-claims-to-quality', 'environment': config.get('environment'), 'team': config.get('logging.team'), 'contact': config.get('logging.contact') })) logger.addHandler(handler) return logger
def get_queue(queue_name=None): """Create and return an SQS Queue.""" if not queue_name: queue_name = config.get('aws.sqs.queue_name') logger.info('Getting SQS queue - {queue}.'.format(queue=queue_name)) access_key_id = config.get('aws.sqs.access_key_id') secret_access_key = config.get('aws.sqs.secret_access_key') session = boto3.session.Session( aws_access_key_id=access_key_id, aws_secret_access_key=secret_access_key, region_name='us-east-1') sqs = session.resource('sqs') queue = sqs.get_queue_by_name(QueueName=queue_name) logger.info('Returning SQS queue - {queue}.'.format(queue=queue_name)) return queue
def get_measure_calculator( measure_number, year=config.get('calculation.measures_year'), single_source_json=measure_reader.load_single_source(), ): """Generate a measure calculator object with correct definition and type.""" try: measure_definition = measure_reader.load_measure_definition( measure_number=measure_number, single_source_json=single_source_json) calculator_class = _get_measure_class(measure_number=measure_number, year=year) kwargs = _get_measure_args(measure_number=measure_number, year=year) return calculator_class(measure_definition=measure_definition, **kwargs) except KeyError: raise KeyError( 'Measure number {measure_number} is not yet supported for {year}.'. format(measure_number=measure_number, year=config.get('calculation.measures_year')))
def _test_access_to_idr_instance( connection_parameters=config.get('teradata.connection.parameters')): idr_endpoint = 'http://' + connection_parameters['system'] + ':1025' try: requests.get(idr_endpoint) except requests.ConnectionError as e: # If the system is able to connect to the IDR instance, the message will be: # "414K(The LAN message Format field is invalid." if 'LAN message Format field is invalid.' in str(e): logger.debug( "You can ping the IDR instance! Let's check access to the database." ) return True else: error_message = 'Unable to ping the IDR instance.' if config.get('teradata.connection.method').upper() == 'JUMP': error_message += ' An error likely occured during SSH tunneling.' else: error_message += ' Likely a network or network-access issue.' raise teradata_errors.TeradataError(error_message)
def test_send_submissions_no_pop_to_submit(self, mock_api_submitter): """Test submission not sent with 0 reporting.""" self.submitter.send_submissions = True measurement_set = get_measurement_set_no_reporting() self.submitter._send_submissions('tin', 'npi', measurement_set) if config.get('submission.filter_out_zero_reporting'): mock_api_submitter.submit_to_measurement_sets_api.assert_not_called( ) else: mock_api_submitter.submit_to_measurement_sets_api.assert_called_once( )
def load_quality_codes( measures=None, json_path=config.get('assets.qpp_single_source_json')[config.get('calculation.measures_year')] ): """ Load quality codes from the single source JSON file and return them as a set of strings. If the measures parameter is provided, restrict to quality codes on those measures. """ with open(json_path, encoding='utf-8') as file: single_source = json.load(file) # Filter out measures that are not claims-based quality measures with performanceOptions. single_source = filter(lambda measure: 'performanceOptions' in measure, single_source) return { code['code'] for measure in single_source for option in measure['performanceOptions'] for code in option['qualityCodes'] if (measures is None or measure['measureId'] in measures) }
def get_discharge_date_query(tins, npis, discharge_period, hidden_codes): """ Query the IDR to determine discharge eligibility for a list of providers. Return query to filter list of eligible (bene, discharge_date). """ start_date = config.get('calculation.start_date') - timedelta(days=discharge_period) end_date = config.get('calculation.end_date') # Note - hidden_codes are quoted in the query by to_sql_list. return DISCHARGE_QUERY.format( start_date=datetime.strftime(start_date, '%Y-%m-%d'), end_date=datetime.strftime(end_date, '%Y-%m-%d'), as_was_date=datetime.strftime( config.get('calculation.as_was_date'), '%Y-%m-%d' ), tins=sql_formatting.to_sql_list(tins), npis=sql_formatting.to_sql_list(npis), access_layer_name=config.get('teradata.access_layer_name'), hidden_codes=sql_formatting.to_sql_list(hidden_codes) )
def _put_to_measurement_sets_api(measurement_set, existing_measurement_set_id): logger.debug('Making PUT request to the measurement-sets API.') endpoint_url = urllib.parse.urljoin(config.get('submission.endpoint'), 'measurement-sets/') url = urllib.parse.urljoin(endpoint_url, str(existing_measurement_set_id)) return requests.put( url=url, data=measurement_set.to_json(), headers=get_headers(), )
def get_mssa_query(bene_sks, encounter_codes, start_date, end_date): """ Query the IDR to find all claim lines related to hospitalization due to MSSA. Return query to get all MSSA claim lines for the beneficiaries. Note - The list returned needs to be grouped into episodes. """ access_layer_name = config.get('teradata.access_layer_name') medicare_vdm_name = config.get('teradata.medicare_vdm_name') return MSSA_QUERY.format( access_layer_name=access_layer_name, medicare_vdm_name=medicare_vdm_name, start_date=datetime.strftime(start_date, '%Y-%m-%d'), end_date=datetime.strftime(end_date, '%Y-%m-%d'), as_was_date=datetime.strftime( config.get('calculation.as_was_date'), '%Y-%m-%d' ), bene_sks=sql_formatting.to_sql_list(bene_sks), encounter_codes=sql_formatting.to_sql_list(encounter_codes) )
def delete_measurement_set_api(measurement_set_id): """Delete a measuremet set by id.""" logger.warning( 'DELETING measurement {} set using the measurement-sets API.'.format( measurement_set_id)) endpoint_url = urllib.parse.urljoin(config.get('submission.endpoint'), 'measurement-sets/') url = urllib.parse.urljoin(endpoint_url, str(measurement_set_id)) response = requests.delete(url=url, headers=get_headers()) _handle_http_error(response, 'delete_measurement_set')
def get_scoring_preview(measurement_set): """Send the submission object to the appropriate API endpoint and get scoring preview.""" logger.debug('Sending measurement set to the score-preview endpoint.') endpoint_url = urllib.parse.urljoin(config.get('submission.endpoint'), 'submissions/score-preview') response = requests.post(url=endpoint_url, data=measurement_set.prepare_for_scoring(), headers=get_headers()) _handle_http_error(response, 'scoring_preview') return response.json()
def _teradata_connection( connection_parameters=config.get('teradata.connection.parameters')): """Return a connection to the IDR in the given environment.""" teradata_config = config.get('teradata.config') uda_exec = _get_uda_exec(teradata_config) session = None # Create a Teradata session. try: session = uda_exec.connect(**connection_parameters) logger.debug('Session created successfully.') except teradata_errors.TeradataError: raise teradata_errors.TeradataError( 'Could not create a new session. You may have a Teradata Driver issue.' ) except (teradata.DatabaseError, teradata.InterfaceError): raise teradata_errors.TeradataError( 'Could not create a new session. You may need to check your credentials.' ) return session
def get_ct_scan_query(bene_date_set): """ Query the IDR to determine when beneficiaries had a CT scan. Return query to get CT scan dates for the beneficiaries. Note - The list returned then needs to be filtered to only keep CT scans which happened on the specified claim date. """ bene_sks, from_dates = zip(*bene_date_set) start_date = min(from_dates) # TODO - Pass in thru_dates as well and use max instead of config. end_date = config.get('calculation.end_date') return CT_SCAN_QUERY.format( access_layer_name=config.get('teradata.access_layer_name'), start_date=datetime.strftime(start_date, '%Y-%m-%d'), end_date=datetime.strftime(end_date, '%Y-%m-%d'), as_was_date=datetime.strftime( config.get('calculation.as_was_date'), '%Y-%m-%d' ), bene_sks=sql_formatting.to_sql_list(bene_sks) )