def test_forward_happypath_emd(self): """test a forward-split: emd""" split_obj = split_utils.SplitInfo(DEMO_SPLIT) raw_emd_data = forecast_utils.fetch_market_history_emd( TEST_CONFIG.get('TEST', 'region_id'), self.test_type_id, data_range=TEST_CONFIG.get('TEST', 'history_count'), config=ROOT_CONFIG ) raw_emd_data1 = forecast_utils.parse_emd_data(raw_emd_data['result']) raw_emd_data = forecast_utils.fetch_market_history_emd( TEST_CONFIG.get('TEST', 'region_id'), self.test_original_id, data_range=TEST_CONFIG.get('TEST', 'history_count'), config=ROOT_CONFIG ) raw_emd_data2 = forecast_utils.parse_emd_data(raw_emd_data['result']) split_data = split_utils.fetch_split_history( TEST_CONFIG.get('TEST', 'region_id'), DEMO_SPLIT['type_id'], api_utils.SwitchCCPSource.EMD, config=ROOT_CONFIG ) ## Doctor data for testing ## min_split_date = split_data.date.min() raw_emd_data1 = prep_raw_data( raw_emd_data1.copy(), min_split_date ) raw_emd_data2 = prep_raw_data( raw_emd_data2.copy(), min_split_date ) pre_split_data = split_data[split_data.date <= split_obj.date_str].reset_index() pre_raw_data = raw_emd_data2[raw_emd_data2.date <= split_obj.date_str].reset_index() post_split_data = split_data[split_data.date > split_obj.date_str].reset_index() post_raw_data = raw_emd_data1[raw_emd_data1.date > split_obj.date_str].reset_index() ## Validate pre/post Split values ## validate_plain_data( post_raw_data, post_split_data ) validate_split_data( pre_raw_data, pre_split_data, split_obj )
def test_fetch_emd_history_fail(config=CONFIG): """happypath test for `fetch_market_history_emd`""" with pytest.raises(requests.exceptions.HTTPError): data = forecast_utils.fetch_market_history_emd( region_id=config.get('TEST', 'region_id'), type_id=config.get('TEST', 'type_id'), data_range=config.get('TEST', 'history_count'), config=config, endpoint_addr='http://www.eveprosper.com/noendpoint') with pytest.raises(exceptions.NoDataReturned): data = forecast_utils.fetch_market_history_emd( region_id=config.get('TEST', 'region_id'), type_id=config.get('TEST', 'bad_typeid'), data_range=config.get('TEST', 'history_count'), config=config)
def test_fetch_emd_history(config=CONFIG): """happypath test for `fetch_market_history_emd`""" data = forecast_utils.fetch_market_history_emd( region_id=config.get('TEST', 'region_id'), type_id=config.get('TEST', 'type_id'), data_range=config.get('TEST', 'history_count'), config=config) ## validate contents ## assert data['version'] == 2 data_keys = data['columns'].split(',') for entry in data['result']: assert set(data_keys) == set(entry['row'].keys()) ## validate count ## try: db_data = helpers.check_db_values( region_id=config.get('TEST', 'region_id'), type_id=config.get('TEST', 'type_id'), data_range=config.get('TEST', 'history_count'), config=config) mismatch_dates = helpers.compare_dates(data['result'], db_data) assert len(mismatch_dates) <= int(config.get('TEST', 'miss_budget')) except Exception as err: pytest.xfail('Unable to validate date counts: {0}/{1}'.format( len(data['result']), config.get('TEST', 'history_count') + '\n\texception= {0}'.format(repr(err))))
def test_fetch_emd_history(config=CONFIG): """happypath test for `fetch_market_history_emd`""" data = forecast_utils.fetch_market_history_emd( region_id=config.get('TEST', 'region_id'), type_id=config.get('TEST', 'type_id'), data_range=config.get('TEST', 'history_count'), config=config ) ## validate contents ## assert data['version'] == 2 data_keys = data['columns'].split(',') for entry in data['result']: assert set(data_keys) == set(entry['row'].keys()) ## validate count ## try: db_data = helpers.check_db_values( region_id=config.get('TEST', 'region_id'), type_id=config.get('TEST', 'type_id'), data_range=config.get('TEST', 'history_count'), config=config ) mismatch_dates = helpers.compare_dates( data['result'], db_data ) assert len(mismatch_dates) <= int(config.get('TEST', 'miss_budget')) except Exception as err: pytest.xfail( 'Unable to validate date counts: {0}/{1}'.format( len(data['result']), config.get('TEST', 'history_count') + '\n\texception= {0}'.format(repr(err)) ))
def test_fetch_emd_history_fail(config=CONFIG): """happypath test for `fetch_market_history_emd`""" with pytest.raises(requests.exceptions.HTTPError): data = forecast_utils.fetch_market_history_emd( region_id=config.get('TEST', 'region_id'), type_id=config.get('TEST', 'type_id'), data_range=config.get('TEST', 'history_count'), config=config, endpoint_addr='http://www.eveprosper.com/noendpoint' ) with pytest.raises(exceptions.NoDataReturned): data = forecast_utils.fetch_market_history_emd( region_id=config.get('TEST', 'region_id'), type_id=config.get('TEST', 'bad_typeid'), data_range=config.get('TEST', 'history_count'), config=config )
def test_forward_happypath_emd(self): """test a forward-split: crest""" split_obj = split_utils.SplitInfo(DEMO_SPLIT) raw_emd_data = forecast_utils.fetch_market_history_emd( TEST_CONFIG.get('TEST', 'region_id'), self.test_type_id, data_range=TEST_CONFIG.get('TEST', 'history_count'), config=ROOT_CONFIG) raw_emd_data1 = forecast_utils.parse_emd_data(raw_emd_data['result']) raw_emd_data = forecast_utils.fetch_market_history_emd( TEST_CONFIG.get('TEST', 'region_id'), self.test_original_id, data_range=TEST_CONFIG.get('TEST', 'history_count'), config=ROOT_CONFIG) raw_emd_data2 = forecast_utils.parse_emd_data(raw_emd_data['result']) split_data = split_utils.fetch_split_history( TEST_CONFIG.get('TEST', 'region_id'), DEMO_SPLIT['type_id'], api_utils.SwitchCCPSource.EMD, config=ROOT_CONFIG) #split_data.to_csv('split_data_emd.csv', index=False) ## Doctor data for testing ## min_split_date = split_data.date.min() raw_emd_data1 = prep_raw_data(raw_emd_data1.copy(), min_split_date) raw_emd_data2 = prep_raw_data(raw_emd_data2.copy(), min_split_date) pre_split_data = split_data[ split_data.date <= split_obj.date_str].reset_index() pre_raw_data = raw_emd_data2[ raw_emd_data2.date <= split_obj.date_str].reset_index() post_split_data = split_data[ split_data.date > split_obj.date_str].reset_index() post_raw_data = raw_emd_data1[ raw_emd_data1.date > split_obj.date_str].reset_index() ## Validate pre/post Split values ## validate_plain_data(post_raw_data, post_split_data) validate_split_data(pre_raw_data, pre_split_data, split_obj)
def test_future_split_emd(self): """valdiate with EMD source""" test_data_emd = split_utils.fetch_split_history( TEST_CONFIG.get('TEST', 'region_id'), self.test_type_id, api_utils.SwitchCCPSource.EMD, data_range=TEST_CONFIG.get('TEST', 'history_count'), config=ROOT_CONFIG) emd_data_raw = forecast_utils.fetch_market_history_emd( TEST_CONFIG.get('TEST', 'region_id'), self.test_type_id, data_range=TEST_CONFIG.get('TEST', 'history_count'), config=ROOT_CONFIG) assert test_data_emd.equals( forecast_utils.parse_emd_data(emd_data_raw['result']))
def test_short_split(self): """make sure escaped if split was too far back""" short_days = floor(DAYS_SINCE_SPLIT / 2) test_data_emd = split_utils.fetch_split_history( TEST_CONFIG.get('TEST', 'region_id'), DEMO_SPLIT['type_id'], api_utils.SwitchCCPSource.EMD, data_range=short_days, config=ROOT_CONFIG) emd_data_raw = forecast_utils.fetch_market_history_emd( TEST_CONFIG.get('TEST', 'region_id'), DEMO_SPLIT['type_id'], data_range=short_days, config=ROOT_CONFIG) assert test_data_emd.equals( forecast_utils.parse_emd_data(emd_data_raw['result']))
def test_future_split_emd(self): """valdiate with EMD source""" test_data_emd = split_utils.fetch_split_history( TEST_CONFIG.get('TEST', 'region_id'), self.test_type_id, fetch_source=api_utils.SwitchCCPSource.EMD, data_range=TEST_CONFIG.get('TEST', 'history_count'), config=ROOT_CONFIG ) emd_data_raw = forecast_utils.fetch_market_history_emd( TEST_CONFIG.get('TEST', 'region_id'), self.test_type_id, data_range=TEST_CONFIG.get('TEST', 'history_count'), config=ROOT_CONFIG ) assert test_data_emd.equals(forecast_utils.parse_emd_data(emd_data_raw['result']))
def test_short_split(self): """make sure escaped if split was too far back""" short_days = floor(DAYS_SINCE_SPLIT/2) test_data_emd = split_utils.fetch_split_history( TEST_CONFIG.get('TEST', 'region_id'), DEMO_SPLIT['type_id'], data_range=short_days, config=ROOT_CONFIG ) emd_data_raw = forecast_utils.fetch_market_history_emd( TEST_CONFIG.get('TEST', 'region_id'), DEMO_SPLIT['type_id'], data_range=short_days, config=ROOT_CONFIG ) assert test_data_emd.equals( forecast_utils.parse_emd_data(emd_data_raw['result']))
def fetch_split_history( region_id, type_id, fetch_source, data_range=400, #split_cache_file=SPLIT_CACHE_FILE, config=api_config.CONFIG, logger=api_config.LOGGER ): """for split items, fetch and stitch the data together Args: region_id (int): EVE Online region_id type_id (int): EVE Online type_id fetch_source (:enum:`api_config.SwitchCCPSource`): which endpoint to fetch data_range (int, optional): how much total data to fetch config (:obj:`configparser.ConfigParser`, optional): config overrides logger (:obj:`logging.logger`, optional): logging handle Returns: (:obj:`pandas.DataFrame`) data from endpoint """ ## Figure out if there's work to do ## if type_id not in api_config.SPLIT_INFO: raise exceptions.NoSplitConfigFound( 'No config set for {0}'.format(type_id) ) split_obj = api_config.SPLIT_INFO[type_id] fetch_id = split_obj.current_typeid() logger.debug(split_obj.__dict__) logger.info( 'fetching data from remote {0} (was {1})'.\ format(type_id, fetch_id) ) ## Get current market data ## if fetch_source == api_config.SwitchCCPSource.EMD: logger.info('--EMD fetch') current_data = forecast_utils.fetch_market_history_emd( region_id, fetch_id, data_range=data_range, config=config, #logger=logger ) current_data = forecast_utils.parse_emd_data(current_data['result']) else: logger.info('--CCP fetch') current_data = crest_utils.fetch_market_history( region_id, fetch_id, mode=fetch_source, config=config, logger=logger ) ## Early exit: split too old or hasn't happened yet ## min_date = datetime_helper(current_data['date'].min()) if min_date > split_obj.split_date or not bool(split_obj): #split is too old OR split hasn't happened yet logger.info('No split work -- Returning current pull') return current_data ## Fetch split data ## logger.info( '--fetching data from cache {0}@{1}'.format( split_obj.original_id, region_id )) split_data = fetch_split_cache_data( region_id, split_obj.original_id, split_date=split_obj.date_str ) if type_id == split_obj.new_id: #adjust the back history logger.info('--splitting old-data') split_data = execute_split( split_data, split_obj ) # vv FIX ME vv: Testable? # elif type_id == split_obj.original_id: #adjust the current data logger.info('--splitting new-data') current_data = execute_split( current_data, split_obj ) # ^^ FIX ME ^^ # else: #pragma: no cover logger.error( 'Unable to map new/old type_ids correctly' + '\n\ttype_id={0}'.format(type_id) + '\n\toriginal_id={0} new_id={1}'.format(split_obj.original_id, split_obj.new_id), exc_info=True ) raise exceptions.MissmatchedTypeIDs( status=500, message='unable to map types to splitcache function' ) logger.info('--combining data') current_data.to_csv('current_data.csv', index=False) split_data.to_csv('split_data.csv', index=False) combined_data = combine_split_history( current_data.copy(), #pass by value, not by reference split_data.copy() ) if fetch_source == api_config.SwitchCCPSource.CREST: logger.info('--Setting CREST datetime') combined_data['date'] = pd.to_datetime(combined_data['date']).\ dt.strftime('%Y-%m-%dT%H:%M:%S') return combined_data
def fetch_split_history( region_id, type_id, fetch_source=api_config.SwitchCCPSource.EMD, data_range=400, config=api_config.CONFIG, logger=logging.getLogger('publicAPI') ): """for split items, fetch and stitch the data together Args: region_id (int): EVE Online region_id type_id (int): EVE Online type_id fetch_source (:enum:`api_config.SwitchCCPSource`): which endpoint to fetch data_range (int): how much total data to fetch config (:obj:`configparser.ConfigParser`): config overrides logger (:obj:`logging.logger`): logging handle Returns: pandas.DataFrame: data from endpoint """ ## Figure out if there's work to do ## if type_id not in api_config.SPLIT_INFO: raise exceptions.NoSplitConfigFound( 'No config set for {0}'.format(type_id) ) split_obj = api_config.SPLIT_INFO[type_id] fetch_id = split_obj.current_typeid() logger.debug(split_obj.__dict__) logger.info( 'fetching data from remote %s (was %s)', type_id, fetch_id ) ## Get current market data ## if fetch_source == api_config.SwitchCCPSource.EMD: logger.info('--EMD fetch') current_data = forecast_utils.fetch_market_history_emd( region_id, fetch_id, data_range=data_range, config=config, ) current_data = forecast_utils.parse_emd_data(current_data['result']) else: logger.info('--CCP fetch') current_data = crest_utils.fetch_market_history( region_id, fetch_id, config=config, logger=logger ) ## Early exit: split too old or hasn't happened yet ## min_date = datetime_helper(current_data['date'].min()) if min_date > split_obj.split_date or not bool(split_obj): #split is too old OR split hasn't happened yet logger.info('No split work -- Returning current pull') return current_data ## Fetch split data ## logger.info( '--fetching data from cache %s@%s', split_obj.original_id, region_id ) split_data = fetch_split_cache_data( region_id, split_obj.original_id, split_date=split_obj.date_str ) if type_id == split_obj.new_id: # adjust the back history logger.info('--splitting old-data') split_data = execute_split( split_data, split_obj ) # vv FIX ME vv: Testable? # elif type_id == split_obj.original_id: # adjust the current data logger.info('--splitting new-data') current_data = execute_split( current_data, split_obj ) # ^^ FIX ME ^^ # else: # pragma: no cover logger.error( 'Unable to map new/old type_ids correctly' + '\n\ttype_id={0}'.format(type_id) + '\n\toriginal_id={0} new_id={1}'.format(split_obj.original_id, split_obj.new_id), exc_info=True ) raise exceptions.MissmatchedTypeIDs( status=500, message='unable to map types to splitcache function' ) logger.info('--combining data') current_data.to_csv('current_data.csv', index=False) split_data.to_csv('split_data.csv', index=False) combined_data = combine_split_history( current_data.copy(), # pass by value, not by reference split_data.copy() ) return combined_data