def test_forward_happypath_emd(self):
"""test a forward-split: emd"""
split_obj = split_utils.SplitInfo(DEMO_SPLIT)
raw_emd_data = forecast_utils.fetch_market_history_emd(
TEST_CONFIG.get('TEST', 'region_id'),
self.test_type_id,
data_range=TEST_CONFIG.get('TEST', 'history_count'),
config=ROOT_CONFIG
)
raw_emd_data1 = forecast_utils.parse_emd_data(raw_emd_data['result'])
raw_emd_data = forecast_utils.fetch_market_history_emd(
TEST_CONFIG.get('TEST', 'region_id'),
self.test_original_id,
data_range=TEST_CONFIG.get('TEST', 'history_count'),
config=ROOT_CONFIG
)
raw_emd_data2 = forecast_utils.parse_emd_data(raw_emd_data['result'])
split_data = split_utils.fetch_split_history(
TEST_CONFIG.get('TEST', 'region_id'),
DEMO_SPLIT['type_id'],
api_utils.SwitchCCPSource.EMD,
config=ROOT_CONFIG
)
## Doctor data for testing ##
min_split_date = split_data.date.min()
raw_emd_data1 = prep_raw_data(
raw_emd_data1.copy(),
min_split_date
)
raw_emd_data2 = prep_raw_data(
raw_emd_data2.copy(),
min_split_date
)
pre_split_data = split_data[split_data.date <= split_obj.date_str].reset_index()
pre_raw_data = raw_emd_data2[raw_emd_data2.date <= split_obj.date_str].reset_index()
post_split_data = split_data[split_data.date > split_obj.date_str].reset_index()
post_raw_data = raw_emd_data1[raw_emd_data1.date > split_obj.date_str].reset_index()
## Validate pre/post Split values ##
validate_plain_data(
post_raw_data,
post_split_data
)
validate_split_data(
pre_raw_data,
pre_split_data,
split_obj
)
def test_write_first_cache(self):
"""test write behavior on first pass (cache-buster mode)"""
self.test_clear_existing_cache() #blowup existing cache again
dummy_data = forecast_utils.parse_emd_data(DEMO_DATA['result'])
forecast_utils.write_prediction_cache(self.region_id,
self.type_id,
dummy_data,
cache_path=self.cache_path)
assert path.isfile(self.cache_filepath)
tdb = TinyDB(self.cache_filepath)
data = tdb.all()[0]
keys_list = [
'cache_date', 'region_id', 'type_id', 'lastWrite', 'prediction'
]
assert set(keys_list) == set(data.keys())
dummy_str_data = dummy_data.to_json(date_format='iso',
orient='records')
cached_data = pd.read_json(data['prediction'])
assert data['prediction'] == dummy_str_data
tdb.close()
def test_write_first_cache(self):
"""test write behavior on first pass (cache-buster mode)"""
self.test_clear_existing_cache() #blowup existing cache again
dummy_data = forecast_utils.parse_emd_data(DEMO_DATA['result'])
forecast_utils.write_prediction_cache(
self.region_id,
self.type_id,
dummy_data,
cache_path=self.cache_path
)
assert path.isfile(self.cache_filepath)
tdb = TinyDB(self.cache_filepath)
data = tdb.all()[0]
keys_list = [
'cache_date',
'region_id',
'type_id',
'lastWrite',
'prediction'
]
assert set(keys_list) == set(data.keys())
dummy_str_data = dummy_data.to_json(
date_format='iso',
orient='records'
)
cached_data = pd.read_json(data['prediction'])
assert data['prediction'] == dummy_str_data
tdb.close()
def test_forward_happypath_emd(self):
"""test a forward-split: crest"""
split_obj = split_utils.SplitInfo(DEMO_SPLIT)
raw_emd_data = forecast_utils.fetch_market_history_emd(
TEST_CONFIG.get('TEST', 'region_id'),
self.test_type_id,
data_range=TEST_CONFIG.get('TEST', 'history_count'),
config=ROOT_CONFIG)
raw_emd_data1 = forecast_utils.parse_emd_data(raw_emd_data['result'])
raw_emd_data = forecast_utils.fetch_market_history_emd(
TEST_CONFIG.get('TEST', 'region_id'),
self.test_original_id,
data_range=TEST_CONFIG.get('TEST', 'history_count'),
config=ROOT_CONFIG)
raw_emd_data2 = forecast_utils.parse_emd_data(raw_emd_data['result'])
split_data = split_utils.fetch_split_history(
TEST_CONFIG.get('TEST', 'region_id'),
DEMO_SPLIT['type_id'],
api_utils.SwitchCCPSource.EMD,
config=ROOT_CONFIG)
#split_data.to_csv('split_data_emd.csv', index=False)
## Doctor data for testing ##
min_split_date = split_data.date.min()
raw_emd_data1 = prep_raw_data(raw_emd_data1.copy(), min_split_date)
raw_emd_data2 = prep_raw_data(raw_emd_data2.copy(), min_split_date)
pre_split_data = split_data[
split_data.date <= split_obj.date_str].reset_index()
pre_raw_data = raw_emd_data2[
raw_emd_data2.date <= split_obj.date_str].reset_index()
post_split_data = split_data[
split_data.date > split_obj.date_str].reset_index()
post_raw_data = raw_emd_data1[
raw_emd_data1.date > split_obj.date_str].reset_index()
## Validate pre/post Split values ##
validate_plain_data(post_raw_data, post_split_data)
validate_split_data(pre_raw_data, pre_split_data, split_obj)
def test_parse_emd_data():
"""happypath test for refactoring EMD data"""
cleandata = forecast_utils.parse_emd_data(DEMO_DATA['result'])
assert isinstance(cleandata, pd.DataFrame) #check output type
headers = list(cleandata.columns.values)
expected_rows = DEMO_DATA['columns'].split(',')
assert set(headers) == set(expected_rows) #check row headers
assert len(cleandata.index) == len(DEMO_DATA['result'])
def test_parse_emd_data():
"""happypath test for refactoring EMD data"""
cleandata = forecast_utils.parse_emd_data(DEMO_DATA['result'])
assert isinstance(cleandata, pd.DataFrame) #check output type
headers = list(cleandata.columns.values)
expected_rows = DEMO_DATA['columns'].split(',')
assert set(headers) == set(expected_rows) #check row headers
assert len(cleandata.index) == len(DEMO_DATA['result'])
def test_future_split_emd(self):
"""valdiate with EMD source"""
test_data_emd = split_utils.fetch_split_history(
TEST_CONFIG.get('TEST', 'region_id'),
self.test_type_id,
api_utils.SwitchCCPSource.EMD,
data_range=TEST_CONFIG.get('TEST', 'history_count'),
config=ROOT_CONFIG)
emd_data_raw = forecast_utils.fetch_market_history_emd(
TEST_CONFIG.get('TEST', 'region_id'),
self.test_type_id,
data_range=TEST_CONFIG.get('TEST', 'history_count'),
config=ROOT_CONFIG)
assert test_data_emd.equals(
forecast_utils.parse_emd_data(emd_data_raw['result']))
def test_short_split(self):
"""make sure escaped if split was too far back"""
short_days = floor(DAYS_SINCE_SPLIT / 2)
test_data_emd = split_utils.fetch_split_history(
TEST_CONFIG.get('TEST', 'region_id'),
DEMO_SPLIT['type_id'],
api_utils.SwitchCCPSource.EMD,
data_range=short_days,
config=ROOT_CONFIG)
emd_data_raw = forecast_utils.fetch_market_history_emd(
TEST_CONFIG.get('TEST', 'region_id'),
DEMO_SPLIT['type_id'],
data_range=short_days,
config=ROOT_CONFIG)
assert test_data_emd.equals(
forecast_utils.parse_emd_data(emd_data_raw['result']))
def test_future_split_emd(self):
"""valdiate with EMD source"""
test_data_emd = split_utils.fetch_split_history(
TEST_CONFIG.get('TEST', 'region_id'),
self.test_type_id,
fetch_source=api_utils.SwitchCCPSource.EMD,
data_range=TEST_CONFIG.get('TEST', 'history_count'),
config=ROOT_CONFIG
)
emd_data_raw = forecast_utils.fetch_market_history_emd(
TEST_CONFIG.get('TEST', 'region_id'),
self.test_type_id,
data_range=TEST_CONFIG.get('TEST', 'history_count'),
config=ROOT_CONFIG
)
assert test_data_emd.equals(forecast_utils.parse_emd_data(emd_data_raw['result']))
def test_short_split(self):
"""make sure escaped if split was too far back"""
short_days = floor(DAYS_SINCE_SPLIT/2)
test_data_emd = split_utils.fetch_split_history(
TEST_CONFIG.get('TEST', 'region_id'),
DEMO_SPLIT['type_id'],
data_range=short_days,
config=ROOT_CONFIG
)
emd_data_raw = forecast_utils.fetch_market_history_emd(
TEST_CONFIG.get('TEST', 'region_id'),
DEMO_SPLIT['type_id'],
data_range=short_days,
config=ROOT_CONFIG
)
assert test_data_emd.equals(
forecast_utils.parse_emd_data(emd_data_raw['result']))
def test_parse_emd_data_fail():
"""make sure behavior is expected for failure"""
with pytest.raises(TypeError):
data = forecast_utils.parse_emd_data(DEMO_DATA)
def fetch_split_history(
region_id,
type_id,
fetch_source,
data_range=400,
#split_cache_file=SPLIT_CACHE_FILE,
config=api_config.CONFIG,
logger=api_config.LOGGER
):
"""for split items, fetch and stitch the data together
Args:
region_id (int): EVE Online region_id
type_id (int): EVE Online type_id
fetch_source (:enum:`api_config.SwitchCCPSource`): which endpoint to fetch
data_range (int, optional): how much total data to fetch
config (:obj:`configparser.ConfigParser`, optional): config overrides
logger (:obj:`logging.logger`, optional): logging handle
Returns:
(:obj:`pandas.DataFrame`) data from endpoint
"""
## Figure out if there's work to do ##
if type_id not in api_config.SPLIT_INFO:
raise exceptions.NoSplitConfigFound(
'No config set for {0}'.format(type_id)
)
split_obj = api_config.SPLIT_INFO[type_id]
fetch_id = split_obj.current_typeid()
logger.debug(split_obj.__dict__)
logger.info(
'fetching data from remote {0} (was {1})'.\
format(type_id, fetch_id)
)
## Get current market data ##
if fetch_source == api_config.SwitchCCPSource.EMD:
logger.info('--EMD fetch')
current_data = forecast_utils.fetch_market_history_emd(
region_id,
fetch_id,
data_range=data_range,
config=config,
#logger=logger
)
current_data = forecast_utils.parse_emd_data(current_data['result'])
else:
logger.info('--CCP fetch')
current_data = crest_utils.fetch_market_history(
region_id,
fetch_id,
mode=fetch_source,
config=config,
logger=logger
)
## Early exit: split too old or hasn't happened yet ##
min_date = datetime_helper(current_data['date'].min())
if min_date > split_obj.split_date or not bool(split_obj):
#split is too old OR split hasn't happened yet
logger.info('No split work -- Returning current pull')
return current_data
## Fetch split data ##
logger.info(
'--fetching data from cache {0}@{1}'.format(
split_obj.original_id, region_id
))
split_data = fetch_split_cache_data(
region_id,
split_obj.original_id,
split_date=split_obj.date_str
)
if type_id == split_obj.new_id: #adjust the back history
logger.info('--splitting old-data')
split_data = execute_split(
split_data,
split_obj
)
# vv FIX ME vv: Testable? #
elif type_id == split_obj.original_id: #adjust the current data
logger.info('--splitting new-data')
current_data = execute_split(
current_data,
split_obj
)
# ^^ FIX ME ^^ #
else: #pragma: no cover
logger.error(
'Unable to map new/old type_ids correctly' +
'\n\ttype_id={0}'.format(type_id) +
'\n\toriginal_id={0} new_id={1}'.format(split_obj.original_id, split_obj.new_id),
exc_info=True
)
raise exceptions.MissmatchedTypeIDs(
status=500,
message='unable to map types to splitcache function'
)
logger.info('--combining data')
current_data.to_csv('current_data.csv', index=False)
split_data.to_csv('split_data.csv', index=False)
combined_data = combine_split_history(
current_data.copy(), #pass by value, not by reference
split_data.copy()
)
if fetch_source == api_config.SwitchCCPSource.CREST:
logger.info('--Setting CREST datetime')
combined_data['date'] = pd.to_datetime(combined_data['date']).\
dt.strftime('%Y-%m-%dT%H:%M:%S')
return combined_data
def fetch_split_history(
region_id,
type_id,
fetch_source=api_config.SwitchCCPSource.EMD,
data_range=400,
config=api_config.CONFIG,
logger=logging.getLogger('publicAPI')
):
"""for split items, fetch and stitch the data together
Args:
region_id (int): EVE Online region_id
type_id (int): EVE Online type_id
fetch_source (:enum:`api_config.SwitchCCPSource`): which endpoint to fetch
data_range (int): how much total data to fetch
config (:obj:`configparser.ConfigParser`): config overrides
logger (:obj:`logging.logger`): logging handle
Returns:
pandas.DataFrame: data from endpoint
"""
## Figure out if there's work to do ##
if type_id not in api_config.SPLIT_INFO:
raise exceptions.NoSplitConfigFound(
'No config set for {0}'.format(type_id)
)
split_obj = api_config.SPLIT_INFO[type_id]
fetch_id = split_obj.current_typeid()
logger.debug(split_obj.__dict__)
logger.info(
'fetching data from remote %s (was %s)',
type_id, fetch_id
)
## Get current market data ##
if fetch_source == api_config.SwitchCCPSource.EMD:
logger.info('--EMD fetch')
current_data = forecast_utils.fetch_market_history_emd(
region_id,
fetch_id,
data_range=data_range,
config=config,
)
current_data = forecast_utils.parse_emd_data(current_data['result'])
else:
logger.info('--CCP fetch')
current_data = crest_utils.fetch_market_history(
region_id,
fetch_id,
config=config,
logger=logger
)
## Early exit: split too old or hasn't happened yet ##
min_date = datetime_helper(current_data['date'].min())
if min_date > split_obj.split_date or not bool(split_obj):
#split is too old OR split hasn't happened yet
logger.info('No split work -- Returning current pull')
return current_data
## Fetch split data ##
logger.info(
'--fetching data from cache %s@%s',
split_obj.original_id, region_id
)
split_data = fetch_split_cache_data(
region_id,
split_obj.original_id,
split_date=split_obj.date_str
)
if type_id == split_obj.new_id: # adjust the back history
logger.info('--splitting old-data')
split_data = execute_split(
split_data,
split_obj
)
# vv FIX ME vv: Testable? #
elif type_id == split_obj.original_id: # adjust the current data
logger.info('--splitting new-data')
current_data = execute_split(
current_data,
split_obj
)
# ^^ FIX ME ^^ #
else: # pragma: no cover
logger.error(
'Unable to map new/old type_ids correctly' +
'\n\ttype_id={0}'.format(type_id) +
'\n\toriginal_id={0} new_id={1}'.format(split_obj.original_id, split_obj.new_id),
exc_info=True
)
raise exceptions.MissmatchedTypeIDs(
status=500,
message='unable to map types to splitcache function'
)
logger.info('--combining data')
current_data.to_csv('current_data.csv', index=False)
split_data.to_csv('split_data.csv', index=False)
combined_data = combine_split_history(
current_data.copy(), # pass by value, not by reference
split_data.copy()
)
return combined_data
def test_parse_emd_data_fail():
"""make sure behavior is expected for failure"""
with pytest.raises(TypeError):
data = forecast_utils.parse_emd_data(DEMO_DATA)
