def test_forward_happypath_esi(self):
"""test a forward-split: ESI"""
split_obj = split_utils.SplitInfo(DEMO_SPLIT)
raw_esi_data1 = crest_utils.fetch_market_history(
TEST_CONFIG.get('TEST', 'region_id'),
self.test_type_id,
config=ROOT_CONFIG
)
raw_esi_data2 = crest_utils.fetch_market_history(
TEST_CONFIG.get('TEST', 'region_id'),
self.test_original_id,
config=ROOT_CONFIG
)
split_data = split_utils.fetch_split_history(
TEST_CONFIG.get('TEST', 'region_id'),
DEMO_SPLIT['type_id'],
fetch_source=api_utils.SwitchCCPSource.ESI,
config=ROOT_CONFIG
)
#split_data.to_csv('split_data_esi.csv', index=False)
## Doctor data for testing ##
min_split_date = split_data.date.min()
raw_esi_data1 = prep_raw_data(
raw_esi_data1.copy(),
min_split_date
)
raw_esi_data2 = prep_raw_data(
raw_esi_data2.copy(),
min_split_date
)
pre_split_data = split_data[split_data.date <= split_obj.date_str].reset_index()
pre_raw_data = raw_esi_data2[raw_esi_data2.date <= split_obj.date_str].reset_index()
post_split_data = split_data[split_data.date > split_obj.date_str].reset_index()
post_raw_data = raw_esi_data1[raw_esi_data1.date > split_obj.date_str].reset_index()
## Validate pre/post Split values ##
validate_plain_data(
post_raw_data,
post_split_data
)
validate_split_data(
pre_raw_data,
pre_split_data,
split_obj
)
def test_fetch_market_history(config=CONFIG):
"""test `fetch_market_history` utility"""
data = crest_utils.fetch_market_history(config.get('TEST', 'region_id'),
config.get('TEST', 'type_id'),
config=ROOT_CONFIG)
assert isinstance(data, pd.DataFrame)
expected_cols = [
'date', 'avgPrice', 'highPrice', 'lowPrice', 'volume', 'orders'
#extra keys:
#'volume_str',
#'orderCountStr'
]
for key in expected_cols:
assert key in data.columns.values
ohlc = crest_utils.data_to_ohlc(data)
assert ohlc['date'].equals(data['date'])
assert ohlc['open'].equals(data['avgPrice'])
assert ohlc['high'].equals(data['highPrice'])
assert ohlc['low'].equals(data['lowPrice'])
assert ohlc['volume'].equals(data['volume'])
assert data['avgPrice'].shift(1).equals(ohlc['close'])
def fetch_esi(
type_id,
region_id,
data_range=400,
logger=logging.getLogger(PROGNAME)
):
"""fetch data from ESI endpoint
Args:
type_id (int): EVE Online type_id
region_id (int): EVE Online region_id
data_range (int, optional): days of back-propogation
logger (:obj:`logging.logger`) logging handle
Returns:
pandas.DataFrame: data from endpoint
"""
logger.info('--Fetching price history: ESI')
if data_range > CREST_MAX:
warning_msg = 'ESI only returns %d days' % CREST_MAX
warnings.warn(warning_msg, UserWarning)
logger.warning(warning_msg)
data = crest_utils.fetch_market_history(
region_id,
type_id,
config=CONFIG,
logger=logger
)
return data.tail(n=data_range)
def fetch_esi(type_id, region_id, data_range=400, logger=LOGGER):
"""fetch data from ESI endpoint
Args:
type_id (int): EVE Online type_id
region_id (int): EVE Online region_id
data_range (int, optional): days of back-propogation
logger (:obj:`logging.logger`, optional) logging handle
Returns:
(:obj:`pandas.DataFrame`): data from endpoint
"""
logger.info('--Fetching price history: ESI')
if data_range > CREST_MAX:
warning_msg = 'ESI only returns %d days' % CREST_MAX
warnings.warn(warning_msg, UserWarning)
logger.warning(warning_msg)
data = crest_utils.fetch_market_history(region_id,
type_id,
mode=api_utils.SwitchCCPSource.ESI,
config=CONFIG,
logger=logger)
return data.tail(n=data_range)
def test_fetch_market_history_esi(config=CONFIG):
"""test `fetch_market_history` utility"""
data = crest_utils.fetch_market_history(
config.get('TEST', 'region_id'),
config.get('TEST', 'type_id'),
config=ROOT_CONFIG,
)
assert isinstance(data, pd.DataFrame)
expected_cols = [
'date',
'avgPrice',
'highPrice',
'lowPrice',
'volume',
'orders'
#extra keys:
#'volume_str',
#'orderCountStr'
]
for key in expected_cols:
assert key in data.columns.values
ohlc = crest_utils.data_to_ohlc(data)
assert ohlc['date'].equals(data['date'])
assert ohlc['open'].equals(data['avgPrice'])
assert ohlc['high'].equals(data['highPrice'])
assert ohlc['low'].equals(data['lowPrice'])
assert ohlc['volume'].equals(data['volume'])
assert data['avgPrice'].shift(1).equals(ohlc['close'])
def test_forward_happypath_crest(self):
"""test a forward-split: crest"""
split_obj = split_utils.SplitInfo(DEMO_SPLIT)
raw_crest_data1 = crest_utils.fetch_market_history(
TEST_CONFIG.get('TEST', 'region_id'),
self.test_type_id,
mode=api_utils.SwitchCCPSource.CREST,
config=ROOT_CONFIG)
raw_crest_data2 = crest_utils.fetch_market_history(
TEST_CONFIG.get('TEST', 'region_id'),
self.test_original_id,
mode=api_utils.SwitchCCPSource.CREST,
config=ROOT_CONFIG)
split_data = split_utils.fetch_split_history(
TEST_CONFIG.get('TEST', 'region_id'),
DEMO_SPLIT['type_id'],
api_utils.SwitchCCPSource.CREST,
config=ROOT_CONFIG)
#split_data.to_csv('split_data_crest.csv', index=False)
## Doctor data for testing ##
min_split_date = split_data.date.min()
raw_crest_data1 = prep_raw_data(raw_crest_data1.copy(), min_split_date)
raw_crest_data2 = prep_raw_data(raw_crest_data2.copy(), min_split_date)
split_date_str = datetime.strftime(split_obj.split_date,
'%Y-%m-%dT%H:%M:%S')
pre_split_data = split_data[
split_data.date <= split_date_str].reset_index()
pre_raw_data = raw_crest_data2[
raw_crest_data2.date <= split_date_str].reset_index()
post_split_data = split_data[
split_data.date > split_date_str].reset_index()
post_raw_data = raw_crest_data1[
raw_crest_data1.date > split_date_str].reset_index()
## Validate pre/post Split values ##
validate_plain_data(post_raw_data, post_split_data)
validate_split_data(pre_raw_data, pre_split_data, split_obj)
def test_future_split_crest(self):
"""validate with CREST source"""
test_data_crest = split_utils.fetch_split_history(
TEST_CONFIG.get('TEST', 'region_id'),
self.test_type_id,
api_utils.SwitchCCPSource.CREST,
config=ROOT_CONFIG)
assert test_data_crest.equals(
crest_utils.fetch_market_history(
TEST_CONFIG.get('TEST', 'region_id'),
self.test_type_id,
mode=api_utils.SwitchCCPSource.CREST,
config=ROOT_CONFIG))
def test_future_split_esi(self):
"""validate on ESI"""
test_data_esi = split_utils.fetch_split_history(
TEST_CONFIG.get('TEST', 'region_id'),
self.test_type_id,
api_utils.SwitchCCPSource.ESI,
config=ROOT_CONFIG)
assert test_data_esi.equals(
crest_utils.fetch_market_history(
TEST_CONFIG.get('TEST', 'region_id'),
self.test_type_id,
mode=api_utils.SwitchCCPSource.ESI,
config=ROOT_CONFIG))
def test_future_split_esi(self):
"""validate on ESI"""
test_data_esi = split_utils.fetch_split_history(
TEST_CONFIG.get('TEST', 'region_id'),
self.test_type_id,
api_utils.SwitchCCPSource.ESI,
config=ROOT_CONFIG
)
assert test_data_esi.equals(
crest_utils.fetch_market_history(
TEST_CONFIG.get('TEST', 'region_id'),
self.test_type_id,
config=ROOT_CONFIG
)
)
def fetch_split_history(
region_id,
type_id,
fetch_source,
data_range=400,
#split_cache_file=SPLIT_CACHE_FILE,
config=api_config.CONFIG,
logger=api_config.LOGGER
):
"""for split items, fetch and stitch the data together
Args:
region_id (int): EVE Online region_id
type_id (int): EVE Online type_id
fetch_source (:enum:`api_config.SwitchCCPSource`): which endpoint to fetch
data_range (int, optional): how much total data to fetch
config (:obj:`configparser.ConfigParser`, optional): config overrides
logger (:obj:`logging.logger`, optional): logging handle
Returns:
(:obj:`pandas.DataFrame`) data from endpoint
"""
## Figure out if there's work to do ##
if type_id not in api_config.SPLIT_INFO:
raise exceptions.NoSplitConfigFound(
'No config set for {0}'.format(type_id)
)
split_obj = api_config.SPLIT_INFO[type_id]
fetch_id = split_obj.current_typeid()
logger.debug(split_obj.__dict__)
logger.info(
'fetching data from remote {0} (was {1})'.\
format(type_id, fetch_id)
)
## Get current market data ##
if fetch_source == api_config.SwitchCCPSource.EMD:
logger.info('--EMD fetch')
current_data = forecast_utils.fetch_market_history_emd(
region_id,
fetch_id,
data_range=data_range,
config=config,
#logger=logger
)
current_data = forecast_utils.parse_emd_data(current_data['result'])
else:
logger.info('--CCP fetch')
current_data = crest_utils.fetch_market_history(
region_id,
fetch_id,
mode=fetch_source,
config=config,
logger=logger
)
## Early exit: split too old or hasn't happened yet ##
min_date = datetime_helper(current_data['date'].min())
if min_date > split_obj.split_date or not bool(split_obj):
#split is too old OR split hasn't happened yet
logger.info('No split work -- Returning current pull')
return current_data
## Fetch split data ##
logger.info(
'--fetching data from cache {0}@{1}'.format(
split_obj.original_id, region_id
))
split_data = fetch_split_cache_data(
region_id,
split_obj.original_id,
split_date=split_obj.date_str
)
if type_id == split_obj.new_id: #adjust the back history
logger.info('--splitting old-data')
split_data = execute_split(
split_data,
split_obj
)
# vv FIX ME vv: Testable? #
elif type_id == split_obj.original_id: #adjust the current data
logger.info('--splitting new-data')
current_data = execute_split(
current_data,
split_obj
)
# ^^ FIX ME ^^ #
else: #pragma: no cover
logger.error(
'Unable to map new/old type_ids correctly' +
'\n\ttype_id={0}'.format(type_id) +
'\n\toriginal_id={0} new_id={1}'.format(split_obj.original_id, split_obj.new_id),
exc_info=True
)
raise exceptions.MissmatchedTypeIDs(
status=500,
message='unable to map types to splitcache function'
)
logger.info('--combining data')
current_data.to_csv('current_data.csv', index=False)
split_data.to_csv('split_data.csv', index=False)
combined_data = combine_split_history(
current_data.copy(), #pass by value, not by reference
split_data.copy()
)
if fetch_source == api_config.SwitchCCPSource.CREST:
logger.info('--Setting CREST datetime')
combined_data['date'] = pd.to_datetime(combined_data['date']).\
dt.strftime('%Y-%m-%dT%H:%M:%S')
return combined_data
def fetch_extended_history(region_id,
type_id,
min_data=MIN_DATA,
crest_range=CREST_RANGE,
config=api_config.CONFIG,
data_range=DEFAULT_RANGE,
logger=LOGGER):
"""fetch data from database
Args:
region_id (int): EVE Online regionID: https://crest-tq.eveonline.com/regions/
type_id (int): EVE Online typeID: https://crest-tq.eveonline.com/types/
cache_buster (bool, optional): skip cache, fetch new data
data_range (int, optional): how far back to fetch data
logger (:obj:`logging.logger`): logging handle
Returns:
(:obj:`pandas.data_frame`): collection of data from database
['date', 'avgPrice', 'highPrice', 'lowPrice', 'volume', 'orders']
"""
logger.info('--fetching history data')
try:
raw_data = fetch_market_history_emd(region_id,
type_id,
data_range,
config=config)
logger.debug(raw_data['result'][:5])
data = parse_emd_data(raw_data['result'])
except Exception as err_msg: #pragma: no cover
logger.warning('ERROR: trouble getting data from EMD' +
'\n\tregion_id={0}'.format(region_id) +
'\n\ttype_id={0}'.format(type_id) +
'\n\tdata_range={0}'.format(data_range),
exc_info=True)
data = []
if len(data) < crest_range: #pragma: no cover
logger.info('--Not enough data found, fetching CREST data')
try:
data = crest_utils.fetch_market_history(region_id,
type_id,
config=config,
logger=logger)
except Exception as err_msg: #pragma: no cover
logger.error('ERROR: trouble getting data from CREST' +
'\n\tregion_id={0}'.format(region_id) +
'\n\ttype_id={0}'.format(type_id),
exc_info=True)
raise exceptions.EMDBadMarketData(
status=500, message='Unable to fetch historical data')
if len(data) < min_data: #pragma: no cover
logger.warning('Not enough data to seed prediction' +
'\n\tregion_id={0}'.format(region_id) +
'\n\ttype_id={0}'.format(type_id) +
'\n\tlen(data)={0}'.format(len(data)))
raise exceptions.ProphetNotEnoughData(
status=500, message='Not enough data to build a prediction')
return data
def fetch_extended_history(
region_id,
type_id,
mode=api_config.SwitchCCPSource.ESI,
min_data=MIN_DATA,
crest_range=CREST_RANGE,
config=api_config.CONFIG,
data_range=DEFAULT_RANGE,
logger=logging.getLogger('publicAPI')
):
"""fetch data from database
Args:
region_id (int): EVE Online regionID: https://crest-tq.eveonline.com/regions/
type_id (int): EVE Online typeID: https://crest-tq.eveonline.com/types/
cache_buster (bool): skip cache, fetch new data
data_range (int): how far back to fetch data
logger (:obj:`logging.logger`): logging handle
Returns:
pandas.DataFrame: collection of data from database
['date', 'avgPrice', 'highPrice', 'lowPrice', 'volume', 'orders']
"""
logger.info('--fetching history data')
try:
raw_data = fetch_market_history_emd(
region_id,
type_id,
data_range,
config=config
)
logger.debug(raw_data['result'][:5])
data = parse_emd_data(raw_data['result'])
except Exception as err_msg: #pragma: no cover
logger.warning(
'ERROR: trouble getting data from EMD' +
'\n\tregion_id={0}'.format(region_id) +
'\n\ttype_id={0}'.format(type_id) +
'\n\tdata_range={0}'.format(data_range),
exc_info=True
)
data = []
if len(data) < crest_range: #pragma: no cover
logger.info('--Not enough data found, fetching CREST data')
try:
data = crest_utils.fetch_market_history(
region_id,
type_id,
config=config,
logger=logger
)
except Exception as err_msg: #pragma: no cover
logger.error(
'ERROR: trouble getting data from CREST' +
'\n\tregion_id={0}'.format(region_id) +
'\n\ttype_id={0}'.format(type_id),
exc_info=True
)
raise exceptions.EMDBadMarketData(
status=500,
message='Unable to fetch historical data'
)
if len(data) < min_data: #pragma: no cover
logger.warning(
'Not enough data to seed prediction' +
'\n\tregion_id={0}'.format(region_id) +
'\n\ttype_id={0}'.format(type_id) +
'\n\tlen(data)={0}'.format(len(data))
)
raise exceptions.ProphetNotEnoughData(
status=500,
message='Not enough data to build a prediction'
)
return data
def fetch_split_history(
region_id,
type_id,
fetch_source=api_config.SwitchCCPSource.EMD,
data_range=400,
config=api_config.CONFIG,
logger=logging.getLogger('publicAPI')
):
"""for split items, fetch and stitch the data together
Args:
region_id (int): EVE Online region_id
type_id (int): EVE Online type_id
fetch_source (:enum:`api_config.SwitchCCPSource`): which endpoint to fetch
data_range (int): how much total data to fetch
config (:obj:`configparser.ConfigParser`): config overrides
logger (:obj:`logging.logger`): logging handle
Returns:
pandas.DataFrame: data from endpoint
"""
## Figure out if there's work to do ##
if type_id not in api_config.SPLIT_INFO:
raise exceptions.NoSplitConfigFound(
'No config set for {0}'.format(type_id)
)
split_obj = api_config.SPLIT_INFO[type_id]
fetch_id = split_obj.current_typeid()
logger.debug(split_obj.__dict__)
logger.info(
'fetching data from remote %s (was %s)',
type_id, fetch_id
)
## Get current market data ##
if fetch_source == api_config.SwitchCCPSource.EMD:
logger.info('--EMD fetch')
current_data = forecast_utils.fetch_market_history_emd(
region_id,
fetch_id,
data_range=data_range,
config=config,
)
current_data = forecast_utils.parse_emd_data(current_data['result'])
else:
logger.info('--CCP fetch')
current_data = crest_utils.fetch_market_history(
region_id,
fetch_id,
config=config,
logger=logger
)
## Early exit: split too old or hasn't happened yet ##
min_date = datetime_helper(current_data['date'].min())
if min_date > split_obj.split_date or not bool(split_obj):
#split is too old OR split hasn't happened yet
logger.info('No split work -- Returning current pull')
return current_data
## Fetch split data ##
logger.info(
'--fetching data from cache %s@%s',
split_obj.original_id, region_id
)
split_data = fetch_split_cache_data(
region_id,
split_obj.original_id,
split_date=split_obj.date_str
)
if type_id == split_obj.new_id: # adjust the back history
logger.info('--splitting old-data')
split_data = execute_split(
split_data,
split_obj
)
# vv FIX ME vv: Testable? #
elif type_id == split_obj.original_id: # adjust the current data
logger.info('--splitting new-data')
current_data = execute_split(
current_data,
split_obj
)
# ^^ FIX ME ^^ #
else: # pragma: no cover
logger.error(
'Unable to map new/old type_ids correctly' +
'\n\ttype_id={0}'.format(type_id) +
'\n\toriginal_id={0} new_id={1}'.format(split_obj.original_id, split_obj.new_id),
exc_info=True
)
raise exceptions.MissmatchedTypeIDs(
status=500,
message='unable to map types to splitcache function'
)
logger.info('--combining data')
current_data.to_csv('current_data.csv', index=False)
split_data.to_csv('split_data.csv', index=False)
combined_data = combine_split_history(
current_data.copy(), # pass by value, not by reference
split_data.copy()
)
return combined_data
def get(self, return_type):
"""GET data from CREST and send out OHLC info"""
args = self.reqparse.parse_args()
#TODO: info archive
LOGGER.info('OHLC {0} Request: {1}'.format(return_type, args))
if return_type not in return_supported_types():
return 'INVALID RETURN FORMAT', 405
mode = api_config.SwitchCCPSource(
api_config.CONFIG.get('GLOBAL', 'crest_or_esi'))
## Validate inputs ##
try:
crest_utils.validate_id('map_regions',
args.get('regionID'),
mode=mode,
config=api_config.CONFIG,
logger=LOGGER)
crest_utils.validate_id('inventory_types',
args.get('typeID'),
mode=mode,
config=api_config.CONFIG,
logger=LOGGER)
except exceptions.ValidatorException as err:
LOGGER.warning('ERROR: unable to validate type/region ids' +
'\n\targs={0}'.format(args),
exc_info=True)
return err.message, err.status
except Exception: #pragma: no cover
LOGGER.error('ERROR: unable to validate type/region ids' +
'args={0}'.format(args),
exc_info=True)
return 'UNHANDLED EXCEPTION', 500
## Fetch CREST ##
try:
#LOGGER.info(api_config.SPLIT_INFO)
if args.get('typeID') in api_config.SPLIT_INFO:
LOGGER.info('FORK: using split utility')
data = split_utils.fetch_split_history(
args.get('regionID'),
args.get('typeID'),
mode,
config=api_config.CONFIG,
logger=LOGGER)
else:
data = crest_utils.fetch_market_history(
args.get('regionID'),
args.get('typeID'),
config=api_config.CONFIG,
mode=mode,
logger=LOGGER)
data = crest_utils.data_to_ohlc(data)
except exceptions.ValidatorException as err: #pragma: no cover
LOGGER.error('ERROR: unable to parse CREST data' +
'\n\targs={0}'.format(args),
exc_info=True)
return err.message, err.status
except Exception: #pragma: no cover
#except Exception as err: #pragma: no cover
LOGGER.error('ERROR: unhandled issue in parsing CREST data' +
'args={0}'.format(args),
exc_info=True)
return 'UNHANDLED EXCEPTION', 500
## Format output ##
if return_type == AcceptedDataFormat.JSON.value:
LOGGER.info('rolling json response')
data_str = data.to_json(path_or_buf=None, orient='records')
message = json.loads(data_str)
elif return_type == AcceptedDataFormat.CSV.value:
LOGGER.info('rolling csv response')
data_str = data.to_csv(
path_or_buf=None,
header=True,
index=False,
columns=['date', 'open', 'high', 'low', 'close', 'volume'])
message = output_csv(data_str, 200)
else: #pragma: no cover
#TODO: CUT?
LOGGER.error('invalid format requested' +
'\n\targs={0}'.format(args) +
'\n\treturn_type={0}'.format(return_type),
exc_info=True)
return 'UNSUPPORTED FORMAT', 500
return message
def get(self, return_type):
"""GET data from CREST and send out OHLC info"""
args = self.reqparse.parse_args()
#TODO: info archive
self.logger.info('OHLC %s Request: %s', return_type, args)
if return_type not in return_supported_types():
return 'INVALID RETURN FORMAT', 405
## Validate inputs ##
try:
crest_utils.validate_id(
'map_regions',
args.get('regionID'),
config=api_config.CONFIG,
logger=self.logger,
)
crest_utils.validate_id(
'inventory_types',
args.get('typeID'),
config=api_config.CONFIG,
logger=self.logger,
)
except exceptions.ValidatorException as err:
self.logger.warning(
'ERROR: unable to validate type/region ids' +
'\n\targs={0}'.format(args),
exc_info=True
)
return err.message, err.status
except Exception: #pragma: no cover
self.logger.error(
'ERROR: unable to validate type/region ids' +
'args={0}'.format(args),
exc_info=True
)
return 'UNHANDLED EXCEPTION', 500
## Fetch CREST ##
try:
#LOGGER.info(api_config.SPLIT_INFO)
if args.get('typeID') in api_config.SPLIT_INFO:
self.logger.info('FORK: using split utility')
data = split_utils.fetch_split_history(
args.get('regionID'),
args.get('typeID'),
config=api_config.CONFIG,
logger=self.logger,
)
else:
data = crest_utils.fetch_market_history(
args.get('regionID'),
args.get('typeID'),
config=api_config.CONFIG,
logger=LOGGER
)
data = crest_utils.data_to_ohlc(data)
except exceptions.ValidatorException as err: #pragma: no cover
self.logger.error(
'ERROR: unable to parse CREST data\n\targs=%s',
args,
exc_info=True
)
return err.message, err.status
except Exception: #pragma: no cover
self.logger.error(
'ERROR: unhandled issue in parsing CREST data\n\targs=%s',
args,
exc_info=True
)
return 'UNHANDLED EXCEPTION', 500
## Format output ##
if return_type == AcceptedDataFormat.JSON.value:
self.logger.info('rolling json response')
data_str = data.to_json(
path_or_buf=None,
orient='records'
)
message = json.loads(data_str)
elif return_type == AcceptedDataFormat.CSV.value:
self.logger.info('rolling csv response')
data_str = data.to_csv(
path_or_buf=None,
header=True,
index=False,
columns=[
'date',
'open',
'high',
'low',
'close',
'volume'
]
)
message = output_csv(data_str, 200)
else: #pragma: no cover
#TODO: CUT?
self.logger.error(
'invalid format requested' +
'\n\targs=%s' +
'\n\treturn_type=%s',
args, return_type,
exc_info=True
)
return 'UNSUPPORTED FORMAT', 500
return message