def SearchForSeriesCode(self, series_code):
"""
:param series_code: str
:return: pandas.DataFrame
"""
# Now for the ugly solution...
flist = glob.glob(os.path.join(self.Directory, '*.xls'))
# Search until we find it; if we don't, puke
for fname in flist:
# This query pattern will work on the "data" sheets; ignore the index.
try:
sheets = pandas.read_excel(fname, sheet_name=None, header=None, index_col=0)
except:
econ_platform_core.log_last_error()
log_warning('Problem with Excel {0}'.format(fname))
continue
for sheet_name in sheets:
sheet = sheets[sheet_name]
list_index = list(sheet.index)
# We ignore sheets that do not match the desired format.
if 'Series ID' not in list_index:
continue
for c in sheet.columns:
if sheet[c]["Series ID"] == series_code:
list_index[0] = 'series_name'
sheet.index = list_index
return sheet[c]
# Did not find it; puke.
raise econ_platform_core.entity_and_errors.TickerNotFoundError('Could not find series ID = {0}'.format(series_code))
def BuildTable(self):
"""
Get all series in all xls in directory (!).
"""
# Now for the ugly solution...
flist = glob.glob(os.path.join(self.Directory, '*.xls'))
# Search until we find it; if we don't, puke
out = []
for fname in flist:
log_debug('Reading %s', fname)
# This query pattern will work on the "data" sheets; ignore the index.
try:
sheets = pandas.read_excel(fname,
sheet_name=None,
header=None,
index_col=0)
except:
econ_platform_core.log_last_error()
log_warning('Problem with Excel {0}'.format(fname))
continue
for sheet_name in sheets:
sheet = sheets[sheet_name]
sheet = self.PatchSheet(sheet)
list_index = list(sheet.index)
# We ignore sheets that do not match the desired format.
for targ_field in self.TickerLabels:
if targ_field not in list_index:
continue
list_index = self.FixIndex(list_index)
sheet.index = list_index
for c in sheet.columns:
try:
[ser, meta] = self.ConvertDFtoSeries(sheet[c])
except SkipColumn:
continue
full_ticker = str(meta.ticker_full)
if full_ticker in self.TableSeries:
ser = ser.combine_first(
self.TableSeries[full_ticker])
self.TableSeries[full_ticker] = ser
else:
self.TableSeries[full_ticker] = ser
self.TableMeta[full_ticker] = meta
def SearchForSeriesCode(self, series_code):
"""
:param series_code: str
:return: pandas.DataFrame
"""
# Now for the ugly solution...
flist = glob.glob(os.path.join(self.Directory, '*.xls'))
# Search until we find it; if we don't, puke
out = []
for fname in flist:
log_debug('Reading %s', fname)
# This query pattern will work on the "data" sheets; ignore the index.
try:
sheets = pandas.read_excel(fname,
sheet_name=None,
header=None,
index_col=0)
except:
econ_platform_core.log_last_error()
log_warning('Problem with Excel {0}'.format(fname))
continue
for sheet_name in sheets:
sheet = sheets[sheet_name]
list_index = list(sheet.index)
# We ignore sheets that do not match the desired format.
targ_field = self.TickerLabel
if targ_field not in list_index:
continue
for c in sheet.columns:
if sheet[c][targ_field] == series_code:
# Fixes ABS spreadsheet
list_index[0] = 'series_name'
sheet.index = list_index
out.append(sheet[c])
# Did not find it; puke.
if len(out) == 0:
raise econ_platform_core.TickerNotFoundError(
'Could not find series ID = {0}'.format(series_code))
else:
return out
def Update(self, ticker, series_meta, provider_wrapper, database_manager):
"""
Procedure to handle updates. The default behaviour is to not update; just retrieve from the
database.
:param ticker: str
:param series_meta: econ_platform_core.SeriesMetadata
:param provider_wrapper: econ_platform_core.ProviderWrapper
:param database_manager: econ_platform_core.DatabaseManager
:return:
"""
last_refresh = series_meta.last_refresh
ticker_str = ticker
if last_refresh is None:
last_refresh = database_manager.GetLastRefresh(series_meta.ticker_full)
# If the developer is too lazy to parse strings...
if type(last_refresh) is str:
last_refresh = dateutil.parser.parse(last_refresh)
nnow = datetime.datetime.now()
if self.NumHours is None:
self.NumHours = econ_platform_core.PlatformConfiguration['UpdateProtocol'].getint('SimpleHours')
age = math.floor(((nnow - last_refresh).total_seconds()) / (60 * 60))
if age < self.NumHours:
log_debug('Series {0} not stale, going to {1}'.format(ticker_str, database_manager.Code))
return database_manager.Retrieve(series_meta)
else:
# The adventure begins!
# For now, refresh entire series.
try:
return self.FetchAndWrite(ticker, series_meta, provider_wrapper, database_manager)
except NoDataError:
log_debug('Series {0} has no new data; marking refreshed'.format(ticker_str))
database_manager.SetLastRefresh(series_meta.ticker_full)
return database_manager.Retrieve(series_meta)
except PlatformError as ex:
# Unable to fetch; just retrieve from the database
# This is perhaps too broad, but we can use whatever is in the database.
econ_platform_core.log_last_error(just_info=True)
print('Could not fetch from provider; using database')
print('Explanation: ' + str(ex))
return database_manager.Retrieve(series_meta)