def SearchForSeriesCode(self, series_code): """ :param series_code: str :return: pandas.DataFrame """ # Now for the ugly solution... flist = glob.glob(os.path.join(self.Directory, '*.xls')) # Search until we find it; if we don't, puke for fname in flist: # This query pattern will work on the "data" sheets; ignore the index. try: sheets = pandas.read_excel(fname, sheet_name=None, header=None, index_col=0) except: econ_platform_core.log_last_error() log_warning('Problem with Excel {0}'.format(fname)) continue for sheet_name in sheets: sheet = sheets[sheet_name] list_index = list(sheet.index) # We ignore sheets that do not match the desired format. if 'Series ID' not in list_index: continue for c in sheet.columns: if sheet[c]["Series ID"] == series_code: list_index[0] = 'series_name' sheet.index = list_index return sheet[c] # Did not find it; puke. raise econ_platform_core.entity_and_errors.TickerNotFoundError('Could not find series ID = {0}'.format(series_code))
def BuildTable(self): """ Get all series in all xls in directory (!). """ # Now for the ugly solution... flist = glob.glob(os.path.join(self.Directory, '*.xls')) # Search until we find it; if we don't, puke out = [] for fname in flist: log_debug('Reading %s', fname) # This query pattern will work on the "data" sheets; ignore the index. try: sheets = pandas.read_excel(fname, sheet_name=None, header=None, index_col=0) except: econ_platform_core.log_last_error() log_warning('Problem with Excel {0}'.format(fname)) continue for sheet_name in sheets: sheet = sheets[sheet_name] sheet = self.PatchSheet(sheet) list_index = list(sheet.index) # We ignore sheets that do not match the desired format. for targ_field in self.TickerLabels: if targ_field not in list_index: continue list_index = self.FixIndex(list_index) sheet.index = list_index for c in sheet.columns: try: [ser, meta] = self.ConvertDFtoSeries(sheet[c]) except SkipColumn: continue full_ticker = str(meta.ticker_full) if full_ticker in self.TableSeries: ser = ser.combine_first( self.TableSeries[full_ticker]) self.TableSeries[full_ticker] = ser else: self.TableSeries[full_ticker] = ser self.TableMeta[full_ticker] = meta
def SearchForSeriesCode(self, series_code): """ :param series_code: str :return: pandas.DataFrame """ # Now for the ugly solution... flist = glob.glob(os.path.join(self.Directory, '*.xls')) # Search until we find it; if we don't, puke out = [] for fname in flist: log_debug('Reading %s', fname) # This query pattern will work on the "data" sheets; ignore the index. try: sheets = pandas.read_excel(fname, sheet_name=None, header=None, index_col=0) except: econ_platform_core.log_last_error() log_warning('Problem with Excel {0}'.format(fname)) continue for sheet_name in sheets: sheet = sheets[sheet_name] list_index = list(sheet.index) # We ignore sheets that do not match the desired format. targ_field = self.TickerLabel if targ_field not in list_index: continue for c in sheet.columns: if sheet[c][targ_field] == series_code: # Fixes ABS spreadsheet list_index[0] = 'series_name' sheet.index = list_index out.append(sheet[c]) # Did not find it; puke. if len(out) == 0: raise econ_platform_core.TickerNotFoundError( 'Could not find series ID = {0}'.format(series_code)) else: return out
def Update(self, ticker, series_meta, provider_wrapper, database_manager): """ Procedure to handle updates. The default behaviour is to not update; just retrieve from the database. :param ticker: str :param series_meta: econ_platform_core.SeriesMetadata :param provider_wrapper: econ_platform_core.ProviderWrapper :param database_manager: econ_platform_core.DatabaseManager :return: """ last_refresh = series_meta.last_refresh ticker_str = ticker if last_refresh is None: last_refresh = database_manager.GetLastRefresh(series_meta.ticker_full) # If the developer is too lazy to parse strings... if type(last_refresh) is str: last_refresh = dateutil.parser.parse(last_refresh) nnow = datetime.datetime.now() if self.NumHours is None: self.NumHours = econ_platform_core.PlatformConfiguration['UpdateProtocol'].getint('SimpleHours') age = math.floor(((nnow - last_refresh).total_seconds()) / (60 * 60)) if age < self.NumHours: log_debug('Series {0} not stale, going to {1}'.format(ticker_str, database_manager.Code)) return database_manager.Retrieve(series_meta) else: # The adventure begins! # For now, refresh entire series. try: return self.FetchAndWrite(ticker, series_meta, provider_wrapper, database_manager) except NoDataError: log_debug('Series {0} has no new data; marking refreshed'.format(ticker_str)) database_manager.SetLastRefresh(series_meta.ticker_full) return database_manager.Retrieve(series_meta) except PlatformError as ex: # Unable to fetch; just retrieve from the database # This is perhaps too broad, but we can use whatever is in the database. econ_platform_core.log_last_error(just_info=True) print('Could not fetch from provider; using database') print('Explanation: ' + str(ex)) return database_manager.Retrieve(series_meta)