def write(self, data, length=None, show_progress=False, invalid_data_behavior='warn'): """Write a stream of minute data. Parameters ---------- data : iterable[(int, pd.DataFrame)] The data to write. Each element should be a tuple of sid, data where data has the following format: columns : ('open', 'high', 'low', 'close', 'volume') open : float64 high : float64 low : float64 close : float64 volume : float64|int64 index : DatetimeIndex of market minutes. A given sid may appear more than once in ``data``; however, the dates must be strictly increasing. show_progress : bool, optional Whether or not to show a progress bar while writing. """ with maybe_show_progress( data, length=length, show_percent=False, show_progress=show_progress, item_show_func=item_show_count(length), label='Compiling five-minute data', ) as it: write_sid = self.write_sid for e in it: write_sid(*e, invalid_data_behavior=invalid_data_behavior)
def _post_process_metadata(self, metadata, cache, show_progress=False): # Create empty data frame using target metadata column names and dtypes final_metadata = pd.DataFrame( columns=self.md_column_names, index=metadata.index, ) # Iterate over the available symbols, loading the asset's raw symbol # data from the cache. The final metadata is computed and recorded in # the appropriate row depending on the asset's id. with maybe_show_progress( metadata.symbol.iteritems(), show_progress, label='Post-processing symbol metadata', item_show_func=item_show_count(len(metadata)), length=len(metadata), show_percent=False, ) as symbols_map: for asset_id, symbol in symbols_map: # Attempt to load data from disk, the cache should have an entry # for each symbol at this point of the execution. If one does # not exist, we should fail. key = '{sym}.daily.frame'.format(sym=symbol) try: raw_data = cache[key] except KeyError: raise ValueError( 'Unable to find cached data for symbol: {0}'.format( symbol)) # Perform and require post-processing of metadata. final_symbol_metadata = self.post_process_symbol_metadata( asset_id, metadata.iloc[asset_id], raw_data, ) # Record symbol's final metadata. final_metadata.iloc[asset_id] = final_symbol_metadata # Register all assets with the bundle's default exchange. final_metadata['exchange'] = self.exchange return final_metadata
def _post_process_metadata(self, metadata, cache, show_progress=False): # Create empty data frame using target metadata column names and dtypes final_metadata = pd.DataFrame( columns=self.md_column_names, index=metadata.index, ) # Iterate over the available symbols, loading the asset's raw symbol # data from the cache. The final metadata is computed and recorded in # the appropriate row depending on the asset's id. with maybe_show_progress( metadata.symbol.iteritems(), show_progress, label='Post-processing symbol metadata', item_show_func=item_show_count(len(metadata)), length=len(metadata), show_percent=False, ) as symbols_map: for asset_id, symbol in symbols_map: # Attempt to load data from disk, the cache should have an # entry for each symbol at this point of the execution. If one # does not exist, we should fail. key = '{sym}.daily.frame'.format(sym=symbol) try: raw_data = cache[key] except KeyError: raise ValueError( 'Unable to find cached data for symbol:' ' {0}'.format(symbol)) # Perform and require post-processing of metadata. final_symbol_metadata = self.post_process_symbol_metadata( asset_id, metadata.iloc[asset_id], raw_data, ) # Record symbol's final metadata. final_metadata.iloc[asset_id] = final_symbol_metadata # Register all assets with the bundle's default exchange. final_metadata['exchange'] = self.exchange return final_metadata
def write(self, data, assets=None, show_progress=False, invalid_data_behavior='warn'): """ Parameters ---------- data : iterable[tuple[int, pandas.DataFrame or bcolz.ctable]] The data chunks to write. Each chunk should be a tuple of sid and the data for that asset. assets : set[int], optional The assets that should be in ``data``. If this is provided we will check ``data`` against the assets and provide better progress information. show_progress : bool, optional Whether or not to show a progress bar while writing. invalid_data_behavior : {'warn', 'raise', 'ignore'}, optional What to do when data is encountered that is outside the range of a uint64. Returns ------- table : bcolz.ctable The newly-written table. """ total = None if assets is None else len(assets) ctx = maybe_show_progress( ( (sid, self.to_ctable(df, invalid_data_behavior)) for sid, df in data ), show_progress=show_progress, label=self.progress_bar_message, item_show_func=item_show_count(total), length=total, show_percent=False, ) with ctx as it: return self._write_internal(it, assets)
def write(self, data, assets=None, show_progress=False, invalid_data_behavior='warn'): """ Parameters ---------- data : iterable[tuple[int, pandas.DataFrame or bcolz.ctable]] The data chunks to write. Each chunk should be a tuple of sid and the data for that asset. assets : set[int], optional The assets that should be in ``data``. If this is provided we will check ``data`` against the assets and provide better progress information. show_progress : bool, optional Whether or not to show a progress bar while writing. invalid_data_behavior : {'warn', 'raise', 'ignore'}, optional What to do when data is encountered that is outside the range of a uint64. Returns ------- table : bcolz.ctable The newly-written table. """ total = None if assets is None else len(assets) ctx = maybe_show_progress( ( (sid, self.to_ctable(df, invalid_data_behavior)) for sid, df in data ), show_progress=show_progress, label=self.progress_bar_message, item_show_func=item_show_count(total), length=total, show_percent=False, ) with ctx as it: return self._write_internal(it, assets)
def _fetch_metadata_frame(self, api_key, cache, retries=DEFAULT_RETRIES, environ=None, show_progress=False): # Setup raw metadata iterator to fetch pages if necessary. raw_iter = self._fetch_metadata_iter(api_key, cache, retries, environ) # Concatenate all frame in iterator to compute a single metadata frame. with maybe_show_progress( raw_iter, show_progress, label='Fetching symbol metadata', item_show_func=item_show_count(), length=3, show_percent=False, ) as blocks: metadata = pd.concat(blocks, ignore_index=True) return metadata
def _fetch_metadata_frame(self, api_key, cache, retries=DEFAULT_RETRIES, environ=None, show_progress=False): # Setup raw metadata iterator to fetch pages if necessary. raw_iter = self._fetch_metadata_iter(api_key, cache, retries, environ) # Concatenate all frame in iterator to compute a single metadata frame. with maybe_show_progress( raw_iter, show_progress, label='Fetching symbol metadata', item_show_func=item_show_count(), length=3, show_percent=False, ) as blocks: metadata = pd.concat(blocks, ignore_index=True) return metadata