def fetch_covariance(self, factor=None, startdate=None, enddate=None, **kwargs): if isinstance(factor, str): factor = factor.find( '_') == -1 and self.prefix + '_' + factor or factor assert factor in self.all_factors[self.model] datetime_index = kwargs.get('datetime_index', self.datetime_index) prevra = kwargs.get('prevra', False) date_check = kwargs.get('date_check', self.date_check) if enddate: enddate = dateutil.compliment_datestring(str(enddate), 1, date_check) enddate = dateutil.parse_date(DATES, enddate, -1)[1] else: enddate = DATES[-1] if startdate: startdate = dateutil.compliment_datestring(str(startdate), -1, date_check) startdate = dateutil.parse_date(DATES, startdate, 1)[1] else: startdate = DATES[0] query = {'date': {'$lte': enddate, '$gte': startdate}} if isinstance(factor, str): query['factor'] = factor proj = {'_id': 0, 'date': 1, 'covariance': 1, 'factor': 1} if prevra: cursor = self.precov.find(query, proj) else: cursor = self.cov.find(query, proj) if factor: res = pd.DataFrame( {row['date']: row['covariance'] for row in cursor}).T if datetime_index: res.index = pd.to_datetime(res.index) else: res = pd.DataFrame({(row['date'], row['factor']): row['covariance'] for row in cursor}).T res = pd.Panel( {date: res.ix[date] for date in res.unstack().index}) if datetime_index: res.items = pd.to_datetime(res.items) del cursor return res
def fetch_intervals(self, dname, date, time, num=None, offset=0, **kwargs): """Return a consecutive interval data ``offset`` is to set offset of ``time``; along with ``date``, they determine the ending datetime. """ date_check = kwargs.get('date_check', self.date_check) reindex = kwargs.get('reindex', self.reindex) date = dateutil.compliment_datestring(str(date), -1, date_check) date = dateutil.parse_date(DATES, date, -1)[1] dateintervals = self.generate_dateintervals(date, time, num=1 if num is None else num, offset=offset) dateindex = pd.to_datetime([dis[0]+' '+dis[1] for dis in dateintervals]) window = [dis[0] for dis in dateintervals] query = {'dname': dname, 'date': {'$gte': window[0], '$lte': window[-1]}, } proj = {'_id': 0, 'dvalue': 1, 'date': 1, 'time': 1} cursor = self.collection.find(query, proj) df = pd.DataFrame({row['date']+' '+row['time']: row['dvalue'] for row in cursor}).T del cursor df.index = pd.to_datetime(df.index) df = df.ix[dateindex] if reindex: df = df.reindex(columns=SIDS) return df.iloc[0] if num is None else df
def fetch_intervals(self, dname, date, time, num=None, offset=0, **kwargs): """Return a consecutive interval data ``offset`` is to set offset of ``time``; along with ``date``, they determine the ending datetime. """ date_check = kwargs.get('date_check', self.date_check) reindex = kwargs.get('reindex', self.reindex) date = dateutil.compliment_datestring(str(date), -1, date_check) date = dateutil.parse_date(DATES, date, -1)[1] dateintervals = self.generate_dateintervals( date, time, num=1 if num is None else num, offset=offset) dateindex = pd.to_datetime( [dis[0] + ' ' + dis[1] for dis in dateintervals]) window = [dis[0] for dis in dateintervals] query = { 'dname': dname, 'date': { '$gte': window[0], '$lte': window[-1] }, } proj = {'_id': 0, 'dvalue': 1, 'date': 1, 'time': 1} cursor = self.collection.find(query, proj) df = pd.DataFrame( {row['date'] + ' ' + row['time']: row['dvalue'] for row in cursor}).T del cursor df.index = pd.to_datetime(df.index) df = df.ix[dateindex] if reindex: df = df.reindex(columns=SIDS) return df.iloc[0] if num is None else df
def record_fetch_history(df, date, backdays=None, delay=0, date_check=False): date = dateutil.compliment_datestring(str(date), -1, date_check) di, date = dateutil.parse_date(DATES, date, -1) di -= delay if backdays is None: window = DATES[:di+1] else: window = DATES[di-backdays+1: di+1] return AlphaBase.record_fetch_window(df, window)
def fetch_history(self, indicator, date, backdays, **kwargs): date_check = kwargs.get('date_check', self.date_check) delay = kwargs.get('delay', self.delay) date = dateutil.compliment_datestring(str(date), -1, date_check) di, date = dateutil.parse_date(DATES, date, -1) di -= delay window = DATES[di - backdays + 1:di + 1] return self.fetch_window(indicator, window, **kwargs)
def fetch_history(self, indicator, date, backdays, **kwargs): date_check = kwargs.get('date_check', self.date_check) delay = kwargs.get('delay', self.delay) date = dateutil.compliment_datestring(str(date), -1, date_check) di, date = dateutil.parse_date(DATES, date, -1) di -= delay window = DATES[di-backdays+1:di+1] return self.fetch_window(indicator, window, **kwargs)
def fetch_history(self, dname, times, date, backdays, **kwargs): """Use :py:meth:`fetch_window` behind the scene.""" date_check = kwargs.get('date_check', self.date_check) delay = kwargs.get('delay', self.delay) date = dateutil.compliment_datestring(str(date), -1, date_check) di, date = dateutil.parse_date(DATES, date, -1) di -= delay window = DATES[di-backdays+1: di+1] return self.fetch_window(dname, times, window, **kwargs)
def fetch_history(self, dname, times, date, backdays, **kwargs): """Use :py:meth:`fetch_window` behind the scene.""" date_check = kwargs.get('date_check', self.date_check) delay = kwargs.get('delay', self.delay) date = dateutil.compliment_datestring(str(date), -1, date_check) di, date = dateutil.parse_date(DATES, date, -1) di -= delay window = DATES[di - backdays + 1:di + 1] return self.fetch_window(dname, times, window, **kwargs)
def fetch_covariance(self, factor=None, startdate=None, enddate=None, **kwargs): if isinstance(factor, str): factor = factor.find('_') == -1 and self.prefix+'_'+factor or factor assert factor in self.all_factors[self.model] datetime_index = kwargs.get('datetime_index', self.datetime_index) prevra = kwargs.get('prevra', False) date_check = kwargs.get('date_check', self.date_check) if enddate: enddate = dateutil.compliment_datestring(str(enddate), 1, date_check) enddate = dateutil.parse_date(DATES, enddate, -1)[1] else: enddate = DATES[-1] if startdate: startdate = dateutil.compliment_datestring(str(startdate), -1, date_check) startdate = dateutil.parse_date(DATES, startdate, 1)[1] else: startdate = DATES[0] query = {'date': {'$lte': enddate, '$gte': startdate}} if isinstance(factor, str): query['factor'] = factor proj = {'_id': 0, 'date': 1, 'covariance': 1, 'factor': 1} if prevra: cursor = self.precov.find(query, proj) else: cursor = self.cov.find(query, proj) if factor: res = pd.DataFrame({row['date']: row['covariance'] for row in cursor}).T if datetime_index: res.index = pd.to_datetime(res.index) else: res = pd.DataFrame({(row['date'], row['factor']): row['covariance'] for row in cursor}).T res = pd.Panel({date: res.ix[date] for date in res.unstack().index}) if datetime_index: res.items = pd.to_datetime(res.items) del cursor return res
def fetch_daily(self, dname, date, offset=0, **kwargs): """Wrapper for :py:meth:`fetch_returns` and :py:meth:`fetch_covariance`. :param str dname: 'returns', 'covariance' """ date_check = kwargs.get('date_check', self.date_check) date = dateutil.compliment_datestring(date, -1, date_check) di, date = dateutil.parse_date(DATES, date, -1) date = DATES[di - offset] if dname == 'covariance': return self.fetch_daily_covariance(date) factor = kwargs.get('factor', None) return self.fetch_returns(factor, [date], **kwargs).iloc[0]
def fetch_daily(self, dname, date, offset=0, **kwargs): """Wrapper for :py:meth:`fetch_returns` and :py:meth:`fetch_covariance`. :param str dname: 'returns', 'covariance' """ date_check = kwargs.get('date_check', self.date_check) date = dateutil.compliment_datestring(date, -1, date_check) di, date = dateutil.parse_date(DATES, date, -1) date = DATES[di-offset] if dname == 'covariance': return self.fetch_daily_covariance(date) factor = kwargs.get('factor', None) return self.fetch_returns(factor, [date], **kwargs).iloc[0]
def filter_daily(self, date, offset=0, parent=None, **kwargs): """Filter out a universe on a certain day. A helper method. :param date: The base point :type date: str, int :param int offset: The offset w.r.t. the ``date``. The actual date is calculated from ``date`` and ``offset``. Default: 0 :param DataFrame parent: The super- or parent-universe to be filtered. Default: None :returns: Series """ date = dateutil.compliment_datestring(str(date), -1, self.date_check) di, date = dateutil.parse_date(DATES, date, -1) date = DATES[di - offset] if isinstance(parent, pd.Series): parent = pd.DataFrame({date: parent}).T return self.filter(date, date, **kwargs).iloc[0]
def filter_daily(self, date, offset=0, parent=None, **kwargs): """Filter out a universe on a certain day. A helper method. :param date: The base point :type date: str, int :param int offset: The offset w.r.t. the ``date``. The actual date is calculated from ``date`` and ``offset``. Default: 0 :param DataFrame parent: The super- or parent-universe to be filtered. Default: None :returns: Series """ date = dateutil.compliment_datestring(str(date), -1, self.date_check) di, date = dateutil.parse_date(DATES, date, -1) date = DATES[di-offset] if isinstance(parent, pd.Series): parent = pd.DataFrame({date: parent}).T return self.filter(date, date, **kwargs).iloc[0]
def fetch_daily(self, *args, **kwargs): """This differs from the default :py:meth:`orca.mongo.base.KDayFetcher.fetch_daily` in only one aspect: when the ``dname`` is not given, this will fetch all factors exposure on ``date``. Also, you can provide one of ('industry', 'style') to fetch exposures to industry/style factors. :returns: Series(if a factor name is given), DataFrame(factor names are in the columns) """ factor, date, offset = None, None, 0 if 'offset' in kwargs: offset = int(kwargs.pop('offset')) # is the first argument a date? try: date = dateutil.compliment_datestring(str(args[0]), -1, True) # yes, it is a date if len(args) > 1: offset = int(args[1]) except ValueError: # the first argument is not a date, presumably, it is the factor name! factor, date = args[0], args[1] # offset provided as the 3rd argument? if len(args) > 2: offset = int(args[2]) if factor is not None and factor not in ('industry', 'style'): return super(BarraExposureFetcher, self).fetch_daily(*args, **kwargs) di, date = dateutil.parse_date(DATES, date, -1) date = DATES[di - offset] reindex = kwargs.get('reindex', self.reindex) query = {'date': date} proj = {'_id': 0, 'dname': 1, 'dvalue': 1} cursor = self.collection.find(query, proj) df = pd.DataFrame({row['dname']: row['dvalue'] for row in cursor}) del cursor if reindex: df = df.reindex(index=SIDS) if factor == 'industry': return df[BarraFetcher.industry_factors[self.model]] elif factor == 'style': return df[BarraFetcher.style_factors[self.model]] return df
def fetch_history(self, *args, **kwargs): date_check = kwargs.get('date_check', self.date_check) delay = kwargs.get('delay', self.delay) try: date = dateutil.compliment_datestring(args[0], -1, date_check) backdays = args[1] dname = self._dname except ValueError: date = dateutil.compliment_datestring(args[1], -1, date_check) backdays = args[2] dname = args[0] date = dateutil.compliment_datestring(date, -1, date_check) di, date = dateutil.parse_date(DATES, date, -1) di -= delay window = DATES[di - backdays + 1:di + 1] return self.fetch_window(dname, window, **kwargs)
def fetch_dates(self, dname, dates, rshift=0, lshift=0, **kwargs): """Use :py:meth:`fetch_window` behind the scene.""" dates_str = dateutil.to_datestr(dates) res, is_df = {}, False for dt, date in zip(dates, dates_str): di, date = dateutil.parse_date(DATES, date, -1) if di-lshift < 0 or di+rshift+1 > len(DATES): continue if rshift+lshift == 0: res[dt] = self.fetch_daily(dname, DATES[di-lshift], **kwargs) if isinstance(res[dt], pd.DataFrame): is_df = True else: res[dt] = self.fetch_window(dname, DATES[di-lshift: di+rshift+1], **kwargs) if rshift+lshift == 0: res = pd.Panel(res).transpose(1, 2, 0) if is_df else pd.DataFrame(res).T return res
def fetch_history(self, *args, **kwargs): date_check = kwargs.get('date_check', self.date_check) delay = kwargs.get('delay', self.delay) try: date = dateutil.compliment_datestring(args[0], -1, date_check) backdays = args[1] dname = self._dname except ValueError: date = dateutil.compliment_datestring(args[1], -1, date_check) backdays = args[2] dname = args[0] date = dateutil.compliment_datestring(date, -1, date_check) di, date = dateutil.parse_date(DATES, date, -1) di -= delay window = DATES[di-backdays+1: di+1] return self.fetch_window(dname, window, **kwargs)
def fetch_daily(self, *args, **kwargs): """This differs from the default :py:meth:`orca.mongo.base.KDayFetcher.fetch_daily` in only one aspect: when the ``dname`` is not given, this will fetch all factors exposure on ``date``. Also, you can provide one of ('industry', 'style') to fetch exposures to industry/style factors. :returns: Series(if a factor name is given), DataFrame(factor names are in the columns) """ factor, date, offset = None, None, 0 if 'offset' in kwargs: offset = int(kwargs.pop('offset')) # is the first argument a date? try: date = dateutil.compliment_datestring(str(args[0]), -1, True) # yes, it is a date if len(args) > 1: offset = int(args[1]) except ValueError: # the first argument is not a date, presumably, it is the factor name! factor, date = args[0], args[1] # offset provided as the 3rd argument? if len(args) > 2: offset = int(args[2]) if factor is not None and factor not in ('industry', 'style'): return super(BarraExposureFetcher, self).fetch_daily(*args, **kwargs) di, date = dateutil.parse_date(DATES, date, -1) date = DATES[di-offset] reindex = kwargs.get('reindex', self.reindex) query = {'date': date} proj = {'_id': 0, 'dname': 1, 'dvalue': 1} cursor = self.collection.find(query, proj) df = pd.DataFrame({row['dname']: row['dvalue'] for row in cursor}) del cursor if reindex: df = df.reindex(index=SIDS) if factor == 'industry': return df[BarraFetcher.industry_factors[self.model]] elif factor == 'style': return df[BarraFetcher.style_factors[self.model]] return df
def fetch_dates(self, dates, rshift=0, lshift=0, **kwargs): """Use :py:meth:`fetch_window` behind the scene.""" dates_str = dateutil.to_datestr(dates) res, is_df = {}, False for dt, date in zip(dates, dates_str): di, date = dateutil.parse_date(DATES, date, -1) if di - lshift < 0 or di + rshift + 1 > len(DATES): continue if rshift + lshift == 0: res[dt] = self.fetch_daily(DATES[di - lshift], **kwargs) if isinstance(res[dt], pd.DataFrame): is_df = True else: res[dt] = self.fetch_window(DATES[di - lshift:di + rshift + 1], **kwargs) if rshift + lshift == 0: res = pd.Panel(res).transpose(1, 2, 0) if is_df else pd.DataFrame(res).T return res
def fetch_idmaps(cls, date=None, barra_key=True): """Fetch barra id - local stock id correspondance. :param boolean barra_key: Whether to use barra ids as keys. Default: True :returns: A ``dict`` """ dates = cls.idmaps.distinct('date') if date is None: date = dates[-1] else: date = dateutil.parse_date(dates, dateutil.compliment_datestring(date, -1, True), -1)[1] query = {'date': str(date)} proj = {'_id': 0, 'idmaps': 1} dct = cls.idmaps.find_one(query, proj)['idmaps'] if barra_key: return dct inv_dct = {} for k, v in dct.iteritems(): if v not in inv_dct or inv_dct[v] > k: inv_dct[v] = k return inv_dct
def fetch_idmaps(cls, date=None, barra_key=True): """Fetch barra id - local stock id correspondance. :param boolean barra_key: Whether to use barra ids as keys. Default: True :returns: A ``dict`` """ dates = cls.idmaps.distinct('date') if date is None: date = dates[-1] else: date = dateutil.parse_date( dates, dateutil.compliment_datestring(date, -1, True), -1)[1] query = {'date': str(date)} proj = {'_id': 0, 'idmaps': 1} dct = cls.idmaps.find_one(query, proj)['idmaps'] if barra_key: return dct inv_dct = {} for k, v in dct.iteritems(): if v not in inv_dct or inv_dct[v] > k: inv_dct[v] = k return inv_dct
def fetch_info(self, dname='name', level=0, date=None, **kwargs): """Fetch industry-name/industry-index correspondance. :param str dname: 'name'(default): fetch industry-name mapping; 'index': fetch industry-index mapping :param int level: Which level of industry is of interest? Default: 0, all 3 levels' information are fetched :rtype: dict """ standard = kwargs.get('standard', self.standard) date_check = kwargs.get('date_check', self.date_check) if date is not None: date = dateutil.parse_date(DATES, dateutil.compliment_datestring(date, -1, date_check), -1)[1] else: date = self.info.distinct('date')[-1] query = {'standard': standard, 'date': date} if level == 0: query.update({'dname': 'industry_%s' % dname}) else: query.update({'dname': 'level%d_%s' % (level, dname)}) proj = {'_id': 0, 'dvalue': 1} return self.info.find_one(query, proj)['dvalue']
def interval_fetch_history(pl, times, date, backdays, delay=0, date_check=False): date = dateutil.compliment_datestring(str(date), -1, date_check) di, date = dateutil.parse_date(DATES, date, -1) di -= delay window = DATES[di-backdays+1: di+1] return AlphaBase.interval_fetch_window(pl, times, window)
def get_date(date, offset=0): """Convenient method to get date with offset.""" di = dateutil.parse_date(DATES, str(date))[0] if di >= offset and len(DATES)-1 >= di+offset: return DATES[di-offset] raise ValueError('Can\'t get date with the specified offset')
def get_date(date, offset=0): """Convenient method to get date with offset.""" di = dateutil.parse_date(DATES, str(date))[0] if di >= offset and len(DATES) - 1 >= di + offset: return DATES[di - offset] raise ValueError('Can\'t get date with the specified offset')