def fetch_covariance(self,
factor=None,
startdate=None,
enddate=None,
**kwargs):
if isinstance(factor, str):
factor = factor.find(
'_') == -1 and self.prefix + '_' + factor or factor
assert factor in self.all_factors[self.model]
datetime_index = kwargs.get('datetime_index', self.datetime_index)
prevra = kwargs.get('prevra', False)
date_check = kwargs.get('date_check', self.date_check)
if enddate:
enddate = dateutil.compliment_datestring(str(enddate), 1,
date_check)
enddate = dateutil.parse_date(DATES, enddate, -1)[1]
else:
enddate = DATES[-1]
if startdate:
startdate = dateutil.compliment_datestring(str(startdate), -1,
date_check)
startdate = dateutil.parse_date(DATES, startdate, 1)[1]
else:
startdate = DATES[0]
query = {'date': {'$lte': enddate, '$gte': startdate}}
if isinstance(factor, str):
query['factor'] = factor
proj = {'_id': 0, 'date': 1, 'covariance': 1, 'factor': 1}
if prevra:
cursor = self.precov.find(query, proj)
else:
cursor = self.cov.find(query, proj)
if factor:
res = pd.DataFrame(
{row['date']: row['covariance']
for row in cursor}).T
if datetime_index:
res.index = pd.to_datetime(res.index)
else:
res = pd.DataFrame({(row['date'], row['factor']): row['covariance']
for row in cursor}).T
res = pd.Panel(
{date: res.ix[date]
for date in res.unstack().index})
if datetime_index:
res.items = pd.to_datetime(res.items)
del cursor
return res
def fetch_intervals(self, dname, date, time, num=None, offset=0, **kwargs):
"""Return a consecutive interval data
``offset`` is to set offset of ``time``; along with ``date``, they determine the ending datetime.
"""
date_check = kwargs.get('date_check', self.date_check)
reindex = kwargs.get('reindex', self.reindex)
date = dateutil.compliment_datestring(str(date), -1, date_check)
date = dateutil.parse_date(DATES, date, -1)[1]
dateintervals = self.generate_dateintervals(date, time, num=1 if num is None else num, offset=offset)
dateindex = pd.to_datetime([dis[0]+' '+dis[1] for dis in dateintervals])
window = [dis[0] for dis in dateintervals]
query = {'dname': dname,
'date': {'$gte': window[0], '$lte': window[-1]},
}
proj = {'_id': 0, 'dvalue': 1, 'date': 1, 'time': 1}
cursor = self.collection.find(query, proj)
df = pd.DataFrame({row['date']+' '+row['time']: row['dvalue'] for row in cursor}).T
del cursor
df.index = pd.to_datetime(df.index)
df = df.ix[dateindex]
if reindex:
df = df.reindex(columns=SIDS)
return df.iloc[0] if num is None else df
def fetch_intervals(self, dname, date, time, num=None, offset=0, **kwargs):
"""Return a consecutive interval data
``offset`` is to set offset of ``time``; along with ``date``, they determine the ending datetime.
"""
date_check = kwargs.get('date_check', self.date_check)
reindex = kwargs.get('reindex', self.reindex)
date = dateutil.compliment_datestring(str(date), -1, date_check)
date = dateutil.parse_date(DATES, date, -1)[1]
dateintervals = self.generate_dateintervals(
date, time, num=1 if num is None else num, offset=offset)
dateindex = pd.to_datetime(
[dis[0] + ' ' + dis[1] for dis in dateintervals])
window = [dis[0] for dis in dateintervals]
query = {
'dname': dname,
'date': {
'$gte': window[0],
'$lte': window[-1]
},
}
proj = {'_id': 0, 'dvalue': 1, 'date': 1, 'time': 1}
cursor = self.collection.find(query, proj)
df = pd.DataFrame(
{row['date'] + ' ' + row['time']: row['dvalue']
for row in cursor}).T
del cursor
df.index = pd.to_datetime(df.index)
df = df.ix[dateindex]
if reindex:
df = df.reindex(columns=SIDS)
return df.iloc[0] if num is None else df
def record_fetch_history(df, date, backdays=None, delay=0, date_check=False):
date = dateutil.compliment_datestring(str(date), -1, date_check)
di, date = dateutil.parse_date(DATES, date, -1)
di -= delay
if backdays is None:
window = DATES[:di+1]
else:
window = DATES[di-backdays+1: di+1]
return AlphaBase.record_fetch_window(df, window)
def fetch_history(self, indicator, date, backdays, **kwargs):
date_check = kwargs.get('date_check', self.date_check)
delay = kwargs.get('delay', self.delay)
date = dateutil.compliment_datestring(str(date), -1, date_check)
di, date = dateutil.parse_date(DATES, date, -1)
di -= delay
window = DATES[di - backdays + 1:di + 1]
return self.fetch_window(indicator, window, **kwargs)
def fetch_history(self, indicator, date, backdays, **kwargs):
date_check = kwargs.get('date_check', self.date_check)
delay = kwargs.get('delay', self.delay)
date = dateutil.compliment_datestring(str(date), -1, date_check)
di, date = dateutil.parse_date(DATES, date, -1)
di -= delay
window = DATES[di-backdays+1:di+1]
return self.fetch_window(indicator, window, **kwargs)
def fetch_history(self, dname, times, date, backdays, **kwargs):
"""Use :py:meth:`fetch_window` behind the scene."""
date_check = kwargs.get('date_check', self.date_check)
delay = kwargs.get('delay', self.delay)
date = dateutil.compliment_datestring(str(date), -1, date_check)
di, date = dateutil.parse_date(DATES, date, -1)
di -= delay
window = DATES[di-backdays+1: di+1]
return self.fetch_window(dname, times, window, **kwargs)
def fetch_history(self, dname, times, date, backdays, **kwargs):
"""Use :py:meth:`fetch_window` behind the scene."""
date_check = kwargs.get('date_check', self.date_check)
delay = kwargs.get('delay', self.delay)
date = dateutil.compliment_datestring(str(date), -1, date_check)
di, date = dateutil.parse_date(DATES, date, -1)
di -= delay
window = DATES[di - backdays + 1:di + 1]
return self.fetch_window(dname, times, window, **kwargs)
def fetch_covariance(self, factor=None, startdate=None, enddate=None, **kwargs):
if isinstance(factor, str):
factor = factor.find('_') == -1 and self.prefix+'_'+factor or factor
assert factor in self.all_factors[self.model]
datetime_index = kwargs.get('datetime_index', self.datetime_index)
prevra = kwargs.get('prevra', False)
date_check = kwargs.get('date_check', self.date_check)
if enddate:
enddate = dateutil.compliment_datestring(str(enddate), 1, date_check)
enddate = dateutil.parse_date(DATES, enddate, -1)[1]
else:
enddate = DATES[-1]
if startdate:
startdate = dateutil.compliment_datestring(str(startdate), -1, date_check)
startdate = dateutil.parse_date(DATES, startdate, 1)[1]
else:
startdate = DATES[0]
query = {'date': {'$lte': enddate, '$gte': startdate}}
if isinstance(factor, str):
query['factor'] = factor
proj = {'_id': 0, 'date': 1, 'covariance': 1, 'factor': 1}
if prevra:
cursor = self.precov.find(query, proj)
else:
cursor = self.cov.find(query, proj)
if factor:
res = pd.DataFrame({row['date']: row['covariance'] for row in cursor}).T
if datetime_index:
res.index = pd.to_datetime(res.index)
else:
res = pd.DataFrame({(row['date'], row['factor']): row['covariance'] for row in cursor}).T
res = pd.Panel({date: res.ix[date] for date in res.unstack().index})
if datetime_index:
res.items = pd.to_datetime(res.items)
del cursor
return res
def fetch_daily(self, dname, date, offset=0, **kwargs):
"""Wrapper for :py:meth:`fetch_returns` and :py:meth:`fetch_covariance`.
:param str dname: 'returns', 'covariance'
"""
date_check = kwargs.get('date_check', self.date_check)
date = dateutil.compliment_datestring(date, -1, date_check)
di, date = dateutil.parse_date(DATES, date, -1)
date = DATES[di - offset]
if dname == 'covariance':
return self.fetch_daily_covariance(date)
factor = kwargs.get('factor', None)
return self.fetch_returns(factor, [date], **kwargs).iloc[0]
def fetch_daily(self, dname, date, offset=0, **kwargs):
"""Wrapper for :py:meth:`fetch_returns` and :py:meth:`fetch_covariance`.
:param str dname: 'returns', 'covariance'
"""
date_check = kwargs.get('date_check', self.date_check)
date = dateutil.compliment_datestring(date, -1, date_check)
di, date = dateutil.parse_date(DATES, date, -1)
date = DATES[di-offset]
if dname == 'covariance':
return self.fetch_daily_covariance(date)
factor = kwargs.get('factor', None)
return self.fetch_returns(factor, [date], **kwargs).iloc[0]
def filter_daily(self, date, offset=0, parent=None, **kwargs):
"""Filter out a universe on a certain day. A helper method.
:param date: The base point
:type date: str, int
:param int offset: The offset w.r.t. the ``date``. The actual date is calculated from ``date`` and ``offset``. Default: 0
:param DataFrame parent: The super- or parent-universe to be filtered. Default: None
:returns: Series
"""
date = dateutil.compliment_datestring(str(date), -1, self.date_check)
di, date = dateutil.parse_date(DATES, date, -1)
date = DATES[di - offset]
if isinstance(parent, pd.Series):
parent = pd.DataFrame({date: parent}).T
return self.filter(date, date, **kwargs).iloc[0]
def filter_daily(self, date, offset=0, parent=None, **kwargs):
"""Filter out a universe on a certain day. A helper method.
:param date: The base point
:type date: str, int
:param int offset: The offset w.r.t. the ``date``. The actual date is calculated from ``date`` and ``offset``. Default: 0
:param DataFrame parent: The super- or parent-universe to be filtered. Default: None
:returns: Series
"""
date = dateutil.compliment_datestring(str(date), -1, self.date_check)
di, date = dateutil.parse_date(DATES, date, -1)
date = DATES[di-offset]
if isinstance(parent, pd.Series):
parent = pd.DataFrame({date: parent}).T
return self.filter(date, date, **kwargs).iloc[0]
def fetch_daily(self, *args, **kwargs):
"""This differs from the default :py:meth:`orca.mongo.base.KDayFetcher.fetch_daily` in only
one aspect: when the ``dname`` is not given, this will fetch all factors exposure on ``date``.
Also, you can provide one of ('industry', 'style') to fetch exposures to industry/style factors.
:returns: Series(if a factor name is given), DataFrame(factor names are in the columns)
"""
factor, date, offset = None, None, 0
if 'offset' in kwargs:
offset = int(kwargs.pop('offset'))
# is the first argument a date?
try:
date = dateutil.compliment_datestring(str(args[0]), -1, True)
# yes, it is a date
if len(args) > 1:
offset = int(args[1])
except ValueError:
# the first argument is not a date, presumably, it is the factor name!
factor, date = args[0], args[1]
# offset provided as the 3rd argument?
if len(args) > 2:
offset = int(args[2])
if factor is not None and factor not in ('industry', 'style'):
return super(BarraExposureFetcher,
self).fetch_daily(*args, **kwargs)
di, date = dateutil.parse_date(DATES, date, -1)
date = DATES[di - offset]
reindex = kwargs.get('reindex', self.reindex)
query = {'date': date}
proj = {'_id': 0, 'dname': 1, 'dvalue': 1}
cursor = self.collection.find(query, proj)
df = pd.DataFrame({row['dname']: row['dvalue'] for row in cursor})
del cursor
if reindex:
df = df.reindex(index=SIDS)
if factor == 'industry':
return df[BarraFetcher.industry_factors[self.model]]
elif factor == 'style':
return df[BarraFetcher.style_factors[self.model]]
return df
def fetch_history(self, *args, **kwargs):
date_check = kwargs.get('date_check', self.date_check)
delay = kwargs.get('delay', self.delay)
try:
date = dateutil.compliment_datestring(args[0], -1, date_check)
backdays = args[1]
dname = self._dname
except ValueError:
date = dateutil.compliment_datestring(args[1], -1, date_check)
backdays = args[2]
dname = args[0]
date = dateutil.compliment_datestring(date, -1, date_check)
di, date = dateutil.parse_date(DATES, date, -1)
di -= delay
window = DATES[di - backdays + 1:di + 1]
return self.fetch_window(dname, window, **kwargs)
def fetch_dates(self, dname, dates, rshift=0, lshift=0, **kwargs):
"""Use :py:meth:`fetch_window` behind the scene."""
dates_str = dateutil.to_datestr(dates)
res, is_df = {}, False
for dt, date in zip(dates, dates_str):
di, date = dateutil.parse_date(DATES, date, -1)
if di-lshift < 0 or di+rshift+1 > len(DATES):
continue
if rshift+lshift == 0:
res[dt] = self.fetch_daily(dname, DATES[di-lshift], **kwargs)
if isinstance(res[dt], pd.DataFrame):
is_df = True
else:
res[dt] = self.fetch_window(dname, DATES[di-lshift: di+rshift+1], **kwargs)
if rshift+lshift == 0:
res = pd.Panel(res).transpose(1, 2, 0) if is_df else pd.DataFrame(res).T
return res
def fetch_history(self, *args, **kwargs):
date_check = kwargs.get('date_check', self.date_check)
delay = kwargs.get('delay', self.delay)
try:
date = dateutil.compliment_datestring(args[0], -1, date_check)
backdays = args[1]
dname = self._dname
except ValueError:
date = dateutil.compliment_datestring(args[1], -1, date_check)
backdays = args[2]
dname = args[0]
date = dateutil.compliment_datestring(date, -1, date_check)
di, date = dateutil.parse_date(DATES, date, -1)
di -= delay
window = DATES[di-backdays+1: di+1]
return self.fetch_window(dname, window, **kwargs)
def fetch_daily(self, *args, **kwargs):
"""This differs from the default :py:meth:`orca.mongo.base.KDayFetcher.fetch_daily` in only
one aspect: when the ``dname`` is not given, this will fetch all factors exposure on ``date``.
Also, you can provide one of ('industry', 'style') to fetch exposures to industry/style factors.
:returns: Series(if a factor name is given), DataFrame(factor names are in the columns)
"""
factor, date, offset = None, None, 0
if 'offset' in kwargs:
offset = int(kwargs.pop('offset'))
# is the first argument a date?
try:
date = dateutil.compliment_datestring(str(args[0]), -1, True)
# yes, it is a date
if len(args) > 1:
offset = int(args[1])
except ValueError:
# the first argument is not a date, presumably, it is the factor name!
factor, date = args[0], args[1]
# offset provided as the 3rd argument?
if len(args) > 2:
offset = int(args[2])
if factor is not None and factor not in ('industry', 'style'):
return super(BarraExposureFetcher, self).fetch_daily(*args, **kwargs)
di, date = dateutil.parse_date(DATES, date, -1)
date = DATES[di-offset]
reindex = kwargs.get('reindex', self.reindex)
query = {'date': date}
proj = {'_id': 0, 'dname': 1, 'dvalue': 1}
cursor = self.collection.find(query, proj)
df = pd.DataFrame({row['dname']: row['dvalue'] for row in cursor})
del cursor
if reindex:
df = df.reindex(index=SIDS)
if factor == 'industry':
return df[BarraFetcher.industry_factors[self.model]]
elif factor == 'style':
return df[BarraFetcher.style_factors[self.model]]
return df
def fetch_dates(self, dates, rshift=0, lshift=0, **kwargs):
"""Use :py:meth:`fetch_window` behind the scene."""
dates_str = dateutil.to_datestr(dates)
res, is_df = {}, False
for dt, date in zip(dates, dates_str):
di, date = dateutil.parse_date(DATES, date, -1)
if di - lshift < 0 or di + rshift + 1 > len(DATES):
continue
if rshift + lshift == 0:
res[dt] = self.fetch_daily(DATES[di - lshift], **kwargs)
if isinstance(res[dt], pd.DataFrame):
is_df = True
else:
res[dt] = self.fetch_window(DATES[di - lshift:di + rshift + 1],
**kwargs)
if rshift + lshift == 0:
res = pd.Panel(res).transpose(1, 2,
0) if is_df else pd.DataFrame(res).T
return res
def fetch_idmaps(cls, date=None, barra_key=True):
"""Fetch barra id - local stock id correspondance.
:param boolean barra_key: Whether to use barra ids as keys. Default: True
:returns: A ``dict``
"""
dates = cls.idmaps.distinct('date')
if date is None:
date = dates[-1]
else:
date = dateutil.parse_date(dates, dateutil.compliment_datestring(date, -1, True), -1)[1]
query = {'date': str(date)}
proj = {'_id': 0, 'idmaps': 1}
dct = cls.idmaps.find_one(query, proj)['idmaps']
if barra_key:
return dct
inv_dct = {}
for k, v in dct.iteritems():
if v not in inv_dct or inv_dct[v] > k:
inv_dct[v] = k
return inv_dct
def fetch_idmaps(cls, date=None, barra_key=True):
"""Fetch barra id - local stock id correspondance.
:param boolean barra_key: Whether to use barra ids as keys. Default: True
:returns: A ``dict``
"""
dates = cls.idmaps.distinct('date')
if date is None:
date = dates[-1]
else:
date = dateutil.parse_date(
dates, dateutil.compliment_datestring(date, -1, True), -1)[1]
query = {'date': str(date)}
proj = {'_id': 0, 'idmaps': 1}
dct = cls.idmaps.find_one(query, proj)['idmaps']
if barra_key:
return dct
inv_dct = {}
for k, v in dct.iteritems():
if v not in inv_dct or inv_dct[v] > k:
inv_dct[v] = k
return inv_dct
def fetch_info(self, dname='name', level=0, date=None, **kwargs):
"""Fetch industry-name/industry-index correspondance.
:param str dname: 'name'(default): fetch industry-name mapping; 'index': fetch industry-index mapping
:param int level: Which level of industry is of interest? Default: 0, all 3 levels' information are fetched
:rtype: dict
"""
standard = kwargs.get('standard', self.standard)
date_check = kwargs.get('date_check', self.date_check)
if date is not None:
date = dateutil.parse_date(DATES, dateutil.compliment_datestring(date, -1, date_check), -1)[1]
else:
date = self.info.distinct('date')[-1]
query = {'standard': standard, 'date': date}
if level == 0:
query.update({'dname': 'industry_%s' % dname})
else:
query.update({'dname': 'level%d_%s' % (level, dname)})
proj = {'_id': 0, 'dvalue': 1}
return self.info.find_one(query, proj)['dvalue']
def interval_fetch_history(pl, times, date, backdays, delay=0, date_check=False):
date = dateutil.compliment_datestring(str(date), -1, date_check)
di, date = dateutil.parse_date(DATES, date, -1)
di -= delay
window = DATES[di-backdays+1: di+1]
return AlphaBase.interval_fetch_window(pl, times, window)
def get_date(date, offset=0):
"""Convenient method to get date with offset."""
di = dateutil.parse_date(DATES, str(date))[0]
if di >= offset and len(DATES)-1 >= di+offset:
return DATES[di-offset]
raise ValueError('Can\'t get date with the specified offset')
def get_date(date, offset=0):
"""Convenient method to get date with offset."""
di = dateutil.parse_date(DATES, str(date))[0]
if di >= offset and len(DATES) - 1 >= di + offset:
return DATES[di - offset]
raise ValueError('Can\'t get date with the specified offset')
