class ComponentsFetcherTestCase(unittest.TestCase):
def setUp(self):
self.fetcher = ComponentsFetcher()
self.dates = dateutil.get_startfrom(DATES, '20140101', 10)
def tearDown(self):
self.fetcher = None
def test_fetch_window1(self):
hs300 = self.fetcher.fetch_window('HS300', self.dates)
self.assertTrue((hs300.sum(axis=1) == 300).all())
def test_fetch_window2(self):
hs300 = self.fetcher.fetch_window('HS300', self.dates, as_bool=False)
self.assertTrue((np.abs(hs300.sum(axis=1) - 100) <= 1).all())
class ComponentsFetcherTestCase(unittest.TestCase):
def setUp(self):
self.fetcher = ComponentsFetcher()
self.dates = dateutil.get_startfrom(DATES, '20140101', 10)
def tearDown(self):
self.fetcher = None
def test_fetch_window1(self):
hs300 = self.fetcher.fetch_window('HS300', self.dates)
self.assertTrue((hs300.sum(axis=1) == 300).all())
def test_fetch_window2(self):
hs300 = self.fetcher.fetch_window('HS300', self.dates, as_bool=False)
self.assertTrue((np.abs(hs300.sum(axis=1)-100) <= 1).all())
import matplotlib.pyplot as plt
from matplotlib.dates import DateFormatter
datefmt = DateFormatter('%Y%m%d')
from matplotlib.backends.backend_pdf import PdfPages
import magic
from orca.mongo.industry import IndustryFetcher
from orca.mongo.index import IndexQuoteFetcher
from orca.mongo.components import ComponentsFetcher
from orca.mongo.sywgquote import SYWGQuoteFetcher
from orca.mongo.kday import UnivFetcher
industry_fetcher = IndustryFetcher(datetime_index=True, reindex=True)
indexquote_fetcher = IndexQuoteFetcher(datetime_index=True)
components_fetcher = ComponentsFetcher(datetime_index=True)
sywgquote_fetcher = SYWGQuoteFetcher(datetime_index=True, use_industry=True)
univ_fetcher = UnivFetcher(datetime_index=True, reindex=True)
from orca.utils import dateutil
from orca.utils.io import read_frame
from orca.operation import api
class Weight(object):
"""Class to analyse portfolio weight decomposition through time."""
def __init__(self, alpha, n, rank=None):
self.alpha = api.format(alpha)
self.rank_alpha = self.alpha.rank(axis=1, ascending=False)
self.rank_alpha = self.rank_alpha[self.rank_alpha <= n]
if rank is None:
import os
from string import Template
import pandas as pd
import logbook
logbook.set_datetime_format('local')
logger = logbook.Logger('composite')
from orca.mongo.barra import BarraFetcher
barra_fetcher = BarraFetcher('short')
from orca.mongo.components import ComponentsFetcher
components_fetcher = ComponentsFetcher(as_bool=False)
def generate_path(path_pattern, date, sid):
return Template(path_pattern).substitute(YYYYMMDD=date,
YYYYMM=date[:6],
YYYY=date[:4],
MM=date[4:6],
DD=date[6:8],
sid=sid)
def prep_composite_lance(account, date, sid, output):
bid_sid = barra_fetcher.fetch_idmaps(date)
path = os.path.join('/home/liulc/trade_' + account, 'barra', date[:4],
date[4:6], date[6:8], 'benchmark.' + date)
from orca.universe.special import (
TickerFilter,
TradingDaysFilter,
ActiveFilter,
ComponentsFilter,
IndustryFilter,
)
from orca.universe.rules import (
startswith,
count_gt,
)
window = dateutil.get_startfrom(DATES, '20140104', 50)
close = QuoteFetcher(datetime_index=True,
reindex=True).fetch_window('close', window)
hs300 = ComponentsFetcher(datetime_index=True,
reindex=True).fetch_window('HS300', window)
sector = IndustryFetcher(datetime_index=True,
reindex=True).fetch_window('sector', window)
class SpecialTestCase(unittest.TestCase):
def test_ticker_filter(self):
sh = TickerFilter(startswith('60')).filter_daily(window[0])
dct = {sid: sid[:2] == '60' for sid in SIDS}
self.assertEqual(sh.to_dict(), dct)
def test_trading_days_filter(self):
trd_filter = TradingDaysFilter(50, count_gt(40), delay=0)
trd1 = trd_filter.filter_daily(window[-1])
trd2 = close.count() > 40
self.assertTrue(series_equal(trd1, trd2))
"""
import os
from string import Template
import pandas as pd
import logbook
logbook.set_datetime_format('local')
logger = logbook.Logger('assets')
from orca.mongo.barra import BarraFetcher
barra_fetcher = BarraFetcher('short')
from orca.mongo.industry import IndustryFetcher
industry_fetcher = IndustryFetcher()
from orca.mongo.components import ComponentsFetcher
components_fetcher = ComponentsFetcher()
def get_board(sid):
if sid[:2] == '60':
return 'SH'
elif sid[:2] == '30':
return 'CYB'
elif sid[:3] == '002':
return 'ZXB'
else:
return 'SZ'
def generate_path(path_pattern, date):
return Template(path_pattern).substitute(YYYYMMDD=date, YYYYMM=date[:6], YYYY=date[:4], MM=date[4:6], DD=date[6:8])
def prep_group(group, name, date, output, add):
"""
import os
from string import Template
import pandas as pd
import logbook
logbook.set_datetime_format('local')
logger = logbook.Logger('assets')
from orca.mongo.barra import BarraFetcher
barra_fetcher = BarraFetcher('short')
from orca.mongo.industry import IndustryFetcher
industry_fetcher = IndustryFetcher()
from orca.mongo.components import ComponentsFetcher
components_fetcher = ComponentsFetcher()
def get_board(sid):
if sid[:2] == '60':
return 'SH'
elif sid[:2] == '30':
return 'CYB'
elif sid[:3] == '002':
return 'ZXB'
else:
return 'SZ'
def generate_path(path_pattern, date):
return Template(path_pattern).substitute(YYYYMMDD=date,
class Performance(object):
"""Class to provide analyser to examine the performance of an alpha from different perspective.
:param alpha: Alpha to be examined, either a well formatted DataFrame or :py:class:`orca.alpha.base.AlphaBase`
"""
mongo_lock = Lock()
quote = QuoteFetcher(datetime_index=True, reindex=True)
index_quote = IndexQuoteFetcher(datetime_index=True)
components = ComponentsFetcher(datetime_index=True, reindex=True)
returns = None
index_returns = {
'HS300': None,
}
index_components = {'HS300': None, 'CS500': None, 'other': None}
@classmethod
def get_returns(cls, startdate):
if cls.returns is None or startdate < cls.returns.index[0]:
with cls.mongo_lock:
cls.returns = cls.quote.fetch(
'returns', startdate=startdate.strftime('%Y%m%d'))
return cls.returns
@classmethod
def get_index_returns(cls, startdate, index='HS300'):
if index not in cls.index_returns or cls.index_returns[
index] is None or startdate < cls.index_returns[index].index[0]:
with cls.mongo_lock:
cls.index_returns[index] = cls.quote.fetch(
'returns', startdate=startdate.strftime('%Y%m%d'))
return cls.index_returns[index]
@classmethod
def get_index_components(cls, startdate, index):
if cls.index_components[
index] is None or startdate < cls.index_components[
index].index[0]:
with cls.mongo_lock:
cls.index_components['HS300'] = cls.components.fetch(
'HS300', startdate=startdate.strftime('%Y%m%d'))
cls.index_components['CS500'] = cls.components.fetch(
'CS500', startdate=startdate.strftime('%Y%m%d'))
cls.index_components['other'] = ~(
cls.index_components['HS300']
| cls.index_components['CS500'])
return cls.index_components[index]
@classmethod
def set_returns(cls, returns):
"""Call this method to set returns so that for future uses, there is no need to interact with MongoDB."""
with cls.mongo_lock:
cls.returns = api.format(returns)
@classmethod
def set_index_returns(cls, index, returns):
"""Call this method to set index returns so that for future uses, there is no need to interact with MongoDB."""
with cls.mongo_lock:
returns.index = pd.to_datetime(returns.index)
cls.index_returns[index] = returns
@classmethod
def set_index_components(cls, index, components):
"""Call this method to set index components data so that for future uses, there is no need to interact with MongoDB."""
with cls.mongo_lock:
cls.index_components[index] = api.format(components).fillna(False)
def __init__(self, alpha):
if isinstance(alpha, AlphaBase):
self.alpha = alpha.get_alphas()
else:
self.alpha = api.format(alpha)
self.alpha = self.alpha[np.isfinite(self.alpha)]
self.startdate = self.alpha.index[0]
def get_original(self):
"""**Be sure** to use this method when either the alpha is neutralized or you know what you are doing."""
return Analyser(self.alpha, Performance.get_returns(self.startdate))
def get_shift(self, n):
return Analyser(self.alpha.shift(n),
Performance.get_returns(self.alpha.index[n]))
def get_longshort(self):
"""Pretend the alpha can be made into a long/short portfolio."""
return Analyser(api.neutralize(self.alpha),
Performance.get_returns(self.startdate))
def get_long(self, index=None):
"""Only analyse the long part."""
return Analyser(self.alpha[self.alpha>0], Performance.get_returns(self.startdate),
Performance.get_index_returns(self.startdate, index=index)) \
if index is not None else \
Analyser(self.alpha[self.alpha>0], Performance.get_returns(self.startdate))
def get_short(self, index=None):
"""Only analyse the short part."""
return Analyser(-self.alpha[self.alpha<0], Performance.get_returns(self.startdate),
Performance.get_index_returns(self.startdate, index=index)) \
if index is not None else \
Analyser(-self.alpha[self.alpha<0], Performance.get_returns(self.startdate))
def get_qtop(self, q, index=None):
"""Only analyse the top quantile as long holding."""
return Analyser(api.qtop(self.alpha, q), Performance.get_returns(self.startdate),
Performance.get_index_returns(self.startdate, index=index)) \
if index is not None else \
Analyser(api.qtop(self.alpha, q), Performance.get_returns(self.startdate))
def get_qbottom(self, q, index=None):
"""Only analyse the bottom quantile as long holding."""
return Analyser(api.qbottom(self.alpha, q), Performance.get_returns(self.startdate),
Performance.get_index_returns(self.startdate, index=index)) \
if index is not None else \
Analyser(api.qbottom(self.alpha, q), Performance.get_returns(self.startdate))
def get_ntop(self, n, index=None):
"""Only analyse the top n stocks as long holding."""
return Analyser(api.top(self.alpha, n), Performance.get_returns(self.startdate),
Performance.get_index_returns(self.startdate, index=index)) \
if index is not None else \
Analyser(api.top(self.alpha, n), Performance.get_returns(self.startdate))
def get_nbottom(self, n, index=None):
"""Only analyse the bottom n stocks as long holding."""
return Analyser(api.bottom(self.alpha, n), Performance.get_returns(self.startdate),
Performance.get_index_returns(self.startdate, index=index)) \
if index is not None else \
Analyser(api.bottom(self.alpha, n), Performance.get_returns(self.startdate))
def get_qtail(self, q):
"""Long the top quantile and at the same time short the bottom quantile."""
return Analyser(
api.qtop(self.alpha, q).astype(int) -
api.qbottom(self.alpha, q).astype(int),
Performance.get_returns(self.startdate))
def get_ntail(self, n):
"""Long the top n stocks and at the same time short the bottom n stocks."""
return Analyser(
api.top(self.alpha, n).astype(int) -
api.bottom(self.alpha, n).astype(int),
Performance.get_returns(self.startdate))
def get_quantiles(self, n):
"""Return a list of analysers for n quantiles."""
return [Analyser(qt, Performance.get_returns(self.startdate)) \
for qt in api.quantiles(self.alpha, n)]
def get_universe(self, univ):
"""Return a performance object for alpha in this universe."""
return Performance(api.intersect(self.alpha, univ))
def get_bms(self):
"""Return a list of 3 performance objects for alphas in HS300, CS500 and other."""
big = Performance.get_index_components(self.startdate,
'HS300').ix[self.alpha.index]
mid = Performance.get_index_components(self.startdate,
'CS500').ix[self.alpha.index]
sml = Performance.get_index_components(self.startdate,
'other').ix[self.alpha.index]
return [self.get_universe(univ) for univ in [big, mid, sml]]
class IntPerformance(object):
"""Class to provide analyser to examine the performance of an interval alpha from different perspective.
:param alpha: Alpha to be examined, either a well formatted DataFrame or :py:class:`orca.alpha.base.AlphaBase`
"""
mongo_lock = Lock()
components = ComponentsFetcher(datetime_index=True, reindex=True)
returns = {
'1min': None,
'5min': None,
'30min': None,
}
index_returns = {
'HS300': {
'1min': None,
'5min': None,
'30min': None,
},
}
@classmethod
def get_returns(cls, startdate, freq):
if cls.returns[freq] is None or startdate < cls.returns[freq].index[
0].date():
with cls.mongo_lock:
returns_fetcher = IntervalReturnsFetcher(freq,
datetime_index=True,
reindex=True)
cls.returns[freq] = returns_fetcher.fetch(
[], startdate=startdate.strftime('%Y%m%d'), as_frame=True)
return cls.returns[freq]
@classmethod
def get_index_returns(cls, startdate, freq, index='HS300'):
if index not in cls.index_returns or cls.index_returns[index][
freq] is None or startdate < cls.index_returns[index][
freq].index[0].date():
with cls.mongo_lock:
index_returns_fetcher = IndexIntervalReturnsFetcher(
freq, datetime_index=True, reindex=True)
cls.index_returns[index] = index_returns_fetcher.fetch(
'returns', startdate=startdate.strftime('%Y%m%d'))
return cls.index_returns[index]
@classmethod
def set_returns(cls, freq, returns):
"""Call this method to set returns so that for future uses, there is no need to interact with MongoDB."""
with cls.mongo_lock:
cls.returns[freq] = api.format(returns)
@classmethod
def set_index_returns(cls, freq, index, returns):
"""Call this method to set index returns so that for future uses, there is no need to interact with MongoDB."""
with cls.mongo_lock:
returns.index = pd.to_datetime(returns.index)
cls.index_returns[index][freq] = returns
def __init__(self, alpha):
if isinstance(alpha, AlphaBase):
self.alpha = alpha.get_alphas()
else:
self.alpha = api.format(alpha)
self.dates = np.unique(self.alpha.index.date)
self.startdate = self.dates[0]
self.freq = len(self.alpha) / len(self.dates)
self.freq = str(240 / self.freq) + 'min'
def get_original(self):
"""**Be sure** to use this method when either the alpha is neutralized or you know what you are doing."""
return IntAnalyser(
self.alpha, IntPerformance.get_returns(self.startdate, self.freq))
def get_longshort(self):
"""Pretend the alpha can be made into a long/short portfolio."""
return IntAnalyser(
api.neutralize(self.alpha),
IntPerformance.get_returns(self.startdate, self.freq))
def get_long(self, index=None):
"""Only analyse the long part."""
return IntAnalyser(self.alpha[self.alpha>0], IntPerformance.get_returns(self.startdate, self.freq),
IntPerformance.get_index_returns(self.startdate, self.freq, index=index)) \
if index is not None else \
IntAnalyser(self.alpha[self.alpha>0], IntPerformance.get_returns(self.startdate, self.freq))
def get_short(self, index=None):
"""Only analyse the short part."""
return IntAnalyser(-self.alpha[self.alpha<0], IntPerformance.get_returns(self.startdate, self.freq),
IntPerformance.get_index_returns(self.startdate, self.freq, index=index)) \
if index is not None else \
IntAnalyser(-self.alpha[self.alpha<0], IntPerformance.get_returns(self.startdate, self.freq))
def get_qtop(self, q, index=None):
"""Only analyse the top quantile as long holding."""
return IntAnalyser(api.qtop(self.alpha, q), IntPerformance.get_returns(self.startdate, self.freq),
IntPerformance.get_index_returns(self.startdate, self.freq, index=index)) \
if index is not None else \
IntAnalyser(api.qtop(self.alpha, q), IntPerformance.get_returns(self.startdate, self.freq))
def get_qbottom(self, q, index=None):
"""Only analyse the bottom quantile as long holding."""
return IntAnalyser(api.qbottom(self.alpha, q), IntPerformance.get_returns(self.startdate, self.freq),
IntPerformance.get_index_returns(self.startdate, self.freq, index=index)) \
if index is not None else \
IntAnalyser(api.qbottom(self.alpha, q), IntPerformance.get_returns(self.startdate, self.freq))
def get_ntop(self, n, index=None):
"""Only analyse the top n stocks as long holding."""
return IntAnalyser(api.top(self.alpha, n), IntPerformance.get_returns(self.startdate, self.freq),
IntPerformance.get_index_returns(self.startdate, self.freq, index=index)) \
if index is not None else \
IntAnalyser(api.top(self.alpha, n), IntPerformance.get_returns(self.startdate, self.freq))
def get_nbottom(self, n, index=None):
"""Only analyse the bottom n stocks as long holding."""
return IntAnalyser(api.bottom(self.alpha, n), IntPerformance.get_returns(self.startdate, self.freq),
IntPerformance.get_index_returns(self.startdate, self.freq, index=index)) \
if index is not None else \
IntAnalyser(api.bottom(self.alpha, n), IntPerformance.get_returns(self.startdate, self.freq))
def get_qtail(self, q):
"""Long the top quantile and at the same time short the bottom quantile."""
return IntAnalyser(
api.qtop(self.alpha, q).astype(int) -
api.qbottom(self.alpha, q).astype(int),
IntPerformance.get_returns(self.startdate, self.freq))
def get_ntail(self, n):
"""Long the top n stocks and at the same time short the bottom n stocks."""
return IntAnalyser(
api.top(self.alpha, n).astype(int) -
api.bottom(self.alpha, n).astype(int),
IntPerformance.get_returns(self.startdate, self.freq))
def get_quantiles(self, n):
"""Return a list of analysers for n quantiles."""
return [IntAnalyser(qt, IntPerformance.get_returns(self.startdate, self.freq)) \
for qt in api.quantiles(self.alpha, n)]
def get_universe(self, univ):
"""Return a performance object for alpha in this universe."""
return IntPerformance(api.intersect_interval(self.alpha, univ))
"""
from orca.mongo.interval import AdjIntervalFetcher
from orca.alpha.base import BacktestingIntervalAlpha
adj = AdjIntervalFetcher('5min')
class MyAlpha(BacktestingIntervalAlpha):
def generate(self, date, time):
self[(date, time)] = adj.fetch_intervals('adj_returns',
date,
time,
offset=self.offset)
if __name__ == '__main__':
from orca.mongo.components import ComponentsFetcher
from orca.operation.api import intersect_interval
start, end = '20140103', '20140131'
alpha = MyAlpha('30min', offset=3)
alpha.run(start, end)
alpha = alpha.get_alphas()
comp = ComponentsFetcher(as_bool=True, datetime_index=True, reindex=True)
HS300 = comp.fetch('HS300', start, end)
alpha_hs300 = intersect_interval(alpha, HS300)
alpha_hs300.to_csv('alpha_int.csv')
"""
.. moduleauthor:: Li, Wang <*****@*****.**>
"""
import re
from itertools import product
import pandas as pd
from orca.perf.performance import Performance
from orca.mongo.kday import UnivFetcher, MiscFetcher
univ_fetcher = UnivFetcher(datetime_index=True, reindex=True)
misc_fetcher = MiscFetcher(datetime_index=True, reindex=True)
from orca.mongo.components import ComponentsFetcher
components_fetcher = ComponentsFetcher(datetime_index=True, reindex=True, as_bool=True)
from base import UpdaterBase
class PerformanceUpdater(UpdaterBase):
def __init__(self, timeout=3*60*60):
UpdaterBase.__init__(self, timeout)
def pre_update(self):
self.dates = self.db.dates.distinct('date')
if not self.skip_monitor:
self.connect_monitor()
def pro_update(self):
pass
def setUp(self):
self.fetcher = ComponentsFetcher()
self.dates = dateutil.get_startfrom(DATES, '20140101', 10)
def setUp(self):
self.fetcher = ComponentsFetcher()
self.dates = dateutil.get_startfrom(DATES, '20140101', 10)