def load_csv(token):
init = request.get_json()
r = Report(init['csv'], init['boy'])
db.set(token, pickle.dumps(r))
r = pickle.loads(db.get(token))
return json.dumps({ "stocks": r.stocks() });
def __init__(self, alg, **kwargs):
self._alg = alg
self._order = kwargs.pop('order', [])
self._cfg = kwargs.pop('cfg', {})
self._reserved = kwargs.pop('reserved', False)
self._debug = kwargs.pop('debug', False)
# hisframe collect args
self._kwargs = {
'opt': kwargs.pop('opt', None),
'starttime': kwargs.pop('starttime', datetime.utcnow() - timedelta(days=30)),
'endtime': kwargs.pop('endtime', datetime.utcnow()),
'targets': ['stock'],
'base': kwargs.pop('base', 'stock'),
'stockids': kwargs.pop('stockids', []),
'traderids': kwargs.pop('traderids', []),
'limit': kwargs.pop('limit', 10),
'debug': self._debug
}
db = "twsealgdb"
db = db if not self._debug else 'test' + db
host, port = MongoDBDriver._host, MongoDBDriver._port
connect(db, host=host, port=port, alias=db)
self._algcoll = switch(AlgStrategyColl, db)
self._id = TwseIdDBHandler(debug=self._debug, opt='twse')
self._report = Report(alg, sort=[('buys', False), ('sells', False), ('portfolio_value', False)], limit=100)
def run(opt='twse', debug=False, limit=0):
# set time window
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
report = Report(
algname=ZombieAlgorithm.__name__,
sort=[('buy_count', False), ('sell_count', False), ('volume', False)], limit=20)
# set debug or normal mode
kwargs = {
'debug': debug,
'limit': limit,
'opt': opt
}
idhandler = TwseIdDBHandler() if kwargs['opt'] == 'twse' else OtcIdDBHandler()
for stockid in idhandler.stock.get_ids(**kwargs):
dbhandler = TwseHisDBHandler() if kwargs['opt'] == 'twse' else OtcHisDBHandler()
dbhandler.stock.ids = [stockid]
data = dbhandler.transform_all_data(starttime, endtime, [stockid], [], 10)
if data.empty:
continue
zombie = ZombieAlgorithm(dbhandler=dbhandler)
results = zombie.run(data).fillna(0)
if results.empty:
continue
report.collect(stockid, results)
print stockid
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'zombie.html')
for stockid in report.iter_stockid():
stream = report.iter_report(stockid, dtype='html')
report.write(stream, "zombie_%s.html" % (stockid))
def run(opt='twse', debug=False, limit=0):
""" as doctest run """
maxlen = 30
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
report = Report(
sort=[('buy_count', False), ('sell_count', False), ('volume', False)], limit=20)
kwargs = {
'debug': debug,
'limit': limit,
'opt': opt
}
idhandler = TwseIdDBHandler(**kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs)
for stockid in idhandler.stock.get_ids():
try:
kwargs = {
'opt': opt,
'targets': ['stock'],
'starttime': starttime,
'endtime': endtime,
'stockids': [stockid],
'traderids': [],
'base': 'stock',
'callback': None,
'limit': 1,
'debug': debug
}
panel, dbhandler = collect_hisframe(**kwargs)
if len(panel[stockid].index) < maxlen:
continue
sim_params = SimulationParameters(
period_start=panel[stockid].index[0],
period_end=panel[stockid].index[-1],
data_frequency='daily',
emission_rate='daily'
)
kdtree = kdtKnnAlgorithm(dbhandler=dbhandler, sim_params=sim_params)
results = kdtree.run(data).fillna(0)
risks = kdtree.perf_tracker.handle_simulation_end()
report.collect(stockid, results, risks)
print "%s pass" %(stockid)
except:
print traceback.format_exc()
continue
if report.report.empty:
return
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'kdtknn.html')
for stockid in report.iter_stockid():
stream = report.iter_report(stockid, dtype='html', has_other=True, has_sideband=True)
report.write(stream, "kdtknn_%s.html" % (stockid))
def run(opt='twse', debug=False, limit=0):
maxlen = 30
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
report = Report(
'kmeans',
sort=[('buys', False), ('sells', False), ('portfolio_value', False)], limit=20)
# set debug or normal mode
kwargs = {
'debug': debug,
'limit': limit,
'opt': opt
}
idhandler = TwseIdDBHandler(**kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs)
for stockid in idhandler.stock.get_ids():
try:
kwargs = {
'opt': opt,
'targets': ['stock'],
'starttime': starttime,
'endtime': endtime,
'stockids': [stockid],
'base': 'stock',
'callback': None,
'limit': 1,
'debug': debug
}
panel, dbhandler = collect_hisframe(**kwargs)
if len(panel[stockid].index) < maxlen:
continue
sim_params = SimulationParameters(
period_start=panel[stockid].index[0],
period_end=panel[stockid].index[-1],
data_frequency='daily',
emission_rate='daily'
)
kmeans = Kmeans(dbhandler=dbhandler, debug=True, sim_params=sim_params)
results = kmeans.run(panel).fillna(0)
risks = kmeans.perf_tracker.handle_simulation_end()
report.collect(stockid, results, risks)
print "%s pass" %(stockid)
except:
print traceback.format_exc()
continue
if report.report.empty:
return
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'kmeans.html')
for stockid in report.iter_symbol():
stream = report.iter_report(stockid, dtype='html')
report.write(stream, "kmeans_%s.html" % (stockid))
def run(opt='twse', debug=False, limit=0):
""" as doctest run """
# set time window
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
# sort factor
report = Report(
algname=DualEMATaLib.__name__,
sort=[('buy_count', False), ('sell_count', False), ('volume', False)], limit=20)
# set debug or normal mode
kwargs = {
'debug': debug,
'limit': limit,
'opt': opt
}
idhandler = TwseIdDBHandler() if kwargs['opt'] == 'twse' else OtcIdDBHandler()
for stockid in idhandler.stock.get_ids(**kwargs):
dbhandler = TwseHisDBHandler() if kwargs['opt'] == 'twse' else OtcHisDBHandler()
dbhandler.stock.ids = [stockid]
data = dbhandler.transform_all_data(starttime, endtime, [stockid], [], 10)
if data.empty:
continue
dualema = DualEMATaLib(dbhandler=dbhandler)
results = dualema.run(data).fillna(0)
if results.empty:
continue
report.collect(stockid, results)
print stockid
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'dualema.html')
for stockid in report.iter_stockid():
stream = report.iter_report(stockid, dtype='html')
report.write(stream, "dualema_%s.html" % (stockid))
# plt
fig = plt.figure()
ax1 = fig.add_subplot(211, ylabel='portfolio value')
results.portfolio_value.plot(ax=ax1)
ax2 = fig.add_subplot(212)
results[['short_ema', 'long_ema']].plot(ax=ax2)
ax2.plot(results.ix[results.buy].index, results.short_ema[results.buy],
'^', markersize=10, color='m')
ax2.plot(results.ix[results.sell].index, results.short_ema[results.sell],
'v', markersize=10, color='k')
plt.legend(loc=0)
plt.gcf().set_size_inches(18, 8)
def main(debug=False, limit=0):
proc = start_service(debug)
# set time window
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
# sort factor
report = Report(
algname=GaussianHmmLib.__name__,
sort=[('buy_count', False), ('sell_count', False), ('volume', False)], limit=20)
# set debug or normal mode
kwargs = {
'debug': debug,
'limit': limit
}
for stockid in TwseIdDBQuery().get_stockids(**kwargs):
dbquery = TwseHisDBQuery()
data = dbquery.get_all_data(
starttime=starttime, endtime=endtime,
stockids=[stockid], traderids=[])
if data.empty:
continue
hmm = GaussianHmmLib(dbquery=dbquery)
hmm.run(data)
hmm.post_run()
def run(opt='twse', debug=False, limit=0):
""" as doctest run """
# set time window
starttime = datetime.utcnow() - timedelta(days=30)
endtime = datetime.utcnow()
# set debug or normal mode
kwargs = {
'debug': debug,
'limit': limit,
'opt': opt
}
# query topbuylist by each trader
# 1590:u'花旗環球', 1440:u'美林'
idhandler = TwseIdDBHandler() if kwargs['opt'] == 'twse' else OtcIdDBHandler()
for traderid in idhandler.trader.get_ids(**kwargs):
dbhandler = TwseHisdbhandler() if kwargs['opt'] == 'twse' else OtcHisDBHandler()
topdt = dbhandler.trader.gettoptrader_data(starttime, endtime, [traderid], 'trader', 'buy', 10)
report = Report(
algname=DarkManAlgorithm.__name__,
sort=[('buy_count', -1), ('sell_count', -1), ('ending_value', -1), ('close', -1)], limit=20)
kwargs = {
'debug': debug,
'limit': limit,
'opt': opt
}
stockids = dbhandler.trader.map_alias([traderid], 'trader', ['topbuy%d' % (i) for i in range(5)])
for stockid in stockids:
dbhandler = TwseHisdbhandler() if kwargs['opt'] == 'twse' else OtcHisDBHandler()
data = dbhandler.transform_all_data(starttime, endtime, [traderid], [stockid], 10)
if data.empty:
continue
darkman = DarkManAlgorithm(dbhandler=dbhandler)
results = darkman.run(data).fillna(0)
if results.empty:
continue
report.collect(stockid, results)
print traderid, stockid
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'darkman_%s.html' % (traderid))
for stockid in report.iter_stockid():
stream = report.iter_report(stockid, dtype='html', has_other=True, has_sideband=True)
report.write(stream, "darkman_%s_%s.html" % (traderid, stockid))
def run(opt='twse', debug=False, limit=0):
""" as doctest run """
# set time window
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
# sort factor
report = Report(
algname=SuperManAlgorithm.__name__,
sort=[('buy_count', False), ('sell_count', False), ('volume', False)], limit=20)
# set debug or normal mode
kwargs = {
'debug': debug,
'limit': limit,
'opt': opt
}
idhandler = TwseIdDBHandler() if kwargs['opt'] == 'twse' else OtcIdDBHandler()
for stockid in idhandler.stock.get_ids(**kwargs):
dbhandler = TwseHisDBHandler() if kwargs['opt'] == 'twse' else OtcHisDBHandler()
dbhandler.stock.ids = [stockid]
data = dbhandler.transform_all_data(starttime, endtime, [stockid], [], 10)
if data.empty:
continue
supman = SuperManAlgorithm(dbhandler=dbhandler)
results = supman.run(data).fillna(0)
if results.empty:
continue
report.collect(stockid, results)
print stockid
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'superman.html')
for stockid in report.iter_stockid():
stream = report.iter_report(stockid, dtype='html', has_other=True, has_sideband=True)
report.write(stream, "superman_%s.html" % (stockid))
class TwseAlgDBHandler(object):
def __init__(self, alg, **kwargs):
self._alg = alg
self._order = kwargs.pop('order', [])
self._cfg = kwargs.pop('cfg', {})
self._reserved = kwargs.pop('reserved', False)
self._debug = kwargs.pop('debug', False)
# hisframe collect args
self._kwargs = {
'opt': kwargs.pop('opt', None),
'starttime': kwargs.pop('starttime', datetime.utcnow() - timedelta(days=30)),
'endtime': kwargs.pop('endtime', datetime.utcnow()),
'targets': ['stock'],
'base': kwargs.pop('base', 'stock'),
'stockids': kwargs.pop('stockids', []),
'traderids': kwargs.pop('traderids', []),
'limit': kwargs.pop('limit', 10),
'debug': self._debug
}
db = "twsealgdb"
db = db if not self._debug else 'test' + db
host, port = MongoDBDriver._host, MongoDBDriver._port
connect(db, host=host, port=port, alias=db)
self._algcoll = switch(AlgStrategyColl, db)
self._id = TwseIdDBHandler(debug=self._debug, opt='twse')
self._report = Report(alg, sort=[('buys', False), ('sells', False), ('portfolio_value', False)], limit=100)
@property
def algcoll(self):
return self._algcoll
def iter_hisframe(self):
if not self._reserved:
stockids = self._kwargs['stockids']
for stockid in stockids:
self._kwargs.update({'stockids': [stockid]})
panel, handler = collect_hisframe(**copy.deepcopy(self._kwargs))
yield stockid, panel, handler
else:
panel, handler = collect_hisframe(**copy.deepcopy(self._kwargs))
yield stockid, panel, handler
def run(self):
for stockid, panel, dbhandler in self.iter_hisframe():
if not panel.empty and dbhandler:
sim_params = SimulationParameters(
period_start=panel[stockid].index[0],
period_end=panel[stockid].index[-1],
data_frequency='daily',
emission_rate='daily')
alg = self._alg(dbhandler, **self._cfg)
self._cfg = alg.cfg
results = alg.run(panel).fillna(0)
risks = alg.perf_tracker.handle_simulation_end()
self._report.collect("%s" %(stockid), results, risks)
def finalize(self, callback=None):
df = self._report.summary()
if callback == self.to_detail:
return callback(df)
if callback == self.insert_summary:
return callback(df)
return df
def delete_summary(self, uid):
pass
def insert_summary(self, df):
keys = [k for k,v in AlgSummaryColl._fields.iteritems() if k not in ['id', '_cls']]
toplist = []
names = df.columns.values.tolist()
algnm = self._alg.__name__.lower()
cfg = json.dumps(dict(self._cfg))
date = self._kwargs['endtime']
for k, v in df.T.to_dict().items():
data = { 'stockid': k }
data.update(v)
toplist.append(AlgSummaryColl(**data))
cursor = self._algcoll.objects(Q(algnm=algnm) & Q(cfg=cfg) & Q(date=date))
cursor = list(cursor)
coll = self._algcoll() if len(cursor) == 0 else cursor[0]
coll.cfg = cfg
coll.algnm = algnm
coll.date = date
coll.toplist = toplist
coll.save()
def to_detail(self, df):
retval = []
stockids = df.T.to_dict().keys()
keys = [k for k,v in AlgDetailColl._fields.iteritems() if k not in ['id', '_cls']]
for stockid in stockids:
df = self._report.iter_report(stockid)
names = df.columns.values.tolist()
for ix, cols in df.iterrows():
coll = {k:cols[k] for k in names if k in keys}
coll.update({
'date': ix,
'stockid': stockid
})
retval.append(coll)
return retval
def query_summary(self, starttime, endtime, cfg, constraint=None, order=None, limit=10, callback=None):
""" return orm
<algorithm>
<watchtime> |<bufwin> | <stockid> | <traderid> | buys | sells ...
20140928 | 5 | 2330 | null | 100 | 100
20140929 | 10 | 2317 | 1440 | 99 | 99
"""
map_f = """
function () {
try {
for(var i=0; i < this.toplist.length; i++) {
var key = { algnm: this.algnm, cfg: this.cfg, stockid: this.toplist[i].stockid };
var portfolio = this.toplist[i].portfolio_value;
var buys = this.toplist[i].buys;
var sells = this.toplist[i].sells;
var used = this.toplist[i].capital_used;
var alpha = this.toplist[i].alpha;
var beta = this.toplist[i].beta;
var sharpe = this.toplist[i].sharpe;
var max_drawdown = this.toplist[i].max_drawdown;
var benchmark = this.toplist[i].benchmark_period_return;
// as constraint/sort key
var value = {
portfolio: portfolio,
buys: buys,
sells: sells,
used: used,
alpha: alpha,
beta: beta,
sharpe: sharpe,
max_drawdown: max_drawdown,
benchmark: benchmark
};
emit(key, value);
}
}
catch(e){
}
finally{
}
}
"""
reduce_f = """
function (key, values) {
var redval = {
portfolio: 0,
buys: 0,
sells: 0,
used: 0,
alpha: 0,
beta: 0,
sharpe: 0,
max_drawdown: 0,
benchmark: 0
};
if (values.length == 1) {
redval.portfolio = values[0].portfolio;
redval.buys = values[0].buys;
redval.sells = values[0].sells;
redval.used = values[0].used;
redval.alpha = values[0].alpha;
redval.beta = values[0].beta;
redval.sharpe = values[0].sharpe;
redval.max_drawdown = values[0].max_drawdown;
redval.benchmark = values[0].benchmark;
}
return redval;
}
"""
finalize_f = """
"""
bufwin = (endtime - starttime).days
cursor = self._algcoll.objects(Q(date__gte=starttime) & Q(date__lte=endtime)).order_by('-date').limit(1)
results = list(cursor.map_reduce(map_f, reduce_f, 'algmap'))
if constraint:
results = filter(constraint, results)
if order:
results = sorted(results, key=order)[:limit]
retval = []
for it in results:
coll = {
# key
'algnm': it.key['algnm'],
'stockid': it.key['stockid'],
'stocknm': self._id.stock.get_name(it.key['stockid']),
'cfg': it.key['cfg'],
'endtime': endtime,
# value
'portfolio': it.value['portfolio'],
'buys': it.value['buys'],
'sells': it.value['sells'],
'used': it.value['used'],
'alpha': it.value['alpha'],
'beta': it.value['beta'],
'sharpe': it.value['sharpe'],
'max_drawdown': it.value['max_drawdown'],
'benchmark': it.value['benchmark']
}
retval.append(coll)
return callback(retval) if callback else retval
def to_pandas(self):
pass
def run(opt='twse', debug=False, limit=0):
""" as doctest run """
# set time window
starttime = datetime.utcnow() - timedelta(days=30)
endtime = datetime.utcnow()
# set debug or normal mode
kwargs = {'debug': debug, 'limit': limit, 'opt': opt}
# query topbuylist by each trader
# 1590:u'花旗環球', 1440:u'美林'
idhandler = TwseIdDBHandler(
) if kwargs['opt'] == 'twse' else OtcIdDBHandler()
for traderid in idhandler.trader.get_ids(**kwargs):
dbhandler = TwseHisdbhandler(
) if kwargs['opt'] == 'twse' else OtcHisDBHandler()
topdt = dbhandler.trader.gettoptrader_data(starttime, endtime,
[traderid], 'trader', 'buy',
10)
report = Report(algname=DarkManAlgorithm.__name__,
sort=[('buy_count', -1), ('sell_count', -1),
('ending_value', -1), ('close', -1)],
limit=20)
kwargs = {'debug': debug, 'limit': limit, 'opt': opt}
stockids = dbhandler.trader.map_alias(
[traderid], 'trader', ['topbuy%d' % (i) for i in range(5)])
for stockid in stockids:
dbhandler = TwseHisdbhandler(
) if kwargs['opt'] == 'twse' else OtcHisDBHandler()
data = dbhandler.transform_all_data(starttime, endtime, [traderid],
[stockid], 10)
if data.empty:
continue
darkman = DarkManAlgorithm(dbhandler=dbhandler)
results = darkman.run(data).fillna(0)
if results.empty:
continue
report.collect(stockid, results)
print traderid, stockid
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'darkman_%s.html' % (traderid))
for stockid in report.iter_stockid():
stream = report.iter_report(stockid,
dtype='html',
has_other=True,
has_sideband=True)
report.write(stream, "darkman_%s_%s.html" % (traderid, stockid))
def run(opt='twse', debug=False, limit=0):
""" as doctest run """
maxlen = 30
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
report = Report(
'dualema',
sort=[('buys', False), ('sells', False), ('portfolio_value', False)], limit=20)
kwargs = {
'debug': debug,
'limit': limit,
'opt': opt
}
# fetch
idhandler = TwseIdDBHandler(**kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs)
for stockid in idhandler.stock.get_ids():
try:
kwargs = {
'opt': opt,
'targets': ['stock'],
'starttime': starttime,
'endtime': endtime,
'stockids': [stockid],
'traderids': [],
'base': 'stock',
'callback': None,
'limit': 1,
'debug': debug
}
panel, dbhandler = collect_hisframe(**kwargs)
if len(panel[stockid].index) < maxlen:
continue
sim_params = SimulationParameters(
period_start=panel[stockid].index[0],
period_end=panel[stockid].index[-1],
data_frequency='daily',
emission_rate='daily'
)
dualema = DualEMA(dbhandler=dbhandler, debug=debug, sim_params=sim_params)
results = dualema.run(panel).fillna(0)
risks = dualema.perf_tracker.handle_simulation_end()
report.collect(stockid, results, risks)
print "%s pass" %(stockid)
except:
print traceback.format_exc()
continue
if report.report.empty:
return
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'dualema.html')
for stockid in report.iter_symbol():
stream = report.iter_report(stockid, dtype='html')
report.write(stream, "dualema_%s.html" % (stockid))
# plot
for stockid in report.iter_symbol():
try:
perf = report.pool[stockid]
fig = plt.figure()
ax1 = fig.add_subplot(211, ylabel='portfolio value')
perf.portfolio_value.plot(ax=ax1)
ax2 = fig.add_subplot(212)
perf[['short_ema', 'long_ema']].plot(ax=ax2)
ax2.plot(perf.ix[perf.buy].index, perf.short_ema[perf.buy],
'^', markersize=10, color='m')
ax2.plot(perf.ix[perf.sell].index, perf.short_ema[perf.sell],
'v', markersize=10, color='k')
plt.legend(loc=0)
plt.gcf().set_size_inches(18, 8)
plt.savefig("dualema_%s.png" %(stockid))
#plt.show()
except:
continue
def run(opt='twse', debug=False, limit=0):
maxlen = 30
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
report = Report(
'bbands',
sort=[('buys', False), ('sells', False), ('portfolio_value', False)], limit=20)
kwargs = {
'debug': debug,
'limit': limit,
'opt': opt
}
idhandler = TwseIdDBHandler(**kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs)
for stockid in idhandler.stock.get_ids():
try:
kwargs = {
'opt': opt,
'targets': ['stock', 'future', 'credit'],
'starttime': starttime,
'endtime': endtime,
'stockids': [stockid],
'traderids': [],
'base': 'stock',
'callback': None,
'limit': 1,
'debug': debug
}
panel, dbhandler = collect_hisframe(**kwargs)
if len(panel[stockid].index) < maxlen:
continue
sim_params = SimulationParameters(
period_start=panel[stockid].index[0],
period_end=panel[stockid].index[-1],
data_frequency='daily',
emission_rate='daily'
)
bbands = BBands(dbhandler=dbhandler, debug=debug, sim_params=sim_params)
results = bbands.run(panel).fillna(0)
risks = bbands.perf_tracker.handle_simulation_end()
report.collect(stockid, results, risks)
print "%s pass" %(stockid)
except:
print traceback.format_exc()
continue
if report.report.empty:
return
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'bbands.html')
for stockid in report.iter_symbol():
stream = report.iter_report(stockid, dtype='html')
report.write(stream, "bbands_%s.html" % (stockid))
for stockid in report.iter_symbol():
perf = report.pool[stockid]
dates = [date2num(i) for i in perf.index[maxlen:]]
quotes = [perf[label][maxlen:].values for label in ['open', 'high', 'low', 'close']]
quotes = zip(*([dates]+quotes))
fig = plt.figure(facecolor='#07000d')
ax1 = plt.subplot2grid((6,4), (1,0), rowspan=4, colspan=4, axisbg='#07000d')
candlestick_ohlc(ax1, quotes, width=.6, colorup='#53c156', colordown='#ff1717')
ax1.plot(dates, perf['upper'][maxlen:].values, '#e1edf9', label='upper', linewidth=1.5)
ax1.plot(dates, perf['middle'][maxlen:].values, '#e1edf9', label='middle', linewidth=1.5)
ax1.plot(dates, perf['lower'][maxlen:].values, '#e1edf9', label='lower', linewidth=1.5)
ax1.grid(True, color='w')
ax1.xaxis.set_major_locator(mticker.MaxNLocator(10))
ax1.xaxis.set_major_formatter(DateFormatter('%Y-%m-%d'))
ax1.yaxis.label.set_color("w")
ax1.spines['bottom'].set_color("#5998ff")
ax1.spines['top'].set_color("#5998ff")
ax1.spines['left'].set_color("#5998ff")
ax1.spines['right'].set_color("#5998ff")
ax1.tick_params(axis='y', colors='w')
plt.gca().yaxis.set_major_locator(mticker.MaxNLocator(prune='upper'))
ax1.tick_params(axis='x', colors='w')
plt.ylabel('Stock price and Volume')
bbLeg = plt.legend(loc=9, ncol=2, prop={'size':7}, fancybox=True, borderaxespad=0.)
bbLeg.get_frame().set_alpha(0.4)
textEd = pylab.gca().get_legend().get_texts()
pylab.setp(textEd[0:6], color = 'w')
ax1v = ax1.twinx()
ax1v.fill_between(dates, 0, perf['volume'][maxlen:].values, facecolor='#00ffe8', alpha=.4)
ax1v.axes.yaxis.set_ticklabels([])
ax1v.grid(False)
###Edit this to 3, so it's a bit larger
ax1v.set_ylim(0, 3*perf['volume'][maxlen:].values.max())
ax1v.spines['bottom'].set_color("#5998ff")
ax1v.spines['top'].set_color("#5998ff")
ax1v.spines['left'].set_color("#5998ff")
ax1v.spines['right'].set_color("#5998ff")
ax1v.tick_params(axis='x', colors='w')
ax1v.tick_params(axis='y', colors='w')
plt.setp(ax1.get_xticklabels(), visible=False)
plt.subplots_adjust(left=.09, bottom=.14, right=.94, top=.95, wspace=.20, hspace=0)
plt.gcf().set_size_inches(18, 8)
plt.savefig("bbands_%s.png" %(stockid), facecolor=fig.get_facecolor())
def run(opt='twse', debug=False, limit=0):
maxlen = 30
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
report = Report('bbands',
sort=[('buys', False), ('sells', False),
('portfolio_value', False)],
limit=20)
kwargs = {'debug': debug, 'limit': limit, 'opt': opt}
idhandler = TwseIdDBHandler(
**kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs)
for stockid in idhandler.stock.get_ids():
try:
kwargs = {
'opt': opt,
'targets': ['stock', 'future', 'credit'],
'starttime': starttime,
'endtime': endtime,
'stockids': [stockid],
'traderids': [],
'base': 'stock',
'callback': None,
'limit': 1,
'debug': debug
}
panel, dbhandler = collect_hisframe(**kwargs)
if len(panel[stockid].index) < maxlen:
continue
sim_params = SimulationParameters(
period_start=panel[stockid].index[0],
period_end=panel[stockid].index[-1],
data_frequency='daily',
emission_rate='daily')
bbands = BBands(dbhandler=dbhandler,
debug=debug,
sim_params=sim_params)
results = bbands.run(panel).fillna(0)
risks = bbands.perf_tracker.handle_simulation_end()
report.collect(stockid, results, risks)
print "%s pass" % (stockid)
except:
print traceback.format_exc()
continue
if report.report.empty:
return
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'bbands.html')
for stockid in report.iter_symbol():
stream = report.iter_report(stockid, dtype='html')
report.write(stream, "bbands_%s.html" % (stockid))
for stockid in report.iter_symbol():
perf = report.pool[stockid]
dates = [date2num(i) for i in perf.index[maxlen:]]
quotes = [
perf[label][maxlen:].values
for label in ['open', 'high', 'low', 'close']
]
quotes = zip(*([dates] + quotes))
fig = plt.figure(facecolor='#07000d')
ax1 = plt.subplot2grid((6, 4), (1, 0),
rowspan=4,
colspan=4,
axisbg='#07000d')
candlestick_ohlc(ax1,
quotes,
width=.6,
colorup='#53c156',
colordown='#ff1717')
ax1.plot(dates,
perf['upper'][maxlen:].values,
'#e1edf9',
label='upper',
linewidth=1.5)
ax1.plot(dates,
perf['middle'][maxlen:].values,
'#e1edf9',
label='middle',
linewidth=1.5)
ax1.plot(dates,
perf['lower'][maxlen:].values,
'#e1edf9',
label='lower',
linewidth=1.5)
ax1.grid(True, color='w')
ax1.xaxis.set_major_locator(mticker.MaxNLocator(10))
ax1.xaxis.set_major_formatter(DateFormatter('%Y-%m-%d'))
ax1.yaxis.label.set_color("w")
ax1.spines['bottom'].set_color("#5998ff")
ax1.spines['top'].set_color("#5998ff")
ax1.spines['left'].set_color("#5998ff")
ax1.spines['right'].set_color("#5998ff")
ax1.tick_params(axis='y', colors='w')
plt.gca().yaxis.set_major_locator(mticker.MaxNLocator(prune='upper'))
ax1.tick_params(axis='x', colors='w')
plt.ylabel('Stock price and Volume')
bbLeg = plt.legend(loc=9,
ncol=2,
prop={'size': 7},
fancybox=True,
borderaxespad=0.)
bbLeg.get_frame().set_alpha(0.4)
textEd = pylab.gca().get_legend().get_texts()
pylab.setp(textEd[0:6], color='w')
ax1v = ax1.twinx()
ax1v.fill_between(dates,
0,
perf['volume'][maxlen:].values,
facecolor='#00ffe8',
alpha=.4)
ax1v.axes.yaxis.set_ticklabels([])
ax1v.grid(False)
###Edit this to 3, so it's a bit larger
ax1v.set_ylim(0, 3 * perf['volume'][maxlen:].values.max())
ax1v.spines['bottom'].set_color("#5998ff")
ax1v.spines['top'].set_color("#5998ff")
ax1v.spines['left'].set_color("#5998ff")
ax1v.spines['right'].set_color("#5998ff")
ax1v.tick_params(axis='x', colors='w')
ax1v.tick_params(axis='y', colors='w')
plt.setp(ax1.get_xticklabels(), visible=False)
plt.subplots_adjust(left=.09,
bottom=.14,
right=.94,
top=.95,
wspace=.20,
hspace=0)
plt.gcf().set_size_inches(18, 8)
plt.savefig("bbands_%s.png" % (stockid), facecolor=fig.get_facecolor())
def run(opt='twse', debug=False, limit=0):
""" as doctest run """
maxlen = 30
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
report = Report('dualema',
sort=[('buys', False), ('sells', False),
('portfolio_value', False)],
limit=20)
kwargs = {'debug': debug, 'limit': limit, 'opt': opt}
# fetch
idhandler = TwseIdDBHandler(
**kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs)
for stockid in idhandler.stock.get_ids():
try:
kwargs = {
'opt': opt,
'targets': ['stock'],
'starttime': starttime,
'endtime': endtime,
'stockids': [stockid],
'traderids': [],
'base': 'stock',
'callback': None,
'limit': 1,
'debug': debug
}
panel, dbhandler = collect_hisframe(**kwargs)
if len(panel[stockid].index) < maxlen:
continue
sim_params = SimulationParameters(
period_start=panel[stockid].index[0],
period_end=panel[stockid].index[-1],
data_frequency='daily',
emission_rate='daily')
dualema = DualEMA(dbhandler=dbhandler,
debug=debug,
sim_params=sim_params)
results = dualema.run(panel).fillna(0)
risks = dualema.perf_tracker.handle_simulation_end()
report.collect(stockid, results, risks)
print "%s pass" % (stockid)
except:
print traceback.format_exc()
continue
if report.report.empty:
return
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'dualema.html')
for stockid in report.iter_symbol():
stream = report.iter_report(stockid, dtype='html')
report.write(stream, "dualema_%s.html" % (stockid))
# plot
for stockid in report.iter_symbol():
try:
perf = report.pool[stockid]
fig = plt.figure()
ax1 = fig.add_subplot(211, ylabel='portfolio value')
perf.portfolio_value.plot(ax=ax1)
ax2 = fig.add_subplot(212)
perf[['short_ema', 'long_ema']].plot(ax=ax2)
ax2.plot(perf.ix[perf.buy].index,
perf.short_ema[perf.buy],
'^',
markersize=10,
color='m')
ax2.plot(perf.ix[perf.sell].index,
perf.short_ema[perf.sell],
'v',
markersize=10,
color='k')
plt.legend(loc=0)
plt.gcf().set_size_inches(18, 8)
plt.savefig("dualema_%s.png" % (stockid))
#plt.show()
except:
continue
def run(opt='twse', debug=False, limit=0):
""" as doctest run """
maxlen = 5
starttime = datetime.utcnow() - timedelta(days=15)
endtime = datetime.utcnow()
report = Report('besttrader',
sort=[('buys', False), ('sells', False),
('portfolio_value', False)],
limit=20)
kwargs = {'debug': debug, 'limit': limit, 'opt': opt}
# 1590:u'花旗環球', 1440:u'美林'
traderid = '1440'
idhandler = TwseIdDBHandler(
**kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs)
for stockid in idhandler.stock.get_ids():
try:
# pre find traderid as top0
kwargs = {
'opt':
opt,
'targets': ['trader'],
'starttime':
starttime,
'endtime':
endtime,
'stockids': [stockid],
'traderids': [],
'base':
'stock',
'constraint':
lambda x: x.value["ebuyratio"] > 10 or x.value["totalkeepbuy"]
>= 1,
'order':
lambda x: [-x.value["totalvolume"], -x.value["totalbuyratio"]],
'callback':
None,
'limit':
10,
'debug':
debug
}
panel, dbhandler = collect_hisframe(**kwargs)
tops = list(
dbhandler.trader.get_alias([stockid], 'trader',
["top%d" % i for i in range(10)]))
if not tops:
continue
print "%s prefound:%s" % (stockid, tops)
traderid = tops[0] if traderid not in tops else traderid
# run
kwargs = {
'opt': opt,
'targets': ['stock', 'trader', 'future', 'credit'],
'starttime': starttime,
'endtime': endtime,
'stockids': [stockid],
'traderids': [traderid],
'base': 'trader',
'callback': None,
'limit': 10,
'debug': debug
}
panel, dbhandler = collect_hisframe(**kwargs)
if len(panel[stockid].index) < maxlen:
continue
sim_params = SimulationParameters(
period_start=panel[stockid].index[0],
period_end=panel[stockid].index[-1],
data_frequency='daily',
emission_rate='daily')
besttrader = BestTrader(dbhandler=dbhandler,
debug=debug,
sim_params=sim_params)
results = besttrader.run(panel).fillna(0)
risks = besttrader.perf_tracker.handle_simulation_end()
report.collect(stockid, results, risks)
print "%s pass" % (stockid)
except:
print traceback.format_exc()
continue
if report.report.empty:
return
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'besttrader.html')
for stockid in report.iter_symbol():
stream = report.iter_report(stockid, dtype='html')
report.write(stream, "besttrader_%s.html" % (stockid))
# plot
for stockid in report.iter_symbol():
try:
perf = report.pool[stockid]
fig = plt.figure()
ax1 = fig.add_subplot(211, ylabel='portfolio value')
perf.portfolio_value.plot(ax=ax1)
ax2 = fig.add_subplot(212)
perf[['close']].plot(ax=ax2)
ax2.plot(perf.ix[perf.buy].index,
perf.close[perf.buy],
'^',
markersize=10,
color='m')
ax2.plot(perf.ix[perf.sell].index,
perf.close[perf.sell],
'v',
markersize=10,
color='k')
plt.legend(loc=0)
plt.gcf().set_size_inches(18, 8)
plt.savefig("besttrader_%s_%s.png" % (traderid, stockid))
#plt.show()
except:
continue
def run(opt='twse', debug=False, limit=0):
maxlen = 30
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
report = Report('kmeans',
sort=[('buys', False), ('sells', False),
('portfolio_value', False)],
limit=20)
# set debug or normal mode
kwargs = {'debug': debug, 'limit': limit, 'opt': opt}
idhandler = TwseIdDBHandler(
**kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs)
for stockid in idhandler.stock.get_ids():
try:
kwargs = {
'opt': opt,
'targets': ['stock'],
'starttime': starttime,
'endtime': endtime,
'stockids': [stockid],
'base': 'stock',
'callback': None,
'limit': 1,
'debug': debug
}
panel, dbhandler = collect_hisframe(**kwargs)
if len(panel[stockid].index) < maxlen:
continue
sim_params = SimulationParameters(
period_start=panel[stockid].index[0],
period_end=panel[stockid].index[-1],
data_frequency='daily',
emission_rate='daily')
kmeans = Kmeans(dbhandler=dbhandler,
debug=True,
sim_params=sim_params)
results = kmeans.run(panel).fillna(0)
risks = kmeans.perf_tracker.handle_simulation_end()
report.collect(stockid, results, risks)
print "%s pass" % (stockid)
except:
print traceback.format_exc()
continue
if report.report.empty:
return
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'kmeans.html')
for stockid in report.iter_symbol():
stream = report.iter_report(stockid, dtype='html')
report.write(stream, "kmeans_%s.html" % (stockid))
def run(opt='twse', debug=False, limit=0):
""" as doctest run """
# set time window
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
# sort factor
report = Report(algname=DualEMATaLib.__name__,
sort=[('buy_count', False), ('sell_count', False),
('volume', False)],
limit=20)
# set debug or normal mode
kwargs = {'debug': debug, 'limit': limit, 'opt': opt}
idhandler = TwseIdDBHandler(
) if kwargs['opt'] == 'twse' else OtcIdDBHandler()
for stockid in idhandler.stock.get_ids(**kwargs):
dbhandler = TwseHisDBHandler(
) if kwargs['opt'] == 'twse' else OtcHisDBHandler()
dbhandler.stock.ids = [stockid]
data = dbhandler.transform_all_data(starttime, endtime, [stockid], [],
10)
if data.empty:
continue
dualema = DualEMATaLib(dbhandler=dbhandler)
results = dualema.run(data).fillna(0)
if results.empty:
continue
report.collect(stockid, results)
print stockid
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'dualema.html')
for stockid in report.iter_stockid():
stream = report.iter_report(stockid, dtype='html')
report.write(stream, "dualema_%s.html" % (stockid))
# plt
fig = plt.figure()
ax1 = fig.add_subplot(211, ylabel='portfolio value')
results.portfolio_value.plot(ax=ax1)
ax2 = fig.add_subplot(212)
results[['short_ema', 'long_ema']].plot(ax=ax2)
ax2.plot(results.ix[results.buy].index,
results.short_ema[results.buy],
'^',
markersize=10,
color='m')
ax2.plot(results.ix[results.sell].index,
results.short_ema[results.sell],
'v',
markersize=10,
color='k')
plt.legend(loc=0)
plt.gcf().set_size_inches(18, 8)
def run(opt='twse', debug=False, limit=0):
""" as doctest run """
maxlen = 30
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
report = Report(
'randforest',
sort=[('buys', False), ('sells', False), ('portfolio_value', False)], limit=20)
kwargs = {
'debug': debug,
'limit': limit,
'opt': opt
}
# fetch
idhandler = TwseIdDBHandler(**kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs)
for stockid in idhandler.stock.get_ids():
try:
kwargs = {
'opt': opt,
'targets': ['stock', 'future', 'credit'],
'starttime': starttime,
'endtime': endtime,
'stockids': [stockid],
'traderids': [],
'base': 'stock',
'order': [],
'callback': None,
'limit': 1,
'debug': debug
}
panel, dbhandler = collect_hisframe(**kwargs)
if len(panel[stockid].index) < maxlen:
continue
rforest = RandForest(dbhandler=dbhandler)
results = rforest.run(panel).fillna(0)
risks = rforest.perf_tracker.handle_simulation_end()
report.collect(stockid, results, risks)
print "%s pass" %(stockid)
except:
print traceback.format_exc()
continue
if report.report.empty:
return
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'randforest.html')
for stockid in report.iter_symbol():
stream = report.iter_report(stockid, dtype='html')
report.write(stream, "randforest_%s.html" % (stockid))
# plot
for stockid in report.iter_symbol():
try:
perf = report.pool[stockid]
fig = plt.figure()
ax1 = fig.add_subplot(211)
perf.portfolio_value.plot(ax=ax1)
ax1.set_ylabel('portfolio value in $')
ax2 = fig.add_subplot(212)
perf[['close', 'mavg7']].plot(ax=ax2)
perf_trans = perf.ix[[t != [] for t in perf.transactions]]
buys = perf_trans.ix[[t[0]['amount'] > 0 for t in perf_trans.transactions]]
sells = perf_trans.ix[[t[0]['amount'] < 0 for t in perf_trans.transactions]]
ax2.plot(buys.index, perf.close.ix[buys.index],
'^', markersize=10, color='m')
ax2.plot(sells.index, perf.close.ix[sells.index],
'v', markersize=10, color='k')
ax2.set_ylabel('price in $')
plt.legend(loc=0)
plt.savefig("randforest_%s.png" %(stockid))
#plt.show()
except:
continue
def run(opt='twse', debug=False, limit=0):
""" as doctest run """
maxlen = 30
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
report = Report(sort=[('buy_count', False), ('sell_count', False),
('volume', False)],
limit=20)
kwargs = {'debug': debug, 'limit': limit, 'opt': opt}
idhandler = TwseIdDBHandler(
**kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs)
for stockid in idhandler.stock.get_ids():
try:
kwargs = {
'opt': opt,
'targets': ['stock'],
'starttime': starttime,
'endtime': endtime,
'stockids': [stockid],
'traderids': [],
'base': 'stock',
'callback': None,
'limit': 1,
'debug': debug
}
panel, dbhandler = collect_hisframe(**kwargs)
if len(panel[stockid].index) < maxlen:
continue
sim_params = SimulationParameters(
period_start=panel[stockid].index[0],
period_end=panel[stockid].index[-1],
data_frequency='daily',
emission_rate='daily')
kdtree = kdtKnnAlgorithm(dbhandler=dbhandler,
sim_params=sim_params)
results = kdtree.run(data).fillna(0)
risks = kdtree.perf_tracker.handle_simulation_end()
report.collect(stockid, results, risks)
print "%s pass" % (stockid)
except:
print traceback.format_exc()
continue
if report.report.empty:
return
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'kdtknn.html')
for stockid in report.iter_stockid():
stream = report.iter_report(stockid,
dtype='html',
has_other=True,
has_sideband=True)
report.write(stream, "kdtknn_%s.html" % (stockid))