def run(opt='twse', debug=False, limit=0):
""" as doctest run """
# set time window
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
# sort factor
report = Report(algname=DualEMATaLib.__name__,
sort=[('buy_count', False), ('sell_count', False),
('volume', False)],
limit=20)
# set debug or normal mode
kwargs = {'debug': debug, 'limit': limit, 'opt': opt}
idhandler = TwseIdDBHandler(
) if kwargs['opt'] == 'twse' else OtcIdDBHandler()
for stockid in idhandler.stock.get_ids(**kwargs):
dbhandler = TwseHisDBHandler(
) if kwargs['opt'] == 'twse' else OtcHisDBHandler()
dbhandler.stock.ids = [stockid]
data = dbhandler.transform_all_data(starttime, endtime, [stockid], [],
10)
if data.empty:
continue
dualema = DualEMATaLib(dbhandler=dbhandler)
results = dualema.run(data).fillna(0)
if results.empty:
continue
report.collect(stockid, results)
print stockid
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'dualema.html')
for stockid in report.iter_stockid():
stream = report.iter_report(stockid, dtype='html')
report.write(stream, "dualema_%s.html" % (stockid))
# plt
fig = plt.figure()
ax1 = fig.add_subplot(211, ylabel='portfolio value')
results.portfolio_value.plot(ax=ax1)
ax2 = fig.add_subplot(212)
results[['short_ema', 'long_ema']].plot(ax=ax2)
ax2.plot(results.ix[results.buy].index,
results.short_ema[results.buy],
'^',
markersize=10,
color='m')
ax2.plot(results.ix[results.sell].index,
results.short_ema[results.sell],
'v',
markersize=10,
color='k')
plt.legend(loc=0)
plt.gcf().set_size_inches(18, 8)
def __init__(self, crawler):
super(TwseHisTraderSpider, self).__init__()
kwargs = {
'debug': crawler.settings.getbool('GIANT_DEBUG'),
'limit': crawler.settings.getint('GIANT_LIMIT'),
# 'slice': crawler.settings.getint('GIANT_SLICE'),
'opt': 'twse'
}
self._id = TwseIdDBHandler(**kwargs)
def __init__(self, crawler):
super(TwseHisFutureSpider, self).__init__()
kwargs = {
'debug': crawler.settings.getbool('GIANT_DEBUG'),
'limit': crawler.settings.getint('GIANT_LIMIT'),
'opt': 'twse'
}
self._id = TwseIdDBHandler(**kwargs)
self._table = {}
def run(opt='twse', debug=False, limit=0):
maxlen = 30
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
report = Report('kmeans',
sort=[('buys', False), ('sells', False),
('portfolio_value', False)],
limit=20)
# set debug or normal mode
kwargs = {'debug': debug, 'limit': limit, 'opt': opt}
idhandler = TwseIdDBHandler(
**kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs)
for stockid in idhandler.stock.get_ids():
try:
kwargs = {
'opt': opt,
'targets': ['stock'],
'starttime': starttime,
'endtime': endtime,
'stockids': [stockid],
'base': 'stock',
'callback': None,
'limit': 1,
'debug': debug
}
panel, dbhandler = collect_hisframe(**kwargs)
if len(panel[stockid].index) < maxlen:
continue
sim_params = SimulationParameters(
period_start=panel[stockid].index[0],
period_end=panel[stockid].index[-1],
data_frequency='daily',
emission_rate='daily')
kmeans = Kmeans(dbhandler=dbhandler,
debug=True,
sim_params=sim_params)
results = kmeans.run(panel).fillna(0)
risks = kmeans.perf_tracker.handle_simulation_end()
report.collect(stockid, results, risks)
print "%s pass" % (stockid)
except:
print traceback.format_exc()
continue
if report.report.empty:
return
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'kmeans.html')
for stockid in report.iter_symbol():
stream = report.iter_report(stockid, dtype='html')
report.write(stream, "kmeans_%s.html" % (stockid))
def main():
stockids = TwseIdDBHandler().stock.get_ids(debug=True)
stockids = list(stockids)
t = timeit.Timer()
args = ('twsehistrader', 'INFO', 'twsehistrader.log', True, True)
run_scrapy_service.delay(*args).get()
print "run trader scrapy using (%.4f)s/%d" % (t.timeit(), len(stockids))
t = timeit.Timer()
args = ('twsehisstock', 'INFO', 'twsehisstock.log', True, True)
run_scrapy_service.delay(*args).get()
print "run stock scrapy using (%.4f)s/%d" % (t.timeit(), len(stockids))
def run(opt='twse', debug=False, limit=0):
""" as doctest run """
# set time window
starttime = datetime.utcnow() - timedelta(days=30)
endtime = datetime.utcnow()
# set debug or normal mode
kwargs = {'debug': debug, 'limit': limit, 'opt': opt}
# query topbuylist by each trader
# 1590:u'花旗環球', 1440:u'美林'
idhandler = TwseIdDBHandler(
) if kwargs['opt'] == 'twse' else OtcIdDBHandler()
for traderid in idhandler.trader.get_ids(**kwargs):
dbhandler = TwseHisdbhandler(
) if kwargs['opt'] == 'twse' else OtcHisDBHandler()
topdt = dbhandler.trader.gettoptrader_data(starttime, endtime,
[traderid], 'trader', 'buy',
10)
report = Report(algname=DarkManAlgorithm.__name__,
sort=[('buy_count', -1), ('sell_count', -1),
('ending_value', -1), ('close', -1)],
limit=20)
kwargs = {'debug': debug, 'limit': limit, 'opt': opt}
stockids = dbhandler.trader.map_alias(
[traderid], 'trader', ['topbuy%d' % (i) for i in range(5)])
for stockid in stockids:
dbhandler = TwseHisdbhandler(
) if kwargs['opt'] == 'twse' else OtcHisDBHandler()
data = dbhandler.transform_all_data(starttime, endtime, [traderid],
[stockid], 10)
if data.empty:
continue
darkman = DarkManAlgorithm(dbhandler=dbhandler)
results = darkman.run(data).fillna(0)
if results.empty:
continue
report.collect(stockid, results)
print traderid, stockid
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'darkman_%s.html' % (traderid))
for stockid in report.iter_stockid():
stream = report.iter_report(stockid,
dtype='html',
has_other=True,
has_sideband=True)
report.write(stream, "darkman_%s_%s.html" % (traderid, stockid))
def start_requests(self):
kwargs = {
'debug': self.settings.getbool('GIANT_DEBUG'),
'limit': self.settings.getint('GIANT_LIMIT'),
'opt': 'twse'
}
requests = []
for stockid in TwseIdDBHandler().stock.get_ids(**kwargs):
URL = 'http://bsr.twse.com.tw/bshtm/bsMenu.aspx'
request = Request(URL, callback=self.parse, dont_filter=True)
item = TwseHisTraderItem()
item['stockid'] = stockid
request.meta['item'] = item
requests.append(request)
return requests
def main():
stockids = TwseIdDBHandler().stock.get_ids(debug=True)
stockids = list(stockids)
# load payload
args = ('twsehistrader', 'INFO', 'twsehistrader.log', True, True)
run_scrapy_service.delay(*args).get()
args = ('twsehisstock', 'INFO', 'twsehisstock.log', True, True)
run_scrapy_service.delay(*args).get()
t = timeit.Timer()
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
args = ('twse', starttime, endtime, ['2317'])
cursor = run_hisstock_query.delay(*args).get()
print cursor
print "run stock 300d query using (%.4f)s" % (t.timeit())
def run(opt='twse', debug=False, limit=0):
""" as doctest run """
# set time window
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
# sort factor
report = Report(
algname=SuperManAlgorithm.__name__,
sort=[('buy_count', False), ('sell_count', False), ('volume', False)], limit=20)
# set debug or normal mode
kwargs = {
'debug': debug,
'limit': limit,
'opt': opt
}
idhandler = TwseIdDBHandler() if kwargs['opt'] == 'twse' else OtcIdDBHandler()
for stockid in idhandler.stock.get_ids(**kwargs):
dbhandler = TwseHisDBHandler() if kwargs['opt'] == 'twse' else OtcHisDBHandler()
dbhandler.stock.ids = [stockid]
data = dbhandler.transform_all_data(starttime, endtime, [stockid], [], 10)
if data.empty:
continue
supman = SuperManAlgorithm(dbhandler=dbhandler)
results = supman.run(data).fillna(0)
if results.empty:
continue
report.collect(stockid, results)
print stockid
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'superman.html')
for stockid in report.iter_stockid():
stream = report.iter_report(stockid, dtype='html', has_other=True, has_sideband=True)
report.write(stream, "superman_%s.html" % (stockid))
def run(opt='twse', debug=False, limit=0):
""" as doctest run """
maxlen = 30
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
report = Report(
'randforest',
sort=[('buys', False), ('sells', False), ('portfolio_value', False)], limit=20)
kwargs = {
'debug': debug,
'limit': limit,
'opt': opt
}
# fetch
idhandler = TwseIdDBHandler(**kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs)
for stockid in idhandler.stock.get_ids():
try:
kwargs = {
'opt': opt,
'targets': ['stock', 'future', 'credit'],
'starttime': starttime,
'endtime': endtime,
'stockids': [stockid],
'traderids': [],
'base': 'stock',
'order': [],
'callback': None,
'limit': 1,
'debug': debug
}
panel, dbhandler = collect_hisframe(**kwargs)
if len(panel[stockid].index) < maxlen:
continue
rforest = RandForest(dbhandler=dbhandler)
results = rforest.run(panel).fillna(0)
risks = rforest.perf_tracker.handle_simulation_end()
report.collect(stockid, results, risks)
print "%s pass" %(stockid)
except:
print traceback.format_exc()
continue
if report.report.empty:
return
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'randforest.html')
for stockid in report.iter_symbol():
stream = report.iter_report(stockid, dtype='html')
report.write(stream, "randforest_%s.html" % (stockid))
# plot
for stockid in report.iter_symbol():
try:
perf = report.pool[stockid]
fig = plt.figure()
ax1 = fig.add_subplot(211)
perf.portfolio_value.plot(ax=ax1)
ax1.set_ylabel('portfolio value in $')
ax2 = fig.add_subplot(212)
perf[['close', 'mavg7']].plot(ax=ax2)
perf_trans = perf.ix[[t != [] for t in perf.transactions]]
buys = perf_trans.ix[[t[0]['amount'] > 0 for t in perf_trans.transactions]]
sells = perf_trans.ix[[t[0]['amount'] < 0 for t in perf_trans.transactions]]
ax2.plot(buys.index, perf.close.ix[buys.index],
'^', markersize=10, color='m')
ax2.plot(sells.index, perf.close.ix[sells.index],
'v', markersize=10, color='k')
ax2.set_ylabel('price in $')
plt.legend(loc=0)
plt.savefig("randforest_%s.png" %(stockid))
#plt.show()
except:
continue
def run(opt='twse', debug=False, limit=0):
maxlen = 30
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
report = Report('bbands',
sort=[('buys', False), ('sells', False),
('portfolio_value', False)],
limit=20)
kwargs = {'debug': debug, 'limit': limit, 'opt': opt}
idhandler = TwseIdDBHandler(
**kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs)
for stockid in idhandler.stock.get_ids():
try:
kwargs = {
'opt': opt,
'targets': ['stock', 'future', 'credit'],
'starttime': starttime,
'endtime': endtime,
'stockids': [stockid],
'traderids': [],
'base': 'stock',
'callback': None,
'limit': 1,
'debug': debug
}
panel, dbhandler = collect_hisframe(**kwargs)
if len(panel[stockid].index) < maxlen:
continue
sim_params = SimulationParameters(
period_start=panel[stockid].index[0],
period_end=panel[stockid].index[-1],
data_frequency='daily',
emission_rate='daily')
bbands = BBands(dbhandler=dbhandler,
debug=debug,
sim_params=sim_params)
results = bbands.run(panel).fillna(0)
risks = bbands.perf_tracker.handle_simulation_end()
report.collect(stockid, results, risks)
print "%s pass" % (stockid)
except:
print traceback.format_exc()
continue
if report.report.empty:
return
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'bbands.html')
for stockid in report.iter_symbol():
stream = report.iter_report(stockid, dtype='html')
report.write(stream, "bbands_%s.html" % (stockid))
for stockid in report.iter_symbol():
perf = report.pool[stockid]
dates = [date2num(i) for i in perf.index[maxlen:]]
quotes = [
perf[label][maxlen:].values
for label in ['open', 'high', 'low', 'close']
]
quotes = zip(*([dates] + quotes))
fig = plt.figure(facecolor='#07000d')
ax1 = plt.subplot2grid((6, 4), (1, 0),
rowspan=4,
colspan=4,
axisbg='#07000d')
candlestick_ohlc(ax1,
quotes,
width=.6,
colorup='#53c156',
colordown='#ff1717')
ax1.plot(dates,
perf['upper'][maxlen:].values,
'#e1edf9',
label='upper',
linewidth=1.5)
ax1.plot(dates,
perf['middle'][maxlen:].values,
'#e1edf9',
label='middle',
linewidth=1.5)
ax1.plot(dates,
perf['lower'][maxlen:].values,
'#e1edf9',
label='lower',
linewidth=1.5)
ax1.grid(True, color='w')
ax1.xaxis.set_major_locator(mticker.MaxNLocator(10))
ax1.xaxis.set_major_formatter(DateFormatter('%Y-%m-%d'))
ax1.yaxis.label.set_color("w")
ax1.spines['bottom'].set_color("#5998ff")
ax1.spines['top'].set_color("#5998ff")
ax1.spines['left'].set_color("#5998ff")
ax1.spines['right'].set_color("#5998ff")
ax1.tick_params(axis='y', colors='w')
plt.gca().yaxis.set_major_locator(mticker.MaxNLocator(prune='upper'))
ax1.tick_params(axis='x', colors='w')
plt.ylabel('Stock price and Volume')
bbLeg = plt.legend(loc=9,
ncol=2,
prop={'size': 7},
fancybox=True,
borderaxespad=0.)
bbLeg.get_frame().set_alpha(0.4)
textEd = pylab.gca().get_legend().get_texts()
pylab.setp(textEd[0:6], color='w')
ax1v = ax1.twinx()
ax1v.fill_between(dates,
0,
perf['volume'][maxlen:].values,
facecolor='#00ffe8',
alpha=.4)
ax1v.axes.yaxis.set_ticklabels([])
ax1v.grid(False)
###Edit this to 3, so it's a bit larger
ax1v.set_ylim(0, 3 * perf['volume'][maxlen:].values.max())
ax1v.spines['bottom'].set_color("#5998ff")
ax1v.spines['top'].set_color("#5998ff")
ax1v.spines['left'].set_color("#5998ff")
ax1v.spines['right'].set_color("#5998ff")
ax1v.tick_params(axis='x', colors='w')
ax1v.tick_params(axis='y', colors='w')
plt.setp(ax1.get_xticklabels(), visible=False)
plt.subplots_adjust(left=.09,
bottom=.14,
right=.94,
top=.95,
wspace=.20,
hspace=0)
plt.gcf().set_size_inches(18, 8)
plt.savefig("bbands_%s.png" % (stockid), facecolor=fig.get_facecolor())
def run(opt='twse', debug=False, limit=0):
""" as doctest run """
maxlen = 30
starttime = datetime.utcnow() - timedelta(days=300)
endtime = datetime.utcnow()
report = Report('dualema',
sort=[('buys', False), ('sells', False),
('portfolio_value', False)],
limit=20)
kwargs = {'debug': debug, 'limit': limit, 'opt': opt}
# fetch
idhandler = TwseIdDBHandler(
**kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs)
for stockid in idhandler.stock.get_ids():
try:
kwargs = {
'opt': opt,
'targets': ['stock'],
'starttime': starttime,
'endtime': endtime,
'stockids': [stockid],
'traderids': [],
'base': 'stock',
'callback': None,
'limit': 1,
'debug': debug
}
panel, dbhandler = collect_hisframe(**kwargs)
if len(panel[stockid].index) < maxlen:
continue
sim_params = SimulationParameters(
period_start=panel[stockid].index[0],
period_end=panel[stockid].index[-1],
data_frequency='daily',
emission_rate='daily')
dualema = DualEMA(dbhandler=dbhandler,
debug=debug,
sim_params=sim_params)
results = dualema.run(panel).fillna(0)
risks = dualema.perf_tracker.handle_simulation_end()
report.collect(stockid, results, risks)
print "%s pass" % (stockid)
except:
print traceback.format_exc()
continue
if report.report.empty:
return
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'dualema.html')
for stockid in report.iter_symbol():
stream = report.iter_report(stockid, dtype='html')
report.write(stream, "dualema_%s.html" % (stockid))
# plot
for stockid in report.iter_symbol():
try:
perf = report.pool[stockid]
fig = plt.figure()
ax1 = fig.add_subplot(211, ylabel='portfolio value')
perf.portfolio_value.plot(ax=ax1)
ax2 = fig.add_subplot(212)
perf[['short_ema', 'long_ema']].plot(ax=ax2)
ax2.plot(perf.ix[perf.buy].index,
perf.short_ema[perf.buy],
'^',
markersize=10,
color='m')
ax2.plot(perf.ix[perf.sell].index,
perf.short_ema[perf.sell],
'v',
markersize=10,
color='k')
plt.legend(loc=0)
plt.gcf().set_size_inches(18, 8)
plt.savefig("dualema_%s.png" % (stockid))
#plt.show()
except:
continue
def run(opt='twse', debug=False, limit=0):
""" as doctest run """
maxlen = 5
starttime = datetime.utcnow() - timedelta(days=15)
endtime = datetime.utcnow()
report = Report('besttrader',
sort=[('buys', False), ('sells', False),
('portfolio_value', False)],
limit=20)
kwargs = {'debug': debug, 'limit': limit, 'opt': opt}
# 1590:u'花旗環球', 1440:u'美林'
traderid = '1440'
idhandler = TwseIdDBHandler(
**kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs)
for stockid in idhandler.stock.get_ids():
try:
# pre find traderid as top0
kwargs = {
'opt':
opt,
'targets': ['trader'],
'starttime':
starttime,
'endtime':
endtime,
'stockids': [stockid],
'traderids': [],
'base':
'stock',
'constraint':
lambda x: x.value["ebuyratio"] > 10 or x.value["totalkeepbuy"]
>= 1,
'order':
lambda x: [-x.value["totalvolume"], -x.value["totalbuyratio"]],
'callback':
None,
'limit':
10,
'debug':
debug
}
panel, dbhandler = collect_hisframe(**kwargs)
tops = list(
dbhandler.trader.get_alias([stockid], 'trader',
["top%d" % i for i in range(10)]))
if not tops:
continue
print "%s prefound:%s" % (stockid, tops)
traderid = tops[0] if traderid not in tops else traderid
# run
kwargs = {
'opt': opt,
'targets': ['stock', 'trader', 'future', 'credit'],
'starttime': starttime,
'endtime': endtime,
'stockids': [stockid],
'traderids': [traderid],
'base': 'trader',
'callback': None,
'limit': 10,
'debug': debug
}
panel, dbhandler = collect_hisframe(**kwargs)
if len(panel[stockid].index) < maxlen:
continue
sim_params = SimulationParameters(
period_start=panel[stockid].index[0],
period_end=panel[stockid].index[-1],
data_frequency='daily',
emission_rate='daily')
besttrader = BestTrader(dbhandler=dbhandler,
debug=debug,
sim_params=sim_params)
results = besttrader.run(panel).fillna(0)
risks = besttrader.perf_tracker.handle_simulation_end()
report.collect(stockid, results, risks)
print "%s pass" % (stockid)
except:
print traceback.format_exc()
continue
if report.report.empty:
return
# report summary
stream = report.summary(dtype='html')
report.write(stream, 'besttrader.html')
for stockid in report.iter_symbol():
stream = report.iter_report(stockid, dtype='html')
report.write(stream, "besttrader_%s.html" % (stockid))
# plot
for stockid in report.iter_symbol():
try:
perf = report.pool[stockid]
fig = plt.figure()
ax1 = fig.add_subplot(211, ylabel='portfolio value')
perf.portfolio_value.plot(ax=ax1)
ax2 = fig.add_subplot(212)
perf[['close']].plot(ax=ax2)
ax2.plot(perf.ix[perf.buy].index,
perf.close[perf.buy],
'^',
markersize=10,
color='m')
ax2.plot(perf.ix[perf.sell].index,
perf.close[perf.sell],
'v',
markersize=10,
color='k')
plt.legend(loc=0)
plt.gcf().set_size_inches(18, 8)
plt.savefig("besttrader_%s_%s.png" % (traderid, stockid))
#plt.show()
except:
continue
def __init__(self, **kwargs):
self._coll = kwargs.pop('coll', None)
self._debug = kwargs.pop('debug', False)
kwargs = {'id': {'debug': self._debug, 'opt': 'twse'}}
self._id = TwseIdDBHandler(**kwargs['id'])
self._ids = []