def run(opt='twse', debug=False, limit=0): # set time window starttime = datetime.utcnow() - timedelta(days=300) endtime = datetime.utcnow() report = Report( algname=ZombieAlgorithm.__name__, sort=[('buy_count', False), ('sell_count', False), ('volume', False)], limit=20) # set debug or normal mode kwargs = { 'debug': debug, 'limit': limit, 'opt': opt } idhandler = TwseIdDBHandler() if kwargs['opt'] == 'twse' else OtcIdDBHandler() for stockid in idhandler.stock.get_ids(**kwargs): dbhandler = TwseHisDBHandler() if kwargs['opt'] == 'twse' else OtcHisDBHandler() dbhandler.stock.ids = [stockid] data = dbhandler.transform_all_data(starttime, endtime, [stockid], [], 10) if data.empty: continue zombie = ZombieAlgorithm(dbhandler=dbhandler) results = zombie.run(data).fillna(0) if results.empty: continue report.collect(stockid, results) print stockid # report summary stream = report.summary(dtype='html') report.write(stream, 'zombie.html') for stockid in report.iter_stockid(): stream = report.iter_report(stockid, dtype='html') report.write(stream, "zombie_%s.html" % (stockid))
def start_requests(self): kwargs = { 'debug': self.settings.getbool('GIANT_DEBUG'), 'limit': self.settings.getint('GIANT_LIMIT'), 'opt': 'otc' } requests = [] for stockid in OtcIdDBHandler().stock.get_ids(**kwargs): for mon in range(2, -1, -1): timestamp = datetime.utcnow() - relativedelta(months=mon) if mon == 0: if timestamp.day == 1 and timestamp.hour <= 14: continue URL = ( 'http://www.gretai.org.tw/ch/stock/aftertrading/' + 'daily_trading_info/st43_download.php?d=%(year)d/%(mon)02d&' + 'stkno=%(stock)s') % { 'year': timestamp.year - 1911, 'mon': timestamp.month, 'stock': stockid } request = Request(URL, callback=self.parse, dont_filter=True) item = OtcHisStockItem() item['stockid'] = stockid request.meta['item'] = item requests.append(request) return requests
def test_on_run(self): ids = OtcIdDBHandler().stock.get_ids(debug=True) ids = list(ids) t = timeit.Timer() args = ('otchisstock', 'INFO', './log/otchisstock.log', True, True) run_scrapy_service.delay(*args).get() print "scrapy otchisstock used %.4f(s)/%d(u)" % (t.timeit(), len(ids)) self.assertTrue(self.has_pass('./log/otchisstock.log'))
def __init__(self, crawler): super(OtcHisStockSpider, self).__init__() kwargs = { 'debug': crawler.settings.getbool('GIANT_DEBUG'), 'limit': crawler.settings.getint('GIANT_LIMIT'), 'opt': 'otc' } self._id = OtcIdDBHandler(**kwargs)
def __init__(self, alg, **kwargs): super(OtcAlgDBHandler, self).__init__(alg, **kwargs) db = "otcalgdb" db = db if not self._debug else 'test' + db host, port = MongoDBDriver._host, MongoDBDriver._port connect(db, host=host, port=port, alias=db) self._sumycoll = switch(AlgSummaryColl, db) self._id = OtcIdDBHandler(debug=self._debug, opt='otc')
def run(opt='twse', debug=False, limit=0): """ as doctest run """ maxlen = 5 starttime = datetime.utcnow() - timedelta(days=15) endtime = datetime.utcnow() kwargs = {'debug': debug, 'limit': limit, 'opt': opt} # 1590:u'花旗環球', 1440:u'美林' traderid = '1440' idhandler = TwseIdDBHandler( **kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs) for stockid in idhandler.stock.get_ids(): try: # pre find traderid as top0 kwargs = { 'opt': opt, 'targets': ['trader'], 'starttime': starttime, 'endtime': endtime, 'stockids': [stockid], 'traderids': [], 'base': 'stock', 'callback': None, 'limit': 10, 'debug': True } panel, dbhandler = collect_hisframe(**kwargs) tops = list( dbhandler.trader.get_alias([stockid], 'trader', ["top%d" % i for i in range(10)])) print "prefound:%s" % (tops) traderid = tops[0] if traderid not in tops else traderid # run kwargs = { 'opt': opt, 'targets': ['stock', 'trader', 'future', 'credit'], 'starttime': starttime, 'endtime': endtime, 'stockids': [stockid], 'traderids': [traderid], 'base': 'trader', 'callback': None, 'limit': 10, 'debug': True } panel, dbhandler = collect_hisframe(**kwargs) if len(panel[stockid].index) < maxlen: continue besttrader = BestTraderAlgorithm(dbhandler=dbhandler, debug=debug) results = besttrader.run(panel).fillna(0) report.collect(stockid, results) print "%s pass" % (stockid) except: print traceback.format_exc() continue if report.report.empty: return
def run(opt='twse', debug=False, limit=0): """ as doctest run """ # set time window starttime = datetime.utcnow() - timedelta(days=300) endtime = datetime.utcnow() # sort factor report = Report(algname=DualEMATaLib.__name__, sort=[('buy_count', False), ('sell_count', False), ('volume', False)], limit=20) # set debug or normal mode kwargs = {'debug': debug, 'limit': limit, 'opt': opt} idhandler = TwseIdDBHandler( ) if kwargs['opt'] == 'twse' else OtcIdDBHandler() for stockid in idhandler.stock.get_ids(**kwargs): dbhandler = TwseHisDBHandler( ) if kwargs['opt'] == 'twse' else OtcHisDBHandler() dbhandler.stock.ids = [stockid] data = dbhandler.transform_all_data(starttime, endtime, [stockid], [], 10) if data.empty: continue dualema = DualEMATaLib(dbhandler=dbhandler) results = dualema.run(data).fillna(0) if results.empty: continue report.collect(stockid, results) print stockid # report summary stream = report.summary(dtype='html') report.write(stream, 'dualema.html') for stockid in report.iter_stockid(): stream = report.iter_report(stockid, dtype='html') report.write(stream, "dualema_%s.html" % (stockid)) # plt fig = plt.figure() ax1 = fig.add_subplot(211, ylabel='portfolio value') results.portfolio_value.plot(ax=ax1) ax2 = fig.add_subplot(212) results[['short_ema', 'long_ema']].plot(ax=ax2) ax2.plot(results.ix[results.buy].index, results.short_ema[results.buy], '^', markersize=10, color='m') ax2.plot(results.ix[results.sell].index, results.short_ema[results.sell], 'v', markersize=10, color='k') plt.legend(loc=0) plt.gcf().set_size_inches(18, 8)
def run(opt='twse', debug=False, limit=0): maxlen = 30 starttime = datetime.utcnow() - timedelta(days=300) endtime = datetime.utcnow() report = Report('kmeans', sort=[('buys', False), ('sells', False), ('portfolio_value', False)], limit=20) # set debug or normal mode kwargs = {'debug': debug, 'limit': limit, 'opt': opt} idhandler = TwseIdDBHandler( **kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs) for stockid in idhandler.stock.get_ids(): try: kwargs = { 'opt': opt, 'targets': ['stock'], 'starttime': starttime, 'endtime': endtime, 'stockids': [stockid], 'base': 'stock', 'callback': None, 'limit': 1, 'debug': debug } panel, dbhandler = collect_hisframe(**kwargs) if len(panel[stockid].index) < maxlen: continue sim_params = SimulationParameters( period_start=panel[stockid].index[0], period_end=panel[stockid].index[-1], data_frequency='daily', emission_rate='daily') kmeans = Kmeans(dbhandler=dbhandler, debug=True, sim_params=sim_params) results = kmeans.run(panel).fillna(0) risks = kmeans.perf_tracker.handle_simulation_end() report.collect(stockid, results, risks) print "%s pass" % (stockid) except: print traceback.format_exc() continue if report.report.empty: return # report summary stream = report.summary(dtype='html') report.write(stream, 'kmeans.html') for stockid in report.iter_symbol(): stream = report.iter_report(stockid, dtype='html') report.write(stream, "kmeans_%s.html" % (stockid))
def run(opt='twse', debug=False, limit=0): """ as doctest run """ # set time window starttime = datetime.utcnow() - timedelta(days=30) endtime = datetime.utcnow() # set debug or normal mode kwargs = {'debug': debug, 'limit': limit, 'opt': opt} # query topbuylist by each trader # 1590:u'花旗環球', 1440:u'美林' idhandler = TwseIdDBHandler( ) if kwargs['opt'] == 'twse' else OtcIdDBHandler() for traderid in idhandler.trader.get_ids(**kwargs): dbhandler = TwseHisdbhandler( ) if kwargs['opt'] == 'twse' else OtcHisDBHandler() topdt = dbhandler.trader.gettoptrader_data(starttime, endtime, [traderid], 'trader', 'buy', 10) report = Report(algname=DarkManAlgorithm.__name__, sort=[('buy_count', -1), ('sell_count', -1), ('ending_value', -1), ('close', -1)], limit=20) kwargs = {'debug': debug, 'limit': limit, 'opt': opt} stockids = dbhandler.trader.map_alias( [traderid], 'trader', ['topbuy%d' % (i) for i in range(5)]) for stockid in stockids: dbhandler = TwseHisdbhandler( ) if kwargs['opt'] == 'twse' else OtcHisDBHandler() data = dbhandler.transform_all_data(starttime, endtime, [traderid], [stockid], 10) if data.empty: continue darkman = DarkManAlgorithm(dbhandler=dbhandler) results = darkman.run(data).fillna(0) if results.empty: continue report.collect(stockid, results) print traderid, stockid # report summary stream = report.summary(dtype='html') report.write(stream, 'darkman_%s.html' % (traderid)) for stockid in report.iter_stockid(): stream = report.iter_report(stockid, dtype='html', has_other=True, has_sideband=True) report.write(stream, "darkman_%s_%s.html" % (traderid, stockid))
def start_requests(self): kwargs = { 'debug': self.settings.getbool('GIANT_DEBUG'), 'limit': self.settings.getint('GIANT_LIMIT'), 'opt': 'otc' } requests = [] for stockid in OtcIdDBHandler().stock.get_ids(**kwargs): URL = ( 'http://www.gretai.org.tw/web/stock/aftertrading/' + 'broker_trading/brokerBS.php?l=zh-tw&stk_code=%(stock)s') % { 'stock': stockid } request = Request(URL, callback=self.parse, dont_filter=True) item = OtcHisTraderItem() item['stockid'] = stockid request.meta['item'] = item requests.append(request) return requests
def run(opt='twse', debug=False, limit=0): """ as doctest run """ maxlen = 30 starttime = datetime.utcnow() - timedelta(days=300) endtime = datetime.utcnow() report = Report( 'randforest', sort=[('buys', False), ('sells', False), ('portfolio_value', False)], limit=20) kwargs = { 'debug': debug, 'limit': limit, 'opt': opt } # fetch idhandler = TwseIdDBHandler(**kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs) for stockid in idhandler.stock.get_ids(): try: kwargs = { 'opt': opt, 'targets': ['stock', 'future', 'credit'], 'starttime': starttime, 'endtime': endtime, 'stockids': [stockid], 'traderids': [], 'base': 'stock', 'order': [], 'callback': None, 'limit': 1, 'debug': debug } panel, dbhandler = collect_hisframe(**kwargs) if len(panel[stockid].index) < maxlen: continue rforest = RandForest(dbhandler=dbhandler) results = rforest.run(panel).fillna(0) risks = rforest.perf_tracker.handle_simulation_end() report.collect(stockid, results, risks) print "%s pass" %(stockid) except: print traceback.format_exc() continue if report.report.empty: return # report summary stream = report.summary(dtype='html') report.write(stream, 'randforest.html') for stockid in report.iter_symbol(): stream = report.iter_report(stockid, dtype='html') report.write(stream, "randforest_%s.html" % (stockid)) # plot for stockid in report.iter_symbol(): try: perf = report.pool[stockid] fig = plt.figure() ax1 = fig.add_subplot(211) perf.portfolio_value.plot(ax=ax1) ax1.set_ylabel('portfolio value in $') ax2 = fig.add_subplot(212) perf[['close', 'mavg7']].plot(ax=ax2) perf_trans = perf.ix[[t != [] for t in perf.transactions]] buys = perf_trans.ix[[t[0]['amount'] > 0 for t in perf_trans.transactions]] sells = perf_trans.ix[[t[0]['amount'] < 0 for t in perf_trans.transactions]] ax2.plot(buys.index, perf.close.ix[buys.index], '^', markersize=10, color='m') ax2.plot(sells.index, perf.close.ix[sells.index], 'v', markersize=10, color='k') ax2.set_ylabel('price in $') plt.legend(loc=0) plt.savefig("randforest_%s.png" %(stockid)) #plt.show() except: continue
def __init__(self, coll): super(OtcTraderHisDBHandler, self).__init__(coll) self._iddbhandler = OtcIdDBHandler()
def run(opt='twse', debug=False, limit=0): maxlen = 30 starttime = datetime.utcnow() - timedelta(days=300) endtime = datetime.utcnow() report = Report('bbands', sort=[('buys', False), ('sells', False), ('portfolio_value', False)], limit=20) kwargs = {'debug': debug, 'limit': limit, 'opt': opt} idhandler = TwseIdDBHandler( **kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs) for stockid in idhandler.stock.get_ids(): try: kwargs = { 'opt': opt, 'targets': ['stock', 'future', 'credit'], 'starttime': starttime, 'endtime': endtime, 'stockids': [stockid], 'traderids': [], 'base': 'stock', 'callback': None, 'limit': 1, 'debug': debug } panel, dbhandler = collect_hisframe(**kwargs) if len(panel[stockid].index) < maxlen: continue sim_params = SimulationParameters( period_start=panel[stockid].index[0], period_end=panel[stockid].index[-1], data_frequency='daily', emission_rate='daily') bbands = BBands(dbhandler=dbhandler, debug=debug, sim_params=sim_params) results = bbands.run(panel).fillna(0) risks = bbands.perf_tracker.handle_simulation_end() report.collect(stockid, results, risks) print "%s pass" % (stockid) except: print traceback.format_exc() continue if report.report.empty: return # report summary stream = report.summary(dtype='html') report.write(stream, 'bbands.html') for stockid in report.iter_symbol(): stream = report.iter_report(stockid, dtype='html') report.write(stream, "bbands_%s.html" % (stockid)) for stockid in report.iter_symbol(): perf = report.pool[stockid] dates = [date2num(i) for i in perf.index[maxlen:]] quotes = [ perf[label][maxlen:].values for label in ['open', 'high', 'low', 'close'] ] quotes = zip(*([dates] + quotes)) fig = plt.figure(facecolor='#07000d') ax1 = plt.subplot2grid((6, 4), (1, 0), rowspan=4, colspan=4, axisbg='#07000d') candlestick_ohlc(ax1, quotes, width=.6, colorup='#53c156', colordown='#ff1717') ax1.plot(dates, perf['upper'][maxlen:].values, '#e1edf9', label='upper', linewidth=1.5) ax1.plot(dates, perf['middle'][maxlen:].values, '#e1edf9', label='middle', linewidth=1.5) ax1.plot(dates, perf['lower'][maxlen:].values, '#e1edf9', label='lower', linewidth=1.5) ax1.grid(True, color='w') ax1.xaxis.set_major_locator(mticker.MaxNLocator(10)) ax1.xaxis.set_major_formatter(DateFormatter('%Y-%m-%d')) ax1.yaxis.label.set_color("w") ax1.spines['bottom'].set_color("#5998ff") ax1.spines['top'].set_color("#5998ff") ax1.spines['left'].set_color("#5998ff") ax1.spines['right'].set_color("#5998ff") ax1.tick_params(axis='y', colors='w') plt.gca().yaxis.set_major_locator(mticker.MaxNLocator(prune='upper')) ax1.tick_params(axis='x', colors='w') plt.ylabel('Stock price and Volume') bbLeg = plt.legend(loc=9, ncol=2, prop={'size': 7}, fancybox=True, borderaxespad=0.) bbLeg.get_frame().set_alpha(0.4) textEd = pylab.gca().get_legend().get_texts() pylab.setp(textEd[0:6], color='w') ax1v = ax1.twinx() ax1v.fill_between(dates, 0, perf['volume'][maxlen:].values, facecolor='#00ffe8', alpha=.4) ax1v.axes.yaxis.set_ticklabels([]) ax1v.grid(False) ###Edit this to 3, so it's a bit larger ax1v.set_ylim(0, 3 * perf['volume'][maxlen:].values.max()) ax1v.spines['bottom'].set_color("#5998ff") ax1v.spines['top'].set_color("#5998ff") ax1v.spines['left'].set_color("#5998ff") ax1v.spines['right'].set_color("#5998ff") ax1v.tick_params(axis='x', colors='w') ax1v.tick_params(axis='y', colors='w') plt.setp(ax1.get_xticklabels(), visible=False) plt.subplots_adjust(left=.09, bottom=.14, right=.94, top=.95, wspace=.20, hspace=0) plt.gcf().set_size_inches(18, 8) plt.savefig("bbands_%s.png" % (stockid), facecolor=fig.get_facecolor())
def run(opt='twse', debug=False, limit=0): """ as doctest run """ maxlen = 30 starttime = datetime.utcnow() - timedelta(days=300) endtime = datetime.utcnow() report = Report('dualema', sort=[('buys', False), ('sells', False), ('portfolio_value', False)], limit=20) kwargs = {'debug': debug, 'limit': limit, 'opt': opt} # fetch idhandler = TwseIdDBHandler( **kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs) for stockid in idhandler.stock.get_ids(): try: kwargs = { 'opt': opt, 'targets': ['stock'], 'starttime': starttime, 'endtime': endtime, 'stockids': [stockid], 'traderids': [], 'base': 'stock', 'callback': None, 'limit': 1, 'debug': debug } panel, dbhandler = collect_hisframe(**kwargs) if len(panel[stockid].index) < maxlen: continue sim_params = SimulationParameters( period_start=panel[stockid].index[0], period_end=panel[stockid].index[-1], data_frequency='daily', emission_rate='daily') dualema = DualEMA(dbhandler=dbhandler, debug=debug, sim_params=sim_params) results = dualema.run(panel).fillna(0) risks = dualema.perf_tracker.handle_simulation_end() report.collect(stockid, results, risks) print "%s pass" % (stockid) except: print traceback.format_exc() continue if report.report.empty: return # report summary stream = report.summary(dtype='html') report.write(stream, 'dualema.html') for stockid in report.iter_symbol(): stream = report.iter_report(stockid, dtype='html') report.write(stream, "dualema_%s.html" % (stockid)) # plot for stockid in report.iter_symbol(): try: perf = report.pool[stockid] fig = plt.figure() ax1 = fig.add_subplot(211, ylabel='portfolio value') perf.portfolio_value.plot(ax=ax1) ax2 = fig.add_subplot(212) perf[['short_ema', 'long_ema']].plot(ax=ax2) ax2.plot(perf.ix[perf.buy].index, perf.short_ema[perf.buy], '^', markersize=10, color='m') ax2.plot(perf.ix[perf.sell].index, perf.short_ema[perf.sell], 'v', markersize=10, color='k') plt.legend(loc=0) plt.gcf().set_size_inches(18, 8) plt.savefig("dualema_%s.png" % (stockid)) #plt.show() except: continue
def run(opt='twse', debug=False, limit=0): """ as doctest run """ maxlen = 5 starttime = datetime.utcnow() - timedelta(days=15) endtime = datetime.utcnow() report = Report('besttrader', sort=[('buys', False), ('sells', False), ('portfolio_value', False)], limit=20) kwargs = {'debug': debug, 'limit': limit, 'opt': opt} # 1590:u'花旗環球', 1440:u'美林' traderid = '1440' idhandler = TwseIdDBHandler( **kwargs) if kwargs['opt'] == 'twse' else OtcIdDBHandler(**kwargs) for stockid in idhandler.stock.get_ids(): try: # pre find traderid as top0 kwargs = { 'opt': opt, 'targets': ['trader'], 'starttime': starttime, 'endtime': endtime, 'stockids': [stockid], 'traderids': [], 'base': 'stock', 'constraint': lambda x: x.value["ebuyratio"] > 10 or x.value["totalkeepbuy"] >= 1, 'order': lambda x: [-x.value["totalvolume"], -x.value["totalbuyratio"]], 'callback': None, 'limit': 10, 'debug': debug } panel, dbhandler = collect_hisframe(**kwargs) tops = list( dbhandler.trader.get_alias([stockid], 'trader', ["top%d" % i for i in range(10)])) if not tops: continue print "%s prefound:%s" % (stockid, tops) traderid = tops[0] if traderid not in tops else traderid # run kwargs = { 'opt': opt, 'targets': ['stock', 'trader', 'future', 'credit'], 'starttime': starttime, 'endtime': endtime, 'stockids': [stockid], 'traderids': [traderid], 'base': 'trader', 'callback': None, 'limit': 10, 'debug': debug } panel, dbhandler = collect_hisframe(**kwargs) if len(panel[stockid].index) < maxlen: continue sim_params = SimulationParameters( period_start=panel[stockid].index[0], period_end=panel[stockid].index[-1], data_frequency='daily', emission_rate='daily') besttrader = BestTrader(dbhandler=dbhandler, debug=debug, sim_params=sim_params) results = besttrader.run(panel).fillna(0) risks = besttrader.perf_tracker.handle_simulation_end() report.collect(stockid, results, risks) print "%s pass" % (stockid) except: print traceback.format_exc() continue if report.report.empty: return # report summary stream = report.summary(dtype='html') report.write(stream, 'besttrader.html') for stockid in report.iter_symbol(): stream = report.iter_report(stockid, dtype='html') report.write(stream, "besttrader_%s.html" % (stockid)) # plot for stockid in report.iter_symbol(): try: perf = report.pool[stockid] fig = plt.figure() ax1 = fig.add_subplot(211, ylabel='portfolio value') perf.portfolio_value.plot(ax=ax1) ax2 = fig.add_subplot(212) perf[['close']].plot(ax=ax2) ax2.plot(perf.ix[perf.buy].index, perf.close[perf.buy], '^', markersize=10, color='m') ax2.plot(perf.ix[perf.sell].index, perf.close[perf.sell], 'v', markersize=10, color='k') plt.legend(loc=0) plt.gcf().set_size_inches(18, 8) plt.savefig("besttrader_%s_%s.png" % (traderid, stockid)) #plt.show() except: continue
def __init__(self, **kwargs): super(OtcFutureHisDBHandler, self).__init__(**copy.deepcopy(kwargs)) kwargs = {'id': {'debug': kwargs['debug'], 'opt': 'otc'}} self._id = OtcIdDBHandler(**kwargs['id'])