def tick_by_tick(market, entry_funcs, exit_funcs, interval=var.default_interval, smas=var.default_smas, emas=var.default_emas, refresh_interval=1, from_file=True, plot=False, log_level=2): ''' Simulates a working bot, in realtime or in faster speed, using pre own data from DB or file, to test an autonomous bot on a specific market. Args: markets(string): list with markets to backtest or empty to run all available markets. entry_funcs(list): list of entry functions to test. exit_funcs(list): list of entry functions to test. interval(string): time between measures. smas(list): list of SMA values to use. emas(list): list of EMA values to use. refresh_interval(int): Refresh rate. main_coins(list): ''' #log_level: # 0 - Only presents total. # 1 - Writes logs to file. # 2 - Writes logs to file and prints on screen. # Default is 2. plt.ion() var.global_log_level = log_level signal.signal(signal.SIGINT, signal_handler) date = [0, 0] total = 0 #market = check_market_name(market) entry_points_x = [] entry_points_y = [] exit_points_x = [] exit_points_y = [] if type(entry_funcs) is not list: entry_funcs = [entry_funcs] if type(exit_funcs) is not list: exit_funcs = [exit_funcs] print '[Market analysis]: ' + market + '\n' if from_file: try: data = get_data_from_file(market, interval=interval) except Exception as e: log(str(e), 1) log('[ERROR] Can\'t find ' + market + ' in files.', 1) return 0 data_init = data #if type(_date[0]) is str: # date[0], date[1] = time_to_index(data, _date) #if date[1] == 0: # data = data[date[0]:] #else: # data = data[date[0]:date[1]] else: try: data = get_historical_data(market, interval=interval, init_date=_date[0], end_date=_date[1]) date[0], date[1] = 0, len(data) data_init = data except Exception as e: log(str(e), 1) log('[ERROR] Can\'t find ' + market + ' in BD.', 1) return 0 #continue aux_buy = False buy_price = 0 high_price = 0 plt.show() #Tests several functions. for i in xrange(len(data) - 50): start_time = time() #print data_init.Last.iloc[i] if not aux_buy: if is_time_to_buy(data[i:i + 50], entry_funcs, smas, emas): buy_price = data_init.Ask.iloc[i + 49 + date[0]] high_price = buy_price entry_points_x.append(i + 49) entry_points_y.append(data_init.Ask.iloc[i + 49 + date[0]]) if exit_funcs: aux_buy = True print str(data_init.time.iloc[i + 49 + date[0]]) + \ ' [BUY] @ ' + str(data_init.Ask.iloc[i + 49 + date[0]]) + '\n' else: # Used for trailing stop loss. if data_init.Last.iloc[i + 49 + date[0]] > high_price: high_price = data_init.Last.iloc[i + 49 + date[0]] if is_time_to_exit(data[i:i + 50], exit_funcs, smas, emas, stop=0, bought_at=buy_price, max_price=high_price): exit_points_x.append(i + 49) exit_points_y.append(data_init.Bid.iloc[i + 49 + date[0]]) aux_buy = False total += round( ((data_init.Bid.iloc[i + 49 + date[0]] - buy_price) / buy_price) * 100, 2) print str(data_init.time.iloc[i + 49 + date[0]]) + \ ' [SELL]@ ' + str(data_init.Bid.iloc[i + 49 + date[0]]) + '\n' print '[P&L] > ' + str(total) + '%.' + '\n' #plt.plot(data.Last.iloc[i:i+50]) #plt.draw() #plt.clf() # In case of processing time is bigger than *refresh_interval* doesn't sleep. if refresh_interval - (time() - start_time) > 0: sleep(refresh_interval - (time() - start_time)) return total
def backtest(markets, entry_funcs, exit_funcs=[], _date=[0, 0], smas=var.default_smas, emas=var.default_emas, interval=var.default_interval, plot=False, to_file=True, from_file=False, base_market='BTC', log_level=1, mp_level="medium"): ''' Backtests strategies. Args: markets(list): list with markets to backtest or empty to test all available markets. entry_funcs(list): list of entry functions to test. exit_funcs(list): list of entry functions to test. _date(list): init and end point to backtest. Ex: '1-1-2017 11:10' smas(list): list of SMA values to use. emas(list): list of EMA values to use. interval(string): time between measures. plot(bool): plot data. to_file(bool): plot to file. from_file(bool): get data from file. base_market(string): base market to use. log_level(int): log level - 0-2. mp_level(string): multiprocessing level - [low, medium, high]. Returns: bool: returns True in case of success. log_level: 0 - Only presents total. 1 - Writes logs to file. 2 - Writes logs to file and prints on screen. Default is 2. ''' if not from_file: # Connects to DB. try: db_client = connect_db() except Exception as e: log(str(e), 0, log_level) sys.exit(1) else: db_client = 0 # For all markets. if not len(markets): y = 'y' #raw_input("Want to run all markets? ") if y == 'y': if from_file: markets = get_markets_on_files(interval, base=base_market) else: markets = get_markets_list(base=base_market) else: log("Without files to analyse.", 0, log_level) # Prevents errors from markets and funcs as str. if not isinstance(markets, list): markets = [markets] if not isinstance(entry_funcs, list): entry_funcs = [entry_funcs] if not isinstance(exit_funcs, list): exit_funcs = [exit_funcs] # For selected markets. if from_file: markets = manage_files(markets, interval=interval) log(str(len(markets)) + " files/chunks to analyse...", 1, log_level) # Create a multiprocessing Pool pool = Pool(num_processors(mp_level)) # Display information about pool. total = pool.map( partial(backtest_market, entry_funcs, exit_funcs, interval, _date, smas, emas, from_file, to_file, plot, db_client, log_level), markets) pool.close() pool.join() log(" Total > " + str(sum(total)), 1, log_level) return sum(total)
def backtest_market(entry_funcs, exit_funcs, interval, _date, smas, emas, from_file, to_file, plot, db_client, log_level, market): ''' Backtests strategies for a specific market. Args: entry_funcs(list): list of entry functions to test. exit_funcs(list): list of entry functions to test. interval(string): time between measures. _date(list): init and end point to backtest. smas(list): list of SMA values to use. emas(list): list of EMA values to use. to_file(bool): plot to file. from_file(bool): get data from file. plot(bool): plot data. markets(string): list with markets to backtest or empty to test all available markets. Returns: float: returns backtests profit & loss value for applied strategies. ''' date = [0, 0] total = 0 #market = check_market_name(market) entry_points_x = [] entry_points_y = [] exit_points_x = [] exit_points_y = [] full_log = '[Market analysis]: ' + market + '\n' if from_file: try: data = get_data_from_file(market, interval=interval) except Exception as e: log(str(e), 0, log_level) log('[ERROR] Can\'t find ' + market + ' in files.', 0, log_level) return 0 data_init = data if type(_date[0]) is str: date[0], date[1] = time_to_index(data, _date) if date[1] == 0: data = data[date[0]:] else: data = data[date[0]:date[1]] else: try: data = get_historical_data(market, interval=interval, init_date=_date[0], end_date=_date[1]) date[0], date[1] = 0, len(data) data_init = data except Exception as e: log(str(e), 0, log_level) log('[ERROR] Can\'t find ' + market + ' in BD.', 0, log_level) return 0 #continue aux_buy = False buy_price = 0 high_price = 0 #Tests several functions. for i in xrange(len(data) - 50): if not aux_buy: if is_time_to_buy(data[i:i + 50], entry_funcs, smas, emas): buy_price = data_init.Ask.iloc[i + 49 + date[0]] high_price = buy_price entry_points_x.append(i + 49) entry_points_y.append(data_init.Ask.iloc[i + 49 + date[0]]) if exit_funcs: aux_buy = True full_log += str(data_init.time.iloc[i + 49 + date[0]]) + \ ' [BUY] @ ' + str(data_init.Ask.iloc[i + 49 + date[0]]) + '\n' else: # Used for trailing stop loss. if data_init.Last.iloc[i + 49 + date[0]] > high_price: high_price = data_init.Last.iloc[i + 49 + date[0]] if is_time_to_exit(data[i:i + 50], exit_funcs, smas, emas, stop=0, bought_at=buy_price, max_price=high_price): exit_points_x.append(i + 49) exit_points_y.append(data_init.Bid.iloc[i + 49 + date[0]]) aux_buy = False total += round( ((data_init.Bid.iloc[i + 49 + date[0]] - buy_price) / buy_price) * 100, 2) full_log += str(data_init.time.iloc[i + 49 + date[0]]) + \ ' [SELL]@ ' + str(data_init.Bid.iloc[i + 49 + date[0]]) + '\n' full_log += '[P&L] > ' + str(total) + '%.' + '\n' del data_init # Use plot_data for just a few markets. If you try to run plot_data for several markets, # computer can start run really slow. try: if plot: plot_data(data, name=market, date=[0, 0], smas=smas, emas=[], entry_points=(entry_points_x, entry_points_y), exit_points=(exit_points_x, exit_points_y), show_smas=True, show_emas=False, show_bbands=True, to_file=to_file) except Exception as e: log("[ERROR] Ploting data: " + str(e), 0, log_level) del data #if len(exit_points_x): # log(market + ' > ' + str(total), log_level) log('[' + market + '][TOTAL]> ' + str(total) + '%.', 0, log_level) log(full_log, 1, log_level) if isnan(total): return 0 return total
def log_debug(s): if config.DEBUG: log(s)
def realtime(entry_funcs, exit_funcs, interval=var.default_interval, smas=var.default_smas, emas=var.default_volume_emas, refresh_interval=10, simulation=True, main_coins=["BTC", "USDT"], log_level=1): ''' Bot using realtime data, doesn't need DB or csv files to work. Args: entry_funcs(list): list of entry functions to test. exit_funcs(list): list of entry functions to test. markets(string): list with markets to backtest or empty to run all available markets. interval(string): time between measures. smas(list): list of SMA values to use. emas(list): list of EMA values to use. refresh_interval(int): Data refresh rate. simulation(bool): Defines if it's running as a simulation or real money mode. main_coins(list): ''' #log_level: # 0 - Only presents total. # 1 - Writes logs to file. # 2 - Writes logs to file and prints on screen. # Default is 2. #var.global_log_level = log_level signal.signal(signal.SIGINT, signal_handler) validate = smas[-1] + 5 if not isinstance(entry_funcs, list): entry_funcs = [entry_funcs] if not isinstance(exit_funcs, list): exit_funcs = [exit_funcs] buy_list = {} # Owned coins list. coins = {} if simulation: bt = Bittrex('', '') log("Starting Bot", 1, log_level) #print 'starting...' else: try: bt = RiskManagement(var.ky, var.sct) except: log("[Error] Couldn't connect to Bittrex", 0, log_level) sys.exit(1) while True: start_time = time() markets = bt.get_market_summaries()['result'] for market in markets: # Needed to pass unicode to string. market_name = str(market['MarketName']) # Checks if pair is included in main coins. if market_name.split('-')[0] in main_coins: # Checks if market already exists in analysed coins. if market_name in coins: # Checks if has enough data to analyse. if coins[market_name] == validate: locals()[market_name] = pd.DataFrame.append( locals()[market_name], [market]).tail(validate) # If not adds data and keep going. else: locals()[market_name] = pd.DataFrame.append( locals()[market_name], [market]) coins[market_name] += 1 continue # If not adds coin to system. else: locals()[market_name] = pd.DataFrame([market]) coins[market_name] = 1 continue data = locals()[market_name].rename(index=str, columns={ "OpenBuyOrders": "OpenBuy", "OpenSellOrders": "OpenSell" }) # Checks if coin is in buy portfolio and looks for a sell opportunity. if market_name in buy_list: # Needed to make use of stop loss and trailing stop loss functions. if buy_list[market_name]['max_price'] < data.Bid.iloc[-1]: buy_list[market_name]['max_price'] = data.Bid.iloc[-1] if is_time_to_exit( data, exit_funcs, bought_at=buy_list[market_name]['bought_at'], max_price=buy_list[market_name]['max_price'], count=buy_list[market_name]['count']): if not simulation: #REAL sell_res = bt.sell( market_name, buy_list[market_name]['quantity'], data.Bid.iloc[-1]) # MUDAR sold_at = sell_res log('[SELL]@ ' + str(sold_at) +\ ' > ' + market_name, 1, log_level) log('[P&L] ' + market_name + '> ' +\ str(round(((sold_at-\ buy_list[market_name]['bought_at'])/\ buy_list[market_name]['bought_at'])*100,2)) +\ '%.', 1, log_level) else: #SIMULATION #print '> https://bittrex.com/Market/Index?MarketName=' + market_name log('[SELL]@ ' + str(data.Bid.iloc[-1]) +\ ' > ' + market_name, 1, log_level) log('[P&L] ' + market_name + '> ' +\ str(round(((data.Bid.iloc[-1]-\ buy_list[market_name]['bought_at'])/\ buy_list[market_name]['bought_at'])*100,2)) +\ '%.', 1, log_level) del buy_list[market_name] #if not time to exit increment count. else: buy_list[market_name]['count'] += 1 # if has no coins from a certain market checks if is time to buy. else: if is_time_to_buy(data, entry_funcs): if not simulation: #REAL sucs, msg = bt.buy(market, data.Ask.iloc[-1] * 1.01) if sucs: buy_list[market_name] = {} buy_list[market_name]['bought_at'] = msg[0] buy_list[market_name]['max_price'] = msg[0] buy_list[market_name]['quantity'] = msg[1] buy_list[market_name]['count'] = 0 log('[BUY]@ ' + str(msg[0]) +\ #' > ' + funcs.func_name +\ ' > ' + market_name, 1, log_level) else: log( "[XXXX] Could not buy @ " + data.Ask.iloc[-1] * 1.01 + "\n[MSG>] " + msg, 0, log_level) else: #SIMULATION buy_list[market_name] = {} buy_list[market_name]['bought_at'] = data.Ask.iloc[ -1] buy_list[market_name]['max_price'] = data.Ask.iloc[ -1] buy_list[market_name]['quantity'] = 1 buy_list[market_name]['count'] = 0 #print '> https://bittrex.com/Market/Index?MarketName='+market_name log('[BUY]@ ' + str(data.Ask.iloc[-1]) +\ #' > ' + funcs.func_name +\ ' > ' + market_name, 1, log_level) # In case of processing time is bigger than *refresh_interval* doesn't sleep. if refresh_interval - (time() - start_time) > 0: sleep(refresh_interval - (time() - start_time))
def log_info(s): log(s)
def crawl(sport, year, division, org, game, url, neutral=False): global data data = data.format(sport, year, division) gamename = game.replace('/', '.') def readFlag(flag): if not os.path.exists(os.path.join(data, org, gamename)): os.mkdir(os.path.join(data, org, gamename)) return os.path.exists(os.path.join(data, org, gamename, flag)) def setFlag(flag): with open(os.path.join(data, org, gamename, flag), 'w') as f: pass if neutral and not readFlag(".neutral"): setFlag(".neutral") filename = os.path.join(data, org, gamename, "{}.csv") if not readFlag(".done"): try: gamelink = urljoin(domain, url) log("{} {} {} {} {} {}".format(sport, year, division, org, game, dumpURL(gamelink))) gs = parseURL(gamelink) sleep(2) gamescore = None gameinfo = None periods = [] teams = [] nextPeriod = 0 for table in gs.select("div.header_menu a"): if ( table["href"] == "#" or not ( table["href"].startswith("/game/box_score") or table["href"].startswith("/game/play_by_play") ) ): continue tablelink = urljoin(domain, table["href"]) print2("{} \033[4m{}\033[0m".format(table.text.strip(), tablelink)) ts = parseURL(tablelink) if gamescore is None: gamescore = parseTable(ts.select("table:nth-of-type(1)")[0]) dumpTable( gamescore, filename.format("Score") ) if gameinfo is None: gameinfo = transposeTable( parseTable(ts.select("table:nth-of-type(3)")[0]) + parseTable(ts.select("table:nth-of-type(4)")[0]) ) dumpTable( gameinfo, filename.format("Info") ) teams = [gamescore[1][0].text.strip(), gamescore[2][0].text.strip()] periods = [v.text.strip() for v in gamescore[0][1:]] if table["href"].startswith("/game/box_score"): if table.text.strip() == "Box Score": sfilename = filename.format("Box Score - {}") else: sfilename = filename.format(periods[nextPeriod] + " - {}") nextPeriod += 1 dumpTable( parseTable(ts.select("table:nth-of-type(5)")[0], header=1), sfilename.format(teams[0]) ) dumpTable( parseTable(ts.select("table:nth-of-type(6)")[0], header=1), sfilename.format(teams[1]) ) elif table["href"].startswith("/game/play_by_play"): sfilename = filename.format("Play by Play - {}") for (i, period) in enumerate(periods[:-1]): dumpTable( parseTable(ts.select("table:nth-of-type({})".format(6 + 2 * i))[0], header=0), sfilename.format(period) ) sleep(2) if gamescore == gameinfo == None: raise Exception("Not a game.") setFlag(".done") sleep(2) except Exception as e: print2(colored("Error: ", "red"), e) finally: print2() if not readFlag(".parsed"): try: gamelink = urljoin(domain, url) log("{} {} {} {} {} {}".format(sport, year, division, org, game, dumpURL(gamelink))) print2("Parsing...") gamescore = loadTable(filename.format("Score")) sfilename = filename.format("Box Score - {}") teams = [gamescore[1][0], gamescore[2][0]] with open(filename.format("Box Score - All (Parsed)"), "w") as af: for team in teams: boxScore = parseBoxScore( sfilename.format(team), filename.format("Info"), team, "All" ) rawDumpTable(boxScore[(0 if team == teams[0] else 1):], af) sfilename = filename.format("Play by Play - {}") periods = gamescore[0][1:] with open(filename.format("Play by Play - All (Parsed)"), "w") as af: for period in periods[:-1]: playByPlay = parsePlayByPlay( sfilename.format(period), period, filename.format("Info") ) rawDumpTable(playByPlay[(0 if period == periods[0] else 1):], af) setFlag(".parsed") except Exception as e: print2(colored("Error: ", "red"), e) finally: print2()
sleep(2) for org in s.select("table a"): orgname = org.text.strip() if filterOrg != None and filterOrg != orgname: continue try: if not os.path.exists(os.path.join(data, orgname)): os.mkdir(os.path.join(data, orgname)) orglink = urljoin(domain, org['href']) print2() log("{} {} {} {} {}".format(sport, year, division, orgname, dumpURL(orglink))) cs = parseURL(orglink) for link in cs.select("#contentarea > a"): if link.text.strip() == "Roster": tq.enqueue( crawlTeam, sport, year, division, org.text.strip(), "Roster", urljoin(domain, link["href"]), 1, at_front=atFront )