def __init__(self, file, logLimit, exchange=''): self.jsonOutputFile = file self.jsonOutput = {} self.clearStatusValues() self.jsonOutputLog = RingBuffer(logLimit) self.jsonOutput['exchange'] = exchange self.jsonOutput['label'] = Config.get("BOT", "label", "Lending Bot")
def __init__(self, api_key, secret): self.APIKey = api_key self.Secret = secret self.req_per_sec = 6 self.req_time_log = RingBuffer(self.req_per_sec) self.lock = threading.RLock() socket.setdefaulttimeout(int(Config.get("BOT", "timeout", 30, 1, 180)))
def __init__(self, cfg): self.cfg = cfg self.APIKey = self.cfg.get("API", "apikey", None) self.Secret = self.cfg.get("API", "secret", None) self.req_per_sec = 6 self.req_time_log = RingBuffer(self.req_per_sec) self.lock = threading.RLock() socket.setdefaulttimeout(int(Config.get("BOT", "timeout", 30, 1, 180)))
def get_rate_list(self, cur, seconds): """ Query the database (cur) for rates that are within the supplied number of seconds and now. :param cur: The currency (database) to remove data from :param seconds: The number of seconds between the oldest order returned and now. :return: A pandas DataFrame object with named columns ('time', 'rate0', 'rate1',...) """ # Request more data from the DB than we need to allow for skipped seconds request_seconds = int(seconds * 1.1) full_list = Config.get_all_currencies() if isinstance(cur, sqlite.Connection): db_con = cur else: if cur not in full_list: raise ValueError( "{0} is not a valid currency, must be one of {1}".format( cur, full_list)) if cur not in self.currencies_to_analyse: return [] db_con = self.create_connection(cur) price_levels = ['rate0'] rates = self.get_rates_from_db(db_con, from_date=time.time() - request_seconds, price_levels=price_levels) if len(rates) == 0: return [] df = pd.DataFrame(rates) columns = ['time'] columns.extend(price_levels) try: df.columns = columns except: if self.ma_debug_log: print("DEBUG:get_rate_list: cols: {0} rates:{1} db:{2}".format( columns, rates, db_con)) raise # convert unixtimes to datetimes so we can resample df.time = pd.to_datetime(df.time, unit='s') # If we don't have enough data return df, otherwise the resample will fill out all values with the same data. # Missing data tolerance allows for a percentage to be ignored and filled in by resampling. if len(df) < seconds * (self.data_tolerance / 100): return df # Resample into 1 second intervals, average if we get two in the same second and fill any empty spaces with the # previous value df = df.resample('1s', on='time').mean().ffill() return df
def __init__(self, cfg, log): super(Poloniex, self).__init__(cfg, log) self.cfg = cfg self.log = log self.APIKey = self.cfg.get("API", "apikey", None) self.Secret = self.cfg.get("API", "secret", None) self.req_per_period = 6 self.default_req_period = 1000 # milliseconds self.req_period = self.default_req_period self.req_time_log = RingBuffer(self.req_per_period) self.lock = threading.RLock() socket.setdefaulttimeout(int(Config.get("BOT", "timeout", 30, 1, 180))) self.api_debug_log = self.cfg.getboolean("BOT", "api_debug_log")
def get_rate_list(self, cur): FULL_LIST = Config.get_all_currencies() if cur not in FULL_LIST: raise ValueError( "{0} is not a valid currency, must be one of {1}".format( cur, FULL_LIST)) if cur not in self.open_files: return [] with open(self.open_files[cur], 'r') as f: reader = csv.reader(f) rates = [] for row in reader: rates.append(row[1]) rates = map(float, rates) return rates
def get_rate_list(self, cur, seconds): """ Query the database (cur) for rates that are within the supplied number of seconds and now. :param cur: The currency (database) to remove data from :param seconds: The number of seconds between the oldest order returned and now. :return: A pandas DataFrame object with named columns ('time', 'rate0', 'rate1',...) """ # Request more data from the DB than we need to allow for skipped seconds request_seconds = int(seconds * 1.1) full_list = Config.get_all_currencies() if isinstance(cur, sqlite.Connection): db_con = cur else: if cur not in full_list: raise ValueError("{0} is not a valid currency, must be one of {1}".format(cur, full_list)) if cur not in self.currencies_to_analyse: return [] db_con = self.create_connection(cur) price_levels = ['rate0'] rates = self.get_rates_from_db(db_con, from_date=time.time() - request_seconds, price_levels=price_levels) if len(rates) == 0: return [] df = pd.DataFrame(rates) columns = ['time'] columns.extend(price_levels) try: df.columns = columns except: if self.ma_debug_log: print("DEBUG:get_rate_list: cols: {0} rates:{1} db:{2}".format(columns, rates, db_con)) raise # convert unixtimes to datetimes so we can resample df.time = pd.to_datetime(df.time, unit='s') # If we don't have enough data return df, otherwise the resample will fill out all values with the same data. # Missing data tolerance allows for a percentage to be ignored and filled in by resampling. if len(df) < seconds * (self.data_tolerance / 100): return df # Resample into 1 second intervals, average if we get two in the same second and fill any empty spaces with the # previous value df = df.resample('1s', on='time').mean().ffill() return df
def create_connection(self, cur, db_path=None, db_type='sqlite3'): """ Create a connection to the sqlite DB. This will create a new file if one doesn't exist. We can use :memory: here for db_path if we don't want to store the data on disk :param cur: The currency (database) in the DB :param db_path: DB directory :return: Connection object or None """ if db_path is None: prefix = Config.get_exchange() db_path = os.path.join(self.db_dir, '{0}-{1}.db'.format(prefix, cur)) try: con = sqlite.connect(db_path) return con except Error as ex: print(ex.message)
def update_market_thread(self, cur, levels=None): """ This is where the main work is done for recording the market data. The loop will not exit and continuously polls Poloniex for the current loans in the book. :param cur: The currency (database) to remove data from :param levels: The depth of offered rates to store """ if levels is None: levels = self.recorded_levels db_con = self.create_connection(cur) while True: try: raw_data = self.api.return_loan_orders(cur, levels)['offers'] except Exception as ex: self.print_traceback(ex, "Error in returning data from Poloniex") market_data = [] for i in xrange(levels): market_data.append(str(raw_data[i]['rate'])) market_data.append(str(raw_data[i]['amount'])) market_data.append('0') # Percentile field not being filled yet. insert_sql = "INSERT INTO loans (" for level in xrange(levels): insert_sql += "rate{0}, amnt{0}, ".format(level) insert_sql += "percentile) VALUES ({0});".format( ','.join(market_data)) # percentile = 0 with db_con: try: db_con.execute(insert_sql) except Exception as ex: self.print_traceback( ex, "Error inserting market data into DB") if Config.get_exchange() == 'BITFINEX': # We don't have a coach for bitfinex, so sleep here time.sleep(5)
import modules.Lending as Lending parser = argparse.ArgumentParser() # Start args. parser.add_argument("-cfg", "--config", help="Location of custom configuration file, overrides settings below") parser.add_argument("-dry", "--dryrun", help="Make pretend orders", action="store_true") args = parser.parse_args() # End args. # Start handling args. dry_run = bool(args.dryrun) if args.config: config_location = args.config else: config_location = "default.cfg" # End handling args. Config.init([config_location]) # Config format: Config.get(category, option, default_value=False, lower_limit=False, upper_limit=False) # A default_value "None" means that the option is required and the bot will not run without it. # Do not use lower or upper limit on any config options which are not numbers. # Define the variable from the option in the module where you use it. api_key = Config.get("API", "apikey", None) api_secret = Config.get("API", "secret", None) auto_renew = int(Config.get("BOT", "autorenew", None, 0, 1)) output_currency = Config.get("BOT", "outputCurrency", "BTC") end_date = Config.get("BOT", "endDate") json_output_enabled = Config.has_option("BOT", "jsonfile") and Config.has_option("BOT", "jsonlogsize") def timestamp(): ts = time.time() return datetime.datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S")
dry_run = bool(args.dryrun) if args.config: config_location = args.config else: config_location = 'default.cfg' # End handling args. ''' # Config format: Config.get(category, option, default_value=False, lower_limit=False, upper_limit=False) # A default_value "None" means that the option is required and the bot will not run without it. # Do not use lower or upper limit on any config options which are not numbers. # Define the variable from the option in the module where you use it. dry_run = False config_location = 'default.cfg' Config.init(config_location) output_currency = Config.get('BOT', 'outputCurrency', 'BTC') end_date = Config.get('BOT', 'endDate') exchange = Config.get_exchange() json_output_enabled = Config.has_option( 'BOT', 'jsonfile') and Config.has_option('BOT', 'jsonlogsize') jsonfile = Config.get('BOT', 'jsonfile', '') username = sys.argv[6] # Configure logging #log = Logger(jsonfile, Decimal(Config.get('BOT', 'jsonlogsize', 200)), exchange) print("JSON file Name:") print(Config.get('BOT', 'jsonfile', '') + username + '_botlog.json')
import os from modules.LanguageMenu import LanguageMenu from modules.MainMenu import MainMenu import pygame from pygame.locals import * from xml.sax import make_parser from modules.Configuration import * from modules.Constants import Constants if __name__=="__main__": parser = make_parser() curHandler = Configuration() parser.setContentHandler(curHandler) parser.parse(open("config/config.xml")) pygame.init() Constants.SCREEN=pygame.display.set_mode(curHandler.getRes()) pygame.display.set_caption(curHandler.getName()) pygame.mouse.set_visible(curHandler.getMouseVisibility()) pygame.mixer.init() l = LanguageMenu() l.run() m = MainMenu() m.run()
"--config", help="Location of custom configuration file, overrides settings below") parser.add_argument("-dry", "--dryrun", help="Make pretend orders", action="store_true") args = parser.parse_args() # End args. # Start handling args. dry_run = bool(args.dryrun) if args.config: config_location = args.config else: config_location = 'default.cfg' # End handling args. Config.init(config_location) # Config format: Config.get(category, option, default_value=False, lower_limit=False, upper_limit=False) # A default_value "None" means that the option is required and the bot will not run without it. # Do not use lower or upper limit on any config options which are not numbers. # Define the variable from the option in the module where you use it. output_currency = Config.get('BOT', 'outputCurrency', 'BTC') end_date = Config.get('BOT', 'endDate') json_output_enabled = Config.has_option( 'BOT', 'jsonfile') and Config.has_option('BOT', 'jsonlogsize') log = Logger(Config.get('BOT', 'jsonfile', ''), Decimal(Config.get('BOT', 'jsonlogsize', -1))) api = Poloniex(Config.get("API", "apikey", None), Config.get("API", "secret", None)) MaxToLend.init(Config, log) Data.init(api, log)
def lend(): try: open('lendingbot.py', 'r') except IOError: os.chdir(os.path.dirname(sys.argv[0])) # Allow relative paths parser = argparse.ArgumentParser() # Start args. parser.add_argument("-cfg", "--config", help="Location of custom configuration file, overrides settings below") parser.add_argument("-dry", "--dryrun", help="Make pretend orders", action="store_true") args = parser.parse_args() # End args. # Start handling args. dry_run = bool(args.dryrun) if args.config: config_location = args.config else: config_location = 'default.cfg' # End handling args. # Config format: Config.get(category, option, default_value=False, lower_limit=False, upper_limit=False) # A default_value "None" means that the option is required and the bot will not run without it. # Do not use lower or upper limit on any config options which are not numbers. # Define the variable from the option in the module where you use it. Config.init(config_location) output_currency = Config.get('BOT', 'outputCurrency', 'BTC') end_date = Config.get('BOT', 'endDate') exchange = Config.get_exchange() json_output_enabled = Config.has_option('BOT', 'jsonfile') and Config.has_option('BOT', 'jsonlogsize') jsonfile = Config.get('BOT', 'jsonfile', '') # Configure web server web_server_enabled = Config.getboolean('BOT', 'startWebServer') if web_server_enabled: if json_output_enabled is False: # User wants webserver enabled. Must have JSON enabled. Force logging with defaults. json_output_enabled = True jsonfile = Config.get('BOT', 'jsonfile', 'www/botlog.json') import modules.WebServer as WebServer WebServer.initialize_web_server(Config) # Configure logging log = Logger(jsonfile, Decimal(Config.get('BOT', 'jsonlogsize', 200)), exchange) # initialize the remaining stuff api = ExchangeApiFactory.createApi(exchange, Config, log) MaxToLend.init(Config, log) Data.init(api, log) Config.init(config_location, Data) notify_conf = Config.get_notification_config() if Config.has_option('MarketAnalysis', 'analyseCurrencies'): from modules.MarketAnalysis import MarketAnalysis # Analysis.init(Config, api, Data) analysis = MarketAnalysis(Config, api) analysis.run() else: analysis = None Lending.init(Config, api, log, Data, MaxToLend, dry_run, analysis, notify_conf) # load plugins PluginsManager.init(Config, api, log, notify_conf) print 'Welcome to ' + Config.get("BOT", "label", "Lending Bot") + ' on ' + exchange try: while True: try: Data.update_conversion_rates(output_currency, json_output_enabled) PluginsManager.before_lending() Lending.transfer_balances() Lending.cancel_all() Lending.lend_all() PluginsManager.after_lending() log.refreshStatus(Data.stringify_total_lent(*Data.get_total_lent()), Data.get_max_duration(end_date, "status")) log.persistStatus() sys.stdout.flush() time.sleep(Lending.get_sleep_time()) except KeyboardInterrupt: # allow existing the main bot loop raise except Exception as ex: log.log_error(ex.message) log.persistStatus() if 'Invalid API key' in ex.message: print "!!! Troubleshooting !!!" print "Are your API keys correct? No quotation. Just plain keys." exit(1) elif 'Nonce must be greater' in ex.message: print "!!! Troubleshooting !!!" print "Are you reusing the API key in multiple applications? Use a unique key for every application." exit(1) elif 'Permission denied' in ex.message: print "!!! Troubleshooting !!!" print "Are you using IP filter on the key? Maybe your IP changed?" exit(1) elif 'timed out' in ex.message: print "Timed out, will retry in " + str(Lending.get_sleep_time()) + "sec" elif isinstance(ex, BadStatusLine): print "Caught BadStatusLine exception from Poloniex, ignoring." elif 'Error 429' in ex.message: additional_sleep = max(130.0-Lending.get_sleep_time(), 0) sum_sleep = additional_sleep + Lending.get_sleep_time() log.log_error('IP has been banned due to many requests. Sleeping for {} seconds'.format(sum_sleep)) time.sleep(additional_sleep) # Ignore all 5xx errors (server error) as we can't do anything about it (https://httpstatuses.com/) elif isinstance(ex, URLError): print "Caught {0} from exchange, ignoring.".format(ex.message) elif isinstance(ex, ApiError): print "Caught {0} reading from exchange API, ignoring.".format(ex.message) else: print traceback.format_exc() print "v{0} Unhandled error, please open a Github issue so we can fix it!".format(Data.get_bot_version()) if notify_conf['notify_caught_exception']: log.notify("{0}\n-------\n{1}".format(ex, traceback.format_exc()), notify_conf) sys.stdout.flush() time.sleep(Lending.get_sleep_time()) except KeyboardInterrupt: if web_server_enabled: WebServer.stop_web_server() PluginsManager.on_bot_exit() log.log('bye') print 'bye' os._exit(0) # Ad-hoc solution in place of 'exit(0)' TODO: Find out why non-daemon thread(s) are hanging on exit
args = parser.parse_args() # End args. # Start handling args. dry_run = bool(args.dryrun) if args.config: config_location = args.config else: config_location = 'default.cfg' # End handling args. # Config format: Config.get(category, option, default_value=False, lower_limit=False, upper_limit=False) # A default_value "None" means that the option is required and the bot will not run without it. # Do not use lower or upper limit on any config options which are not numbers. # Define the variable from the option in the module where you use it. Config.init(config_location) output_currency = Config.get('BOT', 'outputCurrency', 'BTC') end_date = Config.get('BOT', 'endDate') exchange = Config.get_exchange() json_output_enabled = Config.has_option('BOT', 'jsonfile') and Config.has_option('BOT', 'jsonlogsize') jsonfile = Config.get('BOT', 'jsonfile', '') # Configure web server web_server_enabled = Config.getboolean('BOT', 'startWebServer') if web_server_enabled: if json_output_enabled is False: # User wants webserver enabled. Must have JSON enabled. Force logging with defaults. json_output_enabled = True jsonfile = Config.get('BOT', 'jsonfile', 'www/botlog.json')
import os from modules.LanguageMenu import LanguageMenu from modules.MainMenu import MainMenu import pygame from pygame.locals import * from xml.sax import make_parser from modules.Configuration import * from modules.Constants import Constants if __name__ == "__main__": parser = make_parser() curHandler = Configuration() parser.setContentHandler(curHandler) parser.parse(open("config/config.xml")) pygame.init() Constants.SCREEN = pygame.display.set_mode(curHandler.getRes()) pygame.display.set_caption(curHandler.getName()) pygame.mouse.set_visible(curHandler.getMouseVisibility()) pygame.mixer.init() l = LanguageMenu() l.run() m = MainMenu() m.run()
args = parser.parse_args() # End args. # Start handling args. dry_run = bool(args.dryrun) if args.config: config_location = args.config else: config_location = 'default.cfg' # End handling args. # Config format: Config.get(category, option, default_value=False, lower_limit=False, upper_limit=False) # A default_value "None" means that the option is required and the bot will not run without it. # Do not use lower or upper limit on any config options which are not numbers. # Define the variable from the option in the module where you use it. Config.init(config_location) output_currency = Config.get('BOT', 'outputCurrency', 'BTC') end_date = Config.get('BOT', 'endDate') exchange = Config.get_exchange() json_output_enabled = Config.has_option( 'BOT', 'jsonfile') and Config.has_option('BOT', 'jsonlogsize') jsonfile = Config.get('BOT', 'jsonfile', '') # Configure web server web_server_enabled = Config.getboolean('BOT', 'startWebServer') if web_server_enabled: if json_output_enabled == False: # User wants webserver enabled. Must have JSON enabled. Force logging with defaults. json_output_enabled = True
except IOError: os.chdir(os.path.dirname(sys.argv[0])) # Allow relative paths parser = argparse.ArgumentParser() # Start args. parser.add_argument("-cfg", "--config", help="Location of custom configuration file, overrides settings below") parser.add_argument("-dry", "--dryrun", help="Make pretend orders", action="store_true") args = parser.parse_args() # End args. # Start handling args. dry_run = bool(args.dryrun) if args.config: config_location = args.config else: config_location = 'default.cfg' # End handling args. Config.init(config_location) # Config format: Config.get(category, option, default_value=False, lower_limit=False, upper_limit=False) # A default_value "None" means that the option is required and the bot will not run without it. # Do not use lower or upper limit on any config options which are not numbers. # Define the variable from the option in the module where you use it. output_currency = Config.get('BOT', 'outputCurrency', 'BTC') end_date = Config.get('BOT', 'endDate') json_output_enabled = Config.has_option('BOT', 'jsonfile') and Config.has_option('BOT', 'jsonlogsize') log = Logger(Config.get('BOT', 'jsonfile', ''), Config.get('BOT', 'jsonlogsize', -1)) api = Poloniex(Config.get("API", "apikey", None), Config.get("API", "secret", None)) MaxToLend.init(Config, log) Data.init(api, log) Config.init(config_location, Data) if Config.has_option('BOT', 'analyseCurrencies'):
import pandas as pd # Hack to get relative imports - probably need to fix the dir structure instead but we need this at the minute for # pytest to work import os, sys, inspect currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) parentdir = os.path.dirname(currentdir) sys.path.insert(0, parentdir) from modules.MarketAnalysis import MarketAnalysis from modules.Configuration import get_all_currencies from modules.Poloniex import Poloniex import modules.Configuration as Config import modules.Data as Data Config.init('default.cfg', Data) api = Poloniex(Config, None) Data.init(api, None) MA = MarketAnalysis(Config, api) def new_db(): db_con = MA.create_connection(None, ':memory:') MA.create_rate_table(db_con, 3) return db_con def random_rates(): return lists(floats(min_value=0.00001, max_value=100, allow_nan=False, allow_infinity=False), min_size=0, max_size=100).example()
import time # Hack to get relative imports - probably need to fix the dir structure instead but we need this at the minute for # pytest to work import os, sys, inspect currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) parentdir = os.path.dirname(currentdir) sys.path.insert(0, parentdir) from modules.Poloniex import Poloniex import modules.Configuration as Config import modules.Data as Data import threading Config.init('default.cfg', Data) api = Poloniex(Config.get("API", "apikey", None), Config.get("API", "secret", None)) def multiple_api_queries(n): try: for i in xrange(n): # print 'api_query ' + str(i + 1) thread1 = threading.Thread(target=api.return_open_loan_offers) thread1.start() except Exception as e: assert False, 'api_query ' + str(i + 1) + ':' + e.message # Test fast api calls # def test_multiple_calls(): # multiple_api_queries(9)
# Hack to get relative imports - probably need to fix the dir structure instead but we need this at the minute for # pytest to work import os, sys, inspect currentdir = os.path.dirname( os.path.abspath(inspect.getfile(inspect.currentframe()))) parentdir = os.path.dirname(currentdir) sys.path.insert(0, parentdir) from modules.MarketAnalysis import MarketAnalysis from modules.Configuration import FULL_LIST from modules.Poloniex import Poloniex import modules.Configuration as Config import modules.Data as Data Config.init('default.cfg', Data) api = Poloniex(Config.get("API", "apikey", None), Config.get("API", "secret", None)) Data.init(api, None) MA = MarketAnalysis(Config, api) def create_dummy_rate_file(rate_file): rates = lists(floats(min_value=0.00001, allow_nan=False, allow_infinity=False), min_size=0, max_size=100).example() max_year = datetime.datetime.now().year date_times = lists(datetimes(min_year=2016, max_year=max_year), min_size=len(rates),
""" import argparse import pandas as pd from modules.Poloniex import Poloniex, PoloniexApiError import modules.Configuration as Config # Fetch configuration for the API key & secret inside parser = argparse.ArgumentParser() parser.add_argument("-cfg", "--config", help="Custom config file") args = parser.parse_args() if args.config: config_location = args.config else: config_location = 'default.cfg' Config.init(config_location) api = Poloniex(Config.get('API', 'apikey', None), Config.get('API', 'secret', None)) active_loans = api.return_active_loans() active_loans_sum = pd.DataFrame.from_records(active_loans['provided']).query('currency == "BTC"')['amount'].map(lambda x: x.replace('.','')).astype('int').sum() idle_loan_balance = api.return_available_account_balances("lending")["lending"] # idle_loan_balance = int(idle_loan_balance.replace('.', '')) if idle_loan_balance == []: idle_loan_balance = 0 else: idle_loan_balance = int(idle_loan_balance["BTC"].replace('.', '')) print("Idle: {0}".format(idle_loan_balance)) print("Active: {0}".format(active_loans_sum)) print("Total satoshi: {0}".format(idle_loan_balance + active_loans_sum))