def __init__(self, json_file, json_log_size, exchange):
self._lent = ''
self._daysRemaining = ''
self.output = JsonOutput(json_file, json_log_size, exchange)
self.compactLog = bool(Config.get('BOT', 'jsonlogcompact', False))
self.refreshStatus()
self.persistStatus()
def __init__(self, file, logLimit, exchange=''):
self.jsonOutputFile = file
self.jsonOutput = {}
self.clearStatusValues()
self.jsonOutputLog = RingBuffer(logLimit)
self.jsonOutput['exchange'] = exchange
self.jsonOutput['label'] = Config.get("BOT", "label", "Lending Bot")
def __init__(self, cfg, weblog):
super(Poloniex, self).__init__(cfg, weblog)
self.logger = logging.getLogger(__name__)
self.req_per_period = 6
self.default_req_period = 1000 # milliseconds
self.req_period = self.default_req_period
self.req_time_log = RingBuffer(self.req_per_period)
self.lock = threading.RLock()
self.timeout = int(Config.get("BOT", "timeout", 30, 1, 180))
self.url_public = 'https://poloniex.com/public'
self.url_private = 'https://poloniex.com/tradingApi'
def get_rate_list(self, cur, seconds):
"""
Query the database (cur) for rates that are within the supplied number of seconds and now.
:param cur: The currency (database) to remove data from
:param seconds: The number of seconds between the oldest order returned and now.
:return: A pandas DataFrame object with named columns ('time', 'rate0', 'rate1',...)
"""
# Request more data from the DB than we need to allow for skipped seconds
request_seconds = int(seconds * 1.1)
full_list = Config.get_all_currencies()
if isinstance(cur, sqlite.Connection):
db_con = cur
else:
if cur not in full_list:
raise ValueError(
"{0} is not a valid currency, must be one of {1}".format(
cur, full_list))
if cur not in self.currencies_to_analyse:
return []
db_con = self.create_connection(cur)
price_levels = ['rate0']
rates = self.get_rates_from_db(db_con,
from_date=time.time() - request_seconds,
price_levels=price_levels)
if len(rates) == 0:
return []
df = pd.DataFrame(rates)
columns = ['time']
columns.extend(price_levels)
try:
df.columns = columns
except Exception as ex:
self.logger.error(
"get_rate_list: cols: {0} rates:{1} db:{2}".format(
columns, rates, db_con))
raise ex
# convert unixtimes to datetimes so we can resample
df.time = pd.to_datetime(df.time, unit='s')
# If we don't have enough data return df, otherwise the resample will fill out all values with the same data.
# Missing data tolerance allows for a percentage to be ignored and filled in by resampling.
if len(df) < seconds * (self.data_tolerance / 100):
return df
# Resample into 1 second intervals, average if we get two in the same second and fill any empty spaces with the
# previous value
df = df.resample('1s', on='time').mean().ffill()
return df
def create_connection(self, cur, db_path=None, db_type='sqlite3'):
"""
Create a connection to the sqlite DB. This will create a new file if one doesn't exist. We can use :memory:
here for db_path if we don't want to store the data on disk
:param cur: The currency (database) in the DB
:param db_path: DB directory
:return: Connection object or None
"""
if db_path is None:
prefix = Config.get_exchange()
db_path = os.path.join(self.db_dir,
'{0}-{1}.db'.format(prefix, cur))
try:
con = sqlite.connect(db_path)
return con
except Error as ex:
self.logger.error(ex.message)
# pytest to work
import os
import sys
import inspect
currentdir = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0, parentdir)
from coinlendingbot.MarketAnalysis import MarketAnalysis # nopep8
from coinlendingbot.Configuration import get_all_currencies # nopep8
from coinlendingbot.Poloniex import Poloniex # nopep8
import coinlendingbot.Configuration as Config # nopep8
import coinlendingbot.Data as Data # nopep8
Config.init(open('poloniex_test.cfg'), Data)
api = Poloniex(Config, None)
Data.init(api, None)
MA = MarketAnalysis(Config, api)
def new_db():
db_con = MA.create_connection(None, ':memory:')
MA.create_rate_table(db_con, 3)
return db_con
def random_rates():
return lists(floats(min_value=0.00001,
max_value=100,
allow_nan=False,
def main(config, logconfig, dryrun):
logging.config.fileConfig(logconfig)
logger = logging.getLogger('main')
logger.debug('config: {}, logconfig: {}, dryrun: {}'.format(
config, logconfig, dryrun))
Config.init(config)
output_currency = Config.get('BOT', 'outputCurrency', 'BTC')
end_date = Config.get('BOT', 'endDate')
exchange = Config.get_exchange()
json_output_enabled = Config.has_option(
'BOT', 'jsonfile') and Config.has_option('BOT', 'jsonlogsize')
jsonfile = Config.get('BOT', 'jsonfile', '')
# Configure web server
web_server_enabled = Config.getboolean('BOT', 'startWebServer')
if web_server_enabled:
logger.info('Web server enabled.')
if json_output_enabled is False:
# User wants webserver enabled. Must have JSON enabled. Force logging with defaults.
json_output_enabled = True
jsonfile = Config.get('BOT', 'jsonfile', 'www/botweblog.json')
WebServer.initialize_web_server(Config)
# Configure logging to display on webpage
weblog = Logger(jsonfile, Decimal(Config.get('BOT', 'jsonlogsize', 200)),
exchange)
welcome = 'Welcome to {} on {}'.format(
Config.get("BOT", "label", "Lending Bot"), exchange)
logger.info(welcome)
weblog.log(welcome)
# initialize the remaining stuff
api = ExchangeApiFactory.createApi(exchange, Config, weblog)
MaxToLend.init(Config, weblog)
Data.init(api, weblog)
Config.init(config, Data)
notify_conf = Config.get_notification_config()
if Config.has_option('MarketAnalysis', 'analyseCurrencies'):
logger.info('MarketAnalysis enabled.')
from coinlendingbot.MarketAnalysis import MarketAnalysis
# Analysis.init(Config, api, Data)
analysis = MarketAnalysis(Config, api)
analysis.run()
else:
analysis = None
Lending.init(Config, api, weblog, Data, MaxToLend, dryrun, analysis,
notify_conf)
# load plugins
PluginsManager.init(Config, api, weblog, notify_conf)
try:
while True:
try:
logger.info('New round.')
Data.update_conversion_rates(output_currency,
json_output_enabled)
PluginsManager.before_lending()
Lending.transfer_balances()
Lending.cancel_all()
Lending.lend_all()
PluginsManager.after_lending()
weblog.refreshStatus(
Data.stringify_total_lent(*Data.get_total_lent()),
Data.get_max_duration(end_date, "status"))
weblog.persistStatus()
logger.info('Round finished.')
time.sleep(Lending.get_sleep_time())
except KeyboardInterrupt:
# allow existing the main bot loop
raise
except Exception as ex:
exc_type, exc_value, exc_traceback = sys.exc_info()
logger.debug(ex)
logger.debug(repr(traceback.format_tb(exc_traceback)))
weblog.log_error(ex.message)
weblog.persistStatus()
if 'Invalid API key' in ex.message:
logger.critical(
"!!! Troubleshooting !!! Are your API keys correct? No quotation. Just plain keys."
)
exit(1)
elif 'Nonce must be greater' in ex.message:
logger.critical(
"!!! Troubleshooting !!! Are you reusing the API key in multiple applications? "
+ "Use a unique key for every application.")
exit(1)
elif 'Permission denied' in ex.message:
logger.critical(
"!!! Troubleshooting !!! Are you using IP filter on the key?"
)
exit(1)
elif 'timed out' in ex.message:
logger.warn("Timed out, will retry in " +
str(Lending.get_sleep_time()) + "sec")
elif isinstance(ex, BadStatusLine):
logger.warn(
"Caught BadStatusLine exception from Poloniex, ignoring."
)
elif 'Error 429' in ex.message:
additional_sleep = max(130.0 - Lending.get_sleep_time(), 0)
sum_sleep = additional_sleep + Lending.get_sleep_time()
msg = 'IP has been banned due to many requests. Sleeping for {} seconds'.format(
sum_sleep)
weblog.log_error(msg)
logger.warn(msg)
if Config.has_option('MarketAnalysis',
'analyseCurrencies'):
if api.req_period <= api.default_req_period * 1.5:
api.req_period += 3
logger.warn(
"Caught ERR_RATE_LIMIT, sleeping capture and increasing request delay. Current"
+ " {0}ms".format(api.req_period))
weblog.log_error(
'Expect this 130s ban periodically when using MarketAnalysis, '
+ 'it will fix itself')
time.sleep(additional_sleep)
# Ignore all 5xx errors (server error) as we can't do anything about it (https://httpstatuses.com/)
elif isinstance(ex, URLError):
logger.error("Caught {0} from exchange, ignoring.".format(
ex.message))
elif isinstance(ex, ApiError):
logger.error(
"Caught {0} reading from exchange API, ignoring.".
format(ex.message))
else:
logger.error(traceback.format_exc())
logger.error(
"v{0} Unhandled error, please open a Github issue so we can fix it!"
.format(Data.get_bot_version()))
if notify_conf['notify_caught_exception']:
weblog.notify(
"{0}\n-------\n{1}".format(ex,
traceback.format_exc()),
notify_conf)
time.sleep(Lending.get_sleep_time())
except KeyboardInterrupt:
if web_server_enabled:
WebServer.stop_web_server()
PluginsManager.on_bot_exit()
weblog.log('bye')
logger.info('bye')
# Hack to get relative imports - probably need to fix the dir structure instead but we need this at the minute for
# pytest to work
import os
import sys
import inspect
currentdir = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0, parentdir)
from coinlendingbot.Bitfinex import Bitfinex # nopep8
import coinlendingbot.Configuration as Config # nopep8
import coinlendingbot.Data as Data # nopep8
Config.init(open('bitfinex_test.cfg'), Data)
api = Bitfinex(Config, None)
start_time = time.time()
def multiple_api_queries(n):
try:
for i in range(n):
print("Thread {}".format(i + 1))
thread1 = threading.Thread(target=call_get_open_loan_offers,
args=[(i + 1)])
thread1.start()
except Exception as e:
assert False, 'Thread ' + str(i + 1) + ':' + e.message