def initialize_databases(database_name, markets, logger=None):
"""
Create new database for data collection and new table for trades
:param database_name:
:param markets:
:param logger:
:return:
"""
db = Database(hostname=HOSTNAME,
username=USERNAME,
password=PASSCODE,
database_name='develop',
logger=logger)
# Create database if does not exist
db.create_database(database_name)
db.create_database(TRADEBOT_DATABASE)
db.close()
base_db = Database(hostname=HOSTNAME,
username=USERNAME,
password=PASSCODE,
database_name=database_name,
logger=logger)
# Create tables if does not exist
for market in markets:
base_db.create_price_table(market)
base_db.close()
tradebot_db = Database(hostname=HOSTNAME,
username=USERNAME,
password=PASSCODE,
database_name=TRADEBOT_DATABASE,
logger=logger)
tradebot_db.create_trade_table(database_name)
tradebot_db.close()
def run_tradebot(control_queue, data_queue, pending_order_queue,
completed_order_queue, markets, amount_per_call, table_name,
logger, skip_list):
"""
Run trading algorithm on real-time market data
:param pending_order_queue: Queue to pass orders to manager
:param completed_order_queue: Queue to recieve completed orders from manager
:param control_queue: Queue to control tradebot
:param data_queue: Queue to receive data from scraper
:param markets: List of markets
:param amount_per_call: Amount to purchase per buy order
:param table_name: Name of SQL table to record data
:param logger: Main logger
:param skip_list: List of markets to skip
:return:
"""
market_data = {}
market_status = {}
for market in markets:
if market not in skip_list:
market_status[market] = BittrexStatus(market=market)
market_data[market] = BittrexData(market=market)
# Initialize SQL database connection
db = Database(hostname=HOSTNAME,
username=USERNAME,
password=PASSCODE,
database_name=TRADEBOT_DATABASE,
logger=logger)
# Initialize Bittrex object
bittrex = Bittrex(api_key=BITTREX_CREDENTIALS.get('key'),
api_secret=BITTREX_CREDENTIALS.get('secret'),
api_version='v1.1')
try:
while True:
# Receive data from scraper
scraper_data = data_queue.get()
# Add received scraper data from running data
for market, entry in scraper_data.items():
if market not in skip_list:
if entry.get(
'wprice'
) > 0: # Temporary fix for entries with 0 price
market_data[market].datetime.append(
entry.get('datetime'))
market_data[market].wprice.append(entry.get('wprice'))
market_data[market].buy_volume.append(
entry.get('buy_volume'))
market_data[market].sell_volume.append(
entry.get('sell_volume'))
# Check if any orders completed
if not completed_order_queue.empty():
while not completed_order_queue.empty():
completed_order = completed_order_queue.get()
order_market = completed_order.market
# Update market statuses with completed orders
if completed_order.type == OrderType.BUY.name:
# Check if buy order skipped
if completed_order.status == OrderStatus.SKIPPED.name:
market_status[order_market].bought = False
market_status[order_market].buy_signal = None
logger.info(
'Tradebot: Received skipped buy order. Skipping buy order for {}.'
.format(order_market))
else:
market_status[
order_market].buy_order = completed_order
logger.info(
'Tradebot: Received completed buy order for {}.'
.format(order_market))
else:
market_status[
order_market].sell_order = completed_order
logger.info(
'Tradebot: Received completed sell order for {}.'.
format(order_market))
status = market_status[order_market]
# Completed buy and sell order for single market
if status.buy_order.status == OrderStatus.COMPLETED.name and \
status.sell_order.status == OrderStatus.COMPLETED.name:
profit = (status.sell_order.final_total +
status.buy_order.final_total).quantize(
BittrexConstants.DIGITS)
percent = (profit * Decimal(-100) /
status.buy_order.final_total).quantize(
Decimal(10)**-4)
formatted_buy_time = format_time(
status.buy_order.closed_time,
"%Y-%m-%d %H:%M:%S")
formatted_sell_time = format_time(
status.sell_order.closed_time,
"%Y-%m-%d %H:%M:%S")
logger.info(
'Tradebot: completed buy/sell order for {}.'.
format(order_market))
db.insert_query(
table_name,
format_tradebot_entry(
order_market, formatted_buy_time,
status.buy_signal,
status.buy_order.actual_price,
status.buy_order.final_total,
formatted_sell_time, status.sell_signal,
status.sell_order.actual_price,
status.sell_order.final_total, profit,
percent))
# Reset buy/sell orders and buy/sell signals
status.clear_orders()
status.buy_signal = None
status.sell_signal = None
else:
logger.error(
'Tradebot: Attempted to insert INCOMPLETE BUY and SELL order into database.'
)
for market in scraper_data.keys():
data = market_data.get(market)
if market not in skip_list:
if len(data.datetime) > 65:
# Clear the first entries
data.clear_first()
status = market_status.get(market)
run_algorithm(data, status, amount_per_call,
pending_order_queue, logger)
if not control_queue.empty():
signal = control_queue.get()
if signal == 'STOP':
logger.info('Tradebot: Stopping tradebot ...')
break
except Exception: # TODO: change back to ConnectionError and ValueError
logger.error(format_exc())
#logger.error(e)
#logger.info('Tradebot: Stopping tradebot ...')
finally:
db.close()
logger.info('Tradebot: Database connection closed.')
def run_scraper(control_queue,
database_name,
logger,
markets=MARKETS,
interval=60,
sleep_time=5):
"""
Run scraper to pull data from Bittrex
:param control_queue: Queue to control scraper
:param database_name: Name of database
:param logger: Main logger
:param markets: List of active markets
:param interval: Duration between entries into database
:param sleep_time: Duration between API calls
:return:
"""
# Initialize database object
initialize_databases(database_name, markets, logger=logger)
db = Database(hostname=HOSTNAME,
username=USERNAME,
password=PASSCODE,
database_name=database_name,
logger=logger)
# Initialize Bittrex object
bittrex_request = Bittrex(api_key=BITTREX_CREDENTIALS.get('key'),
api_secret=BITTREX_CREDENTIALS.get('secret'),
dispatch=return_request_input,
api_version='v1.1')
# Initialize variables
run_tradebot = False
proxy_indexes = list(range(len(PROXIES)))
working_data = {}
current_datetime = datetime.now().astimezone(tz=None)
current_datetime = {k: current_datetime for k in MARKETS}
last_price = {
k: Decimal(0).quantize(BittrexConstants.DIGITS)
for k in MARKETS
}
weighted_price = {
k: Decimal(0).quantize(BittrexConstants.DIGITS)
for k in MARKETS
}
try:
with FuturesSession(max_workers=20) as session:
while True:
shuffle(proxy_indexes)
start = time()
response_dict = get_data(MARKETS,
bittrex_request,
session,
PROXIES,
proxy_indexes,
logger=logger)
working_data, current_datetime, last_price, weighted_price, entries = \
process_data(response_dict, working_data, current_datetime, last_price, weighted_price, logger,
interval)
if run_tradebot:
tradebot_entries = {k: entries.get(k)[-1] for k in entries}
SCRAPER_TRADEBOT_QUEUE.put(tradebot_entries)
if entries:
formatted_entries = list(
chain.from_iterable([[(x, *format_bittrex_entry(y))
for y in entries[x]]
for x in entries]))
db.insert_transaction_query(formatted_entries)
stop = time()
run_time = stop - start
if run_time > 5:
logger.info(
'Scraper: Total time: {0:.2f}s'.format(run_time))
if run_time < sleep_time:
sleep(sleep_time - run_time)
if not control_queue.empty():
signal = control_queue.get()
if signal == "START TRADEBOT":
run_tradebot = True
logger.info("Scraper: Starting tradebot ...")
elif signal == "STOP TRADEBOT":
run_tradebot = False
logger.info("Scraper: Stopping tradebot ...")
elif signal == "STOP":
logger.info("Scraper: Stopping scraper ...")
break
except ConnectionError as e:
logger.debug('ConnectionError: {}. Exiting ...'.format(e))
finally:
db.close()
logger.info("Scraper: Stopped scraper.")
logger.info("Scraper: Database connection closed.")