class Base(ABC):
"""
Base class for data back-filling
"""
arg_parser = configargparse.get_argument_parser()
arg_parser.add('-c',
'--config',
is_config_file=True,
help='config file path',
default='mosquito.ini')
arg_parser.add(
'--pairs',
help=
'Pairs to backfill. For ex. [BTC_ETH, BTC_* (to get all BTC_* prefixed pairs]'
)
arg_parser.add("--all",
help='Backfill data for ALL currencies',
action='store_true')
arg_parser.add("--days",
help="Number of days to backfill",
required=True,
type=int,
default=1)
arg_parser.add('-v', '--verbosity', help='Verbosity', action='store_true')
logging.config.fileConfig('logging.ini')
def __init__(self):
super(Base, self).__init__()
args = self.arg_parser.parse_known_args()[0]
self.exchange = Exchange()
self.exchange_name = self.exchange.get_exchange_name()
self.db = self.initialize_db(args)
@staticmethod
def initialize_db(args):
"""
DB Initializer
"""
db = args.db
port = int(args.db_port)
url = args.db_url
# Init DB
client = MongoClient(url, port)
return client[db]
def get_backfill_pairs(self, backfill_all_pairs=False, pairs_list=None):
"""
Returns list of exchange pairs that were ordered to backfill
"""
all_pairs = self.exchange.get_pairs()
if backfill_all_pairs:
return all_pairs
elif pairs_list is not None:
# tmp_pairs = [pairs_list] # OLD
tmp_pairs = pairs_list.replace(" ", "").split(',') # MODIFIED
pairs = []
# Handle * suffix pairs
for pair in tmp_pairs:
if '*' in pair:
prefix = pair.replace('*', '')
pairs_list = [p for p in all_pairs if prefix in p]
pairs.extend(pairs_list)
# remove duplicates
pairs = list(set(pairs))
else:
pairs.append(pair)
return pairs
@abstractmethod
def run(self):
"""
Backfill/fetch data
"""
pass
class Base(ABC):
"""
Base class for all simulation types (sim, paper, trade)
"""
ticker_df = pd.DataFrame()
pairs = []
exchange = None
balance = None
def __init__(self, args, config_file, trade_mode):
super(Base, self).__init__()
self.args = args
self.config = self.initialize_config(config_file)
self.exchange = Exchange(args, config_file, trade_mode)
self.transaction_fee = self.exchange.get_transaction_fee()
self.ticker_df = pd.DataFrame()
self.verbosity = int(self.config['General']['verbosity'])
self.pairs = self.process_input_pairs(self.config['Trade']['pairs'])
self.pair_delimiter = self.exchange.get_pair_delimiter()
self.last_tick_epoch = 0
def get_pair_delimiter(self):
"""
Returns exchanges pair delimiter
"""
return self.pair_delimiter
def process_input_pairs(self, in_pairs):
all_pairs = self.exchange.get_pairs()
if in_pairs == 'all':
print('setting_all_pairs')
return all_pairs
else:
pairs = []
parsed_pairs = in_pairs.replace(" ", "").split(',')
for in_pair in parsed_pairs:
if '*' in in_pair:
prefix = in_pair.replace('*', '')
pairs_list = [p for p in all_pairs if prefix in p]
pairs.extend(pairs_list)
# remove duplicates
pairs = list(set(pairs))
else:
pairs.append(in_pair)
return pairs
def prefetch(self, min_ticker_size, ticker_interval):
"""
Method pre-fetches data to ticker buffer
"""
prefetch_epoch_size = ticker_interval * min_ticker_size * 60
prefetch_days = math.ceil(prefetch_epoch_size / 86400)
# Prefetch/Backfill data
for pair in self.pairs:
args = Namespace(pair=pair, days=prefetch_days, all=False)
backfill(args)
# Load data to our ticker buffer
prefetch_epoch_size = ticker_interval * min_ticker_size * 60
epoch_now = int(time.time())
prefetch_epoch = epoch_now - prefetch_epoch_size
print('Going to prefetch data of size (minutes): ',
ticker_interval * min_ticker_size)
df = pd.DataFrame()
while prefetch_epoch < epoch_now:
data = self.exchange.get_offline_ticker(prefetch_epoch, self.pairs)
df = df.append(data, ignore_index=True)
prefetch_epoch += (ticker_interval * 60)
print('Fetching done..')
return df
@staticmethod
def initialize_config(config_file):
config = configparser.ConfigParser()
config.read(config_file)
return config
def get_pairs(self):
"""
Returns the pairs the bot is working with
"""
return self.pairs
@abstractmethod
def get_next(self, interval):
"""
Gets next data set
:param interval:
:return: New data in DataFrame
"""
pass
def get_balance(self):
"""
Returns current balance
"""
return self.balance
def sell_all_assets(self, trades, wallet, pair_to_hold):
"""
Sells all available assets in wallet
"""
assets = wallet.copy()
del assets['BTC']
for asset, amount in assets.items():
if amount == 0.0:
continue
pair = 'BTC' + self.pair_delimiter + asset
# If we have the same pair that we want to buy, lets not sell it
if pair == pair_to_hold:
continue
ticker = self.ticker_df.loc[self.ticker_df['pair'] == pair]
if ticker.empty:
print('No currency data for pair: ' + pair + ', skipping')
continue
close_price = ticker['close'].iloc[0]
fee = self.transaction_fee * float(amount) / 100.0
print('txn fee:', fee, ', balance before: ', amount, ', after: ',
amount - fee)
print(
colored(
'Sold: ' + str(amount) + ', pair: ' + pair + ', price: ' +
str(close_price), 'yellow'))
amount -= fee
earned_balance = close_price * amount
root_symbol = 'BTC'
currency = wallet[root_symbol]
# Store trade history
trades.loc[len(trades)] = [
ticker['date'].iloc[0], pair, close_price, 'sell'
]
wallet[root_symbol] = currency + earned_balance
wallet[asset] = 0.0
def trade(self, actions, wallet, trades, force_sell=True):
"""
force_sell: Sells ALL assets before buying new one
Simulate currency buy/sell (places fictive buy/sell orders).
Returns remaining / not - processed actions
"""
self.balance = wallet
if self.ticker_df.empty:
print('Can not trade with empty dataframe, skipping trade')
return actions
for action in actions:
# If action is None, just skip it
if action.action == ts.none:
actions.remove(action)
continue
# If we are forcing_sell, we will first sell all our assets
if force_sell:
self.sell_all_assets(trades, wallet, action.pair)
# Get pairs current closing price
(currency_symbol,
asset_symbol) = tuple(re.split('[-_]', action.pair))
ticker = self.ticker_df.loc[self.ticker_df['pair'] == action.pair]
close_price = action.rate
currency_balance = asset_balance = 0.0
if currency_symbol in wallet:
currency_balance = wallet[currency_symbol]
if asset_symbol in wallet:
asset_balance = wallet[asset_symbol]
if action.buy_sell_all:
action.amount = self.get_buy_sell_all_amount(wallet, action)
fee = self.transaction_fee * float(action.amount) / 100.0
# *** Buy ***
if action.action == ts.buy:
if currency_balance <= 0:
print('Want to buy ' + action.pair +
', not enough money, or everything already bought..')
actions.remove(action)
continue
print(
colored(
'Bought: ' + str(action.amount) + ', pair: ' +
action.pair + ', price: ' + str(close_price), 'green'))
wallet[asset_symbol] = asset_balance + action.amount - fee
wallet[currency_symbol] = currency_balance - (action.amount *
action.rate)
# Append trade
trades.loc[len(trades)] = [
ticker['date'].iloc[0], action.pair, close_price, 'buy'
]
actions.remove(action)
continue
# *** Sell ***
elif action.action == ts.sell:
if asset_balance <= 0:
print('Want to sell ' + action.pair +
', not enough assets, or everything already sold..')
actions.remove(action)
continue
print(
colored(
'Sold: ' + str(action.amount) + '' + action.pair +
', price: ' + str(close_price), 'yellow'))
wallet[currency_symbol] = currency_balance + (
(action.amount - fee) * action.rate)
wallet[asset_symbol] = asset_balance - action.amount
# Append trade
trades.loc[len(trades)] = [
ticker['date'].iloc[0], action.pair, close_price, 'sell'
]
actions.remove(action)
continue
self.balance = wallet
return actions
def get_buy_sell_all_amount(self, wallet, action):
"""
Calculates total amount for ALL assets in wallet
"""
if action.action == TradeState.none:
return 0.0
if action.rate == 0.0:
print(
colored(
'Got zero rate!. Can not calc. buy_sell_amount for pair: '
+ action.pair, 'red'))
return 0.0
(symbol_1, symbol_2) = tuple(action.pair.split(self.pair_delimiter))
amount = 0.0
if action.action == TradeState.buy and symbol_1 in wallet:
assets = wallet.get(symbol_1)
amount = assets / action.rate
elif action.action == TradeState.sell and symbol_2 in wallet:
assets = wallet.get(symbol_2)
amount = assets
if amount <= 0.0:
return 0.0
return amount
class WalletLense:
"""
Lense: Returns actual wallet statistics with simple daily digest (winners / losers)
"""
arg_parser = configargparse.get_argument_parser()
analysis_days = 1
time_intervals_hours = [1, 3, 6, 12, 24]
def __init__(self):
self.args = self.arg_parser.parse_known_args()[0]
print(colored('Starting lense on exchange: ' + self.args.exchange, 'yellow'))
self.exchange = Exchange()
self.postman = Postman()
def get_stats(self):
"""
Returns current statistics (market and wallet)
"""
self.fetch_last_ticker(self.analysis_days)
df_candles = self.get_ticker(self.exchange.get_pairs(), self.analysis_days)
df_candles.to_csv('test_ticker.csv', index=False)
# df_candles = pd.read_csv('test_ticker.csv')
# Parse all df's to html
html_body = ['<html><body>']
html_body.append('<img src="https://user-images.githubusercontent.com/1301154/33856783-88f8e426-dec9-11e7-8371-ead4ef95006a.png" alt="Mosquito" width="330" height="258">')
winners, losers = self.get_winners_losers(df_candles)
html_body.append(self.parse_winners_losers_to_html(winners, losers))
# wallet_stats = self.get_wallet_stats(ticker)
print('wallet stats:')
html_body.append('</body></html>')
self.send_email(html_body)
@staticmethod
def df_to_html(df, header, bar_color='lightblue'):
"""
Converts DataFrame to html text
"""
df_header = '<h3>' + header + '</h1>'
table = (df.style.set_properties(**{'font-size': '9pt', 'font-family': 'Calibri'})
.set_precision(3)
# .background_gradient(subset=['price_change'], cmap='PuBu', low=-100.0, high=100.0)
.set_table_styles([{'selector': '.row_heading, .blank', 'props': [('display', 'none;')]}])
.bar(subset=['price_change'], color=bar_color)
.render()
)
return df_header + table
def send_email(self, body_list):
"""
Sending email module
"""
body = ''
for body_item in body_list:
body += '\n' + body_item
self.postman.send_mail('mosquito_stats', body)
def get_wallet_stats(self, ticker):
"""
Returns simple wallet stats
"""
wallet = self.exchange.get_balances()
return wallet
def parse_winners_losers_to_html(self, winners, losers):
"""
Converts Winners and Losers df's to html
"""
html = '<h2> Winners </h2>'
grouped_winners = winners.groupby(['hour_spam'])
for key, df in grouped_winners:
df = df.drop(['hour_spam'], axis=1)
# Reorder columns
df = df[['price_change', 'pair', 'V', 'Vq']]
html += self.df_to_html(df, str(key) + '-Hour', bar_color='lightgreen')
html += '<hr>'
html += '<h2> Losers </h2>'
grouped_losers = losers.groupby(['hour_spam'])
for key, df in grouped_losers:
df = df.drop(['hour_spam'], axis=1)
df = df.sort_values(['price_change'], ascending=[1])
df['price_change'] = df['price_change'] * -1.0
# Reorder columns
df = df[['price_change', 'pair', 'V', 'Vq']]
html += self.df_to_html(df, str(key) + '-Hour', bar_color='lightpink')
return html
def get_ticker(self, pairs, history_days):
"""
Gets ticker for given list of pairs and given perion
"""
print('Getting tickers.. (might take a while)')
ticker_to = int(time.time())
ticker_from = ticker_to - (24 * history_days * 3600)
df_pairs_ticker = pd.DataFrame()
for pair in pairs:
ticker_list = self.exchange.get_candles(pair, ticker_from, ticker_to, period=1800)
df_ticker = pd.DataFrame(ticker_list)
df_ticker['pair'] = pair
df_pairs_ticker = df_pairs_ticker.append(df_ticker, ignore_index=True)
return df_pairs_ticker
def get_winners_losers(self, df_ticker):
"""
Get winners/losers
"""
grouped = df_ticker.groupby(['pair'])
df_stats = pd.DataFrame()
for name, df_group in grouped:
pair_stat = self.get_pair_stats(name, df_group, self.time_intervals_hours)
df_s = pd.DataFrame(pair_stat)
df_stats = df_stats.append(df_s, ignore_index=True)
grouped_stats = df_stats.groupby(['hour_spam'])
winners = pd.DataFrame()
losers = pd.DataFrame()
for interval, df_group in grouped_stats:
sorted_intervals = df_group.sort_values('price_change', ascending=False)
winners = winners.append(sorted_intervals.head(5))
losers = losers.append(sorted_intervals.tail(5))
return winners, losers
def get_pair_stats(self, pair, df, hour_intervals):
"""
Returns statistics summary
"""
df_now = df.tail(1)
date_end = df_now['date'].iloc[0]
dates = df['date']
stats = []
for hour_interval in hour_intervals:
next_epoch = hour_interval * 3600
closest_date_idx = self.find_nearest(dates, date_end - next_epoch)
closest_df = df.loc[closest_date_idx]
df_interval = df.loc[df['date'] == closest_df.date]
pair_stats = self.calc_pair_stats(df_now.iloc[0], df_interval.iloc[0])
pair_stats['pair'] = pair
pair_stats['hour_spam'] = hour_interval
stats.append(pair_stats)
return stats
@staticmethod
def calc_pair_stats(ticker_now, ticker_past):
stats = dict()
price_change = ((float(ticker_past.close) * 100.0) / float(ticker_now.close)) - 100.0
volume_perc_change = ((float(ticker_past.volume) * 100.0) / float(ticker_now.volume)) - 100.0
quote_volume_perc_change = ((float(ticker_past.quoteVolume) * 100.0) / float(ticker_now.quoteVolume)) - 100.0
stats['price_change'] = price_change
stats['V'] = volume_perc_change
stats['Vq'] = quote_volume_perc_change
return stats
@staticmethod
def find_nearest(array, value):
idx = (np.abs(array - value)).argmin()
return idx
def fetch_last_ticker(self, prefetch_days):
"""
Prefetch data for all pairs for given days
"""
self.args.days = prefetch_days
self.args.all = True
backfill(self.args)