def get_markets(self) -> List[Exchange.Market]:
try:
markets: dict = self._api.get_exchange_info()
except (BinanceAPIException, BinanceRequestException) as e:
if isinstance(e, BinanceAPIException):
# TODO: Raise an exception when rate limited. Probably overkill
# because it's unlikely over 1200 requests will be sent per
# minute.
if e.status_code == 429:
self._log.error("Being rate limited.")
elif e.status_code == 418:
self._log.error("IP banned for violation of rate limits.")
self._log.error("Markets could not be retrieved.")
return []
markets = markets["symbols"]
self._log.debug(f"Retrieved {len(markets)} markets.")
# TODO: Filter out inactive markets (m["status"] != "TRADING")
return list(
map(
lambda m: Exchange.Market(
m["symbol"],
Exchange.Currency(m["baseAsset"], None, m[
"baseAssetPrecision"]),
Exchange.Currency(m["quoteAsset"], None, m[
"quotePrecision"]), self._get_step_size(m["filters"])),
markets))
def __init__(self):
args = self.arg_parser.parse_known_args()[0]
self.blueprint_days = args.days
self.ticker_size = int(args.ticker_size)
self.blueprint_end_time = int(time.time())
self.start_time = self.blueprint_end_time - int(
self.blueprint_days) * 86400
self.ticker_epoch = self.start_time
self.exchange = Exchange(None)
self.pairs = common.parse_pairs(self.exchange, args.pairs)
blueprints_module = common.load_module('ai.blueprints.', args.features)
self.blueprint = blueprints_module(self.pairs)
self.max_buffer_size = int(
int(args.buffer_size) * (1440 / self.ticker_size) *
len(self.pairs))
self.df_buffer = pd.DataFrame()
self.df_blueprint = pd.DataFrame()
self.output_dir = args.output_dir
self.export_file_name = self.get_output_file_path(
self.output_dir, self.blueprint.name)
self.export_file_initialized = False
# Crete output dir
if not os.path.exists(self.out_dir):
os.makedirs(self.out_dir)
def __init__(self, name, public_key, private_key):
Exchange.__init__(self, name)
#Keep as separate instance
self.CCXT = CCXT_Interface()
self.is_authenticated = self.CCXT.exchange_connect(name)
#Default info load:
self._load_markets()
def __init__(self):
self.args = self.arg_parser.parse_known_args()[0]
print(
colored('Starting lense on exchange: ' + self.args.exchange,
'yellow'))
self.exchange = Exchange()
self.postman = Postman()
def __init__(self,
name,
public_key,
private_key,
poll_time_s=5,
tick_tock=True):
Exchange.__init__(self, name)
self.pending_cancel = {}
self.client = Qryptos(public_key, private_key)
self.client.API_URL = 'https://api.liquid.com'
self.internal_to_external_id = {}
self.external_to_internal_id = {}
self.open_orders_by_exchange_id = {}
self.symbol_to_product = {}
self.markets_following = {}
product_list = self.client.get_products()
for product in product_list:
if 'product_type' in product and product[
'product_type'] == 'CurrencyPair':
self.symbol_to_product[
product['currency_pair_code']] = product['id']
self.poll_time_s = poll_time_s
self.tick_tock = tick_tock
self.name = name
if tick_tock is True:
threading.Timer(self.poll_time_s, self.on_tick).start()
def __init__(self, name, public_key=None, private_key=None):
Exchange.__init__(self, name)
TransferService.__init__(self)
self.client = Client(public_key, private_key)
self.user_data_service = UserDataService(self.client,
self.notify_callbacks, name)
self.is_authenticated = (public_key is not None) and (private_key
is not None)
self.order_book_services = {}
self.callbacks = {}
def __init__(self, trade_mode):
super(Base, self).__init__()
self.args = self.arg_parser.parse_known_args()[0]
self.exchange = Exchange(trade_mode)
self.transaction_fee = self.exchange.get_transaction_fee()
self.ticker_df = pd.DataFrame()
self.verbosity = self.args.verbosity
self.pairs = common.parse_pairs(self.exchange, self.args.pairs)
self.fixed_trade_amount = float(self.args.fixed_trade_amount)
self.pair_delimiter = self.exchange.get_pair_delimiter()
self.last_tick_epoch = 0
def trade_history():
"""
Gets sample trades history
"""
exchange = Exchange()
end = time.time()
start = end - 3600
data = exchange.get_market_history(start=start,
end=end,
currency_pair='BTC_ETH')
print(data)
def __init__(self, args, config_file, trade_mode):
super(Base, self).__init__()
self.args = args
self.config = self.initialize_config(config_file)
self.exchange = Exchange(args, config_file, trade_mode)
self.transaction_fee = self.exchange.get_transaction_fee()
self.ticker_df = pd.DataFrame()
self.verbosity = int(self.config['General']['verbosity'])
self.pairs = self.process_input_pairs(self.config['Trade']['pairs'])
self.pair_delimiter = self.exchange.get_pair_delimiter()
self.last_tick_epoch = 0
def __init__(self, name, public_key, private_key):
Exchange.__init__(self, name)
self.ob_ws = OrderBookSocket(self)
self.ex_ws = ExecutionsSocket(self)
self.ex_ws.authenticate(public_key, private_key)
self.markets_following = {}
self.open_orders = []
self.rest_client = Bittrex(public_key,
private_key,
api_version=API_V1_1)
def __init__(self, trade_mode):
super(Base, self).__init__(
) #crea con init di ABC. ABC è una classe che aiuta a costruire per ereditarietà
self.args = self.arg_parser.parse_known_args()[0]
self.exchange = Exchange(trade_mode)
self.transaction_fee = self.exchange.get_transaction_fee()
self.ticker_df = pd.DataFrame()
self.verbosity = self.args.verbosity
self.pairs = common.parse_pairs(self.exchange, self.args.pairs)
self.fixed_trade_amount = float(self.args.fixed_trade_amount)
self.pair_delimiter = self.exchange.get_pair_delimiter()
self.last_tick_epoch = 0
def get_markets(currency: Exchange.Currency) -> \
DefaultDict[str, List[Exchange.Market]]:
g.log.debug(f"Getting markets for base currency {currency.symbol}.")
quotes: Tuple[str] = tuple(g.config["order"]["quote_currencies"].keys())
cases: str = "\n".join(f"when '{ex}' then {i}"
for i, ex in enumerate(quotes))
g.db.cursor.execute(
f"""
select
e.name,
em.name,
m.base,
(select precision
from currencies
where symbol = m.base
limit 1) base_prec,
m.quote,
(select precision
from currencies
where symbol = m.base
limit 1) quote_prec,
m.step
from markets m
join exchange_markets em
on m.id = em.market_id
join exchanges e
on em.exchange_id = e.id
where
base = upper(?) and
lower(quote) in ({", ".join(["?"] * len(quotes))})
order by
e.name,
case lower(m.quote)
{cases}
end
""", (currency.symbol, ) + quotes)
results: List[Tuple[str, str, str, int, str, int,
Union[str, None]]] = g.db.cursor.fetchall()
markets: DefaultDict[str, List[Exchange.Market]] = defaultdict(list)
g.log.debug(f"Retrieved {len(results)} markets.")
for ex, market, base, base_prec, quote, quote_prec, step in results:
markets[ex].append(
Exchange.Market(market, Exchange.Currency(base, None, base_prec),
Exchange.Currency(quote, None, quote_prec),
Decimal(step) if step else None))
return markets
def parse_currency(text: str) -> Optional[Exchange.Currency]:
"""Finds a currency in a string.
Searches for a currency in the string :any:`text` and returns the first
match.
Notes
-----
Queries the database for all currencies and iterates them. Uses a regular
expression to find a match in the text and constructs a
:class:`~Exchange.Currency` from it.
Parameters
----------
text: str
The string in which to search for a currency.
Returns
-------
Exchange.Currency or None
The first matching currency found in the string, or :any:`None` if no
matches are found.
"""
log.debug("Searching for a currency in the text.")
g.db.cursor.execute("select * from currencies")
return next(
(Exchange.Currency(symbol, name, precision)
for symbol, name, precision in g.db.cursor.fetchall()
if re.search((name if g.config["search_currency_name"] else "") +
r"\s*?[(\[{]+?\s*?" + symbol +
r"\s*?[)\]}]", text, re.IGNORECASE)), None)
def __init__(self,
name,
public_key,
private_key,
poll_time_s=5,
tick_tock=True):
Exchange.__init__(self, name)
self.open_orders_by_exchange_id = {}
self.external_to_internal_id = {}
self.internal_to_external_id = {}
self.poll_time_s = poll_time_s
self.tick_tock = True
self.markets_following = {}
self.client = Api(public_key, private_key)
self.open_orders = []
if tick_tock is True:
threading.Timer(self.poll_time_s, self.on_tick).start()
def get_markets(self) -> List[Exchange.Market]:
markets: dict = self._api.get_markets()
if not markets["success"]:
self._log.error("Markets could not be retrieved: "
f"{markets['message']}")
return []
markets = markets["result"]
self._log.debug(f"Retrieved {len(markets)} markets.")
# TODO: Filter out inactive markets (not m["IsActive"])?
return list(map(lambda m: Exchange.Market(
m["MarketName"],
Exchange.Currency(
m["MarketCurrency"],
m["MarketCurrencyLong"],
None),
Exchange.Currency(
m["BaseCurrency"],
m["BaseCurrencyLong"],
None),
None),
markets))
def __init__(self,
exchange: Exchange,
period_start: datetime,
period_end=None,
interval=60):
self.launchedAt = datetime.now()
# Try to find dataset
dataset = Dataset().get({
"exchange": exchange.name.lower(),
"currency": exchange.currency.lower(),
"asset": exchange.asset.lower(),
"periodStart": period_start,
"periodEnd": period_end,
"candleSize": interval
})
if dataset and len(dataset) > 0:
print(dataset)
print(dataset[0])
print("Dataset found: " + dataset[0]['uuid'])
price = Price()
for prices in price.query('get', {"dataset": dataset[0]['uuid']}):
for price in prices:
print(price)
newPrice = Price()
newPrice.populate(price)
exchange.strategy.set_price(newPrice)
exchange.strategy.run()
else:
print("Dataset not found, external API call to " + exchange.name)
for price in exchange.historical_symbol_ticker_candle(
period_start, period_end, interval):
exchange.strategy.set_price(price)
exchange.strategy.run()
execution_time = datetime.now() - self.launchedAt
print('Execution time: ' + str(execution_time.total_seconds()) +
' seconds')
sys.exit(0)
class WalletLense:
"""
Lense: Returns actual wallet statistics with simple daily digest (winners / losers)
"""
arg_parser = configargparse.get_argument_parser()
analysis_days = 1
time_intervals_hours = [1, 3, 6, 12, 24]
def __init__(self):
self.args = self.arg_parser.parse_known_args()[0]
print(colored('Starting lense on exchange: ' + self.args.exchange, 'yellow'))
self.exchange = Exchange()
self.postman = Postman()
def get_stats(self):
"""
Returns current statistics (market and wallet)
"""
self.fetch_last_ticker(self.analysis_days)
df_candles = self.get_ticker(self.exchange.get_pairs(), self.analysis_days)
df_candles.to_csv('test_ticker.csv', index=False)
# df_candles = pd.read_csv('test_ticker.csv')
# Parse all df's to html
html_body = ['<html><body>']
html_body.append('<img src="https://user-images.githubusercontent.com/1301154/33856783-88f8e426-dec9-11e7-8371-ead4ef95006a.png" alt="Mosquito" width="330" height="258">')
winners, losers = self.get_winners_losers(df_candles)
html_body.append(self.parse_winners_losers_to_html(winners, losers))
# wallet_stats = self.get_wallet_stats(ticker)
print('wallet stats:')
html_body.append('</body></html>')
self.send_email(html_body)
@staticmethod
def df_to_html(df, header, bar_color='lightblue'):
"""
Converts DataFrame to html text
"""
df_header = '<h3>' + header + '</h1>'
table = (df.style.set_properties(**{'font-size': '9pt', 'font-family': 'Calibri'})
.set_precision(3)
# .background_gradient(subset=['price_change'], cmap='PuBu', low=-100.0, high=100.0)
.set_table_styles([{'selector': '.row_heading, .blank', 'props': [('display', 'none;')]}])
.bar(subset=['price_change'], color=bar_color)
.render()
)
return df_header + table
def send_email(self, body_list):
"""
Sending email module
"""
body = ''
for body_item in body_list:
body += '\n' + body_item
self.postman.send_mail('mosquito_stats', body)
def get_wallet_stats(self, ticker):
"""
Returns simple wallet stats
"""
wallet = self.exchange.get_balances()
return wallet
def parse_winners_losers_to_html(self, winners, losers):
"""
Converts Winners and Losers df's to html
"""
html = '<h2> Winners </h2>'
grouped_winners = winners.groupby(['hour_spam'])
for key, df in grouped_winners:
df = df.drop(['hour_spam'], axis=1)
# Reorder columns
df = df[['price_change', 'pair', 'V', 'Vq']]
html += self.df_to_html(df, str(key) + '-Hour', bar_color='lightgreen')
html += '<hr>'
html += '<h2> Losers </h2>'
grouped_losers = losers.groupby(['hour_spam'])
for key, df in grouped_losers:
df = df.drop(['hour_spam'], axis=1)
df = df.sort_values(['price_change'], ascending=[1])
df['price_change'] = df['price_change'] * -1.0
# Reorder columns
df = df[['price_change', 'pair', 'V', 'Vq']]
html += self.df_to_html(df, str(key) + '-Hour', bar_color='lightpink')
return html
def get_ticker(self, pairs, history_days):
"""
Gets ticker for given list of pairs and given perion
"""
print('Getting tickers.. (might take a while)')
ticker_to = int(time.time())
ticker_from = ticker_to - (24 * history_days * 3600)
df_pairs_ticker = pd.DataFrame()
for pair in pairs:
ticker_list = self.exchange.get_candles(pair, ticker_from, ticker_to, period=1800)
df_ticker = pd.DataFrame(ticker_list)
df_ticker['pair'] = pair
df_pairs_ticker = df_pairs_ticker.append(df_ticker, ignore_index=True)
return df_pairs_ticker
def get_winners_losers(self, df_ticker):
"""
Get winners/losers
"""
grouped = df_ticker.groupby(['pair'])
df_stats = pd.DataFrame()
for name, df_group in grouped:
pair_stat = self.get_pair_stats(name, df_group, self.time_intervals_hours)
df_s = pd.DataFrame(pair_stat)
df_stats = df_stats.append(df_s, ignore_index=True)
grouped_stats = df_stats.groupby(['hour_spam'])
winners = pd.DataFrame()
losers = pd.DataFrame()
for interval, df_group in grouped_stats:
sorted_intervals = df_group.sort_values('price_change', ascending=False)
winners = winners.append(sorted_intervals.head(5))
losers = losers.append(sorted_intervals.tail(5))
return winners, losers
def get_pair_stats(self, pair, df, hour_intervals):
"""
Returns statistics summary
"""
df_now = df.tail(1)
date_end = df_now['date'].iloc[0]
dates = df['date']
stats = []
for hour_interval in hour_intervals:
next_epoch = hour_interval * 3600
closest_date_idx = self.find_nearest(dates, date_end - next_epoch)
closest_df = df.loc[closest_date_idx]
df_interval = df.loc[df['date'] == closest_df.date]
pair_stats = self.calc_pair_stats(df_now.iloc[0], df_interval.iloc[0])
pair_stats['pair'] = pair
pair_stats['hour_spam'] = hour_interval
stats.append(pair_stats)
return stats
@staticmethod
def calc_pair_stats(ticker_now, ticker_past):
stats = dict()
price_change = ((float(ticker_past.close) * 100.0) / float(ticker_now.close)) - 100.0
volume_perc_change = ((float(ticker_past.volume) * 100.0) / float(ticker_now.volume)) - 100.0
quote_volume_perc_change = ((float(ticker_past.quoteVolume) * 100.0) / float(ticker_now.quoteVolume)) - 100.0
stats['price_change'] = price_change
stats['V'] = volume_perc_change
stats['Vq'] = quote_volume_perc_change
return stats
@staticmethod
def find_nearest(array, value):
idx = (np.abs(array - value)).argmin()
return idx
def fetch_last_ticker(self, prefetch_days):
"""
Prefetch data for all pairs for given days
"""
self.args.days = prefetch_days
self.args.all = True
backfill(self.args)
def __init__(self, name, public_key, private_key):
Exchange.__init__(self, name)
self.order_book_socket = OrderBookSocket()
self.markets_following = {}
self.rest_client = RestClient(public_key, private_key)
self.open_orders = []
def __init__(self, name, public_key, private_key):
Exchange.__init__(self, name)
def __init__(self, key: str, secret: str):
Exchange.__init__(key, secret)
self.name = self.__class__.__name__
self.client = CoinGeckoAPI()
def get_currencies() -> List[Exchange.Currency]:
g.log.debug("Getting currencies from CoinMarketCap.")
return list(
map(lambda c: Exchange.Currency(c["symbol"], c["name"], None),
Market().ticker(limit=0)))
class Base(ABC):
"""
Base class for all simulation types (sim, paper, trade)
"""
ticker_df = pd.DataFrame()
pairs = []
exchange = None
balance = None
arg_parser = configargparse.get_argument_parser()
def __init__(self, trade_mode):
super(Base, self).__init__()
self.args = self.arg_parser.parse_known_args()[0]
self.exchange = Exchange(trade_mode)
self.transaction_fee = self.exchange.get_transaction_fee()
self.ticker_df = pd.DataFrame()
self.verbosity = self.args.verbosity
self.pairs = common.parse_pairs(self.exchange, self.args.pairs)
self.fixed_trade_amount = float(self.args.fixed_trade_amount)
self.pair_delimiter = self.exchange.get_pair_delimiter()
self.last_tick_epoch = 0
def get_pair_delimiter(self):
"""
Returns exchanges pair delimiter
"""
return self.pair_delimiter
def prefetch(self, min_ticker_size, ticker_interval):
"""
Method pre-fetches data to ticker buffer
"""
prefetch_epoch_size = ticker_interval * min_ticker_size * 60
prefetch_days = math.ceil(prefetch_epoch_size / 86400)
# Prefetch/Backfill data
self.args.days = prefetch_days
orig_pair = self.args.pairs
backfill_candles = Candles()
backfill_candles.run()
self.args.pairs = orig_pair
# Load data to our ticker buffer
prefetch_epoch_size = ticker_interval * min_ticker_size * 60
epoch_now = int(time.time())
prefetch_epoch = epoch_now - prefetch_epoch_size
print('Going to prefetch data of size (minutes): ',
ticker_interval * min_ticker_size)
df = pd.DataFrame()
while prefetch_epoch < epoch_now:
data = self.exchange.get_offline_ticker(prefetch_epoch, self.pairs)
df = df.append(data, ignore_index=True)
prefetch_epoch += (ticker_interval * 60)
print('Fetching done..')
return df
def get_pairs(self):
"""
Returns the pairs the bot is working with
"""
return self.pairs
@abstractmethod
def get_next(self, interval_in_min):
"""
Gets next data set
:param interval_in_min:
:return: New data in DataFrame
"""
pass
def get_balance(self):
"""
Returns current balance
"""
# Remove all items with zero amount
self.balance = {k: v for k, v in self.balance.items() if v != 0}
return self.balance
def sell_all_assets(self, trades, wallet, pair_to_hold):
"""
Sells all available assets in wallet
"""
assets = wallet.copy()
del assets['BTC']
for asset, amount in assets.items():
if amount == 0.0:
continue
pair = 'BTC' + self.pair_delimiter + asset
# If we have the same pair that we want to buy, lets not sell it
if pair == pair_to_hold:
continue
ticker = self.ticker_df.loc[self.ticker_df['pair'] == pair]
if ticker.empty:
print('No currency data for pair: ' + pair + ', skipping')
continue
close_price = ticker['close'].iloc[0]
fee = self.transaction_fee * float(amount) / 100.0
print('txn fee:', fee, ', balance before: ', amount, ', after: ',
amount - fee)
print(
colored(
'Sold: ' + str(amount) + ', pair: ' + pair + ', price: ' +
str(close_price), 'yellow'))
amount -= fee
earned_balance = close_price * amount
root_symbol = 'BTC'
currency = wallet[root_symbol]
# Store trade history
trades.loc[len(trades)] = [
ticker['date'].iloc[0], pair, close_price, 'sell'
]
wallet[root_symbol] = currency + earned_balance
wallet[asset] = 0.0
def trade(self, actions, wallet, trades, force_sell=True):
"""
force_sell: Sells ALL assets before buying new one
Simulate currency buy/sell (places fictive buy/sell orders).
Returns remaining / not - processed actions
"""
self.balance = wallet
if self.ticker_df.empty:
print('Can not trade with empty dataframe, skipping trade')
return actions
for action in actions:
# If action is None, just skip it
if action.action == TradeState.none:
actions.remove(action)
continue
# If we are forcing_sell, we will first sell all our assets
if force_sell:
self.sell_all_assets(trades, wallet, action.pair)
# Get pairs current closing price
(currency_symbol,
asset_symbol) = tuple(re.split('[-_]', action.pair))
ticker = self.ticker_df.loc[self.ticker_df['pair'] == action.pair]
if len(ticker.index) == 0:
print('Could not find pairs ticker, skipping trade')
continue
close_price = action.rate
currency_balance = asset_balance = 0.0
if currency_symbol in wallet:
currency_balance = wallet[currency_symbol]
if asset_symbol in wallet:
asset_balance = wallet[asset_symbol]
if action.buy_sell_mode == BuySellMode.all:
action.amount = self.get_buy_sell_all_amount(wallet, action)
elif action.buy_sell_mode == BuySellMode.fixed:
action.amount = self.get_fixed_trade_amount(wallet, action)
fee = self.transaction_fee * float(action.amount) / 100.0
# *** Buy ***
if action.action == TradeState.buy:
if currency_balance <= 0:
print('Want to buy ' + action.pair +
', not enough money, or everything already bought..')
actions.remove(action)
continue
print(
colored(
'Bought: ' + str(action.amount) + ', pair: ' +
action.pair + ', price: ' + str(close_price), 'green'))
wallet[asset_symbol] = asset_balance + action.amount - fee
wallet[currency_symbol] = currency_balance - (action.amount *
action.rate)
# Append trade
trades.loc[len(trades)] = [
ticker['date'].iloc[0], action.pair, close_price, 'buy'
]
actions.remove(action)
continue
# *** Sell ***
elif action.action == TradeState.sell:
if asset_balance <= 0:
print('Want to sell ' + action.pair +
', not enough assets, or everything already sold..')
actions.remove(action)
continue
print(
colored(
'Sold: ' + str(action.amount) + '' + action.pair +
', price: ' + str(close_price), 'yellow'))
wallet[currency_symbol] = currency_balance + (
(action.amount - fee) * action.rate)
wallet[asset_symbol] = asset_balance - action.amount
# Append trade
trades.loc[len(trades)] = [
ticker['date'].iloc[0], action.pair, close_price, 'sell'
]
actions.remove(action)
continue
self.balance = wallet
return actions
def get_buy_sell_all_amount(self, wallet, action):
"""
Calculates total amount for ALL assets in wallet
"""
if action.action == TradeState.none:
return 0.0
if action.rate == 0.0:
print(
colored(
'Got zero rate!. Can not calc. buy_sell_amount for pair: '
+ action.pair, 'red'))
return 0.0
(symbol_1, symbol_2) = tuple(action.pair.split(self.pair_delimiter))
amount = 0.0
if action.action == TradeState.buy and symbol_1 in wallet:
assets = wallet.get(symbol_1)
amount = assets / action.rate
elif action.action == TradeState.sell and symbol_2 in wallet:
assets = wallet.get(symbol_2)
amount = assets
if amount <= 0.0:
return 0.0
return amount
def get_fixed_trade_amount(self, wallet, action):
"""
Calculates fixed trade amount given action
"""
if action.action == TradeState.none:
return 0.0
if action.rate == 0.0:
print(
colored(
'Got zero rate!. Can not calc. buy_sell_amount for pair: '
+ action.pair, 'red'))
return 0.0
(symbol_1, symbol_2) = tuple(action.pair.split(self.pair_delimiter))
amount = 0.0
if action.action == TradeState.buy and symbol_1 in wallet:
assets = self.fixed_trade_amount
amount = assets / action.rate
elif action.action == TradeState.sell and symbol_2 in wallet:
assets = wallet.get(symbol_2)
amount = assets
if amount <= 0.0:
return 0.0
return amount
class Blueprint:
"""
Main module for generating and handling datasets for AI. Application will generate datasets including
future target/output parameters.
"""
arg_parser = configargparse.get_argument_parser()
arg_parser.add('--days', help='Days to start blueprint from', default=30)
arg_parser.add(
'-f',
'--features',
help='Blueprints module name to be used to generated features',
required=True)
arg_parser.add('--ticker_size',
help='Size of the candle ticker (minutes)',
default=5)
arg_parser.add('--pairs', help='Pairs to blueprint')
arg_parser.add('-v', '--verbosity', help='Verbosity', action='store_true')
arg_parser.add("--buffer_size",
help="Maximum Buffer size (days)",
default=30)
arg_parser.add("--output_dir", help="Output directory")
features_list = None
exchange = None
blueprint = None
out_dir = 'out/blueprints/'
def __init__(self):
args = self.arg_parser.parse_known_args()[0]
self.blueprint_days = args.days
self.ticker_size = int(args.ticker_size)
self.blueprint_end_time = int(time.time())
self.start_time = self.blueprint_end_time - int(
self.blueprint_days) * 86400
self.ticker_epoch = self.start_time
self.exchange = Exchange(None)
self.pairs = common.parse_pairs(self.exchange, args.pairs)
blueprints_module = common.load_module('ai.blueprints.', args.features)
self.blueprint = blueprints_module(self.pairs)
self.max_buffer_size = int(
int(args.buffer_size) * (1440 / self.ticker_size) *
len(self.pairs))
self.df_buffer = pd.DataFrame()
self.df_blueprint = pd.DataFrame()
self.output_dir = args.output_dir
self.export_file_name = self.get_output_file_path(
self.output_dir, self.blueprint.name)
self.export_file_initialized = False
# Crete output dir
if not os.path.exists(self.out_dir):
os.makedirs(self.out_dir)
@staticmethod
def get_output_file_path(dir_path, blueprint_name):
filename = 'blueprint_' + blueprint_name + '_' + str(int(
time.time())) + '.csv'
if dir_path:
if not dir_path.endswith(os.path.sep):
dir_path += os.path.sep
filename = dir_path + filename
return filename
def print_progress_dot(self, counter):
"""
Prints progress
"""
if counter % 100 == 0:
print('.', end='', flush=True)
if counter > 101:
counter = 0
self.write_to_file()
return counter + 1
def write_to_file(self):
"""
Writes df to file
"""
if self.df_blueprint.empty:
print('Blueprint is empty, nothing to write to file.')
return
export_df = self.df_blueprint.copy()
dropping_columns = ['_id', 'id', 'curr_1', 'curr_2', 'exchange']
df_columns = self.blueprint.get_feature_names()
df_columns = [x for x in df_columns if x not in dropping_columns]
export_df = export_df.drop(dropping_columns, axis=1)
export_df = export_df[df_columns]
dt = export_df.tail(1).date.iloc[0]
dt_string = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(dt))
print('saving,..(last df date: ' + dt_string + ')')
if not self.export_file_initialized:
export_df.to_csv(self.out_dir + self.export_file_name,
index=False,
columns=df_columns)
self.export_file_initialized = True
else:
export_df.to_csv(self.out_dir + self.export_file_name,
mode='a',
header=False,
index=False,
columns=df_columns)
self.df_blueprint = self.df_blueprint[0:0]
def run(self):
"""
Calculates and stores dataset
"""
info_text = 'Starting generating data for Blueprint ' + self.blueprint.name + ' :back-days ' + \
self.blueprint_days + ' (This might take several hours/days,.so please stay back and relax)'
print(colored(info_text, 'yellow'))
dot_counter = 0
while True:
# Get new dataset
df = self.exchange.get_offline_ticker(self.ticker_epoch,
self.pairs)
# Check if the simulation is finished
if self.ticker_epoch >= self.blueprint_end_time:
self.write_to_file()
return
# Store df to buffer
if not self.df_buffer.empty:
df = df[list(self.df_buffer)]
self.df_buffer = self.df_buffer.append(df, ignore_index=True)
else:
self.df_buffer = self.df_buffer.append(df, ignore_index=True)
self.df_buffer = common.handle_buffer_limits(
self.df_buffer, self.max_buffer_size)
scan_df = self.blueprint.scan(self.df_buffer, self.ticker_size)
if not scan_df.empty:
dot_counter = self.print_progress_dot(dot_counter)
self.df_blueprint = self.df_blueprint.append(scan_df,
ignore_index=True)
self.ticker_epoch += self.ticker_size * 60
def __init__(self, api_key, api_secret):
Exchange.__init__(self, api_key, api_secret)
self.initialize_headers()
def __init__(self, api_key, api_secret):
Exchange.__init__(self, api_key, api_secret)
class Base(ABC):
"""
Base class for data back-filling
"""
arg_parser = configargparse.get_argument_parser()
arg_parser.add('-c',
'--config',
is_config_file=True,
help='config file path',
default='mosquito.ini')
arg_parser.add(
'--pairs',
help=
'Pairs to backfill. For ex. [BTC_ETH, BTC_* (to get all BTC_* prefixed pairs]'
)
arg_parser.add("--all",
help='Backfill data for ALL currencies',
action='store_true')
arg_parser.add("--days",
help="Number of days to backfill",
required=True,
type=int,
default=1)
arg_parser.add('-v', '--verbosity', help='Verbosity', action='store_true')
logging.config.fileConfig('logging.ini')
def __init__(self):
super(Base, self).__init__()
args = self.arg_parser.parse_known_args()[0]
self.exchange = Exchange()
self.exchange_name = self.exchange.get_exchange_name()
self.db = self.initialize_db(args)
@staticmethod
def initialize_db(args):
"""
DB Initializer
"""
db = args.db
port = int(args.db_port)
url = args.db_url
# Init DB
client = MongoClient(url, port)
return client[db]
def get_backfill_pairs(self, backfill_all_pairs=False, pairs_list=None):
"""
Returns list of exchange pairs that were ordered to backfill
"""
all_pairs = self.exchange.get_pairs()
if backfill_all_pairs:
return all_pairs
elif pairs_list is not None:
# tmp_pairs = [pairs_list] # OLD
tmp_pairs = pairs_list.replace(" ", "").split(',') # MODIFIED
pairs = []
# Handle * suffix pairs
for pair in tmp_pairs:
if '*' in pair:
prefix = pair.replace('*', '')
pairs_list = [p for p in all_pairs if prefix in p]
pairs.extend(pairs_list)
# remove duplicates
pairs = list(set(pairs))
else:
pairs.append(pair)
return pairs
@abstractmethod
def run(self):
"""
Backfill/fetch data
"""
pass
def __init__(self):
super(Base, self).__init__()
args = self.arg_parser.parse_known_args()[0]
self.exchange = Exchange()
self.exchange_name = self.exchange.get_exchange_name()
self.db = self.initialize_db(args)