def update_24hour_statistics():
"""
Loads the 24 hour market data statistics from coinbase pro
into our database.
"""
client = PublicClient()
products = Product.objects.all()
stats_list = []
for product in products:
s = client.get_product_24hr_stats(product.product_id)
if 'message' in s:
print(
f'API rate limit exceeded with {product.product_id}. retrying in 10'
)
sleep(10)
s = client.get_product_24hr_stats(product.product_id)
if 'message' in s:
continue
print(f'Packaging {product} with marketdata.')
package = [product, s]
stats_list.append(package)
sleep(1)
added_count = 0
print('Inserting producted data to database.')
for s in stats_list:
insert_product_stats(s[0], s[1])
added_count += 1
return added_count
def __init__(self, index=4, weight=True):
"""
Creates an instance of the Indicator class\n
Args:
index (int): type of price to work with, close price is default
weight (bool): If
"""
self.__new_client = PublicClient(timeout=45)
self.candles = []
self.close_array = []
self.date_array = []
self.np_array = []
self.index = index
self.weight = weight
class Indicator:
"""
Base class for any indicator.\n
Coinbase returns data in the following way: [1415398768, 0.32, 4.2, 0.35, 4.2, 12.3],
indicators work with close prices as default, therefore we append data from index 4.\n
If the indicator is weighted, certain amount of elements back are needed,
weight = True will get that many elements automatically
"""
def __init__(self, index=4, weight=True):
"""
Creates an instance of the Indicator class\n
Args:
index (int): type of price to work with, close price is default
weight (bool): If
"""
self.__new_client = PublicClient(timeout=45)
self.candles = []
self.close_array = []
self.date_array = []
self.np_array = []
self.index = index
self.weight = weight
def set_candles(self, product, callback, begin, granularity):
"""
Makes requests to the Coinbase API for Historic rates\n
Args:
product (str): A valid Coinbase product
callback (int): Number of seconds to go back from begin
begin (int): Last second to get rates from
granularity (int): Number of seconds
"""
self.candles = self.__new_client.get_product_historic_rates(product_id=product, start=callback, end=begin, granularity=granularity)
return self.candles
def set_dates(self):
"""
Puts of the dates in a list an reverses it the same method. That is because when we
testing strategies sometimes we wanna see when it happened and we this method we just have to
reference the same index as the one in the closing prices list.\n
:return: Reversed list of dates in Unix form
"""
p = 0
while p < len(self.candles):
self.date_array.append(self.candles[p][0])
p = p + 1
self.date_array.reverse()
return self.date_array
def set_indicator(self):
for candle in self.candles:
self.close_array.append(float(candle[self.index]))
self.close_array.reverse()
self.np_array = numpy.array(self.close_array)
def get_index(self, index):
return float(self.close_array[index])
def load_currencies():
"""
Load currencies into database that are on the coinbase pro exchange.
first check if they are in the database before adding them.
"""
currency_list = Currency.objects.all() # currencies already added
client = PublicClient() # connect client to coinbase pro
coinbase_currency_list = client.get_currencies() # get currency list
added_count = 0
for currency in coinbase_currency_list:
if currency['details']['type'] == 'crypto': # only add crytos
pass # I think I will add all of them now. f**k it.
c, created = Currency.objects.get_or_create(symbol=currency['id'],
name=currency['name'])
c.save()
if created:
added_count += 1
return added_count
def load_products():
"""
Load products into database that are on the coinbase pro exchange.
only add if they are not already in database.
"""
product_list = Product.objects.all()
client = PublicClient()
coinbase_product_list = client.get_products()
added_count = 0
for product in coinbase_product_list:
base = Currency.objects.get(symbol=product['base_currency'])
quote = Currency.objects.get(symbol=product['quote_currency'])
p, created = Product.objects.get_or_create(
product_id=product['id'],
base_currency=base,
quote_currency=quote,
display_name=product['display_name'])
p.save()
if created:
added_count += 1
return added_count
def __init__(self):
super(CoinbaseClient, self).__init__()
self.public_client = PublicClient()
api_key = self.settings['api_key']
secret = self.settings['api_secret']
passphrase = self.settings['api_pass']
api_url = self.settings['api_url']
if api_key and secret and passphrase and api_url:
auth_client = AuthenticatedClient(api_key,
secret,
passphrase,
api_url=api_url)
if auth_client:
self.auth_client = auth_client
else:
return Exception(
"Error while loading the authenticated client")
self._load_accounts()
def format_data():
pub_client = PublicClient()
products = pub_client.get_products()
data = {}
for product in products:
current_dat = {}
current_dat['base order min'] = float(product['base_min_size'])
current_dat['base resolution'] = np.abs(
int(np.log10(float(product['base_increment']))))
current_dat['resolution'] = np.abs(
int(np.log10(float(product['quote_increment']))))
current_dat['avoid'] = product['post_only'] or product[
'limit_only'] or product['cancel_only']
data[product['id']] = current_dat
if product['quote_currency'] == 'USD':
data[product['base_currency']] = current_dat
print('{')
for id in data:
print(" '" + id + "'" + ":" + str(data[id]) + ",")
print('}')
class Coinbase(TradeAPI):
def __init__(self):
self.public_client = PublicClient()
def get_available_currencies(self):
# TODO: Update currencies on server startup
return [
Currency(currency_id=product["id"], base_currency=product["base_currency"],
quote_currency=product["quote_currency"],
name=cryptocurrency_icons.get_currency_name(product["base_currency"]),
color=cryptocurrency_icons.get_currency_color(product["base_currency"]))
for product in self.public_client.get_products()
]
def get_historic_rates(self, currency_id, start_date, end_date, granularity):
return [
Candle(currency_id=currency_id, granularity=granularity, time=datetime.utcfromtimestamp(candle[0]),
low=candle[1], high=candle[2], open=candle[3], close=candle[4],
volume=candle[5])
for candle in
self.public_client.get_product_historic_rates(currency_id, start_date.isoformat(), end_date.isoformat(),
granularity)
]
pass
def __init__(self, api_key: str=None,
secret_key: str=None,
passphrase: str=None):
super().__init__()
if any(i is None for i in [api_key, secret_key, passphrase]):
self.client = PublicClient()
else:
self.client = AuthenticatedClient(api_key, secret_key, passphrase)
self.products = self.client.get_products()
self.filters = {product['id']: {
'min_order_size': Decimal(product['base_min_size']),
'max_order_size': Decimal(product['base_max_size']),
'order_step': Decimal('1e-8'),
'price_step': Decimal(product['quote_increment']),
'base': product['quote_currency'],
'commodity': product['base_currency']
} for product in self.products}
class CoinBaseProApi(DataSource):
def __init__(self):
self.data_source = PublicClient()
def get_dataframe(self, product_id_list, granularity, max_count=300):
dataframes = []
for product_id in product_id_list:
dataframes.append(
self.data_source.get_product_historic_rates(
product_id, granularity))
candle_dfs = pd.concat(
dataframes,
axis=1,
keys=[product_id for product_id in product_id_list])
print(candle_dfs)
return candle_dfs
def __init__(self,
trading_type,
verbose,
api_key='',
api_secret='',
api_passphrase='',
**kwargs):
self._trading_type = trading_type
self._verbose = verbose
if trading_type == TradingType.BACKTEST:
raise NotImplementedError()
if self._trading_type == TradingType.SANDBOX:
super().__init__(ExchangeType('coinbaseprosandbox'))
else:
super().__init__(ExchangeType('coinbasepro'))
auth = api_key and api_secret and api_passphrase
self._public_client = PublicClient()
if trading_type == TradingType.SANDBOX:
self._auth_client = AuthenticatedClient(
api_key,
api_secret,
api_passphrase,
api_url="https://api-public.sandbox.pro.coinbase.com"
) if auth else None
else:
self._auth_client = AuthenticatedClient(
api_key, api_secret, api_passphrase) if auth else None
# TODO
self._subscriptions = []
self._ws_client = WebsocketClient(url="wss://ws-feed.pro.coinbase.com",
products="BTC-USD")
def get_historic_candles_for_product(
cb_public_client: PublicClient,
db: Database,
historical_data_suffix: str,
product: dict,
start_time: datetime,
end_time: datetime,
m_granularity=Granularity.ONE_MINUTE) -> None:
granularity_desc = next(key for key, val in granularity_option_map.items()
if val == m_granularity)
collection_name = f'{product["id"].lower()}-{granularity_desc}-{historical_data_suffix}'
collection = db[collection_name]
current_time = start_time
time_diff = end_time - start_time
time_increment = granularity_time_increment_map[m_granularity]
request_count = 0
bar_title = f'Processing {product["id"]}'
bar_length = seconds_to_hours(time_diff)
total_record_count = 0
with tqdm(total=bar_length, desc=bar_title) as progress_bar:
while current_time < end_time:
# coinbase pro public api has a rate limit of 3 requests per second.
# So for every three requests we will wait 1 second.
if request_count == 3:
sleep(1)
request_count = 0
previous_time = current_time
current_time = previous_time + time_increment
candle_stick_data = cb_public_client.get_product_historic_rates(
product["id"],
start=previous_time.isoformat(),
end=current_time.isoformat(),
granularity=300)
# put our data into a pandas dataframe for easy processing
df = pandas.DataFrame(candle_stick_data,
columns=[
'timestamp', 'low', 'high', 'open',
'close', 'volume'
])
records = get_new_records(df, collection)
if len(records):
total_record_count = total_record_count + len(records)
collection.insert_many(records)
request_count = request_count + 1
progress_bar.update(seconds_to_hours(time_increment))
progress_bar.desc = f'Processing {product["id"]} : Record count {total_record_count}'
from cbpro import PublicClient
client = PublicClient()
writer = open("dict.py", "w")
data = client.get_products()
count = 0
writer.write("new_dict = {\n")
for line in data:
if line['quote_currency'] == "USD":
writer.write(" \"" + line['id'] + "\": \"" +
str(len(line['base_increment']) - 2) + "\",\n")
writer.write("}")
def __init__(self):
self.public_client = PublicClient()
class CoinbaseClient(Provider):
PROVIDER_ID = 'coinbase'
def __init__(self):
super(CoinbaseClient, self).__init__()
self.public_client = PublicClient()
api_key = self.settings['api_key']
secret = self.settings['api_secret']
passphrase = self.settings['api_pass']
api_url = self.settings['api_url']
if api_key and secret and passphrase and api_url:
auth_client = AuthenticatedClient(api_key,
secret,
passphrase,
api_url=api_url)
if auth_client:
self.auth_client = auth_client
else:
return Exception(
"Error while loading the authenticated client")
self._load_accounts()
#TODO: move to provider in general form when have more than 1 provider enabled
@property
def accounts(self):
accounts_snapshot = []
for account in self._accounts:
price = self.get_current_price(account)
snapshot = account.snapshot(price=price)
accounts_snapshot.append(snapshot)
return accounts_snapshot
def _load_accounts(self):
try:
accounts = self.auth_client.get_accounts()
valid_accounts = []
for record in accounts:
if "balance" not in record:
return Exception(
"Expecting balance field from account record")
if float(record["balance"]) > 0:
crypto_account = CryptoAccount(record)
valid_accounts.append(crypto_account)
except Exception as e:
return Exception("Error while loading accounts", e)
finally:
self._accounts = valid_accounts
def get_current_price(self, account):
product_id = account.alias.upper()
if product_id != 'USD-USD':
ticker = self.get_ticker(product_id)
price = ticker['price']
if type(price) != float:
price = float(price)
else:
price = 1
return price
def get_account_history(self, account):
account_history = islice(
self.auth_client.get_account_history(account.id), 300)
history = list(account_history)
for h in history:
details = h["details"]
add_fields = {}
for key, value in details.items():
add_fields[key] = value
h.update(add_fields)
h.pop("details")
return history
def get_all_accounts_history(self):
all_history = []
for account in self._accounts:
history = self.get_account_history(account)
all_history.append([h for h in history])
return all_history
def get_fills(self, params: dict = {}):
try:
if "product_id" in params:
product_id = params["product_id"]
fills = self.auth_client.get_fills(product_id=product_id)
elif "order_id" in params:
order_id = params["order_id"]
fills = self.auth_client.get_fills(order_id=order_id)
else:
return Exception(
"Missing required paramaters product_id or order_id")
if fills:
return fills
else:
return Exception(
"Error while trying to load all accounts fills")
except Exception as e:
return e
def get_ticker(self, product_id):
ticker = self.public_client.get_product_ticker(product_id=product_id)
return ticker
def __init__(self):
self.data_source = PublicClient()
def scrape_and_save_order_books(
sym_list,
file_name_base='/Users/rjh2nd/PycharmProjects/CryptoNeuralNet/CryptoBot/HistoricalData/order_books/',
unique_id='',
run_time=24 * 3600):
err_counter = 0
old_row_dict = dict((sym, [0]) for sym in sym_list)
old_fill_dict = dict((sym, ['N/A']) for sym in sym_list)
sym = 'N/A'
public_client = PublicClient()
start_time = time()
elapsed_time = 0
file_name = None
local_file_path = None
fill_file_name = None
local_fill_file_path = None
while (err_counter < 10) and (elapsed_time < run_time):
try:
for sym in sym_list:
#TODO get rid of need to repeat processes for every loop
file_name = sym + '_historical_order_books' + unique_id + '.csv'
local_file_path = file_name_base + file_name
old_row = old_row_dict[sym]
new_row, header_names = get_single_order_book_row(
sym, public_client)
if new_row is None:
# For errors
continue
old_row_dict[sym] = new_row
if old_row == new_row:
# Don't waste time saving repeated rows
continue
save_single_row(local_file_path, new_row, header_names)
err_counter = 0
current_time = time()
elapsed_time = current_time - start_time
except Exception as e:
err_counter = print_err_msg(
'scrape and save ' + sym + ' order_book', e, err_counter)
try:
for sym in sym_list:
prod_id = sym.upper() + '-USD'
recent_fill = list(
islice(
public_client.get_product_trades(product_id=prod_id),
1))[0]
sleep(0.5)
if type(recent_fill) == str:
print('Error: recent_fill is a str, recent_fill = ' +
recent_fill)
continue
recent_fill_side = recent_fill['side']
last_fill_side = old_fill_dict[sym]
old_fill_dict[sym] = recent_fill_side
if last_fill_side == recent_fill_side:
continue
fill_file_name = sym + '_fills' + unique_id + '.csv'
local_fill_file_path = file_name_base + fill_file_name
save_single_row(local_fill_file_path,
list(recent_fill.values()),
list(recent_fill.keys()))
except Exception as e:
err_counter = print_err_msg(' get last fill for ' + sym, e,
err_counter)
return file_name, local_file_path, fill_file_name, local_fill_file_path
import time
from cbpro import PublicClient
import json
import schedule
from utils import setup_custom_logger
from utils import create_snapshot_string
# Do we want to get fixed currency-pairs or dynamic?
CONFIG_MODE = 'fixed'
logger = setup_custom_logger('snapshot-crawler')
product_ids = ['BTC-USD', 'ETH-USD', 'BTC-EUR', 'XRP-USD', 'EOS-USD']
client = PublicClient()
targets = {
pr_id: gzip.open(create_snapshot_string(pr_id), 'a+')
for pr_id in product_ids
}
def job(target_csv_files):
logger.info('Opening new csv files for the next day')
for csv_file in target_csv_files:
csv_file.close()
target_csv_files.clear()
for prod_id in product_ids:
targets[prod_id] = gzip.open(create_snapshot_string(prod_id), 'a+')
return
import numpy as np
import scipy.stats as sps
import matplotlib.pyplot as plt
from cbpro import PublicClient
from app_methods import get_time
client = PublicClient()
data = client.get_product_historic_rates(product_id="ETH-USD",
start=get_time(60000),
end=get_time(0),
granularity=300)
avr = 0
candlesticks = []
for candle in data:
candlesticks.append(float(candle[4]))
new_sum = sum(candlesticks)
avr = new_sum / len(candlesticks)
print(len(candlesticks))
mu = avr
sigma = len(candlesticks)
# define the normal distribution and PDF
dist = sps.norm(loc=mu, scale=sigma)
x = np.linspace(dist.ppf(.001), dist.ppf(.999))
y = dist.pdf(x)
# calculate PPFs