def compare_totle_and_cexs_same_fee(cex_names, base, quote, trade_size, books,
order_type):
cex_savings = {}
# this hardcodes sourceAmount=trade_size when from_token is ETH
from_token, to_token = (quote, base) if order_type == 'buy' else (base,
quote)
from_token, to_token, params = v2_compare_prices.get_from_to_params(
order_type, base, quote, trade_size)
totle_quote = totle_client.try_swap(totle_client.name(),
from_token,
to_token,
params=params,
verbose=False)
if totle_quote:
for cex_name in cex_names:
try:
cex_price = best_price_with_fees(trade_size, books[cex_name],
order_type, FIXED_FEE_PCT)
cex_savings[cex_name] = get_savings(cex_name, cex_price,
totle_quote, base,
trade_size, order_type)
except ValueError as e:
print(
f"{cex_name} raised {e} which resulted in no price quote for {order_type} {base} / {trade_size} {quote}"
)
return cex_savings, totle_quote
def compare_totle_and_aggs_parallel(from_token,
to_token,
from_amount,
usd_trade_size=None):
agg_savings = {}
totle_quote = totle_client.try_swap(totle_client.name(),
from_token,
to_token,
params={'fromAmount': from_amount},
verbose=False,
debug=False)
if totle_quote:
# print(f"SUCCESSFUL getting Totle API Quote buying {to_token} with {from_amount} {from_token}")
futures_agg = {}
with concurrent.futures.ThreadPoolExecutor() as executor:
for agg_client in AGG_CLIENTS:
future = executor.submit(agg_client.get_quote,
from_token,
to_token,
from_amount=from_amount)
futures_agg[future] = agg_client.name()
for f in concurrent.futures.as_completed(futures_agg):
agg_name = futures_agg[f]
agg_quote = f.result()
if agg_quote:
# print(f"SUCCESSFUL getting {agg_name} quote for buying {to_token} with {from_amount} {from_token}")
if agg_quote['price'] == 0:
print(
f"DIVISION BY ZERO: {agg_name} buying {to_token} with {from_amount} {from_token} returned a price of {agg_quote['price']}"
)
continue
savings = get_savings(agg_name,
agg_quote['price'],
totle_quote,
to_token,
usd_trade_size or from_amount,
'buy',
agg_quote=agg_quote,
quote_token=from_token,
print_savings=False)
print(
f"Totle saved {savings['pct_savings']:.2f} percent vs {agg_name} buying {to_token} with {from_amount} {from_token} on {savings['totle_used']}"
)
agg_savings[agg_name] = savings
else:
print(
f"FAILED getting {agg_name} quote: had no price quote for buying {to_token} with {from_amount} {from_token}"
)
else:
print(
f"FAILED getting Totle API Quote buying {to_token} with {from_amount} {from_token}"
)
return agg_savings
def compare_to_totle(base, quote, order_type, trade_size, exchange, ex_price):
"""Returns a savings_data dict comparing price (in *spent* token) to totle's price"""
from_token, to_token, params = get_from_to_params(order_type, base, quote,
trade_size)
totle_quote = totle_client.try_swap(totle_client.name(),
from_token,
to_token,
params=params,
verbose=False)
if totle_quote:
return get_savings(exchange, ex_price, totle_quote, base, trade_size,
order_type)
else:
print(
f"Compare {order_type} {base}/{quote} trade size={trade_size} got no result from Totle"
)
def get_token_prices(tokens):
cmc_data = json.load(open(f'data/cmc_tokens.json'))['data']
usd_prices = {
t['symbol']: float(t['quote']['USD']['price'])
for t in cmc_data if t['symbol'] in tokens and t['platform']
['token_address'] not in token_utils.ADDRESSES_TO_FILTER_OUT
}
# if 'WETH' in usd_prices and 'ETH' not in usd_prices: usd_prices['ETH'] = usd_prices['WETH']
print(f"usd_prices={json.dumps(usd_prices, indent=3)}")
skipped_tokens, missing_tokens = set(), set(tokens) - set(usd_prices)
print(f"CMC had prices for {len(usd_prices)}/{len(tokens)} tokens.")
if len(missing_tokens) > 0:
print(
f"Querying Totle for prices on the remaining {len(missing_tokens)} tokens ({missing_tokens})"
)
for missing_token in missing_tokens:
totle_quote = totle_client.try_swap(totle_client.name(),
'USDC',
missing_token,
params={'fromAmount': 1},
verbose=False,
debug=False)
if totle_quote: # set the from_amount so it's roughly the same across all swaps
print(
f"totle_quote for {missing_token} = {totle_quote['price']}"
)
usd_prices[missing_token] = totle_quote['price']
else:
# If we can't get a price from CMC or Totle, then just discard this token. Other aggs may have the pair, but if you can't
# buy it for ETH on Totle, then it is essentially not a "tradable" token as curated by Totle, and thus not in this study.
skipped_tokens.add(missing_token)
if any(skipped_tokens):
raise ValueError(
f"Skipping {skipped_tokens} because we couldn't get a price from CMC or Totle"
)
return usd_prices
def compare_totle_and_cexs(cex_name_client,
base,
quote,
trade_size,
books,
order_type,
totle_quote=None,
fee_override=None):
print(
f"compare_totle_and_cexs client_names = {list(cex_name_client.keys())}"
)
cex_savings = {}
if order_type == 'buy':
from_token, to_token, params = quote, base, {'fromAmount': trade_size}
else:
from_token, to_token, params = base, quote, {'toAmount': trade_size}
totle_quote = totle_quote or totle_client.try_swap(totle_client.name(),
from_token,
to_token,
params=params,
verbose=False)
if totle_quote:
for cex_name_c, cex_client in cex_name_client.items():
if books.get(cex_name_c):
try:
cex_price = best_price_with_fees(
trade_size, books[cex_name_c], order_type, fee_override
or cex_client.fee_pct())
cex_savings[cex_name_c] = get_savings(
cex_name_c, cex_price, totle_quote, base, trade_size,
order_type)
except ValueError as e:
print(
f"{cex_name_c} raised {e} which resulted in no price quote for {order_type} {base} / {trade_size} {quote}"
)
else:
print(f"No books for {base}/{quote} {cex_name_c}")
return cex_savings
def get_token_prices(tokens=None):
all_tradable_tokens = tokens or token_utils.tradable_tokens()
all_tradable_tokens.pop('ETH')
# TODO: remove ETH it's not really an ERC-20
cmc_data = json.load(open(f'data/cmc_tokens.json'))['data']
usd_prices = {
t['symbol']: float(t['quote']['USD']['price'])
for t in cmc_data if t['symbol'] in all_tradable_tokens
}
skipped_tokens, missing_tokens = set(
), set(all_tradable_tokens) - set(usd_prices)
print(
f"CMC had prices for {len(usd_prices)}/{len(all_tradable_tokens)} tokens. Querying Totle for prices on the remaining {len(missing_tokens)} tokens"
)
for missing_token in missing_tokens:
if missing_token == 'CETH':
usd_prices[missing_token] = 2.83
else:
totle_quote = totle_client.try_swap(totle_client.name(),
missing_token,
'ETH',
params={'toAmount': 0.1},
verbose=False,
debug=False)
if totle_quote: # set the from_amount so it's roughly the same across all swaps
usd_prices[missing_token] = ETH_PRICE / totle_quote['price']
else:
# If we can't get a price from CMC or Totle, then just discard this token. Other aggs may have the pair, but if you can't
# buy it for ETH on Totle, then it is essentially not a "tradable" token as curated by Totle, and thus not in this study.
skipped_tokens.add(missing_token)
print(
f"Skipping {skipped_tokens} because we couldn't get a price from CMC or Totle"
)
return usd_prices
def pct_savings_gen(per_pair_savings):
"""Generates a sequence of (pair, trade_size, agg/exchange, [pct_savings]) for all leaves in the given dict"""
for pair, ts_ex_savings in sorted(per_pair_savings.items()):
for trade_size, ex_savings in ts_ex_savings.items():
for exchange, pct_savings in ex_savings.items():
yield pair, trade_size, exchange, pct_savings
########################################################################################################################
# JSON file aggregation functions
DEX_AG = dexag_client.name()
ONE_INCH = oneinch_client.name()
PARASWAP = paraswap_client.name()
TOTLE_EX = totle_client.name()
AGG_NAMES = [DEX_AG, ONE_INCH, PARASWAP]
JSON_DATA_DIR = f"{os.path.dirname(os.path.abspath(__file__))}/order_splitting_data"
@functools.lru_cache()
def get_all_splits_by_agg(files=None):
"""Returns an aggregated dict of split data, i.e. token: {trade_size: {agg: [{dex: pct, dex: pct}, {...}, ...]}}"""
files = files or glob.glob(f'{JSON_DATA_DIR}/2019*ts_splits_by_agg.json')
tok_ts_splits_by_agg = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
for f in files:
for token, ts_splits_by_agg in json.load(open(f)).items():
for ts, agg_splits in ts_splits_by_agg.items():
for agg, split in agg_splits.items():
tok_ts_splits_by_agg[token][ts][agg].append(split)
def do_summary():
# agg_pairs = summarize_csv('outputs/totle_vs_agg_supported_tokens_2019-12-07_11:06:53.csv')
# print(f"agg_pairs['Totle'] ({len(agg_pairs['Totle'])} pairs) = {agg_pairs['Totle']}")
# exit(0)
csv_files = [
'totle_vs_agg_supported_tokens_2019-12-05_18:47:50.csv',
'totle_vs_agg_supported_tokens_2019-12-06_11:25:17.csv',
'totle_vs_agg_supported_tokens_2019-12-06_17:55:13.csv',
'totle_vs_agg_supported_tokens_2019-12-07_13:04:21.csv',
'totle_vs_agg_supported_tokens_2019-12-07_11:06:53.csv',
'totle_vs_agg_supported_tokens_2019-12-07_17:09:20.csv',
'totle_vs_agg_supported_tokens_2019-12-07_17:54:29.csv'
]
agg_pairs = defaultdict(set)
for csv_file in csv_files:
agg_pairs_f = summarize_csv(f'outputs/{csv_file}')
for agg_name, pairs in agg_pairs_f.items():
agg_pairs[agg_name] |= set(pairs)
agg_tokens = {}
for agg_name, pairs in agg_pairs.items():
agg_tokens[agg_name] = set(sum(map(list, pairs), []))
print("\n\n")
for agg, pairs in agg_pairs.items():
print(f"{agg} supports a total of {len(pairs)} ERC-20/ERC-20 pairs")
totle_pairs, totle_tokens = agg_pairs[totle_client.name()], agg_tokens[
totle_client.name()]
all_other_agg_exc_tokens, all_other_agg_exc_pairs = set(), set()
summary_csv = [
'Competitor,"Overlap","Exclusive to Totle","Exclusive to Competitor"'
]
for other_agg_name in agg_pairs:
if other_agg_name == totle_client.name(): continue
other_agg_tokens = agg_tokens[other_agg_name]
totle_exc_tokens = set(totle_tokens) - set(other_agg_tokens)
other_agg_exc_tokens = set(other_agg_tokens) - set(totle_tokens)
all_other_agg_exc_tokens |= other_agg_exc_tokens
print(
f"\nTotle supports {len(totle_exc_tokens)} tokens that {other_agg_name} doesn't: {totle_exc_tokens}"
)
print(
f"{other_agg_name} supports {len(other_agg_exc_tokens)} tokens that Totle doesn't: {other_agg_exc_tokens}"
)
other_agg_pairs = agg_pairs[other_agg_name]
totle_exc_pairs = set(totle_pairs) - set(other_agg_pairs)
other_agg_exc_pairs = set(other_agg_pairs) - set(totle_pairs)
all_other_agg_exc_pairs |= other_agg_exc_pairs
overlap = set(totle_pairs) & set(other_agg_pairs)
summary_csv.append(
f"{other_agg_name},{len(overlap)},{len(totle_exc_pairs)},{len(other_agg_exc_pairs)}"
)
# print(f"\nTotle and {other_agg_name} overlap in {len(overlap)} ERC-20/ERC-20 pairs")
# print(f"Totle supports {len(totle_has)} pairs that {other_agg_name} doesn't: Totle: {len(set(totle_pairs))} {other_agg_name}: {len(set(other_agg_pairs))} diff={len(set(totle_pairs))-len(set(other_agg_pairs))}")
# print(f"{other_agg_name} supports {len(other_agg_has)} pairs that Totle doesn't:")
exc_pair_involving_exc_token = [
p for p in other_agg_exc_pairs
if p[0] in other_agg_exc_tokens or p[1] in other_agg_exc_tokens
]
print(
f"Of {other_agg_name}'s {len(other_agg_exc_pairs)} exclusive pairs, {len(exc_pair_involving_exc_token)} involved {other_agg_exc_tokens}"
)
print(f"\n\n\nall_other_agg_exc_tokens={all_other_agg_exc_tokens}")
all_exc_pair_involving_exc_token = [
p for p in all_other_agg_exc_pairs
if p[0] in all_other_agg_exc_tokens or p[1] in all_other_agg_exc_tokens
]
print(
f"Of all {len(all_other_agg_exc_pairs)} pairs exclusive to competitors, {len(all_exc_pair_involving_exc_token)} involved {all_other_agg_exc_tokens}"
)
bad_pairs = all_other_agg_exc_pairs - set(all_exc_pair_involving_exc_token)
bad_pairs_strs = sorted([f"{p[0]}/{p[1]}" for p in bad_pairs])
print(
f"The {len(bad_pairs)} pairs exclusive to competitors not involving these {len(all_other_agg_exc_tokens)} tokens were: {bad_pairs_strs}"
)
bad_tokens = set(sum(map(list, bad_pairs), []))
print(
f"These pairs were various combinations of the following {len(bad_tokens)} tokens: {bad_tokens}"
)
bad_tokens = {'CETH', 'MLN', 'LRC'}
for t in bad_tokens:
for agg_name, pairs in agg_pairs.items():
agg_has_bad = [p for p in pairs if t in p]
print(f"{agg_name} {t} pairs: {agg_has_bad}")
totle_has_bad = [p for p in totle_pairs if t in p]
print(f"Totle has {len(totle_has_bad)} pairs with {t}")
print(f"\n\n")
for row in summary_csv:
print(row)
print(f"\n\nOverlap Pairs")
overlap_pairs = []
dexag_supported_tokens = dexag_client.supported_tokens()
for o_pair in agg_pairs[paraswap_client.name(
)]: # iterate through the agg with the least pairs
if all([o_pair in pairs for pairs in agg_pairs.values()]):
# dexag no longer supports certain tokens and pairs constructed from them, so we add this constraint
if o_pair[0] in dexag_supported_tokens and o_pair[
1] in dexag_supported_tokens:
overlap_pairs.append(o_pair)
else:
print(
f"{o_pair} was removed because DEXAG does not support one of the tokens"
)
print(
f"there are {len(overlap_pairs)} pairs supported by Totle and all competitors"
)
print(f"OVERLAP_PAIRS={overlap_pairs}")
print(f"\n\nTotle Supported Pairs CSV")
print("base,quote")
for base, quote in sorted(totle_pairs):
print(f"{base},{quote}")
def test_basics():
print(totle_client.name())
print(totle_client.exchanges())
print(totle_client.enabled_exchanges())
print(totle_client.data_exchanges())
def compare_dex_prices(token,
supported_pairs,
non_liquid_tokens,
liquid_dexs,
order_type,
params=None,
verbose=True,
debug=False):
"""Returns a dict of dex: savings_data for Totle and other DEXs"""
kw_params = {
k: v
for k, v in vars().items() if k in ['params', 'verbose', 'debug']
}
savings = {}
# TODO: replace token with from_token, to_token
if order_type == 'buy':
trade_size = params['fromAmount']
from_token, to_token, bidask = ('ETH', token, 'ask')
elif order_type == 'sell':
trade_size = params['toAmount']
from_token, to_token, bidask = (token, 'ETH', 'bid')
else:
raise ValueError(f"order_type must be either 'buy' or 'sell'")
totle_ex = totle_client.name()
# Get the best price using Totle's aggregated order books
totle_quote = totle_client.try_swap(totle_ex, from_token, to_token,
**kw_params)
if totle_quote:
totle_used = totle_quote['totleUsed']
swap_prices = {totle_ex: totle_quote['price']}
# we only need to pre-populate of if len(totle_used) <= 1, otherwise dexs_to_compare will include all dexes
if len(totle_used) == 1:
supported_pairs[totle_used[0]].append([from_token, to_token])
# Compare to best prices from other DEXs
# don't compare to the one that Totle used, unless Totle used multiple DEXs
dexs_to_compare = [
dex for dex in liquid_dexs
if dex != totle_used[0] or len(totle_used) > 1
]
for dex in dexs_to_compare:
dex_sd = totle_client.try_swap(dex,
from_token,
to_token,
exchange=dex,
**kw_params)
if dex_sd:
swap_prices[dex] = dex_sd['price']
if swap_prices[dex] < 0.0:
raise ValueError(
f"{dex} had an invalid price={swap_prices[dex]}")
supported_pairs[dex].append([from_token, to_token])
other_dexs = [k for k in swap_prices if k != totle_ex]
if other_dexs: # there is data to compare
totle_price = swap_prices[totle_ex]
for e in other_dexs:
# totle_price assumed lower
pct_savings = get_pct_savings(totle_price, swap_prices[e])
savings[e] = get_savings(e,
swap_prices[e],
totle_quote,
token,
trade_size,
order_type,
quote_token='ETH',
print_savings=True)
print(
f"Totle saved {pct_savings:.2f} percent vs {e} {order_type}ing {token} on {totle_used} trade size={trade_size} ETH"
)
else:
print(
f"Could not compare {token} prices. Only valid price was {swap_prices}"
)
# although we'll likely get the same result at higher trade sizes, don't over-
# optimize. Past data shows there are more liquid tokens at higher trade sizes
# than what we get with this optimization
# non_liquid_tokens.append(token)
else:
non_liquid_tokens.append(token)
return savings