def list(self):
df = self._list()
set_print_settings()
if df.empty:
print('There are no datasets available yet.')
else:
print(df)
def compare_current_with_last_candle(self, exchange, assets, end_dt,
freq, data_frequency, data_portal):
"""
Creates DataFrames from the bundle and exchange for the specified
data set.
Parameters
----------
exchange: Exchange
assets
end_dt
bar_count
freq
data_frequency
data_portal
Returns
-------
"""
data = dict()
assets = sorted(assets, key=lambda a: a.symbol)
log_catcher = TestHandler()
with log_catcher:
symbols = [asset.symbol for asset in assets]
print(
'comparing data for {}/{} with {} timeframe on {}'.format(
exchange.name, symbols, freq, end_dt
)
)
data['candle'] = data_portal.get_history_window(
assets=assets,
end_dt=end_dt,
bar_count=1,
frequency=freq,
field='close',
data_frequency=data_frequency,
)
set_print_settings()
print(
'the bundle first / last row:\n{}'.format(
data['candle'].iloc[[-1]]
)
)
current = data_portal.get_spot_value(
assets=assets,
field='close',
dt=end_dt,
data_frequency=data_frequency,
)
data['current'] = pd.Series(data=current, index=assets)
print(
'the current price:\n{}'.format(
data['current']
)
)
pass
def compare_current_with_last_candle(self, exchange, assets, end_dt,
freq, data_frequency, data_portal):
"""
Creates DataFrames from the bundle and exchange for the specified
data set.
Parameters
----------
exchange: Exchange
assets
end_dt
bar_count
freq
data_frequency
data_portal
Returns
-------
"""
data = dict()
assets = sorted(assets, key=lambda a: a.symbol)
log_catcher = TestHandler()
with log_catcher:
symbols = [asset.symbol for asset in assets]
print(
'comparing data for {}/{} with {} timeframe on {}'.format(
exchange.name, symbols, freq, end_dt
)
)
data['candle'] = data_portal.get_history_window(
assets=assets,
end_dt=end_dt,
bar_count=1,
frequency=freq,
field='close',
data_frequency=data_frequency,
)
set_print_settings()
print(
'the bundle first / last row:\n{}'.format(
data['candle'].iloc[[-1]]
)
)
current = data_portal.get_spot_value(
assets=assets,
field='close',
dt=end_dt,
data_frequency=data_frequency,
)
data['current'] = pd.Series(data=current, index=assets)
print(
'the current price:\n{}'.format(
data['current']
)
)
pass
def analyze(context, perf):
set_print_settings()
transaction_df = extract_transactions(perf)
print('the transactions:\n{}'.format(transaction_df))
orders_df = extract_orders(perf)
print('the orders:\n{}'.format(orders_df))
stats = get_pretty_stats(perf, show_tail=False, num_rows=5)
print('the stats:\n{}'.format(stats))
pass
def analyze(context, perf):
set_print_settings()
transaction_df = extract_transactions(perf)
print('the transactions:\n{}'.format(transaction_df))
orders_df = extract_orders(perf)
print('the orders:\n{}'.format(orders_df))
stats = get_pretty_stats(perf, show_tail=False, num_rows=5)
print('the stats:\n{}'.format(stats))
pass
def handle_data(context, data):
prices = data.history(symbol('xlm_eth'),
fields=['open', 'high', 'low', 'close'],
bar_count=50,
frequency='1T')
set_print_settings()
print(prices.tail(10))
context.data.append(prices)
context.i = context.i + 1
if context.i == 3:
context.interrupt_algorithm()
def list(self):
data_sources = self.mkt_contract.functions.getAllProviders().call()
data = []
for index, data_source in enumerate(data_sources):
if index > 0:
if 'test' not in Web3.toText(data_source).lower():
data.append(dict(dataset=self.to_text(data_source)))
df = pd.DataFrame(data)
set_print_settings()
if df.empty:
print('There are no datasets available yet.')
else:
print(df)
def initialize(context):
# This initialize function sets any data or variables that you'll use in
# your algorithm. For instance, you'll want to define the trading pair (or
# trading pairs) you want to backtest. You'll also want to define any
# parameters or values you're going to use.
# In our example, we're looking at Neo in Ether.
df = get_dataset('testmarketcap2') # type: pd.DataFrame
# Picking a specific date in our DataFrame
first_dt = df.index.get_level_values(0)[0]
# Since we use a MultiIndex with date / symbol, picking a date will
# result in a new DataFrame for the selected date with a single
# symbol index
df = df.xs(first_dt, level=0)
# Keep only the top coins by market cap
df = df.loc[df['market_cap_usd'].isin(df['market_cap_usd'].nlargest(100))]
set_print_settings()
df.sort_values(by=['market_cap_usd'], ascending=True, inplace=True)
print('the marketplace data:\n{}'.format(df))
# Pick the 5 assets with the lowest market cap for trading
quote_currency = 'eth'
exchange = context.exchanges[next(iter(context.exchanges))]
symbols = [
a.symbol for a in exchange.assets if a.start_date < context.datetime
]
context.assets = []
for currency, price in df['market_cap_usd'].iteritems():
if len(context.assets) >= 5:
break
s = '{}_{}'.format(currency.decode('utf-8'), quote_currency)
if s in symbols:
context.assets.append(symbol(s))
context.base_price = None
context.current_day = None
context.RSI_OVERSOLD = 55
context.RSI_OVERBOUGHT = 60
context.CANDLE_SIZE = '5T'
context.start_time = time.time()
def get_withdraw_amount(self, dataset=None):
if dataset is None:
df_sets = self._list()
if df_sets.empty:
print('There are no datasets available yet.')
return
set_print_settings()
while True:
print(df_sets)
dataset_num = input(
'Choose the dataset you want to '
'get the withdraw amount for [0..{}]: '.format(
df_sets.size - 1))
try:
dataset_num = int(dataset_num)
except ValueError:
print(
'Enter a number between 0 and {}'.format(df_sets.size -
1))
else:
if dataset_num not in range(0, df_sets.size):
print('Enter a number between 0 and {}'.format(
df_sets.size - 1))
else:
dataset = df_sets.iloc[dataset_num]['dataset']
break
dataset = dataset.lower()
address = self.choose_pubaddr()[0]
provider_info = self.mkt_contract.functions.getDataProviderInfo(
Web3.toHex(dataset.encode())).call()
if not provider_info[4]:
print('The requested "{}" dataset is not registered in '
'the Data Marketplace.'.format(dataset))
return
withdraw_amount = from_grains(
self.mkt_contract.functions.getWithdrawAmount(
Web3.toHex(dataset.encode())).call())
print('{} ENG'.format(withdraw_amount))
def test_get_candles(self):
log.info('retrieving candles')
candles = self.exchange.get_candles(
freq='1T',
assets=[self.exchange.get_asset('eth_btc')],
bar_count=200,
# start_dt=pd.to_datetime('2017-09-01', utc=True),
)
for asset in candles:
df = pd.DataFrame(candles[asset])
df.set_index('last_traded', drop=True, inplace=True)
set_print_settings()
print('got {} candles'.format(len(df)))
print(df.head(10))
print(df.tail(10))
pass
def ingest(self, ds_name=None, start=None, end=None, force_download=False):
if ds_name is None:
df_sets = self._list()
if df_sets.empty:
print('There are no datasets available yet.')
return
set_print_settings()
while True:
print(df_sets)
dataset_num = input('Choose the dataset you want to '
'ingest [0..{}]: '.format(df_sets.size -
1))
try:
dataset_num = int(dataset_num)
except ValueError:
print(
'Enter a number between 0 and {}'.format(df_sets.size -
1))
else:
if dataset_num not in range(0, df_sets.size):
print('Enter a number between 0 and {}'.format(
df_sets.size - 1))
else:
ds_name = df_sets.iloc[dataset_num]['dataset']
break
# ds_name = ds_name.lower()
# TODO: catch error conditions
provider_info = self.mkt_contract.functions.getDataProviderInfo(
Web3.toHex(ds_name)).call()
if not provider_info[4]:
print('The requested "{}" dataset is not registered in '
'the Data Marketplace.'.format(ds_name))
return
address, address_i = self.choose_pubaddr()
fns = self.mkt_contract.functions
check_sub = fns.checkAddressSubscription(address,
Web3.toHex(ds_name)).call()
if check_sub[0] != address or self.to_text(check_sub[1]) != ds_name:
print('You are not subscribed to dataset "{}" with address {}. '
'Plese subscribe first.'.format(ds_name, address))
return
if not check_sub[5]:
print('Your subscription to dataset "{}" expired on {} UTC.'
'Please renew your subscription by running:\n'
'catalyst marketplace subscribe --dataset={}'.format(
ds_name, pd.to_datetime(check_sub[4], unit='s',
utc=True), ds_name))
if 'key' in self.addresses[address_i]:
key = self.addresses[address_i]['key']
secret = self.addresses[address_i]['secret']
else:
key, secret = get_key_secret(address,
self.addresses[address_i]['wallet'])
headers = get_signed_headers(ds_name, key, secret)
log.info('Starting download of dataset for ingestion...')
r = requests.post(
'{}/marketplace/ingest'.format(AUTH_SERVER),
headers=headers,
stream=True,
)
if r.status_code == 200:
log.info('Dataset downloaded successfully. Processing dataset...')
bundle_folder = get_data_source_folder(ds_name)
shutil.rmtree(bundle_folder, ignore_errors=True)
target_path = get_temp_bundles_folder()
try:
decoder = MultipartDecoder.from_response(r)
# with maybe_show_progress(
# iter(decoder.parts),
# True,
# label='Processing files') as part:
counter = 1
for part in decoder.parts:
log.info("Processing file {} of {}".format(
counter, len(decoder.parts)))
h = part.headers[b'Content-Disposition'].decode('utf-8')
# Extracting the filename from the header
name = re.search(r'filename="(.*)"', h).group(1)
filename = os.path.join(target_path, name)
with open(filename, 'wb') as f:
# for chunk in part.content.iter_content(
# chunk_size=1024):
# if chunk: # filter out keep-alive new chunks
# f.write(chunk)
f.write(part.content)
self.process_temp_bundle(ds_name, filename)
counter += 1
except NonMultipartContentTypeException:
response = r.json()
raise MarketplaceHTTPRequest(
request='ingest dataset',
error=response,
)
else:
raise MarketplaceHTTPRequest(
request='ingest dataset',
error=r.status_code,
)
log.info('{} ingested successfully'.format(ds_name))
def subscribe(self, dataset=None):
if dataset is None:
df_sets = self._list()
if df_sets.empty:
print('There are no datasets available yet.')
return
set_print_settings()
while True:
print(df_sets)
dataset_num = input(
'Choose the dataset you want to '
'subscribe to [0..{}]: '.format(df_sets.size - 1))
try:
dataset_num = int(dataset_num)
except ValueError:
print(
'Enter a number between 0 and {}'.format(df_sets.size -
1))
else:
if dataset_num not in range(0, df_sets.size):
print('Enter a number between 0 and {}'.format(
df_sets.size - 1))
else:
dataset = df_sets.iloc[dataset_num]['dataset']
break
dataset = dataset.lower()
address = self.choose_pubaddr()[0]
provider_info = self.mkt_contract.functions.getDataProviderInfo(
Web3.toHex(dataset)).call()
if not provider_info[4]:
print('The requested "{}" dataset is not registered in '
'the Data Marketplace.'.format(dataset))
return
grains = provider_info[1]
price = from_grains(grains)
subscribed = self.mkt_contract.functions.checkAddressSubscription(
address, Web3.toHex(dataset)).call()
if subscribed[5]:
print('\nYou are already subscribed to the "{}" dataset.\n'
'Your subscription started on {} UTC, and is valid until '
'{} UTC.'.format(
dataset, pd.to_datetime(subscribed[3],
unit='s',
utc=True),
pd.to_datetime(subscribed[4], unit='s', utc=True)))
return
print('\nThe price for a monthly subscription to this dataset is'
' {} ENG'.format(price))
print('Checking that the ENG balance in {} is greater than {} '
'ENG... '.format(address, price),
end='')
wallet_address = address[2:]
balance = self.web3.eth.call({
'from':
address,
'to':
self.eng_contract_address,
'data':
'0x70a08231000000000000000000000000{}'.format(wallet_address)
})
try:
balance = Web3.toInt(balance) # web3 >= 4.0.0b7
except TypeError:
balance = Web3.toInt(hexstr=balance) # web3 <= 4.0.0b6
if balance > grains:
print('OK.')
else:
print('FAIL.\n\nAddress {} balance is {} ENG,\nwhich is lower '
'than the price of the dataset that you are trying to\n'
'buy: {} ENG. Get enough ENG to cover the costs of the '
'monthly\nsubscription for what you are trying to buy, '
'and try again.'.format(address, from_grains(balance),
price))
return
while True:
agree_pay = input('Please confirm that you agree to pay {} ENG '
'for a monthly subscription to the dataset "{}" '
'starting today. [default: Y] '.format(
price, dataset)) or 'y'
if agree_pay.lower() not in ('y', 'n'):
print("Please answer Y or N.")
else:
if agree_pay.lower() == 'y':
break
else:
return
print('Ready to subscribe to dataset {}.\n'.format(dataset))
print('In order to execute the subscription, you will need to sign '
'two different transactions:\n'
'1. First transaction is to authorize the Marketplace contract '
'to spend {} ENG on your behalf.\n'
'2. Second transaction is the actual subscription for the '
'desired dataset'.format(price))
tx = self.eng_contract.functions.approve(
self.mkt_contract_address,
grains,
).buildTransaction({
'from': address,
'nonce': self.web3.eth.getTransactionCount(address)
})
signed_tx = self.sign_transaction(tx)
try:
tx_hash = '0x{}'.format(
bin_hex(self.web3.eth.sendRawTransaction(signed_tx)))
print('\nThis is the TxHash for this transaction: {}'.format(
tx_hash))
except Exception as e:
print('Unable to subscribe to data source: {}'.format(e))
return
self.check_transaction(tx_hash)
print('Waiting for the first transaction to succeed...')
while True:
try:
if self.web3.eth.getTransactionReceipt(tx_hash).status:
break
else:
print('\nTransaction failed. Aborting...')
return
except AttributeError:
pass
for i in range(0, 10):
print('.', end='', flush=True)
time.sleep(1)
print('\nFirst transaction successful!\n'
'Now processing second transaction.')
tx = self.mkt_contract.functions.subscribe(
Web3.toHex(dataset), ).buildTransaction({
'from':
address,
'nonce':
self.web3.eth.getTransactionCount(address)
})
signed_tx = self.sign_transaction(tx)
try:
tx_hash = '0x{}'.format(
bin_hex(self.web3.eth.sendRawTransaction(signed_tx)))
print('\nThis is the TxHash for this transaction: '
'{}'.format(tx_hash))
except Exception as e:
print('Unable to subscribe to data source: {}'.format(e))
return
self.check_transaction(tx_hash)
print('Waiting for the second transaction to succeed...')
while True:
try:
if self.web3.eth.getTransactionReceipt(tx_hash).status:
break
else:
print('\nTransaction failed. Aborting...')
return
except AttributeError:
pass
for i in range(0, 10):
print('.', end='', flush=True)
time.sleep(1)
print('\nSecond transaction successful!\n'
'You have successfully subscribed to dataset {} with'
'address {}.\n'
'You can now ingest this dataset anytime during the '
'next month by running the following command:\n'
'catalyst marketplace ingest --dataset={}'.format(
dataset, address, dataset))
def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
freq, data_frequency, data_portal):
"""
Creates DataFrames from the bundle and exchange for the specified
data set.
Parameters
----------
exchange: Exchange
assets
end_dt
bar_count
freq
data_frequency
data_portal
Returns
-------
"""
data = dict()
log_catcher = TestHandler()
with log_catcher:
symbols = [asset.symbol for asset in assets]
print(
'comparing data for {}/{} with {} timeframe until {}'.format(
exchange.name, symbols, freq, end_dt
)
)
data['bundle'] = data_portal.get_history_window(
assets=assets,
end_dt=end_dt,
bar_count=bar_count,
frequency=freq,
field='close',
data_frequency=data_frequency,
)
set_print_settings()
print(
'the bundle first / last row:\n{}'.format(
data['bundle'].iloc[[-1, 0]]
)
)
candles = exchange.get_candles(
end_dt=end_dt,
freq=freq,
assets=assets,
bar_count=bar_count,
)
data['exchange'] = get_candles_df(
candles=candles,
field='close',
freq=freq,
bar_count=bar_count,
end_dt=end_dt,
)
print(
'the exchange first / last row:\n{}'.format(
data['exchange'].iloc[[-1, 0]]
)
)
for source in data:
df = data[source]
path, folder = output_df(
df, assets, '{}_{}'.format(freq, source)
)
print('saved {} test results: {}'.format(end_dt, folder))
assert_frame_equal(
right=data['bundle'],
left=data['exchange'],
check_less_precise=1,
)
try:
assert_frame_equal(
right=data['bundle'],
left=data['exchange'],
check_less_precise=min([a.decimals for a in assets]),
)
except Exception as e:
print('Some differences were found within a 1 decimal point '
'interval of confidence: {}'.format(e))
with open(os.path.join(folder, 'compare.txt'), 'w+') as handle:
handle.write(e.args[0])
pass
def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
freq, data_frequency, data_portal, field):
"""
Creates DataFrames from the bundle and exchange for the specified
data set.
Parameters
----------
exchange: Exchange
assets
end_dt
bar_count
freq
data_frequency
data_portal
Returns
-------
"""
data = dict()
log_catcher = TestHandler()
with log_catcher:
symbols = [asset.symbol for asset in assets]
print(
'comparing {} for {}/{} with {} timeframe until {}'.format(
field, exchange.name, symbols, freq, end_dt
)
)
data['bundle'] = data_portal.get_history_window(
assets=assets,
end_dt=end_dt,
bar_count=bar_count,
frequency=freq,
field=field,
data_frequency=data_frequency,
)
set_print_settings()
print(
'the bundle data:\n{}'.format(
data['bundle']
)
)
candles = exchange.get_candles(
end_dt=end_dt,
freq=freq,
assets=assets,
bar_count=bar_count,
)
data['exchange'] = get_candles_df(
candles=candles,
field=field,
freq=freq,
bar_count=bar_count,
end_dt=end_dt,
)
print(
'the exchange data:\n{}'.format(
data['exchange']
)
)
for source in data:
df = data[source]
path, folder = output_df(
df, assets, '{}_{}'.format(freq, source)
)
print('saved {} test results: {}'.format(end_dt, folder))
assert_frame_equal(
right=data['bundle'][:-1],
left=data['exchange'][:-1],
check_less_precise=1,
)
try:
assert_frame_equal(
right=data['bundle'][:-1],
left=data['exchange'][:-1],
check_less_precise=min([a.decimals for a in assets]),
)
except Exception as e:
print(
'Some differences were found within a 1 decimal point '
'interval of confidence: {}'.format(e)
)
with open(os.path.join(folder, 'compare.txt'), 'w+') as handle:
handle.write(e.args[0])
pass
def withdraw(self, dataset=None):
if dataset is None:
df_sets = self._list()
if df_sets.empty:
print('There are no datasets available yet.')
return
set_print_settings()
while True:
print(df_sets)
dataset_num = input('Choose the dataset you want to '
'withdraw from [0..{}]: '.format(
df_sets.size - 1))
try:
dataset_num = int(dataset_num)
except ValueError:
print('Enter a number between 0 and {}'.format(
df_sets.size - 1))
else:
if dataset_num not in range(0, df_sets.size):
print('Enter a number between 0 and {}'.format(
df_sets.size - 1))
else:
dataset = df_sets.iloc[dataset_num]['dataset']
break
dataset = dataset.lower()
address = self.choose_pubaddr()[0]
provider_info = self.mkt_contract.functions.getDataProviderInfo(
Web3.toHex(dataset.encode())
).call()
if not provider_info[4]:
print('The requested "{}" dataset is not registered in '
'the Data Marketplace.'.format(dataset))
return
try:
tx = self.mkt_contract.functions.withdrawProvider(
Web3.toHex(dataset.encode()),
).buildTransaction(
{'from': address,
'nonce': self.web3.eth.getTransactionCount(address)}
)
signed_tx = self.sign_transaction(tx)
tx_hash = '0x{}'.format(
bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
)
print(
'\nThis is the TxHash for this transaction: {}'.format(tx_hash)
)
except Exception as e:
print('Unable to withdraw: {}'.format(e))
return
self.check_transaction(tx_hash)
print('Waiting for the transaction to succeed...')
while True:
try:
if self.web3.eth.getTransactionReceipt(tx_hash).status:
break
else:
print('\nTransaction failed. Aborting...')
return
except AttributeError:
pass
for i in range(0, 10):
print('.', end='', flush=True)
time.sleep(1)
print('\nTransaction successful!\n'
'You have successfully withdrawn your earned ENG\n')
