def __init__(self,
_api_url='https://api.darwinex.com',
_api_name='quotes',
_version=1.0):
super(DWX_Quotes_API, self).__init__(_api_url, _api_name, _version)
self._graphics = DWX_Graphics_Helpers()
class DWX_Quotes_API(DWX_API):
def __init__(self,
_api_url='https://api.darwinex.com',
_api_name='quotes',
_version=1.0):
super(DWX_Quotes_API, self).__init__(_api_url, _api_name, _version)
self._graphics = DWX_Graphics_Helpers()
# Note: Pass _json=False and _stream=True to self._Call_API_() to
# receive request object instead of JSON text output.
##########################################################################
def _stream_quotes_(self,
_endpoint='/quotes',
_symbols=['THA.4.12', 'LVS.4.20']):
# Create session for streaming quotes
_s = Session()
# Construct DARWIN list for sending to the endpoint
_data = "{\"productNames\": [ \"" + "\",\"".join(_symbols) + "\" ]}"
# Retreive Request object for session
_ret = self._Call_API_(_endpoint,
_type='POST',
_data=_data,
_json=False,
_stream=True)
# Send the request
_resp = _s.send(_ret.prepare(), stream=True, verify=True)
# Yield output
for _l in _resp.iter_lines():
if _l:
yield _l
##########################################################################
def _process_stream_(self,
_symbols=[
'DWZ.4.7', 'DWC.4.20', 'LVS.4.20', 'SYO.4.24',
'YZZ.4.20'
],
_plot=True):
# Create empty dataframe
self._df = pd.DataFrame(columns=_symbols)
# Create graphing objects if _plot==True
if _plot:
# Create fig, ax
if len(_symbols) > 1:
_fig, _ax = plt.subplots(nrows=1,
ncols=len(_symbols),
figsize=(20, 4))
else:
_fig, _ax = plt.subplots(figsize=(10, 6))
# for line in self.streaming(symbols=_symbols):
for _ret in self._stream_quotes_(_symbols=_symbols):
# Parse string to dict
_stream = eval(_ret)
# Extract values of interest.
_darwin = _stream['productName']
_quote = _stream['quote']
_timestamp = _stream['timestamp']
# Add to DataFrame
_quotes = [np.nan for x in range(self._df.shape[1])]
_quotes[_symbols.index(_darwin)] = _quote
_row = pd.Series(index=_symbols, data=_quotes, name=_timestamp)
self._df = self._df.append(_row)
self._df.fillna(method='ffill', inplace=True)
self._df.fillna(method='bfill', inplace=True)
if _plot == False:
print(_stream)
else:
# Axis to pass
if len(_symbols) > 1:
_axp = _ax[_symbols.index(_darwin)]
else:
_axp = _ax
# Call plotter function from self._graphics
self._graphics._mpl_plot_axis_(
plt,
_axp,
#_ax[_symbols.index(_darwin)],
self._df,
_darwin,
'Last 100 ticks',
'Quote',
'#00fa9a',
0.5,
100,
'#07335B',
{'fontname': 'Courier New'})
def __init__(self, **kwargs):
super(DWX_Info_API, self).__init__(**kwargs)
self._graphics = DWX_Graphics_Helpers()
class DWX_Info_API(DWX_API):
def __init__(self, **kwargs):
super(DWX_Info_API, self).__init__(**kwargs)
self._graphics = DWX_Graphics_Helpers()
#########################################################
# Function: Get Quotes for all DARWINs in list _symbols #
#########################################################
def _Get_Historical_Quotes_(self, _symbols=['THA.4.12','LVS.4.20'],
_start='', #_start=pd.to_datetime('today') - pd.to_timedelta(240, 'D'),
_end='', #_end=pd.to_datetime('today'),
_endpoint='/products/{}/history/quotes',
_plot_title='DWX_Info_API: def _Get_Quotes_() Example',
_plot=False,
_delay=0.01):
if isinstance(_symbols, list):
_dict = {}
_count = 1
for darwin in _symbols:
print('\r[DarwinInfoAPI] Getting Quotes for DARWIN {} / {}: ${}'.format(_count, len(_symbols), darwin),
end='', flush=True)
try:
# If dates provided, attach query parameters to endpoint
if _start not in ['', np.nan]:
_ep = _endpoint.format(darwin) + '?start={}&end={}'.format(
int(_start.timestamp())*1000,
int(_end.timestamp())*1000)
else:
_ep = _endpoint.format(darwin)
# Construct DataFrame from returned JSON
_dict[darwin] = pd.DataFrame(data=self._Call_API_(_ep, 'GET', ''))
# Assign column names
_dict[darwin].columns = ['timestamp', darwin]
# Convert ms timestamp to datetime
_dict[darwin].timestamp = pd.to_datetime(_dict[darwin].timestamp, unit='ms')
# Crop to data before today's date
_dict[darwin] = _dict[darwin][_dict[darwin].timestamp.dt.date < pd.to_datetime('today').date()]
# Set index to timestamp
_dict[darwin] = _dict[darwin].set_index('timestamp')
# Sleep
if _delay > 0:
time.sleep(_delay)
except Exception as ex:
print('[ERROR] Something went wrong while looking up ${}'.format(darwin))
_exstr = "Exception Type {0}. Args:\n{1!r}"
_msg = _exstr.format(type(ex).__name__, ex.args)
print(_msg)
continue
# Update counter
_count += 1
_retdf = pd.concat([_df for _df in _dict.values() if isinstance(_df, pd.DataFrame)], axis=1)
_retdf.columns = _symbols
if _plot:
self._graphics._plotly_dataframe_scatter_(_custom_filename='example_quotes.html',
_dir_prefix='MISC/',
_df=_retdf,
_x_title='EOD Timestamp',
_y_title='DARWIN Quote',
_main_title=_plot_title)
return _retdf
else:
print('[ERROR] Please specify symbols as Python list []')
#########################################################################
def _Get_Historical_Scores_(self,
_symbols=['THA.4.12','LVS.4.20'],
_endpoint='/products/{}/history/badges',
_plot_title='DWX_Info_API: def _Get_Historical_Scores_() Example',
_plot=False,
_delay=0.01):
if isinstance(_symbols, list):
_dict = {}
_count = 1
# Badges dict for later use
_badge_cols = ['eod_ts','Dp','Ex','Mc','Rs',
'Ra','Os','Cs','Rp','Rm',
'Dc','La','Pf','Cp','Ds',
'fcal_ts','lcal_ts']
for darwin in _symbols:
print('\r[DarwinInfoAPI] Getting Scores for DARWIN {} / {}: ${}'.format(_count, len(_symbols), darwin),
end='', flush=True)
try:
# Construct DataFrame from returned JSON
_dict[darwin] = self._Call_API_(_endpoint.format(darwin), 'GET', '')
_dict[darwin] = [_dict[darwin][i][:2] + [x for x in _dict[darwin][i][2]] + _dict[darwin][i][-2:] for i in range(len(_dict[darwin]))]
_dict[darwin] = pd.DataFrame(data=_dict[darwin], index=[_dict[darwin][i][0] for i in range(len(_dict[darwin]))])
# Assign column names
_dict[darwin].columns = _badge_cols
# Convert ms timestamp to datetime
_dict[darwin].eod_ts = pd.to_datetime(_dict[darwin].eod_ts, unit='ms')
_dict[darwin].fcal_ts = pd.to_datetime(_dict[darwin].fcal_ts, unit='ms')
_dict[darwin].lcal_ts = pd.to_datetime(_dict[darwin].lcal_ts, unit='ms')
# Set index to eod_ts
_dict[darwin] = _dict[darwin].set_index('eod_ts')
# Sleep
if _delay > 0:
time.sleep(_delay)
except Exception as ex:
print('[ERROR] Something went wrong while looking up ${}'.format(darwin))
_exstr = "Exception Type {0}. Args:\n{1!r}"
_msg = _exstr.format(type(ex).__name__, ex.args)
print(_msg)
continue
# Update counter
_count += 1
# If only one symbol provided, plot scores via Plotly
if len(_symbols) == 1:
if _plot:
self._graphics._plotly_dataframe_scatter_(_custom_filename='example_scores.html',
_dir_prefix='MISC/',
_df=_dict[_symbols[0]].drop(['fcal_ts','lcal_ts','Ds','Dp'],
axis=1).loc[:, ],
_x_title='EOD Timestamp',
_y_title='Score / Investment Attribute',
_main_title=_plot_title)
return _dict
else:
print('[ERROR] Please specify symbols as Python list []')
#########################################################################
def _Get_DARWIN_Universe_(self,
_status='ALL',
_endpoint='/products{}',
_query_string='?status={}&page={}&per_page={}',
_page=0,
_perPage=50,
_iterate=True,
_delay=0.01):
# Get first batch
try:
print('[DarwinInfoAPI] Getting first {} DARWINs..'.format(_perPage))
_darwins = self._Call_API_(_endpoint \
.format(_query_string \
.format(_status, _page, _perPage)),
_type='GET',
_data='')
except Exception as ex:
_exstr = "Exception Type {0}. Args:\n{1!r}"
_msg = _exstr.format(type(ex).__name__, ex.args)
print(_msg)
return None
if _iterate:
# Calculate number of pages
_pages = int(_darwins['totalPages'])
# Keep 'content' list of DARWINs, discard everything else
_darwins = _darwins['content']
print('[API] {} pages of {} DARWINs each found.. iterating, stand by! :muscle:\n'
.format(_pages, _perPage))
# Iterate
for i in range(_page + 1, _pages):
print('\r[DarwinInfoAPI] Getting page {} of {}'.format(i+1, _pages), end='', flush=True)
try:
_darwins = _darwins + self._Call_API_(_endpoint \
.format(_query_string \
.format(_status, i, _perPage)),
_type='GET',
_data='')['content']
# Sleep my child.. until next time..
if _delay > 0:
time.sleep(_delay)
except Exception as ex:
_exstr = "Exception Type {0}. Args:\n{1!r}"
_msg = _exstr.format(type(ex).__name__, ex.args)
print(_msg)
continue
# Return dict
return pd.DataFrame(_darwins)
#########################################################################
"""
Available filters (as of 2019-05-29):
return, drawdown, investors, trades, trader_equity, return_drawdown,
divergence, days_in_darwinex, current_investment_usd, var, d-score,
Ex, Mc, Rs, Ra, Os, Cs, R+, R-, Dc, La, Pf, Sc
For the complete specification, please see:
--
https://api.darwinex.com/store/apis/info?name=DarwinInfoAPI
&version=1.5&provider=admin#/default/filterProductsUsingPOST
Example Usage:
# Create API object
_info = DWX_Info_API()
# 1
_info._Get_Filtered_DARWINS_(_filters=[['drawdown',-10,0,'6m'],
['return',5,100,'1m']],
_order=['return','12m','DESC'],
_page=0,
_perPage=50)
# 2
_info._Get_Filtered_DARWINS_(_filters=[['d-score',50,100,'actual']],
_order=['return','12m','DESC'],
_page=0,
_perPage=50)
"""
def _Get_Filtered_DARWINS_(self,
_endpoint='/products',
_filters=[['drawdown',-10,0,'6m'],
['return',5,100,'1m']],
_order=['return','12m','DESC'],
_page=0,
_perPage=50,
_delay=0.01):
# Construct filter
_json = dict(filter=[dict(name=_filters[i][0],
options=[dict(max=_filters[i][2],
min=_filters[i][1],
period=_filters[i][3])]) \
for i in range(len(_filters))],
order=_order[2],
orderField=_order[0],
page=_page,
perPage=_perPage,
period=_order[1])
_rets = []
# Execute
while _json['page'] != -1:
print('\r[DarwinInfoAPI] Getting page {} of DARWINs that satisfy criteria..' \
.format(_json['page']), end='', flush=True)
try:
_ret = self._Call_API_(_endpoint,
_type='POST',
_data=str(_json).replace('\'', '"'))
if len(_ret) > 0:
# Update page number
_json['page'] += 1
# Add output to running list
_rets = _rets + _ret
# Sleep between calls
time.sleep(_delay)
else:
_json['page'] = -1 # done
except AssertionError:
print('[ERROR] name, period, min and max lists must be the same length.')
return None
# Return DataFrame
return pd.DataFrame(_rets)
#########################################################################
"""
Example Usage:
# from/to dates
_info._Get_DARWIN_OHLC_Candles_( _symbols=['KVL'], \
_from_dt=str(pd.Timestamp('now') \
- pd.tseries.offsets.BDay(7)), \
_to_dt=pd.Timestamp('now'), \
_resolution='1m')['KVL']
# timeframes
_info._Get_DARWIN_OHLC_Candles_( _symbols=['KVL'], \
_timeframe='/1D', \
_resolution='1m')['KVL']
"""
def _Get_DARWIN_OHLC_Candles_(self,
_symbols=['KVL'],
_resolution='1m', # 1m, 5m, 15m, 30m, 1h, 4h, 1d, 1w, 1mn
_from_dt='2019-05-31 12:00:00', # UTC
_to_dt=str(pd.Timestamp('now')),
_timeframe='/1D', # 1D, 1W, 1M, 3M, 6M, 1Y, 2Y, ALL
_endpoint='/products/{}/candles{}',
_delay=0.01):
# from/to endpoint given priority. change as necessary.
if _from_dt != '':
# Convert human-readable datetimes to millisecond EPOCHs
_from_epoch = int(pd.Timestamp(_from_dt).timestamp())
_to_epoch = int(pd.Timestamp(_to_dt).timestamp())
_query_string = f'?resolution={_resolution}&from={_from_epoch}\
&to={_to_epoch}'
elif _timeframe != '':
_query_string = f'{_timeframe}?resolution={_resolution}'
else:
print('[KERNEL] Inputs not recognized.. please try again.')
return None
_candles = {}
for _darwin in _symbols:
try:
print('[DarwinInfoAPI] Getting Candles for {}..'.format(_darwin))
_d = self._Call_API_(_endpoint \
.format(_darwin,
_query_string),
_type='GET',
_data='')
# Parse data into DataFrame
_candles[_darwin] = pd.DataFrame(
data=[_row['candle'] for _row in _d['candles']],
index=[_row['timestamp'] for _row in _d['candles']])
# Convert timestamp EPOCHs to human-readable datetimes
_candles[_darwin].index = pd.to_datetime(_candles[_darwin].index, unit='s')
except Exception as ex:
_exstr = "Exception Type {0}. Args:\n{1!r}"
_msg = _exstr.format(type(ex).__name__, ex.args)
print(_msg)
return None
return _candles
def __init__(self):
self._api = DWX_Info_API()
self._dataset = None
self._graphics = DWX_Graphics_Helpers()
class Test_Filter_Visualizations():
def __init__(self):
self._api = DWX_Info_API()
self._dataset = None
self._graphics = DWX_Graphics_Helpers()
##########################################################################
"""
Excerpt from https://blog.darwinex.com/identify-overfit-trading-strategies/:
Typically, three combinations of scores for the above attributes
demonstrate consistent performance between backtests and live trading
(when accompanied by High Ex, High Mc, High Rs scores):
1) Low Cp | High Os/Cs | High Pf | High R-
2) Moderate Cp | High La | High Pf
3) High Pf | Very High R+/R- or Dc | Moderate La
"""
def _run_(self,
_filters={
'Return/Drawdown Default':
[['drawdown',-10,0,'6m'],
['return',5,100,'1m']],
'Ds > 75': [['d-score',75,100,'actual']],
'Pf > 9.5': [['Pf',9.5,10,'actual']],
'R+ > 9.5': [['R+',9.5,10,'actual']],
'Os > 9.5': [['Os',9.5,10,'actual']],
'Cp<3 | Os>7 | Cs>7 | Pf>7 | R->7 | Ex>7 | Mc>7 | Rs>7':
[['Sc',0,2.99,'actual'],
['Os',7.01,10,'actual'],
['Cs',7.01,10,'actual'],
['Pf',7.01,10,'actual'],
['R-',7.01,10,'actual'],
['Ex',7.01,10,'actual'],
['Rs',7.01,10,'actual'],
['Mc',7.01,10,'actual']],
'4<Cp<6 | La>9 | Pf>9 | Ex>7 | Mc>7 | Rs>7':
[['Sc',4.01,5.99,'actual'],
['La',9.01,10,'actual'],
['Pf',9.01,10,'actual'],
['Ex',7.01,10,'actual'],
['Rs',7.01,10,'actual'],
['Mc',7.01,10,'actual']]
}):
if self._dataset is None:
# 1) Get filtered DARWINs
_filtered_darwins = {_name: self._api._Get_Filtered_DARWINS_(
_filters=_filter) \
.productName \
.unique() \
.tolist()
for _name, _filter in _filters.items()}
# 2) Get DARWIN Quotes for all filtered DARWINs, regardless of filter.
_darwins = []
for _filter in _filtered_darwins.values():
_darwins = _darwins + _filter
_quotes = self._api._Get_Historical_Quotes_( \
_symbols= list(set(_darwins)))
# 3) Create dict of go.Scatter lists (for each filter)
_scatter_dict = {_name: self._graphics._generate_scatter_list_(
_quotes[_filtered_darwins[_name]])
for _name in _filtered_darwins.keys()}
# 4) Create stacked list of Scatter objects for updatemenus later.
# We'll use this to enable disable traces on a Plotly chart.
_scatter_stack = []
for _name, _scatter_list in _scatter_dict.items():
print(f'[INFO] Filter "{_name}" has {len(_scatter_list)} DARWINs')
_scatter_stack = _scatter_stack + _scatter_list
# 5) Create default (disabled) updatemenus toggle list
# _updatemenus_toggle = [False for i in range(len(_scatter_stack))]
##################################################################
# Return list for Plotly toggle
def _enable_filter_(_name, _stack_length):
# Get index of _name in _scatter_dict
_index = list(_scatter_dict.keys()).index(_name)
# Get size of scatter list for this _name
_size = len(_scatter_dict[_name])
# Create toggle list
_ret = []
if _index == 0:
_ret = [True for i in range(0, _size, 1)] + \
[False for i in range(_size, _stack_length, 1)]
else:
# Toggle everything before _index as False
for _i in range(0, _index, 1):
_ret = _ret + [False for _x in range(
len(_scatter_dict[
list(_scatter_dict.keys())[_i]]))]
# Now toggle True for this _index
_ret = _ret + [True for _i in range(len(_scatter_dict[_name]))]
# Now toggle False for everything after this _index
_ret = _ret + [False for _i in range(len(_ret), _stack_length, 1)]
return _ret
##################################################################
# Test
# return _enable_filter_(list(_scatter_dict.keys())[1], len(_scatter_stack))
# 6) Create updatemenus object for Plotly
_updatemenus = list([
dict(#type="buttons",
active=-1,
buttons=list([
dict(label = _name,
method = 'update', # update
args = [{'visible': _enable_filter_(_name, len(_scatter_stack))},
{'title': _name}])
for _name in _scatter_dict.keys()]),
)
])
# Save data in memory
self._dataset = _scatter_stack
# Plot
self._graphics._plotly_multi_scatter_(_data=self._dataset,
_title='DARWIN Filter Visualizer v1.0',
_updatemenus=_updatemenus,
_dir_prefix='MISC/')
from MINIONS.dwx_graphics_helpers import DWX_Graphics_Helpers
from plotly.offline import init_notebook_mode
from scipy.stats import zscore
import pickle, warnings
import pandas as pd
import numpy as np
################################
# Some configuration for later #
################################
warnings.simplefilter("ignore") # Suppress warnings
init_notebook_mode(connected=True)
################################
# Create DWX Graphics Helpers object for later
_graphics = DWX_Graphics_Helpers()
# Load DataFrame of DARWIN quotes (Daily precision) from pickle archive.
quotes = pickle.load(
open('../DATA/jn_all_quotes_active_deleted_12062019.pkl', 'rb'))
# Remove non-business days (consider Monday to Friday only)
quotes = quotes[quotes.index.dayofweek < 5]
# Load FX Market Volatility data upto 2019-06-17 (for evaluation later)
voldf = pd.read_csv('../DATA/volatility.beginning.to.2019-06-17.csv',
index_col=0,
parse_dates=True,
infer_datetime_format=True)
# Print DARWINs in dataset