def get_correlation(data: dict, sectors: list, **kwargs) -> dict:
"""Get Correlation
Arguments:
data {dict} -- downloaded data
sectors {list} -- sector list
Optional Arguments:
plot_output {bool} -- (default: {True})
clock {uint64_t} -- time for prog_bar (default: {None})
Returns:
dict -- object with correlations
"""
plot_output = kwargs.get('plot_output', True)
clock = kwargs.get('clock')
PERIOD_LENGTH = [100, 50, 25]
corr_data = dict()
if '^GSPC' in data.keys():
tot_len = len(data['^GSPC']['Close'])
start_pt = max(PERIOD_LENGTH)
tot_count = (tot_len - start_pt) * 11 * len(PERIOD_LENGTH)
if tot_count < 25000:
divisor = 50
else:
divisor = 250
pbar_count = np.ceil(tot_count / float(divisor))
progress_bar = ProgressBar(pbar_count,
name="Correlation Composite Index",
offset=clock)
progress_bar.start()
corrs = {}
dates = data['^GSPC'].index[start_pt:tot_len]
net_correlation = []
legend = []
counter = 0
for period in PERIOD_LENGTH:
nc = [0.0] * (tot_len - start_pt)
for sector in sectors:
corrs[sector] = []
for i in range(start_pt, tot_len):
_, rsqd = beta_comparison_list(
data[sector]['Close'][i - period:i],
data['^GSPC']['Close'][i - period:i])
corrs[sector].append(rsqd)
nc[i - start_pt] += rsqd
counter += 1
if counter == divisor:
progress_bar.uptick()
counter = 0
net_correlation.append(nc.copy())
legend.append('Corr-' + str(period))
norm_corr = []
for nc_period in net_correlation:
max_ = np.max(nc_period)
norm = [x / max_ for x in nc_period]
norm_corr.append(norm)
net_correlation = norm_corr.copy()
dual_plotting(net_correlation,
data['^GSPC']['Close'][start_pt:tot_len],
x=dates,
y1_label=legend,
y2_label='S&P500',
title='CCI Net Correlation',
saveFig=(not plot_output),
filename='CCI_net_correlation.png')
str_dates = []
for date in dates:
str_dates.append(date.strftime("%Y-%m-%d"))
corr_data['tabular'] = {}
for i, nc_period in enumerate(net_correlation):
corr_data[legend[i]] = {}
corr_data[legend[i]]['data'] = nc_period.copy()
corr_data[legend[i]]['date'] = str_dates.copy()
corr_data['tabular'][legend[i]] = nc_period.copy()
corr_data['tabular']['date'] = str_dates.copy()
progress_bar.end()
return corr_data
def run_dev(script: list):
"""Run Development Script
Script that is for implementing new content
Arguments:
script {list} -- dataset, funds, periods, config
Returns:
dict -- analysis object of fund data
"""
dataset = script[0]
funds = script[1]
periods = script[2]
config = script[3]
# Start of automated process
analysis = {}
clock = start_clock()
for fund_name in funds:
if fund_name in SKIP_INDEXES:
continue
fund_print = INDEXES.get(fund_name, fund_name)
print("")
print(f"~~{fund_print}~~")
create_sub_temp_dir(fund_name, sub_periods=config['period'])
analysis[fund_name] = {}
analysis[fund_name]['metadata'] = get_api_metadata(
fund_name,
max_close=max(dataset[periods[0]][fund_name]['Close']),
data=dataset[periods[0]][fund_name])
###################### START OF PERIOD LOOPING #############################
for i, period in enumerate(periods):
fund_data = {}
fund = dataset[period][fund_name]
start = date_extractor(fund.index[0], _format='str')
end = date_extractor(fund.index[-1], _format='str')
fund_data['dates_covered'] = {'start': str(start), 'end': str(end)}
fund_data['name'] = fund_name
fund_print2 = fund_print + f" ({period}) "
p = ProgressBar(config['process_steps'],
name=fund_print2,
offset=clock)
p.start()
fund_data['statistics'] = get_high_level_stats(fund)
fund_data['clustered_osc'] = cluster_oscs(fund,
function='all',
filter_thresh=3,
name=fund_name,
plot_output=False,
progress_bar=p,
view=period)
fund_data['full_stochastic'] = full_stochastic(fund,
name=fund_name,
plot_output=False,
out_suppress=False,
progress_bar=p,
view=period)
fund_data['rsi'] = RSI(fund,
name=fund_name,
plot_output=False,
out_suppress=False,
progress_bar=p,
view=period)
fund_data['ultimate'] = ultimate_oscillator(fund,
name=fund_name,
plot_output=False,
out_suppress=False,
progress_bar=p,
view=period)
fund_data['awesome'] = awesome_oscillator(fund,
name=fund_name,
plot_output=False,
progress_bar=p,
view=period)
fund_data['momentum_oscillator'] = momentum_oscillator(
fund,
name=fund_name,
plot_output=False,
progress_bar=p,
view=period)
fund_data['on_balance_volume'] = on_balance_volume(
fund,
plot_output=False,
name=fund_name,
progress_bar=p,
view=period)
fund_data['simple_moving_average'] = triple_moving_average(
fund,
plot_output=False,
name=fund_name,
progress_bar=p,
view=period)
fund_data['exp_moving_average'] = triple_exp_mov_average(
fund,
plot_output=False,
name=fund_name,
progress_bar=p,
view=period)
fund_data['sma_swing_trade'] = moving_average_swing_trade(
fund,
plot_output=False,
name=fund_name,
progress_bar=p,
view=period)
fund_data['ema_swing_trade'] = moving_average_swing_trade(
fund,
function='ema',
plot_output=False,
name=fund_name,
progress_bar=p,
view=period)
fund_data['hull_moving_average'] = hull_moving_average(
fund,
plot_output=False,
name=fund_name,
progress_bar=p,
view=period)
fund_data['macd'] = mov_avg_convergence_divergence(
fund,
plot_output=False,
name=fund_name,
progress_bar=p,
view=period)
fund_data['bear_bull_power'] = bear_bull_power(fund,
plot_output=False,
name=fund_name,
progress_bar=p,
view=period)
fund_data['total_power'] = total_power(fund,
plot_output=False,
name=fund_name,
progress_bar=p,
view=period)
fund_data['bollinger_bands'] = bollinger_bands(fund,
plot_output=False,
name=fund_name,
progress_bar=p,
view=period)
fund_data['commodity_channels'] = commodity_channel_index(
fund,
plot_output=False,
name=fund_name,
progress_bar=p,
view=period)
fund_data['rate_of_change'] = rate_of_change_oscillator(
fund,
plot_output=False,
name=fund_name,
progress_bar=p,
view=period)
fund_data['know_sure_thing'] = know_sure_thing(fund,
plot_output=False,
name=fund_name,
progress_bar=p,
view=period)
fund_data['average_true_range'] = average_true_range(
fund,
plot_output=False,
name=fund_name,
progress_bar=p,
view=period)
fund_data['adx'] = average_directional_index(
fund,
atr=fund_data['average_true_range']['tabular'],
plot_output=False,
name=fund_name,
progress_bar=p,
view=period)
fund_data['parabolic_sar'] = parabolic_sar(
fund,
adx_tabular=fund_data['adx']['tabular'],
plot_output=False,
name=fund_name,
progress_bar=p,
view=period)
fund_data['demand_index'] = demand_index(fund,
plot_output=False,
name=fund_name,
progress_bar=p,
view=period)
if 'no_index' not in config['state']:
strength, match_data = relative_strength(
fund_name,
full_data_dict=dataset[period],
config=config,
plot_output=False,
meta=analysis[fund_name]['metadata'],
progress_bar=p,
period=period,
interval=config['interval'][i],
view=period)
fund_data['relative_strength'] = strength
fund_data['statistics']['risk_ratios'] = risk_comparison(
fund,
dataset[period]['^GSPC'],
dataset[period]['^IRX'],
sector_data=match_data)
p.uptick()
# Support and Resistance Analysis
fund_data['support_resistance'] = find_resistance_support_lines(
fund,
name=fund_name,
plot_output=False,
progress_bar=p,
view=period)
# Feature Detection Block
fund_data['features'] = {}
fund_data['features'][
'head_shoulders'] = feature_detection_head_and_shoulders(
fund,
name=fund_name,
plot_output=False,
progress_bar=p,
view=period)
fund_data['candlesticks'] = candlesticks(fund,
name=fund_name,
plot_output=False,
view=period,
progress_bar=p)
fund_data['price_gaps'] = analyze_price_gaps(fund,
name=fund_name,
plot_output=False,
progress_bar=p,
view=period)
# Get Trendlines
fund_data['trendlines'] = get_trendlines(
fund,
name=fund_name,
plot_output=False,
progress_bar=p,
view=period,
meta=analysis[fund_name]['metadata'])
# Various Fund-specific Metrics
fund_data['futures'] = future_returns(fund, progress_bar=p)
# Parse through indicators and pull out latest signals (must be last)
fund_data['last_signals'] = assemble_last_signals(
fund_data,
fund=fund,
name=fund_name,
view=period,
progress_bar=p,
plot_output=False)
p.end()
analysis[fund_name][period] = fund_data
analysis[fund_name]['synopsis'] = generate_synopsis(analysis,
name=fund_name)
return analysis, clock
def composite_index(data: dict,
sectors: list,
index_dict: dict,
plot_output=True,
bond_type='Treasury',
index_type='BND',
**kwargs):
"""Composite Index
Arguments:
data {dict} -- Data of composites
sectors {list} -- sector list
index_dict {dict} -- data pulled from sectors.json file (can be None)
Keyword Arguments:
plot_output {bool} -- (default: {True})
bond_type {str} -- (default: {'Treasury'})
index_type {str} -- (default: {'BND'})
Optional Args:
clock {uint64_t} -- timekeeping for prog_bar (default: {None})
Returns:
list -- composite signal, plots, dates
"""
clock = kwargs.get('clock')
progress = len(sectors) + 2
p = ProgressBar(
progress, name=f'{bond_type} Bond Composite Index', offset=clock)
p.start()
composite = []
for tick in sectors:
if tick != index_type:
cluster = cluster_oscs(
data[tick], plot_output=False, function='market', wma=False)
graph = cluster['tabular']
p.uptick()
composite.append(graph)
composite2 = []
for i in range(len(composite[0])):
s = 0.0
for j in range(len(composite)):
s += float(composite[j][i])
composite2.append(s)
p.uptick()
composite2 = windowed_moving_avg(composite2, 3, data_type='list')
max_ = np.max(np.abs(composite2))
composite2 = [x / max_ for x in composite2]
key = list(data.keys())[0]
data_to_plot = bond_type_index_generator(
data, index_dict, bond_type=bond_type)
dates = dates_extractor_list(data[key])
if plot_output:
dual_plotting(data_to_plot, composite2, y1_label=index_type,
y2_label='BCI', title=f'{bond_type} Bond Composite Index')
else:
dual_plotting(data_to_plot, composite2,
y1_label=index_type, y2_label='BCI',
title=f'{bond_type} Bond Composite Index', x=dates,
saveFig=True, filename=f'{bond_type}_BCI.png')
p.uptick()
return composite2, data_to_plot, dates
def market_composite_index(**kwargs) -> dict:
"""Market Composite Index (MCI)
Optional Args:
config {dict} -- controlling config dictionary (default: {None})
plot_output {bool} -- True to render plot in realtime (default: {True})
period {str / list} -- time period for data (e.g. '2y') (default: {None})
clock {uint64_t} -- time for prog_bar (default: {None})
data {pd.DataFrame} -- dataset with sector funds (default: {None})
returns:
list -- dict contains all mci information, dict fund content, sector list
"""
config = kwargs.get('config')
period = kwargs.get('period')
clock = kwargs.get('clock')
plot_output = kwargs.get('plot_output', True)
data = kwargs.get('data')
sectors = kwargs.get('sectors')
if config is not None:
period = config['period']
properties = config['properties']
elif period is None:
print(
f"{ERROR_COLOR}ERROR: config and period both provided {period} " +
f"for market_composite_index{NORMAL_COLOR}")
return {}
else:
# Support for release 1 versions
period = period
properties = dict()
properties['Indexes'] = {}
properties['Indexes']['Market Sector'] = True
# Validate each index key is set to True in the --core file
if properties is not None:
if 'Indexes' in properties.keys():
props = properties['Indexes']
if 'Market Sector' in props.keys():
if props['Market Sector'] == True:
mci = dict()
if data is None or sectors is None:
data, sectors = metrics_initializer(period=period)
if data:
p = ProgressBar(len(sectors) * 2 + 5,
name='Market Composite Index',
offset=clock)
p.start()
composite = composite_index(data,
sectors,
plot_output=plot_output,
progress_bar=p)
correlations = composite_correlation(
data,
sectors,
plot_output=plot_output,
progress_bar=p)
mci['tabular'] = {'mci': composite}
mci['correlations'] = correlations
p.end()
return mci, data, sectors
return {}, None, None
def type_composite_index(**kwargs) -> list:
"""Type Composite Index (MCI)
Similar to MCI, TCI compares broader market types (sensitive, cyclical, and defensive)
Optional Args:
config {dict} -- controlling config dictionary (default: {None})
plot_output {bool} -- True to render plot in realtime (default: {True})
period {str / list} -- time period for data (e.g. '2y') (default: {None})
clock {float} -- time for prog_bar (default: {None})
data {pd.DataFrame} -- fund datasets (default: {None})
sectors {list} -- list of sectors (default: {None})
returns:
list -- dict contains all tci information, data, sectors
"""
config = kwargs.get('config')
period = kwargs.get('period')
plot_output = kwargs.get('plot_output', True)
clock = kwargs.get('clock')
data = kwargs.get('data')
sectors = kwargs.get('sectors')
if config is not None:
period = config['period']
properties = config['properties']
elif period is None:
print(
f"{ERROR_COLOR}ERROR: config and period both provided {period} " +
f"for type_composite_index{NORMAL}")
return {}
else:
# Support for release 1 versions
period = period
properties = dict()
properties['Indexes'] = {}
properties['Indexes']['Type Sector'] = True
# Validate each index key is set to True in the --core file
if properties is not None:
if 'Indexes' in properties.keys():
props = properties['Indexes']
if 'Type Sector' in props.keys():
if props['Type Sector'] == True:
m_data = get_metrics_content()
if data is None or sectors is None:
data, sectors = metrics_initializer(m_data,
period='2y')
if data:
p = ProgressBar(19,
name='Type Composite Index',
offset=clock)
p.start()
tci = dict()
composite = {}
for sect in sectors:
cluster = cluster_oscs(data[sect],
plot_output=False,
function='market',
wma=False,
progress_bar=p)
graph = cluster['tabular']
composite[sect] = graph
defensive = type_composites(composite,
m_data,
type_type='Defensive')
p.uptick()
sensitive = type_composites(composite,
m_data,
type_type='Sensitive')
p.uptick()
cyclical = type_composites(composite,
m_data,
type_type='Cyclical')
p.uptick()
d_val = weighted_signals(data,
m_data,
type_type='Defensive')
p.uptick()
s_val = weighted_signals(data,
m_data,
type_type='Sensitive')
p.uptick()
c_val = weighted_signals(data,
m_data,
type_type='Cyclical')
p.uptick()
d_val = windowed_moving_avg(d_val, 3, data_type='list')
c_val = windowed_moving_avg(c_val, 3, data_type='list')
s_val = windowed_moving_avg(s_val, 3, data_type='list')
p.uptick()
tci['defensive'] = {
"tabular": d_val,
"clusters": defensive
}
tci['sensitive'] = {
"tabular": s_val,
"clusters": sensitive
}
tci['cyclical'] = {
"tabular": c_val,
"clusters": cyclical
}
dates = data['VGT'].index
if plot_output:
dual_plotting(y1=d_val,
y2=defensive,
y1_label='Defensive Index',
y2_label='Clustered Osc',
title='Defensive Index',
x=dates)
dual_plotting(y1=s_val,
y2=sensitive,
y1_label='Sensitive Index',
y2_label='Clustered Osc',
title='Sensitive Index',
x=dates)
dual_plotting(y1=c_val,
y2=cyclical,
y1_label='Cyclical Index',
y2_label='Clustered Osc',
title='Cyclical Index',
x=dates)
generic_plotting(
[d_val, s_val, c_val],
legend=['Defensive', 'Sensitive', 'Cyclical'],
title='Type Indexes',
x=dates)
else:
generic_plotting(
[d_val, s_val, c_val],
legend=['Defensive', 'Sensitive', 'Cyclical'],
title='Type Indexes',
x=dates,
saveFig=True,
ylabel='Normalized "Price"',
filename='tci.png')
p.end()
return tci, data, sectors
return {}, None, None
def get_correlation(data: dict, sectors: list, **kwargs) -> dict:
"""Get Correlation
Arguments:
data {dict} -- downloaded data
sectors {list} -- sector list
Optional Arguments:
plot_output {bool} -- (default: {True})
clock {uint64_t} -- time for prog_bar (default: {None})
Returns:
dict -- object with correlations
"""
plot_output = kwargs.get('plot_output', True)
clock = kwargs.get('clock')
PERIOD_LENGTH = [100, 50, 25]
WEIGHTS = [1.5, 1.25, 1.0]
corr_data = dict()
if '^GSPC' in data.keys():
tot_len = len(data['^GSPC']['Close'])
start_pt = max(PERIOD_LENGTH)
tot_count = (tot_len - start_pt) * 11 * len(PERIOD_LENGTH)
if tot_count < 25000:
divisor = 50
else:
divisor = 250
pbar_count = np.ceil(tot_count / float(divisor))
progress_bar = ProgressBar(
pbar_count, name="Correlation Composite Index", offset=clock)
progress_bar.start()
corrs = {}
dates = data['^GSPC'].index[start_pt:tot_len]
net_correlation = []
legend = []
counter = 0
for period in PERIOD_LENGTH:
nc = [0.0] * (tot_len-start_pt)
for sector in sectors:
corrs[sector] = []
for i in range(start_pt, tot_len):
_, rsqd = beta_comparison_list(
data[sector]['Close'][i-period:i], data['^GSPC']['Close'][i-period:i])
corrs[sector].append(rsqd)
nc[i-start_pt] += rsqd
counter += 1
if counter == divisor:
progress_bar.uptick()
counter = 0
net_correlation.append(nc.copy())
legend.append('Corr-' + str(period))
norm_corr = []
for nc_period in net_correlation:
max_ = np.max(nc_period)
norm = [x / max_ for x in nc_period]
norm_corr.append(norm)
net_correlation = norm_corr.copy()
dual_plotting(net_correlation,
data['^GSPC']['Close'][start_pt:tot_len],
x=dates,
y1_label=legend,
y2_label='S&P500',
title='CCI Net Correlation',
saveFig=(not plot_output),
filename='CCI_net_correlation.png')
str_dates = []
for date in dates:
str_dates.append(date.strftime("%Y-%m-%d"))
corr_data['tabular'] = {}
for i, nc_period in enumerate(net_correlation):
corr_data[legend[i]] = {}
corr_data[legend[i]]['data'] = nc_period.copy()
corr_data[legend[i]]['date'] = str_dates.copy()
corr_data['tabular'][legend[i]] = nc_period.copy()
corr_data['tabular']['date'] = str_dates.copy()
overall_signal = [0.0] * len(corr_data['tabular']['date'])
for i, period in enumerate(PERIOD_LENGTH):
string = f"Corr-{period}"
corr_signal = corr_data['tabular'].get(string, [])
if len(corr_signal) > 0:
weight = WEIGHTS[i]
for j, csl in enumerate(corr_signal):
overall_signal[j] += csl * weight
signal_line = simple_moving_avg(overall_signal, 20, data_type='list')
dual_plotting(data['^GSPC']['Close'][start_pt:tot_len],
[overall_signal, signal_line],
x=dates,
y1_label='S&P500',
y2_label=['Overall Signal', '20d-SMA'],
title='Overall Correlation Signal',
saveFig=(not plot_output),
filename='CCI_overall_correlation.png')
diff_signal = [x - signal_line[i]
for i, x in enumerate(overall_signal)]
dual_plotting(data['^GSPC']['Close'][start_pt:tot_len],
diff_signal,
x=dates,
y1_label='S&P500',
y2_label='Corr - Signal Line',
title='Diff Correlation Signal',
saveFig=(not plot_output),
filename='CCI_diff_correlation.png')
corr_data['tabular']['overall'] = overall_signal
corr_data['tabular']['signal_line'] = signal_line
corr_data['tabular']['diff_signal'] = diff_signal
progress_bar.end()
return corr_data