def itch_data_plot_generator(tickers, dates):
"""Generates all the analysis and plots from the ITCH data.
:param tickers: list of the string abbreviation of the stocks to be
analized (i.e. ['AAPL', 'MSFT']).
:param dates: list of strings with the date of the data to be extracted
(i.e. ['2008-01-02', '2008-01-03]).
:return: None -- The function saves the data in a file and does not return
a value.
"""
# Parallel computing
with mp.Pool(processes=mp.cpu_count()) as pool:
# Basic functions
pool.starmap(
itch_data_analysis_data_extraction.itch_midpoint_second_data,
iprod(tickers, dates))
pool.starmap(
itch_data_analysis_data_extraction.itch_trade_signs_second_data,
iprod(tickers, dates))
# Plot
pool.starmap(itch_data_plot_data_extraction.itch_midpoint_second_plot,
iprod(tickers, [dates]))
return None
def __init__(self, num_vars, num_neighbors, rule, mode="RULENUMBER", dim=1):
if dim <= 0:
raise ValueError("dim must be strictly positive")
elif dim == 1 and not isinstance(num_vars, list):
num_vars = [num_vars]
if not isinstance(num_vars, list) or len(num_vars) != dim:
raise ValueError("num_vars should be list with %d elements" % dims)
all_vars = np.array(list(iprod(*[list(range(d)) for d in num_vars])))
total_num_vars = len(all_vars)
all_var_ndxs = {tuple(v): ndx for ndx, v in enumerate(all_vars)}
neighbor_offsets = np.array(list(iprod(*[list(range(-num_neighbors, num_neighbors + 1)) for d in num_vars])))
if mode == "FUNC":
updaterule = self._updatefunc_to_truthtables(len(neighbor_offsets), rule)
elif mode == "RULENUMBER":
all_neighbor_cnt = (2 * num_neighbors) ** dim
updaterule = list(int2tuple(rule, 2 ** (all_neighbor_cnt + 1)))
elif mode == "TRUTHTABLE":
updaterule = rule
else:
raise ValueError("Unknown mode %s" % mode)
rules = []
for v in all_vars:
conns = []
coffsets = v + neighbor_offsets
conn_address = zip(*[(coffsets[:, d] % num_vars[d]) for d in range(dim)])
conns = [all_var_ndxs[v] for v in conn_address]
rules.append([tuple(v), conns, updaterule])
super(CellularAutomaton, self).__init__(rules=rules, mode="TRUTHTABLES")
def taq_daily_data_extract(tickers, year):
""" Extracts data to daily CSV files.
Extract and filter the data for every day of a year in HDF5 files.
:param tickers: list of the string abbreviation of the stocks to be
analyzed (i.e. ['AAPL', 'MSFT']).
:param year: string of the year to be analyzed (i.e '2016').
:return: None -- The function saves the data in a file and does not return
a value.
"""
# Extract daily data
print('Extracting daily data')
# Parallel computing
with mp.Pool(processes=mp.cpu_count()) as pool:
pool.starmap(taq_data_extract, iprod(tickers, ['quotes'], [year]))
# Parallel computing
with mp.Pool(processes=mp.cpu_count()) as pool:
pool.starmap(taq_data_extract, iprod(tickers, ['trades'], [year]))
# Delete CSV folder
# Obtain the absolute path of the current file and split it
abs_path = os.path.abspath(__file__).split('/')
# Take the path from the start to the project folder
root_path = '/'.join(abs_path[:abs_path.index('project') + 1])
f_path = root_path + f'/taq_data/csv_year_data_{year}/'
# Remove CSV folder
subprocess.call(f'rm -r {f_path}', shell=True)
return None
def pointGroup(self):
from itertools import product as iprod
import numpy.linalg as LA
# Find the symmetry operators and centers where the basis is unchanged
# Use the least common siesta atom type to find possible centers
abundance = [N.sum(N.array(self.basis.snr)==snr) for snr in self.basis.snr]
whichsnr = self.basis.snr[N.where(N.array(abundance)==N.min(abundance))[0][0]]
Ulist,a1,a2,a3 = self.pointU33, self.a1, self.a2, self.a3
pointU, pointO = [], []
for iU, U in enumerate(Ulist):
SIO.printDone(iU, len(Ulist), 'Looking for point group')
xyz= self.basis.xyz[N.where(self.basis.snr==whichsnr)[0]]
# Find centers by solving U(x_i-o)==x_j-o +R where o is center, R is lattice vector
centers = []
A = U-N.eye(3)
for i1, i2, i3 in iprod(range(-1, 2),repeat=3):
for ii,jj in iprod(range(len(xyz)),repeat=2):
sol = LA.lstsq(A,mm(U,xyz[ii,:].transpose())-xyz[jj,:].transpose()+(a1*i1+a2*i2+a3*i3))[0]
correct = not N.any(N.abs(mm(A,sol)-mm(U, xyz[ii,:].transpose())+xyz[jj,:].transpose()-(a1*i1+a2*i2+a3*i3))>1e-6)
if correct: centers += [sol]
centers = myUnique2(moveIntoCell(N.array(centers),a1,a2,a3,self.accuracy),self.accuracy)
# All centers are not correct since we only looked at one atom type and
# remove repeats of the same symmetry by looking at ipvi, i.e., which atom moves onto which
origin, ipiv = [], []
for o in centers:
xyz, nxyz = self.basis.xyz, mm(U,self.basis.xyz.transpose()-o.reshape((3,1))).transpose()+o
xyz, nxyz = moveIntoCell(xyz,a1,a2,a3,self.accuracy), moveIntoCell(nxyz,a1,a2,a3,self.accuracy)
thisipiv = []
for ix,x in enumerate(xyz):
for iy,y in enumerate(nxyz):
if N.allclose(x,y,atol=self.accuracy):
thisipiv+=[iy]
trueSym = len(thisipiv)==self.basis.NN and N.allclose(N.array(self.basis.snr)[thisipiv],self.basis.snr) and N.allclose(N.sort(thisipiv),N.arange(self.basis.NN))
if trueSym and len(myIntersect(N.array(ipiv),N.array([thisipiv]),self.accuracy))==0:
origin, ipiv = origin+[o], ipiv+[thisipiv]
if len(origin)>0:
pointU+=[U]
pointO+=[origin]
self.U33, self.origo = pointU, pointO
print("Symmetry: %i point symmetry operations (with rotation centers) found for lattice+basis."%len(pointU))
# Calculate rank of operations
self.rankU33, sign = [], []
for U in self.U33:
tmp, ii = U, 1
while ii<7 and not N.allclose(N.eye(3),tmp,atol=self.accuracy):
tmp, ii = mm(tmp,U), ii+1
if ii > 6: sys.exit('Symmetry error: rank >6 !!')
self.rankU33 += [ii]
sign += [N.linalg.det(U)]
print "Symmetry: rank*det"
print N.array(self.rankU33)*sign
return
def pointGroup(self):
from itertools import product as iprod
import numpy.linalg as LA
# Find the symmetry operators and centers where the basis is unchanged
# Use the least common siesta atom type to find possible centers
abundance = [N.sum(N.array(self.basis.snr) == snr) for snr in self.basis.snr]
whichsnr = self.basis.snr[N.where(N.array(abundance) == N.min(abundance))[0][0]]
Ulist, a1, a2, a3 = self.pointU33, self.a1, self.a2, self.a3
pointU, pointO = [], []
for iU, U in enumerate(Ulist):
SIO.printDone(iU, len(Ulist), 'Looking for point group')
xyz = self.basis.xyz[N.where(self.basis.snr == whichsnr)[0]]
# Find centers by solving U(x_i-o)==x_j-o +R where o is center, R is lattice vector
centers = []
A = U-N.eye(3)
for i1, i2, i3 in iprod(range(-1, 2), repeat=3):
for ii, jj in iprod(range(len(xyz)), repeat=2):
sol = LA.lstsq(A, mm(U, xyz[ii, :].transpose())-xyz[jj, :].transpose()+(a1*i1+a2*i2+a3*i3))[0]
correct = not N.any(N.abs(mm(A, sol)-mm(U, xyz[ii, :].transpose())+xyz[jj, :].transpose()-(a1*i1+a2*i2+a3*i3)) > 1e-6)
if correct: centers += [sol]
centers = myUnique2(moveIntoCell(N.array(centers), a1, a2, a3, self.accuracy), self.accuracy)
# All centers are not correct since we only looked at one atom type and
# remove repeats of the same symmetry by looking at ipvi, i.e., which atom moves onto which
origin, ipiv = [], []
for o in centers:
xyz, nxyz = self.basis.xyz, mm(U, self.basis.xyz.transpose()-o.reshape((3, 1))).transpose()+o
xyz, nxyz = moveIntoCell(xyz, a1, a2, a3, self.accuracy), moveIntoCell(nxyz, a1, a2, a3, self.accuracy)
thisipiv = []
for x in xyz:
for iy, y in enumerate(nxyz):
if N.allclose(x, y, atol=self.accuracy):
thisipiv += [iy]
trueSym = len(thisipiv) == self.basis.NN and N.allclose(N.array(self.basis.snr)[thisipiv], self.basis.snr) and N.allclose(N.sort(thisipiv), N.arange(self.basis.NN))
if trueSym and len(myIntersect(N.array(ipiv), N.array([thisipiv]), self.accuracy)) == 0:
origin, ipiv = origin+[o], ipiv+[thisipiv]
if len(origin) > 0:
pointU += [U]
pointO += [origin]
self.U33, self.origo = pointU, pointO
print("Symmetry: %i point symmetry operations (with rotation centers) found for lattice+basis."%len(pointU))
# Calculate rank of operations
self.rankU33, sign = [], []
for U in self.U33:
tmp, ii = U, 1
while ii < 7 and not N.allclose(N.eye(3), tmp, atol=self.accuracy):
tmp, ii = mm(tmp, U), ii+1
if ii > 6: sys.exit('Symmetry error: rank >6 !!')
self.rankU33 += [ii]
sign += [N.linalg.det(U)]
print "Symmetry: rank*det"
print N.array(self.rankU33)*sign
return
def taq_data_plot_generator(tickers, year, shifts):
"""Generates all the analysis and plots from the TAQ data.
:param tickers: list of the string abbreviation of the stocks to be
analyzed (i.e. ['AAPL', 'MSFT']).
:param year: string of the year to be analyzed (i.e '2016').
:param shifts: list of integers greater than zero (i.e. [1, 10, 50]).
:return: None -- The function saves the data in a file and does not return
a value.
"""
date_list = taq_data_tools_responses_physical_shift \
.taq_bussiness_days(year)
# Especific functions
# Self-response
for ticker in tickers:
for shift in shifts:
taq_data_analysis_responses_physical_shift \
.taq_self_response_year_responses_physical_shift_data(ticker,
year,
shift)
ticker_prod = iprod(tickers, tickers)
# ticker_prod = [('AAPL', 'MSFT'), ('MSFT', 'AAPL'),
# ('GS', 'JPM'), ('JPM', 'GS'),
# ('CVX', 'XOM'), ('XOM', 'CVX'),
# ('GOOG', 'MA'), ('MA', 'GOOG'),
# ('CME', 'GS'), ('GS', 'CME'),
# ('RIG', 'APA'), ('APA', 'RIG')]
# Cross-response
for ticks in ticker_prod:
for shift in shifts:
taq_data_analysis_responses_physical_shift \
.taq_cross_response_year_responses_physical_shift_data(
ticks[0], ticks[1], year, shift)
# Parallel computing
with mp.Pool(processes=mp.cpu_count()) as pool:
# Plot
pool.starmap(
taq_data_plot_responses_physical_shift.
taq_self_response_year_avg_responses_physical_shift_plot,
iprod(tickers, [year], [shifts]))
# Parallel computing
with mp.Pool(processes=mp.cpu_count()) as pool:
# Plot
pool.starmap(
taq_data_plot_responses_physical_shift.
taq_cross_response_year_avg_responses_physical_shift_plot,
iprod(tickers, tickers, [year], [shifts]))
return None
def taq_data_plot_generator(tickers, year, tau, taus_p):
"""Generates all the analysis and plots from the TAQ data.
:param tickers: list of the string abbreviation of the stocks to be
analyzed (i.e. ['AAPL', 'MSFT']).
:param year: string of the year to be analyzed (i.e '2016').
:param taus: Integer great than zero (i.e. 1000).
:param taus_p: list of integers great than zero (i.e. [1, 10, 50]).
:return: None -- The function saves the data in a file and does not return
a value.
"""
# Specific functions
# Self-response
for ticker in tickers:
for tau_p in taus_p:
taq_data_analysis_responses_physical_short_long \
.taq_self_response_year_responses_physical_short_long_data(
ticker, year, tau, tau_p)
ticker_prod = iprod(tickers, tickers)
# ticker_prod = [('AAPL', 'MSFT'), ('MSFT', 'AAPL'),
# ('GS', 'JPM'), ('JPM', 'GS'),
# ('CVX', 'XOM'), ('XOM', 'CVX'),
# ('GOOG', 'MA'), ('MA', 'GOOG'),
# ('CME', 'GS'), ('GS', 'CME'),
# ('RIG', 'APA'), ('APA', 'RIG')]
# Cross-response and cross-correlator
for ticks in ticker_prod:
for tau_p in taus_p:
taq_data_analysis_responses_physical_short_long \
.taq_cross_response_year_responses_physical_short_long_data(
ticks[0], ticks[1], year, tau, tau_p)
# Parallel computing
with mp.Pool(processes=mp.cpu_count()) as pool:
# Plot
pool.starmap(
taq_data_plot_responses_physical_short_long.
taq_self_response_year_avg_responses_physical_short_long_plot,
iprod(tickers, [year], [tau], taus_p))
# Parallel computing
with mp.Pool(processes=mp.cpu_count()) as pool:
# Plot
pool.starmap(
taq_data_plot_responses_physical_short_long.
taq_cross_response_year_avg_responses_physical_short_long_plot,
iprod(tickers, tickers, [year], [tau], taus_p))
return None
def hist_data_plot_generator(fx_pairs: List[str], years: List[str]) -> None:
"""Generates all the analysis and plots from the HIST data.
:param fx_pairs: list of the string abbreviation of the forex pairs to be
analyzed (i.e. ['eur_usd', 'gbp_usd']).
:param years: list of the string of the year to be analyzed
(i.e. ['2016', '2017']).
:return: None -- The function saves the data in a file and does not return
a value.
"""
# Specific functions
fx_pair: str
for fx_pair in fx_pairs:
for year in years:
# Self-response
hist_data_analysis_responses_physical \
.hist_fx_self_response_year_responses_physical_data(fx_pair,
year)
# Parallel computing
with mp.Pool(processes=mp.cpu_count()) as pool:
# Plot
pool.starmap(hist_data_plot_responses_physical
.hist_fx_self_response_year_avg_responses_physical_plot,
iprod(fx_pairs, years))
def hist_data_plot_generator(years: List[str], intervals: List[str]) -> None:
"""Generates all the analysis and plots from the HIST data.
:param fx_pairs: list of the string abbreviation of the forex pairs to be
analyzed (i.e. ['eur_usd', 'gbp_usd']).
:param years: list of the string of the year to be analyzed
(i.e. ['2016', '2017']).
:param intervals: list of string of the interval to be analyzed
(i.e. ['week', 'month', 'quarter', 'year'])
:return: None -- The function saves the data in a file and does not return
a value.
"""
# Parallel computing
with mp.Pool(processes=mp.cpu_count()) as pool:
# Basic functions
pool.starmap(
hist_data_analysis_eigenvectors_physical.
hist_fx_eigenvectors_physical_data, iprod(years, intervals))
# Specific functions
year: str
for year in years:
interval: str
for interval in intervals:
hist_data_plot_eigenvectors_physical. \
hist_fx_eigenvectors_physical_plot(year, interval)
def hist_data_plot_generator(fx_pairs: List[str], years: List[str],
weeks: Tuple[str, ...]) -> None:
"""Generates all the analysis and plots from the HIST data.
:param fx_pairs: list of the string abbreviation of the forex pairs to be
analyzed (i.e. ['eur_usd', 'gbp_usd']).
:param years: list of the strings of the years to be analyzed
(i.e. ['2016', '2017']).
:param weeks: tuple of the strings of the weeks to be analyzed
(i.e. ('01', '02')).
:return: None -- The function saves the data in a file and does not return
a value.
"""
fx_pair: str
year: str
for fx_pair in fx_pairs:
for year in years:
# Data extraction
hist_data_analysis_extraction \
.hist_fx_data_extraction_week(fx_pair, year)
# Parallel computing
with mp.Pool(processes=mp.cpu_count()) as pool:
# Basic functions
pool.starmap(hist_data_analysis_extraction.hist_fx_midpoint_trade_data,
iprod(fx_pairs, years, weeks))
for fx_pair in fx_pairs:
for year in years:
# Plot
hist_data_plot_extraction \
.hist_fx_midpoint_year_plot(fx_pair, year, weeks)
def iter_all_dict_combinations_ordered(varied_dict):
"""
Same as all_dict_combinations but preserves order
"""
tups_list = [[(key, val) for val in val_list]
for (key, val_list) in six.iteritems(varied_dict)]
dict_iter = (OrderedDict(tups) for tups in iprod(*tups_list))
return dict_iter
def all_dict_combinations(varied_dict):
"""
all_dict_combinations
Args:
varied_dict (dict): a dict with lists of possible parameter settings
Returns:
list: dict_list a list of dicts correpsonding to all combinations of params settings
CommandLine:
python -m utool.util_dict --test-all_dict_combinations
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_dict import * # NOQA
>>> import utool as ut
>>> varied_dict = {'logdist_weight': [0.0, 1.0], 'pipeline_root': ['vsmany'], 'sv_on': [True, False, None]}
>>> dict_list = all_dict_combinations(varied_dict)
>>> result = str(ut.list_str(dict_list))
>>> print(result)
[
{'logdist_weight': 0.0, 'pipeline_root': 'vsmany', 'sv_on': True,},
{'logdist_weight': 0.0, 'pipeline_root': 'vsmany', 'sv_on': False,},
{'logdist_weight': 0.0, 'pipeline_root': 'vsmany', 'sv_on': None,},
{'logdist_weight': 1.0, 'pipeline_root': 'vsmany', 'sv_on': True,},
{'logdist_weight': 1.0, 'pipeline_root': 'vsmany', 'sv_on': False,},
{'logdist_weight': 1.0, 'pipeline_root': 'vsmany', 'sv_on': None,},
]
[
{'pipeline_root': 'vsmany', 'sv_on': True, 'logdist_weight': 0.0,},
{'pipeline_root': 'vsmany', 'sv_on': True, 'logdist_weight': 1.0,},
{'pipeline_root': 'vsmany', 'sv_on': False, 'logdist_weight': 0.0,},
{'pipeline_root': 'vsmany', 'sv_on': False, 'logdist_weight': 1.0,},
{'pipeline_root': 'vsmany', 'sv_on': None, 'logdist_weight': 0.0,},
{'pipeline_root': 'vsmany', 'sv_on': None, 'logdist_weight': 1.0,},
]
Ignore:
print(x)
print(y)
print(ut.hz_str('\n'.join(list(x)), '\n'.join(list(y))))
"""
#tups_list = [[(key, val) for val in val_list]
# if isinstance(val_list, (list, tuple))
# else [(key, val_list)]
# for (key, val_list) in six.iteritems(varied_dict)]
tups_list = [[(key, val) for val in val_list]
if isinstance(val_list, (list, tuple))
else [(key, val_list)]
for (key, val_list) in iteritems_sorted(varied_dict)]
dict_list = [dict(tups) for tups in iprod(*tups_list)]
#dict_list = [{key: val for (key, val) in tups} for tups in iprod(*tups_list)]
#from collections import OrderedDict
#dict_list = [OrderedDict([(key, val) for (key, val) in tups]) for tups in iprod(*tups_list)]
return dict_list
def taq_cross_response_year_responses_physical_shift_data(ticker_i, ticker_j,
year, shift):
"""Computes the cross-response of a year.
Using the taq_cross_response_day_responses_physical_shift_data function
computes the cross-response function for a year.
:param ticker_i: string of the abbreviation of the stock to be analyzed
(i.e. 'AAPL').
:param ticker_j: string of the abbreviation of the stock to be analyzed
(i.e. 'AAPL').
:param year: string of the year to be analyzed (i.e '2016').
:param shift: integer great than zero (i.e. 50).
:return: tuple -- The function returns a tuple with numpy arrays.
"""
if (ticker_i == ticker_j):
# Self-response
return None
else:
function_name = taq_cross_response_year_responses_physical_shift_data\
.__name__
taq_data_tools_responses_physical_shift \
.taq_function_header_print_data(function_name, ticker_i, ticker_j,
year, '', '')
dates = taq_data_tools_responses_physical_shift \
.taq_bussiness_days(year)
cross_values = []
args_prod = iprod([ticker_i], [ticker_j], dates, [shift])
# Parallel computation of the cross-responses. Every result is appended
# to a list
with mp.Pool(processes=mp.cpu_count()) as pool:
cross_values.append(pool.starmap(
taq_cross_response_day_responses_physical_shift_data,
args_prod))
# To obtain the total cross-response, I sum over all the cross-response
# values and all the amount of trades (averaging values)
cross_v_final = np.sum(cross_values[0], axis=0)
cross_response_val = cross_v_final[0] / cross_v_final[1]
cross_response_avg = cross_v_final[1]
# Saving data
taq_data_tools_responses_physical_shift \
.taq_save_data(f'{function_name}_shift_{shift}',
cross_response_val, ticker_i, ticker_j, year, '',
'')
return (cross_response_val, cross_response_avg)
def taq_trade_sign_cross_correlator_year_responses_physical_data(ticker_i,
ticker_j,
year):
"""Computes the trade sign-cross correlator of a year.
Using the taq_trade_sign_cross_correlator_day_responses_physical_data
function computes the cross-correlator function for a year.
:param ticker_i: string of the abbreviation of the stock to be analyzed
(i.e. 'AAPL').
:param ticker_j: string of the abbreviation of the stock to be analyzed
(i.e. 'AAPL').
:param year: string of the year to be analyzed (i.e '2016').
:return: tuple -- The function returns a tuple with numpy arrays.
"""
if (ticker_i == ticker_j):
# Self-response
return None
else:
function_name = \
taq_trade_sign_cross_correlator_year_responses_physical_data \
.__name__
taq_data_tools_responses_physical \
.taq_function_header_print_data(function_name, ticker_i, ticker_j,
year, '', '')
dates = taq_data_tools_responses_physical.taq_bussiness_days(year)
cross_values = []
args_prod = iprod([ticker_i], [ticker_j], dates)
# Parallel computation of the cross-correlator. Every result is
# appended to a list
with mp.Pool(processes=mp.cpu_count()) as pool:
cross_values.append(pool.starmap(
taq_trade_sign_cross_correlator_day_responses_physical_data,
args_prod))
# To obtain the total cross-correlator, I sum over all the
# cross-correlator values and all the amount of trades
# (averaging values)
cross_v_final = np.sum(cross_values[0], axis=0)
cross_correlator_val = cross_v_final[0] / cross_v_final[1]
cross_correlator_avg = cross_v_final[1]
# Saving data
taq_data_tools_responses_physical \
.taq_save_data(function_name, cross_correlator_val, ticker_i,
ticker_j, year, '', '')
return (cross_correlator_val, cross_correlator_avg)
def makeHumanReadable(self, a1, a2, a3):
from itertools import product as iprod
# TODO! Does not give "Human" readbility ...
# Choose vectors that give the smallest angles between a1..a3
# Sort on length
ipiv = N.argsort(distance(N.array([a1, a2, a3])))
na1 = [a1, a2, a3][ipiv[0]]
na2 = [a1, a2, a3][ipiv[1]]
na3 = [a1, a2, a3][ipiv[2]]
a1, a2, a3 = na1, na2, na3
# Make possible n a1 + m a2 + l a3
poss = []
for i1, i2, i3 in iprod(range(-1, 2), repeat=3):
poss += [a1 * i1 + a2 * i2 + a3 * i3]
poss = N.array(poss)
# possiblities for a1, a2, a3 based on length
poss1 = poss[N.where(
N.abs(distance(poss) - distance(a1)) < self.accuracy)]
poss2 = poss[N.where(
N.abs(distance(poss) - distance(a2)) < self.accuracy)]
poss3 = poss[N.where(
N.abs(distance(poss) - distance(a3)) < self.accuracy)]
poss = N.concatenate((poss1, poss2, poss3)) # Keep duplicates
# Choose the one fitting the PBC best
vec = self.pbc[0, :]
scal = mm(vec, N.transpose(poss))
a1 = poss[N.where(N.sort(scal)[-1] == scal)[0][0]]
# Remove linear dependent
poss=poss[N.where(N.abs(N.abs(mm(poss, a1))-\
N.abs(distance(poss)*distance(a1)))>self.accuracy)]
# Choose the one fitting the PBC best
vec = self.pbc[1, :]
scal = mm(vec, N.transpose(poss))
a2 = poss[N.where(N.sort(scal)[-1] == scal)[0][0]]
# Remove linear dependent
poss=poss[N.where(N.abs(N.abs(mm(poss, a2))-\
N.abs(distance(poss)*distance(a2)))>self.accuracy)]
# Choose the one fitting the PBC best
vec = self.pbc[2, :]
scal = mm(vec, N.transpose(poss))
a3 = poss[N.where(N.sort(scal)[-1] == scal)[0][0]]
if N.linalg.det([a1, a2, a3]) < -self.accuracy:
a3 = -a3
return a1, a2, a3
def extract_detector_negatives(hs, output_dir, batch_extract_kwargs):
from itertools import product as iprod
negreg_dir = join(output_dir, 'negatives', 'regions')
negall_dir = join(output_dir, 'negatives', 'whole')
negreg_fmt = join(negreg_dir, 'gx%d_wix%d_hix%d_neg.png')
negall_fmt = join(negall_dir, 'gx%d_all_neg.png')
helpers.ensuredir(negall_dir)
helpers.ensuredir(negreg_dir)
print('[train] extract_negatives')
gx_list = hs.get_valid_gxs()
nChips_list = np.array(hs.gx2_nChips(gx_list))
aif_list = np.array(hs.gx2_aif(gx_list))
# Find images where there are completely negative. They have no animals.
#is_negative = np.logical_and(aif_list, nChips_list)
is_completely_negative = np.logical_and(aif_list, nChips_list == 0)
negall_gxs = gx_list[np.where(is_completely_negative)[0]]
gfpath_list = []
cfpath_list = []
roi_list = []
def add_neg_eg(roi, gfpath, cfpath):
roi_list.append(roi)
gfpath_list.append(gfpath)
cfpath_list.append(cfpath)
width_split = 2
(uw, uh) = batch_extract_kwargs['uniform_size']
for gx in negall_gxs:
gfpath = hs.gx2_gname(gx, full=True)
# Add whole negative image
(gw, gh) = hs.gx2_image_size(gx)
roi = (0, 0, gw, gh)
add_neg_eg(roi, gfpath, negall_fmt % (gx))
# Add negative regions
w_step = gw // width_split
h_step = int(round(gh * (w_step / gw)))
nHeights, nWidths = gh // h_step, gw // w_step
if nWidths < 2 or nHeights < 1:
continue
for wix, hix in iprod(xrange(nWidths), xrange(nHeights)):
x, y = wix * w_step, hix * h_step
w, h = w_step, h_step
roi = (x, y, w, h)
add_neg_eg(roi, gfpath, negreg_fmt % (gx, wix, hix))
theta_list = [0] * len(roi_list)
cc2.batch_extract_chips(gfpath_list, cfpath_list, roi_list, theta_list,
**batch_extract_kwargs)
def CMHtest(var1,var2,controls=None): [] if controls == None else controls=controls """ In progress """ stratum = list(iprod([unique(i) for i in controls])) for i,k in enumerate(stratum): n_k = 0 return None
def hist_download_all_data(fx_pairs: List[str], years: List[str]) -> None:
"""Downloads all the HIST data.
:param fx_pairs: list of the string abbreviation of the forex pairs to be
analyzed (i.e. ['eur_usd', 'gbp_usd']).
:param years: list of the strings of the years to be analyzed
(i.e. ['2016', '2017]).
:return: None -- The function saves the data in a file and does not return
a value.
"""
with mp.Pool(processes=mp.cpu_count()) as pool:
pool.starmap(hist_download_data, iprod(fx_pairs, years))
def makeHumanReadable(self,a1,a2,a3):
from itertools import product as iprod
# TODO! Does not give "Human" readbility ...
# Choose vectors that give the smallest angles between a1..a3
# Sort on length
ipiv = N.argsort(distance(N.array([a1,a2,a3])))
na1 = [a1,a2,a3][ipiv[0]]
na2 = [a1,a2,a3][ipiv[1]]
na3 = [a1,a2,a3][ipiv[2]]
a1, a2, a3 = na1, na2, na3
# Make possible n a1 + m a2 + l a3
poss = []
for i1, i2, i3 in iprod(range(-1, 2),repeat=3):
poss += [a1*i1+a2*i2+a3*i3]
poss = N.array(poss)
# possiblities for a1, a2, a3 based on length
poss1 = poss[N.where(N.abs(distance(poss)-distance(a1))<self.accuracy)]
poss2 = poss[N.where(N.abs(distance(poss)-distance(a2))<self.accuracy)]
poss3 = poss[N.where(N.abs(distance(poss)-distance(a3))<self.accuracy)]
poss = N.concatenate((poss1,poss2,poss3)) # Keep duplicates
# Choose the one fitting the PBC best
vec = self.pbc[0,:]
scal = mm(vec,N.transpose(poss))
a1=poss[N.where(N.sort(scal)[-1]==scal)[0][0]]
# Remove linear dependent
poss=poss[N.where(N.abs(N.abs(mm(poss,a1))-\
N.abs(distance(poss)*distance(a1)))>self.accuracy)]
# Choose the one fitting the PBC best
vec = self.pbc[1,:]
scal = mm(vec,N.transpose(poss))
a2=poss[N.where(N.sort(scal)[-1]==scal)[0][0]]
# Remove linear dependent
poss=poss[N.where(N.abs(N.abs(mm(poss,a2))-\
N.abs(distance(poss)*distance(a2)))>self.accuracy)]
# Choose the one fitting the PBC best
vec = self.pbc[2,:]
scal = mm(vec,N.transpose(poss))
a3=poss[N.where(N.sort(scal)[-1]==scal)[0][0]]
if N.linalg.det([a1,a2,a3])<-self.accuracy:
a3=-a3
return a1,a2,a3
def get_gaussimg(width=3, resolution=7):
half_width = width / 2
gauss_xs = np.linspace(-half_width, half_width, resolution)
gauss_ys = np.linspace(-half_width, half_width, resolution)
gaussspace_xys = np.array(list(iprod(gauss_xs, gauss_ys)))
gausspace_score = np.array([pdf_norm2d(x, y) for (x, y) in gaussspace_xys])
gausspace_score -= gausspace_score.min()
gausspace_score /= gausspace_score.max()
size = (resolution, resolution)
gaussimg = gausspace_score.reshape(size).T
gaussimg = np.array(gaussimg, dtype=np.float32)
return gaussimg
def taq_quotes_trades_year_avg_spread_data(tickers, year):
"""Obtain the quotes and trades statistics for a year.
Using the taq_quotes_trades_day_avg_spread_data function computes the
statistics of the average spread, number of quotes and number of trades
for a year.
:param tickers: list of the string abbreviation of the stocks to be
analyzed (i.e. ['AAPL', 'MSFT']).
:param year: string of the year to be analyzed (i.e '2016').
:return: None -- The function saves the data in a file and does not return
a value.
"""
function_name = taq_quotes_trades_year_avg_spread_data.__name__
# Pandas DataFrame to store the data
spread_stats = pd.DataFrame(
columns=['Ticker', 'Avg_Quotes', 'Avg_Trades', 'Avg_Spread'])
dates = taq_data_tools_avg_spread.taq_bussiness_days(year)
for idx, ticker in enumerate(tickers):
taq_data_tools_avg_spread \
.taq_function_header_print_data(function_name, ticker, ticker,
year, '', '')
stat = []
args_prod = iprod([ticker], dates)
# Parallel computation of the statistics. Every result is appended to
# a list
with mp.Pool(processes=mp.cpu_count()) as pool:
stat.append(pool.starmap(taq_quotes_trades_day_avg_spread_data,
args_prod))
# To obtain the average of the year, I average all the results of the
# corresponding values (number quotes, trades and avg spread)
stat_year = np.nanmean(stat[0], axis=0)
stat_year = np.round(stat_year, decimals=2)
spread_stats.loc[idx] = [ticker] + list(stat_year)
spread_stats.sort_values(by='Avg_Spread', inplace=True)
spread_stats.to_csv(f'../taq_avg_spread_{year}.csv')
print(spread_stats)
return None
def taq_data_plot_generator(tickers, dates):
"""Generates all the analysis and plots from the TAQ data.
:param tickers: list of the string abbreviation of the stocks to be
analized (i.e. ['AAPL', 'MSFT']).
:param dates: list of strings with the date of the data to be extracted
(i.e. ['2008-01-02', '2008-01-03]).
:return: None -- The function saves the data in a file and does not return
a value.
"""
# Parallel computing
with mp.Pool(processes=mp.cpu_count()) as pool:
# Basic functions
pool.starmap(
taq_data_analysis_responses_second.taq_midpoint_second_data,
iprod(tickers, dates))
pool.starmap(
taq_data_analysis_responses_second.taq_trade_signs_second_data,
iprod(tickers, dates))
# Especific functions
for ticker in tickers:
# Self-response
taq_data_analysis_responses_second \
.taq_self_response_week_responses_second_data(ticker, dates)
# Plot
taq_data_plot_responses_second \
.taq_midpoint_second_plot(ticker, dates)
taq_data_plot_responses_second \
.taq_self_response_week_avg_responses_second_plot(ticker, dates)
return None
def data_plot_generator(dates: List[List[str]], time_steps: List[str]) -> None:
"""Generates all the analysis and plots from the data.
:param dates: list of lists of the string of the dates to be analyzed
(i.e. [['1980-01', '2020-12'], ['1980-01', '2020-12']).
:param time_steps: list of the string of the time step of the data
(i.e. ['1m', '2m', '5m']).
:return: None -- The function saves the data in a file and does not return
a value.
"""
# Parallel computing
with mp.Pool(processes=mp.cpu_count()) as pool:
# Specific functions
pool.starmap(
exact_distributions_correlation_analysis.returns_data,
iprod(dates, time_steps),
)
pool.starmap(
exact_distributions_correlation_analysis.normalized_returns_data,
iprod(dates, time_steps),
)
pool.starmap(
exact_distributions_correlation_analysis.
aggregated_dist_returns_market_data,
iprod(dates, time_steps),
)
# Plot
pool.starmap(exact_distributions_correlation_plot.returns_plot,
iprod(dates, time_steps))
pool.starmap(
exact_distributions_correlation_plot.
aggregated_dist_returns_market_plot,
iprod(dates, time_steps),
)
def latticeGroup(self):
from itertools import product as iprod
# Generate all unitary matrices that take a1...a3 to
# lattice points
a1, a2, a3 = self.a1, self.a2, self.a3
b1, b2, b3 = self.b1, self.b2, self.b3
points, dist = [], []
# Generate lattice points
for i1, i2, i3 in iprod(range(-2, 3), repeat=3):
points += [i1 * a1 + i2 * a2 + i3 * a3]
dist += [distance(points[-1])]
points, dist = N.array(points), N.array(dist)
a = [a1, a2, a3]
targets = [[], [], []]
# Choose all possible target points for a1...a3 with same length
for ii in range(3):
targets[ii] = points[N.where(
abs(dist - distance(a[ii])) < self.accuracy)]
# Make all possible combinations of targets for a1..a3
possible = myPermute(targets)
# Generate unitary transformation from a1, a2, a3 -> possible
Ulist = []
for ii in possible:
# Generate symmetry operations
U = ii[0].reshape(3, 1)*b1.reshape(1, 3)+\
ii[1].reshape(3, 1)*b2.reshape(1, 3)+\
ii[2].reshape(3, 1)*b3.reshape(1, 3)
if N.allclose(mm(U, N.transpose(U)), N.eye(3), atol=self.accuracy):
Ulist += [U]
self.pointU33 = Ulist
print "Symmetry: %i point symmetry operations found for lattice" % len(
Ulist)
sign = N.array([N.linalg.det(ii) for ii in Ulist])
rotn = []
for ii in Ulist:
a, M = ii, 1
while not N.allclose(N.eye(3), a, atol=self.accuracy):
a, M = mm(a, ii), M + 1
rotn += [M]
self.pointRank = rotn
print "Symmetry: rank * determinant"
print sign * self.pointRank
return
def taq_quotes_trades_year_statistics_data(tickers, year):
"""Obtain the quotes and trades statistics for a year.
Using the taq_quotes_trades_day_statistics_data function computes the
statistics of the average spread, number of quotes and number of trades
for a year.
:param tickers: list of the string abbreviation of the stocks to be
analyzed (i.e. ['AAPL', 'MSFT']).
:param year: string of the year to be analyzed (i.e '2016').
:return: None -- The function saves the data in a file and does not return
a value.
"""
function_name = taq_quotes_trades_year_statistics_data.__name__
# Create a file to save the info
file = open('../taq_quotes_trades_year_statistics_data.csv', 'a+')
file.write('Ticker, avg_quotes, avg_trades, avg_spread\n')
for ticker in tickers:
taq_data_tools_statistics \
.taq_function_header_print_data(function_name, ticker, ticker,
year, '', '')
dates = taq_data_tools_statistics.taq_bussiness_days(year)
stat = []
args_prod = iprod([ticker], dates)
# Parallel computation of the statistics. Every result is appended to
# a list
with mp.Pool(processes=mp.cpu_count()) as pool:
stat.append(
pool.starmap(taq_quotes_trades_day_statistics_data, args_prod))
# To obtain the average of the year, I average all the results of the
# corresponding values (number quotes, trades and avg spread)
stat_year = np.nanmean(stat[0], axis=0)
# Write data in file
file.write(f'{ticker}, {stat_year[0]:.0f}, {stat_year[1]:.0f},' +
f' {stat_year[2]:.2f}\n')
file.close
return None
def ln_aggregated_dist_returns_market_data(dates: List[str], time_step: str,
window: str) -> None:
"""Computes the aggregated distribution of returns for a market.
:param dates: List of the interval of dates to be analyzed
(i.e. ['1980-01', '2020-12']).
:param time_step: time step of the data (i.e. '1m', '2m', '5m', ...).
:param window: window time to compute the volatility (i.e. '60').
:return: None -- The function saves the data in a file and does not return
a value.
"""
function_name: str = ln_aggregated_dist_returns_market_data.__name__
local_normalization_tools \
.function_header_print_data(function_name, dates, time_step, window)
try:
# Load data
stocks_name: pd.DataFrame = pickle.load(open(
f'../data/correlation_matrix/returns_data_{dates[0]}_{dates[1]}'
+ f'_step_{time_step}.pickle', 'rb')).columns[:20]
agg_ret_mkt_list: List[float] = []
stocks_comb: Iterator[Tuple[Any, ...]] = icomb(stocks_name, 2)
args_prod: Iterator[Any] = iprod([dates], [time_step], stocks_comb,
[window])
with mp.Pool(processes=mp.cpu_count()) as pool:
agg_ret_mkt_list.extend(pool.starmap(
ln_aggregated_dist_returns_pair_data, args_prod))
agg_ret_mkt_list_flat = [val for sublist in agg_ret_mkt_list for val in sublist]
agg_ret_mkt_series: pd.Series = pd.Series(agg_ret_mkt_list_flat)
# Saving data
local_normalization_tools \
.save_data(agg_ret_mkt_series, function_name, dates, time_step,
window)
del agg_ret_mkt_list
del agg_ret_mkt_series
except FileNotFoundError as error:
print('No data')
print(error)
print()
def hist_quotes_trades_year_avg_spread_data(fx_pairs: List[str],
year: str) -> None:
"""Obtain the quotes and trades statistics for a year.
Using the hist_quotes_trades_day_avg_spread_data function computes the
statistics of the average spread, number of quotes and number of trades
for a year.
:param fx_pairs: list of the string abbreviation of the fx pairs to be
analyzed (i.e. ['eur_usd', 'gbp_usd']).
:param year: String of the years to be analyzed (i.e '2016').
:return: None -- The function saves the data in a file and does not return
a value.
"""
function_name: str = hist_quotes_trades_year_avg_spread_data.__name__
# Pandas DataFrame to store the data
spread_stats: pd.DataFrame = pd.DataFrame(
columns=['FxPair', 'Avg_Quotes', 'Avg_Spread'])
weeks: Tuple[str, ...] = hist_data_tools_avg_spread.hist_weeks()
idx: int
fx_pair: str
for idx, fx_pair in enumerate(fx_pairs):
hist_data_tools_avg_spread \
.hist_function_header_print_data(function_name, fx_pair, year, '')
stat: List[Any] = []
args_prod: Iterator[Tuple[str, ...]] = iprod([fx_pair], [year], weeks)
# Parallel computation of the statistics. Every result is appended to
# a list
with mp.Pool(processes=mp.cpu_count()) as pool:
stat.append(pool.starmap(hist_quotes_trades_day_avg_spread_data,
args_prod))
# To obtain the average of the year, I average all the results of the
# corresponding values (number quotes, trades and avg spread)
stat_year: List[str] = list(np.ceil(np.nanmean(stat[0], axis=0)))
spread_stats.loc[idx] = [fx_pair] + stat_year
spread_stats.sort_values(by='Avg_Spread', inplace=True)
spread_stats.to_csv(f'../hist_avg_spread_{year}.csv')
print(spread_stats)
def latticeGroup(self):
from itertools import product as iprod
# Generate all unitary matrices that take a1...a3 to
# lattice points
a1, a2, a3= self.a1, self.a2, self.a3
b1, b2, b3= self.b1, self.b2, self.b3
points, dist = [], []
# Generate lattice points
for i1, i2, i3 in iprod(range(-2, 3),repeat=3):
points +=[i1*a1+i2*a2+i3*a3]
dist+=[distance(points[-1])]
points, dist = N.array(points), N.array(dist)
a = [a1, a2, a3]
targets = [[], [], []]
# Choose all possible target points for a1...a3 with same length
for ii in range(3):
targets[ii]=points[N.where(abs(dist-distance(a[ii]))<self.accuracy)]
# Make all possible combinations of targets for a1..a3
possible = myPermute(targets)
# Generate unitary transformation from a1, a2, a3 -> possible
Ulist = []
for ii in possible:
# Generate symmetry operations
U = ii[0].reshape(3,1)*b1.reshape(1,3)+\
ii[1].reshape(3,1)*b2.reshape(1,3)+\
ii[2].reshape(3,1)*b3.reshape(1,3)
if N.allclose(mm(U,N.transpose(U)),N.eye(3), atol=self.accuracy):
Ulist+=[U]
self.pointU33 = Ulist
print "Symmetry: %i point symmetry operations found for lattice"%len(Ulist)
sign = N.array([N.linalg.det(ii) for ii in Ulist])
rotn = []
for ii in Ulist:
a, M = ii, 1
while not N.allclose(N.eye(3),a,atol=self.accuracy):
a, M =mm(a,ii), M+1
rotn+=[M]
self.pointRank = rotn
print "Symmetry: rank * determinant"
print sign*self.pointRank
return
def taq_trade_sign_self_correlator_year_responses_physical_data(ticker, year):
"""Computes the trade sign self-correlator of a year.
Using the taq_trade_sign_self_correlator_day_responses_physical_data
function computes the self-correlator function for a year.
:param ticker: string of the abbreviation of the stock to be analyzed
(i.e. 'AAPL').
:param year: string of the year to be analyzed (i.e '2016').
:return: tuple -- The function returns a tuple with numpy arrays.
"""
function_name = \
taq_trade_sign_self_correlator_year_responses_physical_data.__name__
taq_data_tools_responses_physical \
.taq_function_header_print_data(function_name, ticker, ticker, year,
'', '')
dates = taq_data_tools_responses_physical.taq_bussiness_days(year)
self_values = []
args_prod = iprod([ticker], dates)
# Parallel computation of the self-correlator. Every result is appended to
# a list
with mp.Pool(processes=mp.cpu_count()) as pool:
self_values.append(pool.starmap(
taq_trade_sign_self_correlator_day_responses_physical_data,
args_prod))
# To obtain the total self-correlator, I sum over all the self-correlator
# values and all the amount of trades (averaging values)
self_v_final = np.sum(self_values[0], axis=0)
self_correlator_val = self_v_final[0] / self_v_final[1]
self_correlator_avg = self_v_final[1]
# Saving data
taq_data_tools_responses_physical \
.taq_save_data(function_name, self_correlator_val, ticker, ticker,
year, '', '')
return (self_correlator_val, self_correlator_avg)
def taq_self_response_week_responses_second_data(ticker, dates):
"""Computes the self-response of a year.
Using the taq_self_response_day_responses_second_data function computes the
self-response function for a year.
:param ticker: string of the abbreviation of stock to be analized
(i.e. 'AAPL').
:param dates: list of strings with the date of the data to be extracted
(i.e. ['2008-01-02', '2008-01-03]).
:return: tuple -- The function returns a tuple with numpy arrays.
"""
year = dates[0].split('-')[0]
function_name = taq_self_response_week_responses_second_data.__name__
taq_data_tools_responses_second \
.taq_function_header_print_data(function_name, ticker, ticker, year,
'', '')
self_values = []
args_prod = iprod([ticker], dates)
# Parallel computation of the self-responses. Every result is appended to
# a list
with mp.Pool(processes=mp.cpu_count()) as pool:
self_values.append(
pool.starmap(taq_self_response_day_responses_second_data,
args_prod))
# To obtain the total self-response, I sum over all the self-response
# values and all the amount of trades (averaging values)
self_v_final = np.sum(self_values[0], axis=0)
self_response_val = self_v_final[0] / self_v_final[1]
self_response_avg = self_v_final[1]
# Saving data
taq_data_tools_responses_second \
.taq_save_data(f"{function_name}_{dates[0].split('-')[1]}",
self_response_val, ticker, ticker, year, '', '')
return (self_response_val, self_response_avg)
def taq_self_response_year_trade_shift_data(ticker, year, tau):
"""Computes the self response of a year.
Using the taq_self_response_day_trade_shift_data function computes the
self-response function for a year.
:param ticker: string of the abbreviation of stock to be analyzed
(i.e. 'AAPL').
:param year: string of the year to be analyzed (i.e '2016').
:param tau: integer great than zero (i.e. 50).
:return: tuple -- The function returns a tuple with numpy arrays.
"""
function_name = taq_self_response_year_trade_shift_data.__name__
taq_data_tools_trade_shift \
.taq_function_header_print_data(function_name, ticker, ticker, year,
'', '')
dates = taq_data_tools_trade_shift.taq_bussiness_days(year)
self_values = []
args_prod = iprod([ticker], dates, [tau])
# Parallel computation of the self-responses. Every result is appended to
# a list
with mp.Pool(processes=mp.cpu_count()) as pool:
self_values.append(
pool.starmap(taq_self_response_day_trade_shift_data, args_prod))
# To obtain the total self-response, I sum over all the self-response
# values and all the amount of trades (averaging values)
self_v_final = np.sum(self_values[0], axis=0)
self_response_val = self_v_final[0] / self_v_final[1]
self_response_avg = self_v_final[1]
# Saving data
taq_data_tools_trade_shift \
.taq_save_data(f'{function_name}_tau_{tau}', self_response_val, ticker,
ticker, year, '', '')
return (self_response_val, self_response_avg)
def all_dict_combinations_lbls(varied_dict):
""" returns a label for each variation in a varydict.
It tries to not be oververbose and returns only what parameters are varied
in each label.
CommandLine:
python -m utool.util_dict --test-all_dict_combinations_lbls
Example:
>>> # ENABLE_DOCTEST
>>> import utool
>>> varied_dict = {'logdist_weight': [0.0, 1.0], 'pipeline_root': ['vsmany'], 'sv_on': [True, False, None]}
>>> comb_lbls = utool.all_dict_combinations_lbls(varied_dict)
>>> result = (utool.list_str(comb_lbls))
>>> print(result)
[
"(('logdist_weight', 0.0), ('sv_on', True))",
"(('logdist_weight', 0.0), ('sv_on', False))",
"(('logdist_weight', 0.0), ('sv_on', None))",
"(('logdist_weight', 1.0), ('sv_on', True))",
"(('logdist_weight', 1.0), ('sv_on', False))",
"(('logdist_weight', 1.0), ('sv_on', None))",
]
[
"(('sv_on', True), ('logdist_weight', 0.0))",
"(('sv_on', True), ('logdist_weight', 1.0))",
"(('sv_on', False), ('logdist_weight', 0.0))",
"(('sv_on', False), ('logdist_weight', 1.0))",
"(('sv_on', None), ('logdist_weight', 0.0))",
"(('sv_on', None), ('logdist_weight', 1.0))",
]
#ut.list_type_profile(comb_lbls)
"""
# TODO dont do sorted if it an ordered dict
multitups_list = [[(key, val) for val in val_list]
for key, val_list in iteritems_sorted(varied_dict)
if isinstance(val_list, (list, tuple)) and len(val_list) > 1]
comb_lbls = list(map(str, list(iprod(*multitups_list))))
return comb_lbls
def hist_fx_self_response_year_responses_physical_data(
fx_pair: str, year: str) -> Tuple[np.ndarray, ...]:
"""Computes the self-response of a year.
Using the hist_self_response_year_responses_physical_data function computes
the self-response function for a year.
:param ticker: string of the abbreviation of stock to be analyzed
(i.e. 'AAPL').
:param year: string of the year to be analyzed (i.e '2016').
:return: tuple -- The function returns a tuple with numpy arrays.
"""
function_name: str = hist_fx_self_response_year_responses_physical_data \
.__name__
hist_data_tools_responses_physical \
.hist_function_header_print_data(function_name, fx_pair, year, '')
weeks: Tuple[str, ...] = hist_data_tools_responses_physical.hist_weeks()
self_values: List[np.ndarray] = []
args_prod: Iterator[Tuple[str, ...]] = iprod([fx_pair], [year], weeks)
# Parallel computation of the self-responses. Every result is appended to
# a list
with mp.Pool(processes=mp.cpu_count()) as pool:
self_values.append(
pool.starmap(hist_fx_self_response_week_responses_physical_data,
args_prod))
# To obtain the total self-response, I sum over all the self-response
# values and all the amount of trades (averaging values)
self_v_final: np.ndarray = np.sum(self_values[0], axis=0)
self_response_val: np.ndarray = self_v_final[0] / self_v_final[1]
self_response_avg: np.ndarray = self_v_final[1]
# Saving data
if (not os.path.isdir(
f'../../hist_data/responses_physical_{year}/{function_name}/')):
try:
os.mkdir(
f'../../hist_data/responses_physical_{year}/{function_name}/')
print('Folder to save data created')
except FileExistsError:
print('Folder exists. The folder was not created')
if (not os.path.isdir(
f'../../hist_data/responses_physical_{year}/{function_name}/' +
f'{fx_pair}/')):
try:
os.mkdir(
f'../../hist_data/responses_physical_{year}/{function_name}/' +
f'{fx_pair}/')
print('Folder to save data created')
except FileExistsError:
print('Folder exists. The folder was not created')
hist_data_tools_responses_physical \
.hist_save_data(self_response_val, fx_pair, year)
return (self_response_val, self_response_avg)
def group_interactions(Smean, Aij, mwhere, return_all=0, remove_diag=1):
'''Compute mu,gamma... per group, given interaction matrix Aij and mwhere=[ [idxs of group 1], [...]]'''
from itertools import product as iprod
interactions = [[[
Aij[i, j] for i, j in iprod(m1, m2)
if ifelse(remove_diag, i != j, True)
] for m2 in mwhere] for m1 in mwhere]
Amean = np.array([[getmean(i) for i in j] for j in interactions])
Astd = np.array([[getstd(i) for i in j] for j in interactions])
Arowstd = np.array(
[[getstd([getmean(Aij[i, m2]) for i in m1]) for m2 in mwhere]
for m1 in mwhere])
intsym = [[[
Aij[i, j] * Aij[j, i] for i, j in iprod(m1, m2) if i != j or im1 != im2
] for im2, m2 in enumerate(mwhere)] for im1, m1 in enumerate(mwhere)]
Asym = np.array([[getmean(i) for i in j] for j in intsym])
conn = np.array([[getmean(np.array(i) != 0) for i in j]
for j in interactions])
mu = -Smean * Amean
sigma = np.sqrt(Smean) * Astd
sigrow = Smean * Arowstd
# gamma=(Asym-Amean*Amean.T )/(Astd*Astd.T +10**-15 )
def getcorr(m1, m2, same=0):
if len(m2) > 1 and len(m1) > 1:
if same:
m = m1
offdiag = (np.eye(len(m)) == 0)
mat = Aij[np.ix_(m, m)]
if np.std(mat[offdiag]):
return np.corrcoef(mat[offdiag], mat.T[offdiag])[0, 1]
else:
if np.std(Aij[np.ix_(m1, m2)]) and np.std(Aij[np.ix_(m2, m1)]):
return np.corrcoef(Aij[np.ix_(m1, m2)].ravel(),
Aij[np.ix_(m2, m1)].T.ravel())[0, 1]
return 0
gamma = np.array([[getcorr(m1, m2) for im2, m2 in enumerate(mwhere)]
for im1, m1 in enumerate(mwhere)])
#
# def getcorr(m):
# if len(m):
# return np.corrcoef(Aij[np.ix_(m, m)][np.eye(len(m)) == 0],
# Aij[np.ix_(m, m)].T[np.eye(len(m)) == 0])[0, 1]
# return 0
gamma[np.eye(gamma.shape[0]) > 0] = [getcorr(m, m, same=1) for m in mwhere]
try:
assert not np.isnan(sigma).any()
except:
print 'SIGMA NAN'
code_debugger()
gamma[np.isnan(gamma)] = 0
try:
assert np.max(np.abs(gamma)) <= 1
except:
print '|GAMMA| >1'
code_debugger()
if return_all:
return conn, mu, sigma, sigrow, gamma, Amean, Astd, Asym
return conn, mu, sigma, sigrow, gamma
def get_sift_collection(sift, aff=None, bin_color=BLACK, arm1_color=RED,
arm2_color=BLACK, arm_alpha=1.0, arm1_lw=1.0,
arm2_lw=2.0, circ_alpha=.5, **kwargs):
"""
Creates a collection of SIFT matplotlib patches
get_sift_collection
Args:
sift (?):
aff (None):
bin_color (ndarray):
arm1_color (ndarray):
arm2_color (ndarray):
arm_alpha (float):
arm1_lw (float):
arm2_lw (float):
circ_alpha (float):
Returns:
?: coll_tup
CommandLine:
python -m plottool.mpl_sift --test-get_sift_collection
Example:
>>> from plottool.mpl_sift import * # NOQA
>>> sift = testdata_sifts()[0]
>>> aff = None
>>> bin_color = array([ 0., 0., 0., 1.])
>>> arm1_color = array([ 1., 0., 0., 1.])
>>> arm2_color = array([ 0., 0., 0., 1.])
>>> arm_alpha = 1.0
>>> arm1_lw = 0.5
>>> arm2_lw = 1.0
>>> circ_alpha = 0.5
>>> coll_tup = get_sift_collection(sift, aff, bin_color, arm1_color, arm2_color, arm_alpha, arm1_lw, arm2_lw, circ_alpha)
>>> print(coll_tup)
"""
# global offset scale adjustments
if aff is None:
aff = mpl.transforms.Affine2D()
MULTI_COLORED_ARMS = kwargs.pop('multicolored_arms', False)
_kwarm = kwargs.copy()
_kwarm.update(dict(head_width=1e-10, length_includes_head=False, transform=aff, color=[1, 1, 0]))
_kwcirc = dict(transform=aff)
arm_patches = []
DSCALE = 0.25 # Descriptor scale factor
ARMSCALE = 1.5 # Arm length scale factor
XYSCALE = 0.5 # Position scale factor
XYOFFST = -0.75 # Position offset
NORI, NX, NY = 8, 4, 4 # SIFT BIN CONSTANTS
NBINS = NX * NY
discrete_ori = (np.arange(0, NORI) * (TAU / NORI))
# Arm magnitude and orientations
arm_mag = sift / 255.0
arm_ori = np.tile(discrete_ori, (NBINS, 1)).flatten()
# Arm orientation in dxdy format
arm_dxy = np.array(list(zip(*_cirlce_rad2xy(arm_ori, arm_mag))))
# Arm locations and dxdy index
yxt_gen = iprod(range(NY), range(NX), range(NORI))
# Circle x,y locations
yx_gen = iprod(range(NY), range(NX))
# Draw 8 directional arms in each of the 4x4 grid cells
arm_args_list = []
for y, x, t in yxt_gen:
#print('y=%r, x=%r, t=%r' % (y, x, t))
index = (y * NX * NORI) + (x * NORI) + (t)
(dx, dy) = arm_dxy[index]
arm_x = (x * XYSCALE) + XYOFFST # MULTIPLY BY -1 to invert X axis
arm_y = (y * XYSCALE) + XYOFFST
arm_dy = (dy * DSCALE) * ARMSCALE
arm_dx = (dx * DSCALE) * ARMSCALE
_args = [arm_x, arm_y, arm_dx, arm_dy]
arm_args_list.append(_args)
for _args in arm_args_list:
arm_patch = mpl.patches.FancyArrow(*_args, **_kwarm)
arm_patches.append(arm_patch)
#print('len(arm_patches) = %r' % (len(arm_patches),))
# Draw circles around each of the 4x4 grid cells
circle_patches = []
for y, x in yx_gen:
circ_xy = (x * XYSCALE + XYOFFST, y * XYSCALE + XYOFFST)
circ_radius = DSCALE
circle_patches += [mpl.patches.Circle(circ_xy, circ_radius, **_kwcirc)]
circ_coll = _circl_collection(circle_patches, bin_color, circ_alpha)
arm2_coll = _arm_collection(arm_patches, arm2_color, arm_alpha, arm2_lw)
if MULTI_COLORED_ARMS:
# Hack in same colorscheme for arms as the sift bars
ori_colors = color_fns.distinct_colors(16)
coll_tup = [circ_coll, arm2_coll]
coll_tup += [_arm_collection(_, color, arm_alpha, arm1_lw)
for _, color in zip(ut.ichunks(arm_patches, 8), ori_colors)]
coll_tup = tuple(coll_tup)
else:
# Just use a single color for all the arms
arm1_coll = _arm_collection(arm_patches, arm1_color, arm_alpha, arm1_lw)
coll_tup = (circ_coll, arm2_coll, arm1_coll)
return coll_tup
def test_draw_keypoint_main():
r"""
CommandLine:
python -m pyhesaff.tests.test_draw_keypoint --test-test_draw_keypoint_main --show
Example:
>>> # DISABLE_DOCTEST
>>> from pyhesaff.tests.test_draw_keypoint import * # NOQA
>>> test_draw_keypoint_main()
>>> ut.show_if_requested()
"""
from plottool import draw_func2 as df2
from plottool import mpl_keypoint
import vtool.keypoint as ktool # NOQA
# TODO: Gui tests yield:
# Jul 13 13:14:53 www.longerdog.com Python[23974] <Error>: This user is not allowed access to the window system right now.
# don't do window access without --show
TAU = 2 * np.pi
# Hack these directions to be relative to gravity
#RIGHT = ((0 * TAU / 4) - ktool.GRAVITY_THETA) % TAU
DOWN = ((1 * TAU / 4) - ktool.GRAVITY_THETA) % TAU
#LEFT = ((2 * TAU / 4) - ktool.GRAVITY_THETA) % TAU
#UP = ((3 * TAU / 4) - ktool.GRAVITY_THETA) % TAU
def test_keypoint(xscale=1, yscale=1, ori=DOWN, skew=0):
# Test Keypoint
x, y = 0, 0
iv11, iv21, iv22 = xscale, skew, yscale
kp = np.array([x, y, iv11, iv21, iv22, ori])
# Test SIFT descriptor
sift = np.zeros(128)
sift[ 0: 8] = 1
sift[ 8:16] = .5
sift[16:24] = .0
sift[24:32] = .5
sift[32:40] = .8
sift[40:48] = .8
sift[48:56] = .1
sift[56:64] = .2
sift[64:72] = .3
sift[72:80] = .4
sift[80:88] = .5
sift[88:96] = .6
sift[96:104] = .7
sift[104:112] = .8
sift[112:120] = .9
sift[120:128] = 1
sift = sift / np.sqrt((sift ** 2).sum())
sift = np.round(sift * 255)
kpts = np.array([kp])
sifts = np.array([sift])
return kpts, sifts
def square_axis(ax, s=3):
ax.set_xlim(-s, s)
ax.set_ylim(-s, s)
ax.set_aspect('equal')
ax.invert_yaxis()
df2.set_xticks([])
df2.set_yticks([])
def test_shape(ori=0, skew=0, xscale=1, yscale=1, pnum=(1, 1, 1), fnum=1):
df2.figure(fnum=fnum, pnum=pnum)
kpts, sifts = test_keypoint(ori=ori, skew=skew, xscale=xscale, yscale=yscale)
ax = df2.gca()
square_axis(ax)
mpl_keypoint.draw_keypoints(ax, kpts, sifts=sifts, ell_color=df2.ORANGE, ori=True,
rect_color=df2.DARK_RED,
ori_color=df2.DEEP_PINK, eig_color=df2.PINK,
rect=True, eig=True, bin_color=df2.RED,
arm1_color=df2.YELLOW, arm2_color=df2.BLACK)
kptsstr = '\n'.join(ktool.get_kpts_strs(kpts))
#print(kptsstr)
df2.upperleft_text(kptsstr)
title = 'xyscale=(%.1f, %.1f),\n skew=%.1f, ori=%.2ftau' % (xscale, yscale, skew, ori / TAU)
df2.set_title(title)
df2.dark_background()
return kpts, sifts
np.set_printoptions(precision=3)
#THETA1 = DOWN
#THETA2 = (DOWN + DOWN + RIGHT) / 3
#THETA3 = (DOWN + RIGHT) / 2
#THETA4 = (DOWN + RIGHT + RIGHT) / 3
#THETA5 = RIGHT
nRows = 2
nCols = 4
import plottool as pt
#pnum_ = pt.pnum_generator(nRows, nCols).next
pnum_ = pt.pnum_generator(nRows, nCols)
#def pnum_(px=None):
# global px_
# if px is None:
# px_ += 1
# px = px_
# return (nRows, nCols, px)
MIN_ORI = ut.get_argval('--min-ori', float, DOWN)
MAX_ORI = ut.get_argval('--max-ori', float, DOWN + TAU - .2)
MIN_X = .5
MAX_X = 2
MIN_SWEW = ut.get_argval('--min-skew', float, 0)
MAX_SKEW = ut.get_argval('--max-skew', float, 1)
MIN_Y = .5
MAX_Y = 2
#kp_list = []
for row, col in iprod(range(nRows), range(nCols)):
#print((row, col))
alpha = col / (nCols)
beta = row / (nRows)
xsca = (MIN_X * (1 - alpha)) + (MAX_X * (alpha))
ori = (MIN_ORI * (1 - alpha)) + (MAX_ORI * (alpha))
skew = (MIN_SWEW * (1 - beta)) + (MAX_SKEW * (beta))
ysca = (MIN_Y * (1 - beta)) + (MAX_Y * (beta))
kpts, sifts = test_shape(pnum=six.next(pnum_),
ori=ori,
skew=skew,
xscale=xsca,
yscale=ysca)
def taq_cross_response_year_avg_responses_time_shift_plot(tickers, year,
shifts):
"""Plots the average of the cross-response average for a year for each
shift.
:param tickers: list of strings of the abbreviation of the stock to be
analized (i.e. ['AAPL', 'MSFT']).
:param year: string of the year to be analized (i.e '2008')
:param shifts: list of integers greater than zero (i.e. [1, 10, 50]).
:return: None -- The function saves the plot in a file and does not return
a value.
"""
ticker_couples = list(iprod(tickers, tickers))
try:
function_name = \
taq_cross_response_year_avg_responses_time_shift_plot.__name__
for shift in shifts:
figure = plt.figure(figsize=(16, 9))
avg_val = np.zeros(10000)
# Figure with the average of different stocks for the same shift
for couple in ticker_couples:
ticker_i = couple[0]
ticker_j = couple[1]
if (ticker_i == ticker_j):
# Self-response
pass
else:
# Load data
cross = pickle.load(open(''.join((
f'../../taq_data/responses_time_shift_data'
+ f'_{year}/taq_cross_response_year'
+ f'_responses_time_shift_data_shift'
+ f'_{shift}/taq_cross_response_year'
+ f'_responses_time_shift_data_shift'
+ f'_{shift}_{year}_{ticker_i}i'
+ f'_{ticker_j}j.pickle').split()), 'rb'))
plt.semilogx(cross, linewidth=3, alpha=0.1,
label=f'{ticker_i}-{ticker_j}')
avg_val += cross
plt.semilogx(avg_val / 30, linewidth=5,
label='Average')
plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.2), ncol=7,
fontsize=15)
plt.title(f'Cross-response - shift {shift}s', fontsize=40)
plt.xlabel(r'$\tau \, [s]$', fontsize=35)
plt.ylabel(r'$R_{ij}(\tau)$', fontsize=35)
plt.xticks(fontsize=25)
plt.yticks(fontsize=25)
plt.xlim(1, 10000)
# plt.ylim(4 * 10 ** -5, 9 * 10 ** -5)
plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
plt.grid(True)
plt.tight_layout()
# Plotting
plt.savefig(''.join((
f'../taq_plot/taq_data_plot_responses_time_shift_average/'
+ f'{function_name}_{shift}.png').split()))
return None
except FileNotFoundError as e:
print('No data')
print(e)
print()
return None
def observe(self, dict_in):
"""
Loads observation model parameters into a dictionary,
performs the forward model and provides an initial solution.
Args:
dict_in (dict): Dictionary which will be overwritten with
all of the observation model parameters, forward model
observation 'y', and initial estimate 'x_0'.
"""
warnings.simplefilter("ignore", np.ComplexWarning)
#########################################
#fetch observation model parameters here#
#########################################
if (self.str_type[:11] == 'convolution'
or self.str_type == 'compressed_sensing'):
wrf = self.get_val('wienerfactor', True)
str_domain = self.get_val('domain', False)
noise_pars = defaultdict(int) #build a dict to generate the noise
noise_pars['seed'] = self.get_val('seed', True)
noise_pars['variance'] = self.get_val('noisevariance', True)
noise_pars['distribution'] = self.get_val('noisedistribution',
False)
noise_pars['mean'] = self.get_val('noisemean', True)
noise_pars['interval'] = self.get_val('noiseinterval',
True) #uniform
noise_pars['size'] = dict_in['x'].shape
dict_in['noisevariance'] = noise_pars['variance']
if self.str_type == 'compressed_sensing':
noise_pars['complex_noise'] = 1
if dict_in['noisevariance'] > 0:
dict_in['n'] = noise_gen(noise_pars)
else:
dict_in['n'] = 0
elif self.str_type == 'classification':
#partition the classification dataset into an 'observed' training set
#and an unobserved evaluation/test set, and generate features
dict_in['x_train'] = {}
dict_in['x_test'] = {}
dict_in['y_label'] = {}
dict_in['x_feature'] = {}
dict_in['n_training_samples'] = 0
dict_in['n_testing_samples'] = 0
shuffle = self.get_val('shuffle', True)
if shuffle:
shuffleseed = self.get_val('shuffleseed', True)
training_proportion = self.get_val('trainingproportion', True)
classes = dict_in['x'].keys()
#partition and generate numeric class labels
for _class_index, _class in enumerate(classes):
class_size = len(dict_in['x'][_class])
training_size = int(training_proportion * class_size)
dict_in['n_training_samples'] += training_size
dict_in['n_testing_samples'] += class_size - training_size
if shuffle:
np.random.seed(shuffleseed)
indices = np.random.permutation(class_size)
else:
indices = np.array(range(class_size), dtype='uint16')
dict_in['x_train'][_class] = indices[:training_size]
dict_in['x_test'][_class] = indices[training_size:]
dict_in['y_label'][_class] = _class_index
else:
raise ValueError('unsupported observation model')
################################################
#compute the forward model and initial estimate#
################################################
if self.str_type == 'convolution':
H = self.Phi
H.set_output_fourier(False)
dict_in['Hx'] = H * dict_in['x']
dict_in['y'] = dict_in['Hx'] + dict_in['n']
#regularized Wiener filtering in Fourier domain
H.set_output_fourier(True)
dict_in['x_0'] = real(
ifftn(~H * dict_in['y'] /
(H.get_spectrum_sq() + wrf * noise_pars['variance'])))
# dict_in['x_0'] = real(ifftn(~H * dict_in['y'])) %testing only
H.set_output_fourier(False)
#compute bsnr
self.compute_bsnr(dict_in, noise_pars)
elif self.str_type == 'convolution_downsample':
Phi = self.Phi
#this order is important in the config file
D = Phi.ls_ops[1]
H = Phi.ls_ops[0]
H.set_output_fourier(False)
if self.get_val('spatialblur', True):
dict_in['Phix'] = D * convolve(dict_in['x'], H.kernel, 'same')
dict_in['Hxpn'] = convolve(dict_in['x'], H.kernel,
'same') + dict_in['n']
else:
dict_in['Phix'] = Phi * dict_in['x']
dict_in['Hxpn'] = H * dict_in['x'] + dict_in['n']
dict_in['Hx'] = dict_in['Phix']
#the version of y without downsampling
dict_in['DHxpn'] = np.zeros((D * dict_in['Hxpn']).shape)
if dict_in['n'].__class__.__name__ == 'ndarray':
dict_in['n'] = D * dict_in['n']
dict_in['y'] = dict_in['Hx'] + dict_in['n']
DH = fftn(Phi * nd_impulse(dict_in['x'].shape))
DHt = conj(DH)
Hty = fftn(D * (~Phi * dict_in['y']))
HtDtDH = np.real(DHt * DH)
# dict_in['x_0'] = ~D*real(ifftn(Hty /
# (HtDtDH +
# wrf * noise_pars['variance'])))
dict_in['x_0'] = ~D * dict_in['y']
#optional interpolation
xdim = dict_in['x'].ndim
xshp = dict_in['x'].shape
if self.get_val('interpinitialsolution', True):
if xdim == 2:
if self.get_val('useimresize', True):
interp_vals = imresize(
dict_in['y'],
tuple(D.ds_factor *
np.asarray(dict_in['y'].shape)),
interp='bicubic')
else:
grids = np.mgrid[[
slice(0, xshp[j]) for j in xrange(xdim)
]]
grids = tuple(
[grids[i] for i in xrange(grids.shape[0])])
sampled_coords = np.mgrid[[
slice(D.offset[j], xshp[j], D.ds_factor[j])
for j in xrange(xdim)
]]
values = dict_in['x_0'][[
coord.flatten() for coord in sampled_coords
]]
points = np.vstack([
sampled_coords[i, Ellipsis].flatten()
for i in xrange(sampled_coords.shape[0])
]).transpose() #pts to interp
interp_vals = griddata(points,
values,
grids,
method='cubic',
fill_value=0.0)
else:
values = dict_in[
'y'] #we're not using blank values, different interpolation scheme..
dsfactors = np.asarray(
[int(D.ds_factor[j]) for j in xrange(values.ndim)])
valshpcorrect = (
np.asarray(values.shape) -
np.asarray(xshp, dtype='uint16') / dsfactors)
valshpcorrect = valshpcorrect / np.asarray(dsfactors,
dtype='float32')
interp_coords = iprod(*[
np.arange(0, values.shape[j] - valshpcorrect[j], 1.0 /
D.ds_factor[j]) for j in xrange(values.ndim)
])
interp_coords = np.array([el for el in interp_coords
]).transpose()
interp_vals = map_coordinates(values,
interp_coords,
order=3,
mode='nearest').reshape(xshp)
# interp_vals = map_coordinates(values,interp_coords,order=3,mode='nearest')
# cut off the edges
# if xdim == 2:
# interp_vals = interp_vals[0:xshp[0],0:xshp[1]]
# else:
interp_vals = interp_vals[0:xshp[0], 0:xshp[1], 0:xshp[2]]
dict_in['x_0'] = interp_vals
elif self.get_val('inputinitialsoln', False) != '':
init_soln_inputsec = Input(
self.ps_parameters, self.get_val('inputinitialsoln',
False))
dict_in['x_0'] = init_soln_inputsec.read({}, True)
self.compute_bsnr(dict_in, noise_pars)
elif self.str_type == 'convolution_poisson':
dict_in['mp'] = self.get_val('maximumphotonspervoxel', True)
dict_in['b'] = self.get_val('background', True)
H = self.Phi
if str_domain == 'fourier':
H.set_output_fourier(False) #return spatial domain object
orig_shape = dict_in['x'].shape
Hspec = np.zeros(orig_shape)
dict_in['r'] = H * dict_in['x']
k = dict_in['mp'] / nmax(dict_in['r'])
dict_in['r'] = k * dict_in['r']
#normalize the output image to have the same
#maximum photon count as the ouput image
dict_in['x'] = k * dict_in['x']
dict_in['x'] = crop_center(
dict_in['x'], dict_in['r'].shape).astype('float32')
#the spatial domain measurements, before photon counts
dict_in['fb'] = dict_in['r'] + dict_in['b']
#lambda of the poisson distn
noise_pars['ary_mean'] = dict_in['fb']
#specifying the poisson distn
noise_distn2 = self.get_val('noisedistribution2', False)
noise_pars['distribution'] = noise_distn2
#generating quantized (uint16) poisson measurements
# dict_in['y'] = (noise_gen(noise_pars)+dict_in['n']).astype('uint16').astype('int32')
dict_in['y'] = noise_gen(noise_pars) + crop_center(
dict_in['n'], dict_in['fb'].shape)
dict_in['y'][dict_in['y'] < 0] = 0
elif str_domain == 'evaluation': #are given the observation, which is stored in 'x'
dict_in['y'] = dict_in.pop('x')
else:
raise Exception('domain not supported: ' + str_domain)
dict_in['x_0'] = ((~H) * (dict_in['y'])).astype(dtype='float32')
dict_in['y_padded'] = pad_center(dict_in['y'],
dict_in['x_0'].shape)
elif self.str_type == 'compressed_sensing':
Fu = self.Phi
dict_in['Hx'] = Fu * dict_in['x']
dict_in['y'] = dict_in['Hx'] + dict_in['n']
dict_in['x_0'] = (~Fu) * dict_in['y']
dict_in['theta_0'] = angle(dict_in['x_0'])
dict_in['theta_0'] = su.phase_unwrap(dict_in['theta_0'],
dict_in['dict_global_lims'],
dict_in['ls_local_lim_secs'])
dict_in['magnitude_0'] = nabs(dict_in['x_0'])
if self.get_val('maskinitialsoln', True):
dict_in['theta_0'] *= dict_in['mask']
dict_in['magnitude_0'] *= dict_in['mask']
dict_in['x_0'] = dict_in['magnitude_0'] * exp(
1j * dict_in['theta_0'])
self.compute_bsnr(dict_in, noise_pars)
#store the wavelet domain version of the ground truth
if np.iscomplexobj(dict_in['x']):
dict_in['w'] = [
self.W * dict_in['x'].real, self.W * dict_in['x'].imag
]
else:
dict_in['w'] = [self.W * dict_in['x']]
def epochs_aggregated_dist_returns_market_data(
dates: List[str], time_step: str, window: str, K_value: str, norm: str = "long"
) -> None:
"""Computes the aggregated distribution of returns for a market.
:param dates: List of the interval of dates to be analyzed
(i.e. ['1980-01-01', '2020-12-31']).
:param time_step: time step of the data (i.e. '1m', '1h', '1d', '1wk',
'1mo').
:param window: window time to compute the volatility (i.e. '25').
:param K_value: number of companies to be used (i.e. '80', 'all').
:norm: define if the normalization is made in the complete time series or
in each epoch. Default 'long', 'short' is the other option.
:return: None -- The function saves the data in a file and does not return
a value.
"""
function_name: str = epochs_aggregated_dist_returns_market_data.__name__
epochs_tools.function_header_print_data(
function_name, dates, time_step, window, K_value
)
try:
# Load name of the stocks
if K_value == "all":
stocks_name: pd.DataFrame = pickle.load(
open(
f"../data/epochs/returns_data_{dates[0]}_{dates[1]}_step"
+ f"_{time_step}_win__K_.pickle",
"rb",
)
)[:200]
else:
stocks_name = pickle.load(
open(
f"../data/epochs/returns_data_{dates[0]}_{dates[1]}_step"
+ f"_{time_step}_win__K_.pickle",
"rb",
)
).sample(n=int(K_value), axis="columns")
agg_ret_mkt_list: List[List[float]] = []
# Combination of stock pairs
stocks_comb: Iterable[Tuple[Any, ...]] = icomb(stocks_name, 2)
args_prod: Iterable[Any] = iprod(
[dates], [time_step], stocks_comb, [window], [norm]
)
# Parallel computing
with mp.Pool(processes=mp.cpu_count()) as pool:
agg_ret_mkt_list.extend(
pool.starmap(epochs_aggregated_dist_returns_pair_data, args_prod)
)
# Flatten the list
agg_ret_mkt_list_flat: List[float] = [
val for sublist in agg_ret_mkt_list for val in sublist
]
agg_ret_mkt_series: pd.Series = pd.Series(agg_ret_mkt_list_flat)
print(f"mean = {agg_ret_mkt_series.mean()}")
print(f"std = {agg_ret_mkt_series.std()}")
# Saving data
epochs_tools.save_data(
agg_ret_mkt_series,
function_name + "_" + norm,
dates,
time_step,
window,
K_value,
)
del agg_ret_mkt_list
del agg_ret_mkt_series
except FileNotFoundError as error:
print("No data")
print(error)
print()
MAX_ORI = utool.get_argval('--max-ori', float, DOWN + TAU - .2)
MIN_X = .5
MAX_X = 2
MIN_SWEW = utool.get_argval('--min-skew', float, 0)
MAX_SKEW = utool.get_argval('--max-skew', float, 1)
MIN_Y = .5
MAX_Y = 2
kp_list = []
if __name__ == '__main__':
for row, col in iprod(range(nRows), range(nCols)):
#print((row, col))
alpha = col / (nCols)
beta = row / (nRows)
xsca = (MIN_X * (1 - alpha)) + (MAX_X * (alpha))
ori = (MIN_ORI * (1 - alpha)) + (MAX_ORI * (alpha))
skew = (MIN_SWEW * (1 - beta)) + (MAX_SKEW * (beta))
ysca = (MIN_Y * (1 - beta)) + (MAX_Y * (beta))
kpts, sifts = test_shape(pnum=pnum_(),
ori=ori,
skew=skew,
xscale=xsca,
yscale=ysca)
#print('+----')
#kp_list.append(kpts[0])
def ranges(*ls):
return iprod(*[rang(l) for l in ls])