if __name__ == "__main__":
import time
t0 = time.time()
## Artificial 1dim random time series
shape = (2000, )
ts = np.random.random(shape)
## Computing measure by binning
pars_ret, nbins = {'l': 8, 'center': 0, 'excluded': False}, 5
windret = WindowsRetriever(shape, pars_ret)
binsdesc = NBinsHistogramDesc(nbins)
cat_ts = binsdesc.set_global_info(ts, transform=True)
gret = RetrieverManager(windret)
feats_ret = FeaturesManager(ImplicitFeatures(cat_ts, out_type='ndarray',
descriptormodel=binsdesc),
maps_vals_i=cat_ts)
# feats_ret = FeaturesManager(cat_ts, descriptormodels=binsdesc,
# maps_vals_i=cat_ts)
spdesc = SpatialDescriptorModel(gret, feats_ret)
net = spdesc.compute()
# Compare with the expected result
try:
np.testing.assert_allclose(net.sum(1)/net.sum(), 1./nbins, atol=0.03)
except AssertionError as e:
print(e)
print time.time()-t0
# Interpolate to points
t_int = np.arange(0, np.floor(t_s[-1]+0.5))
nt_new = len(t_int)
# Retrievers
ret0 = KRetriever(locs=t_s.reshape((n_t, 1)), info_ret=1,
autolocs=t_int.reshape((len(t_int), 1)), ifdistance=True)
ret1 = CircRetriever(locs=t_s.reshape((n_t, 1)), info_ret=.9,
autolocs=t_int.reshape((len(t_int), 1)),
ifdistance=True)
## Easy interpolation using scipy
vals_nn = griddata(t_s.reshape((n_t, 1)), values.reshape((n_t, 1)),
t_int.reshape((len(t_int), 1)), fill_value=0.).ravel()
## Using pySpatialTools framework
interpolator = Interpolator('null', {}, 'null', {})
feats = ImplicitFeatures(values.reshape((len(values), 1)),
descriptormodel=interpolator, out_type='ndarray')
feats_ret = FeaturesManager(feats, maps_vals_i=('matrix', nt_new, nt_new))
interpolation = SpatialDescriptorModel(ret0, feats_ret)
vals_nn_new = interpolation.compute().ravel()
## Plot interpolation
plt.plot(t_s, values, label='original')
plt.plot(range(nt_new), vals_nn, label='scipy interpolation')
plt.plot(range(nt_new), vals_nn_new, label='pst 1-nn interpolation')
legend = plt.legend(loc='upper right', shadow=True)
plt.show()
# interpolator = Interpolator('gaussian', pars_w[1], f_dens1, pars_d)
def relative_descriptors(self, i, neighs_info, desc_i, desc_neigh, vals_i):
"General default relative descriptors."
descriptors = desc_i[:]
ind_life = desc_i.ravel().astype(bool)
ind_dead = np.logical_not(desc_i.ravel())
descriptors[ind_life] = np.logical_or(desc_neigh[ind_life] == 2,
desc_neigh[ind_life] == 3)
descriptors[ind_dead] = desc_neigh[ind_dead] == 3
return descriptors
if __name__ == "__main__":
## Initial variables
nx, ny, nt = 10, 10, 100
pars_ret = {'l': 3, 'center': 0, 'excluded': True}
## Initialization
initial_state = np.random.randint(0, 2, nx*ny)
windret = WindowsRetriever((nx, ny), pars_ret=pars_ret)
gret = RetrieverManager(windret)
conw_ev = ConwayEvolution()
## Evolution
state = initial_state[:]
m_vals_i = create_mapper_vals_i(('matrix', len(state), len(state)))
for i in range(nt):
state =\
FeaturesManager(ImplicitFeatures(state, descriptormodel=conw_ev),
maps_vals_i=m_vals_i)
spdesc = SpatialDescriptorModel(gret, state)
state = spdesc.compute()[:, :, 0].astype(int)
# Retrievers
perturbations = JitterLocations(0.001, 250)
ret0 = KRetriever(locs=n_points, info_ret=nclass*nneighs,
autolocs=points, ifdistance=False,
perturbations=perturbations)
ret1 = CircRetriever(locs=n_points, info_ret=0.025,
autolocs=points, ifdistance=False,
perturbations=perturbations)
############### WARNING: TOTEST (not shape)
# Features and descriptor
desc = CountDescriptor()
names = [str(e) for e in range(nclass)]
feats = ImplicitFeatures(classes, descriptormodel=desc, # names=names,
perturbations=perturbations)
feats_ret = FeaturesManager(feats, maps_vals_i=np.zeros((len(points), 1)))
measurer = SpatialDescriptorModel(ret0, feats_ret)
measure = measurer.compute()
###########################################################################
######### Statistical tests
## Detect which one is the more stable under jitter and permutation
new_points = np.vstack([n_points, points])
new_classes = np.hstack([classes, np.ones(len(points))*nclass])
# Perturbations
jitter_perturbs = JitterLocations(0.001, 250)
perm_perturbs = PermutationPerturbation((len(new_points), 250))
## 1st test (jitter) [TODO: error heterogeneous shape retrievers]
# a) using K-neighs neighborhood
ret0 = KRetriever(locs=new_points[:], info_ret=nclass*nneighs,
def test():
n, nx, ny = 100, 100, 100
m, rei = 3, 5
locs = np.random.random((n, 2))*10
## Retrievers management
ret0 = KRetriever(locs, 3, ifdistance=True)
ret1 = CircRetriever(locs, .3, ifdistance=True)
#countdesc = CountDescriptor()
## Other functions
def map_indices(s, i):
if s._pos_inputs is not None:
return s._pos_inputs.start + s._pos_inputs.step*i
else:
return i
def halting_f(signum, frame):
raise Exception("Not error time.")
## Random exploration functions
def random_pos_space_exploration(pos_possibles):
selected, indices = [], []
for i in range(len(pos_possibles)):
sel, ind = random_pos_exploration(pos_possibles[i])
selected.append(sel)
indices.append(ind)
return selected, indices
def random_pos_exploration(possibles):
## Selection
i_pos = np.random.randint(0, len(possibles))
return possibles[i_pos], i_pos
## Impossibles
def impossible_instantiation(selected, p, ret, feat):
i_ret, sel, agg, pert = p
p_ind, m_ind, n_desc, i_feat = selected
checker = False
## Not implemented perturbation over explicit features
if pert is not None:
if type(feat) == np.ndarray:
if len(feat.shape) == 3:
checker = True
elif isinstance(feat, ExplicitFeatures):
checker = True
elif isinstance(feat, FeaturesManager):
check_aux = []
for i in range(len(feat.features)):
check_aux.append(isinstance(feat.features[i],
ExplicitFeatures))
checker = any(check_aux)
return checker
def compulsary_instantiation_errors(selected, p, ret, feat):
i_ret, sel, agg, pert = p
p_ind, m_ind, n_desc, i_feat = selected
checker = False
## Cases
if p_ind == []:
checker = True
## Compulsary failing instantiation
if not checker:
return
try:
boolean = False
SpatialDescriptorModel(retrievers=ret, featurers=feat,
mapselector_spdescriptor=sel,
pos_inputs=p_ind, map_indices=m_ind,
perturbations=pert, aggregations=agg,
name_desc=n_desc)
boolean = True
except:
if boolean:
raise Exception("It has to halt here.")
return checker
def test_methods(methods, input_):
"""Test proper methods output for selectors indications."""
# print methods, input_
assert(len(methods) == 3)
assert(methods[0] in [True, False])
if methods[1] is None:
assert(methods[2] is None)
elif type(input_) == int:
assert(type(methods[1]) == tuple)
assert(type(methods[2]) == tuple)
assert(len(methods[1]) == 2)
assert(len(methods[2]) == 3)
assert(all([len(e) == 2 for e in methods[2]]))
assert(all([type(e) == tuple for e in methods[2]]))
else:
assert(type(input_) == list)
assert(type(methods[1]) == list)
assert(type(methods[2]) == list)
assert(len(methods[1]) == len(input_))
assert(len(methods[2]) == len(input_))
for i in range(len(methods[1])):
assert(type(methods[1][i]) == tuple)
assert(type(methods[2][i]) == tuple)
assert(len(methods[1][i]) == 2)
assert(len(methods[2][i]) == 3)
assert(all([len(e) == 2 for e in methods[2][i]]))
assert(all([type(e) == tuple for e in methods[2][i]]))
###########################################################################
###########################################################################
######## Testing aggregations preparation
## Testing all possible aggregation_in
agg_f_ret = None
desc_in, desc_out = AvgDescriptor(), AvgDescriptor()
feats = ImplicitFeatures(np.random.random((100, 10)),
descriptormodel=AvgDescriptor())
agg_in = agg_f_ret, desc_in, {}, {}, desc_out
res = _parse_aggregation_feat(agg_in, feats)
assert(type(res) == tuple)
assert(len(res) == 5)
agg_in = agg_f_ret, desc_in, {}, {}
res = _parse_aggregation_feat(agg_in, feats)
assert(type(res) == tuple)
assert(len(res) == 5)
agg_in = agg_f_ret, {}, {}
res = _parse_aggregation_feat(agg_in, feats)
assert(type(res) == tuple)
assert(len(res) == 5)
agg_in = agg_f_ret, desc_in, desc_out
res = _parse_aggregation_feat(agg_in, feats)
assert(type(res) == tuple)
assert(len(res) == 5)
agg_in = (agg_f_ret, )
res = _parse_aggregation_feat(agg_in, feats)
assert(type(res) == tuple)
assert(len(res) == 5)
# Creation standard aggregation_info
disc = GridSpatialDisc((5, 5), xlim=(0, 1), ylim=(0, 1))
locs = np.random.random((100, 2))
regs = disc.discretize(locs)
disc_info = locs, regs, disc
retriever_in = (KRetriever, {'info_ret': 4})
retriever_out = (KRetriever, {'info_ret': 4})
aggregating = avgregionlocs_outretriever, (avgregionlocs_outretriever, )
aggregation_info = disc_info, retriever_in, retriever_out, aggregating
# Creation of aggregation objects
aggretriever = create_aggretriever(aggregation_info)
assert(isinstance(aggretriever, BaseRetriever))
aggfeatures = create_aggfeatures(aggregation_info, feats)
assert(isinstance(aggfeatures, BaseFeatures))
###########################################################################
###########################################################################
######## Testing instantiation spdesc
## TODO: bool_input_idx=False
# Aggregation
disc = GridSpatialDisc((5, 5), xlim=(0, 1), ylim=(0, 1))
retriever_in = (KRetriever, {'info_ret': 4})
retriever_out = (KRetriever, {'info_ret': 4})
aggregating = avgregionlocs_outretriever, (avgregionlocs_outretriever, )
aggregation_info = disc, retriever_in, retriever_out, aggregating
# Locs and retrievers
n_in, n_out = 50, 50 # TODO: Different sizes and easy manage
locs_input = np.random.random((n_in, 2))
locs1 = np.random.random((n_out, 2))
locs2 = np.random.random((n_out, 2))
# Features
aggfeats = np.random.random((n_out, m, rei))
featsarr0 = np.random.random((n_out, m))
featsarr1 = np.random.random((n_out, m))
featsarr2 = np.vstack([np.random.randint(0, 10, n_out)
for i in range(m)]).T
def new_retrievers_creation():
ret0 = KRetriever(locs1, autolocs=locs_input, info_ret=3,
bool_input_idx=True)
ret1 = [ret0, CircRetriever(locs2, info_ret=0.1, autolocs=locs_input,
bool_input_idx=True)]
ret2 = RetrieverManager(ret0)
pos_rets = [ret0, ret1, ret2]
return pos_rets
pos_rets = range(3)
## Possible feats
def new_features_creation():
feats0 = ExplicitFeatures(aggfeats)
feats1 = ImplicitFeatures(featsarr0)
feats2 = FeaturesManager(ExplicitFeatures(aggfeats))
pos_feats = [feats0, feats1, aggfeats, featsarr0, feats2]
return pos_feats
pos_feats = range(5)
# Selectors
arrayselector0 = np.zeros((n_in, 8))
arrayselector1 = np.zeros((n_in, 2)), np.zeros((n_in, 6))
arrayselector2 = np.zeros((n_in, 2)), tuple([np.zeros((n_in, 2))]*3)
functselector0 = lambda idx: ((0, 0), ((0, 0), (0, 0), (0, 0)))
functselector1 = lambda idx: (0, 0), lambda idx: ((0, 0), (0, 0), (0, 0))
tupleselector0 = (0, 0), (0, 0, 0, 0, 0, 0)
tupleselector1 = (0, 0, 0, 0, 0, 0, 0, 0)
tupleselector2 = (0, 0), ((0, 0), (0, 0), (0, 0))
listselector = None
selobj = Sp_DescriptorSelector(*arrayselector1)
pos_selectors = [None, arrayselector0, arrayselector1, arrayselector2,
functselector0, functselector1,
tupleselector0, tupleselector1, tupleselector2,
Sp_DescriptorSelector(arrayselector0)]
pos_agg = [None]
## Perturbations
reindices0 = np.arange(n_out)
reindices = np.vstack([reindices0]+[np.random.permutation(n_out)
for i in range(rei-1)]).T
perturbation = PermutationPerturbation(reindices)
pos_pert = [None, perturbation]
## Random exploration
pos_loop_ind = [None, 20, (0, n_in, 1), slice(0, n_in, 1), []]
pos_loop_mapin = [None, map_indices]
pos_name_desc = [None, '', 'random_desc']
# Possible feats
# Random exploration possibilities
pos_random = [pos_loop_ind, pos_loop_mapin, pos_name_desc, pos_feats]
possibilities = [pos_rets, pos_selectors, pos_agg, pos_pert]
s = 0
for p in product(*possibilities):
i_ret, sel, agg, pert = p
## Random exploration of parameters
selected, indices = random_pos_space_exploration(pos_random)
p_ind, m_ind, n_desc, i_feat = selected
## Classes renewal
rets_cand = new_retrievers_creation()
feats_cand = new_features_creation()
# Retrievers
ret = rets_cand[i_ret]
feat = feats_cand[i_feat]
# print indices
# print p, selected
## Impossible cases
checker1 = impossible_instantiation(selected, p, ret, feat)
checker2 = compulsary_instantiation_errors(selected, p, ret, feat)
if checker1 or checker2:
continue
## Testing instantiation
spdesc = SpatialDescriptorModel(retrievers=ret, featurers=feat,
mapselector_spdescriptor=sel,
pos_inputs=p_ind, map_indices=m_ind,
perturbations=pert, aggregations=agg,
name_desc=n_desc)
# print s
#### Function testing
## Auxiliar functions
spdesc.add_perturbations(pert)
spdesc.set_loop(p_ind, m_ind)
spdesc._map_indices(spdesc, 0)
for i in spdesc.iter_indices():
methods = spdesc._get_methods(i)
test_methods(methods, i)
methods = spdesc._get_methods(0)
test_methods(methods, 0)
methods = spdesc._get_methods(10)
test_methods(methods, 10)
methods = spdesc._get_methods([0])
test_methods(methods, [0])
methods = spdesc._get_methods([0, 1, 2])
test_methods(methods, [0, 1, 2])
desc = spdesc._compute_descriptors(10)
desc = spdesc._compute_descriptors([10])
desc = spdesc._compute_descriptors([0, 1, 2])
desc = spdesc.compute(10)
desc = spdesc.compute([10])
desc = spdesc.compute([0, 1, 2])
#Retrieverdriven
aux_i = 0
for desc_i, vals_i in spdesc.compute_nets_i():
assert(len(desc_i) == len(vals_i))
assert(len(desc_i) == spdesc.featurers.k_perturb+1)
aux_i += 1
if aux_i == 100:
break
aux_i = 0
for desc_ik, vals_ik in spdesc.compute_net_ik():
aux_i += 1
if aux_i == 100:
break
## Loops
# for idx in spdesc.iter_indices():
# break
# for vals_ik, desc_ik in spdesc.compute_net_ik():
# #assert(vals_ik)
# #assert(desc_ik)
# break
# for desc_i, vals_i in spdesc.compute_net_i():
# #assert(vals_ik)
# #assert(desc_ik)
# break
## Global computations
# try:
# signal.signal(signal.SIGALRM, halting_f)
# signal.alarm(0.01) # 0.01 seconds
# spdesc.compute()
# except Exception as e:
# logi = e == "Not error time."
# if not logi:
# spdesc.compute()
# try:
# signal.signal(signal.SIGALRM, halting_f)
# signal.alarm(0.01) # 0.01 seconds
# spdesc._compute_nets()
# except Exception as e:
# logi = e == "Not error time."
# if not logi:
# spdesc._compute_nets()
# try:
# ## Testing compute_retdriven
# signal.signal(signal.SIGALRM, halting_f)
# signal.alarm(0.01) # 0.01 seconds
# spdesc._compute_retdriven()
# except Exception as e:
# logi = e == "Not error time."
# if not logi:
# spdesc._compute_retdriven()
# try:
# logfile = Logger('logfile.log')
# signal.signal(signal.SIGALRM, halting_f)
# signal.alarm(0.01) # 0.01 seconds
# spdesc.compute_process(logfile, lim_rows=100000, n_procs=0)
# os.remove('logfile.log')
# except Exception as e:
# os.remove('logfile.log')
# logi = e == "Not error time."
# if not logi:
# spdesc.compute_process(logfile, lim_rows=100000, n_procs=0)
## Testing aggregations
if len(spdesc.retrievers) == len(spdesc.featurers):
spdesc.add_aggregations(aggregation_info)
else:
spdesc.add_aggregations(aggregation_info, ([0], [0]))
s += 1
feats1 = ImplicitFeatures(featsarr0)
m_vals_i = np.random.randint(0, 5, 50)
ret = CircRetriever(locs1, autolocs=locs_input, info_ret=3,
bool_input_idx=True)
feat = FeaturesManager(feats1, maps_vals_i=m_vals_i, mode='sequential',
descriptormodels=None)
spdesc = SpatialDescriptorModel(retrievers=ret, featurers=feat,
mapselector_spdescriptor=None,
perturbations=perturbation,
aggregations=None, name_desc=n_desc)
## Complete processes
spdesc.compute()
logfile = Logger('logfile.log')
spdesc.compute_process(logfile, lim_rows=100000, n_procs=0)
os.remove('logfile.log')
spdesc._compute_nets()
spdesc._compute_retdriven()
## Model functions
spdesc.fit(np.arange(20), np.random.random(20))
spdesc.predict(np.arange(20))
############
### Auxiliar functions
####
spdesc = _spdesc_parsing_creation(ret, feat)
assert(isinstance(spdesc, SpatialDescriptorModel))
res = create_aggfeatures(spdesc, None)
assert(isinstance(res, ExplicitFeatures))
###########################################################################
###########################################################################
spdesc_temp = SpatioTemporalDescriptorModel(spdesc)
indices = np.arange(10)
y = np.random.random(10)
spdesc_temp = spdesc_temp.fit(indices, y)
spdesc_temp.predict(indices)
spdesc_temp = SpatioTemporalDescriptorModel([spdesc, spdesc])
indices = np.arange(20)
y = np.random.random(20)
spdesc_temp = spdesc_temp.fit(indices, y)
spdesc_temp.predict(indices)
