def _format_aggregations(self, aggregations, i_r=(None, None)): """Prepare and add aggregations to retrievers and features. Parameters ---------- aggregations: tuple the aggregation information. i_r: tuple the indices of retriever and features to use in the aggregation. """ if aggregations is None: return if type(aggregations) == list: for i in range(len(aggregations)): self._format_aggregations(aggregations[i], i_r) if type(aggregations) == tuple: ## Prepare instructions i_ret = i_r[0] i_ret = range(len(self.retrievers)) if i_ret is None else i_ret i_ret = [i_ret] if type(i_ret) != list else i_ret i_feat = i_r[1] i_feat = range(len(self.featurers)) if i_feat is None else i_feat i_feat = [i_feat] * len(i_ret) if type(i_feat) != list else i_feat ## Assert correctness assert len(i_ret) == len(i_feat) ## Main loop for i in range(len(i_ret)): ## Preparing information to retriever number i_ret ret = self.retrievers.retrievers[i_ret[i]] agg_0 = _discretization_information_creation(aggregations[0], ret) aggregations_i = tuple([agg_0] + list(aggregations[1:])) # Add aggregation to retrievers new_ret = create_aggretriever(aggregations_i) self.retrievers.add_aggregations(new_ret) # Add aggregations to features i_feat_i = [i_feat[i]] if type(i_feat[i]) == int else i_feat[i] for j in i_feat_i: new_features = create_aggfeatures(aggregations_i, self.featurers.features[j]) self.featurers.add_aggregations(new_features)
def test(): n, nx, ny = 100, 100, 100 m, rei = 3, 5 locs = np.random.random((n, 2))*10 ## Retrievers management ret0 = KRetriever(locs, 3, ifdistance=True) ret1 = CircRetriever(locs, .3, ifdistance=True) #countdesc = CountDescriptor() ## Other functions def map_indices(s, i): if s._pos_inputs is not None: return s._pos_inputs.start + s._pos_inputs.step*i else: return i def halting_f(signum, frame): raise Exception("Not error time.") ## Random exploration functions def random_pos_space_exploration(pos_possibles): selected, indices = [], [] for i in range(len(pos_possibles)): sel, ind = random_pos_exploration(pos_possibles[i]) selected.append(sel) indices.append(ind) return selected, indices def random_pos_exploration(possibles): ## Selection i_pos = np.random.randint(0, len(possibles)) return possibles[i_pos], i_pos ## Impossibles def impossible_instantiation(selected, p, ret, feat): i_ret, sel, agg, pert = p p_ind, m_ind, n_desc, i_feat = selected checker = False ## Not implemented perturbation over explicit features if pert is not None: if type(feat) == np.ndarray: if len(feat.shape) == 3: checker = True elif isinstance(feat, ExplicitFeatures): checker = True elif isinstance(feat, FeaturesManager): check_aux = [] for i in range(len(feat.features)): check_aux.append(isinstance(feat.features[i], ExplicitFeatures)) checker = any(check_aux) return checker def compulsary_instantiation_errors(selected, p, ret, feat): i_ret, sel, agg, pert = p p_ind, m_ind, n_desc, i_feat = selected checker = False ## Cases if p_ind == []: checker = True ## Compulsary failing instantiation if not checker: return try: boolean = False SpatialDescriptorModel(retrievers=ret, featurers=feat, mapselector_spdescriptor=sel, pos_inputs=p_ind, map_indices=m_ind, perturbations=pert, aggregations=agg, name_desc=n_desc) boolean = True except: if boolean: raise Exception("It has to halt here.") return checker def test_methods(methods, input_): """Test proper methods output for selectors indications.""" # print methods, input_ assert(len(methods) == 3) assert(methods[0] in [True, False]) if methods[1] is None: assert(methods[2] is None) elif type(input_) == int: assert(type(methods[1]) == tuple) assert(type(methods[2]) == tuple) assert(len(methods[1]) == 2) assert(len(methods[2]) == 3) assert(all([len(e) == 2 for e in methods[2]])) assert(all([type(e) == tuple for e in methods[2]])) else: assert(type(input_) == list) assert(type(methods[1]) == list) assert(type(methods[2]) == list) assert(len(methods[1]) == len(input_)) assert(len(methods[2]) == len(input_)) for i in range(len(methods[1])): assert(type(methods[1][i]) == tuple) assert(type(methods[2][i]) == tuple) assert(len(methods[1][i]) == 2) assert(len(methods[2][i]) == 3) assert(all([len(e) == 2 for e in methods[2][i]])) assert(all([type(e) == tuple for e in methods[2][i]])) ########################################################################### ########################################################################### ######## Testing aggregations preparation ## Testing all possible aggregation_in agg_f_ret = None desc_in, desc_out = AvgDescriptor(), AvgDescriptor() feats = ImplicitFeatures(np.random.random((100, 10)), descriptormodel=AvgDescriptor()) agg_in = agg_f_ret, desc_in, {}, {}, desc_out res = _parse_aggregation_feat(agg_in, feats) assert(type(res) == tuple) assert(len(res) == 5) agg_in = agg_f_ret, desc_in, {}, {} res = _parse_aggregation_feat(agg_in, feats) assert(type(res) == tuple) assert(len(res) == 5) agg_in = agg_f_ret, {}, {} res = _parse_aggregation_feat(agg_in, feats) assert(type(res) == tuple) assert(len(res) == 5) agg_in = agg_f_ret, desc_in, desc_out res = _parse_aggregation_feat(agg_in, feats) assert(type(res) == tuple) assert(len(res) == 5) agg_in = (agg_f_ret, ) res = _parse_aggregation_feat(agg_in, feats) assert(type(res) == tuple) assert(len(res) == 5) # Creation standard aggregation_info disc = GridSpatialDisc((5, 5), xlim=(0, 1), ylim=(0, 1)) locs = np.random.random((100, 2)) regs = disc.discretize(locs) disc_info = locs, regs, disc retriever_in = (KRetriever, {'info_ret': 4}) retriever_out = (KRetriever, {'info_ret': 4}) aggregating = avgregionlocs_outretriever, (avgregionlocs_outretriever, ) aggregation_info = disc_info, retriever_in, retriever_out, aggregating # Creation of aggregation objects aggretriever = create_aggretriever(aggregation_info) assert(isinstance(aggretriever, BaseRetriever)) aggfeatures = create_aggfeatures(aggregation_info, feats) assert(isinstance(aggfeatures, BaseFeatures)) ########################################################################### ########################################################################### ######## Testing instantiation spdesc ## TODO: bool_input_idx=False # Aggregation disc = GridSpatialDisc((5, 5), xlim=(0, 1), ylim=(0, 1)) retriever_in = (KRetriever, {'info_ret': 4}) retriever_out = (KRetriever, {'info_ret': 4}) aggregating = avgregionlocs_outretriever, (avgregionlocs_outretriever, ) aggregation_info = disc, retriever_in, retriever_out, aggregating # Locs and retrievers n_in, n_out = 50, 50 # TODO: Different sizes and easy manage locs_input = np.random.random((n_in, 2)) locs1 = np.random.random((n_out, 2)) locs2 = np.random.random((n_out, 2)) # Features aggfeats = np.random.random((n_out, m, rei)) featsarr0 = np.random.random((n_out, m)) featsarr1 = np.random.random((n_out, m)) featsarr2 = np.vstack([np.random.randint(0, 10, n_out) for i in range(m)]).T def new_retrievers_creation(): ret0 = KRetriever(locs1, autolocs=locs_input, info_ret=3, bool_input_idx=True) ret1 = [ret0, CircRetriever(locs2, info_ret=0.1, autolocs=locs_input, bool_input_idx=True)] ret2 = RetrieverManager(ret0) pos_rets = [ret0, ret1, ret2] return pos_rets pos_rets = range(3) ## Possible feats def new_features_creation(): feats0 = ExplicitFeatures(aggfeats) feats1 = ImplicitFeatures(featsarr0) feats2 = FeaturesManager(ExplicitFeatures(aggfeats)) pos_feats = [feats0, feats1, aggfeats, featsarr0, feats2] return pos_feats pos_feats = range(5) # Selectors arrayselector0 = np.zeros((n_in, 8)) arrayselector1 = np.zeros((n_in, 2)), np.zeros((n_in, 6)) arrayselector2 = np.zeros((n_in, 2)), tuple([np.zeros((n_in, 2))]*3) functselector0 = lambda idx: ((0, 0), ((0, 0), (0, 0), (0, 0))) functselector1 = lambda idx: (0, 0), lambda idx: ((0, 0), (0, 0), (0, 0)) tupleselector0 = (0, 0), (0, 0, 0, 0, 0, 0) tupleselector1 = (0, 0, 0, 0, 0, 0, 0, 0) tupleselector2 = (0, 0), ((0, 0), (0, 0), (0, 0)) listselector = None selobj = Sp_DescriptorSelector(*arrayselector1) pos_selectors = [None, arrayselector0, arrayselector1, arrayselector2, functselector0, functselector1, tupleselector0, tupleselector1, tupleselector2, Sp_DescriptorSelector(arrayselector0)] pos_agg = [None] ## Perturbations reindices0 = np.arange(n_out) reindices = np.vstack([reindices0]+[np.random.permutation(n_out) for i in range(rei-1)]).T perturbation = PermutationPerturbation(reindices) pos_pert = [None, perturbation] ## Random exploration pos_loop_ind = [None, 20, (0, n_in, 1), slice(0, n_in, 1), []] pos_loop_mapin = [None, map_indices] pos_name_desc = [None, '', 'random_desc'] # Possible feats # Random exploration possibilities pos_random = [pos_loop_ind, pos_loop_mapin, pos_name_desc, pos_feats] possibilities = [pos_rets, pos_selectors, pos_agg, pos_pert] s = 0 for p in product(*possibilities): i_ret, sel, agg, pert = p ## Random exploration of parameters selected, indices = random_pos_space_exploration(pos_random) p_ind, m_ind, n_desc, i_feat = selected ## Classes renewal rets_cand = new_retrievers_creation() feats_cand = new_features_creation() # Retrievers ret = rets_cand[i_ret] feat = feats_cand[i_feat] # print indices # print p, selected ## Impossible cases checker1 = impossible_instantiation(selected, p, ret, feat) checker2 = compulsary_instantiation_errors(selected, p, ret, feat) if checker1 or checker2: continue ## Testing instantiation spdesc = SpatialDescriptorModel(retrievers=ret, featurers=feat, mapselector_spdescriptor=sel, pos_inputs=p_ind, map_indices=m_ind, perturbations=pert, aggregations=agg, name_desc=n_desc) # print s #### Function testing ## Auxiliar functions spdesc.add_perturbations(pert) spdesc.set_loop(p_ind, m_ind) spdesc._map_indices(spdesc, 0) for i in spdesc.iter_indices(): methods = spdesc._get_methods(i) test_methods(methods, i) methods = spdesc._get_methods(0) test_methods(methods, 0) methods = spdesc._get_methods(10) test_methods(methods, 10) methods = spdesc._get_methods([0]) test_methods(methods, [0]) methods = spdesc._get_methods([0, 1, 2]) test_methods(methods, [0, 1, 2]) desc = spdesc._compute_descriptors(10) desc = spdesc._compute_descriptors([10]) desc = spdesc._compute_descriptors([0, 1, 2]) desc = spdesc.compute(10) desc = spdesc.compute([10]) desc = spdesc.compute([0, 1, 2]) #Retrieverdriven aux_i = 0 for desc_i, vals_i in spdesc.compute_nets_i(): assert(len(desc_i) == len(vals_i)) assert(len(desc_i) == spdesc.featurers.k_perturb+1) aux_i += 1 if aux_i == 100: break aux_i = 0 for desc_ik, vals_ik in spdesc.compute_net_ik(): aux_i += 1 if aux_i == 100: break ## Loops # for idx in spdesc.iter_indices(): # break # for vals_ik, desc_ik in spdesc.compute_net_ik(): # #assert(vals_ik) # #assert(desc_ik) # break # for desc_i, vals_i in spdesc.compute_net_i(): # #assert(vals_ik) # #assert(desc_ik) # break ## Global computations # try: # signal.signal(signal.SIGALRM, halting_f) # signal.alarm(0.01) # 0.01 seconds # spdesc.compute() # except Exception as e: # logi = e == "Not error time." # if not logi: # spdesc.compute() # try: # signal.signal(signal.SIGALRM, halting_f) # signal.alarm(0.01) # 0.01 seconds # spdesc._compute_nets() # except Exception as e: # logi = e == "Not error time." # if not logi: # spdesc._compute_nets() # try: # ## Testing compute_retdriven # signal.signal(signal.SIGALRM, halting_f) # signal.alarm(0.01) # 0.01 seconds # spdesc._compute_retdriven() # except Exception as e: # logi = e == "Not error time." # if not logi: # spdesc._compute_retdriven() # try: # logfile = Logger('logfile.log') # signal.signal(signal.SIGALRM, halting_f) # signal.alarm(0.01) # 0.01 seconds # spdesc.compute_process(logfile, lim_rows=100000, n_procs=0) # os.remove('logfile.log') # except Exception as e: # os.remove('logfile.log') # logi = e == "Not error time." # if not logi: # spdesc.compute_process(logfile, lim_rows=100000, n_procs=0) ## Testing aggregations if len(spdesc.retrievers) == len(spdesc.featurers): spdesc.add_aggregations(aggregation_info) else: spdesc.add_aggregations(aggregation_info, ([0], [0])) s += 1 feats1 = ImplicitFeatures(featsarr0) m_vals_i = np.random.randint(0, 5, 50) ret = CircRetriever(locs1, autolocs=locs_input, info_ret=3, bool_input_idx=True) feat = FeaturesManager(feats1, maps_vals_i=m_vals_i, mode='sequential', descriptormodels=None) spdesc = SpatialDescriptorModel(retrievers=ret, featurers=feat, mapselector_spdescriptor=None, perturbations=perturbation, aggregations=None, name_desc=n_desc) ## Complete processes spdesc.compute() logfile = Logger('logfile.log') spdesc.compute_process(logfile, lim_rows=100000, n_procs=0) os.remove('logfile.log') spdesc._compute_nets() spdesc._compute_retdriven() ## Model functions spdesc.fit(np.arange(20), np.random.random(20)) spdesc.predict(np.arange(20)) ############ ### Auxiliar functions #### spdesc = _spdesc_parsing_creation(ret, feat) assert(isinstance(spdesc, SpatialDescriptorModel)) res = create_aggfeatures(spdesc, None) assert(isinstance(res, ExplicitFeatures)) ########################################################################### ########################################################################### spdesc_temp = SpatioTemporalDescriptorModel(spdesc) indices = np.arange(10) y = np.random.random(10) spdesc_temp = spdesc_temp.fit(indices, y) spdesc_temp.predict(indices) spdesc_temp = SpatioTemporalDescriptorModel([spdesc, spdesc]) indices = np.arange(20) y = np.random.random(20) spdesc_temp = spdesc_temp.fit(indices, y) spdesc_temp.predict(indices)