def test_randomized(self): for i in range(4): print(i) size = 257 d1 = np.empty([size, size]) d2 = np.empty(size) for i, j in np.ndindex((size, size)): d1[i, j] = random.randrange(1, 100) for i in range(size): d2[i] = random.randrange(1, 100) start = dt() np_solution = np.linalg.solve(d1, d2) print("Numpy solution lasted {}.".format(dt() - start)) print(str(np_solution[:100])) start = dt() my_solution = solve(d1, d2) print("My solution lasted {}.".format(dt() - start)) print(str(my_solution)) for i, j in zip(my_solution, np_solution): if not np.isclose(i, j): print(i, j) self.assertAlmostEqual(i, j)
def solve(self): self.VC = [] self.sc = [] self.MUC = [] self.VSF = [] self.ssf = [] self.MUSF = [] self.VSM = [] self.ssm = [] self.MUSM = [] for t in range(self.T, 0, -1): # couples: print('solving for t = {}'.format(t)) try: Vnext = self.VC[0] MUnext = self.MUC[0] except: Vnext = 3 * (np.zeros(self.v_couple_shape[-1]), ) MUnext = np.zeros(self.v_couple_shape[-1]) Vthis, MUthis, s = iteration_couples(self, t, Vnext, MUnext) self.sc = [s] + self.sc self.VC = [Vthis] + self.VC self.MUC = [MUthis] + self.MUC print('couples done, time {}'.format(dt() - self.t0)) # singles: try: Vnext = self.VSF[0] MUnext = self.MUSF[0] except: Vnext = np.zeros(self.v_sf_shape[-1]) MUnext = np.zeros(self.v_sf_shape[-1]) Vthis, MUthis, s = iteration_singles(self, t, Vnext, MUnext, True) self.ssf = [s] + self.ssf self.VSF = [Vthis] + self.VSF self.MUSF = [MUthis] + self.MUSF print('single female done, time {}'.format(dt() - self.t0)) try: Vnext = self.VSM[0] MUnext = self.MUSM[0] except: Vnext = np.zeros(self.v_sm_shape[-1]) MUnext = np.zeros(self.v_sm_shape[-1]) Vthis, MUthis, s = iteration_singles(self, t, Vnext, MUnext, False) self.ssm = [s] + self.ssm self.VSM = [Vthis] + self.VSM self.MUSM = [MUthis] + self.MUSM print('single male done, time {}'.format(dt() - self.t0))
def opencv_call(x): start_time = dt() H = c_long(x.shape[0]) W = c_long(x.shape[1]) P = x.ctypes.data_as(c_void_p) # The C pointer double_me_lib.double_me(P, W, H) rt = (dt() - start_time) * 1000 # print("OpenCV C code took %d ms for %d floats" % (rt, x.size)) return rt
def __init__(self, **kwagrs): self.setup = Setup(**kwagrs) self.t0 = dt() s = self.setup self.T = self.setup.T self.v_couple_shape = [(s.na_c, s.nexo_t[t], s.ntheta_coarse) for t in range(self.T)] self.v_sf_shape = [(s.na_s, s.n_zf) for t in range(self.T)] self.v_sm_shape = [(s.na_s, s.n_zm) for t in range(self.T)] print('setup created, time {}'.format(dt() - self.t0)) self.solve()
def evaluate( model, img_emb, txt_emb, lengths, device, shared_size=128, return_sims=False ): model.eval() _metrics_ = ('r1', 'r5', 'r10', 'medr', 'meanr') begin_pred = dt() img_emb = torch.tensor(img_emb).to(device) txt_emb = torch.tensor(txt_emb).to(device) end_pred = dt() sims = model.compute_pairwise_similarity( model.similarity, img_emb, txt_emb, lengths, shared_size=shared_size ) print(sims.min(), sims.max(), sims.mean()) # sims = model.get_sim_matrix( # embed_a=img_emb, embed_b=txt_emb, # lens=lengths, # ) div = sims.shape[1] / sims.shape[0] samp_sim = sims[:,np.arange(0, sims.shape[1], div).astype(np.int)] val_loss = model.multimodal_criterion(samp_sim) sims = layers.tensor_to_numpy(sims) end_sim = dt() i2t_metrics = i2t(sims) t2i_metrics = t2i(sims) rsum = np.sum(i2t_metrics[:3]) + np.sum(t2i_metrics[:3]) i2t_metrics = {f'i2t_{k}': v for k, v in zip(_metrics_, i2t_metrics)} t2i_metrics = {f't2i_{k}': v for k, v in zip(_metrics_, t2i_metrics)} metrics = { 'pred_time': end_pred-begin_pred, 'sim_time': end_sim-end_pred, 'val_loss': val_loss, } metrics.update(i2t_metrics) metrics.update(t2i_metrics) metrics['rsum'] = rsum if return_sims: return metrics, sims return metrics
def updateProgBar(curIter, totalIter, t0, barLength=20, decimals=0): """ Update progress bar. Place this function anywhere in a loop where you want to keep track of the loop's progress. Parameters ---------- curIter : int The current iteration. totalIter : int The total number of iterations. t0 : numeric The start time of the operation (in seconds). barLength : int, optional The length of the progress bar. The default is 20. decimals : int, optional The number of decimal places to use for tracking miliseconds. The default is 0 Returns ------- None. """ status = "Working..." progress = float(curIter) / float(totalIter) if isinstance(progress, int): progress = float(progress) if progress >= 1: progress = 1 status = "Finished!..." block = int(round(barLength * progress)) text = "\rPercent: [{0}] {1:.2f}% iter: {2}/{3} {4} Elapsed: {5}, Estimated: {6}".format( "#" * block + "-" * (barLength - block), round(progress * 100.0, 2), curIter, totalIter, status, pretty_print_time(t0, dt(), decimals=decimals), pretty_print_time((dt() - t0) / curIter * (totalIter - curIter), decimals=decimals)) if progress >= 1: sys.stdout.write(text + "\r\n") sys.stdout.flush() else: sys.stdout.write(text) sys.stdout.flush()
def evaluate( model, img_emb, txt_emb, lengths, device, shared_size=128, return_sims=False ): model.eval() _metrics_ = ('r1', 'r5', 'r10', 'medr', 'meanr') begin_pred = dt() img_emb = torch.FloatTensor(img_emb).to(device) txt_emb = torch.FloatTensor(txt_emb).to(device) end_pred = dt() sims = model.get_sim_matrix_shared( embed_a=img_emb, embed_b=txt_emb, lens=lengths, shared_size=shared_size ) sims = layers.tensor_to_numpy(sims) end_sim = dt() i2t_metrics = i2t(sims) t2i_metrics = t2i(sims) rsum = np.sum(i2t_metrics[:3]) + np.sum(t2i_metrics[:3]) i2t_metrics = {f'i2t_{k}': v for k, v in zip(_metrics_, i2t_metrics)} t2i_metrics = {f't2i_{k}': v for k, v in zip(_metrics_, t2i_metrics)} metrics = { 'pred_time': end_pred-begin_pred, 'sim_time': end_sim-end_pred, } metrics.update(i2t_metrics) metrics.update(t2i_metrics) metrics['rsum'] = rsum if return_sims: return metrics, sims return metrics
def _forward(self, batch, lang_iters, epoch): self.model.train() self.optimizer.zero_grad() begin_forward = dt() multimodal_loss = self._forward_multimodal_loss(batch) total_lang_loss, loss_info = self._get_multilanguage_total_loss( lang_iters) total_loss = multimodal_loss + total_lang_loss total_loss.backward() norm = 0. if self.clip_grad > 0: norm = clip_grad_norm_(self.model.parameters(), self.clip_grad) self.optimizer.step() if self.lr_scheduler is not None: self.lr_scheduler.step() end_backward = dt() batch_time = end_backward - begin_forward return self._update_train_info(batch_time, multimodal_loss, total_loss, epoch, norm, loss_info)
def ForwardSelectionAnalysis(self, cv_iterations=5, random_state=123, default_penalty=0.01, for_stride=2, reduce_features_by=30, red_metric='bic', verbose=True, t0=None): """ Run very basic forward selection procedure on the model. Parameters ---------- default_penalty : float, optional The penalty term to be applied to models where the penalty is otherwise unspecified. The default is 0.01. for_stride : int Window size of the coefficients to test in the forward selection procedure. For example, stride=1 is 1,2,3,4,..., stride=2 is 1,3,5,7,... The default is 2. reduce_features_by : int, optional Number of features to evaluate in the final model. The default is 30. red_metric : str, optional Metric to use for finding optimal penalty value. The default is 'bic'. verbose : bool, optional Print results of the process. The default is True. t0 : float, optional Initial start time for the process. The default is None. Returns ------- None. """ # TODO: make final feature selection automatic self._fsa_analysis_arglist = { 'cv_iterations': cv_iterations, 'random_state': random_state, 'default_penalty': default_penalty, 'for_stride': for_stride, 'reduce_features_by': reduce_features_by, 'red_metric': red_metric, 'verbose': verbose, 't0': t0 } if t0 is None: t0 = dt() self.plotAICs() aic_type = {'aic': 0, 'aicc': 1, 'bic': 2, 'ebic': 3, 'misclass': 4} self.opt_c = self.reduction[np.where( self.aics[:, aic_type[red_metric]] == np.min( self.aics[:, aic_type[red_metric]]))][0] self.optimal_num_params = np.array(self.nonzero_coefs)[np.where( self.aics[:, aic_type[red_metric]] == np.min( self.aics[:, aic_type[red_metric]]))][0] if verbose: print_time("\nCreating Model with Optimal Penalty Value...", t0, te=dt()) self.build_model('lasso_reduction_mod', cv_iterations, self.opt_c, random_state) coef = self.steps['lasso_reduction_mod']['model'].coef_.reshape(-1, ) coef_ord = pd.DataFrame( np.abs(coef), index=self.train.loc[:, self.cur_pred_list].columns, columns=["Importance"]) coef_ord = coef_ord.sort_values('Importance', ascending=False) keep_cols = list(coef_ord[coef_ord["Importance"] > 0].index) keep_cols.append(self.target_var) self.train = self.train.loc[:, keep_cols] self.cur_pred_list = self.train.columns[np.where( self.target_var != self.train.columns)[0]] self.steps['lasso_reduction_mod']['keep_cols'] = keep_cols keep_cols = [] coef_ord = [] coef = [] if verbose: print_time( "\nCreating Model with Optimal Penalty Value After Removing Zeroed Parameters...\n", t0, te=dt()) self.build_model('lasso_reduction_mod_reduced', cv_iterations, 1, random_state) coef_red = self.steps['lasso_reduction_mod_reduced'][ 'model'].coef_.reshape(-1, 1) coef_ord_red = pd.DataFrame( np.concatenate( (np.abs(coef_red), coef_red, 100.0 * (np.exp(coef_red) - 1)), axis=1), index=self.train.loc[:, self.cur_pred_list].columns, columns=["Importance", "Coefficiants", "% increase in Prob"]) coef_ord_red = coef_ord_red.sort_values('Importance', ascending=False) self.steps['lasso_reduction_mod_reduced'][ 'coef_ord_red'] = coef_ord_red self.forwardElimMetricCalc(coef_ord_red, for_stride, random_state) self.plotForwardMetrics() cols_to_include = list(coef_ord_red.index[:reduce_features_by]) self.cur_pred_list = cols_to_include if verbose: print_time("\nCreating Final Model...", t0, te=dt()) self.build_model('final_mod', cv_iterations, 1, random_state) coef_red_final = self.steps['final_mod']['model'].coef_.reshape(-1, 1) coef_ord_red_final = pd.DataFrame( np.concatenate((np.abs(coef_red_final), coef_red_final, 100.0 * (np.exp(coef_red_final) - 1)), axis=1), index=self.train.loc[:, cols_to_include].columns, columns=["Importance", "Coefficiants", "% increase in Prob"]) coef_ord_red_final = coef_ord_red_final.sort_values('Importance', ascending=False) self.coef_ord_red_final = coef_ord_red_final self.Xtrain = self.train.loc[:, cols_to_include] self.Ytrain = self.train.loc[:, self.target_var] self.Ypred_train = self.steps['final_mod']['model'].predict( self.Xtrain) self.Yprob_train = self.steps['final_mod']['model'].predict_proba( self.Xtrain) self.Xtest = self.holdout.loc[:, cols_to_include] self.Ytest = self.holdout.loc[:, self.target_var] self.Ypred_test = self.steps['final_mod']['model'].predict(self.Xtest) self.Yprob_test = self.steps['final_mod']['model'].predict_proba( self.Xtest)[:, 1] if verbose: print_time("\nFinished...", t0, te=dt())
def BaselineDropProcedures(self, freshStart=True, downsample=False, dropCols=None, dropCor=True, corr_cutoff=0.9, dropVar=True, dropVarTol=0.001, cv_iterations=5, random_state=123, default_penalty=0.01, verbose=True, t0=None): """ Create Baseline model and begin dropping features from the model based on given criteria. Parameters ---------- freshStart : bool, optional Whether to use a previous result, or start from scrath. The default is True. downsample : bool, optional Whether or not to downsample the majority class. The default is False. dropCols : list, optional List of columns to initially drop. The default is None. dropCor : bool, optional Drop variables with high correlations. The default is True. corr_cutoff : float, optional Cutoff for removing high correlation variables. Anything below the cutoff is kept. The default is 0.9. dropVar : bool, optional Drop variables with very low variance. Does nothing if data is standardized. The default is True. dropVarTol : float, optional Cutoff value for low variance. The default is 0.001. cv_iterations : int, optional The number of cross validation steps. The default is 5. random_state : int, optional The random seed for the process. The default is 123. default_penalty : float, optional The penalty term to be applied to models where the penalty is otherwise unspecified. The default is 0.01. verbose : bool, optional Print steps as they complete. The default is True. t0 : float, optional The start time of the process. For internal use. The default is None. Returns ------- None. """ self._bdp_analysis_arglist = { 'freshStart': freshStart, 'downsample': downsample, 'dropCols': dropCols, 'dropCor': dropCor, 'corr_cutoff': corr_cutoff, 'dropVar': dropVar, 'dropVarTol': dropVarTol, 'cv_iterations': cv_iterations, 'random_state': random_state, 'default_penalty': default_penalty, 'verbose': verbose, 't0': t0 } if t0 is None: t0 = dt() if freshStart: if not self.low_memory: self.train = self.train_raw self.holdout = self.holdout_raw else: print("Can't run fresh start in low memory mode.") if verbose: print_time("\nCreating Baseline Model...", t0, te=dt()) self.build_model('baseline_mod', cv_iterations, default_penalty, random_state) if verbose: print_time("\nExecuting Column Drop Procedures...", t0, te=dt()) if dropCols is not None: if verbose: print_time("\nDropping Pre-Determined Columns...", t0, te=dt()) cols = list(self.train.columns) # what I used to do... #idx = list(np.where([True if sum((True if reject in col else False for reject in dropCols))>0 else False for col in cols])[0]) idx = [ i for i, col in zip(range(len(cols)), cols) if col in dropCols ] label_idx = list(self.train.columns[idx]) self.train = self.train.drop(label_idx, axis=1) self.cur_pred_list = self.train.columns[np.where( self.target_var != self.train.columns)[0]] self.num_dropped_cols['dropCol_list'] = len(dropCols) if verbose: print_time("\nDropped " + str(len(dropCols)) + " Predefined Columns...", t0, te=dt()) if dropVar: if verbose: print_time("\nDropping Low Variance Columns...", t0, te=dt()) varDropList = naiveVarDrop( self.train, list(self.train.columns[np.where( self.target_var != self.train.columns)[0]]), tol=dropVarTol, asList=True) self.train = self.train.drop(varDropList, axis=1) self.varDropList = varDropList self.cur_pred_list = self.train.columns[np.where( self.target_var != self.train.columns)[0]] self.num_dropped_cols['varDropList'] = len(varDropList) if verbose: print_time("\nDropped " + str(len(varDropList)) + " Low-Variance Columns...", t0, te=dt()) varDropList = [] if dropCor: if verbose: print_time("\nGenerating/Recovering Correlation Matrix...", t0, te=dt()) if self.corr_mat is None or self.y_corr is None: self.GenCorrStats(is_raw=False) if self.highCorr is None: self.genHighCorrs(corr_cutoff) if verbose: print_time("\nDropping High Correlations...", t0, te=dt()) HCdropList = dropHighCorrs(self.train, self.highCorr, asList=True, print_=False) self.train = self.train.drop(HCdropList, axis=1) self.HCdropList = HCdropList self.num_dropped_cols['HCdropList'] = len(HCdropList) if verbose: print_time("\nDropped " + str(len(HCdropList)) + " Correlated Columns...", t0, te=dt()) HCdropList = [] self.cur_pred_list = self.train.columns[np.where( self.target_var != self.train.columns)[0]] self.plot_y_corr() if verbose: total_dropped = 0 for key in self.num_dropped_cols.keys(): total_dropped += self.num_dropped_cols[key] print_time("\nDropped " + str(total_dropped) + " Columns...", t0, te=dt()) if verbose: print_time("\nCreating Post-Drop Baseline Model...", t0, te=dt()) self.build_model('postdrop_baseline_mod', cv_iterations, default_penalty, random_state) if verbose: print_time("\nFinished Baseline Drop...", t0, te=dt())
def forwardElimMetricCalc(self, coef_ord_red, stride, random_state): """ Run a very basic forward elimination procedure, and calculate model metrics. Parameters ---------- coef_ord_red : list A list of model coefficients, ordered by importance. stride : int Window size of the coefficients to test in the forward selection procedure. For example, stride=1 is 1,2,3,4,..., stride=2 is 1,3,5,7,... random_state : int Random seed for procedure. Returns ------- None. """ num_params = np.arange(1, self.train.shape[1] - 1, stride) # f1, acc, sens, spec, auc metrics = np.zeros(shape=(len(num_params), 10)) t0 = dt() for i, parm in enumerate(num_params): cur_mod_reg = LogisticRegression(C=self.opt_c, max_iter=10000, penalty="l1", solver='liblinear', random_state=random_state) # Cross-Val fully reduced model cols_to_include = list(coef_ord_red.index[0:parm]) cur_mod_cv_results = crossVal(self.train.loc[:, cols_to_include], self.train.loc[:, self.target_var], 5, cur_mod_reg, print_=False) metrics[i, 0:2] = ( np.mean(cur_mod_cv_results['Out of Sample']["Accuracy"]), np.std(cur_mod_cv_results['Out of Sample']["Accuracy"], ddof=1)) metrics[i, 2:4] = (np.mean(cur_mod_cv_results['Out of Sample']["F1"]), np.std(cur_mod_cv_results['Out of Sample']["F1"], ddof=1)) metrics[i, 4:6] = ( np.mean(cur_mod_cv_results['Out of Sample']["Sens/Recall"]), np.std(cur_mod_cv_results['Out of Sample']["Sens/Recall"], ddof=1)) metrics[i, 6:8] = ( np.mean(cur_mod_cv_results['Out of Sample']["Specificity"]), np.std(cur_mod_cv_results['Out of Sample']["Specificity"], ddof=1)) metrics[i, 8:10] = (np.mean( cur_mod_cv_results['Out of Sample']["AUC"]), np.std(cur_mod_cv_results['Out of Sample']["AUC"], ddof=1)) updateProgBar(i + 1, len(num_params), t0) self.forwardElimMetrics = metrics self.num_params_forward = num_params
if __name__ == '__main__': import numpy as np try: print(len(mdl.V)) except: from dill import load mdl = load(open('mdl.pkl','rb+')) np.random.seed(12) q = dt() o = AgentsEst(mdl,T=30,verbose=False) ss_val = o.ss_val kf_val = o.kf_val print('total time is {:02.1f} sec'.format(dt()-q)) ''' pmeet_np = [] ppreg_np = [] for t in range(4): print('t = {}'.format(t))
def train_epoch( self, train_loader, lang_loaders, epoch, valid_loaders=[], log_interval=50, valid_interval=500, path='' ): lang_iters = [ DataIterator( loader=loader, device=self.device, non_stop=True ) for loader in lang_loaders ] pbar = lambda x: x if self.master: pbar = lambda x: tqdm( x, total=len(x), desc='Steps ', leave=False, ) for batch in pbar(train_loader): self.model.train() # Update progress bar self.optimizer.zero_grad() begin_forward = dt() multimodal_loss = self.model.forward_multimodal_loss(batch) iteration = self.model.multimodal_criterion.iteration adjusted_iter = self.world_size * iteration # Cross-language update total_lang_loss = 0. loss_info = {} for lang_iter in lang_iters: lang_data = lang_iter.next() lang_loss = self.model.forward_multilanguage_loss(*lang_data) total_lang_loss += lang_loss loss_info[f'train_loss_{str(lang_iter)}'] = lang_loss total_loss = multimodal_loss + total_lang_loss total_loss.backward() norm = 0. if self.clip_grad > 0: norm = clip_grad_norm_( self.model.parameters(), self.clip_grad ) self.optimizer.step() if self.lr_scheduler is not None: self.lr_scheduler.step() end_backward = dt() batch_time = end_backward-begin_forward train_info = Dict({ 'loss': multimodal_loss, 'iteration': iteration, 'total_loss': total_loss, 'k': self.model.multimodal_criterion.k, 'batch_time': batch_time, 'countdown': self.count, 'epoch': epoch, 'norm': norm, }) train_info.update(loss_info) for param_group in self.optimizer.param_groups: if 'name' in param_group: train_info.update({f"lr_{param_group['name']}": param_group['lr']}) else: train_info.update({'lr_base': param_group['lr']}) if self.master: logger.tb_log_dict( tb_writer=self.tb_writer, data_dict=train_info, iteration=iteration, prefix='train' ) if iteration % valid_interval == 0: # Run evaluation metrics, metric_value = self.evaluate_loaders(valid_loaders) # Update early stop variables # and save checkpoint if metric_value < self.best_val: self.count -= 1 elif not self.save_all: self.count = self.early_stop self.best_val = metric_value if self.master: self.save( path=self.path, is_best=(metric_value >= self.best_val), args=self.args, rsum=metric_value, ) # Log updates for metric, values in metrics.items(): self.tb_writer.add_scalar(metric, values, iteration) # Early stop if self.count == 0 and self.master: self.sysoutlog('\n\nEarly stop\n\n') return False if iteration % log_interval == 0 and self.master: helper.print_tensor_dict(train_info, print_fn=self.sysoutlog) if self.log_histograms: logger.log_param_histograms( self.model, self.tb_writer, iteration=self.model.multimodal_criterion.iteration, ) return True
def fit(self, X, y): """Fit the model Parameters ---------- X : numpy array The feature (or design) matrix. y : numpy array The response variable. Returns ------- self Updates internal attributes, such as `coef_` and `intercept_`. """ t0 = dt() X = check_array(X, force_all_finite='allow-nan', estimator=self, copy=True) if not self.has_constant: X = add_constant(X, prepend=True) if self.start is not None: pass else: if self.verbose: print_time("Initializing Coefficients...", t0, dt(), backsn=True) if self.initialize_weights == 'sklearn': C = self.C if C is None: C = 1 # TODO: update with Weibull Regression starting values ### If using sklearn version 0.23.2, can use this line instead #mod = glm_pois(alpha=1/C, fit_intercept=False, max_iter=1000).fit(X, y) #self.start = mod.coef_.reshape(-1, ) ### else, use statsmodels mod = glm_pois(y, X, family=Poisson()).fit() self.start = mod.params.reshape(-1, ) if self.extra_params > 0: self.start = np.concatenate( (self.start, np.repeat(1, self.extra_params))) elif self.initialize_weights == 'ones': self.start = np.ones(shape=X.shape[1] + self.extra_params) elif self.initialize_weights == 'random': self.start = np.random.normal(X.shape[1] + self.extra_params) else: self.start = np.zeros(shape=X.shape[1] + self.extra_params) if self.verbose: print_time("Beginning MCMC...", t0, dt(), backsn=True) postArgs = { 'X': X, 'Y': y, 'l_scale': self.C if self.C is None else 2 * self.C } algo_res = applyMCMC(st=self.start, ni=self.niter, lp=self.lpost, algo=self.algo, postArgs=postArgs, algoOpts=self.algo_options, sd=self.retry_sd, max_tries=self.retry_max_tries) self.mcmc_params = algo_res['parameters'] self.prev_vals = algo_res['prev_vals'] self.coef_, self.intercept_, self.extra_params_sum_ = self._create_coefs( self.mcmc_params, self.param_summary, self.extra_params) self.n_iter_ = self.niter #get model summaries weights = _check_sample_weight(None, X) y_pred = self.predict(X[:, 1:]) y_mean = np.average(y, weights=weights) dev = np.sum(weights * (2 * (xlogy(y, y / y_pred) - y + y_pred))) dev_null = np.sum(weights * (2 * (xlogy(y, y / y_mean) - y + y_mean))) self.deviance_ = dev self.null_deviance_ = dev_null self.pearson_residuals_ = (y - y_pred) / np.sqrt(y_pred) self.pearson_chi2_ = np.sum(self.pearson_residuals_**2) self.model_d2_ = 1 - dev / dev_null self.df_model_ = X.shape[1] - 1 self.df_residuals_ = X.shape[0] - X.shape[1] self.dispersion_scale_ = self.pearson_chi2_ / self.df_residuals_ self.dispersion_scale_sqrt_ = np.sqrt(self.dispersion_scale_) return self
from tsdst.metrics import r2 from tsdst.nn.model import NeuralNetwork def cust_r2(y_true, y_pred): return np.mean([r2(y_true[:, i], y_pred[:, i]) for i in range(y_true.shape[1])]) X_og, Y_og = make_regression(n_samples=1000, n_features=50, n_informative=20, n_targets=1, bias=0.0, effective_rank=None, tail_strength=0.5, noise=0.0, shuffle=True, coef=False, random_state=42) t0 = dt() X_train, X_test, Y_train, Y_test = train_test_split(X_og, Y_og, test_size=0.3, random_state=42) Y_train = Y_train.reshape(X_train.shape[0], -1) Y_test = Y_test.reshape(X_test.shape[0], -1) model = { 'hidden0': {'depth': 10, 'activation': 'relu', 'derivative': 'relu_der', 'activation_args': {}, 'initializer': 'he_uniform', 'dropout_keep_prob': 1, 'lambda': 0.01,
def fit(self, X, y): """Fit the model Parameters ---------- X : numpy array The feature (or design) matrix. y : numpy array The response variable. Returns ------- self Updates internal attributes, such as `coef_` and `intercept_`. """ t0 = dt() X = check_array(X, force_all_finite='allow-nan', estimator=self, copy=True) if not self.has_constant: X = add_constant(X, prepend=True) if self.start is not None: pass else: if self.verbose: print_time("Initializing Coefficients...", t0, dt(), backsn=True) if self.initialize_weights == 'sklearn': C = self.C if C is None: C = 1 mod = glm_lr(C=C, solver='liblinear', penalty='l1', fit_intercept=False).fit(X, y) self.start = mod.coef_.reshape(-1, ) if self.extra_params > 0: self.start = np.concatenate( (self.start, np.repeat(1, self.extra_params))) elif self.initialize_weights == 'ones': self.start = np.ones(shape=X.shape[1] + self.extra_params) elif self.initialize_weights == 'random': self.start = np.random.normal(size=(X.shape[1] + self.extra_params, )) else: self.start = np.zeros(shape=X.shape[1] + self.extra_params) if self.verbose: print_time("Beginning MCMC...", t0, dt(), backsn=True) postArgs = { 'X': X, 'Y': y, 'l_scale': self.C if self.C is None else 2 * self.C } algo_res = applyMCMC(st=self.start, ni=self.niter, lp=self.lpost, algo=self.algo, postArgs=postArgs, algoOpts=self.algo_options, sd=self.retry_sd, max_tries=self.retry_max_tries) self.mcmc_params = algo_res['parameters'] self.prev_vals = algo_res['prev_vals'] self.coef_, self.intercept_, self.extra_params_sum_ = self._create_coefs( self.mcmc_params, self.param_summary, self.extra_params) self.n_iter_ = self.niter self.classes_ = np.unique(y) if self.over_dispersion: self.dispersion_estimation_ = self.extra_params_sum_[-1] else: self.dispersion_estimation_ = None if self.verbose: print_time("Finished MCMC. Stored Coefficients...", t0, dt(), backsn=True) return self
def fit(self, X, y): """Fit the model Parameters ---------- X : numpy array The feature (or design) matrix. y : numpy array The response variable. Returns ------- self Updates internal attributes, such as `coef_` and `intercept_`. """ t0 = dt() X = check_array(X, force_all_finite='allow-nan', estimator=self, copy=True) if not self.has_constant: X = add_constant(X, prepend=True) if self.start is not None: pass else: if self.verbose: print_time("Initializing Coefficients...", t0, dt(), backsn=True) if self.initialize_weights == 'sklearn': C = self.C if C is None: C = 1 try: from sklearn.linear_model import PoissonRegressor as glm_pois_sk mod = glm_pois_sk(alpha=1 / C, fit_intercept=False, max_iter=1000).fit(X, y) self.start = mod.coef_.reshape(-1, ) except ImportError: print( 'Older sklearn, no PoissonRegressor. Using statsmodels instead' ) mod = glm_pois(y, X, family=Poisson()).fit() self.start = mod.params.reshape(-1, ) if self.extra_params > 0: self.start = np.concatenate( (self.start, np.repeat(1, self.extra_params))) elif self.initialize_weights == 'ones': self.start = np.ones(shape=X.shape[1] + self.extra_params) elif self.initialize_weights == 'random': self.start = np.random.normal(size=(X.shape[1] + self.extra_params, )) else: self.start = np.zeros(shape=X.shape[1] + self.extra_params) if self.verbose: print_time("Beginning MCMC...", t0, dt(), backsn=True) postArgs = { 'X': X, 'Y': y, 'l_scale': self.C if self.C is None else 2 * self.C } algo_res = applyMCMC(st=self.start, ni=self.niter, lp=self.lpost, algo=self.algo, postArgs=postArgs, algoOpts=self.algo_options, sd=self.retry_sd, max_tries=self.retry_max_tries) self.mcmc_params = algo_res['parameters'] self.prev_vals = algo_res['prev_vals'] self.coef_, self.intercept_, self.extra_params_sum_ = self._create_coefs( self.mcmc_params, self.param_summary, self.extra_params) self.n_iter_ = self.niter # get model summaries if self.over_dispersion: self.dispersion_delta_ = self.extra_params_sum_[-1] self.dispersion_estimation_ = 1 / (1 - self.dispersion_delta_)**2 else: self.dispersion_delta_ = 0 self.dispersion_estimation_ = None ddu = self._deviance_dispersion_update(X[:, 1:], y, sample_weight=None) self.deviance_ = ddu['deviance_'] self.null_deviance_ = ddu['null_deviance_'] self.pearson_residuals_ = ddu['pearson_residuals_'] self.pearson_chi2_ = ddu['pearson_chi2_'] self.model_d2_ = ddu['model_d2_'] self.df_model_ = ddu['df_model_'] self.df_residuals_ = ddu['df_residuals_'] self.df_total_ = ddu['df_total_'] self.dispersion_scale_ = ddu['dispersion_scale_'] self.dispersion_scale_sqrt_ = ddu['dispersion_scale_sqrt_'] return self
continue else: break #최대값 넘길시 20으로 재조정 opportunity = min(opportunity, 20) # ====================================== # =============구구단 파트============== # ====================================== count: int = 0 probs: list[str] = [] correctAns: int = 0 wrongAns: int = 0 timeout: int = 3 #전체 시험 타이머 시작 initialtime = dt() st: float = 0.0 #이제 그냥 가장 앞 n개만 뽑으면 됨 print("문제시작:", end="") while count < opportunity: #그냥 앞에서부터 하나씩 긁어오기만 해도 충-분 realAns = (googoolist[count][0] * googoolist[count][1]) #문제 타이머 시작 st = dt() try: #인풋 받기 inp = int(input("%dX%d = " % googoolist[count])) #정답을 맞추었고 if (inp == realAns): #시간안에 풀으면 정답처리 if ((dt() - st) < 3):
num_iterations=n_iters, optimizer='adam', optimizer_args={ 'learning_rate': 0.001, 'beta1': 0.9, 'beta2': 0.999, 'eps': 1e-8 }, m_scale=1, bn_tol=1e-6, bn_momentum=0, shuffle=False, print_cost=True, random_state=42) t0 = dt() nn = nn.fit(X_train, Y_train_oh) t1 = dt() print('tsdst Runtime (s):', t1 - t0) ################################################################# ################### Comparison With Keras ####################### ################################################################# from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense, Activation, BatchNormalization from tensorflow.keras.regularizers import l1, l2, l1_l2 from tensorflow.keras.initializers import RandomUniform, TruncatedNormal from tensorflow.keras.optimizers import Adam
def latinHypercube1D(data, sampleSize, random_state=None, shuffle_after=True, sort_=True, sort_method="quicksort", sort_cols=None, stratified=True, bin_placement="random", verbose=False): """ Creates a sample from a Dataframe (data). If replace=True, this function can be used for bootstrap samples. Parameters ---------- data : numpy array or pandas dataframe The design or feature matrix (with response). sampleSize : int The number of samples to return. random_state : int, optional The random seed of the process. The default is None. shuffle_after : bool, optional Shuffle the results after being sampled. The default is True. sort_ : bool, optional Sort the data (only set to False if sending presorted data). The default is True. sort_method : str, optional Numpy sort method. The default is "quicksort". sort_cols : list, optional The columns to include in the sorting. The default is None. stratified : bool, optional Create a stratified sample. The default is True. bin_placement : str, optional Method for placing the edges on the sampling bins. The default is "random". verbose : bool, optional Print the results of the process. The default is False. Raises ------ ValueError Raised if invalid bin_placement is passed. Returns ------- LHC : pandas dataframe or numpy array The samples. """ t0 = None if verbose: # Initialize time for printing t0 = dt() print_time("\nInitializing...", t0, te=dt()) # Initialize sortedData = None df = False # Check if DataFrame for convenience later if isinstance(data, pd.DataFrame): df = True if sort_: if verbose: print_time("\nSorting...", t0, te=dt()) # Convert to DataFrame, if not already (for easy # column sorting control) # TODO: see if column control could be equally as easy without # converting to DF if df: sortedData = data.copy(deep=True) else: sortedData = pd.DataFrame(data) if sort_cols is not None: sortedData = sortedData.sort_values(sort_cols, axis=0, kind=sort_method) else: sortedData = sortedData.sort_values(list(sortedData.columns), axis=0, kind=sort_method) if random_state is not None: np.random.seed(random_state) if verbose: print_time("\nShaping...", t0, te=dt()) if sortedData is not None: sortedData = sortedData.values.reshape(data.shape[0], -1) else: sortedData = np.array(data).reshape(data.shape[0], -1) rows = sortedData.shape[0] cols = sortedData.shape[1] LHC = np.zeros(shape=(sampleSize, cols), dtype=sortedData.dtypes) splits = sampleSize if verbose: print_time("\nCreating the bins...", t0, te=dt()) if stratified: high = int(np.ceil(rows / sampleSize)) low = int(np.floor(rows / sampleSize)) rem = rows % sampleSize if rem != 0: rem2 = sampleSize - rem if bin_placement == "random": f_array = np.repeat((high, low), (rem, rem2)) np.random.shuffle(f_array) elif bin_placement == "spaced": if rem > rem2: r1 = np.repeat(high, rem) start = 1 end = rem + 1 a1 = np.arange(start, end, np.floor(rem / rem2), dtype=np.int)[:rem2] f_array = np.insert(r1, a1, low) else: r1 = np.repeat(low, rem2) start = 1 end = rem2 + 1 a1 = np.arange(start, end, np.floor(rem2 / rem), dtype=np.int)[:rem] f_array = np.insert(r1, a1, high) else: raise ValueError( "Not a valid bin placement. Change to 'spaced' or " "'random'. To order the bins from high to low, change " "stratified to False") else: f_array = np.repeat(high, sampleSize) splits = np.cumsum(f_array)[:-1] Splits = np.array_split(sortedData, splits) if verbose: print_time("\nSampling...", t0, te=dt()) t1 = dt() for i, sample in enumerate(Splits): LHC[i, :] = sample[np.random.choice(sample.shape[0], 1)] if verbose: updateProgBar(i + 1, sampleSize, t1) if shuffle_after: if verbose: print_time("\nShuffling...", t0, te=dt()) np.random.shuffle(LHC) if df: if verbose: print_time("\nConverting to DataFrame...", t0, te=dt()) LHC = pd.DataFrame(LHC, columns=data.columns) if verbose: print_time("\nFinished...", t0, te=dt()) return LHC
def RunFullAnalysis(self, freshStart=True, downsample=False, dropCols=None, dropCor=True, corr_cutoff=0.9, dropVar=True, dropVarTol=0.001, cv_iterations=5, random_state=123, default_penalty=0.01, verbose=True, try_parallel=True, for_stride=2, reduce_features_by=30, num_cs=100, red_metric='bic', red_log_low=-5, red_log_high=1, n_jobs=1, remove_msg=True, chunk=False): """ Performs a quick analysis by first considering which features/variables may be uninformative predictors based on preliminary models. Then, it performs a very basic forward selection procedure to limit predictors in the final model. Currently, this only supports Logistic Regression. Parameters ---------- freshStart : bool, optional Whether to use a previous result, or start from scrath. The default is True. downsample : bool, optional Whether or not to downsample the majority class. The default is False. dropCols : list, optional List of columns to initially drop. The default is None. dropCor : bool, optional Drop variables with high correlations. The default is True. corr_cutoff : float, optional Cutoff for removing high correlation variables. Anything below the cutoff is kept. The default is 0.9. dropVar : bool, optional Drop variables with very low variance. Does nothing if data is standardized. The default is True. dropVarTol : float, optional Cutoff value for low variance. The default is 0.001. cv_iterations : int, optional The number of cross validation steps. The default is 5. random_state : int, optional The random seed for the process. The default is 123. default_penalty : float, optional The penalty term to be applied to models where the penalty is otherwise unspecified. The default is 0.01. verbose : bool, optional Print steps as they complete. The default is True. try_parallel : bool, optional Try to perform the operation in parallel. If it fails, it will continue without parallel operations. The default is True. for_stride : int, optional The gap between variables evaluated for the forward selection metric. The default is 2. reduce_features_by : int, optional The size of the final model. The default is 30. num_cs : int, optional number of penalty values to test (i.e. number of models to evaluate during AIC calculations). The default is 100. red_metric : str, optional The metric used to optimize model reductions. The default is 'bic'. red_log_low : float, optional The logspace minimum value for evaluating the L1 penalty. The default is -5. red_log_high : float, optional The logspace maximum value for evaluating the L1 penalty.. The default is 1. n_jobs : int, optional The number of processes to attempt. The default is 1. remove_msg : bool, optional Remove messagepacks. The default is True. chunk : bool, optional Chunk msgpacks? The default is False. Returns ------- None. """ t0 = dt() self._full_analysis_arglist = { 'freshStart': freshStart, 'downsample': downsample, 'dropCols': dropCols, 'dropCor': dropCor, 'corr_cutoff': corr_cutoff, 'dropVar': dropVar, 'dropVarTol': dropVarTol, 'cv_iterations': cv_iterations, 'random_state': random_state, 'default_penalty': default_penalty, 'try_parallel': try_parallel, 'for_stride': for_stride, 'reduce_features_by': reduce_features_by, 'num_cs': num_cs, 'red_metric': red_metric, 'red_log_low': red_log_low, 'red_log_high': red_log_high, 'n_jobs': n_jobs, 'verbose': verbose, 'remove_msg': remove_msg } self.BaselineDropProcedures(freshStart=freshStart, downsample=downsample, dropCols=dropCols, dropCor=dropCor, corr_cutoff=corr_cutoff, dropVar=dropVar, dropVarTol=dropVarTol, cv_iterations=cv_iterations, random_state=random_state, default_penalty=default_penalty, verbose=verbose, t0=t0) if verbose: print_time("\nCreating Lasso Reduction Models...", t0, te=dt()) self.reduction = np.logspace(red_log_low, red_log_high, num_cs) if try_parallel: try_again = True while (try_again): try: self.calcAICs(num_cs, try_parallel, n_jobs, random_state=random_state, remove_msg=remove_msg, chunk=chunk) try_again = False except (MemoryError): n_jobs = int(n_jobs - (n_jobs / 2)) print('\n\tReduced n_jobs to ', n_jobs) if n_jobs == 1: try_again = False if not try_again: self.calcAICs(num_cs, False, n_jobs=1, random_state=random_state, remove_msg=remove_msg, chunk=chunk) else: self.calcAICs(num_cs, False, n_jobs=1, random_state=random_state, remove_msg=remove_msg, chunk=chunk) self.ForwardSelectionAnalysis(cv_iterations=cv_iterations, random_state=random_state, default_penalty=default_penalty, for_stride=for_stride, reduce_features_by=reduce_features_by, red_metric=red_metric, verbose=verbose, t0=t0)
int: the number of the prime numbers smaller than num """ currentno = 2 #check from 2 count =0 #check all number smaller or equal to n #it seems that while is faster than for... while currentno <= num: if(isPrime(currentno)): count = count +1 currentno = currentno+1 return count #test code startt = dt() #initialize Timer print("prime numbers in 0-10 : %d"%numberOfPrime(10)) print("eleapsed time : %.2fms"%((dt()-startt)*1000)) startt = dt() #initialize Timer print("prime numbers in 0-100 : %d"%numberOfPrime(100)) print("eleapsed time : %.2fms"%((dt()-startt)*1000)) startt = dt() #initialize Timer print("prime numbers in 0-1000 : %d"%numberOfPrime(1000)) print("eleapsed time : %.2fms"%((dt()-startt)*1000)) startt = dt() #initialize Timer print("prime numbers in 0-10000 : %d"%numberOfPrime(10000)) print("eleapsed time : %.2fms"%((dt()-startt)*1000)) startt = dt() #initialize Timer print("prime numbers in 0-100000 : %d"%numberOfPrime(100000)) print("eleapsed time : %.2fms"%((dt()-startt)*1000))
#initialize variables result = False i = 1 #check the number smaller than square while i < num**(0.5): i = i + 1 #if the number can be devided by other number, its not prime if (num % i == 0): result = True #return the value return not (result) n: int = 0 primenumbers: list[int] = [2] startt: float = dt() try: n = int(input("type the number:")) except ValueError: exit # if error, just exit the program currentno = 2 #check from 2 count = 0 #check all number smallor or equal to n while currentno <= n: if (isprime(currentno)): primenumbers.append(currentno) count = count + 1 currentno = currentno + 1 print("%d이하 소수의 갯수 : %d" % (n, count))
def numpy_call(x): start_time = dt() x *= 2.0 rt = (dt() - start_time) * 1000 # print("Python numpy took %d ms for %d floats" % (rt, x.size)) return rt
# Authors: Jaideep Reddy, Deepika Bisht (BML Munjal University Gurgaon, India) # # Last Modified:18-07-2021 #importing multiprocessing libraries from timeit import default_timer as dt import numpy as np from multiprocessing import Process, Array, Value #provide the following cores = 18 # no of cores nodes = 4 # no of nodes gsize = 0.5 # grid size node = 0 # node number st = dt() veckey = list(QgsProject.instance().mapLayers().keys())[ 1] # Get the key for map/india layer instancelayer = QgsProject.instance().mapLayers()[ veckey] # Use key to get the map/india layer key = list( QgsProject.instance().mapLayers().keys())[0] # Get the key for grid layer grid = QgsProject.instance().mapLayers()[ key] # Use grid key to get the grid layer # This function : counter() returns the count of boxes that contains a part of coastline / intersects with coastline. def counter(gridfts, instvectors, arr, i): cnt = 0 for feature in gridfts: # Get the features of map within boundary of grid feature
for pn in primenumbers: if (pn > (num**(0.5))): break #exit the loop if all prime numbers are checked if (num % pn) == 0: return False #2nd check : devide by the numbers within the maximum primenumbers - root(num) for nn in range(pn, int(num**0.5)): if (num % nn) == 0: return False #For now, the number is prime number so add it to the list primenumbers.append(num) return True startt = dt() #initialize Timer n: int = 0 try: n = int(input("type the number:")) except ValueError: exit # if error, just exit the program currentno = 2 #check from 2 count = 0 #check all number smallor or equal to n while currentno <= n: if (isprime_imp(currentno)): count = count + 1 currentno = currentno + 1 print("%d이하 소수의 갯수 : %d" % (n, count)) print("목록", primenumbers.__str__())
W = c_long(x.shape[1]) P = x.ctypes.data_as(c_void_p) # The C pointer # The C function header is: # void double_me(void *buffer, const int W, const int H); # Lets call the inplace C function: double_me_lib.double_me(P, W, H) # Operations are inplace there for we can see the result in same array. print("Same Array after c code:\n", x) # Now Let's time it x = np.random.randn(10000, 10000).astype(np.float32) start_time = dt() x = np.ascontiguousarray(x) print("Flatten took %d ms for %d floats" % ((dt() - start_time) * 1000, x.size)) def numpy_call(x): start_time = dt() x *= 2.0 rt = (dt() - start_time) * 1000 # print("Python numpy took %d ms for %d floats" % (rt, x.size)) return rt def opencv_call(x): start_time = dt()
def forwardSelection(XY, target_var, model, metric='bic', family='gaussian', verbose=True, n_jobs=1, early_stop=False, perc_min=0.01, stop_at_p=1000, stop_when=5, use_probabilities=False, return_type='all', serialze_flavor='feather', use_threads=True): """ A forward selection algorithm for classification only right now. Still needs some work. Note: it is a known bug for this function to fail when n_jobs > 1 while using sypder. This is because of an issue with spyder and the p_prog_simp function. I have not been able to discover why, or provide a fix. If you run a script with this function from the console, it should run just fine. Parameters ---------- XY : pandas dataframe The combined independent variables/features and reponse/target variable. target_var : str The column containing the target (or response) variable. model : sklearn, or similar An unfitted model. Any model that has a fit and predict method. metric : str The AIC metric to be used, for example, aic, aicc, bic, ebic, hastie, or kwano. family : str The family of distributions to calculate log-likelihood from. verbose : bool, optional Output the steps and progress as it completes. The default is True. n_jobs : int, optional If greater than 1, perform operation in parallel. The default is 1. perc_min : float, optional --NOT IMPLEMENTED--. The percent change minimum for breaking early, if no meanigful change in metric is detected. The default is 0.01. stop_at_p : int, optional The number of selections to stop at. In other words, it selects up to `p` predictors, even if the optimal model is not yet found and more tests could be done. The default is 1000. stop_when : int, optional Stop the operations when the metric no longer continues to decrease, after evaluating the next stop_when columns. The default is 5. use_probabilities : bool, optional Whether or not to use predicted probabilities as the only other feature in the set, or to use the actual features in performing the forward selection. The default is False. return_type : str, optional Which object to return, can be either list, model, data, or all. The default is 'all'. serialize_flavor : str Which mode of downsaving data to use, currently supports 'feather' and 'msgpack'. Unfortunately, as of pandas 0.25.0, msgpack is no longer supported. use_threads : bool, optional Use threads instead of processes for parallel operation.f Note: using feather requires pyarrow Returns ------- list, model, data, or tuple of objects Either return as list, model, data, or all. The default is 'all'. """ t0 = dt() total_possible_complexity = int(((XY.shape[1]-1)**2) - ((XY.shape[1]-1)*((XY.shape[1]-1) - 1)/2)) if early_stop: if stop_at_p >= (XY.shape[1] - 1): complexity = total_possible_complexity stop_at_p = XY.shape[1] - 1 else: complexity = np.sum((XY.shape[1] - 1) - np.arange(0, stop_at_p)) else: complexity = total_possible_complexity stop_at_p = XY.shape[1] - 1 if verbose: print_time("Problem Complexity: " + str(complexity) + " Iterations Needed, " + str(total_possible_complexity) + " Possible...", t0, te=dt(), backsn=True) # Use this for an exhaustive search #all_combos = list(powerset(X.columns)) final_mod_cols = [] leftover_cols = [x for x in XY.columns if x != target_var] currentScore = np.inf #early_stop_counter = 0 loop_counter = 0 if n_jobs > 1: if verbose: print_time("\nPreparing Parallel Operation...", t0, te=dt()) save_name, save_path, save_ext, num_chunks = _prepare_for_parallel(XY) arg = {'save_path': save_path, 'save_name': save_name, 'save_ext': save_ext, 'metric': metric, 'model': model, 'target_var': target_var, 'family': family, 'use_probabilities': use_probabilities, 'mod_cols': final_mod_cols, 'Yprob': None } for i in range(XY.shape[1] - 1): scores = [] if use_probabilities and i > 1: # create current model and probability array initial_fit = model.fit(XY.loc[:, final_mod_cols], XY.loc[:, target_var]) Yprob = initial_fit.predict_proba(XY.loc[:, final_mod_cols])[:, 1] #print('\nprob gen step: ', Yprob.shape, "\n") if n_jobs > 1: arg['Yprob'] = Yprob if n_jobs > 1: if verbose: print_time("\nPerforming " + str(i + 1) + " of " + str(stop_at_p) + " Steps...", t0, te=dt(), backsn=True) loop_arg = [{'new_predictor': [col]} for col in leftover_cols] scores = p_prog_simp(arg, loop_arg, _doParallelForward, n_jobs, use_threads=use_threads) #return scores if verbose: loop_counter += len(loop_arg) updateProgBar(loop_counter, complexity, t0) else: for k, col in enumerate(leftover_cols): if use_probabilities and i > 1: #print('loop step: ', XY.loc[:, col].values.shape, "\n") XY_prob = pd.DataFrame({'Yprob': Yprob, col: XY.loc[:, col].values}, index=XY.index) score = _calcForwardAICs(XY_prob, XY.loc[:, target_var], model, metric, family) else: score = _calcForwardAICs(XY.loc[:, final_mod_cols + [col]], XY.loc[:, target_var], model, metric, family) scores.append(score) if verbose: loop_counter += 1 updateProgBar(loop_counter, complexity, t0) minScore = np.min(scores) if minScore > currentScore: print("\nNo further imporovement in the model. Breaking...") break else: currentScore = minScore minScoreLoc = np.where(minScore == np.array(scores))[0][0] final_mod_cols.append(leftover_cols[minScoreLoc]) del leftover_cols[minScoreLoc] # could be useful if I wanted to do something like accuracy, # but for AIC/BIC, this isn't necessary #if np.abs(percentIncrease(minScore, prev_minScore)) < perc_min: # early_stop_counter += 1 #else: # early_stop_counter = 0 # #if early_stop_counter >= stop_when: # print('Stopped Early') # break #if i >= stop_at_p: # print('Stopped at ', str(i), ' selected predictors') # break if n_jobs > 1: file_ = save_path + save_name + save_ext try: os.remove(file_) except FileNotFoundError: print('Could Not find file ', file_, 'Continuing...') if return_type == 'list': return final_mod_cols elif return_type == 'model': return model.fit(XY.loc[:, final_mod_cols], XY.loc[:, target_var]) elif return_type == 'data': return XY.loc[:, final_mod_cols] else: return final_mod_cols, model.fit(XY.loc[:, final_mod_cols], XY.loc[:, target_var]), XY.loc[:, final_mod_cols]
import numpy as np from sklearn.preprocessing import StandardScaler from timeit import default_timer as dt begin = dt() print('1') train = np.load('/A/VSE/data/coco_tensor_precomp/train_ims.npy') end = begin - dt() print('Done') exit() train = train.mean(-1).mean(-1) scaler = StandardScaler() train = scaler.fit_transform(train) print(train.shape) print('2') valid = np.load('/A/VSE/data/coco_tensor_precomp/val_ims.npy') valid = valid.mean(-1).mean(-1) valid = scaler.transform(valid) print(valid.shape) print('3') test = np.load('/A/VSE/data/coco_tensor_precomp/test_ims.npy') test = test.mean(-1).mean(-1) test = scaler.transform(test) print(test.shape) print('4') ftest = np.load('/A/VSE/data/f30k_tensor_precomp/test_ims.npy') ftest = ftest.mean(-1).mean(-1) ftest = scaler.transform(ftest) print(ftest.shape) print('5')
def main(args): global Driver_Instance rospy.init_node("autonomous", anonymous=True) config = yaml.load(open("./config/config.yaml"), Loader=yaml.FullLoader) Driver_Instance = Driver(gainL=args.GAIN_L, gainA=args.GAIN_A) LRS_Instance = LRS(Driver_Instance, clean=args.fresh) # Subscribe to relevant topics and services rospy.Subscriber(config["odometry_topic"], Odometry, route) # Wait for elevation map to be available rospy.loginfo("Waiting for topics to become online...") rospy.rostime.wallsleep(0.1) rospy.wait_for_message(config["odometry_topic"], Odometry, timeout=10) rospy.loginfo("OK!") try: # ROS Loop rospy.loginfo("Running the control loop") while not rospy.core.is_shutdown(): if Driver_Instance.flag == DriverStatus.HALT: try: ans = input("Execute? [y/n]") if ans == "n": exit(0) except ValueError: print("Invalid input") continue Driver_Instance.flag = DriverStatus.NORMAL start = dt() px, py, _ = Driver_Instance.get_current_loc() if LRS_Instance.has_landmarks(): AR = LRS_Instance.dictionary valid = 0 markers = [] for m_id, m in AR.items(): if m_id == 0 or m_id == 1: valid += 1 markers.append(m.loc()) if valid != 2: continue x, y = get_next((px, py), markers, scale=2, tol=0.8) rospy.logwarn( "\nx={:.2f} :: y={:.2f} :: in {:.2f} ms\n".format( x, y, 1000 * (dt() - start))) Driver_Instance.go_to_position(x, y) rospy.rostime.wallsleep(0.1) except (Exception, rospy.ROSException, KeyboardInterrupt): exc_type, exc_value, exc_traceback = sys.exc_info() rospy.logfatal("Program crashed or halted") traceback.print_exception(exc_type, exc_value, exc_traceback, limit=2, file=sys.stdout) rospy.core.signal_shutdown("exited") exit(1)