def test_epoch(epoch, experiment): testloaders, testsets = experiment.create_test_dataloaders() use_cuda = experiment.use_cuda net = experiment.net summaries = experiment.summaries criterion = experiment.criterion net.eval() utils.set_random_seeds(1234) with torch.no_grad(): for i, (testloader, testname) in enumerate(testloaders): stats = get_stats() print("Testing on {}".format(testname)) for batch_idx, input_set in enumerate(testloader): experiment.step = epoch * len(experiment.trainloader) + int( batch_idx / len(testloader) * len(experiment.trainloader)) experiment.iter = batch_idx torch.cuda.empty_cache() inputs, targets = input_set if use_cuda: inputs = inputs.cuda() targets = targets.cuda() # inputs, targets = experiment.data_preprocessing(inputs) # inputs, targets = Variable(inputs, requires_grad=False), Variable(targets, requires_grad=False) pred = torch.clamp(net(inputs), 0.0, 1.0) batch_loss = criterion(pred, targets) loss = batch_loss.mean() stats["loss"].update(loss.data) psnr_iter = metrics.psnr(pred, targets, maxval=1).mean().data ssim_iter = metrics.ssim(pred, targets) stats["psnr"].update(psnr_iter, pred.size(0)) stats["ssim"].update(ssim_iter.data, pred.size(0)) progress_bar( batch_idx, len(testloader), 'Loss: %.5f | PSNR: %.2f | SSIM: %.3f' % (stats["loss"].avg, stats["psnr"].avg, stats["ssim"].avg)) # save predicted image learned_img = Image.fromarray( (255 * pred[0, 0].cpu().data.numpy()).astype(np.uint8)) filename = os.path.join( './n3net-results', testsets[0][i].at(batch_idx).split( '/home/pacole2/Projects/datasets/DeepLesionTestPreprocessed/miniStudies/' )[1]) directory = os.path.dirname(filename) if not os.path.exists(directory): os.makedirs(directory) learned_img.save(os.path.join(filename)) del pred, inputs, targets add_summary(experiment, summaries, testname + "/epoch", epoch) for k, stat in stats.items(): add_summary(experiment, summaries, testname + "/" + k, stat.avg)
def train_mixup(nbEpochs=1): best_test_acc = 0 train_size = len(training_generator) train_size = len(training_generator.dataset) test_size = len(eval_generator) test_size = len(eval_generator.dataset) for epoch in range(nbEpochs): train_loss = 0.0 train_acc = 0.0 cnn.train() for index_batch, (inputs, labels) in enumerate(training_generator): inputs, labels = inputs.to(device), labels.long().to(device) inputs, lbl_a, lbl_b, lam = mixup_data(inputs, labels, alpha) #?? from torch.autograd import Variable #inputs, lbl_a, lbl_b = map(Variable, (inputs, lbl_a, lbl_b)) optimizer.zero_grad() outputs = cnn(inputs) #loss = criterion(outputs, labels) loss = mixup_criterion(criterion, outputs, lbl_a, lbl_b, lam) _, predicted = torch.max(outputs.data, 1) #train_acc += (predicted == labels).sum().item() train_acc += (lam * (predicted == lbl_a).sum().item() + (1 - lam) * (predicted == lbl_b).sum().item()) loss.backward() optimizer.step() train_loss += loss.item() progress_bar(index_batch, len(training_generator)) train_loss /= train_size train_acc /= train_size print("Train at Epoch:", epoch, " loss:", train_loss, " accuracy:", 100.0 * train_acc) test_loss = 0.0 test_acc = 0.0 cnn.eval() with torch.no_grad(): for index_batch, (inputs, labels) in enumerate(eval_generator): inputs, labels = inputs.to(device), labels.long().to(device) outputs = cnn(inputs) loss = criterion(outputs, labels) _, predicted = torch.max(outputs.data, 1) test_acc += (predicted == labels).sum().item() test_loss += loss.item() progress_bar(index_batch, len(eval_generator)) test_loss /= test_size test_acc /= test_size print("Test at Epoch:", epoch, " loss:", test_loss, " accuracy:", 100.0 * test_acc) lr = linear_adjust_learning_rate(epoch) save_result.append( [epoch, lr, train_loss, train_acc, test_loss, test_acc]) if epoch > 30 and test_acc > best_test_acc: best_test_acc = test_acc #Add save model save_model("Dorfer2", str(epoch))
def bootstrap_for_secondary(func2, block, samples, show_progressbar, *args): """Bootstrap for secondary observables. Every element of *arg is a list of two element of the form args[i]=[func_i, vec_i] and the final result is func2(<func_0(vec_0)>, ..,<func_n(vec_n)>) with blocksize "block" for blocking and "samples" resampling show_progressbar: if =1 show the progressbar """ if not isinstance(block, int): print("ERROR: blocksize has to be an integer!") sys.exit(1) if block<1: print("ERROR: blocksize has to be positive!") sys.exit(1) secondary_samples=np.empty(samples, dtype=np.float) for sample in range(samples): if show_progressbar==1: pb.progress_bar(sample, samples) primary_samples=[] numblocks=int(len(args[0][1])/block) end = block * numblocks resampling = np.random.randint(0,numblocks,size=numblocks) for arg in args: func_l, vec_l = arg # cut vec_in to have a number of columns multiple of "block" and apply "func" data=func_l(vec_l[:end]) #block block_sum_data=bs.blocksum(data, block)/float(block) #sample average tmp = np.average([block_sum_data[i] for i in resampling]) primary_samples.append(tmp) aux=func2(primary_samples) secondary_samples[sample]=aux ris=np.mean(secondary_samples) err=np.std(secondary_samples, ddof=1) return ris, err
def train_epoch(model, criterion, optimizer, train_loader, device=torch.device('cuda'), dtype=torch.float, collector=None): model.train() train_loss = 0 for batch_idx, batch_data in enumerate(train_loader): input, target, extra = batch_data['input'], batch_data[ 'target'], batch_data['extra'] input = input.to(device, dtype) if isinstance(target, torch.Tensor): target = target.to(device, dtype) elif isinstance(target, dict): for k in target: if isinstance(target[k], torch.Tensor): target[k] = target[k].to(device, dtype) #print('solver target[heatmap]: ', target['heatmap'].dtype) optimizer.zero_grad() output = model(input) loss = criterion(output, target) loss.backward() optimizer.step() if collector is None: train_loss += loss.item() progress_bar(batch_idx, len(train_loader), 'Loss: {0:.4e}'.format(train_loss / (batch_idx + 1))) #print('loss: {0: .4e}'.format(train_loss/(batch_idx+1))) else: model.eval() with torch.no_grad(): extra['batch_idx'], extra['loader_len'], extra[ 'batch_avg_loss'] = batch_idx, len( train_loader), loss.item() collector({ 'model': model, 'input': input, 'target': target, 'output': output, 'extra': extra }) # Keep train mode model.train()
def test_epoch(epoch, experiment): testloaders = experiment.create_test_dataloaders() use_cuda = experiment.use_cuda net = experiment.net summaries = experiment.summaries criterion = experiment.criterion net.eval() utils.set_random_seeds(1234) with torch.no_grad(): for testloader, testname in testloaders: stats = get_stats() print("Testing on {}".format(testname)) for batch_idx, inputs in enumerate(testloader): experiment.step = epoch * len(experiment.trainloader) + int( batch_idx / len(testloader) * len(experiment.trainloader)) experiment.iter = batch_idx torch.cuda.empty_cache() if use_cuda: inputs = inputs.cuda() inputs, targets = experiment.data_preprocessing(inputs) # CLAMP values to [0,1] after adding noise inputs = torch.clamp(inputs, min=0, max=1) inputs, targets = Variable( inputs, requires_grad=False), Variable(targets, requires_grad=False) pred = net(inputs) batch_loss = criterion(pred, targets) loss = batch_loss.mean() stats["loss"].update(loss.data) psnr_iter = metrics.psnr(pred, targets, maxval=1).mean().data ssim_iter = metrics.ssim(pred, targets) stats["psnr"].update(psnr_iter, pred.size(0)) stats["ssim"].update(ssim_iter.data, pred.size(0)) progress_bar( batch_idx, len(testloader), 'Loss: %.5f | PSNR: %.2f | SSIM: %.3f' % (stats["loss"].avg, stats["psnr"].avg, stats["ssim"].avg)) del pred, inputs, targets add_summary(experiment, summaries, testname + "/epoch", epoch) for k, stat in stats.items(): add_summary(experiment, summaries, testname + "/" + k, stat.avg)
def train(nbEpochs=1): best_test_acc = 0 train_size = len(training_generator.dataset) test_size = len(eval_generator.dataset) for epoch in range(nbEpochs): train_loss = 0.0 train_acc = 0.0 cnn.train() for index_batch, (inputs, labels) in enumerate(training_generator): inputs, labels = inputs.to(device), labels.long().to(device) optimizer.zero_grad() outputs = cnn(inputs) loss = criterion(outputs, labels) _, predicted = torch.max(outputs.data, 1) train_acc += (predicted == labels).sum().item() loss.backward() optimizer.step() train_loss += loss.item() progress_bar(index_batch, len(training_generator)) train_loss /= train_size train_acc /= train_size print("Train at Epoch:", epoch, " loss:", train_loss, " accuracy:", 100.0 * train_acc) test_loss = 0.0 test_acc = 0.0 cnn.eval() with torch.no_grad(): for index_batch, (inputs, labels) in enumerate(eval_generator): inputs, labels = inputs.to(device), labels.long().to(device) outputs = cnn(inputs) loss = criterion(outputs, labels) _, predicted = torch.max(outputs.data, 1) test_acc += (predicted == labels).sum().item() test_loss += loss.item() progress_bar(index_batch, len(eval_generator)) test_loss /= test_size test_acc /= test_size print("Test at Epoch:", epoch, " loss:", test_loss, " accuracy:", 100.0 * test_acc) lr = linear_adjust_learning_rate(epoch) save_result.append( [epoch, lr, train_loss, train_acc, test_loss, test_acc]) if epoch > 30 and test_acc > best_test_acc: best_test_acc = test_acc
def val_epoch(model, criterion, val_loader, device=torch.device('cuda'), dtype=torch.float, collector=None): model.eval() val_loss = 0 with torch.no_grad(): for batch_idx, batch_data in enumerate(val_loader): input, target, extra = batch_data['input'], batch_data[ 'target'], batch_data['extra'] input = input.to(device, dtype) if isinstance(target, torch.Tensor): target = target.to(device, dtype) elif isinstance(target, dict): for k in target: if isinstance(target[k], torch.Tensor): target[k] = target[k].to(device, dtype) output = model(input) loss = criterion(output, target) if collector is None: val_loss += loss.item() progress_bar( batch_idx, len(val_loader), 'Loss: {0:.4e}'.format(val_loss / (batch_idx + 1))) #print('loss: {0: .4e}'.format(val_loss/(batch_idx+1))) else: extra['batch_idx'], extra['loader_len'], extra[ 'batch_avg_loss'] = batch_idx, len( val_loader), loss.item() collector({ 'model': model, 'input': input, 'target': target, 'output': output, 'extra': extra }) # Keep eval mode model.eval()
async def refresh_applist(dryrun, skip, from_scratch=False, max_apps=None): local_applist = await steam.get_applist_from_local() if (skip): return local_applist async with web.Session(limit_per_host=20) as webSession: foreign_applist = await steam.get_applist_from_server( webSession, max_apps) styledprint.print_info('Apps in server:', len(foreign_applist)) styledprint.print_info('Apps in cache at start:', len(local_applist)) calname = 'refresh_applist' try: tasks = [] for name in foreign_applist: for app in foreign_applist[name]: if ((not from_scratch) and (name in local_applist) and (app in local_applist[name])): continue appid, typ = app link = steam.get_store_link(appid, typ) f = asyncio.ensure_future( steam.get_page(link, name, webSession)) f.add_done_callback( functools.partial(poolsubmit, calname, get_newgame_info, name, appid, typ, tasks)) tasks.append(f) if (len(tasks)): styledprint.print_info('async tasks:') await asyncio.gather(progressbar.progress_bar(tasks)) except Exception as e: styledprint.print_error( 'Error happened while running the async loop:', e) styledprint.print_error(traceback.format_exc()) pass fs = parallelism.wait_calname(calname) for f in fs: ext_applist = f.result() merge_applists(local_applist, ext_applist) styledprint.print_info( 'Apps in cache at end (duplicate names not in the count):', len(local_applist)) if (not dryrun): logging.debug('not dryrun so saving local_applist to disk') await steam.save_applist_to_local(local_applist) games = utils.DictCaseInsensitive() for game in local_applist: if (not game.lower().endswith('demo')): games[game] = local_applist[game] styledprint.print_info('Apps in cleaned cache:', len(games)) return games
def timer(seconds: int, prefix: str) -> None: """ Simplify usage of progress_bar since many of the options are going to be the same throughout """ remaining = seconds while True: progress = remaining / seconds * 100 progress_bar( progress=progress, length=30, complete=0, msg_complete="Requesting new data...", msg_prefix=prefix + f' ({time.strftime("%M:%S", time.gmtime(remaining))} remaining): ', suppress_nl=True, ) time.sleep(1) remaining -= 1 if remaining < 0: break
def start_loop(subGames, options, cachedgames, steamgames, winedb, cleansteamgames, cleanwinedb, urlsmapping, cpuCount): asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) webSession = web.Session(limit_per_host=((20 / cpuCount) + 1)) tasks = [] for name in subGames: game = subGames[name] tasks.append( asyncio.ensure_future(get_game_info(options, game, cachedgames, steamgames, winedb, cleansteamgames, cleanwinedb, name, urlsmapping, webSession), loop=loop)) loop.run_until_complete( asyncio.gather(progressbar.progress_bar(tasks), loop=loop)) loop.run_until_complete(webSession.close()) return subGames
async def get_ratings(): URL = 'https://appdb.winehq.org/objectManager.php?sClass=application&sTitle=Browse+Applications&iappVersion-ratingOp0=5&sOrderBy=appName&bAscending=true&iItemsPerPage=200&sappVersion-ratingData0=' ratings = utils.DictCaseInsensitive() async with web.Session(limit_per_host=200) as webSession: tasks = [] for e in Rating: tasks.append( asyncio.ensure_future(get_forOneRating(URL, e.name, webSession))) await asyncio.gather(progressbar.progress_bar(tasks)) for task in tasks: apps = task.result()[0] rating = task.result()[1] for app in apps: if (app[0] in ratings): ratings[app[0]].append(WineApp(app[1], rating)) else: ratings[app[0]] = [WineApp(app[1], rating)] return ratings
def fit(self, X): """Actual implementation of K-SVD algorithm. Args: - X: numpy 2d-array of dimensions : (len(signal) = D.shape[0], n_samples) TODO: add a stopping condition like an epsilon (and return corresponding number of iterations """ #Check wether data is coherent if self.D.shape[0] != X.shape[0]: raise TypeError("Supplied X matrix is not " "coherent with dictionary dimensions: you " "should have same number of lines for " "both the dictionary and the input data ") #ProgressBar setup print "Training dictionary over {} iterations".format(self.n_iter) progress = progressbar.progress_bar(self.n_iter) #self.n_iter iterations for it in range(self.n_iter): #Step 1: Compute sparse representation of X #given current dictionary D gamma = orthogonal_mp(self.D, X, n_nonzero_coefs = self.K, precompute = self.precompute) #Step 2: Adjust dictionary D and sparse #representation gamma at the same time #column by column for j in range(self.D.shape[1]): #Compute I = {indices of the signals in X #whose representations use jth column of D I = self.find_indices(gamma, j) #If one column is not used, it won't be until #the algorithm actually stops, which is a shame #So, we use heuristics: we set teh values of the #column to the worst represented columns of #X matrix if I == []: #find worst represented column in X d = self.worst_represented(gamma, X) #normalize d = d/np.linalg.norm(d) #set D column to d self.D[:,j] = d #jump to the next column optimization continue #Set D_j to zero self.D[:,j] = np.zeros_like(self.D[:,j]) #From now, we use a certain number of tricks #explained in [1] to accelerate the (therefore #approximate) K-SVD algorithm #TODO: try to understand better... -> maybe we could #solve the equations in the report ;) g = gamma[j,:][I].T d = X[:,I].dot(g) - self.D.dot(gamma[:,I].dot(g)) if d.sum() != 0: d = d/np.linalg.norm(d) g = (X[:,I].T).dot(d) - ((self.D.dot(gamma[:,I])).T).dot(d) #Store new values self.D[:,j] = d gamma[j,:][I] = g.T #Update progress bar progress.update(it) print(' Done!')
def train_epoch(experiment): use_cuda = experiment.use_cuda net = experiment.net optimizer = experiment.optimizer summaries = experiment.summaries criterion = experiment.criterion epoch = experiment.epoch lr = experiment.base_lr * experiment.learning_rate_decay(epoch) for group in experiment.optimizer.param_groups: group['lr'] = lr print('\nEpoch: %d, Learning rate: %f, Expdir %s' % (epoch, lr, experiment.expname)) net.train() stats = get_stats() trainloader = experiment.trainloader for batch_idx, inputs in enumerate(trainloader): experiment.epoch_frac = float(batch_idx) / len(trainloader) experiment.step = epoch * len(trainloader) + batch_idx experiment.iter = batch_idx ''' B = inputs.shape[0] for b in range(B): max_v = torch.max(inputs[b, :, :, :]) inputs[b, :,:,:] *= 1.0/max_v ''' if use_cuda: inputs = inputs.cuda() optimizer.zero_grad() inputs, targets = experiment.data_preprocessing(inputs) #np.save('/mnt/Lab-Kellman/Share/temp/inputs.npy', inputs.cpu().detach().numpy()) #np.save('/mnt/Lab-Kellman/Share/temp/targets.npy', targets.cpu().detach().numpy()) inputs, targets = Variable(inputs, requires_grad=False), Variable( targets, requires_grad=False) pred = net(inputs) batch_loss = criterion(pred, targets) loss = batch_loss.mean() psnr_iter = metrics.psnr(pred, targets, maxval=torch.max(targets)).mean().data ssim_iter = metrics.ssim(pred, targets) loss_v = loss.data stats["loss"].update(loss.data, pred.size(0)) stats["psnr"].update(psnr_iter, pred.size(0)) stats["ssim"].update(ssim_iter.data, pred.size(0)) loss.backward() del (loss) optimizer.step() if batch_idx % 10 == 0: experiment.writer.add_scalars('train/psnr', { 'psnr': stats["psnr"].ema, 'loss': stats["loss"].ema }, epoch * len(trainloader) + batch_idx) progress_bar( batch_idx, len(trainloader), 'Batch: %05d | Loss: %.5f | PSNR: %.2f | SSIM: %.3f' % (batch_idx, stats["loss"].ema, stats["psnr"].ema, stats["ssim"].ema)) if batch_idx % (len(trainloader) // 20) == 0: #progress_bar(batch_idx, len(trainloader),"") #print("Batch {:05d}, ".format(batch_idx), end='') #for k,stat in stats.items(): # print("{}: {:.4f}, ".format(stat.name, stat.avg), end='') #print("") dump_dir = '/mnt/Lab-Kellman/RawData/MachinLearning_Labelled_data/denoising/perf_training_record' fname = 'inputs_epoch_%d__batch_%d.npy' % (epoch, batch_idx) np.save(os.path.join(dump_dir, fname), inputs.detach().cpu().numpy()) fname = 'targets_epoch_%d__batch_%d.npy' % (epoch, batch_idx) np.save(os.path.join(dump_dir, fname), targets.detach().cpu().numpy()) fname = 'pred_epoch_%d__batch_%d.npy' % (epoch, batch_idx) np.save(os.path.join(dump_dir, fname), pred.detach().cpu().numpy()) stop = (lr == 0) progress_bar( batch_idx, len(trainloader), 'Loss: %.5f | PSNR: %.2f | SSIM: %.3f' % (stats["loss"].avg, stats["psnr"].avg, stats["ssim"].avg)) # test the network #add_summary(experiment, summaries, "train/epoch", epoch) #for k,stat in stats.items(): # add_summary(experiment, summaries, "train/" + k, stat.avg) print("") return stop
def train_epoch(experiment): use_cuda = experiment.use_cuda net = experiment.net optimizer = experiment.optimizer summaries = experiment.summaries criterion = experiment.criterion epoch = experiment.epoch lr = experiment.base_lr * experiment.learning_rate_decay(epoch) for group in experiment.optimizer.param_groups: group['lr'] = lr print('\nEpoch: %d, Learning rate: %f, Expdir %s' % (epoch, lr, experiment.expname)) net.train() stats = get_stats() trainloader = experiment.trainloader for batch_idx, inputs in enumerate(trainloader): experiment.epoch_frac = float(batch_idx) / len(trainloader) experiment.step = epoch * len(trainloader) + batch_idx experiment.iter = batch_idx if use_cuda: inputs = inputs.cuda() optimizer.zero_grad() inputs, targets = experiment.data_preprocessing_Blind_SFM(inputs) inputs, targets = Variable(inputs, requires_grad=False), Variable( targets, requires_grad=False) pred = net(inputs) batch_loss = criterion(pred, targets) loss = batch_loss.mean() psnr_iter = metrics.psnr(pred, targets, maxval=1).mean().data ssim_iter = metrics.ssim(pred, targets) stats["loss"].update(loss.data, pred.size(0)) stats["psnr"].update(psnr_iter, pred.size(0)) stats["ssim"].update(ssim_iter.data, pred.size(0)) loss.backward() del (loss) optimizer.step() if batch_idx % (len(trainloader) // 10) == 0: progress_bar(batch_idx, len(trainloader), "") print("Batch {:05d}, ".format(batch_idx), end='') for k, stat in stats.items(): print("{}: {:.4f}, ".format(stat.name, stat.avg), end='') print("") progress_bar( batch_idx, len(trainloader), 'Loss: %.5f | PSNR: %.2f | SSIM: %.3f' % (stats["loss"].ema, stats["psnr"].ema, stats["ssim"].ema)) stop = (lr == 0) progress_bar( batch_idx, len(trainloader), 'Loss: %.5f | PSNR: %.2f | SSIM: %.3f' % (stats["loss"].avg, stats["psnr"].avg, stats["ssim"].avg)) add_summary(experiment, summaries, "train/epoch", epoch) for k, stat in stats.items(): add_summary(experiment, summaries, "train/" + k, stat.avg) print("") return stop
def sample(draws, step, start=None, trace=None, tune=None, progressbar=True, model=None, random_seed=None): """ Draw a number of samples using the given step method. Multiple step methods supported via compound step method returns the amount of time taken. Parameters ---------- draws : int The number of samples to draw step : function A step function start : dict Starting point in parameter space (or partial point) Defaults to trace.point(-1)) if there is a trace provided and model.test_point if not (defaults to empty dict) trace : NpTrace or list Either a trace of past values or a list of variables to track (defaults to None) tune : int Number of iterations to tune, if applicable (defaults to None) progressbar : bool Flag for progress bar model : Model (optional if in `with` context) """ model = modelcontext(model) draws = int(draws) seed(random_seed) if start is None: start = {} if isinstance(trace, NpTrace) and len(trace) > 0: trace_point = trace.point(-1) trace_point.update(start) start = trace_point else: test_point = model.test_point.copy() test_point.update(start) start = test_point if not isinstance(trace, NpTrace): if trace is None: trace = model.named_vars try: trace = NpTrace(trace.values()) except AttributeError: trace = NpTrace(list(trace)) try: step = step_methods.CompoundStep(step) except TypeError: pass point = Point(start, model=model) progress = progress_bar(draws) for i in xrange(draws): if (i == tune): step = stop_tuning(step) point = step.step(point) trace = trace.record(point) if progressbar: progress.update(i) return trace
def integrator_yerr(x, y, dy, xmin, xmax, samples, spline_order, \ plot_spline=1, plot_band=1, plot_distribution=1, save_figs=0, show_progressbar=1): """ Compute the integral of the data in the range [xmin, xmax] using a spline interpolation of order "spline_order" and using "samples" bootstrap samples to evaluate the errors. plot_spline: if =1 plot the optimal fit together with the data plot_band: if =1 plot the 1std band together with data plot_distribtion: if =1 plot the bootstrapped distributions of the parameters save_figs: if =1 save the figures in png insted of displaying them show_progressbar: if =1 show the progressbar return the estimated value of the integral, its error and the bootstrap samples of the integral. """ mask = ((x <= xmax) & (x >= xmin)) x = x[mask] y = y[mask] dy = dy[mask] band_size = 1000 data_length = len(x) # array to store the bootstrapped results boot_sample = np.empty(samples, dtype=np.float) if plot_band == 1: x_band = np.linspace(xmin, xmax, band_size) boot_band = np.empty((band_size, samples), dtype=np.float) for i in range(samples): if show_progressbar == 1: pb.progress_bar(i, samples) # bootstrap sample booty = y + np.random.normal(0, dy, data_length) ris = 1 err = 1 # spline interpolation s = interp.UnivariateSpline(x, booty, k=spline_order, s=0) # integrate the spline interpolation boot_sample[i] = s.integral(xmin, xmax) if plot_band == 1: boot_band[:, i] = s(x_band) # optimal parameters and errors ris = np.mean(boot_sample) err = np.std(boot_sample, ddof=1) if plot_spline == 1: x_aux = np.linspace(xmin, xmax, 1000) s = interp.UnivariateSpline(x, y, k=spline_order, s=0) y_aux = s(x_aux) plt.figure('Plot of the spline interpolation') plt.xlim(0.9 * xmin, 1.1 * xmax) plt.errorbar(x, y, yerr=dy, fmt='ob', ms=5) plt.plot(x_aux, y_aux, 'r-') if plot_band == 1: band_mean = np.mean(boot_band, axis=1) band_std = np.std(boot_band, axis=1) plt.plot(x_band, band_mean + band_std, 'g-') plt.plot(x_band, band_mean - band_std, 'g-') if save_figs == 1: plt.savefig('spline.png') else: plt.show() if plot_distribution == 1: plt.figure('Bootstrapped distribution of the integral') plt.xlabel('value of the integral') plt.ylabel('distribution histogram') plt.hist(boot_sample, bins='auto') if save_figs == 1: plt.savefig('boot_integral.png') else: plt.show() return ris, err, boot_sample
def fit(self, X): """Actual implementation of K-SVD algorithm. Args: - X: numpy 2d-array of dimensions : (len(signal) = D.shape[0], n_samples) TODO: add a stopping condition like an epsilon (and return corresponding number of iterations """ #Check wether data is coherent if self.D.shape[0] != X.shape[0]: raise TypeError("Supplied X matrix is not " "coherent with dictionary dimensions: you " "should have same number of lines for " "both the dictionary and the input data ") #ProgressBar setup print "Training dictionary over {} iterations".format(self.n_iter) progress = progressbar.progress_bar(self.n_iter) #self.n_iter iterations for it in range(self.n_iter): #Step 1: Compute sparse representation of X #given current dictionary D gamma = orthogonal_mp(self.D, X, n_nonzero_coefs=self.K, precompute=self.precompute) #Step 2: Adjust dictionary D and sparse #representation gamma at the same time #column by column for j in range(self.D.shape[1]): #Compute I = {indices of the signals in X #whose representations use jth column of D I = self.find_indices(gamma, j) #If one column is not used, it won't be until #the algorithm actually stops, which is a shame #So, we use heuristics: we set teh values of the #column to the worst represented columns of #X matrix if I == []: #find worst represented column in X d = self.worst_represented(gamma, X) #normalize d = d / np.linalg.norm(d) #set D column to d self.D[:, j] = d #jump to the next column optimization continue #Set D_j to zero self.D[:, j] = np.zeros_like(self.D[:, j]) #From now, we use a certain number of tricks #explained in [1] to accelerate the (therefore #approximate) K-SVD algorithm #TODO: try to understand better... -> maybe we could #solve the equations in the report ;) g = gamma[j, :][I].T d = X[:, I].dot(g) - self.D.dot(gamma[:, I].dot(g)) if d.sum() != 0: d = d / np.linalg.norm(d) g = (X[:, I].T).dot(d) - ((self.D.dot(gamma[:, I])).T).dot(d) #Store new values self.D[:, j] = d gamma[j, :][I] = g.T #Update progress bar progress.update(it) print(' Done!')
def sample(draws, step, start=None, trace=None, tune=None, progressbar=True, model=None, random_seed=None): """ Draw a number of samples using the given step method. Multiple step methods supported via compound step method returns the amount of time taken. Parameters ---------- draws : int The number of samples to draw step : function A step function start : dict Starting point in parameter space (or partial point) Defaults to trace.point(-1)) if there is a trace provided and model.test_point if not (defaults to empty dict) trace : NpTrace or list Either a trace of past values or a list of variables to track (defaults to None) tune : int Number of iterations to tune, if applicable (defaults to None) progressbar : bool Flag for progress bar model : Model (optional if in `with` context) """ model = modelcontext(model) draws = int(draws) seed(random_seed) if start is None: start = {} if isinstance(trace, NpTrace) and len(trace) > 0: trace_point = trace.point(-1) trace_point.update(start) start = trace_point else: test_point = model.test_point.copy() test_point.update(start) start = test_point if not isinstance(trace, NpTrace): if trace is None: trace = model.named_vars try: trace = NpTrace(trace.values()) except AttributeError: trace = NpTrace(list(trace)) try: step = step_methods.CompoundStep(step) except TypeError: pass point = Point(start, model=model) progress = progress_bar(draws) try: for i in xrange(draws): if (i == tune): step = stop_tuning(step) point = step.step(point) trace = trace.record(point) if progressbar: progress.update(i) except KeyboardInterrupt: pass finally: return trace
def fit_with_yerr(x, y, dy, xmin, xmax, func, params, samples, \ stop_param=1.0e-15, plot_fit=1, plot_band=1, plot_residuals=1, \ plot_distribution=1, save_figs=0, show_progressbar=1): """ Perform a fit to data on [xmin, xmax] with the function func(x, param), using "samples" bootstrap samples to evaluate the errors. stop_param: stopping parameter for the least square regression plot_fit: if =1 plot the optimal fit together with the data plot_band: if =1 plot the 1std band together with data plot_residuals: if =1 plot residuals after convergence plot_distribtion: if =1 plot the bootstrapped distributions of the parameters save_figs: if =1 save the figures in png insted of displaying them show_progressbar: if =1 show the progressbar return the optimal vales of the parameters, their errors, the value of chi^2,the number of dof, the p-value and the bootstrap samples of the parameters. """ mask = ((x<=xmax) & (x>=xmin)) x=x[mask] y=y[mask] dy=dy[mask] band_size=1000 data_length=len(x) # array to store the bootstrapped results boot_sample=np.empty((len(params), samples), dtype=np.float) if plot_band==1: x_band=np.linspace(xmin, xmax, band_size) boot_band=np.empty((band_size, samples), dtype=np.float) for i in range(samples): if show_progressbar==1: pb.progress_bar(i, samples) # bootstrap sample booty=y+np.random.normal(0, dy, data_length) # least square regression ris = opt.leastsq(_residuals_yerr, params, ftol=stop_param, args=(x, booty, dy, func)) boot_sample[:,i]=ris[0] if plot_band==1: boot_band[:,i]=func(x_band, ris[0]) # optimal parameters and errors ris=np.mean(boot_sample, axis=1) err=np.std(boot_sample, axis=1, ddof=1) # auxilliary stuff opt_res=_residuals_yerr(ris, x, y, dy, func) chi2=np.sum(opt_res*opt_res) dof=data_length - len(params) pvalue=1.0 - stats.chi2.cdf(chi2, dof) if plot_fit==1: x_aux=np.linspace(xmin, xmax, 1000) y_aux=func(x_aux, ris) plt.figure('Best fit (chi2/dof=%.4f/%d=%f)' % (chi2, dof, chi2/dof)) plt.xlim(0.9*xmin, 1.1*xmax) plt.errorbar(x, y, yerr=dy, fmt='ob', ms=5) plt.plot(x_aux,y_aux,'r-') if plot_band==1: band_mean=np.mean(boot_band, axis=1) band_std=np.std(boot_band, axis=1) plt.plot(x_band, band_mean + band_std,'g-') plt.plot(x_band, band_mean - band_std,'g-') if save_figs==1: plt.savefig('fit.png') else: plt.show() if plot_residuals==1: x_aux=np.linspace(xmin, xmax, 1000) y_aux=np.ones(len(x_aux)) plt.figure('Residuals') plt.xlim(0.9*xmin, 1.1*xmax) plt.errorbar(x, opt_res, yerr=1, fmt='ob', ms=5) plt.plot(x_aux, -2*y_aux, 'g:') plt.plot(x_aux, -y_aux, 'r--') plt.plot(x_aux, 0*y_aux, 'r-') plt.plot(x_aux, y_aux, 'r--') plt.plot(x_aux, 2*y_aux, 'g:') if save_figs==1: plt.savefig('residuals.png') else: plt.show() if plot_distribution==1: for i in range(len(params)): plt.figure('Bootstrapped distribution of param[%d]' % i) plt.xlabel('param[%d] values' % i) plt.ylabel('distribution histogram') plt.hist(boot_sample[i], bins='auto') if save_figs==1: plt.savefig('param'+str(i)+'.png') else: plt.show() return ris, err, chi2, dof, pvalue, boot_sample