def get_sub_tensor(year, home_ids, data_type): if data_type == 'missing': raw_data = np.load("../data/data-2013-2017-missing.npy", allow_pickle=True).item() ORDER = APPLIANCE_ORDER_MISSING if data_type == 'filter': raw_data = np.load("../data/data-2013-2017-missing-filtered.npy", allow_pickle=True).item() ORDER = APPLIANCE_ORDER_MISSING if data_type == 'observed': raw_data = np.load("../data/data-2013-2017-observed-filtered.npy", allow_pickle=True).item() ORDER = APPLIANCE_ORDER_OBSERVED if data_type == 'artificial': raw_data = np.load( "../data/data-2013-2017-observed-filtered-artificial.npy", allow_pickle=True).item() ORDER = APPLIANCE_ORDER_OBSERVED data_year = raw_data[year] tensor = np.empty((len(home_ids), len(ORDER), 12)) tensor[:] = np.NaN for i, home_id in enumerate(home_ids): for j, appliance in enumerate(ORDER): try: tensor[i, j, :12] = data_year[home_id][appliance].values except: continue return tensor
def re_scatter(url_re, url_init, threshold = 0.05): # Scatter plot the relative errors given by the prediction of neural networks. fig = plt.figure() ax = fig.add_subplot(111, projection='3d', num = 8000) redata = np.load(url_re) re = redata['re'] initdata = np.load(url_init) initconditions = initdata['x0'] xs = initconditions[:, 1] ys = initconditions[:, 2] zs = initconditions[:, 3] sup_index = re > threshold re[sup_index] = 1 sub_index = re <= threshold re[sub_index] = 0 # rate gives the ratio of well fitted trajectories. rate = (num - np.count_nonzero(re))/num print('Rate = {:+1.3e}'.format(rate)) color = re p = ax.scatter(xs, ys, zs, c=color, cmap='viridis') fig.colorbar(p) ax.set_xlabel('x2') ax.set_ylabel('x3') ax.set_zlabel('x4') plt.show()
def summarize_res(sname, datasize): print(sname) res = [] times = [] for i in range(100): PATH = ROOT_PATH + "/MMR_IVs/results/zoo/" + sname + "/" filename = os.path.join( PATH, str(date.today()), 'LMO_errs_{}_nystr_prodkern_{}.npy'.format(i, datasize)) if os.path.exists(filename): tmp_res = np.load(filename, allow_pickle=True) if tmp_res[-1] is not None: res += [tmp_res[-1]] time_path = os.path.join( PATH, str(date.today()), '/LMO_errs_{}_nystr_prodkern_{}_time.npy'.format(i, datasize)) if os.path.exists(time_path): t = np.load(time_path) times += [t] res = np.array(res) times = np.array(times) res = remove_outliers(res) times = np.sort(times)[:80] print(times) print('mean, std: ', np.mean(res), np.std(res)) print('time: ', np.mean(times), np.std(times))
def read_image_data_knn(): print('Reading image data ...') train_x = np.load('../../Data/data_train_knn.npy') train_y = np.load('../../Data/train_labels_knn.npy') test_x = np.load('../../Data/data_test.npy') return (train_x, train_y, test_x)
def read_image_data(): # Loading the dataset print('Reading image data ...') temp = np.load('../../Data/data_train.npz') train_x = temp['data_train'] temp = np.load('../../Data/labels_train.npz') train_y_integers = temp['labels_train'] temp = np.load('../../Data/data_test.npz') test_x = temp['data_test'] return (train_x, train_y_integers, test_x)
def _load_data(self, path_state_action, path_delta): """ Loads the state-action and delta values :param path_state_action: Path where the state-actions are stored :param path_delta: Path where the delta values are stored :return: """ self.state_action_pairs = np.load(open(f"{path_state_action}", "rb")) self.state_delta = np.load(open(f"{path_delta}", "rb")) self.data_loaded = True
def load(self, folder): # Load mu if self.d == 0: w = np.load(os.path.join(folder, 'coef0.npy')).reshape(-1) else: w = np.load(os.path.join(folder, 'w.npy')) V = np.load(os.path.join(folder, 'V.npy')) self.V = V[:self.n_users] self.item_embed = V[self.n_users:] self.w = w[:self.n_users] self.item_bias = w[self.n_users:] print('w user', self.w.shape) print('w item', self.item_bias.shape)
def load_caltech100(images_fname, labels_fname): # if images(64).npy or output_labels(64).npy missing then # print('Generating data because it does not exist. Note that this may take a while') # gen_data() one_hot = lambda x, K: np.array(x[:, None] == np.arange(K)[None, :], dtype=int) images = np.load(images_fname) output_labels = np.load(labels_fname) output_labels = np.load(labels_fname) train_images, valid_images, train_labels, valid_labels = train_test_split( images, output_labels, test_size=0.20, random_state=1729) train_labels = one_hot(train_labels, 101) valid_labels = one_hot(valid_labels, 101) return train_images, train_labels, valid_images, valid_labels
def construct_cp_df(critfinder_row): finder_kwargs = critfinder_row.finder_kwargs finder_dir = os.path.dirname(critfinder_row.finder_json) output_dir = os.path.join(finder_dir, OUTPUT_DIR_NAME) output_npzs = [np.load(os.path.join(output_dir, elem)) for elem in os.listdir(output_dir) if elem.endswith("npz")] row_dictionaries = [] for output_npz in output_npzs: row_dictionary = {} row_dictionary.update(finder_kwargs) if "theta" in output_npz.keys(): row_dictionary["thetas"] = output_npz["theta"] row_dictionary["final_theta"] = row_dictionary["thetas"][-1] row_dictionary["run_length"] = len(row_dictionary["thetas"]) if "f_theta" in output_npz.keys(): row_dictionary["losses"] = output_npz["f_theta"] row_dictionary["final_loss"] = row_dictionary["losses"][-1] if "g_theta" in output_npz.keys(): row_dictionary["squared_grad_norms"] = 2 * output_npz["g_theta"] row_dictionary["final_squared_grad_norm"] = row_dictionary["squared_grad_norms"][-1] row_dictionaries.append(row_dictionary) return pd.DataFrame(row_dictionaries)
def mfvi_elbo(model): """ computes the elbo value for the saved MFVI model """ lnpdf, D, param_names = make_model(model) mfvi = vi.DiagMvnBBVI(lnpdf, D, lnpdf_is_vectorized=True) mfvi_lam = np.load(model + "_output/mfvi.npy") elbo_val = mfvi.elbo_mc(mfvi_lam, n_samps=Nsamp) return elbo_val
def apply_rotation(obj, coord_old, src_folder): coord_vec_ls = [] for i in range(3): f = os.path.join(src_folder, 'coord{}_vec.npy'.format(i)) coord_vec_ls.append(np.load(f)) s = obj.shape coord0_vec, coord1_vec, coord2_vec = coord_vec_ls coord_old = np.tile(coord_old, [s[0], 1]) coord1_old = coord_old[:, 0] coord2_old = coord_old[:, 1] coord_old = np.stack([coord0_vec, coord1_old, coord2_old], axis=1).transpose() # print(sess.run(coord_old)) obj_channel_ls = np.split(obj, s[3], 3) obj_rot_channel_ls = [] for channel in obj_channel_ls: channel_flat = channel.flatten() ind = coord_old[0] * (s[1] * s[2]) + coord_old[1] * s[2] + coord_old[2] ind = ind.astype('int') obj_chan_new_val = channel_flat[ind] obj_rot_channel_ls.append(np.reshape(obj_chan_new_val, s[:-1])) obj_rot = np.stack(obj_rot_channel_ls, axis=3) return obj_rot
def re_processing(url_load, url_re): # url_load gives the url for loading the optimal weight of NN. # url_re gives the url for saving the computed relative errors. global inputs, targets data = np.load(url_load) opt_weights = data['optweights'].item() x_scaler = data['x_scaler'].item() y_scaler = data['y_scaler'].item() RelativeError = [] for datafile in range(8000): x_traj_test, y_traj_test = sample_multitraj(datafile, datafile+1) inputs = x_scaler.transform(x_traj_test) targets = y_scaler.transform(y_traj_test) outputs = nn_encode_foward_decode(opt_weights, inputs) outputs = y_scaler.inverse_transform(outputs) targets = y_scaler.inverse_transform(targets) re = np.mean([np.linalg.norm(targets[i] - outputs[i]) / np.linalg.norm(targets[i]) for i in range(len(targets))]) print('sample {:d} relative training norm error {:+1.4e}'.format(datafile, re)) RelativeError.append(re) np.savez(url_re, re = RelativeError, url_load = url_load)
def load_feature_array(class_name, feature_folder='inception_features'): """ class_name can be airplane, automobile, ... or wholedata. """ npy_path = cifar10_file(feature_folder, '{}.npy'.format(class_name)) array = np.load(npy_path) return array
def load_data_array(class_name): """ class_name can be airplane, automobile, .... """ npy_path = cifar10_file('data', '{}.npy'.format(class_name)) array = np.load(npy_path) return array
def load_folded(file): """ Load a folded value and its pattern from a file together with any additional data. Note that ``pattern`` must be registered with ``register_pattern_json`` to use ``load_folded``. Parameters --------------- file: String or file A file or filename of data saved with ``save_folded``. Returns ----------- folded_val: The folded value of the saved parameter. pattern: The ``paragami`` pattern of the saved parameter. data: The data as returned from ``np.load``. Additional saved values will exist as keys of ``data``. """ data = np.load(file) pattern = get_pattern_from_json(str(data['pattern_json'])) folded_val = pattern.fold(data['flat_val'], free=False) return folded_val, pattern, data
def classification_data(seed=0): """ Load 2D data. 2 Classes. Class labels generated from a 2-2-1 network. :param seed: random number seed :return: """ npr.seed(seed) data = np.load("./data/2D_toy_data_linear.npz") x = data['x'] y = data['y'] ids = np.arange(x.shape[0]) npr.shuffle(ids) # 75/25 split num_train = int(np.round(0.01 * x.shape[0])) x_train = x[ids[:num_train]] y_train = y[ids[:num_train]] x_test = x[ids[num_train:]] y_test = y[ids[num_train:]] mu = np.mean(x_train, axis=0) std = np.std(x_train, axis=0) x_train = (x_train - mu) / std x_test = (x_test - mu) / std train_stats = dict() train_stats['mu'] = mu train_stats['sigma'] = std return x_train, y_train, x_test, y_test, train_stats
def test(): # Pick a trajectory and check the prediction of the nn on this trajectory global inputs, targets data = np.load('data/sample_1/optweights_tanh_minmax_random1000shuffle_G20_layer2_2.npz') opt_weights = data['optweights'].item() x_scaler = data['x_scaler'].item() y_scaler = data['y_scaler'].item() # the number of the trajectory picked datafile = 4412 x_traj_test, y_traj_test = sample_multitraj(datafile, datafile+1) print(x_traj_test[0,:]) inputs = x_scaler.transform(x_traj_test) targets = y_scaler.transform(y_traj_test) outputs = nn_encode_foward_decode(opt_weights, inputs) x0 = inputs[0,:].reshape((1,4)) outputs = np.concatenate((x0, outputs), axis = 0) targets = np.concatenate((x0, targets), axis = 0) outputs = y_scaler.inverse_transform(outputs) targets = y_scaler.inverse_transform(targets) re = np.mean([np.linalg.norm(targets[i] - outputs[i]) / np.linalg.norm(targets[i]) for i in range(len(targets))]) # re = np.mean([np.linalg.norm(targets[i] - outputs[i])**2 for i in range(len(targets))]) print('sample Relative training norm error {:+1.4e}'.format(re)) figplot(outputs, url = None)
def npvi_elbo(model, ncomp): """ computes the elbo value for saved NPVI models """ lnpdf, D, param_names = make_model(model) npvi = vi.NPVI(lnpdf, D=D) npvi_theta = np.load(model + "_output/npvi_%d-comp.npy" % ncomp) print " npvi n comp: ", npvi_theta.shape[0] elbo_val = npvi.mc_elbo(npvi_theta, nsamps=Nsamp) return elbo_val
def get_grid_psi(data_set='BTLS'): fgridpsi = '%s_grid_psi.npy' % dataset.lower() if os.path.isfile(fgridpsi): return np.load(fgridpsi) else: grid_psi = np.stack([psi(tt) for tt in theta_xygrid]) np.save(fgridpsi, grid_psi) return grid_psi
def load_data_array(self, class_name): """ class_name can be airplane, automobile, .... """ filename = '{}.npy'.format(class_name) npy_path = self.data_path('data', filename) array = np.load(npy_path) return array
def load(self, desc=""): fn = self.default_file_name() if len(desc) != 0: fn += "_%s" % desc p = np.load(fn + ".npz") self.model.ps = p["gen"] self.model.sync_ps()
def fetch_trajectory(trajectory_path): if trajectory_path.endswith(".npz"): results_npz = np.load(trajectory_path) trajectory = results_npz["theta"] else: raise NotImplementedError( "trajectory_path {} not understood".format(trajectory_path)) return trajectory
def restore_param_arrays_from_checkpoint(self): arr = np.load( os.path.join(self.output_folder, 'opt_params_checkpoint.npy')) if len(self.params_list) > 0: for i, param_name in enumerate(self.params_list): self.params_whole_array_dict[param_name] = arr[i] return
def load_DR10QSO_train_test_idx(split_type="random", spectra_loc=SPECTRA_LOC, in_plate_dir=True): if not split_type in split_types: raise ValueError('split_type %s not found!' % split_type) # grab the train/test split f = file(join(TRAIN_TEST_SPLIT_LOC, "split_%s.bin" % split_type), 'rb') train_idx = np.load(f) test_idx = np.load(f) f.close() # select DR10Q_v2 file dr10filepath = join(DR10QSO_LOC, 'DR10Q_v2.fits') dr10file = fitsio.FITS(dr10filepath) qso_psf_flux = dr10file[1]['PSFFLUX'].read() qso_psf_flux_ivar = dr10file[1]['IVAR_PSFFLUX'].read() qso_z = dr10file[1]['Z_VI'].read() qso_psf_mags = dr10file[1]['PSFMAG'].read() # remove zero'd fluxes zero_idx = np.any(qso_psf_flux == 0, axis=1) print " found %d rows with zero fluxes (removing from training)" % zero_idx.sum( ) if zero_idx.sum() > 0: for bad_idx in np.where(zero_idx)[0]: train_idx = train_idx[train_idx != bad_idx] # construct NERSC file paths print "constructing NERSC file paths to %d quasar spectra" % ( qso_z.shape[0]) qso_ids = dr10file[1][['PLATE', 'MJD', 'FIBERID']].read() if in_plate_dir: spec_url_template = join(spectra_loc, "%04d/spec-%04d-%05d-%04d.fits") qso_files = [ spec_url_template % (qid[0], qid[0], qid[1], qid[2]) for qid in qso_ids ] else: spec_url_template = join(spectra_loc, "spec-%04d-%05d-%04d.fits") qso_files = [ spec_url_template % (qid[0], qid[1], qid[2]) for qid in qso_ids ] return qso_psf_flux, qso_psf_flux_ivar, qso_psf_mags, qso_z, \ qso_files, train_idx, test_idx
def __init__(self, configurations, parameters, controls): self.configurations = configurations self.parameters = parameters self.controls = controls flat_args, self.unflatten = flatten(self.controls) self.gx = grad(self.cost) self.J = jacobian(self.forward) self.hx = hessian_vector_product(self.cost) self.hvp = hvp(self.hx) y0, t_total, N_total, number_group, population_proportion, \ t_control, number_days_per_control_change, number_control_change_times, number_time_dependent_controls = configurations self.N_total = N_total self.number_group = number_group self.t_control = t_control self.dimension = len(self.t_control) self.number_time_dependent_controls = number_time_dependent_controls self.y0 = y0 self.t_total = t_total self.interpolation = piecewiseLinear self.load_data(fips) self.initialization() if number_group > 1: # contact matrix school_closure = True # calendar from February 15th weekday = [2, 3, 4, 5, 6] # calendar from April 1st # weekday = [0, 1, 2, 5, 6] # calendar from May 1st # weekday = [0, 3, 4, 5, 6] calendar = np.zeros(1000 + 1, dtype=int) # set work days as 1 and school days as 2 for i in range(1001): if np.remainder(i, 7) in weekday: calendar[i] = 1 if not school_closure: # and i < 45 calendar[i] = 2 self.calendar = calendar contact = np.load("utils/contact_matrix.npz") self.c_home = contact["home"] self.c_school = contact["school"] self.c_work = contact["work"] self.c_other = contact["other"] self.contact_full = self.c_home + 5. / 7 * ( (1 - school_closure) * self.c_school + self.c_work) + self.c_other
def load_model(self): """Loads the last checkpoint to continue training or for evaluation.""" file = np.load(self.checkpoint_path) self.epoch = file['epoch'] self.fourier_freq, self.fourier_offset = file[ 'fourier_features'] # feature parameters self.θ, self.Σ = file['θ'], file['Σ'] # policy parameters self.α, self.η = file['α'], file['η'] # dual parameters print(f"LOADED Model at epoch {self.epoch}")
def load_redshift_samples(fname): bname = os.path.splitext(fname)[0] th_samples = np.load(bname + ".npy") with open(bname + ".pkl", 'rb') as handle: ll_samps = pickle.load(handle) q_idx = pickle.load(handle) qso_info = pickle.load(handle) chain_idx = pickle.load(handle) return th_samples, ll_samps, q_idx, qso_info, chain_idx
def summarize_res(sname, datasize): print(sname) res = [] times = [] for i in range(100): PATH = ROOT_PATH + "/our_methods/results/mendelian/" + sname + "/" filename = PATH + 'LMO_errs_{}_nystr_{}.npy'.format(i, datasize) if os.path.exists(filename): tmp_res = np.load(filename, allow_pickle=True) if tmp_res[-1] is not None: res += [tmp_res[-1]] time_path = PATH + '/LMO_errs_{}_nystr_{}_time.npy'.format(i, datasize) if os.path.exists(time_path): t = np.load(time_path) times += [t] res = np.array(res) res = remove_outliers(res) print('mean, std: ', np.mean(res), np.std(res))
def load_basis_samples(fname): bname = os.path.splitext(fname)[0] th_samples = np.load(bname + ".npy") with open(bname + ".pkl", 'rb') as handle: ll_samps = pickle.load(handle) lam0 = pickle.load(handle) lam0_delta = pickle.load(handle) parser = pickle.load(handle) chain_idx = pickle.load(handle) return th_samples, ll_samps, lam0, lam0_delta, parser, chain_idx
def generate_dataset(self, cachefile='data/decoy-mnist.npz'): if cachefile and os.path.exists(cachefile): cache = np.load(cachefile) data = tuple([cache[f] for f in sorted(cache.files)]) else: data = self._generate_dataset(os.path.dirname(cachefile)) if cachefile: np.savez(cachefile, *data) self.Xr, self.X, self.y, self.E, self.Xtr, self.Xt, self.yt, self.Et = data self.status.initialized = True
def load_feat_array(): """Open the numpy array of all graphs' data.""" return np.load('../data/feat_array.npy')