def get_sub_tensor(year, home_ids, data_type):
if data_type == 'missing':
raw_data = np.load("../data/data-2013-2017-missing.npy",
allow_pickle=True).item()
ORDER = APPLIANCE_ORDER_MISSING
if data_type == 'filter':
raw_data = np.load("../data/data-2013-2017-missing-filtered.npy",
allow_pickle=True).item()
ORDER = APPLIANCE_ORDER_MISSING
if data_type == 'observed':
raw_data = np.load("../data/data-2013-2017-observed-filtered.npy",
allow_pickle=True).item()
ORDER = APPLIANCE_ORDER_OBSERVED
if data_type == 'artificial':
raw_data = np.load(
"../data/data-2013-2017-observed-filtered-artificial.npy",
allow_pickle=True).item()
ORDER = APPLIANCE_ORDER_OBSERVED
data_year = raw_data[year]
tensor = np.empty((len(home_ids), len(ORDER), 12))
tensor[:] = np.NaN
for i, home_id in enumerate(home_ids):
for j, appliance in enumerate(ORDER):
try:
tensor[i, j, :12] = data_year[home_id][appliance].values
except:
continue
return tensor
def re_scatter(url_re, url_init, threshold = 0.05):
# Scatter plot the relative errors given by the prediction of neural networks.
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d', num = 8000)
redata = np.load(url_re)
re = redata['re']
initdata = np.load(url_init)
initconditions = initdata['x0']
xs = initconditions[:, 1]
ys = initconditions[:, 2]
zs = initconditions[:, 3]
sup_index = re > threshold
re[sup_index] = 1
sub_index = re <= threshold
re[sub_index] = 0
# rate gives the ratio of well fitted trajectories.
rate = (num - np.count_nonzero(re))/num
print('Rate = {:+1.3e}'.format(rate))
color = re
p = ax.scatter(xs, ys, zs, c=color, cmap='viridis')
fig.colorbar(p)
ax.set_xlabel('x2')
ax.set_ylabel('x3')
ax.set_zlabel('x4')
plt.show()
def summarize_res(sname, datasize):
print(sname)
res = []
times = []
for i in range(100):
PATH = ROOT_PATH + "/MMR_IVs/results/zoo/" + sname + "/"
filename = os.path.join(
PATH, str(date.today()),
'LMO_errs_{}_nystr_prodkern_{}.npy'.format(i, datasize))
if os.path.exists(filename):
tmp_res = np.load(filename, allow_pickle=True)
if tmp_res[-1] is not None:
res += [tmp_res[-1]]
time_path = os.path.join(
PATH, str(date.today()),
'/LMO_errs_{}_nystr_prodkern_{}_time.npy'.format(i, datasize))
if os.path.exists(time_path):
t = np.load(time_path)
times += [t]
res = np.array(res)
times = np.array(times)
res = remove_outliers(res)
times = np.sort(times)[:80]
print(times)
print('mean, std: ', np.mean(res), np.std(res))
print('time: ', np.mean(times), np.std(times))
def read_image_data_knn():
print('Reading image data ...')
train_x = np.load('../../Data/data_train_knn.npy')
train_y = np.load('../../Data/train_labels_knn.npy')
test_x = np.load('../../Data/data_test.npy')
return (train_x, train_y, test_x)
def read_image_data():
# Loading the dataset
print('Reading image data ...')
temp = np.load('../../Data/data_train.npz')
train_x = temp['data_train']
temp = np.load('../../Data/labels_train.npz')
train_y_integers = temp['labels_train']
temp = np.load('../../Data/data_test.npz')
test_x = temp['data_test']
return (train_x, train_y_integers, test_x)
def _load_data(self, path_state_action, path_delta):
"""
Loads the state-action and delta values
:param path_state_action: Path where the state-actions are stored
:param path_delta: Path where the delta values are stored
:return:
"""
self.state_action_pairs = np.load(open(f"{path_state_action}", "rb"))
self.state_delta = np.load(open(f"{path_delta}", "rb"))
self.data_loaded = True
def load(self, folder):
# Load mu
if self.d == 0:
w = np.load(os.path.join(folder, 'coef0.npy')).reshape(-1)
else:
w = np.load(os.path.join(folder, 'w.npy'))
V = np.load(os.path.join(folder, 'V.npy'))
self.V = V[:self.n_users]
self.item_embed = V[self.n_users:]
self.w = w[:self.n_users]
self.item_bias = w[self.n_users:]
print('w user', self.w.shape)
print('w item', self.item_bias.shape)
def load_caltech100(images_fname, labels_fname):
# if images(64).npy or output_labels(64).npy missing then
# print('Generating data because it does not exist. Note that this may take a while')
# gen_data()
one_hot = lambda x, K: np.array(x[:, None] == np.arange(K)[None, :],
dtype=int)
images = np.load(images_fname)
output_labels = np.load(labels_fname)
output_labels = np.load(labels_fname)
train_images, valid_images, train_labels, valid_labels = train_test_split(
images, output_labels, test_size=0.20, random_state=1729)
train_labels = one_hot(train_labels, 101)
valid_labels = one_hot(valid_labels, 101)
return train_images, train_labels, valid_images, valid_labels
def construct_cp_df(critfinder_row):
finder_kwargs = critfinder_row.finder_kwargs
finder_dir = os.path.dirname(critfinder_row.finder_json)
output_dir = os.path.join(finder_dir, OUTPUT_DIR_NAME)
output_npzs = [np.load(os.path.join(output_dir, elem))
for elem in os.listdir(output_dir)
if elem.endswith("npz")]
row_dictionaries = []
for output_npz in output_npzs:
row_dictionary = {}
row_dictionary.update(finder_kwargs)
if "theta" in output_npz.keys():
row_dictionary["thetas"] = output_npz["theta"]
row_dictionary["final_theta"] = row_dictionary["thetas"][-1]
row_dictionary["run_length"] = len(row_dictionary["thetas"])
if "f_theta" in output_npz.keys():
row_dictionary["losses"] = output_npz["f_theta"]
row_dictionary["final_loss"] = row_dictionary["losses"][-1]
if "g_theta" in output_npz.keys():
row_dictionary["squared_grad_norms"] = 2 * output_npz["g_theta"]
row_dictionary["final_squared_grad_norm"] = row_dictionary["squared_grad_norms"][-1]
row_dictionaries.append(row_dictionary)
return pd.DataFrame(row_dictionaries)
def mfvi_elbo(model):
""" computes the elbo value for the saved MFVI model """
lnpdf, D, param_names = make_model(model)
mfvi = vi.DiagMvnBBVI(lnpdf, D, lnpdf_is_vectorized=True)
mfvi_lam = np.load(model + "_output/mfvi.npy")
elbo_val = mfvi.elbo_mc(mfvi_lam, n_samps=Nsamp)
return elbo_val
def apply_rotation(obj, coord_old, src_folder):
coord_vec_ls = []
for i in range(3):
f = os.path.join(src_folder, 'coord{}_vec.npy'.format(i))
coord_vec_ls.append(np.load(f))
s = obj.shape
coord0_vec, coord1_vec, coord2_vec = coord_vec_ls
coord_old = np.tile(coord_old, [s[0], 1])
coord1_old = coord_old[:, 0]
coord2_old = coord_old[:, 1]
coord_old = np.stack([coord0_vec, coord1_old, coord2_old], axis=1).transpose()
# print(sess.run(coord_old))
obj_channel_ls = np.split(obj, s[3], 3)
obj_rot_channel_ls = []
for channel in obj_channel_ls:
channel_flat = channel.flatten()
ind = coord_old[0] * (s[1] * s[2]) + coord_old[1] * s[2] + coord_old[2]
ind = ind.astype('int')
obj_chan_new_val = channel_flat[ind]
obj_rot_channel_ls.append(np.reshape(obj_chan_new_val, s[:-1]))
obj_rot = np.stack(obj_rot_channel_ls, axis=3)
return obj_rot
def re_processing(url_load, url_re):
# url_load gives the url for loading the optimal weight of NN.
# url_re gives the url for saving the computed relative errors.
global inputs, targets
data = np.load(url_load)
opt_weights = data['optweights'].item()
x_scaler = data['x_scaler'].item()
y_scaler = data['y_scaler'].item()
RelativeError = []
for datafile in range(8000):
x_traj_test, y_traj_test = sample_multitraj(datafile, datafile+1)
inputs = x_scaler.transform(x_traj_test)
targets = y_scaler.transform(y_traj_test)
outputs = nn_encode_foward_decode(opt_weights, inputs)
outputs = y_scaler.inverse_transform(outputs)
targets = y_scaler.inverse_transform(targets)
re = np.mean([np.linalg.norm(targets[i] - outputs[i]) / np.linalg.norm(targets[i]) for i in range(len(targets))])
print('sample {:d} relative training norm error {:+1.4e}'.format(datafile, re))
RelativeError.append(re)
np.savez(url_re, re = RelativeError, url_load = url_load)
def load_feature_array(class_name, feature_folder='inception_features'):
"""
class_name can be airplane, automobile, ... or wholedata.
"""
npy_path = cifar10_file(feature_folder, '{}.npy'.format(class_name))
array = np.load(npy_path)
return array
def load_data_array(class_name):
"""
class_name can be airplane, automobile, ....
"""
npy_path = cifar10_file('data', '{}.npy'.format(class_name))
array = np.load(npy_path)
return array
def load_folded(file):
"""
Load a folded value and its pattern from a file together with any
additional data.
Note that ``pattern`` must be registered with ``register_pattern_json``
to use ``load_folded``.
Parameters
---------------
file: String or file
A file or filename of data saved with ``save_folded``.
Returns
-----------
folded_val:
The folded value of the saved parameter.
pattern:
The ``paragami`` pattern of the saved parameter.
data:
The data as returned from ``np.load``. Additional saved values will
exist as keys of ``data``.
"""
data = np.load(file)
pattern = get_pattern_from_json(str(data['pattern_json']))
folded_val = pattern.fold(data['flat_val'], free=False)
return folded_val, pattern, data
def classification_data(seed=0):
"""
Load 2D data. 2 Classes. Class labels generated from a 2-2-1 network.
:param seed: random number seed
:return:
"""
npr.seed(seed)
data = np.load("./data/2D_toy_data_linear.npz")
x = data['x']
y = data['y']
ids = np.arange(x.shape[0])
npr.shuffle(ids)
# 75/25 split
num_train = int(np.round(0.01 * x.shape[0]))
x_train = x[ids[:num_train]]
y_train = y[ids[:num_train]]
x_test = x[ids[num_train:]]
y_test = y[ids[num_train:]]
mu = np.mean(x_train, axis=0)
std = np.std(x_train, axis=0)
x_train = (x_train - mu) / std
x_test = (x_test - mu) / std
train_stats = dict()
train_stats['mu'] = mu
train_stats['sigma'] = std
return x_train, y_train, x_test, y_test, train_stats
def test():
# Pick a trajectory and check the prediction of the nn on this trajectory
global inputs, targets
data = np.load('data/sample_1/optweights_tanh_minmax_random1000shuffle_G20_layer2_2.npz')
opt_weights = data['optweights'].item()
x_scaler = data['x_scaler'].item()
y_scaler = data['y_scaler'].item()
# the number of the trajectory picked
datafile = 4412
x_traj_test, y_traj_test = sample_multitraj(datafile, datafile+1)
print(x_traj_test[0,:])
inputs = x_scaler.transform(x_traj_test)
targets = y_scaler.transform(y_traj_test)
outputs = nn_encode_foward_decode(opt_weights, inputs)
x0 = inputs[0,:].reshape((1,4))
outputs = np.concatenate((x0, outputs), axis = 0)
targets = np.concatenate((x0, targets), axis = 0)
outputs = y_scaler.inverse_transform(outputs)
targets = y_scaler.inverse_transform(targets)
re = np.mean([np.linalg.norm(targets[i] - outputs[i]) / np.linalg.norm(targets[i]) for i in range(len(targets))])
# re = np.mean([np.linalg.norm(targets[i] - outputs[i])**2 for i in range(len(targets))])
print('sample Relative training norm error {:+1.4e}'.format(re))
figplot(outputs, url = None)
def npvi_elbo(model, ncomp):
""" computes the elbo value for saved NPVI models """
lnpdf, D, param_names = make_model(model)
npvi = vi.NPVI(lnpdf, D=D)
npvi_theta = np.load(model + "_output/npvi_%d-comp.npy" % ncomp)
print " npvi n comp: ", npvi_theta.shape[0]
elbo_val = npvi.mc_elbo(npvi_theta, nsamps=Nsamp)
return elbo_val
def get_grid_psi(data_set='BTLS'):
fgridpsi = '%s_grid_psi.npy' % dataset.lower()
if os.path.isfile(fgridpsi):
return np.load(fgridpsi)
else:
grid_psi = np.stack([psi(tt) for tt in theta_xygrid])
np.save(fgridpsi, grid_psi)
return grid_psi
def load_data_array(self, class_name):
"""
class_name can be airplane, automobile, ....
"""
filename = '{}.npy'.format(class_name)
npy_path = self.data_path('data', filename)
array = np.load(npy_path)
return array
def load(self, desc=""):
fn = self.default_file_name()
if len(desc) != 0:
fn += "_%s" % desc
p = np.load(fn + ".npz")
self.model.ps = p["gen"]
self.model.sync_ps()
def fetch_trajectory(trajectory_path):
if trajectory_path.endswith(".npz"):
results_npz = np.load(trajectory_path)
trajectory = results_npz["theta"]
else:
raise NotImplementedError(
"trajectory_path {} not understood".format(trajectory_path))
return trajectory
def restore_param_arrays_from_checkpoint(self):
arr = np.load(
os.path.join(self.output_folder, 'opt_params_checkpoint.npy'))
if len(self.params_list) > 0:
for i, param_name in enumerate(self.params_list):
self.params_whole_array_dict[param_name] = arr[i]
return
def load_DR10QSO_train_test_idx(split_type="random",
spectra_loc=SPECTRA_LOC,
in_plate_dir=True):
if not split_type in split_types:
raise ValueError('split_type %s not found!' % split_type)
# grab the train/test split
f = file(join(TRAIN_TEST_SPLIT_LOC, "split_%s.bin" % split_type), 'rb')
train_idx = np.load(f)
test_idx = np.load(f)
f.close()
# select DR10Q_v2 file
dr10filepath = join(DR10QSO_LOC, 'DR10Q_v2.fits')
dr10file = fitsio.FITS(dr10filepath)
qso_psf_flux = dr10file[1]['PSFFLUX'].read()
qso_psf_flux_ivar = dr10file[1]['IVAR_PSFFLUX'].read()
qso_z = dr10file[1]['Z_VI'].read()
qso_psf_mags = dr10file[1]['PSFMAG'].read()
# remove zero'd fluxes
zero_idx = np.any(qso_psf_flux == 0, axis=1)
print " found %d rows with zero fluxes (removing from training)" % zero_idx.sum(
)
if zero_idx.sum() > 0:
for bad_idx in np.where(zero_idx)[0]:
train_idx = train_idx[train_idx != bad_idx]
# construct NERSC file paths
print "constructing NERSC file paths to %d quasar spectra" % (
qso_z.shape[0])
qso_ids = dr10file[1][['PLATE', 'MJD', 'FIBERID']].read()
if in_plate_dir:
spec_url_template = join(spectra_loc, "%04d/spec-%04d-%05d-%04d.fits")
qso_files = [
spec_url_template % (qid[0], qid[0], qid[1], qid[2])
for qid in qso_ids
]
else:
spec_url_template = join(spectra_loc, "spec-%04d-%05d-%04d.fits")
qso_files = [
spec_url_template % (qid[0], qid[1], qid[2]) for qid in qso_ids
]
return qso_psf_flux, qso_psf_flux_ivar, qso_psf_mags, qso_z, \
qso_files, train_idx, test_idx
def __init__(self, configurations, parameters, controls):
self.configurations = configurations
self.parameters = parameters
self.controls = controls
flat_args, self.unflatten = flatten(self.controls)
self.gx = grad(self.cost)
self.J = jacobian(self.forward)
self.hx = hessian_vector_product(self.cost)
self.hvp = hvp(self.hx)
y0, t_total, N_total, number_group, population_proportion, \
t_control, number_days_per_control_change, number_control_change_times, number_time_dependent_controls = configurations
self.N_total = N_total
self.number_group = number_group
self.t_control = t_control
self.dimension = len(self.t_control)
self.number_time_dependent_controls = number_time_dependent_controls
self.y0 = y0
self.t_total = t_total
self.interpolation = piecewiseLinear
self.load_data(fips)
self.initialization()
if number_group > 1:
# contact matrix
school_closure = True
# calendar from February 15th
weekday = [2, 3, 4, 5, 6]
# calendar from April 1st
# weekday = [0, 1, 2, 5, 6]
# calendar from May 1st
# weekday = [0, 3, 4, 5, 6]
calendar = np.zeros(1000 + 1, dtype=int)
# set work days as 1 and school days as 2
for i in range(1001):
if np.remainder(i, 7) in weekday:
calendar[i] = 1
if not school_closure: # and i < 45
calendar[i] = 2
self.calendar = calendar
contact = np.load("utils/contact_matrix.npz")
self.c_home = contact["home"]
self.c_school = contact["school"]
self.c_work = contact["work"]
self.c_other = contact["other"]
self.contact_full = self.c_home + 5. / 7 * (
(1 - school_closure) * self.c_school +
self.c_work) + self.c_other
def load_model(self):
"""Loads the last checkpoint to continue training or for evaluation."""
file = np.load(self.checkpoint_path)
self.epoch = file['epoch']
self.fourier_freq, self.fourier_offset = file[
'fourier_features'] # feature parameters
self.θ, self.Σ = file['θ'], file['Σ'] # policy parameters
self.α, self.η = file['α'], file['η'] # dual parameters
print(f"LOADED Model at epoch {self.epoch}")
def load_redshift_samples(fname):
bname = os.path.splitext(fname)[0]
th_samples = np.load(bname + ".npy")
with open(bname + ".pkl", 'rb') as handle:
ll_samps = pickle.load(handle)
q_idx = pickle.load(handle)
qso_info = pickle.load(handle)
chain_idx = pickle.load(handle)
return th_samples, ll_samps, q_idx, qso_info, chain_idx
def summarize_res(sname, datasize):
print(sname)
res = []
times = []
for i in range(100):
PATH = ROOT_PATH + "/our_methods/results/mendelian/" + sname + "/"
filename = PATH + 'LMO_errs_{}_nystr_{}.npy'.format(i, datasize)
if os.path.exists(filename):
tmp_res = np.load(filename, allow_pickle=True)
if tmp_res[-1] is not None:
res += [tmp_res[-1]]
time_path = PATH + '/LMO_errs_{}_nystr_{}_time.npy'.format(i, datasize)
if os.path.exists(time_path):
t = np.load(time_path)
times += [t]
res = np.array(res)
res = remove_outliers(res)
print('mean, std: ', np.mean(res), np.std(res))
def load_basis_samples(fname):
bname = os.path.splitext(fname)[0]
th_samples = np.load(bname + ".npy")
with open(bname + ".pkl", 'rb') as handle:
ll_samps = pickle.load(handle)
lam0 = pickle.load(handle)
lam0_delta = pickle.load(handle)
parser = pickle.load(handle)
chain_idx = pickle.load(handle)
return th_samples, ll_samps, lam0, lam0_delta, parser, chain_idx
def load_basis_samples(fname):
bname = os.path.splitext(fname)[0]
th_samples = np.load(bname + ".npy")
with open(bname + ".pkl", 'rb') as handle:
ll_samps = pickle.load(handle)
lam0 = pickle.load(handle)
lam0_delta = pickle.load(handle)
parser = pickle.load(handle)
chain_idx = pickle.load(handle)
return th_samples, ll_samps, lam0, lam0_delta, parser, chain_idx
def generate_dataset(self, cachefile='data/decoy-mnist.npz'):
if cachefile and os.path.exists(cachefile):
cache = np.load(cachefile)
data = tuple([cache[f] for f in sorted(cache.files)])
else:
data = self._generate_dataset(os.path.dirname(cachefile))
if cachefile:
np.savez(cachefile, *data)
self.Xr, self.X, self.y, self.E, self.Xtr, self.Xt, self.yt, self.Et = data
self.status.initialized = True
def load_feat_array():
"""Open the numpy array of all graphs' data."""
return np.load('../data/feat_array.npy')
