def setup(save_path, layers=[1,2,3], curve_num=5000, chunk_size=1000, noisy=False, xray=False,
show_plots=True, generate_data=True, train_classifier=True, train_regressor=True,
classifer_epochs=2, regressor_epochs=2):
"""Sets up the pipeline for predictions on .dat files by generating data and training.
Args:
save_path (string): a path to the directory where data and models will be saved.
layers (list): a list of layers to generate and train for.
curve_num (int): the number of curves to generate per layer.
chunk_size (int): the size of chunks to use in the h5 storage of images for curves.
noisy (Boolean): whether to add noise to generated data.
xray (Boolean): whether to use an x-ray probe or not.
show_plots (Boolean): whether to display classification confusion matrix and regression plots or not.
generate_data (Boolean): whether to generate data or use existing data.
train_classifier (Boolean): whether to train the classifier or not.
train_regressor (Boolean): whether to train the regressors or not.
classifer_epochs (int): the number of epochs to train the classifier for.
regressor_epochs (int): the number of epochs to train the regressor for.
"""
if generate_data:
print("-------------- Data Generation ------------")
for layer in layers: #Generate curves for each layer specified.
print(">>> Generating {}-layer curves".format(layer))
if xray: #Generate data using x-ray probe.
structures = XRayGenerator.generate(curve_num, layer)
XRayGenerator.save(save_path + "/data", LAYERS_STR[layer], structures, noisy=noisy)
else: #Generate data using neutron probe.
structures = NeutronGenerator.generate(curve_num, layer)
NeutronGenerator.save(save_path + "/data", LAYERS_STR[layer], structures, noisy=noisy)
print(">>> Creating images for {}-layer curves".format(layer))
save_path_layer = data_path_layer = save_path + "/data/{}".format(LAYERS_STR[layer])
#Create images for the generated curves, ready for input to the classifier and regressors.
generate_images(data_path_layer, save_path_layer, [layer], xray=xray, chunk_size=chunk_size, display_status=False)
layers_paths = [save_path + "/data/{}".format(LAYERS_STR[layer]) for layer in layers]
merge(save_path + "/data", layers_paths, display_status=False) #Merge the curves for each layer for classification.
print("\n-------------- Classification -------------")
if train_classifier:
print(">>> Training classifier")
classify(save_path + "/data/merged", save_path, train=True, epochs=classifer_epochs, show_plots=show_plots) #Train the classifier.
else:
print(">>> Loading classifier")
load_path = save_path + "/classifier/full_model.h5" #Load a classifier.
classify(save_path + "/data/merged", load_path=load_path, train=False, show_plots=show_plots)
print("\n---------------- Regression ---------------")
for layer in layers: #Train or load regressors for each layer that we are setting up for.
data_path_layer = save_path + "/data/{}".format(LAYERS_STR[layer])
if train_regressor:
print(">>> Training {}-layer regressor".format(LAYERS_STR[layer]))
regress(data_path_layer, layer, save_path, epochs=regressor_epochs, show_plots=show_plots, xray=xray) #Train the regressor.
else:
print(">>> Loading {}-layer regressor".format(LAYERS_STR[layer]))
load_path_layer = save_path + "/{}-layer-regressor/full_model.h5".format(LAYERS_STR[layer]) #Load an existing regressor.
regress(data_path_layer, layer, load_path=load_path_layer, train=False, show_plots=show_plots, xray=xray)
print()
def align_shape(model, img):
init_shape = model.m
r = model.R
for cascade in range(config.n_cascades):
scale = 1 / (config.scale_factor**(config.n_cascades - (cascade + 1)))
img = cv2.resize(img, (np.floor(img.shape[1] * scale).astype(
np.int), np.floor(img.shape[0] * scale).astype(np.int)),
interpolation=cv2.INTER_CUBIC)
init_shape = init_shape * scale
# extract local features
desc, desc_size = local_descriptors.hog(img, init_shape, cascade)
# regressing
delta_shape = regression.regress(np.array(desc), np.array(r[cascade]))
# estimate new shape
orig_delta_shape = np.reshape(delta_shape, (-1, 2)) * np.tile([
np.max(init_shape[:, 0]) - np.min(init_shape[:, 0]),
np.max(init_shape[:, 1]) - np.min(init_shape[:, 1])
], (init_shape.shape[0],
1)) # de-normalize estimated delta_shape. reshape along rows
aligned_shape = (init_shape - orig_delta_shape) / scale
init_shape = aligned_shape
return aligned_shape
def test_regress(self):
# test data taken from https://www.statisticshowto.datasciencecentral.com/probability-and-statistics/regression-analysis/find-a-linear-regression-equation/
test_data_x = [43, 21, 25, 42, 57, 59]
test_data_y = [99, 65, 79, 75, 87, 81]
expected_result = { 'intercept': 65.1416, 'slope': 0.385225 }
actual_result = regress(test_data_x, test_data_y)
self.assertAlmostEqual(expected_result['slope'], actual_result['slope'], 6)
self.assertAlmostEqual(expected_result['intercept'], actual_result['intercept'], 4)
for n in range(len(d) - s):
si = max(d[n:n + s], key=lambda a: a[1])
if si[1] > mi + 0.75 * r:
sr.add(si)
if i.find('zero') < 0:
left = set()
right = set()
for si in sr:
if si[0] < m:
left.add(si)
else:
right.add(si)
lm = max(left, key=lambda a: a[1])
rm = max(right, key=lambda a: a[1])
print('diff:', rm[0] - lm[0])
print('{}\t{}'.format(lm[0], mi + r / 2))
print('{}\t{}'.format(rm[0], mi + r / 2))
print()
for name, d in data.points_data.items():
print(name)
slope, intercept, std_err = regression.regress(d)
n = 4.668e-8
print('error:', std_err)
print('slope:', slope)
print('g_err:', std_err / n)
print('g:', slope / n)
print()
# lambert_yticks(ax1, ytick)
ax2.gridlines(xlocs=xtick, ylocs=ytick,color='gray', linestyle='--', linewidth=0.5)
ax2.xaxis.set_major_formatter(LONGITUDE_FORMATTER)
ax2.yaxis.set_major_formatter(LATITUDE_FORMATTER)
# export the figure for monthly plot
plt.savefig('./figures/arctic_jpl_mascon.pdf', dpi=300, facecolor='w', edgecolor='w',
orientation='landscape', papertype=None, format=None,
transparent=False, bbox_inches="tight", pad_inches=None,
frameon=None)
# performing the regression
taxis=time_convert(ds['z'].time+1).tarray_month2year()
reg=regress(taxis)
reg.multivar_regress(ds['z'].isel(lon=0,lat=0).values,predef_var='semisea_sea_lin')
dimname=reg.dm_order
beta=np.zeros([ds['z'].lon.shape[0],ds['z'].lat.shape[0],len(dimname)])+np.nan
se=np.zeros([ds['z'].lon.shape[0],ds['z'].lat.shape[0],len(dimname)])+np.nan
# import time
# start = time.time()
ii=0
for i in ds['z'].lon.values :
jj=0
print i
for j in ds['z'].lat.values :
if ~ds['z'].sel(lon=i,lat=j).sum(skipna=False).isnull():
meridN_w = -int_w * rhow * g / f.sel(
lat=sel_lat
) / rho0 * 1e-6 # m^2 * kg/m^3 * m/s^2 / (1/s) / (kg/m^3) *1E-6 = m^3/s * Sv/m^3 = Sv
meridN_e = int_e * rhow * g / f.sel(
lat=sel_lat
) / rho0 * 1e-6 # m^2 * kg/m^3 * m/s^2 / (1/s) / (kg/m^3) *1E-6 = m^3/s * Sv/m^3 = Sv
meridN = meridN_e + meridN_w
# ---
# ### Regression to remove linear trend/ linear trend + seasonal + semiseasonal
from time_convert import time_convert
ts = time_convert(meridN.time)
yeardate = ts.tarray_month2year()
from regression import regress
reg = regress(axis=yeardate)
dict1 = reg.multivar_regress(meridN.values, predef_var='semisea_sea_lin')
list_variates = dict1['list_variates']
betas = dict1['beta']
linmodel = betas[0] * list_variates[0] + betas[1] * list_variates[1]
model = dict1['model']
res = meridN.values - model
reslin = meridN.values - linmodel
meridN_res = xr.DataArray(res, coords={"time": meridN.time}, dims="time")
meridN_reslin = xr.DataArray(reslin, coords={"time": meridN.time}, dims="time")
fig = plt.figure(figsize=[10, 5])
ax1 = fig.add_axes([0, 0, 1.5, 1])
meridN.plot(ax=ax1,
color='#d62728',
marker='o',
def learn_one_cascade(current_cascade, images, ground_truth_shapes,
temp_shapes):
# scale factor of current cascade
scale = 1 / (config.scale_factor**(config.n_cascades -
(current_cascade + 1)))
delta_shapes = []
descs = [
] # local descriptors may have various length according to different scales
for i in range(images.shape[2]):
if config.verbose:
print('Cascade: {:d}\tSample: {:d}\n'.format(
current_cascade + 1, i + 1))
# compute the initial shape
# for the first cascade, the mean shape is used as the initial shape
# align the mean shape according to the bounding box of the groundtruth shape (why to do so?)
if current_cascade == 0:
# here to align the mean shape according to the bounding box of the groundtruth shape
pass
# resize image and shape according to current cascade
img = Image.fromarray(images[:, :, i])
img = img.resize(
(np.floor(img.size[0] * scale), np.floor(img.size[1] * scale)),
Image.BICUBIC)
shape = temp_shapes[:, :, i] * scale
# extract local descriptors
desc, desc_size = local_descriptors.hog(np.asarray(img), shape,
current_cascade)
descs.append(
desc
) # shape of descs: n*m, n is the number of samples, m is the length of local descriptors of one image
# compute delta shape (current shape minus true shape)
delta_shape = shape - ground_truth_shapes[:, :, i] * scale
delta_shape = np.true_divide(
delta_shape,
np.tile([
np.max(shape[:, 0]) - np.min(shape[:, 0]),
np.max(shape[:, 1]) - np.min(shape[:, 1])
], (shape.shape[0], 1))) # normalize delta_shape
delta_shapes.append(delta_shape.ravel().tolist()) # reshape along rows
# solving multivariant linear regression
if config.verbose: print('Solving linear regression problem...\n')
delta_shapes = np.array(delta_shapes)
descs = np.array(descs)
# (X'*X+eye(descs.shape[1])*alpha)\X'*Y
a = descs.T.dot(descs) + (np.eye(descs.shape[1]) *
config.alpha[current_cascade])
b = np.linalg.inv(a).dot(descs.T)
R = b.dot(delta_shapes)
# update shapes
if config.verbose: print('Updating shapes...\n')
temp_delta_shapes = regression.regress(
descs, R) # n_training_sample by n_points * 2
new_shapes = np.ndarray(temp_shapes.shape)
for i in range(images.shape[2]):
shape = temp_shapes[:, :, i] * scale
orig_delta_shapes = np.reshape(
temp_delta_shapes[i, :], (-1, 2)) * np.tile([
np.max(shape[:, 0]) - np.min(shape[:, 0]),
np.max(shape[:, 1]) - np.min(shape[:, 1])
], (shape.shape[0],
1)) # de-normalize estimated delta_shape. reshape along rows
new_shapes[:, :, i] = (shape - orig_delta_shapes) / scale
# compute error
err = np.zeros((images.shape[2], 1))
for i in range(images.shape[2]):
err[i] = error_compute.rms_error(new_shapes[:, :, i],
ground_truth_shapes[:, :, i])
rms = np.mean(err) * 100
return R, new_shapes, rms