def _initiate_scalers(self, rmCol): ''' Scaling for control device: linear minmax to [0.1, 0.9] Scaling for power: logarithmic element-wise minmax to [0.1, 0.9] ''' # Treat rmCol if rmCol is None: nrmCol = 0 else: nrmCol = len(rmCol) # Find limits minh = np.inf maxh = 0 minp = np.full(self.n_y[1] - nrmCol, np.inf) maxp = np.zeros(self.n_y[1] - nrmCol) for rec in self.pwrl: # Min and max of all sbh (flattened array) minh = min(minh, np.amin(rec.inx)) maxh = max(maxh, np.amax(rec.inx)) # Min and max of each column (e.g. fuel element for MITR) minp = np.amin(np.vstack((minp, np.amin(rec.iny, axis=0))), axis=0) maxp = np.amax(np.vstack((maxp, np.amax(rec.iny, axis=0))), axis=0) # Save limits self.minh = minh self.maxh = maxh self.minp = minp self.maxp = maxp # Create MMS objects self.heightMMS = MMS((0.1, 0.9)) self.heightMMS.fit([[minh], [maxh]]) self.powerMMS = MMS((0.1, 0.9)) self.powerMMS.fit(np.log([minp, maxp]))
def _gradient_(self): data = self.img_gray.ravel().copy() fvs = np.zeros((len(data), self.neighborhood)).astype('uint8') img_gray_pad = np.pad(self.img_gray.copy(), 1, mode='edge') cnt = 0 for i in range(1, self.img_gray.shape[1], 1): for j in range(1, self.img_gray.shape[0], 1): if (self.neighborhood == 8): fvs[cnt, 0] = data[cnt] - img_gray_pad[j - 1, i - 1] fvs[cnt, 1] = data[cnt] - img_gray_pad[j - 1, i] fvs[cnt, 2] = data[cnt] - img_gray_pad[j - 1, i + 1] fvs[cnt, 3] = data[cnt] - img_gray_pad[j, i - 1] fvs[cnt, 4] = data[cnt] - img_gray_pad[j, i + 1] fvs[cnt, 5] = data[cnt] - img_gray_pad[j + 1, i - 1] fvs[cnt, 6] = data[cnt] - img_gray_pad[j + 1, i] fvs[cnt, 7] = data[cnt] - img_gray_pad[j + 1, i + 1] elif (self.neighborhood == 4): fvs[cnt, 0] = data[cnt] - img_gray_pad[j, i - 1] fvs[cnt, 1] = data[cnt] - img_gray_pad[j, i + 1] fvs[cnt, 2] = data[cnt] - img_gray_pad[j - 1, i] fvs[cnt, 3] = data[cnt] - img_gray_pad[j + 1, i] cnt = cnt + 1 fvs = MMS().fit_transform(fvs) ##returns vector that is neighborhood wide of data length return fvs
def loadMat(fn): f = sio.loadmat(fn) events = f['events'].astype(np.int16) sounds = f['sounds'].astype(np.float16) rates = MMS(f['rates'].squeeze()) return events, sounds, rates
def test_fit1(self): scale_param = self.get_scale_param() scale_param.scale_column_idx = [] scale_param.feat_upper = 2 scale_param.feat_lower = 1 scale_obj = MinMaxScale(scale_param) fit_instance = scale_obj.fit(self.table_instance) column_min_value = scale_obj.column_min_value column_max_value = scale_obj.column_max_value for i, line in enumerate(self.test_data): for j, value in enumerate(line): if value > 2: self.test_data[i][j] = 2 elif value < 1: self.test_data[i][j] = 1 scaler = MMS() scaler.fit(self.test_data) self.assertListEqual(self.get_table_instance_feature(fit_instance), np.around(scaler.transform(self.test_data), 6).tolist()) data_min = list(scaler.data_min_) data_max = list(scaler.data_max_) self.assertListEqual(column_min_value, data_min) self.assertListEqual(column_max_value, data_max) transform_data = scale_obj.transform(self.table_instance) self.assertListEqual(self.get_table_instance_feature(fit_instance), self.get_table_instance_feature(transform_data))
def test_cols_select_fit_and_transform_repeat(self): scale_column_idx = [1, 1, 2, 2, 4, 5, 5] min_max_scaler = MinMaxScaler(mode='normal', area='col', scale_column_idx=scale_column_idx, feat_upper=None, feat_lower=None, out_upper=None, out_lower=None) fit_data, cols_transform_value = min_max_scaler.fit( self.table_instance) scaler = MMS() scaler.fit(self.test_data) mms_transform_data = np.around(scaler.transform(self.test_data), 4).tolist() for i, line in enumerate(mms_transform_data): for j, cols in enumerate(line): if j not in scale_column_idx: mms_transform_data[i][j] = self.test_data[i][j] self.assertListEqual(self.get_table_instance_feature(fit_data), mms_transform_data) transform_data = min_max_scaler.transform(self.table_instance, cols_transform_value) self.assertListEqual(self.get_table_instance_feature(transform_data), mms_transform_data)
def normalise_numeric_features(X, standardisation=False, means=True, stdev=True): """ Normalisation for numeric features :param X: A numpy matrix of the data. First axis corresponding to instances, second axis corresponding to samples :param standardisation: Whether standardisation needs to be done instead of normalisation. Default: False :param means: Whether the mean should be normalised. Default: True :param stdev: Whether the standard devation should be normalised. Default: True :return: X and features with numeric features normalised. """ column_types = column_types_dataset(X, categorical=False) for i in range(len(column_types)): if column_types[i]: if standardisation: # Standardisation scaler = MMS([0, 1]) X[:, i:i + 1] = scaler.fit_transform(X[:, i:i + 1]) else: # Normalisation scaler = SS(means, stdev) X[:, i:i + 1] = scaler.fit_transform(X[:, i:i + 1]) return X
def get_feature_vectors(img_rgb, process='intensity', neighborhood=8, radius=1, METHOD='uniform'): from skimage.color import rgb2ycbcr as r2y from skimage.feature import local_binary_pattern as LBP import cv2 img_gray = cv2.cvtColor(img_rgb.copy(), cv2.COLOR_RGB2GRAY) feats = np.expand_dims(img_gray.ravel().copy(), 1) feats = MMS().fit_transform(feats) get_features = fvs(img_rgb, neighborhood=neighborhood, radius=radius, METHOD=METHOD) if ('intensity' in process): temp = get_features._intensity_() feats = np.concatenate((feats, temp), axis=1) if ('gradient' in process): temp = get_features._gradient_() feats = np.concatenate((feats, temp), axis=1) if ('texture' in process): temp = get_features._texture_() feats = np.concatenate((feats, temp), axis=1) if ('cb_channel' in process): temp = get_features._ycbcr_mask() feats = np.concatenate((feats, temp), axis=1) return feats
def test_fit_col_feat(self): feat_upper = [0.8, 5, 6, 7, 8, 9] feat_lower = [0.5, 2, 3, 3, 4, 4.0] out_upper = 2.0 out_lower = -1.0 min_max_scaler = MinMaxScaler(mode='normal', area='col', feat_upper=feat_upper, feat_lower=feat_lower, out_upper=out_upper, out_lower=out_lower) fit_data, cols_transform_value = min_max_scaler.fit(self.table_instance) new_data = [] for data in self.test_data: tmp_data = [] for i in range(len(data)): if data[i] > feat_upper[i]: tmp_data.append(feat_upper[i]) elif data[i] < feat_lower[i]: tmp_data.append(feat_lower[i]) else: tmp_data.append(data[i]) new_data.append(tmp_data) feature_range = (out_lower, out_upper) scaler = MMS(feature_range) scaler.fit(new_data) self.assertListEqual(self.get_table_instance_feature(fit_data), np.around(scaler.transform(new_data), 4).tolist())
def readTestData(): testData = np.loadtxt('data/test.csv', delimiter=',', skiprows=1) xTest = testData[:, 1:31] scale = MMS() allX = scale.fit_transform(xTest) indexTest = list(testData[:, 0]) return [allX, indexTest]
def readTrainingData(): data = np.loadtxt('data/training.csv', delimiter=',', skiprows=1, converters={32: lambda x: int(x == 's'.encode('utf-8'))}) allY = data[:, 32] allX = data[:, 1:31] allW = data[:, 31] scale = MMS() allX = scale.fit_transform(allX) np.random.seed(42) r = np.random.rand(allY.shape[0]) xTrain = allX[r <= 0.4] yTrain = allY[r <= 0.4] wTrain = allW[r <= 0.4] xValid = allX[r > 0.7] yValid = allY[r > 0.7] wValid = allW[r > 0.7] v = np.random.rand(yValid.shape[0]) xCrossValid = xValid[v <= 0.5] yCrossValid = yValid[v <= 0.5] wCrossValid = wValid[v <= 0.5] xTestValid = xValid[v > 0.5] yTestValid = yValid[v > 0.5] wTestValid = wValid[v > 0.5] return [ xTrain, yTrain, wTrain, xCrossValid, yCrossValid, wCrossValid, xTestValid, yTestValid, wTestValid ]
def _texture_(self): n_pts = 8 * self.radius lbp_img = LBP(self.img_gray.copy(), n_pts, self.radius, self.METHOD) fvs = np.expand_dims(lbp_img.ravel(), 1) fvs = MMS().fit_transform(fvs) ##returns vector that is the length of img_gray.shape[0]*img_gray.shape[1] return fvs
def test_fit_instance_default(self): min_max_scaler = MinMaxScaler(mode='normal', area='all', feat_upper=None, feat_lower=None, out_upper=None, out_lower=None) fit_instance, cols_transform_value = min_max_scaler.fit(self.table_instance) scaler = MMS() scaler.fit(self.test_data) self.assertListEqual(self.get_table_instance_feature(fit_instance), np.around(scaler.transform(self.test_data), 4).tolist()) self.assertListEqual(cols_transform_value, self.sklearn_attribute_format(scaler, [0, 1]))
def test_fit_col_out(self): min_max_scaler = MinMaxScaler(mode='normal', area='col', feat_upper=None, feat_lower=None, out_upper=2, out_lower=-1) fit_data, cols_transform_value = min_max_scaler.fit(self.table_instance) feature_range = (-1, 2) scaler = MMS(feature_range) scaler.fit(self.test_data) self.assertListEqual(self.get_table_instance_feature(fit_data), np.around(scaler.transform(self.test_data), 4).tolist()) self.assertListEqual(cols_transform_value, self.sklearn_attribute_format(scaler, feature_range))
def test_fit5(self): scale_column_idx = [1, 2, 4] scale_names = ['fid1', 'fid2', 'fid1000'] scale_param = self.get_scale_param() scale_param.mode = "cap" # scale_param.area = "col" scale_param.feat_upper = 0.8 scale_param.feat_lower = 0.2 scale_param.scale_names = scale_names scale_param.scale_col_indexes = [2, 4] scale_obj = MinMaxScale(scale_param) fit_instance = scale_obj.fit(self.table_instance) column_min_value = scale_obj.column_min_value column_max_value = scale_obj.column_max_value raw_data = copy.deepcopy(self.test_data) gt_cap_lower_list = [0, 2, 2, 2, 3, 1] gt_cap_upper_list = [1, 8, 8, 8, 7, 8] for i, line in enumerate(self.test_data): for j, value in enumerate(line): if value > gt_cap_upper_list[j]: self.test_data[i][j] = gt_cap_upper_list[j] elif value < gt_cap_lower_list[j]: self.test_data[i][j] = gt_cap_lower_list[j] scaler = MMS() scaler.fit(self.test_data) sklearn_transform_data = np.around(scaler.transform(self.test_data), 6).tolist() for i, line in enumerate(sklearn_transform_data): for j, cols in enumerate(line): if j not in scale_column_idx: sklearn_transform_data[i][j] = raw_data[i][j] fit_data = np.round(self.get_table_instance_feature(fit_instance), 6).tolist() self.assertListEqual(fit_data, sklearn_transform_data) for i, line in enumerate(sklearn_transform_data): for j, cols in enumerate(line): if j not in scale_column_idx: sklearn_transform_data[i][j] = raw_data[i][j] data_min = list(scaler.data_min_) data_max = list(scaler.data_max_) self.assertListEqual(column_min_value, data_min) self.assertListEqual(column_max_value, data_max) transform_data = scale_obj.transform(self.table_instance) self.assertListEqual(self.get_table_instance_feature(fit_instance), self.get_table_instance_feature(transform_data))
def kpca_preprocess_features(X): mms = MMS(feature_range=(-0.5, 0.5)) n_rows, n_cols = X.shape for j in range(n_cols): col_max = max(X[:, j]) col_min = min(X[:, j]) if col_max <= 1.0 and col_min >= -1.0: pass else: X[:, j] = power(X[:, j], 0.125) X = mms.fit_transform(X) return X
def test_fit4(self): scale_column_idx = [1, 2, 4] scale_param = self.get_scale_param() # scale_param.area = "col" scale_param.feat_upper = 2 scale_param.feat_lower = 1 scale_param.scale_col_indexes = scale_column_idx scale_obj = MinMaxScale(scale_param) fit_instance = scale_obj.fit(self.table_instance) column_min_value = scale_obj.column_min_value column_max_value = scale_obj.column_max_value raw_data = copy.deepcopy(self.test_data) for i, line in enumerate(self.test_data): for j, value in enumerate(line): if j in scale_column_idx: if value > 2: self.test_data[i][j] = 2 elif value < 1: self.test_data[i][j] = 1 scaler = MMS() scaler.fit(self.test_data) sklearn_transform_data = np.around(scaler.transform(self.test_data), 6).tolist() for i, line in enumerate(sklearn_transform_data): for j, cols in enumerate(line): if j not in scale_column_idx: sklearn_transform_data[i][j] = raw_data[i][j] self.assertListEqual(self.get_table_instance_feature(fit_instance), sklearn_transform_data) for i, line in enumerate(sklearn_transform_data): for j, cols in enumerate(line): if j not in scale_column_idx: sklearn_transform_data[i][j] = raw_data[i][j] data_min = list(scaler.data_min_) data_max = list(scaler.data_max_) self.assertListEqual(column_min_value, data_min) self.assertListEqual(column_max_value, data_max) transform_data = scale_obj.transform(self.table_instance) self.assertListEqual(self.get_table_instance_feature(fit_instance), self.get_table_instance_feature(transform_data))
def test_fit_feat(self): feat_upper = 8 feat_lower = 3 out_upper = 2 out_lower = -1 min_max_scaler = MinMaxScaler(mode='normal', area='all', feat_upper=feat_upper, feat_lower=feat_lower, out_upper=out_upper, out_lower=out_lower) fit_data, cols_transform_value = min_max_scaler.fit( self.table_instance) new_data = [] for data in self.test_data: tmp_data = [] for i in range(len(data)): if data[i] > feat_upper: tmp_data.append(feat_upper) elif data[i] < feat_lower: tmp_data.append(feat_lower) else: tmp_data.append(data[i]) new_data.append(tmp_data) feature_range = (out_lower, out_upper) scaler = MMS(feature_range) scaler.fit(new_data) self.assertListEqual(self.get_table_instance_feature(fit_data), np.around(scaler.transform(new_data), 4).tolist()) sklearn_res = self.sklearn_attribute_format(scaler, [out_lower, out_upper]) # self.assertListEqual(cols_transform_value, [(feat_lower, feat_upper, out_lower, out_upper)]) for i in range(len(sklearn_res)): if sklearn_res[i][0] != feat_lower: tmp_res = list(sklearn_res[i]) tmp_res[0] = feat_lower sklearn_res[i] = tuple(tmp_res) if sklearn_res[i][1] != feat_upper: tmp_res = list(sklearn_res[i]) tmp_res[1] = feat_upper sklearn_res[i] = tuple(tmp_res) self.assertListEqual(cols_transform_value[0], sklearn_res)
def test_fit_instance_default(self): scale_param = self.get_scale_param() scale_param.scale_col_indexes = -1 scale_obj = MinMaxScale(scale_param) fit_instance = scale_obj.fit(self.table_instance) column_min_value = scale_obj.column_min_value column_max_value = scale_obj.column_max_value scaler = MMS() scaler.fit(self.test_data) self.assertListEqual(self.get_table_instance_feature(fit_instance), np.around(scaler.transform(self.test_data), 6).tolist()) data_min = list(scaler.data_min_) data_max = list(scaler.data_max_) self.assertListEqual(column_min_value, data_min) self.assertListEqual(column_max_value, data_max) transform_data = scale_obj.transform(self.table_instance) self.assertListEqual(self.get_table_instance_feature(fit_instance), self.get_table_instance_feature(transform_data))
def _ycbcr_mask(self): img_ycrcb = r2y(self.img_rgb) y_channel = img_ycrcb[:, :, 0] mean = np.mean(y_channel.copy()) std = np.std(y_channel.copy()) y_channel[y_channel < (mean - std)] = 0 y_channel[y_channel != 0] = 1 y_channel[y_channel == 0] = 255 y_channel[y_channel == 1] = 0 cr_channel = img_ycrcb[:, :, 1] mean = np.mean(cr_channel.copy()) std = np.std(cr_channel.copy()) cr_channel[cr_channel < (mean - std)] = 0 cr_channel[cr_channel != 0] = 1 cr_channel[cr_channel == 0] = 255 cr_channel[cr_channel == 1] = 0 y_cr_or = np.logical_and(y_channel, np.uint8(1 - cr_channel)).astype('uint8') y_cr_or[y_cr_or != 0] = 255 cb_channel = img_ycrcb[:, :, 2] mean = np.mean(cb_channel.copy()) std = np.std(cb_channel.copy()) cb_channel[cb_channel < (mean - std)] = 0 cb_channel[cb_channel != 0] = 1 cb_channel[cb_channel == 0] = 255 cb_channel[cb_channel == 1] = 0 cb_y_cr_or = np.logical_or(y_cr_or, cb_channel).astype('uint8') cb_y_cr_or[cb_y_cr_or != 0] = 1 SE = np.ones((3, 3)).astype('uint8') final_mask = cv2.morphologyEx(cb_y_cr_or.copy(), cv2.MORPH_OPEN, SE).astype('uint8') mask_on_img = np.multiply(final_mask, self.img_gray) fvs = np.expand_dims(mask_on_img.ravel(), 1) fvs = MMS().fit_transform(fvs) ##returns vector the length of img_gray.shape[0]*img_gray.shape[1] return fvs
encoder = LE() encoder.fit(Y) encoded_Y = encoder.transform(Y) new_Y = pd.DataFrame(encoded_Y) print('\n new_Y.head(10)') print(new_Y.head(10)) # convert X and new_Y to numpy arrays X = X.values Y = new_Y.values # minmax X scaler = MMS(feature_range=(0, 1)) rescaledX = scaler.fit_transform(X) # split data into train and test datasets X_train, X_test, Y_train, Y_test = m_s.train_test_split( rescaledX, Y, test_size=validation_size, random_state=seed) print('\n basic preprocessing finished...') # create the model by doing the ff.. # 1. instantiate the Sequential class # 2. add initial DEnse layer with 108 input neurons, 'normal' # kernel_initializer, and 'relu' activation function # 3. add dense output layer with 1 neuron, 'normal' kernel_initializer # and 'sigmoid' activation function # 4. compile the model using the 'binary_cross_entrophy' loss function
def _normalization(self, array): a = array[array != 0].reshape(-1, 1) array[array != 0] = MMS().fit_transform(a).reshape(-1) + 1e-3 return array
print "{} Recall & {} Precision".format(featrscored[1], featpscored[1]) ranks1 = np.argsort(FeatureValue)[::-1] print "All features for Updated Featurelist" for n in range(len(features_list)-1): print "{} feature: {} ({})".format(n, features_list[n+1], FeatureValue[ranks1[n]]) ranks2 = np.argsort(FeatureValueDefault)[::-1] print "All features for Default Featurelist" for n in range(len(default_features_list)-1): print "{} feature: {} ({})".format(n, default_features_list[n+1], FeatureValueDefault[ranks2[n]]) #create scaler MMSs = MMS() #Create Transformers SKBt = SelectKBest(k=6) PCAt = PCA(random_state=42) FUt = FeatureUnion([("kbest", SKBt), ("pca", PCAt)]) #Create Classifiers GNBc = GaussianNB() SVCc = SVC(kernel="linear", random_state=42) DTCc = DecisionTreeClassifier(random_state=42) RFc = RandomForestClassifier(random_state=42) #final parameter section #create scaler MMSs = MMS() #Create Transformers
import matplotlib.pyplot as plot import pandas as pnd from keras.models import Sequential from keras.layers import Dense from keras.layers import LSTM as lest from keras.layers import Dropout from sklearn.preprocessing import MinMaxScaler as MMS # taking in the set for training dataSetTrain = pnd.read_csv('GSPTrain.csv') SetTrain = dataSetTrain.iloc[:, 1:2].values # Features ofor the scale sc = MMS(feature_range=(0, 1)) SetTrain_scaled = sc.fit_transform(SetTrain) trainedTime = [] trainedPrice = [] for i in range(60, 500): trainedTime.append(SetTrain_scaled[i - 60:i, 0]) trainedPrice.append(SetTrain_scaled[i, 0]) trainedTime, trainedPrice = np.array(trainedTime), np.array(trainedPrice) # Reshaping trainedTime = np.reshape(trainedTime, (trainedTime.shape[0], trainedTime.shape[1], 1)) # Creating the RNN
def inverse_transform(self, x): x = self._to_numpy(x) return (x - self.data_min_) * self.scale_ def partial_fit(self, x): self._check_type(x) x = self._to_numpy(x) self.data_min_ = np.minimum(self.data_min_, np.abs(x).min(axis=0)) self.data_max_ = np.maximum(self.data_max_, np.abs(x).max(axis=0)) self.data_range_ = self.data_max_ - self.data_min_ self.scale_ = (self.feature_range[0] - self.feature_range[0]) / self.data_range_ self.min_ = self.feature_range[0] - self.data_min_ * self.scale_ self.n_samples_seen_ += len(x) return self if __name__ == "__main__": from sklearn.preprocessing import MinMaxScaler as MMS X = [[-1, 2], [-0.5, 6], [0, 10], [1, 18]] scaler = MMS() print(scaler.fit(X).transform(X)) print(scaler.scale_) print(scaler.min_) my_scaler = MinMaxScaler() print(scaler.fit(X).transform(X)) print(scaler.scale_) print(scaler.min_)
hidden = sae.transform(X, sess) hidden.dump("hidden.npk") # Recon = sae.getRecon(X, sess) # Image.fromarray(I.tile_raster_images(X=Recon,img_shape=(28,28), tile_shape=(10, 10),tile_spacing=(1, 1))).save(r"Recon.png") if __name__ == '__main__': sparse_list = [[i] for i in np.arange(0.0, 0.5, 0.05)] folder = r'/home/zc8304/Documents/Raytheon/CleanedUserProg212' topics = [folder + '/' + x for x in os.listdir(folder)] for f in topics: case = f.split('/')[-1].split('+')[1] if not os.path.isdir(case): os.makedirs(case) os.chdir(case) X = np.load(f) index = np.random.choice(range(X.shape[0]), size=8000, replace=False) X = X[index] X = MMS().fit_transform(X) dim = [X.shape[1], X.shape[1]] for sparse in sparse_list: sparse_case_name = "_".join([str(x) for x in sparse]) if not os.path.isdir(sparse_case_name): os.makedirs(sparse_case_name) os.chdir(sparse_case_name) one_run(X=X, dim=dim, sparsities=sparse, iteration=300) os.chdir('../') os.chdir('../')
import numpy as np import pandas as pd import matplotlib.pyplot as plt import seaborn as sns from sklearn.model_selection import train_test_split as TTS import tensorflow as tf from sklearn.preprocessing import minmax_scale as MMS # READ DATA dfile = 'TensorFlow/TensorFlowBasics/cal_housing_clean.csv' df = pd.read_csv(dfile) # SCALE AND TRAIN TEST SPLIT tcol = 'medianHouseValue' scaled = MMS(df.drop(tcol, axis=1), feature_range=(0, 1), copy=True) X = pd.DataFrame(scaled, columns=list(df.drop(tcol, axis=1).columns)) y = df[tcol] X_train, X_test, y_train, y_test = TTS(X, y, test_size=0.3, random_state=64) # CREATE FEATURE COLUMNS fcols = [tf.feature_column.numeric_column(c) for c in list(X.columns)] print(fcols) # INPUT FUNCTION input_func_train = tf.estimator.inputs.pandas_input_fn(x=X_train, y=y_train, num_epochs=10000, num_threads=8, batch_size=1000, shuffle=True)
def __init__(self, lat=44.75, lon=-80.3125, height=1, grad_height=1, dates=[[2013, 5, 6, 7], [2014, 5, 6, 7]], when='20132014050607', add_time=True, add_day=False, scale='none', finite_diff='center', ignore=None): heights = [ -0.006, 0.058, 0.189, 0.320, 0.454, 0.589, 0.726, 0.864, # 16 heights 1.004, 1.146, 1.290, 1.436, 1.584, 1.759, 1.988, 2.249 ] self.heights = heights if ignore: self.ignore = ignore else: self.ignore = 'none' self.grad_height = grad_height self.scale_type = scale self.times = np.linspace(0.5, 23.5, 24)[1::3] self.dates = list_dates(dates) # dates = [[2013,5,6,7],[2014,5,6,7]] self.ground_vars = ['TS', 'SWGDN'] self.level_vars = ['T', 'U', 'V', 'QV', 'P', 'PT', 'UV'] self.grad_vars = ['U', 'V', 'PT', 'UV'] self.all_level_vars = [ var + str(i) for var in self.level_vars for i in range(15) ] self.all_grad_vars = [ var + 'G' + str(i) for var in self.grad_vars for i in range(1, 14) ] self.height_level_vars = [var + str(height) for var in self.level_vars] self.height_grad_vars = [ var + 'G' + str(grad_height) for var in self.grad_vars ] self.vars = self.ground_vars + self.all_level_vars + self.all_grad_vars self.all_vars = self.vars + ['PBLH'] + ['TIME'] hrs_idx = { 'halfnight': [0, 2, 3, 4, 5, 6], 'allnight': [3, 4, 5, 6], 'halfday': [4, 5, 6, 7], 'midnight': [1, 2, 3, 4, 5, 6] } dir_load = os.path.join(os.getcwd(), 'data') fname = when + 'LAT' + str(lat) + 'LON' + str(lon) + 'LVL0-14vers2.npy' data = np.load(os.path.join(dir_load, fname), encoding='latin1')[()] g, gas_const, P_0, r = 9.81, 287.058, 1e3, 0.61 # GET PRESSURES, POTENTIAL TEMPERATURE, WIND SPEED MAGNITUDE for heights 0 - 14 for date in self.dates: data[date + 'P0'] = data[date + 'PS'] data[date + 'UV0'] = data[date + 'U0']**2 + data[date + 'V0']**2 data[date + 'PT0'] = data[date + 'T0'] * (P_0 / data[date + 'P0'])**0.286 data[date + 'PTV0'] = (1 - r * data[date + 'QV0']) * data[date + 'PT0'] for lvl in range(1, len(heights) - 1): h = (heights[lvl - 1] - heights[lvl]) * 1e3 T_ave = 0.5 * (data[date + 'T' + str(lvl - 1)] + data[date + 'T' + str(lvl)]) P_lower_level = data[date + 'P' + str(lvl - 1)] exp_ratio = np.exp(g * h / gas_const / T_ave) print(date + str(lvl) + 'e^(gh/RT)=', exp_ratio) data[date + 'P' + str(lvl)] = P_lower_level * exp_ratio U_level, V_level = data[date + 'U' + str(lvl)], data[date + 'V' + str(lvl)] data[date + 'UV' + str(lvl)] = U_level**2 + V_level**2 T_level = data[date + 'T' + str(lvl)] P_level = data[date + 'P' + str(lvl)] QV_level = data[date + 'QV' + str(lvl)] PT = T_level * (P_0 / P_level)**0.286 data[date + 'PT' + str(lvl)] = PT data[date + 'PTV' + str(lvl)] = (1 - r * QV_level) * PT print( 'Calculated Pressures, potential temperatures and wind speed magnitudes' ) # GET VERTICAL GRADIENTS BY FINITE DIFFERENCES for date, lvl in product(self.dates, range(1, len(heights) - 2)): for var in ['U', 'V', 'PT', 'PTV']: if finite_diff == 'forward': delta_var = data[date + var + str(lvl + 1)] - data[date + var + str(lvl)] h = heights[lvl + 1] - heights[lvl] data[date + var + 'G' + str(lvl)] = delta_var / h elif finite_diff == 'back': delta_var = data[date + var + str(lvl)] - data[date + var + str(lvl - 1)] h = heights[lvl] - heights[lvl - 1] data[date + var + 'G' + str(lvl)] = delta_var / h else: delta_var = data[date + var + str(lvl + 1)] - data[date + var + str(lvl - 1)] h = heights[lvl + 1] - heights[lvl - 1] data[date + var + 'G' + str(lvl)] = delta_var / h data[date + 'UVG' + str(lvl)] = data[date + 'UG' + str(lvl)] ** 2 + \ data[date + 'VG' + str(lvl)] ** 2 print('Calculated Vertical Gradients') # ADD TIME or DAY for date in self.dates: # date is like '201305' days = int(data[date + 'TS'].shape[0] / 8) if add_time: data[date + 'TIME'] = np.tile(self.times, days) if add_day: data[date + 'DAY'] = np.array([[i] * 8 for i in range(1, days + 1) ]).reshape(-1) # TRIM DATA for date in self.dates: data[date + 'shape'] = [data[date + var].shape[0] for var in self.vars] print(date, data[date + 'shape']) if max(data[date + 'shape']) != min(data[date + 'shape']): for var in (self.vars + ['PBLH']): data[date + var] = data[date + var][:min(data[date + 'shape'])] if ignore: for date in self.dates: # date is like '201305' days = int(data[date + 'TS'].shape[0] / 8) n = hrs_idx[ignore] idx = [ n[i] + 8 * k for k in range(days) for i in range(len(n)) ] for var in self.all_vars: data[date + var] = data[date + var][idx] print(date + var, data[date + var].shape[0]) # SCALING self.scale = { 'TS': 300.0, 'SWGDN': 1000.0, 'PBLH': 1000.0, 'TIME': 10.0, 'QV': 0.01, 'U': 10.0, 'V': 10.0, 'UV': 10.0, 'P': 900.0, 'T': 100.0, 'PT': 100.0, 'UG': 10.0, 'VG': 10.0, 'UVG': 100.0, 'PTG': 10.0 } for date in self.dates: data[date + 'PBLH'] /= self.scale['PBLH'] for date, var in product(self.dates, self.ground_vars): if scale == 'custom': data[date + var] /= self.scale[var] elif scale == 'maxmin': min_pblh = min(data[date + 'PBLH']) max_pblh = max(data[date + 'PBLH']) X = data[date + var] data[date + var] = MMS(X, feature_range=(min_pblh, max_pblh)) elif scale == 'remove_mean': data[date + var] -= np.mean(data[date + var]) for date, var, lvl in product(self.dates, self.level_vars, range(len(heights) - 1)): if scale == 'custom': data[date + var + str(lvl)] /= self.scale[var] elif scale == 'maxmin': min_pblh = min(data[date + 'PBLH']) max_pblh = max(data[date + 'PBLH']) X = data[date + var + str(lvl)] data[date + var + str(lvl)] = MMS(X, feature_range=(min_pblh, max_pblh)) elif scale == 'remove_mean': data[date + var + str(lvl)] -= np.mean(data[date + var + str(lvl)]) for date, var, lvl in product(self.dates, self.grad_vars, range(1, len(heights) - 2)): if scale == 'custom': data[date + var + 'G' + str(lvl)] /= self.scale[var] elif scale == 'maxmin': min_pblh = min(data[date + 'PBLH']) max_pblh = max(data[date + 'PBLH']) X = data[date + var + 'G' + str(lvl)] data[date + var + 'G' + str(lvl)] = MMS( X, feature_range=(min_pblh, max_pblh)) elif scale == 'remove_mean': data[date + var + str(lvl)] -= np.mean(data[date + var + str(lvl)]) #print('scaled : '+var+' on '+date) self.data = data
def plot_graph(graph=nx.tutte_graph(), layout=nx.kamada_kawai_layout): """ Make plotly visualization of networkx graph. node_size -> betweeness centrality node_color -> closeness centrality """ b_cents = nx.betweenness_centrality(graph) c_cents = nx.closeness_centrality(graph) d_cents = graph.degree() edge_x = [] edge_y = [] pos = layout(graph) for edge in graph.edges: x0, y0 = pos[edge[0]] x1, y1 = pos[edge[1]] edge_x.append(x0) edge_x.append(x1) edge_x.append(None) edge_y.append(y0) edge_y.append(y1) edge_y.append(None) edge_trace = go.Scatter( x=edge_x, y=edge_y, line=dict(width=0.5, color='#888'), hoverinfo='none', mode='lines') node_x = [] node_y = [] node_texts = [] node_degrees = [] node_closenesses = [] node_betweenesses = [] for node in graph.nodes(): x, y = pos[node] node_x.append(x) node_y.append(y) node_degree = d_cents[node] node_closeness = c_cents[node] node_betweeness = b_cents[node] node_degrees.append(node_degree) node_closenesses.append(node_closeness) node_betweenesses.append(node_betweeness) node_text = str(node) + "<br>Degree: " + str(node_degree) node_texts.append(node_text) node_trace = go.Scatter( x=node_x, y=node_y, mode='markers', hoverinfo='text', marker=dict( opacity=1, showscale=True, colorscale='Jet', reversescale=False, color=[], size=[], colorbar=dict( thickness=15, title='Closeness', xanchor='left', titleside='right' ), line=dict( width=1, color='Black' ), ) ) node_degrees = np.array(node_degrees) node_closenesses = np.array(node_closenesses) node_betweenesses = np.array(node_betweenesses) size_scaler = MMS(feature_range=(7.5, 17.5)) node_betweenesses = size_scaler.fit_transform(node_betweenesses.reshape(-1,1)).ravel() node_trace.marker.color = node_closenesses node_trace.marker.size = node_betweenesses node_trace.text = node_texts fig = go.Figure( data=[ edge_trace, node_trace ], layout=go.Layout( autosize=True, titlefont_size=16, showlegend=False, hovermode='closest', margin=dict(b=20,l=5,r=5,t=40), xaxis=dict(showgrid=False, zeroline=False, showticklabels=False), yaxis=dict(showgrid=False, zeroline=False, showticklabels=False)) ) iplot(fig)
input_dim=1, activation='relu', kernel_initializer=weight_init, bias_initializer=bias_init)) model.add( Dense(10, activation='relu', kernel_initializer=weight_init, bias_initializer=bias_init)) model.add(Dense(1, kernel_initializer=weight_init, bias_initializer=bias_init)) model.compile(optimizer=optimizers.RMSprop(0.01), loss='mse') x = np.linspace(-2 * np.pi, 2 * np.pi, 1000).reshape(-1, 1) y = np.sin(x) mms = MMS() x_train = mms.fit_transform(x) y_train = mms.fit_transform(y) #tensorboard = TensorBoard(log_dir='log') #model.fit(x, y, batch_size=10, epochs=100, shuffle=True, callbacks=[tensorboard]) #model.fit(x_train, y_train, batch_size=10, epochs=500, shuffle=True) #print('\n'.join(['%s:%s' % item for item in model.__dict__.items()])) #model.fit(x[500:], y[500:], batch_size=10, epochs=100, shuffle=True) #model.train_on_batch(x[500:], y[500:]) #model.train_on_batch(x[:500], y[:500]) for step in range(100): indexes = np.random.randint(1000, size=32) loss = model.train_on_batch(x[indexes], y[indexes]) #loss = model.train_on_batch(x, y) print("loss: {} after {} step".format(loss, step + 1))