def _initiate_scalers(self, rmCol):
'''
Scaling for control device: linear minmax to [0.1, 0.9]
Scaling for power: logarithmic element-wise minmax to [0.1, 0.9]
'''
# Treat rmCol
if rmCol is None:
nrmCol = 0
else:
nrmCol = len(rmCol)
# Find limits
minh = np.inf
maxh = 0
minp = np.full(self.n_y[1] - nrmCol, np.inf)
maxp = np.zeros(self.n_y[1] - nrmCol)
for rec in self.pwrl:
# Min and max of all sbh (flattened array)
minh = min(minh, np.amin(rec.inx))
maxh = max(maxh, np.amax(rec.inx))
# Min and max of each column (e.g. fuel element for MITR)
minp = np.amin(np.vstack((minp, np.amin(rec.iny, axis=0))), axis=0)
maxp = np.amax(np.vstack((maxp, np.amax(rec.iny, axis=0))), axis=0)
# Save limits
self.minh = minh
self.maxh = maxh
self.minp = minp
self.maxp = maxp
# Create MMS objects
self.heightMMS = MMS((0.1, 0.9))
self.heightMMS.fit([[minh], [maxh]])
self.powerMMS = MMS((0.1, 0.9))
self.powerMMS.fit(np.log([minp, maxp]))
def _gradient_(self):
data = self.img_gray.ravel().copy()
fvs = np.zeros((len(data), self.neighborhood)).astype('uint8')
img_gray_pad = np.pad(self.img_gray.copy(), 1, mode='edge')
cnt = 0
for i in range(1, self.img_gray.shape[1], 1):
for j in range(1, self.img_gray.shape[0], 1):
if (self.neighborhood == 8):
fvs[cnt, 0] = data[cnt] - img_gray_pad[j - 1, i - 1]
fvs[cnt, 1] = data[cnt] - img_gray_pad[j - 1, i]
fvs[cnt, 2] = data[cnt] - img_gray_pad[j - 1, i + 1]
fvs[cnt, 3] = data[cnt] - img_gray_pad[j, i - 1]
fvs[cnt, 4] = data[cnt] - img_gray_pad[j, i + 1]
fvs[cnt, 5] = data[cnt] - img_gray_pad[j + 1, i - 1]
fvs[cnt, 6] = data[cnt] - img_gray_pad[j + 1, i]
fvs[cnt, 7] = data[cnt] - img_gray_pad[j + 1, i + 1]
elif (self.neighborhood == 4):
fvs[cnt, 0] = data[cnt] - img_gray_pad[j, i - 1]
fvs[cnt, 1] = data[cnt] - img_gray_pad[j, i + 1]
fvs[cnt, 2] = data[cnt] - img_gray_pad[j - 1, i]
fvs[cnt, 3] = data[cnt] - img_gray_pad[j + 1, i]
cnt = cnt + 1
fvs = MMS().fit_transform(fvs)
##returns vector that is neighborhood wide of data length
return fvs
def loadMat(fn):
f = sio.loadmat(fn)
events = f['events'].astype(np.int16)
sounds = f['sounds'].astype(np.float16)
rates = MMS(f['rates'].squeeze())
return events, sounds, rates
def test_fit1(self):
scale_param = self.get_scale_param()
scale_param.scale_column_idx = []
scale_param.feat_upper = 2
scale_param.feat_lower = 1
scale_obj = MinMaxScale(scale_param)
fit_instance = scale_obj.fit(self.table_instance)
column_min_value = scale_obj.column_min_value
column_max_value = scale_obj.column_max_value
for i, line in enumerate(self.test_data):
for j, value in enumerate(line):
if value > 2:
self.test_data[i][j] = 2
elif value < 1:
self.test_data[i][j] = 1
scaler = MMS()
scaler.fit(self.test_data)
self.assertListEqual(self.get_table_instance_feature(fit_instance),
np.around(scaler.transform(self.test_data), 6).tolist())
data_min = list(scaler.data_min_)
data_max = list(scaler.data_max_)
self.assertListEqual(column_min_value, data_min)
self.assertListEqual(column_max_value, data_max)
transform_data = scale_obj.transform(self.table_instance)
self.assertListEqual(self.get_table_instance_feature(fit_instance),
self.get_table_instance_feature(transform_data))
def test_cols_select_fit_and_transform_repeat(self):
scale_column_idx = [1, 1, 2, 2, 4, 5, 5]
min_max_scaler = MinMaxScaler(mode='normal',
area='col',
scale_column_idx=scale_column_idx,
feat_upper=None,
feat_lower=None,
out_upper=None,
out_lower=None)
fit_data, cols_transform_value = min_max_scaler.fit(
self.table_instance)
scaler = MMS()
scaler.fit(self.test_data)
mms_transform_data = np.around(scaler.transform(self.test_data),
4).tolist()
for i, line in enumerate(mms_transform_data):
for j, cols in enumerate(line):
if j not in scale_column_idx:
mms_transform_data[i][j] = self.test_data[i][j]
self.assertListEqual(self.get_table_instance_feature(fit_data),
mms_transform_data)
transform_data = min_max_scaler.transform(self.table_instance,
cols_transform_value)
self.assertListEqual(self.get_table_instance_feature(transform_data),
mms_transform_data)
def normalise_numeric_features(X,
standardisation=False,
means=True,
stdev=True):
""" Normalisation for numeric features
:param X: A numpy matrix of the data. First axis corresponding to instances, second axis corresponding to samples
:param standardisation: Whether standardisation needs to be done instead of normalisation. Default: False
:param means: Whether the mean should be normalised. Default: True
:param stdev: Whether the standard devation should be normalised. Default: True
:return: X and features with numeric features normalised.
"""
column_types = column_types_dataset(X, categorical=False)
for i in range(len(column_types)):
if column_types[i]:
if standardisation:
# Standardisation
scaler = MMS([0, 1])
X[:, i:i + 1] = scaler.fit_transform(X[:, i:i + 1])
else:
# Normalisation
scaler = SS(means, stdev)
X[:, i:i + 1] = scaler.fit_transform(X[:, i:i + 1])
return X
def get_feature_vectors(img_rgb,
process='intensity',
neighborhood=8,
radius=1,
METHOD='uniform'):
from skimage.color import rgb2ycbcr as r2y
from skimage.feature import local_binary_pattern as LBP
import cv2
img_gray = cv2.cvtColor(img_rgb.copy(), cv2.COLOR_RGB2GRAY)
feats = np.expand_dims(img_gray.ravel().copy(), 1)
feats = MMS().fit_transform(feats)
get_features = fvs(img_rgb,
neighborhood=neighborhood,
radius=radius,
METHOD=METHOD)
if ('intensity' in process):
temp = get_features._intensity_()
feats = np.concatenate((feats, temp), axis=1)
if ('gradient' in process):
temp = get_features._gradient_()
feats = np.concatenate((feats, temp), axis=1)
if ('texture' in process):
temp = get_features._texture_()
feats = np.concatenate((feats, temp), axis=1)
if ('cb_channel' in process):
temp = get_features._ycbcr_mask()
feats = np.concatenate((feats, temp), axis=1)
return feats
def test_fit_col_feat(self):
feat_upper = [0.8, 5, 6, 7, 8, 9]
feat_lower = [0.5, 2, 3, 3, 4, 4.0]
out_upper = 2.0
out_lower = -1.0
min_max_scaler = MinMaxScaler(mode='normal', area='col', feat_upper=feat_upper, feat_lower=feat_lower,
out_upper=out_upper, out_lower=out_lower)
fit_data, cols_transform_value = min_max_scaler.fit(self.table_instance)
new_data = []
for data in self.test_data:
tmp_data = []
for i in range(len(data)):
if data[i] > feat_upper[i]:
tmp_data.append(feat_upper[i])
elif data[i] < feat_lower[i]:
tmp_data.append(feat_lower[i])
else:
tmp_data.append(data[i])
new_data.append(tmp_data)
feature_range = (out_lower, out_upper)
scaler = MMS(feature_range)
scaler.fit(new_data)
self.assertListEqual(self.get_table_instance_feature(fit_data),
np.around(scaler.transform(new_data), 4).tolist())
def readTestData():
testData = np.loadtxt('data/test.csv', delimiter=',', skiprows=1)
xTest = testData[:, 1:31]
scale = MMS()
allX = scale.fit_transform(xTest)
indexTest = list(testData[:, 0])
return [allX, indexTest]
def readTrainingData():
data = np.loadtxt('data/training.csv',
delimiter=',',
skiprows=1,
converters={32: lambda x: int(x == 's'.encode('utf-8'))})
allY = data[:, 32]
allX = data[:, 1:31]
allW = data[:, 31]
scale = MMS()
allX = scale.fit_transform(allX)
np.random.seed(42)
r = np.random.rand(allY.shape[0])
xTrain = allX[r <= 0.4]
yTrain = allY[r <= 0.4]
wTrain = allW[r <= 0.4]
xValid = allX[r > 0.7]
yValid = allY[r > 0.7]
wValid = allW[r > 0.7]
v = np.random.rand(yValid.shape[0])
xCrossValid = xValid[v <= 0.5]
yCrossValid = yValid[v <= 0.5]
wCrossValid = wValid[v <= 0.5]
xTestValid = xValid[v > 0.5]
yTestValid = yValid[v > 0.5]
wTestValid = wValid[v > 0.5]
return [
xTrain, yTrain, wTrain, xCrossValid, yCrossValid, wCrossValid,
xTestValid, yTestValid, wTestValid
]
def _texture_(self):
n_pts = 8 * self.radius
lbp_img = LBP(self.img_gray.copy(), n_pts, self.radius, self.METHOD)
fvs = np.expand_dims(lbp_img.ravel(), 1)
fvs = MMS().fit_transform(fvs)
##returns vector that is the length of img_gray.shape[0]*img_gray.shape[1]
return fvs
def test_fit_instance_default(self):
min_max_scaler = MinMaxScaler(mode='normal', area='all', feat_upper=None, feat_lower=None, out_upper=None,
out_lower=None)
fit_instance, cols_transform_value = min_max_scaler.fit(self.table_instance)
scaler = MMS()
scaler.fit(self.test_data)
self.assertListEqual(self.get_table_instance_feature(fit_instance),
np.around(scaler.transform(self.test_data), 4).tolist())
self.assertListEqual(cols_transform_value, self.sklearn_attribute_format(scaler, [0, 1]))
def test_fit_col_out(self):
min_max_scaler = MinMaxScaler(mode='normal', area='col', feat_upper=None, feat_lower=None, out_upper=2,
out_lower=-1)
fit_data, cols_transform_value = min_max_scaler.fit(self.table_instance)
feature_range = (-1, 2)
scaler = MMS(feature_range)
scaler.fit(self.test_data)
self.assertListEqual(self.get_table_instance_feature(fit_data),
np.around(scaler.transform(self.test_data), 4).tolist())
self.assertListEqual(cols_transform_value, self.sklearn_attribute_format(scaler, feature_range))
def test_fit5(self):
scale_column_idx = [1, 2, 4]
scale_names = ['fid1', 'fid2', 'fid1000']
scale_param = self.get_scale_param()
scale_param.mode = "cap"
# scale_param.area = "col"
scale_param.feat_upper = 0.8
scale_param.feat_lower = 0.2
scale_param.scale_names = scale_names
scale_param.scale_col_indexes = [2, 4]
scale_obj = MinMaxScale(scale_param)
fit_instance = scale_obj.fit(self.table_instance)
column_min_value = scale_obj.column_min_value
column_max_value = scale_obj.column_max_value
raw_data = copy.deepcopy(self.test_data)
gt_cap_lower_list = [0, 2, 2, 2, 3, 1]
gt_cap_upper_list = [1, 8, 8, 8, 7, 8]
for i, line in enumerate(self.test_data):
for j, value in enumerate(line):
if value > gt_cap_upper_list[j]:
self.test_data[i][j] = gt_cap_upper_list[j]
elif value < gt_cap_lower_list[j]:
self.test_data[i][j] = gt_cap_lower_list[j]
scaler = MMS()
scaler.fit(self.test_data)
sklearn_transform_data = np.around(scaler.transform(self.test_data),
6).tolist()
for i, line in enumerate(sklearn_transform_data):
for j, cols in enumerate(line):
if j not in scale_column_idx:
sklearn_transform_data[i][j] = raw_data[i][j]
fit_data = np.round(self.get_table_instance_feature(fit_instance),
6).tolist()
self.assertListEqual(fit_data, sklearn_transform_data)
for i, line in enumerate(sklearn_transform_data):
for j, cols in enumerate(line):
if j not in scale_column_idx:
sklearn_transform_data[i][j] = raw_data[i][j]
data_min = list(scaler.data_min_)
data_max = list(scaler.data_max_)
self.assertListEqual(column_min_value, data_min)
self.assertListEqual(column_max_value, data_max)
transform_data = scale_obj.transform(self.table_instance)
self.assertListEqual(self.get_table_instance_feature(fit_instance),
self.get_table_instance_feature(transform_data))
def kpca_preprocess_features(X):
mms = MMS(feature_range=(-0.5, 0.5))
n_rows, n_cols = X.shape
for j in range(n_cols):
col_max = max(X[:, j])
col_min = min(X[:, j])
if col_max <= 1.0 and col_min >= -1.0:
pass
else:
X[:, j] = power(X[:, j], 0.125)
X = mms.fit_transform(X)
return X
def test_fit4(self):
scale_column_idx = [1, 2, 4]
scale_param = self.get_scale_param()
# scale_param.area = "col"
scale_param.feat_upper = 2
scale_param.feat_lower = 1
scale_param.scale_col_indexes = scale_column_idx
scale_obj = MinMaxScale(scale_param)
fit_instance = scale_obj.fit(self.table_instance)
column_min_value = scale_obj.column_min_value
column_max_value = scale_obj.column_max_value
raw_data = copy.deepcopy(self.test_data)
for i, line in enumerate(self.test_data):
for j, value in enumerate(line):
if j in scale_column_idx:
if value > 2:
self.test_data[i][j] = 2
elif value < 1:
self.test_data[i][j] = 1
scaler = MMS()
scaler.fit(self.test_data)
sklearn_transform_data = np.around(scaler.transform(self.test_data),
6).tolist()
for i, line in enumerate(sklearn_transform_data):
for j, cols in enumerate(line):
if j not in scale_column_idx:
sklearn_transform_data[i][j] = raw_data[i][j]
self.assertListEqual(self.get_table_instance_feature(fit_instance),
sklearn_transform_data)
for i, line in enumerate(sklearn_transform_data):
for j, cols in enumerate(line):
if j not in scale_column_idx:
sklearn_transform_data[i][j] = raw_data[i][j]
data_min = list(scaler.data_min_)
data_max = list(scaler.data_max_)
self.assertListEqual(column_min_value, data_min)
self.assertListEqual(column_max_value, data_max)
transform_data = scale_obj.transform(self.table_instance)
self.assertListEqual(self.get_table_instance_feature(fit_instance),
self.get_table_instance_feature(transform_data))
def test_fit_feat(self):
feat_upper = 8
feat_lower = 3
out_upper = 2
out_lower = -1
min_max_scaler = MinMaxScaler(mode='normal',
area='all',
feat_upper=feat_upper,
feat_lower=feat_lower,
out_upper=out_upper,
out_lower=out_lower)
fit_data, cols_transform_value = min_max_scaler.fit(
self.table_instance)
new_data = []
for data in self.test_data:
tmp_data = []
for i in range(len(data)):
if data[i] > feat_upper:
tmp_data.append(feat_upper)
elif data[i] < feat_lower:
tmp_data.append(feat_lower)
else:
tmp_data.append(data[i])
new_data.append(tmp_data)
feature_range = (out_lower, out_upper)
scaler = MMS(feature_range)
scaler.fit(new_data)
self.assertListEqual(self.get_table_instance_feature(fit_data),
np.around(scaler.transform(new_data), 4).tolist())
sklearn_res = self.sklearn_attribute_format(scaler,
[out_lower, out_upper])
# self.assertListEqual(cols_transform_value, [(feat_lower, feat_upper, out_lower, out_upper)])
for i in range(len(sklearn_res)):
if sklearn_res[i][0] != feat_lower:
tmp_res = list(sklearn_res[i])
tmp_res[0] = feat_lower
sklearn_res[i] = tuple(tmp_res)
if sklearn_res[i][1] != feat_upper:
tmp_res = list(sklearn_res[i])
tmp_res[1] = feat_upper
sklearn_res[i] = tuple(tmp_res)
self.assertListEqual(cols_transform_value[0], sklearn_res)
def test_fit_instance_default(self):
scale_param = self.get_scale_param()
scale_param.scale_col_indexes = -1
scale_obj = MinMaxScale(scale_param)
fit_instance = scale_obj.fit(self.table_instance)
column_min_value = scale_obj.column_min_value
column_max_value = scale_obj.column_max_value
scaler = MMS()
scaler.fit(self.test_data)
self.assertListEqual(self.get_table_instance_feature(fit_instance),
np.around(scaler.transform(self.test_data), 6).tolist())
data_min = list(scaler.data_min_)
data_max = list(scaler.data_max_)
self.assertListEqual(column_min_value, data_min)
self.assertListEqual(column_max_value, data_max)
transform_data = scale_obj.transform(self.table_instance)
self.assertListEqual(self.get_table_instance_feature(fit_instance),
self.get_table_instance_feature(transform_data))
def _ycbcr_mask(self):
img_ycrcb = r2y(self.img_rgb)
y_channel = img_ycrcb[:, :, 0]
mean = np.mean(y_channel.copy())
std = np.std(y_channel.copy())
y_channel[y_channel < (mean - std)] = 0
y_channel[y_channel != 0] = 1
y_channel[y_channel == 0] = 255
y_channel[y_channel == 1] = 0
cr_channel = img_ycrcb[:, :, 1]
mean = np.mean(cr_channel.copy())
std = np.std(cr_channel.copy())
cr_channel[cr_channel < (mean - std)] = 0
cr_channel[cr_channel != 0] = 1
cr_channel[cr_channel == 0] = 255
cr_channel[cr_channel == 1] = 0
y_cr_or = np.logical_and(y_channel,
np.uint8(1 - cr_channel)).astype('uint8')
y_cr_or[y_cr_or != 0] = 255
cb_channel = img_ycrcb[:, :, 2]
mean = np.mean(cb_channel.copy())
std = np.std(cb_channel.copy())
cb_channel[cb_channel < (mean - std)] = 0
cb_channel[cb_channel != 0] = 1
cb_channel[cb_channel == 0] = 255
cb_channel[cb_channel == 1] = 0
cb_y_cr_or = np.logical_or(y_cr_or, cb_channel).astype('uint8')
cb_y_cr_or[cb_y_cr_or != 0] = 1
SE = np.ones((3, 3)).astype('uint8')
final_mask = cv2.morphologyEx(cb_y_cr_or.copy(), cv2.MORPH_OPEN,
SE).astype('uint8')
mask_on_img = np.multiply(final_mask, self.img_gray)
fvs = np.expand_dims(mask_on_img.ravel(), 1)
fvs = MMS().fit_transform(fvs)
##returns vector the length of img_gray.shape[0]*img_gray.shape[1]
return fvs
encoder = LE()
encoder.fit(Y)
encoded_Y = encoder.transform(Y)
new_Y = pd.DataFrame(encoded_Y)
print('\n new_Y.head(10)')
print(new_Y.head(10))
# convert X and new_Y to numpy arrays
X = X.values
Y = new_Y.values
# minmax X
scaler = MMS(feature_range=(0, 1))
rescaledX = scaler.fit_transform(X)
# split data into train and test datasets
X_train, X_test, Y_train, Y_test = m_s.train_test_split(
rescaledX, Y, test_size=validation_size, random_state=seed)
print('\n basic preprocessing finished...')
# create the model by doing the ff..
# 1. instantiate the Sequential class
# 2. add initial DEnse layer with 108 input neurons, 'normal'
# kernel_initializer, and 'relu' activation function
# 3. add dense output layer with 1 neuron, 'normal' kernel_initializer
# and 'sigmoid' activation function
# 4. compile the model using the 'binary_cross_entrophy' loss function
def _normalization(self, array):
a = array[array != 0].reshape(-1, 1)
array[array != 0] = MMS().fit_transform(a).reshape(-1) + 1e-3
return array
print "{} Recall & {} Precision".format(featrscored[1], featpscored[1])
ranks1 = np.argsort(FeatureValue)[::-1]
print "All features for Updated Featurelist"
for n in range(len(features_list)-1):
print "{} feature: {} ({})".format(n, features_list[n+1], FeatureValue[ranks1[n]])
ranks2 = np.argsort(FeatureValueDefault)[::-1]
print "All features for Default Featurelist"
for n in range(len(default_features_list)-1):
print "{} feature: {} ({})".format(n, default_features_list[n+1], FeatureValueDefault[ranks2[n]])
#create scaler
MMSs = MMS()
#Create Transformers
SKBt = SelectKBest(k=6)
PCAt = PCA(random_state=42)
FUt = FeatureUnion([("kbest", SKBt), ("pca", PCAt)])
#Create Classifiers
GNBc = GaussianNB()
SVCc = SVC(kernel="linear", random_state=42)
DTCc = DecisionTreeClassifier(random_state=42)
RFc = RandomForestClassifier(random_state=42)
#final parameter section
#create scaler
MMSs = MMS()
#Create Transformers
import matplotlib.pyplot as plot
import pandas as pnd
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM as lest
from keras.layers import Dropout
from sklearn.preprocessing import MinMaxScaler as MMS
# taking in the set for training
dataSetTrain = pnd.read_csv('GSPTrain.csv')
SetTrain = dataSetTrain.iloc[:, 1:2].values
# Features ofor the scale
sc = MMS(feature_range=(0, 1))
SetTrain_scaled = sc.fit_transform(SetTrain)
trainedTime = []
trainedPrice = []
for i in range(60, 500):
trainedTime.append(SetTrain_scaled[i - 60:i, 0])
trainedPrice.append(SetTrain_scaled[i, 0])
trainedTime, trainedPrice = np.array(trainedTime), np.array(trainedPrice)
# Reshaping
trainedTime = np.reshape(trainedTime,
(trainedTime.shape[0], trainedTime.shape[1], 1))
# Creating the RNN
def inverse_transform(self, x):
x = self._to_numpy(x)
return (x - self.data_min_) * self.scale_
def partial_fit(self, x):
self._check_type(x)
x = self._to_numpy(x)
self.data_min_ = np.minimum(self.data_min_, np.abs(x).min(axis=0))
self.data_max_ = np.maximum(self.data_max_, np.abs(x).max(axis=0))
self.data_range_ = self.data_max_ - self.data_min_
self.scale_ = (self.feature_range[0] -
self.feature_range[0]) / self.data_range_
self.min_ = self.feature_range[0] - self.data_min_ * self.scale_
self.n_samples_seen_ += len(x)
return self
if __name__ == "__main__":
from sklearn.preprocessing import MinMaxScaler as MMS
X = [[-1, 2], [-0.5, 6], [0, 10], [1, 18]]
scaler = MMS()
print(scaler.fit(X).transform(X))
print(scaler.scale_)
print(scaler.min_)
my_scaler = MinMaxScaler()
print(scaler.fit(X).transform(X))
print(scaler.scale_)
print(scaler.min_)
hidden = sae.transform(X, sess)
hidden.dump("hidden.npk")
# Recon = sae.getRecon(X, sess)
# Image.fromarray(I.tile_raster_images(X=Recon,img_shape=(28,28), tile_shape=(10, 10),tile_spacing=(1, 1))).save(r"Recon.png")
if __name__ == '__main__':
sparse_list = [[i] for i in np.arange(0.0, 0.5, 0.05)]
folder = r'/home/zc8304/Documents/Raytheon/CleanedUserProg212'
topics = [folder + '/' + x for x in os.listdir(folder)]
for f in topics:
case = f.split('/')[-1].split('+')[1]
if not os.path.isdir(case):
os.makedirs(case)
os.chdir(case)
X = np.load(f)
index = np.random.choice(range(X.shape[0]), size=8000, replace=False)
X = X[index]
X = MMS().fit_transform(X)
dim = [X.shape[1], X.shape[1]]
for sparse in sparse_list:
sparse_case_name = "_".join([str(x) for x in sparse])
if not os.path.isdir(sparse_case_name):
os.makedirs(sparse_case_name)
os.chdir(sparse_case_name)
one_run(X=X, dim=dim, sparsities=sparse, iteration=300)
os.chdir('../')
os.chdir('../')
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split as TTS
import tensorflow as tf
from sklearn.preprocessing import minmax_scale as MMS
# READ DATA
dfile = 'TensorFlow/TensorFlowBasics/cal_housing_clean.csv'
df = pd.read_csv(dfile)
# SCALE AND TRAIN TEST SPLIT
tcol = 'medianHouseValue'
scaled = MMS(df.drop(tcol, axis=1), feature_range=(0, 1), copy=True)
X = pd.DataFrame(scaled, columns=list(df.drop(tcol, axis=1).columns))
y = df[tcol]
X_train, X_test, y_train, y_test = TTS(X, y, test_size=0.3, random_state=64)
# CREATE FEATURE COLUMNS
fcols = [tf.feature_column.numeric_column(c) for c in list(X.columns)]
print(fcols)
# INPUT FUNCTION
input_func_train = tf.estimator.inputs.pandas_input_fn(x=X_train,
y=y_train,
num_epochs=10000,
num_threads=8,
batch_size=1000,
shuffle=True)
def __init__(self,
lat=44.75,
lon=-80.3125,
height=1,
grad_height=1,
dates=[[2013, 5, 6, 7], [2014, 5, 6, 7]],
when='20132014050607',
add_time=True,
add_day=False,
scale='none',
finite_diff='center',
ignore=None):
heights = [
-0.006,
0.058,
0.189,
0.320,
0.454,
0.589,
0.726,
0.864, # 16 heights
1.004,
1.146,
1.290,
1.436,
1.584,
1.759,
1.988,
2.249
]
self.heights = heights
if ignore:
self.ignore = ignore
else:
self.ignore = 'none'
self.grad_height = grad_height
self.scale_type = scale
self.times = np.linspace(0.5, 23.5, 24)[1::3]
self.dates = list_dates(dates) # dates = [[2013,5,6,7],[2014,5,6,7]]
self.ground_vars = ['TS', 'SWGDN']
self.level_vars = ['T', 'U', 'V', 'QV', 'P', 'PT', 'UV']
self.grad_vars = ['U', 'V', 'PT', 'UV']
self.all_level_vars = [
var + str(i) for var in self.level_vars for i in range(15)
]
self.all_grad_vars = [
var + 'G' + str(i) for var in self.grad_vars for i in range(1, 14)
]
self.height_level_vars = [var + str(height) for var in self.level_vars]
self.height_grad_vars = [
var + 'G' + str(grad_height) for var in self.grad_vars
]
self.vars = self.ground_vars + self.all_level_vars + self.all_grad_vars
self.all_vars = self.vars + ['PBLH'] + ['TIME']
hrs_idx = {
'halfnight': [0, 2, 3, 4, 5, 6],
'allnight': [3, 4, 5, 6],
'halfday': [4, 5, 6, 7],
'midnight': [1, 2, 3, 4, 5, 6]
}
dir_load = os.path.join(os.getcwd(), 'data')
fname = when + 'LAT' + str(lat) + 'LON' + str(lon) + 'LVL0-14vers2.npy'
data = np.load(os.path.join(dir_load, fname), encoding='latin1')[()]
g, gas_const, P_0, r = 9.81, 287.058, 1e3, 0.61
# GET PRESSURES, POTENTIAL TEMPERATURE, WIND SPEED MAGNITUDE for heights 0 - 14
for date in self.dates:
data[date + 'P0'] = data[date + 'PS']
data[date + 'UV0'] = data[date + 'U0']**2 + data[date + 'V0']**2
data[date +
'PT0'] = data[date + 'T0'] * (P_0 / data[date + 'P0'])**0.286
data[date +
'PTV0'] = (1 - r * data[date + 'QV0']) * data[date + 'PT0']
for lvl in range(1, len(heights) - 1):
h = (heights[lvl - 1] - heights[lvl]) * 1e3
T_ave = 0.5 * (data[date + 'T' + str(lvl - 1)] +
data[date + 'T' + str(lvl)])
P_lower_level = data[date + 'P' + str(lvl - 1)]
exp_ratio = np.exp(g * h / gas_const / T_ave)
print(date + str(lvl) + 'e^(gh/RT)=', exp_ratio)
data[date + 'P' + str(lvl)] = P_lower_level * exp_ratio
U_level, V_level = data[date + 'U' +
str(lvl)], data[date + 'V' + str(lvl)]
data[date + 'UV' + str(lvl)] = U_level**2 + V_level**2
T_level = data[date + 'T' + str(lvl)]
P_level = data[date + 'P' + str(lvl)]
QV_level = data[date + 'QV' + str(lvl)]
PT = T_level * (P_0 / P_level)**0.286
data[date + 'PT' + str(lvl)] = PT
data[date + 'PTV' + str(lvl)] = (1 - r * QV_level) * PT
print(
'Calculated Pressures, potential temperatures and wind speed magnitudes'
)
# GET VERTICAL GRADIENTS BY FINITE DIFFERENCES
for date, lvl in product(self.dates, range(1, len(heights) - 2)):
for var in ['U', 'V', 'PT', 'PTV']:
if finite_diff == 'forward':
delta_var = data[date + var +
str(lvl + 1)] - data[date + var +
str(lvl)]
h = heights[lvl + 1] - heights[lvl]
data[date + var + 'G' + str(lvl)] = delta_var / h
elif finite_diff == 'back':
delta_var = data[date + var +
str(lvl)] - data[date + var +
str(lvl - 1)]
h = heights[lvl] - heights[lvl - 1]
data[date + var + 'G' + str(lvl)] = delta_var / h
else:
delta_var = data[date + var +
str(lvl + 1)] - data[date + var +
str(lvl - 1)]
h = heights[lvl + 1] - heights[lvl - 1]
data[date + var + 'G' + str(lvl)] = delta_var / h
data[date + 'UVG' + str(lvl)] = data[date + 'UG' + str(lvl)] ** 2 + \
data[date + 'VG' + str(lvl)] ** 2
print('Calculated Vertical Gradients')
# ADD TIME or DAY
for date in self.dates: # date is like '201305'
days = int(data[date + 'TS'].shape[0] / 8)
if add_time:
data[date + 'TIME'] = np.tile(self.times, days)
if add_day:
data[date + 'DAY'] = np.array([[i] * 8
for i in range(1, days + 1)
]).reshape(-1)
# TRIM DATA
for date in self.dates:
data[date +
'shape'] = [data[date + var].shape[0] for var in self.vars]
print(date, data[date + 'shape'])
if max(data[date + 'shape']) != min(data[date + 'shape']):
for var in (self.vars + ['PBLH']):
data[date + var] = data[date + var][:min(data[date +
'shape'])]
if ignore:
for date in self.dates: # date is like '201305'
days = int(data[date + 'TS'].shape[0] / 8)
n = hrs_idx[ignore]
idx = [
n[i] + 8 * k for k in range(days) for i in range(len(n))
]
for var in self.all_vars:
data[date + var] = data[date + var][idx]
print(date + var, data[date + var].shape[0])
# SCALING
self.scale = {
'TS': 300.0,
'SWGDN': 1000.0,
'PBLH': 1000.0,
'TIME': 10.0,
'QV': 0.01,
'U': 10.0,
'V': 10.0,
'UV': 10.0,
'P': 900.0,
'T': 100.0,
'PT': 100.0,
'UG': 10.0,
'VG': 10.0,
'UVG': 100.0,
'PTG': 10.0
}
for date in self.dates:
data[date + 'PBLH'] /= self.scale['PBLH']
for date, var in product(self.dates, self.ground_vars):
if scale == 'custom':
data[date + var] /= self.scale[var]
elif scale == 'maxmin':
min_pblh = min(data[date + 'PBLH'])
max_pblh = max(data[date + 'PBLH'])
X = data[date + var]
data[date + var] = MMS(X, feature_range=(min_pblh, max_pblh))
elif scale == 'remove_mean':
data[date + var] -= np.mean(data[date + var])
for date, var, lvl in product(self.dates, self.level_vars,
range(len(heights) - 1)):
if scale == 'custom':
data[date + var + str(lvl)] /= self.scale[var]
elif scale == 'maxmin':
min_pblh = min(data[date + 'PBLH'])
max_pblh = max(data[date + 'PBLH'])
X = data[date + var + str(lvl)]
data[date + var + str(lvl)] = MMS(X,
feature_range=(min_pblh,
max_pblh))
elif scale == 'remove_mean':
data[date + var + str(lvl)] -= np.mean(data[date + var +
str(lvl)])
for date, var, lvl in product(self.dates, self.grad_vars,
range(1,
len(heights) - 2)):
if scale == 'custom':
data[date + var + 'G' + str(lvl)] /= self.scale[var]
elif scale == 'maxmin':
min_pblh = min(data[date + 'PBLH'])
max_pblh = max(data[date + 'PBLH'])
X = data[date + var + 'G' + str(lvl)]
data[date + var + 'G' + str(lvl)] = MMS(
X, feature_range=(min_pblh, max_pblh))
elif scale == 'remove_mean':
data[date + var + str(lvl)] -= np.mean(data[date + var +
str(lvl)])
#print('scaled : '+var+' on '+date)
self.data = data
def plot_graph(graph=nx.tutte_graph(), layout=nx.kamada_kawai_layout):
"""
Make plotly visualization of networkx graph.
node_size -> betweeness centrality
node_color -> closeness centrality
"""
b_cents = nx.betweenness_centrality(graph)
c_cents = nx.closeness_centrality(graph)
d_cents = graph.degree()
edge_x = []
edge_y = []
pos = layout(graph)
for edge in graph.edges:
x0, y0 = pos[edge[0]]
x1, y1 = pos[edge[1]]
edge_x.append(x0)
edge_x.append(x1)
edge_x.append(None)
edge_y.append(y0)
edge_y.append(y1)
edge_y.append(None)
edge_trace = go.Scatter(
x=edge_x, y=edge_y,
line=dict(width=0.5, color='#888'),
hoverinfo='none',
mode='lines')
node_x = []
node_y = []
node_texts = []
node_degrees = []
node_closenesses = []
node_betweenesses = []
for node in graph.nodes():
x, y = pos[node]
node_x.append(x)
node_y.append(y)
node_degree = d_cents[node]
node_closeness = c_cents[node]
node_betweeness = b_cents[node]
node_degrees.append(node_degree)
node_closenesses.append(node_closeness)
node_betweenesses.append(node_betweeness)
node_text = str(node) + "<br>Degree: " + str(node_degree)
node_texts.append(node_text)
node_trace = go.Scatter(
x=node_x, y=node_y,
mode='markers',
hoverinfo='text',
marker=dict(
opacity=1,
showscale=True,
colorscale='Jet',
reversescale=False,
color=[],
size=[],
colorbar=dict(
thickness=15,
title='Closeness',
xanchor='left',
titleside='right'
),
line=dict(
width=1,
color='Black'
),
)
)
node_degrees = np.array(node_degrees)
node_closenesses = np.array(node_closenesses)
node_betweenesses = np.array(node_betweenesses)
size_scaler = MMS(feature_range=(7.5, 17.5))
node_betweenesses = size_scaler.fit_transform(node_betweenesses.reshape(-1,1)).ravel()
node_trace.marker.color = node_closenesses
node_trace.marker.size = node_betweenesses
node_trace.text = node_texts
fig = go.Figure(
data=[
edge_trace,
node_trace
],
layout=go.Layout(
autosize=True,
titlefont_size=16,
showlegend=False,
hovermode='closest',
margin=dict(b=20,l=5,r=5,t=40),
xaxis=dict(showgrid=False, zeroline=False, showticklabels=False),
yaxis=dict(showgrid=False, zeroline=False, showticklabels=False))
)
iplot(fig)
input_dim=1,
activation='relu',
kernel_initializer=weight_init,
bias_initializer=bias_init))
model.add(
Dense(10,
activation='relu',
kernel_initializer=weight_init,
bias_initializer=bias_init))
model.add(Dense(1, kernel_initializer=weight_init, bias_initializer=bias_init))
model.compile(optimizer=optimizers.RMSprop(0.01), loss='mse')
x = np.linspace(-2 * np.pi, 2 * np.pi, 1000).reshape(-1, 1)
y = np.sin(x)
mms = MMS()
x_train = mms.fit_transform(x)
y_train = mms.fit_transform(y)
#tensorboard = TensorBoard(log_dir='log')
#model.fit(x, y, batch_size=10, epochs=100, shuffle=True, callbacks=[tensorboard])
#model.fit(x_train, y_train, batch_size=10, epochs=500, shuffle=True)
#print('\n'.join(['%s:%s' % item for item in model.__dict__.items()]))
#model.fit(x[500:], y[500:], batch_size=10, epochs=100, shuffle=True)
#model.train_on_batch(x[500:], y[500:])
#model.train_on_batch(x[:500], y[:500])
for step in range(100):
indexes = np.random.randint(1000, size=32)
loss = model.train_on_batch(x[indexes], y[indexes])
#loss = model.train_on_batch(x, y)
print("loss: {} after {} step".format(loss, step + 1))
