def newDay(self, weekly, monthly, yearly, current_day):
results = [0, 0, 0]
week = cl.Classify()
history = week.trainNetwork(self.epochs, weekly[:, :7:], weekly[:,
7:8:])
results[0] = week.testNetwork(weekly[:, 1:8:])
if current_day % 15 == 1 or current_day == 0:
month = rg.Regression()
history = month.trainNetwork(self.epochs, monthly[:, :30:],
monthly[:, 30:31:])
self.results_monthly = month.testNetwork(monthly[:, 1:31:])
self.results_monthly = self.results_monthly.flatten()
results[1] = self.results_monthly
if current_day % (365 / 2) == 1 or current_day == 0:
year = rg.Regression()
history = year.trainNetwork(self.epochs, yearly[:, :365:],
yearly[:, 365:366:])
self.results_yearly = year.testNetwork(yearly[:, 1:366:])
self.results_yearly = self.results_yearly.flatten()
results[2] = self.results_yearly
self.results = np.average(results, axis=0)
return self.results
def test_get_mean(self):
test_arr = [1, 2, 3, 4]
# regression = Regression
result = regression.Regression().get_mean(self, test_arr)
expect = 2.5
self.assertEqual(expect, result)
def update(self):
self.logger.info("Labeled points ({0}): ".format(self.num_label))
self.logger.info(self.x_label)
self.logger.info(self.y_label)
if self.model == "regression":
# 1. train the model
self.logger.info("Start training ... ")
reg = regression.Regression(self.x, self.weight, self.ind_label,
self.y_label, self.logger)
reg.train()
# 2. log info
best_new_ind = reg.log(self.x, self.y, self.num_label,
self.ind_label)
elif self.model == "LP":
lp = classification.LP(self.weight, self.ind_label,
self.ind_unlabel, self.y_label, self.logger)
hu = lp.propagate()
best_new_ind = self.ind_unlabel[np.argmax(hu)]
lp.log(hu, best_new_ind, self.x, self.y, self.best_ind)
# 3. update the graph
self.ind_label = np.append(self.ind_label, best_new_ind)
self.y_label = self.y[self.ind_label]
self.x_label = self.x[self.ind_label]
self.ind_unlabel = np.setdiff1d(np.arange(self.n), self.ind_label)
self.x_unlabel = self.x[self.ind_unlabel]
self.num_label = self.num_label + 1
def testActivations(trainSize,
batchSize,
testSize,
nLayers,
nNodes,
nEpochs,
a=0.9,
b=1.6):
# function to approximate
function = lambda s: 1.0 / s**12 - 1.0 / s**6
# approximate on [a,b]
a = 0.9
b = 1.6
regress = regression.Regression(function, int(1e6), int(1e4), int(1e3), 1,
1)
regress.generateData(a, b)
# test different activations
activations = [
tf.nn.relu, tf.nn.relu6, tf.nn.elu, tf.nn.sigmoid, tf.nn.tanh
]
counter = 0
for act in activations:
regress.constructNetwork(nLayers,
nNodes,
activation=act,
wInit='trunc_normal',
bInit='trunc_normal')
regress.train(nEpochs)
counter += 1
def f(S, D):
zs = np.zeros(shape=(D.size, S.size))
print(zs)
for i in range(len(S)):
for j in range(len(D)):
s = S[i][j]
d = D[i][j]
regression = reg.Regression(dim=3)
trajectory = Trajectory(nsamples=s,
integration_time=30,
n_timesteps=10,
density=d)
regression.initialize_samples(1, trajectory=trajectory)
regression.run_model()
print("{}, {}".format(s, d))
e = regression.compute_error(errors=['ge_av_raw'])
print(e)
zs[i][j] = e[0]
return zs
def LennardJonesExample(trainSize=int(1e5),
batchSize=50,
testSize=int(1e4),
nLayers=1,
nNodes=10,
nEpochs=int(1e5),
units='metal',
activation=tf.nn.sigmoid):
"""
Train to reproduce shifted L-J potential to
verify implementation of network and backpropagation in the MD code
This is a 1-dimensional example
"""
function, a, cutoff, functionDerivative = setUpLJPotential('metal',
derivative=True)
regress = regression.Regression(function,
trainSize,
batchSize,
testSize,
1,
1,
functionDerivative=functionDerivative)
regress.generateData(a, cutoff, 'twoBody')
regress.constructNetwork(nLayers, nNodes, activation=activation, \
wInit='normal', bInit='normal')
regress.train(nEpochs)
def _regress1(self, rawdict, side, key, specific_options, tbest):
# rawdict is like {'CO2': x1, 'N2O': x2, ...}
# where x1 and x2 is like{'left':(t,y,Istartstop), 'right':(t,y,Istartstop)}
# and Istartstop is [(Istart, Istop), (Istart, Istop)...]
# (Istartstop are indexes determining
# which part of the data has been used for the regression for one side)
options = self.options.get_options(side, key, specific_options)
t, y = remove_zeros(*rawdict[key][side][:2])
if len(t) == 0:
reg = None
elif 'slope' in options: # If slope was manually set
#(EEB) This manually creates the 'reg' without calling the regression2 function.
#It sets SE's and MSE's to zero, and start/stop to the min and max
#(EEB) items passed to Regression:(self, intercept, slope, se_intercept, se_slope, mse, start, stop)
reg = regression.Regression(
t[0], options['slope'], 0, 0, 0, t[0], t[-1])
#If user manually gave start and stop times, call regression2 via regress_within
elif 'start' in options and 'stop' in options:
reg = regression.regress_within(t, y, options['start'], options['stop'])
#If the regression should use CO2's start and stop points, call regression2 via regress_within
elif key != 'CO2' and options['co2_guides'] and tbest is not None:
reg = regression.regress_within(t, y, *tbest)
#In all other cases, call regression2 via find_best_regression, which will find the best start and stop times.
else:
reg = regression.find_best_regression(
t, y, options['interval'], options['crit'])
if key == 'CO2' and reg is not None:
tbest = reg.start, reg.stop
if reg is not None:
reg.Iswitch = rawdict[key][side][2] # EEB adds switching times to reg, like [(27, 41), (67, 81), (107, 121), (147, 161), (182, 181)]
#Quality check of N2O regressions
if key=='N2O':
try:
reg.signal_range = reg.max_y - reg.min_y
except:
reg.signal_range = 0
try:
reg.curve_factor = abs(reg.slope)/reg.signal_range
except:
reg.curve_factor = -1
if reg.signal_range > 0.14 or (side=='left' and reg.curve_factor>0.0081) or (side=='right' and reg.curve_factor>0.0066) or (reg.pval>0.0001 and reg.pval<0.001 and reg.signal_range >.003) or reg.curve_factor == -1 or reg.signal_range == -1:
reg.quality_check='Outliers likely'
elif reg.min_y < 0.31 or reg.min_y > 0.34 or (reg.slope < 0 and reg.max_y > 0.34):
if reg.pval > 0.001 and reg.signal_range < 0.003:
reg.quality_check='Out of range - possibly zero slope'
elif reg.slope < 0:
reg.quality_check='Out of range and negative'
else:
reg.quality_check='Out of range'
elif reg.pval > 0.001:
if reg.signal_range > 0.003:
reg.quality_check='Fails p-test for other reason'
else:
reg.quality_check='Probably zero slope'
else:
reg.quality_check=''
return reg, tbest
def build(X_train, y_train, X_train_scaled, y_train_scaled, models):
"""Build and return regression models"""
regression = \
reg.Regression(X_train, y_train, X_train_scaled, y_train_scaled)
regressors = []
for model in models:
regressors.append(getattr(regression, model)())
return regressors
def LennardJonesNeighbours(trainSize, batchSize, testSize, nLayers, nNodes, nEpochs, \
neighbours, outputs=1):
function, a, b = setUpLJPotential('LJ')
regress = regression.Regression(function, trainSize, batchSize, testSize,
neighbours, outputs)
regress.generateData(a, b, 'neighbourTwoBody')
regress.constructNetwork(nLayers, nNodes, activation=tf.nn.sigmoid, \
wInit='normal', bInit='normal')
regress.train(nEpochs)
def LennardJonesNeighboursForce(trainSize, batchSize, testSize, nLayers, nNodes, nEpochs, \
neighbours, outputs=4, a=0.8, b=2.5):
function, a, b = setUpLJPotential('LJ', shifted=True)
functionDerivative = lambda t: 12.0 / t**13 - 6.0 / t**7
inputs = neighbours * 4
regress = regression.Regression(function, trainSize, batchSize, testSize, inputs, outputs, \
functionDerivative)
regress.generateData(a, b, 'neighbourTwoBody')
regress.constructNetwork(nLayers, nNodes, activation=tf.nn.sigmoid, \
wInit='normal', bInit='normal')
regress.train(nEpochs)
def lammpsTrainingSi(nLayers=2, nNodes=35, nEpochs=int(1e5), symmFuncType='G5', \
lammpsDir='', outputs=1, activation=tf.nn.sigmoid, \
useFunction=False, forces=False, batch=5, Behler=True, \
klargerj=False, tags=False, learningRate=0.001, RMSEtol=1e-10, nTypes=1,
normalize=False, shiftMean=False, standardize=False,
wInit='uniform', bInit='zeros', constantValue=0.1, stdDev=0.1):
"""
Use neighbour data and energies from lammps with sw-potential
as input and output training data respectively
"""
lammpsDir = "../LAMMPS_test/Silicon/Data/TrainingData/" + lammpsDir + '/'
# get energies from sw lammps
if useFunction:
function, _, _ = getStillingerWeber()
#function, _, _ = getTwoBodySW()
else:
function = None
# these are sampled from lammps
trainSize = batchSize = testSize = inputs = low = high = 0
regress = regression.Regression(function,
trainSize,
batchSize,
testSize,
inputs,
outputs,
learningRate=learningRate,
RMSEtol=RMSEtol)
regress.generateData(low,
high,
'lammpsSi',
symmFuncType=symmFuncType,
dataFolder=lammpsDir,
forces=forces,
batch=batch,
Behler=Behler,
klargerj=klargerj,
tags=tags,
nTypes=nTypes,
normalize=normalize,
shiftMean=shiftMean,
standardize=standardize)
regress.constructNetwork(nLayers,
nNodes,
activation=activation,
wInit=wInit,
bInit=bInit,
constantValue=constantValue,
stdDev=stdDev)
regress.train(nEpochs)
def run_models(samples):
trajectory = Trajectory(nsamples=samples,
integration_time=30,
n_timesteps=15,
pattern=Pattern.grid)
regression = reg.Regression(dim=3)
regression.initialize_samples(ndrifters=samples, trajectory=trajectory)
regression.run_model()
rbf_e = regression.compute_error(errors=["ge_av_raw"])
return rbf_e
def lammpsTrainingSiO2(nLayers=2,
nNodes=10,
nEpochs=int(1e5),
symmFuncType='G5',
activation=tf.nn.sigmoid,
lammpsDir='4Atoms/T1e3N1e4',
forces=False,
batch=5,
learningRate=0.001,
RMSEtol=0.003,
outputs=1,
atomType=0,
nTypes=2,
nAtoms=10):
"""
Use neighbour data and energies from lammps with vashista-potential
as input and output training data respectively
"""
lammpsDir = "../LAMMPS_test/Quartz/Data/TrainingData/" + lammpsDir + '/'
# these are sampled from lammps
function = None
trainSize = batchSize = testSize = inputs = low = high = 0
regress = regression.Regression(function,
trainSize,
batchSize,
testSize,
inputs,
outputs,
learningRate=learningRate,
RMSEtol=RMSEtol)
regress.generateData(low,
high,
'lammpsSiO2',
symmFuncType=symmFuncType,
dataFolder=lammpsDir,
forces=forces,
batch=batch,
atomType=atomType,
nTypes=nTypes,
nAtoms=nAtoms)
regress.constructNetwork(nLayers,
nNodes,
activation=activation,
wInit='xavier',
bInit='constant')
regress.train(nEpochs)
def LennardJonesSymmetryFunctions(trainSize, batchSize, testSize, nLayers, nNodes, nEpochs, \
neighbours, numberOfSymmFunc, symmFuncType, outputs=1, \
units='metal', shifted=False, varyingNeigh=True):
function, a, b = setUpLJPotential('metal', shifted=shifted)
regress = regression.Regression(function, trainSize, batchSize, testSize,
numberOfSymmFunc, outputs)
regress.generateData(a,
b,
'twoBodySymmetry',
neighbours=neighbours,
numberOfSymmFunc=numberOfSymmFunc,
symmFuncType='G2',
varyingNeigh=varyingNeigh)
regress.constructNetwork(nLayers, nNodes, activation=tf.nn.sigmoid, \
wInit='normal', bInit='normal')
regress.train(nEpochs)
def regression(self):
self.Ytrain_mean = np.mean(self.Ytrain)
if self.threshold is not None:
c_threshold = self.threshold - self.Ytrain_mean
else:
c_threshold = None
Ytrain = utils.centre(self.Ytrain)
Ytest = utils.centre(self.Ytrain, self.Ytest)
regress = regression.Regression(Ytrain,
Ytest=Ytest,
Xtrain=self.Xtrain,
Xtest=self.Xtest,
kernel=self.kernel,
cent_threshold=c_threshold)
return regress
def run_models(samples):
trajectory = Trajectory(nsamples=samples,
integration_time=30,
n_timesteps=15,
pattern=Pattern.grid)
div_k = dfk.DivFreeK(3)
curl_k = cfk.CurlFreeK(3)
k = div_k + curl_k
regression = reg.Regression(dim=3)
regression.initialize_samples(ndrifters=samples, trajectory=trajectory)
regression.run_model(kernel=k)
cdk_e = regression.compute_error(errors=["ge_av_raw"])
return cdk_e
def optimization(self):
Ytrain = utils.centre(self.Ytrain)
run = regression.Regression(Xtest=self.Xtest,
Xtrain=self.Xtrain,
Ytrain=Ytrain,
add_noise=0.0,
kernel=self.kernel,
Ytest=None)
sd = run.post_s
p_mean = run.post_mean
# CENTRE XTRAIN AGAIN?
new_x, ind = self.acq_func.compute(self.Xtest,
self.Xtrain,
Ytrain,
sd,
p_mean,
plot=False)
new_obs = self.Ytest[ind]
if len(self.Xtrain) != 2:
if isinstance(new_x, float) == True:
self.Xtrain = np.vstack((self.Xtrain, new_x))
self.Xtest = np.delete(self.Xtest, ind, axis=0)
else:
self.Xtrain.append(new_x)
del self.Xtest[ind]
else:
self.Xtrain[0] = np.vstack((self.Xtrain[0], new_x[0]))
self.Xtrain[1] = list(self.Xtrain[1])
self.Xtrain[1].append(new_x[1])
self.Xtest[0] = np.delete(self.Xtest[0], ind, axis=0)
self.Xtest[1] = list(self.Xtest[1])
del self.Xtest[1][ind]
self.Ytrain = np.vstack((self.Ytrain, new_obs))
self.Ytest = np.delete(self.Ytest, ind, axis=0)
return new_x, new_obs
def main():
# Config
with open('config.json') as json_file:
params = json.load(json_file)
# Scrape web: Prices, fundamentals
crawlers = initialize_crawlers(params)
for crawler in crawlers.values():
crawler.build()
crawler.process()
# Data processing: Daily returns, factor exposures and outlier removal
if params['data_processing']['activate']:
processor = dataProcessing.DataProcessing(params)
processor.process()
# Regressions
if params['regression']['activate']:
reg = regression.Regression(params)
reg.process()
def StillingerWeberSymmetry(trainSize, batchSize, testSize, nLayers, nNodes, nEpochs, \
neighbours, symmFunctype, method,
outputs=1, varyingNeigh=True, dataFolder=''):
"""
Train neural network to simulate tetrahedral Si atoms
methodGenerate random input training data or use xyz-data from lammps
Output training data is calculated with my sw-potential, i.e.
energies not from lammps
method=threeBodySymmetry: random configs
method=lammps: configs from lammps, but not energies
"""
function, a, sigma = getStillingerWeber()
# set limits for input training data
low = 2.0 # checked with lammps Si simulation
# cutoff = sigma*a according to http://lammps.sandia.gov/doc/pair_sw.html
# subtract a small number to prevent division by zero
tol = 1e-14
high = sigma * a + sigma / np.log(tol)
print high
exit(1)
inputs = 0
# train
regress = regression.Regression(function, trainSize, batchSize, testSize,
inputs, outputs)
regress.generateData(low,
high,
method,
neighbours=neighbours,
symmFuncType='G4',
dataFolder=dataFolder,
varyingNeigh=varyingNeigh)
regress.constructNetwork(nLayers, nNodes, activation=tf.nn.sigmoid, \
wInit='normal', bInit='normal')
regress.train(nEpochs)
def performanceTest(maxEpochs, maxLayers, maxNodes):
# function to approximate
function = lambda s: 1.0 / s**12 - 1.0 / s**6
# approximate on [a,b]
a = 0.9
b = 1.6
regress = regression.Regression(function, int(1e6), int(1e4), int(1e3))
regress.generateData(a, b)
# finding optimal value
counter = 0
for layers in range(1, maxLayers + 1, 1):
for nodes in range(layers, maxNodes + 1, 1):
start = timer()
regress.constructNetwork(layers, nodes)
regress.train(maxEpochs)
end = timer()
timeElapsed = end - start
print "Layers: %2d, nodes: %2d, time = %10g" % (layers, nodes,
timeElapsed)
print
if counter == 0:
with open('Tests/timeElapsed', 'w') as outFile:
outStr = "Timing analysis"
outFile.write(outStr + '\n')
with open('Tests/timeElapsed', 'a') as outFile:
outStr = "Layers: %2d, nodes: %2d, time = %10g" % (
layers, nodes, timeElapsed)
outFile.write(outStr + '\n')
counter += 1
# Create indicator variable for North Amierca in both data and diff_data
indicator_name = "North America"
index_name = "ISO_Code"
countries_in_north_america = [
"BHS", "BRB", "BLZ", "CAN", "CRI", "DOM", "SLV", "GTM", "HTI", "HND",
"JAM", "MEX", "NIC", "PAN", "TTO", "USA"
]
for key in data_dict:
data = data_dict[key]
create_indicator_variable(data=data,
indicator_name=indicator_name,
index_name=index_name,
target_index_list=countries_in_north_america)
# prepare regression variables
X_names = ["EFW", "Log RGDP Per Capita Lag", "North America"]
y_name = ["Log RGDP Per Capita"]
# save instance of regression class
reg = regression.Regression()
for key in data_dict:
# call OLS method
data = data_dict[key]
reg.OLS(reg_name=key,
data=data.dropna(),
y_name=y_name,
beta_names=X_names)
print(key, reg.estimates, sep="\n")
print(reg.stats_DF)
print()
# set random seed to make results reproducible
rand_seed_num = 0
torch.manual_seed(rand_seed_num)
Nx = 1000
data_x = torch.rand(Nx, 1)
c1, c2 = 1.0, 1e2
p1, p2 = 1.0, 2.0
data_y = c1 * (data_x**p1) + c2 * (data_x**p2)
total_epoch = 200000
# without freeze integer
reg = regression.Regression(np.array([c1, c2]))
p1_init, p2_init = reg.get_params()
reg.train(data_x,
data_y,
is_freeze_integer=False,
total_epoch=total_epoch,
loss_params_file_name='loss_params_history_freeze_false.npy')
# with freeze integer
reg = regression.Regression(np.array([c1, c2]))
reg.set_params(np.array([p1_init, p2_init]))
reg.train(data_x,
data_y,
is_freeze_integer=True,
total_epoch=total_epoch,
loss_params_file_name='loss_params_history_freeze_true.npy')
import common as ml import regression as r ts = ml.TrainingSet(lambda x:x * 4, 30, -10, 10, 0) reg = r.Regression(ts) reg.general() reg.plot(0) ml.show_plot()
def gridSearchSi(maxLayers=3, minNodes=5, skipNodes=2, maxNodes=30, maxEpochs=1e5, symmFuncType='G5', \
lammpsDir='', outputs=1, activation=tf.nn.sigmoid, \
useFunction=False, forces=False, batch=5, Behler=True, \
klargerj=False, tags=False, learningRate=0.001, RMSEtol=1e-10, nTypes=1,
normalize=False, shiftMean=False, standardize=False,
wInit='uniform', bInit='zeros', constantValue=0.1, stdDev=0.1):
"""
Do a grid search to find a suitable NN architecture
"""
lammpsDir = "../LAMMPS_test/Silicon/Data/TrainingData/" + lammpsDir + '/'
function = None
# these are sampled from lammps
trainSize = batchSize = testSize = inputs = low = high = 0
regress = regression.Regression(function,
trainSize,
batchSize,
testSize,
inputs,
outputs,
learningRate=learningRate,
RMSEtol=RMSEtol)
regress.generateData(low,
high,
'lammpsSi',
symmFuncType=symmFuncType,
dataFolder=lammpsDir,
forces=forces,
batch=batch,
Behler=Behler,
klargerj=klargerj,
tags=tags,
nTypes=nTypes,
normalize=normalize,
shiftMean=shiftMean,
standardize=standardize)
# finding optimal value
counter = 0
for layers in xrange(1, maxLayers + 1):
for nodes in xrange(minNodes, maxNodes + 1, skipNodes):
regress.constructNetwork(layers,
nodes,
activation=activation,
wInit='uniform',
bInit='zeros')
testRMSE, epoch, timeElapsed = regress.train(maxEpochs)
print "Layers: %2d, nodes: %2d, RMSE = %g, Epoch = %d, time = %10g" % (
layers, nodes, testRMSE, epoch, timeElapsed)
print
if counter == 0:
with open('Tests/timeElapsed.txt', 'w') as outFile:
outStr = "Timing analysis"
outFile.write(outStr + '\n')
with open('Tests/timeElapsed.txt', 'a') as outFile:
outStr = "Layers: %2d, nodes: %2d, RMSE: %g, Epoch: %d, time = %10g" % (
layers, nodes, testRMSE, epoch, timeElapsed)
outFile.write(outStr + '\n')
counter += 1
if __name__ == "__main__":
s = np.arange(1, 181, 20)
d = np.arange(0.01, 2.26, 0.25)
S, D = np.meshgrid(s, d)
zs = np.zeros(shape=(len(S), len(D)))
for i in range(len(S)):
for j in range(len(D)):
si = S[i][j]
di = D[i][j]
regression = reg.Regression(dim=3)
trajectory = Trajectory(nsamples=si,
integration_time=30,
n_timesteps=10,
density=di)
regression.initialize_samples(1, trajectory=trajectory)
regression.run_model()
print("{}, {}".format(si, di))
e = regression.compute_error(errors=['ge_av_raw'])
print(e)
def main():
loader = sys.argv[1]
exec_file = sys.argv[2]
runner = regression.Regression(loader, exec_file)
runner.add_check(name=exec_file, check=lambda res: False)
runner.run_checks()
print("Loading: %s" % file2)
df = pd.read_csv(file2, sep=',', header=None)
input_data2 = df.as_matrix()
print("Loading: %s" % file3)
df = pd.read_csv(file3, sep=',', header=None)
input_data3 = df.as_matrix()
# get node position
tag_pos1 = [input_data1[0][0], input_data1[0][1]]
print(tag_pos1)
tag_pos2 = [input_data2[0][0], input_data2[0][1]]
print(tag_pos2)
tag_pos3 = [input_data3[0][0], input_data3[0][1]]
print(tag_pos3)
regressor = regr.Regression()
numit = 3333
# LS (LOS classification)
for a in range(3, 11):
print("%d anchors" % a)
err_vect = []
temp_vect1 = loc.localization_wls(input_data1, tag_pos1, a, regressor,
numit)
temp_vect2 = loc.localization_wls(input_data2, tag_pos2, a, regressor,
numit)
temp_vect3 = loc.localization_wls(input_data3, tag_pos3, a, regressor,
numit)
def work_flow():
"""The work flow of blending several TC OSW.
"""
load_configs.setup_logging()
logger = logging.getLogger(__name__)
# CONFIG
try:
CONFIG = load_configs.load_config()
except Exception as msg:
logger.exception(f'Exception occurred when loading confi: {msg}')
os.makedirs(CONFIG['logging']['dir'], exist_ok=True)
# read commandline arguments, first
full_cmd_arguments = sys.argv
# - further arguments
argument_list = full_cmd_arguments[1:]
try:
arguments, values = getopt.getopt(argument_list, '', gnuOptions)
except getopt.error as err:
# output error, and return with an error code
print(str(err))
sys.exit(2)
input_custom_period = False
input_custom_region = False
specify_basin = False
basin = None
do_match_smap = False
do_regression = False
reg_instructions = None
smogn_target = None
interval = None
do_simulate = False
do_classify = False
classify_instruction = None
tag = None
do_compare = False
draw_sfmr = False
max_windspd = None
force_align_smap = False
do_sfmr = False
sfmr_instructions = None
do_ibtracs = False
ibtracs_instructions = None
do_validation = False
do_check = False
do_sta_ibtracs = False
do_sta_era5_smap = False
do_smart_compare = False
do_merra2 = False
do_match_sfmr = False
do_combine = False
# evaluate given options
for current_argument, current_value in arguments:
if current_argument in ('-p', '--period'):
input_custom_period = True
period_parts = current_value.split(',')
if len(period_parts) != 2:
logger.error((f"""Inputted period is wrong: """
f"""need 2 parameters"""))
elif current_argument in ('-r', '--region'):
input_custom_region = True
region_parts = current_value.split(',')
if len(region_parts) != 4:
logger.error((f"""Inputted region is wrong: """
f"""need 4 parameters"""))
elif current_argument in ('-b', '--basin'):
specify_basin = True
basin_parts = current_value.split(',')
if len(basin_parts) != 1:
logger.error((f"""Inputted basin is wrong: """
f"""must 1 parameters"""))
basin = basin_parts[0]
elif current_argument in ('-e', '--match_smap'):
do_match_smap = True
elif current_argument in ('-g', '--reg'):
do_regression = True
reg_instructions = current_value.split(',')
elif current_argument in ('--smogn_target'):
smogn_target = current_value.split(',')[0]
elif current_argument in ('--interval'):
interval = current_value.split(',')[:2]
elif current_argument in ('--simulate'):
do_simulate = True
simulate_instructions = current_value.split(',')
elif current_argument in ('--classify'):
do_classify = True
classify_instructions = current_value.split(',')
elif current_argument in ('--tag'):
tag = current_value.split(',')[0]
elif current_argument in ('-c', '--compare'):
do_compare = True
compare_instructions = current_value.split(',')
elif current_argument in ('--draw_sfmr'):
head = current_value.split(',')[0]
if head == 'True':
draw_sfmr = True
elif head == 'False':
draw_sfmr = False
else:
logger.error('draw_sfmr must be "True" or "False"')
sys.exit(1)
elif current_argument in ('--max_windspd'):
head = current_value.split(',')[0]
max_windspd = float(head)
elif current_argument in ('--force_align_smap'):
head = current_value.split(',')[0]
if head == 'True':
force_align_smap = True
elif head == 'False':
force_align_smap = False
else:
logger.error('force_align_smap must be "True" or "False"')
sys.exit(1)
elif current_argument in ('-s', '--sfmr'):
do_sfmr = True
elif current_argument in ('-i', '--ibtracs'):
do_ibtracs = True
elif current_argument in ('-v', '--validate'):
do_validation = True
validate_instructions = current_value
elif current_argument in ('-k', '--check'):
do_check = True
elif current_argument in ('--sta_ibtracs'):
do_sta_ibtracs = True
elif current_argument in ('--sta_era5_smap'):
do_sta_era5_smap = True
sources = current_value.split(',')
elif current_argument in ('--smart_compare'):
do_smart_compare = True
elif current_argument in ('--merra2'):
do_merra2 = True
elif current_argument in ('--match_sfmr'):
do_match_sfmr = True
elif current_argument in ('--combine'):
do_combine = True
if not specify_basin:
logger.error('Must specify basin')
exit()
if input_custom_period:
# Period parts
# yyyy-mm-dd-HH-MM-SS
period = [
datetime.datetime.strptime(period_parts[0], '%Y-%m-%d-%H-%M-%S'),
datetime.datetime.strptime(period_parts[1], '%Y-%m-%d-%H-%M-%S')
]
else:
period = [
datetime.datetime(2015, 4, 1, 0, 0, 0),
datetime.datetime.now()
]
train_test_split_dt = datetime.datetime(2019, 1, 1, 0, 0, 0)
if input_custom_region:
# Area parts
custom_region = []
for part in region_parts:
custom_region.append(float(part))
else:
region = [-90, 90, 0, 360]
# Period
logger.info(f'Period: {period}')
# Region
logger.info(f'Region: {region}')
# MySQL Server root password
passwd = '399710'
# Download and read
try:
if do_combine:
combine_table.TableCombiner(CONFIG, period, region, basin, passwd)
if do_match_sfmr:
match_era5_sfmr.matchManager(CONFIG, period, region, basin, passwd,
False)
if do_merra2:
merra2.MERRA2Manager(CONFIG, period, False)
if do_smart_compare:
smart_compare.SmartComparer(CONFIG, period, basin, passwd)
if do_sta_era5_smap:
sta_era5_smap.Statisticer(CONFIG, period, basin, sources, passwd)
if do_sta_ibtracs:
sta_ibtracs.Statisticer(CONFIG, period, basin, passwd)
if do_check:
checker.Checker(CONFIG)
if do_validation:
validate.ValidationManager(CONFIG, period, basin,
validate_instructions)
if do_match_smap:
match_era5_smap.matchManager(CONFIG,
period,
region,
basin,
passwd,
False,
work=True)
if do_classify:
classify.Classifier(CONFIG, period, train_test_split_dt, region,
basin, passwd, False, classify_instructions,
smogn_target)
if do_simulate:
simulate.TCSimulator(CONFIG, period, region, basin, passwd, False,
simulate_instructions)
if do_regression:
# if tag is None:
# logger.error('No model tag')
# exit()
regression.Regression(CONFIG, period, train_test_split_dt, region,
basin, passwd, False, reg_instructions,
smogn_target, tag)
if do_compare:
# if ('smap_prediction' in compare_instructions
# and tag is None):
# logger.error('No model tag')
# exit()
compare_tc.TCComparer(CONFIG, period, region, basin, passwd, False,
compare_instructions, draw_sfmr, max_windspd,
force_align_smap)
# tag)
# ccmp_ = ccmp.CCMPManager(CONFIG, period, region, passwd,
# work_mode='fetch_and_compare')
# era5_ = era5.ERA5Manager(CONFIG, period, region, passwd,
# work=True, save_disk=False, 'scs',
# 'surface_all_vars')
# isd_ = isd.ISDManager(CONFIG, period, region, passwd,
# work_mode='fetch_and_read')
# grid_ = grid.GridManager(CONFIG, region, passwd, run=True)
# satel_scs_ = satel_scs.SCSSatelManager(CONFIG, period,
# region, passwd,
# save_disk=False,
# work=True)
# coverage_ = coverage.CoverageManager(CONFIG, period,
# region, passwd)
if do_ibtracs:
ibtracs_ = ibtracs.IBTrACSManager(CONFIG, period, region, basin,
passwd)
# cwind_ = cwind.CwindManager(CONFIG, period, region, passwd)
# stdmet_ = stdmet.StdmetManager(CONFIG, period, region, passwd)
if do_sfmr:
sfmr_ = sfmr.SfmrManager(CONFIG, period, region, passwd)
# satel_ = satel.SatelManager(CONFIG, period, region, passwd,
# save_disk=False)
# compare_ = compare_offshore.CompareCCMPWithInStu(
# CONFIG, period, region, passwd)
pass
except Exception as msg:
logger.exception('Exception occured when downloading and reading')
try:
# new_reg = reg_scs.NewReg(CONFIG, period, test_period,
# region, passwd, save_disk=True)
# ibtracs_ = ibtracs.IBTrACSManager(CONFIG, test_period,
# region, passwd)
# hwind_ = hwind.HWindManager(CONFIG, test_period, region, passwd)
# era5_ = era5.ERA5Manager(CONFIG, test_period, region, passwd,
# work=True, save_disk=False)
pass
except Exception as msg:
logger.exception('Exception occured when downloading and reading')
logger.info('SWFusion complete.')
import regression as r
import pandas as pd
data_path = "./health.xls"
data = pd.read_excel(data_path)
x = r.panda_to_numpy(
data,
'X2',
'X3',
'X4',
'X5',
)
y = data['X1']
model = r.Regression(y, x)
weights = model.train(epochs=2000, a=0.00000001, print_loss=False)
prediction = model.predict(50, 200, 5, 150)
print(prediction)
# Generate test values
X_test = np.vstack(np.linspace(0, 5, 100))
# Output function (plotted for comparison only)
true_f = np.sin(0.9 * X_test)
# Generate training inputs with noisy output values
# The posterior is generated using these values
X_train = np.vstack(5 * np.random.rand(10, 1))
train_noise = np.vstack(0.05 * np.random.randn(10, 1))
Y_train = np.sin(0.9 * X_train) + train_noise
# Set up model
kern = kernels.RBF(lengthscale=0.2, sig_var=1, noise_var=0.1)
m = regression.Regression(X_test,
X_train,
Y_train,
add_noise=0.1,
kernel=kern,
Ytest=true_f) # Optional: kernel, normalize
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
# Plot the prior and posterior (and random draws from both)
# Note how the posterior uncertainty is large for the regions of the input space where there
# are no training values. The posterior mean visibly deviates from the true mean in these
# regions. Close to the training points, the posterior mean is a good representation of the
# true output function.
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
m.plot_prior()
m.plot_posterior()
