class Scheduler(multiprocessing.Process):
def __init__(self, event, queue): # Args: 1. Event object and 2. Queue
self.event = event # Event object with which Scheduler is poked by oscillator every min
self.dataprovider = Datamanager() # The interface to configurations through sqlite/mysql/python script/xml/json/nagios model
self.queue = queue # The Queue
super(Scheduler, self).__init__()
def run(self):
try:
while True:
self.event.wait() # Wait till oscillator pokes me
for task in self.dataprovider.probs_to_run_now(): # Read the configs and check what to do now
self.queue.put(task) # If anything is to be monitored now, add to Queue
except KeyboardInterrupt:
pass
import sys
from parameters import Parameters
from nsoptcaller import NsoptCaller
from gaussfit import Gaussfit
from datamanager import Datamanager
#from memory_profiler import profile
import numpy as np
import pickle
param = Parameters(interval, samples, center_lecs=lec_center)
nsopt = NsoptCaller()
gauss = Gaussfit()
dm = Datamanager(echo=False)
gauss.save_fig = save_fig
gauss.save_path = save_path
if dm.num_matches(training_tags) <= 0 or dm.num_matches(validation_tags) <= 0:
sys.exit('Check your tags. No matched found in database.')
else:
#Set empty arrays with zeros for training and validation data
train_obs = np.array([0])
train_energy = np.array([0])
train_lecs = np.array(np.zeros(LEC_LENGTH))
#Read database data with the specified tags
for row in dm.read(training_tags):
train_obs = np.vstack((train_obs, row.observable))
train_energy = np.vstack((train_energy, row.energy))
train_lecs = np.vstack((train_lecs, row.LECs))
# Clean up initialized zeros
import numpy as np
from matplotlib import pyplot as plt
from matplotlib import style
from matplotlib2tikz import save as tikz_save
from os import listdir
from os.path import isfile, join
from shutil import move, copyfile
from matplotlib import pyplot as plt
from matplotlib import style
# The kernels we have trained fro
kernels = ['RBF', 'Matern52']
gauss = Gaussfit()
dm = Datamanager(echo=False)
def calculate_valid(val_lecs):
"""Wrapper function to measure time with memory profiler."""
mod_obs, mod_var = gauss.calculate_valid(val_lecs)
return mod_obs, mod_var
LEC_LENGTH = 16
RBF_error = []
Matern52_error = []
samples = range(250, 3001, 250)
# Kernel loop
for kernel in kernels:
LEC_LENGTH = 16
#GPy parameters
kernel = 'Matern52'
lengthscale = 1.
multi_dim = True #Use multi-dimensional (16)
generate_tags = ['sgt' + str(energy[0]) + '-' + str(energy[1]), 'validation' + str(samples),
'single_lec_E_curve50/50' + str(lec_sampling) + str(lec_index)]
# Set up necessary classes)
param = Parameters(interval, samples, center_lecs=lec_center)
nsopt = NsoptCaller()
gauss = Gaussfit()
dm = Datamanager(echo=False)
if dm.num_matches(generate_tags) > 0:
answer = raw_input('Matching data for your tags already exist, add new data as well? (y for yes): ')
if answer == 'y':
continue_generate = True
def get_observable(energies, lecs):
"""Wrapper function to measure time with memory profiler."""
return nsopt.get_nsopt_observable(energies,LECM=lecs)
param.nbr_of_samples = samples
if lec_sampling == 'lhs':
print('lhs')
lecs = param.create_lhs_lecs(energy_interval=energy)
energies = lecs[:,-1]
from gaussfit import Gaussfit
from datamanager import Datamanager
from memory_profiler import profile
import numpy as np
import pickle
def train_gp_model(train_obs, train_lecs):
"""Populate and optimize GP model."""
gauss.populate_gp_model(train_obs, train_lecs, rescale=rescale_data)
gauss.optimize()
save_params = True
gauss = Gaussfit()
dm = Datamanager(echo=False)
kernels = ['RBF', 'Matern52']
samples = range(250, 3001, 250)
for kernel in kernels:
for training_number in xrange(250, 3001, 250):
print("Training kernel {} training_number {}".format(
kernel, training_number))
training_tags = [
'sgt1-150', 'training' + str(training_number), 'E_curvelhs'
]
params_save_path = '/net/data1/ml2017/gpyparams/E_curve_gpy_log/' + str(
kernel) + '_E_curve_training_' + str(
training_number) + '_lhs_sgt1_150_multidim_log.pickle'
def __init__(self, event, queue): # Args: 1. Event object and 2. Queue
self.event = event # Event object with which Scheduler is poked by oscillator every min
self.dataprovider = Datamanager() # The interface to configurations through sqlite/mysql/python script/xml/json/nagios model
self.queue = queue # The Queue
super(Scheduler, self).__init__()
#------------------------------------------------------------------------------------------
# Actual code
# Training dimension, do we want to train on energy or only lecs?
# This is for generation
parameter_dim = LEC_LENGTH
energy_as_param = False
if energy[1] != energy[0]:
parameter_dim += 1
energy_as_param = True
# Set up necessary classes)
param = Parameters(interval, samples, center_lecs=lec_center)
nsopt = NsoptCaller()
gauss = Gaussfit()
dm = Datamanager(echo=False)
gauss.save_fig = save_fig
gauss.save_path = save_fig_path
@profile
def get_observable(energies, lecs):
"""Wrapper function to measure time with memory profiler."""
return nsopt.get_nsopt_observable(energies,LECM=lecs)
@profile
def train_gp_model(train_obs, train_lecs):
"""Populate and optimize GP model."""
gauss.populate_gp_model(train_obs, train_lecs, rescale=rescale_data)
gauss.optimize()
@profile
def main():
"""Controls the program flow determined by user input"""
# Parses arguments supplied by the user.
parser = argparse.ArgumentParser()
parser.add_argument("-v",
"--verbosity",
action="store_true",
help="Show data output")
# Add a group of arguments that can't coexist
group = parser.add_mutually_exclusive_group()
group.add_argument("-f",
"--function",
action="store",
help="Use specified function for GP")
group.add_argument("-n",
"--nsopt",
action="store_true",
help="Use nsopt calculation for GP")
#Add tag handling. Supports multiple arg seperated by space.
group.add_argument("-l",
"--load",
nargs='+',
action='append',
help="Use stored data for GP")
args = parser.parse_args()
# Initialize project class objects
nsopt = NsoptCaller()
gauss = Gaussfit()
dm = Datamanager()
param = Parameters(
0.1,
100,
)
LECS = param.create_lhs_lecs()
# Suggestion: X should have a more intuitive name (energy) //Martin
# Read input arguments to get input vector
X = nsopt.read_ini(args)
# Load all data with the right tags and convert to array
if args.load:
print args.load
data_list = dm.read(args.load[0])
#TODO(Martin) Format data from data objects
observables = []
energies = []
LECs = []
for row in data_list:
observables.append(row.observable)
energies.append(row.energy)
LECs.append(row.LECs)
#data = np.asarray(data)
return None #Returning None for the moment, when LECs are in database this will work
# Do we want to genereate new nsopt values or use specified function.
if args.nsopt:
Y = nsopt.get_nsopt_observable()
Y = np.trim_zeros(Y)
Y = Y.reshape(1, len(Y))
else:
Y = np.sin(X) + np.random.randn(20, 1) * 0.03
# Set up GP-processes and plot output after optimization.
gauss.set_gp_kernel()
if args.load:
#gauss.populate_gp_model(data) #TODO(rikard, Martin) Check data format.
pass
else:
gauss.populate_gp_model(X, Y)
gauss.optimize()
gauss.plot()