def run_premade_map():
# Set up RL
data = Data(ROWS, COLS, load_data=False, epsilon=0.25)
gen = Generator(ROWS, COLS, resolution, size)
vis = Plotter(data, cell_edge, ROWS, COLS)
data.load_racetrack(directory='Sensitivity_Analysis')
data.get_start_line()
data.get_finish_line()
vis.visualize_racetrack()
# Finish setting up RL
env = Environment(data, gen, ROWS, COLS)
mcc = Monte_Carlo_Control(data, ROWS, COLS)
agent = Agent()
print("Saving data")
# Save arrays
data.save_racetrack(directory='Sensitivity_Analysis_2')
data.save_Q_vals(directory='Sensitivity_Analysis_2')
data.save_C_vals(directory='Sensitivity_Analysis_2')
data.save_rewards(directory='Sensitivity_Analysis_2')
data.save_pi(directory='Sensitivity_Analysis_2')
# Train agent
train_agent(mcc,
env,
agent,
data,
plot=True,
image_name='sensitivity_epsilon=0.25')
get_gif('epsilon=0.25', data, vis)
def make_full_report(list_output_results):
list_ultimate_points = []
list_yield_points = []
list_stiffness = []
# Creating subsequent force displacement curves
for i in range(0, len(list_output_results)):
plots = Plotter(list_output_results[i], "plot" + str(i))
plots.plot_xy_data(r"Rotation $\theta$ ($deg$)", r"Torque ($Nm$)")
# storing the ultimates (maximum) and offset yields
list_ultimate_points.append(list_output_results[i].ultimate_point.y)
list_yield_points.append(list_output_results[i].yield_point.y)
list_stiffness.append(list_output_results[i].stiffness)
mean_ultimate = np.mean(np.asarray(list_ultimate_points))
mean_yield = np.mean(np.asarray(list_yield_points))
std_ultimate = np.std(np.asarray(list_ultimate_points))
std_yield = np.std(np.asarray(list_yield_points))
print("mean of ultimates = ", mean_ultimate)
print("mean of yields = ", mean_yield)
return "done"
def run_tests(times: int = 1):
mea = MultiInstanceEAlgorithm(set_specimen=True)
mea.execute_algorithm(times=times)
best, worst, avg, std = mea.get_ratings()
best_len, worst_len, avg_len, std_len = mea.get_ratings_lengths()
case_names = []
map_names = []
y_data = []
iteretion = 0
for name in mea.cases_names:
case_names.append(case_naming(name))
for name in mea.maps_names:
map_names.append(map_naming(name))
for name1 in map_names:
Plotter().barplot_threeway(best[iteretion * 7:iteretion * 7 + 7],
avg[iteretion * 7:iteretion * 7 + 7],
worst[iteretion * 7:iteretion * 7 + 7],
case_names, "Rating", name1,
["Best", "Avg", "Worst"])
# Plotter().scatter_plot(best[iteretion*7:iteretion*7+7],
# best_len[iteretion*7:iteretion*7+7],
# "Ratings", "Genome lengths", name1)
y_data.append(best[iteretion * 7:iteretion * 7 + 7])
print(iteretion, "baprlot values",
best[iteretion * 7:iteretion * 7 + 7])
iteretion += 1
return None
def test_map_making():
data = Data(ROWS, COLS, load_data=False)
gen = Generator(ROWS, COLS, resolution, size)
vis = Plotter(data, cell_edge, ROWS, COLS)
data.racetrack = gen.generate_map()
vis.visualize_racetrack()
def test_dynamic_plotting(self):
plotter = Plotter()
max = 3000
for i in range(max):
plotter.add_values([("loss", (max-i)/max), ("evaluation score", i/max/2), ("second score", 0.3)])
plotter.plot("DynamicTestPlot").savefig("DynamicTestPlot")
self.assertTrue(os.path.exists("DynamicTestPlot.png"))
def _plot(self, series, labels, colors, title, lines=None, today=None,
stacked=False, loc=0, lines_y='bottom', today_y='bottom',
**kwargs):
p = Plotter(fill=stacked, **kwargs)
p.plots(zip(labels, series), stacked=stacked, colors=colors)
self._lines(p, series, lines, lines_y, today, today_y)
plt.legend(loc=loc)
plt.title('%s, stacked' % title if stacked else title)
def __init__(self,
hyperparameters,
should_log=False,
eval_interval=-1,
display_moves=False):
self.hparams = hyperparameters
self.random_seed = round(time.time())
self.should_log = should_log
self.display_moves = display_moves
self.eval_interval = eval_interval
self.random_util = RandomUtil(self.random_seed)
self.base_path = 'results/GA-' + self.random_util.generate_random_string(
6) + '/'
self.plotter = Plotter(self.base_path)
self.episode_count = 0
def __init__(self,
train_data,
test_data,
split_size,
split_type,
data_type,
classifier,
threads,
iter=-1,
clean_stuff=None):
self.data_handler = MultiClassDataHandler(train_data, test_data,
split_size, split_type, iter,
clean_stuff)
self.classifier = classifier
self.threads = threads
self.data_type = data_type
self.plotter = Plotter(style="ggplot")
self.decisions = []
self.infos = []
def run_random_map(plot=False,
directory='Random_Map_Data',
image_name='random_reward_graph',
epsilon=0.1):
# Set up RL
data = Data(ROWS, COLS, load_data=False, epsilon=epsilon)
gen = Generator(ROWS, COLS, resolution, size)
vis = Plotter(data, cell_edge, ROWS, COLS)
# TODO:: Must be able to plot map without needing space bar
# Display maps until we get one we like
while True:
data.racetrack = gen.generate_map()
data.get_start_line()
data.get_finish_line()
vis.visualize_racetrack()
response = input("Is this map suitable for our test? (y/n): ")
if response == 'y':
break
# Finish setting up RL
env = Environment(data, gen, ROWS, COLS)
mcc = Monte_Carlo_Control(data, ROWS, COLS)
agent = Agent()
print("Saving data")
# Save arrays
data.save_racetrack(directory=directory)
data.save_Q_vals(directory=directory)
data.save_C_vals(directory=directory)
data.save_rewards(directory=directory)
data.save_pi(directory=directory)
# Train agent
train_agent(mcc, env, agent, data, plot=plot, image_name=image_name)
if plot:
get_gif(image_name, data, vis)
def __init__(self,
train_data,
test_data,
positive_classes,
split_size,
split_type,
data_type,
classifier,
threads,
iteration=-1,
clean_stuff=None):
self.data_handler = DataHandler(train_data, test_data,
positive_classes, split_size,
split_type, iteration, clean_stuff)
self.classifier = classifier
self.threads = threads
self.iteration = iteration
self.data_type = data_type
self.positive_classes = positive_classes
self.plotter = Plotter(style="ggplot")
self.decisions = []
self.infos = []
def run_test_percentage(times: int = 1):
mea = MultiInstanceEAlgorithm(set_specimen=True)
mea.execute_precengage_values(times)
best, worst, avg, std = mea.get_ratings()
# best_len, worst_len, avg_len, std_len = mea.get_ratings_lengths()
case_names = []
map_names = []
# y_data = []
x_data = [10 * x for x in range(5, 15)]
print(len(best) / 11)
iteretion = 0
# print("best len", len(best))
for name in mea.cases_names:
case_names.append(case_naming(name))
for name in mea.maps_names:
map_names.append(map_naming(name))
for name1 in map_names:
print("XXX")
# Plotter().barplot_threeway(best[iteretion * 7:iteretion * 7 + 7],
# avg[iteretion * 7:iteretion * 7 + 7],
# worst[iteretion * 7:iteretion * 7 + 7],
# case_names, "Rating", name1, ["Best", "Avg", "Worst"])
# Plotter().scatter_plot(best[iteretion*7:iteretion*7+7],
# best_len[iteretion*7:iteretion*7+7],
# "Ratings", "Genome lengths", name1)
Plotter().multiline_plot(
x_data,
best[11 * iteretion:11 * iteretion + 10],
best[10 * iteretion + 10:10 * iteretion + 20],
best[10 * iteretion + 20:10 * iteretion + 30],
x_axis_name="Default parameter pecentage",
y_axis_name="Rating",
plot_title=name1,
legend=["alpha", "mutation_probability", "parent_group_size"])
# y_data.append(best[iteretion * 7:iteretion * 7 + 7])
# print(iteretion, "baprlot values", best[iteretion * 7:iteretion * 7 + 7])
iteretion += 1
from datamodel import Model
from servercomm import URL
from parameterui import ViewController
from excelprinter import ExcelPrinter
from plotting import Plotter
from optparse import OptionParser
parser = OptionParser()
parser.add_option("-p", "--plot",
action="store_true", dest="do_plot", default=False,
help="don't print status messages to stdout")
#def set_plot():
# do_plot = True
if __name__ == "__main__":
(options, args) = parser.parse_args()
print(options)
printer = 0
if (options.do_plot):
printer = Plotter()
else:
printer = ExcelPrinter('data.xlsx')
m = Model(URL, printer)
c = ViewController(m)
Plot("dzres_vs_pt_Sigma", ytitle="#sigma(#delta z_{0}) [cm]", ymin=0.0009, ymax=0.1, **_common),
Plot("ptres_vs_pt_Sigma", ytitle="#sigma(#delta p_{t}/p_{t})", ymin=0.003, ymax=2.2, **_common),
],
legendDy=-0.02, legendDh=-0.01
)
plotter = Plotter([
"DQMData/Run 1/Tracking/Run summary/Track",
"DQMData/Tracking/Track",
"DQMData/Run 1/RecoTrackV/Run summary/Track",
"DQMData/RecoTrackV/Track",
],[
_effandfake1,
_effandfake2,
_dupandfake1,
_dupandfake2,
_effvspos,
_dedx,
_chargemisid,
_hitsAndPt,
_ntracks,
_tuning,
_pulls,
_resolutionsEta,
_resolutionsPt,
])
import collections
_iterModuleMap = collections.OrderedDict([
("initialStepPreSplitting", ["initialStepSeedLayersPreSplitting",
"initialStepSeedsPreSplitting",
"initialStepTrackCandidatesPreSplitting",
import numpy as np from plotting import Plotter a=11E-9 L = 1000 # Length of wet region P_cap = 2*72E-3/a x = np.linspace(0,L,100) print(x) sigma_0 = P_cap*(x/L) r_tip = plt = Plotter(subplot=(2, 1),sharey = True) plt.add_plot(x, sigma_0, marker='r-') plt.show_figure()
def create_plotters_from_config(file_path):
user_specs = yaml_loader(file_path)
# user_specs = OrderedDict(yaml_loader(file_path))
# user_specs = OrderedDict(user_specs)
# 2. Grab schema file
schemas = None
if capnp_schemas_location in user_specs[source_key]:
schemas = user_specs[source_key][capnp_schemas_location]
else:
print 'no schemas location specified'
# 3. Grab source from which to read data (.stk or live transport)
# Checks to see what source is specified (For now cannot be both)
if user_specs[source_key].has_key(stk_file_source):
stk = user_specs[source_key][stk_file_source]
reader = DataReaderFromFile(schemas, stk)
elif user_specs[source_key][kb_tranport_settings_key][
kb_transport_type_key]:
transport_type = user_specs[source_key][kb_tranport_settings_key][
kb_transport_type_key]
hosts = user_specs[source_key][kb_tranport_settings_key][
kb_transport_hosts_key]
kb_name = user_specs[source_key][kb_tranport_settings_key][kb_name_key]
queue_lenght = None
thread_hertz = None
read_threads = None
transport_settings = user_specs[source_key][kb_tranport_settings_key]
#sub_dict = dict(sub_dict)
if transport_settings.has_key(queue_lenght_key):
queue_lenght = transport_settings[queue_lenght_key]
if transport_settings.has_key(read_threads_key):
read_threads = transport_settings[read_threads_key]
if transport_settings.has_key(thread_hertz_key):
thread_hertz = transport_settings[thread_hertz_key]
creator = KnowledgeBaseCreator(kb_name, transport_type, hosts,
queue_lenght, read_threads,
thread_hertz)
kb = creator.get_knowledge_base()
reader = DataReaderFromKB(schemas, kb)
else:
print(
'You have not specified a data source to read from or it is incomplete!'
)
sys.exit()
# TODO: maybe merge types into one?
####################################### Plot Data ###############################################
# 1. Create list of KRs for which the user would like to be plotted.
# Separate lists for Any and Frame type data
if frames_key in user_specs[subkeys_key]:
frame_KRs = user_specs[subkeys_key][frames_key] # list
else:
frame_KRs = {}
if any_type_key in user_specs[subkeys_key]:
any_KRs = user_specs[subkeys_key][any_type_key] # list
else:
any_KRs = {}
has_other_key = False
plot_dict = {}
for key in user_specs[subkeys_key]:
plot_3d = False
points = 0
if not (key == frames_key) and not (key == any_type_key):
has_other_key = True
value = user_specs[subkeys_key][key]
if not (value == None):
sub_plot_list = []
for subkey, value in value.items():
if subkey.startswith(subplot_start_key):
sub_plot_list.append(value.items())
elif subkey == plot_3d_key:
plot_3d = value
elif subkey == points_per_plot_key:
points = value
plot_dict[key] = Plotter(reader,
key,
subkeys=sub_plot_list,
points_per_plot=points,
plot_to_3d=plot_3d)
else:
plot_dict[key] = Plotter(reader, key)
if len(any_KRs) == 0 and len(frame_KRs) == 0 and not (has_other_key):
print('You have not specified any knowledge records!')
sys.exit()
# 2. Frame type data plotting
for key, value in frame_KRs.items():
sub_plot_list = []
plot_3d = False
points = 0
reference_frames = ['geo',
'p1_base_stabilized'] # keep the default values
try:
if plot_3d_key in value.keys():
plot_3d = value[plot_3d_key]
if points_per_plot_key in value.keys():
points = value[points_per_plot_key]
if reference_frame_key in value.keys():
reference_frames = value[reference_frame_key]
# just to find if 'plot_' substring is in a value string
s = [s for s in value.keys() if subplot_start_key in s]
if s:
for subkey in value.keys():
if subplot_start_key in subkey:
sub_plot_list.append(frame_KRs[key][subkey].items())
except:
pass
# if it is empty set to None, so when passing to plotter it doesn't appear as an existing array
if not sub_plot_list:
print sub_plot_list
sub_plot_list = None
plot_dict[key] = Plotter(reader,
data_reader_interface.frames_prefix + '.' +
key,
plot_to_3d=plot_3d,
points_per_plot=points,
subkeys=sub_plot_list,
frames_of_choice=reference_frames)
# 3. Any type data plotting
#TODO: Allow for handling of just specifying key and plotting subkeys automatically
for key, value in any_KRs.items():
sub_plot_list = []
plot_3d = False
points = 0
try:
if plot_3d_key in value.keys():
plot_3d = value[plot_3d_key]
if points_per_plot_key in value.keys():
points = value[points_per_plot_key]
# just to find if 'plot_' substring is in a value string
s = [s for s in value.keys() if subplot_start_key in s]
if s:
for subkey, subvalue in value.items():
if subplot_start_key in subkey:
try:
sub_plot_list.append(
sorted(any_KRs[key][subkey].items()))
except:
sub_plot_list.append(subvalue)
except:
pass
if not sub_plot_list:
sub_plot_list = None
plot_dict[key] = Plotter(reader,
key,
plot_to_3d=plot_3d,
points_per_plot=points,
subkeys=sub_plot_list)
return plot_dict
import yahoo
from plotting import Plotter
import market
import argparse
LOCAL_DATASOURCE = "yahoo.db3"
TEST_SYMBOL = "ENI.MI"
if __name__ == '__main__':
## plotting ##
source = yahoo.LocalSource(LOCAL_DATASOURCE)
symbol = market.Symbol(source, TEST_SYMBOL, None, None, matplotlib=True)
p = Plotter('Simple')
p.draw_simple(symbol)
p.run()
p = Plotter('Candlestick')
p.draw_candlestick(symbol)
p.run()
p = Plotter('Simple with volume')
p.draw_simple_with_volume(symbol)
p.run()
p = Plotter('Simple with volume and OBV')
p.draw_simple_with_volume_obv(symbol)
p.run()
def __init__(self):
self.experiment_name = self.__class__.__name__
self.__plotter__ = Plotter()
self.last_plot = None
self.path = self.config.TIC_TAC_TOE_DIR + "/experiments/artifacts/%s/" % self.experiment_name
# bus_loc = (40,80)
# bus_ori = 170
lresolution = 10
presolution = 1
# initialize obstacles
obstacles = Obs.random_obstacle_generate(dim_range, start, end, N_obs)
# initialize search space
Space = Xspace(dim_range, obstacles)
Car = lidar_car(car_loc, car_ori)
car_lidar_data = Car.lidar_check(Space, lresolution, presolution)
# plot part
handle_plot = Plotter(Space)
# get current axes
ax = plt.gca()
# add search space to axes
ax = handle_plot.plot_2D(ax, obstacles, start, end)
# add agents to space
ax = handle_plot.plot_2D_vehicle(ax, Car)
ax = handle_plot.plot_2D_vehicle(ax, Bus)
ax.axis('equal')
ax.grid()
plt.xlim([0, dim_range[0]])
plt.ylim([0, dim_range[1]])
plt.xlabel('x (m)')
plt.ylabel('y (m)')
else:
a_type_dict = {}
if not (a_type_dict or f_type_dict):
print('You have not specified any knowledge records!')
sys.exit()
######################## Frame type data parsing ##########################
frame_KRs = user_specs['knowledge_record_subkeys']['frame_types']
for key, value in frame_KRs.items():
sub_plot_list = []
print key, value
# print value.keys()
if not value:
plt2 = Plotter(reader, '.gams.frames.' + key)
continue
if 'reference_frame' in value.keys():
reference_frames = frame_KRs[key]['reference_frame']
for subkey in value.keys():
if 'plot' in subkey:
sub_plot_list.append(frame_KRs[key][subkey].items())
plt7 = Plotter(reader,
'.gams.frames.' + key,
frames_of_choice=reference_frames,
subkeys=sub_plot_list,
points_per_plot=10)
else:
plt4 = Plotter(reader,
'.gams.frames' + key,
frames_of_choice=['geo', 'p1_base_footprint'])
def sensitivity_analysis():
# Sensitivity analysis settings for epsilon=0.1
directory = 'Sensitivity_Analysis'
image_name = 'epsilon=0.1'
epsilon = 0.1
# Set up RL
data = Data(ROWS, COLS, load_data=False, epsilon=epsilon)
gen = Generator(ROWS, COLS, resolution, size)
vis = Plotter(data, cell_edge, ROWS, COLS)
# Display maps until we get one we like
while True:
data.racetrack = gen.generate_map()
data.get_start_line()
data.get_finish_line()
vis.visualize_racetrack()
response = input("Is this map suitable for our test? (y/n): ")
if response == 'y':
break
# Finish setting up RL
env = Environment(data, gen, ROWS, COLS)
mcc = Monte_Carlo_Control(data, ROWS, COLS)
agent = Agent()
print("Saving data")
# Save arrays
data.save_racetrack(directory=directory)
data.save_Q_vals(directory=directory)
data.save_C_vals(directory=directory)
data.save_rewards(directory=directory)
data.save_pi(directory=directory)
# Train agent
train_agent(mcc, env, agent, data, plot=True, image_name=image_name)
# Sensitivity analysis settings for epsilon=0.01
image_name = 'epsilon=0.01'
epsilon = 0.01
gif_name = 'epsilon=0.01'
# Set up RL
data2 = Data(ROWS, COLS, load_data=False, epsilon=epsilon)
vis = Plotter(data2, cell_edge, ROWS, COLS)
# Get old map
data2.racetrack = data.racetrack
data2.get_start_line()
data2.get_finish_line()
# Finish setting up RL
env = Environment(data2, gen, ROWS, COLS)
mcc = Monte_Carlo_Control(data2, ROWS, COLS)
print("Saving data")
# Save arrays
data.save_racetrack(directory=directory)
data.save_Q_vals(directory=directory)
data.save_C_vals(directory=directory)
data.save_rewards(directory=directory)
data.save_pi(directory=directory)
# Train agent
train_agent(mcc, env, agent, data2, plot=True, image_name=image_name)
get_gif(gif_name, data2, vis)
class GA_Agent:
hparams = {}
random_seed = 0
eval_interval = -1
display_moves = False
should_log = False
random_util = RandomUtil(0)
plotter = Plotter('none')
base_path = 'none'
episode_count = 0
def __init__(self,
hyperparameters,
should_log=False,
eval_interval=-1,
display_moves=False):
self.hparams = hyperparameters
self.random_seed = round(time.time())
self.should_log = should_log
self.display_moves = display_moves
self.eval_interval = eval_interval
self.random_util = RandomUtil(self.random_seed)
self.base_path = 'results/GA-' + self.random_util.generate_random_string(
6) + '/'
self.plotter = Plotter(self.base_path)
self.episode_count = 0
def __evaluate_ga(self,
blue_coeffs,
red_coeffs,
eval_py_env,
display_moves=False,
no_tests=10):
if display_moves:
episode_path = 'game-drawn/episode-' + str(self.episode_count)
if not os.path.exists(self.base_path + episode_path):
os.makedirs(self.base_path + episode_path)
time_step = eval_py_env.reset()
picture_count = 0
while not time_step.is_last():
observation = time_step.observation
self.plotter.plot_state(
observation,
episode_path + '/' + str(picture_count) + '.jpeg')
picture_count += 1
time_step = eval_py_env.step()
picture_count = 0
self.episode_count += 1
total_return = 0.0
for _ in range(no_tests):
time_step = eval_py_env.reset()
episode_return = 0.0
while not time_step.is_last():
#print(time_step.observation.numpy())
time_step = eval_py_env.step()
episode_return += time_step.reward
total_return += episode_return
return total_return / no_tests
def __plot_coefs(self, single_organism_coefs, file_name):
x = 0.0
x_values = []
function_values = []
def evaluate_function(x):
value = 0.0
x_pow = 1
for coef in single_organism_coefs:
value += coef * x_pow
x_pow *= x
return value
step = 0.03
while x < 10:
function_values.append(evaluate_function(x))
x_values.append(x)
x += step
self.plotter.plot_simple_values(x=x_values,
y=function_values,
directory=file_name)
def train(self, eval_game_params):
hparams = self.hparams
max_degree = hparams['max_parameter_degree']
no_blues = hparams['no_blue_organisms']
no_reds = hparams['no_red_organisms']
no_parameters = hparams['no_parameters']
coef_count = np.power(max_degree + 1, no_parameters)
self.ga_util = GaUtil(self.random_util, coef_count)
blue_coeffs = self.random_util.get_random_matrix(
no_blues, coef_count, [-1000, 1000])
red_coeffs = self.random_util.get_random_matrix(
no_reds, coef_count, [-1000, 1000])
returns = []
# Train the genetic algorithm
no_random = hparams['no_random_start'] * 1.0
random_step = (hparams['no_random_final'] -
no_random) / hparams['no_random_anneal_time']
mutation_factor_range = np.array(
hparams['mutation_factor_range_start'])
mutation_factor_range_final = np.array(
hparams['mutation_factor_range_final'])
mutation_factor_range_step = (mutation_factor_range_final - mutation_factor_range)\
/ hparams['mutation_factor_range_anneal_time']
for generation_number in range(hparams['no_generations']):
prev_blue_organisms = []
prev_red_organisms = []
if generation_number > 0:
prev_blue_organisms = env.dead_blue_organisms
prev_red_organisms = env.dead_red_organisms
env = GameEnv(blue_coeffs, red_coeffs, max_degree,
hparams['food_count'], hparams['board_size'])
#utils.validate_py_environment(py_environment, episodes=5)
# Evaluate the GA
if self.eval_interval > 0 and (generation_number +
1) % self.eval_interval == 0:
eval_blue_coeffs = self.ga_util.get_coeffs_from_best(
prev_blue_organisms, eval_game_params['no_blue_organisms'],
eval_game_params['no_blue_organisms'], 0, [0, 0])
eval_red_coeffs = self.ga_util.get_coeffs_from_best(
prev_red_organisms, eval_game_params['no_red_organisms'],
eval_game_params['no_red_organisms'], 0, [0, 0])
eval_py_env = GameEnv(eval_blue_coeffs, eval_red_coeffs,
max_degree,
eval_game_params['food_count'],
eval_game_params['board_size'])
avg_return = self.__evaluate_ga(blue_coeffs, red_coeffs,
eval_py_env,
self.display_moves)
returns.append(avg_return)
if self.should_log:
print(avg_return)
# Play the game
time_step = env.reset()
while not time_step.is_last():
#print(time_step.observation.numpy())
time_step = env.step()
# Pick best genomes for the next generation
blue_organisms = env.dead_blue_organisms
blue_coeffs = self.ga_util.get_coeffs_from_best(
blue_organisms, no_blues, hparams['no_best'], round(no_random),
mutation_factor_range)
red_organisms = env.dead_red_organisms
red_coeffs = self.ga_util.get_coeffs_from_best(
red_organisms, no_reds, hparams['no_best'], round(no_random),
mutation_factor_range)
# Reduce the number of random organisms and mutation_factor_range
no_random += random_step
mutation_factor_range += mutation_factor_range_step
self.plotter.plot_simple_values(y=list(returns),
directory='score.jpeg')
for single_organism_coefs in red_coeffs:
self.__plot_coefs(single_organism_coefs, 'red-coeffs.jpeg')
for single_organism_coefs in blue_coeffs:
self.__plot_coefs(single_organism_coefs, 'blue-coeffs.jpeg')
hparams['random_seed'] = self.random_seed
self.plotter.dump_to_json(hparams, 'hparams.json')
def create_plotters_from_config(file_path):
user_specs = yaml_loader(file_path)
# user_specs = OrderedDict(yaml_loader(file_path))
# user_specs = OrderedDict(user_specs)
# 2. Grab schema file
if user_specs[source_key][capnp_schemas_location]:
schemas = user_specs[source_key][capnp_schemas_location]
else:
sys.exit('You have not specified schema files')
# 3. Grab source from which to read data (.stk or live transport)
# Checks to see what source is specified (For now cannot be both)
if user_specs[source_key].has_key(stk_file_source):
stk = user_specs[source_key][stk_file_source]
reader = DataReaderFromFile(schemas, stk)
elif user_specs[source_key][kb_tranport_settings_key][
kb_transport_type_key]:
transport_type = user_specs[source_key][kb_tranport_settings_key][
kb_transport_type_key]
hosts = user_specs[source_key][kb_tranport_settings_key][
kb_transport_hosts_key]
kb_name = user_specs[source_key][kb_tranport_settings_key][kb_name_key]
queue_lenght = None
thread_hertz = None
read_threads = None
transport_settings = user_specs[source_key][kb_tranport_settings_key]
#sub_dict = dict(sub_dict)
if transport_settings.has_key(queue_lenght_key):
queue_lenght = transport_settings[queue_lenght_key]
if transport_settings.has_key(read_threads_key):
read_threads = transport_settings[read_threads_key]
if transport_settings.has_key(thread_hertz_key):
thread_hertz = transport_settings[thread_hertz_key]
creator = KnowledgeBaseCreator(kb_name, transport_type, hosts,
queue_lenght, read_threads,
thread_hertz)
kb = creator.get_knowledge_base()
reader = DataReaderFromKB(schemas, kb)
else:
print(
'You have not specified a data source to read from or it is incomplete!'
)
sys.exit()
# TODO: maybe merge types into one?
####################################### Plot Data ###############################################
# 1. Create list of KRs for which the user would like to be plotted.
# Separate lists for Any and Frame type data
if user_specs[subkeys_key][frames_key]:
frame_KRs = user_specs[subkeys_key][frames_key] # list
else:
frame_KRs = {}
if user_specs[subkeys_key][any_type_key]:
any_KRs = user_specs[subkeys_key][any_type_key] # list
else:
any_KRs = {}
has_other_key = False
plot_dict = {}
for key in user_specs[subkeys_key]:
if not (key == frames_key) and not (key == any_type_key):
has_other_key = True
value = user_specs[subkeys_key][key]
if not (value == None):
sub_plot_list = []
for subkey in value.keys():
if 'plot' in subkey:
sub_plot_list.append(value[subkey].items())
plot_dict[key] = Plotter(reader, key, subkeys=sub_plot_list)
else:
plot_dict[key] = Plotter(reader, key)
if len(any_KRs) == 0 and len(frame_KRs) == 0 and not (has_other_key):
print('You have not specified any knowledge records!')
sys.exit()
# 2. Frame type data plotting
for key, value in frame_KRs.items():
sub_plot_list = []
flagger = [0, 0, 0, 0]
try:
if '3D' in value.keys() and value['3D']:
flagger[0] += 1
if 'points_per_plot' in value.keys():
points = value['points_per_plot']
flagger[1] += 1
if 'reference_frame' in value.keys():
reference_frames = value['reference_frame']
flagger[2] += 1
# just to find if 'plot_' substring is in a value string
s = [s for s in value.keys() if 'plot_' in s]
if s:
# if 'plot_' in value.keys():
flagger[3] += 1
for subkey in value.keys():
if 'plot_' in subkey:
sub_plot_list.append(frame_KRs[key][subkey].items())
except:
pass
if flagger == [0, 0, 0, 0]:
plot_dict[key] = Plotter(
reader, data_reader_interface.frames_prefix + '.' + key)
continue
if flagger == [1, 0, 0, 0]:
plot_dict[key] = Plotter(reader,
data_reader_interface.frames_prefix +
'.' + key,
plot_to_3d=True)
continue
if flagger == [0, 1, 0, 0]:
plot_dict[key] = Plotter(reader,
data_reader_interface.frames_prefix +
'.' + key,
points_per_plot=points)
continue
if flagger == [0, 0, 1, 0]:
plot_dict[key] = Plotter(reader,
data_reader_interface.frames_prefix +
'.' + key,
frames_of_choice=reference_frames)
continue
if flagger == [0, 0, 0, 1]:
plot_dict[key] = Plotter(reader,
data_reader_interface.frames_prefix +
'.' + key,
subkeys=sub_plot_list)
continue
if flagger == [1, 1, 0, 0]:
plot_dict[key] = Plotter(reader,
data_reader_interface.frames_prefix +
'.' + key,
points_per_plot=points,
plot_to_3d=True)
continue
if flagger == [0, 1, 1, 0]:
plot_dict[key] = Plotter(reader,
data_reader_interface.frames_prefix +
'.' + key,
points_per_plot=points,
frames_of_choice=reference_frames)
continue
if flagger == [0, 0, 1, 1]:
plot_dict[key] = Plotter(reader,
data_reader_interface.frames_prefix +
'.' + key,
frames_of_choice=reference_frames,
subkeys=sub_plot_list)
continue
if flagger == [1, 0, 1, 0]:
plot_dict[key] = Plotter(reader,
data_reader_interface.frames_prefix +
'.' + key,
plot_to_3d=True,
frames_of_choice=reference_frames)
continue
if flagger == [1, 0, 0, 1]:
plot_dict[key] = Plotter(reader,
data_reader_interface.frames_prefix +
'.' + key,
plot_to_3d=True,
subkeys=sub_plot_list)
continue
if flagger == [0, 1, 0, 1]:
plot_dict[key] = Plotter(reader,
data_reader_interface.frames_prefix +
'.' + key,
points_per_plot=points,
subkeys=sub_plot_list)
continue
if flagger == [1, 1, 1, 0]:
plot_dict[key] = Plotter(reader,
data_reader_interface.frames_prefix +
'.' + key,
points_per_plot=points,
plot_to_3d=True,
frames_of_choice=reference_frames)
continue
if flagger == [0, 1, 1, 1]:
plot_dict[key] = Plotter(reader,
data_reader_interface.frames_prefix +
'.' + key,
points_per_plot=points,
subkeys=sub_plot_list,
frames_of_choice=reference_frames)
continue
if flagger == [1, 0, 1, 1]:
plot_dict[key] = Plotter(reader,
data_reader_interface.frames_prefix +
'.' + key,
plot_to_3d=True,
subkeys=sub_plot_list,
frames_of_choice=reference_frames)
continue
if flagger == [1, 1, 0, 1]:
plot_dict[key] = Plotter(reader,
data_reader_interface.frames_prefix +
'.' + key,
plot_to_3d=True,
subkeys=sub_plot_list,
points_per_plot=points)
continue
if flagger == [1, 1, 1, 1]:
plot_dict[key] = Plotter(reader,
data_reader_interface.frames_prefix +
'.' + key,
points_per_plot=points,
plot_to_3d=True,
frames_of_choice=reference_frames,
subkeys=sub_plot_list)
continue
# 3. Any type data plotting
#TODO: Allow for handling of just specifying key and plotting subkeys automatically
for key, value in any_KRs.items():
sub_plot_list = []
flagger = [0, 0, 0]
try:
if '3D' in value.keys() and value['3D']:
flagger[0] += 1
if 'points_per_plot' in value.keys():
points = value['points_per_plot']
flagger[1] += 1
# just to find if 'plot_' substring is in a value string
s = [s for s in value.keys() if 'plot_' in s]
if s:
flagger[2] += 1
for subkey, subvalue in value.items():
if 'plot_' in subkey:
try:
sub_plot_list.append(
sorted(any_KRs[key][subkey].items()))
except:
sub_plot_list.append(subvalue)
except:
pass
if flagger == [0, 0, 1]:
plot_dict[key] = Plotter(reader, key, subkeys=sub_plot_list)
continue
if flagger == [0, 1, 1]:
plot_dict[key] = Plotter(reader,
key,
subkeys=sub_plot_list,
points_per_plot=points)
continue
if flagger == [1, 0, 1]:
plot_dict[key] = Plotter(reader,
key,
plot_to_3d=True,
subkeys=sub_plot_list)
continue
if flagger == [1, 1, 1]:
plot_dict[key] = Plotter(reader,
key,
plot_to_3d=True,
subkeys=sub_plot_list,
points_per_plot=points)
continue
return plot_dict
_summaryPlots = [
_summary,
_summaryN,
]
_summaryPlotsHp = [
_summaryHp,
_summaryNHp,
]
_packedCandidatePlots = [
_packedCandidateFlow,
_packedCandidateParam1,
_packedCandidateParam2,
_packedCandidateMomVert,
_packedCandidateHits,
]
plotter = Plotter()
def _appendTrackingPlots(lastDirName, name, algoPlots, onlyForPileup=False):
# to keep backward compatibility, this set of plots has empty name
plotter.append(
name, _trackingFolders(lastDirName),
TrackingPlotFolder(*algoPlots,
onlyForPileup=onlyForPileup,
purpose=PlotPurpose.TrackingIteration))
summaryName = ""
if name != "":
summaryName += name + "_"
summaryName += "summary"
plotter.append(
summaryName, _trackingFolders(lastDirName),
def main(model=None, output_dir=None, n_iter=20, n_texts=2000, init_tok2vec=None, plot=False):
if output_dir is not None:
output_dir = Path(output_dir)
if not output_dir.exists():
output_dir.mkdir()
if model is not None:
nlp = spacy.load(model) # load existing spaCy model
print("Loaded model '%s'" % model)
else:
nlp = spacy.blank("en") # create blank Language class
print("Created blank 'en' model")
# add the text classifier to the pipeline if it doesn't exist
# nlp.create_pipe works for built-ins that are registered with spaCy
if "textcat" not in nlp.pipe_names:
textcat = nlp.create_pipe(
"textcat", config={"exclusive_classes": True, "architecture": "simple_cnn"}
)
nlp.add_pipe(textcat, last=True)
# otherwise, get it, so we can add labels to it
else:
textcat = nlp.get_pipe("textcat")
# add label to text classifier
textcat.add_label("POSITIVE")
textcat.add_label("NEGATIVE")
# load the IMDB dataset
print("Loading IMDB data...")
(train_texts, train_cats), (dev_texts, dev_cats) = load_data()
train_texts = train_texts[:n_texts]
train_cats = train_cats[:n_texts]
print(
"Using {} examples ({} training, {} evaluation)".format(
n_texts, len(train_texts), len(dev_texts)
)
)
train_data = list(zip(train_texts, [{"cats": cats} for cats in train_cats]))
dev_data = list(zip(dev_texts, [{"cats": cats} for cats in dev_cats]))
# get names of other pipes to disable them during training
pipe_exceptions = ["textcat", "trf_wordpiecer", "trf_tok2vec"]
other_pipes = [pipe for pipe in nlp.pipe_names if pipe not in pipe_exceptions]
with nlp.disable_pipes(*other_pipes): # only train textcat
optimizer = nlp.begin_training()
if init_tok2vec is not None:
with init_tok2vec.open("rb") as file_:
textcat.model.tok2vec.from_bytes(file_.read())
print("Training the model...")
print("{:^5}\t{:^5}\t{:^5}\t{:^5}".format("LOSS", "P", "R", "F"))
# Set-up the plotter:
if plot:
plotter = Plotter(
title='IMDB Text categorisation training',
ylabels=["Train-loss", "Dev-loss", "Precision", "Recall", "F-score"],
iterations=n_iter,
figsize=(8, 10))
batch_sizes = compounding(4.0, 32.0, 1.001)
for i in range(n_iter):
losses = {}
# batch up the examples using spaCy's minibatch
random.shuffle(train_data)
batches = minibatch(train_data, size=batch_sizes)
for batch in batches:
texts, annotations = zip(*batch)
nlp.update(texts, annotations, sgd=optimizer, drop=0.2, losses=losses)
with textcat.model.use_params(optimizer.averages):
# evaluate on the dev data split off in load_data()
scores = evaluate(nlp.tokenizer, textcat, dev_texts, dev_cats)
dev_losses = {}
random.shuffle(dev_data)
batches = minibatch(dev_data, size=batch_sizes)
for batch in batches:
texts, annotations = zip(*batch)
nlp.update(texts, annotations, sgd=None, losses=dev_losses)
print(
"{0:.3f}\t{1:.3f}\t{2:.3f}\t{3:.3f}".format( # print a simple table
losses["textcat"],
dev_losses["textcat"],
scores["textcat_p"],
scores["textcat_r"],
scores["textcat_f"],
)
)
# Update the plot:
if plot:
plotter.update(y=[
losses["textcat"],
dev_losses["textcat"],
scores["textcat_p"],
scores["textcat_r"],
scores["textcat_f"],
])
# test the trained model
test_text = "This movie sucked"
doc = nlp(test_text)
print(test_text, doc.cats)
if output_dir is not None:
with nlp.use_params(optimizer.averages):
nlp.to_disk(output_dir)
print("Saved model to", output_dir)
# test the saved model
print("Loading from", output_dir)
nlp2 = spacy.load(output_dir)
doc2 = nlp2(test_text)
print(test_text, doc2.cats)
# Keep showing the plot until the plotting window is closed.
if plot:
plotter.keep()
import matplotlib.pyplot as plt
import seaborn as sns
sys.path.insert(1, "../tools")
from analysis import Analyzer
from plotting import Plotter
from training import Trainer
if __name__ == "__main__":
sns.set()
plot_dir = "plots"
plot_file = os.path.join(plot_dir, "rdf.png")
if not os.path.exists(plot_dir):
os.mkdir(plot_dir)
anl = Analyzer()
plter = Plotter()
r_cut = 6.0
r, rdf = anl.calculate_rdf("trajs/training.traj", r_max=r_cut)
rdf[np.nonzero(rdf)] /= max(rdf)
cutoff = plter.polynomial(r, r_cut, gamma=5.0)
plt.plot(r, rdf, label="Radial distribution function")
plt.plot(r, cutoff, label="Polynomial cutoff, gamma=5.0")
plt.legend()
plt.title("Copper radial distribution function")
plt.xlabel("Radial distance [Angstrom]")
plt.ylabel("Radial distribution function (normalized to 1)")
plt.savefig(plot_file)
assert b1 in range(1, 9)
b2 = int(
input(
'Choose action:\n1)Visualize Bigrams\n2)Visualize Terms\n3)Run model grid param search\n4)Run all '
'models kfold\n> '))
assert b2 in range(1, 5)
b3 = int(
input(
'Perform clean file parsing or use pickles?:\n1)Use pickles\n2)Perform clean file parsing\n> '
))
assert b3 in range(1, 3)
plotter = Plotter(threads=b1, ignore_pickles=True, strict=(b3 == 2))
while b2 != 5:
if b2 == 1:
plotter.visualize_bigrams(40)
#print(plotter.prep.best_bigram_scores)
elif b2 == 2:
plotter.visualize_descriptive_terms(200)
#print(plotter.prep.selected_words)
elif b2 == 3:
start = time.time()
