def run_premade_map(): # Set up RL data = Data(ROWS, COLS, load_data=False, epsilon=0.25) gen = Generator(ROWS, COLS, resolution, size) vis = Plotter(data, cell_edge, ROWS, COLS) data.load_racetrack(directory='Sensitivity_Analysis') data.get_start_line() data.get_finish_line() vis.visualize_racetrack() # Finish setting up RL env = Environment(data, gen, ROWS, COLS) mcc = Monte_Carlo_Control(data, ROWS, COLS) agent = Agent() print("Saving data") # Save arrays data.save_racetrack(directory='Sensitivity_Analysis_2') data.save_Q_vals(directory='Sensitivity_Analysis_2') data.save_C_vals(directory='Sensitivity_Analysis_2') data.save_rewards(directory='Sensitivity_Analysis_2') data.save_pi(directory='Sensitivity_Analysis_2') # Train agent train_agent(mcc, env, agent, data, plot=True, image_name='sensitivity_epsilon=0.25') get_gif('epsilon=0.25', data, vis)
def make_full_report(list_output_results): list_ultimate_points = [] list_yield_points = [] list_stiffness = [] # Creating subsequent force displacement curves for i in range(0, len(list_output_results)): plots = Plotter(list_output_results[i], "plot" + str(i)) plots.plot_xy_data(r"Rotation $\theta$ ($deg$)", r"Torque ($Nm$)") # storing the ultimates (maximum) and offset yields list_ultimate_points.append(list_output_results[i].ultimate_point.y) list_yield_points.append(list_output_results[i].yield_point.y) list_stiffness.append(list_output_results[i].stiffness) mean_ultimate = np.mean(np.asarray(list_ultimate_points)) mean_yield = np.mean(np.asarray(list_yield_points)) std_ultimate = np.std(np.asarray(list_ultimate_points)) std_yield = np.std(np.asarray(list_yield_points)) print("mean of ultimates = ", mean_ultimate) print("mean of yields = ", mean_yield) return "done"
def run_tests(times: int = 1): mea = MultiInstanceEAlgorithm(set_specimen=True) mea.execute_algorithm(times=times) best, worst, avg, std = mea.get_ratings() best_len, worst_len, avg_len, std_len = mea.get_ratings_lengths() case_names = [] map_names = [] y_data = [] iteretion = 0 for name in mea.cases_names: case_names.append(case_naming(name)) for name in mea.maps_names: map_names.append(map_naming(name)) for name1 in map_names: Plotter().barplot_threeway(best[iteretion * 7:iteretion * 7 + 7], avg[iteretion * 7:iteretion * 7 + 7], worst[iteretion * 7:iteretion * 7 + 7], case_names, "Rating", name1, ["Best", "Avg", "Worst"]) # Plotter().scatter_plot(best[iteretion*7:iteretion*7+7], # best_len[iteretion*7:iteretion*7+7], # "Ratings", "Genome lengths", name1) y_data.append(best[iteretion * 7:iteretion * 7 + 7]) print(iteretion, "baprlot values", best[iteretion * 7:iteretion * 7 + 7]) iteretion += 1 return None
def test_map_making(): data = Data(ROWS, COLS, load_data=False) gen = Generator(ROWS, COLS, resolution, size) vis = Plotter(data, cell_edge, ROWS, COLS) data.racetrack = gen.generate_map() vis.visualize_racetrack()
def test_dynamic_plotting(self): plotter = Plotter() max = 3000 for i in range(max): plotter.add_values([("loss", (max-i)/max), ("evaluation score", i/max/2), ("second score", 0.3)]) plotter.plot("DynamicTestPlot").savefig("DynamicTestPlot") self.assertTrue(os.path.exists("DynamicTestPlot.png"))
def _plot(self, series, labels, colors, title, lines=None, today=None, stacked=False, loc=0, lines_y='bottom', today_y='bottom', **kwargs): p = Plotter(fill=stacked, **kwargs) p.plots(zip(labels, series), stacked=stacked, colors=colors) self._lines(p, series, lines, lines_y, today, today_y) plt.legend(loc=loc) plt.title('%s, stacked' % title if stacked else title)
def __init__(self, hyperparameters, should_log=False, eval_interval=-1, display_moves=False): self.hparams = hyperparameters self.random_seed = round(time.time()) self.should_log = should_log self.display_moves = display_moves self.eval_interval = eval_interval self.random_util = RandomUtil(self.random_seed) self.base_path = 'results/GA-' + self.random_util.generate_random_string( 6) + '/' self.plotter = Plotter(self.base_path) self.episode_count = 0
def __init__(self, train_data, test_data, split_size, split_type, data_type, classifier, threads, iter=-1, clean_stuff=None): self.data_handler = MultiClassDataHandler(train_data, test_data, split_size, split_type, iter, clean_stuff) self.classifier = classifier self.threads = threads self.data_type = data_type self.plotter = Plotter(style="ggplot") self.decisions = [] self.infos = []
def run_random_map(plot=False, directory='Random_Map_Data', image_name='random_reward_graph', epsilon=0.1): # Set up RL data = Data(ROWS, COLS, load_data=False, epsilon=epsilon) gen = Generator(ROWS, COLS, resolution, size) vis = Plotter(data, cell_edge, ROWS, COLS) # TODO:: Must be able to plot map without needing space bar # Display maps until we get one we like while True: data.racetrack = gen.generate_map() data.get_start_line() data.get_finish_line() vis.visualize_racetrack() response = input("Is this map suitable for our test? (y/n): ") if response == 'y': break # Finish setting up RL env = Environment(data, gen, ROWS, COLS) mcc = Monte_Carlo_Control(data, ROWS, COLS) agent = Agent() print("Saving data") # Save arrays data.save_racetrack(directory=directory) data.save_Q_vals(directory=directory) data.save_C_vals(directory=directory) data.save_rewards(directory=directory) data.save_pi(directory=directory) # Train agent train_agent(mcc, env, agent, data, plot=plot, image_name=image_name) if plot: get_gif(image_name, data, vis)
def __init__(self, train_data, test_data, positive_classes, split_size, split_type, data_type, classifier, threads, iteration=-1, clean_stuff=None): self.data_handler = DataHandler(train_data, test_data, positive_classes, split_size, split_type, iteration, clean_stuff) self.classifier = classifier self.threads = threads self.iteration = iteration self.data_type = data_type self.positive_classes = positive_classes self.plotter = Plotter(style="ggplot") self.decisions = [] self.infos = []
def run_test_percentage(times: int = 1): mea = MultiInstanceEAlgorithm(set_specimen=True) mea.execute_precengage_values(times) best, worst, avg, std = mea.get_ratings() # best_len, worst_len, avg_len, std_len = mea.get_ratings_lengths() case_names = [] map_names = [] # y_data = [] x_data = [10 * x for x in range(5, 15)] print(len(best) / 11) iteretion = 0 # print("best len", len(best)) for name in mea.cases_names: case_names.append(case_naming(name)) for name in mea.maps_names: map_names.append(map_naming(name)) for name1 in map_names: print("XXX") # Plotter().barplot_threeway(best[iteretion * 7:iteretion * 7 + 7], # avg[iteretion * 7:iteretion * 7 + 7], # worst[iteretion * 7:iteretion * 7 + 7], # case_names, "Rating", name1, ["Best", "Avg", "Worst"]) # Plotter().scatter_plot(best[iteretion*7:iteretion*7+7], # best_len[iteretion*7:iteretion*7+7], # "Ratings", "Genome lengths", name1) Plotter().multiline_plot( x_data, best[11 * iteretion:11 * iteretion + 10], best[10 * iteretion + 10:10 * iteretion + 20], best[10 * iteretion + 20:10 * iteretion + 30], x_axis_name="Default parameter pecentage", y_axis_name="Rating", plot_title=name1, legend=["alpha", "mutation_probability", "parent_group_size"]) # y_data.append(best[iteretion * 7:iteretion * 7 + 7]) # print(iteretion, "baprlot values", best[iteretion * 7:iteretion * 7 + 7]) iteretion += 1
from datamodel import Model from servercomm import URL from parameterui import ViewController from excelprinter import ExcelPrinter from plotting import Plotter from optparse import OptionParser parser = OptionParser() parser.add_option("-p", "--plot", action="store_true", dest="do_plot", default=False, help="don't print status messages to stdout") #def set_plot(): # do_plot = True if __name__ == "__main__": (options, args) = parser.parse_args() print(options) printer = 0 if (options.do_plot): printer = Plotter() else: printer = ExcelPrinter('data.xlsx') m = Model(URL, printer) c = ViewController(m)
Plot("dzres_vs_pt_Sigma", ytitle="#sigma(#delta z_{0}) [cm]", ymin=0.0009, ymax=0.1, **_common), Plot("ptres_vs_pt_Sigma", ytitle="#sigma(#delta p_{t}/p_{t})", ymin=0.003, ymax=2.2, **_common), ], legendDy=-0.02, legendDh=-0.01 ) plotter = Plotter([ "DQMData/Run 1/Tracking/Run summary/Track", "DQMData/Tracking/Track", "DQMData/Run 1/RecoTrackV/Run summary/Track", "DQMData/RecoTrackV/Track", ],[ _effandfake1, _effandfake2, _dupandfake1, _dupandfake2, _effvspos, _dedx, _chargemisid, _hitsAndPt, _ntracks, _tuning, _pulls, _resolutionsEta, _resolutionsPt, ]) import collections _iterModuleMap = collections.OrderedDict([ ("initialStepPreSplitting", ["initialStepSeedLayersPreSplitting", "initialStepSeedsPreSplitting", "initialStepTrackCandidatesPreSplitting",
import numpy as np from plotting import Plotter a=11E-9 L = 1000 # Length of wet region P_cap = 2*72E-3/a x = np.linspace(0,L,100) print(x) sigma_0 = P_cap*(x/L) r_tip = plt = Plotter(subplot=(2, 1),sharey = True) plt.add_plot(x, sigma_0, marker='r-') plt.show_figure()
def create_plotters_from_config(file_path): user_specs = yaml_loader(file_path) # user_specs = OrderedDict(yaml_loader(file_path)) # user_specs = OrderedDict(user_specs) # 2. Grab schema file schemas = None if capnp_schemas_location in user_specs[source_key]: schemas = user_specs[source_key][capnp_schemas_location] else: print 'no schemas location specified' # 3. Grab source from which to read data (.stk or live transport) # Checks to see what source is specified (For now cannot be both) if user_specs[source_key].has_key(stk_file_source): stk = user_specs[source_key][stk_file_source] reader = DataReaderFromFile(schemas, stk) elif user_specs[source_key][kb_tranport_settings_key][ kb_transport_type_key]: transport_type = user_specs[source_key][kb_tranport_settings_key][ kb_transport_type_key] hosts = user_specs[source_key][kb_tranport_settings_key][ kb_transport_hosts_key] kb_name = user_specs[source_key][kb_tranport_settings_key][kb_name_key] queue_lenght = None thread_hertz = None read_threads = None transport_settings = user_specs[source_key][kb_tranport_settings_key] #sub_dict = dict(sub_dict) if transport_settings.has_key(queue_lenght_key): queue_lenght = transport_settings[queue_lenght_key] if transport_settings.has_key(read_threads_key): read_threads = transport_settings[read_threads_key] if transport_settings.has_key(thread_hertz_key): thread_hertz = transport_settings[thread_hertz_key] creator = KnowledgeBaseCreator(kb_name, transport_type, hosts, queue_lenght, read_threads, thread_hertz) kb = creator.get_knowledge_base() reader = DataReaderFromKB(schemas, kb) else: print( 'You have not specified a data source to read from or it is incomplete!' ) sys.exit() # TODO: maybe merge types into one? ####################################### Plot Data ############################################### # 1. Create list of KRs for which the user would like to be plotted. # Separate lists for Any and Frame type data if frames_key in user_specs[subkeys_key]: frame_KRs = user_specs[subkeys_key][frames_key] # list else: frame_KRs = {} if any_type_key in user_specs[subkeys_key]: any_KRs = user_specs[subkeys_key][any_type_key] # list else: any_KRs = {} has_other_key = False plot_dict = {} for key in user_specs[subkeys_key]: plot_3d = False points = 0 if not (key == frames_key) and not (key == any_type_key): has_other_key = True value = user_specs[subkeys_key][key] if not (value == None): sub_plot_list = [] for subkey, value in value.items(): if subkey.startswith(subplot_start_key): sub_plot_list.append(value.items()) elif subkey == plot_3d_key: plot_3d = value elif subkey == points_per_plot_key: points = value plot_dict[key] = Plotter(reader, key, subkeys=sub_plot_list, points_per_plot=points, plot_to_3d=plot_3d) else: plot_dict[key] = Plotter(reader, key) if len(any_KRs) == 0 and len(frame_KRs) == 0 and not (has_other_key): print('You have not specified any knowledge records!') sys.exit() # 2. Frame type data plotting for key, value in frame_KRs.items(): sub_plot_list = [] plot_3d = False points = 0 reference_frames = ['geo', 'p1_base_stabilized'] # keep the default values try: if plot_3d_key in value.keys(): plot_3d = value[plot_3d_key] if points_per_plot_key in value.keys(): points = value[points_per_plot_key] if reference_frame_key in value.keys(): reference_frames = value[reference_frame_key] # just to find if 'plot_' substring is in a value string s = [s for s in value.keys() if subplot_start_key in s] if s: for subkey in value.keys(): if subplot_start_key in subkey: sub_plot_list.append(frame_KRs[key][subkey].items()) except: pass # if it is empty set to None, so when passing to plotter it doesn't appear as an existing array if not sub_plot_list: print sub_plot_list sub_plot_list = None plot_dict[key] = Plotter(reader, data_reader_interface.frames_prefix + '.' + key, plot_to_3d=plot_3d, points_per_plot=points, subkeys=sub_plot_list, frames_of_choice=reference_frames) # 3. Any type data plotting #TODO: Allow for handling of just specifying key and plotting subkeys automatically for key, value in any_KRs.items(): sub_plot_list = [] plot_3d = False points = 0 try: if plot_3d_key in value.keys(): plot_3d = value[plot_3d_key] if points_per_plot_key in value.keys(): points = value[points_per_plot_key] # just to find if 'plot_' substring is in a value string s = [s for s in value.keys() if subplot_start_key in s] if s: for subkey, subvalue in value.items(): if subplot_start_key in subkey: try: sub_plot_list.append( sorted(any_KRs[key][subkey].items())) except: sub_plot_list.append(subvalue) except: pass if not sub_plot_list: sub_plot_list = None plot_dict[key] = Plotter(reader, key, plot_to_3d=plot_3d, points_per_plot=points, subkeys=sub_plot_list) return plot_dict
import yahoo from plotting import Plotter import market import argparse LOCAL_DATASOURCE = "yahoo.db3" TEST_SYMBOL = "ENI.MI" if __name__ == '__main__': ## plotting ## source = yahoo.LocalSource(LOCAL_DATASOURCE) symbol = market.Symbol(source, TEST_SYMBOL, None, None, matplotlib=True) p = Plotter('Simple') p.draw_simple(symbol) p.run() p = Plotter('Candlestick') p.draw_candlestick(symbol) p.run() p = Plotter('Simple with volume') p.draw_simple_with_volume(symbol) p.run() p = Plotter('Simple with volume and OBV') p.draw_simple_with_volume_obv(symbol) p.run()
def __init__(self): self.experiment_name = self.__class__.__name__ self.__plotter__ = Plotter() self.last_plot = None self.path = self.config.TIC_TAC_TOE_DIR + "/experiments/artifacts/%s/" % self.experiment_name
# bus_loc = (40,80) # bus_ori = 170 lresolution = 10 presolution = 1 # initialize obstacles obstacles = Obs.random_obstacle_generate(dim_range, start, end, N_obs) # initialize search space Space = Xspace(dim_range, obstacles) Car = lidar_car(car_loc, car_ori) car_lidar_data = Car.lidar_check(Space, lresolution, presolution) # plot part handle_plot = Plotter(Space) # get current axes ax = plt.gca() # add search space to axes ax = handle_plot.plot_2D(ax, obstacles, start, end) # add agents to space ax = handle_plot.plot_2D_vehicle(ax, Car) ax = handle_plot.plot_2D_vehicle(ax, Bus) ax.axis('equal') ax.grid() plt.xlim([0, dim_range[0]]) plt.ylim([0, dim_range[1]]) plt.xlabel('x (m)') plt.ylabel('y (m)')
else: a_type_dict = {} if not (a_type_dict or f_type_dict): print('You have not specified any knowledge records!') sys.exit() ######################## Frame type data parsing ########################## frame_KRs = user_specs['knowledge_record_subkeys']['frame_types'] for key, value in frame_KRs.items(): sub_plot_list = [] print key, value # print value.keys() if not value: plt2 = Plotter(reader, '.gams.frames.' + key) continue if 'reference_frame' in value.keys(): reference_frames = frame_KRs[key]['reference_frame'] for subkey in value.keys(): if 'plot' in subkey: sub_plot_list.append(frame_KRs[key][subkey].items()) plt7 = Plotter(reader, '.gams.frames.' + key, frames_of_choice=reference_frames, subkeys=sub_plot_list, points_per_plot=10) else: plt4 = Plotter(reader, '.gams.frames' + key, frames_of_choice=['geo', 'p1_base_footprint'])
def sensitivity_analysis(): # Sensitivity analysis settings for epsilon=0.1 directory = 'Sensitivity_Analysis' image_name = 'epsilon=0.1' epsilon = 0.1 # Set up RL data = Data(ROWS, COLS, load_data=False, epsilon=epsilon) gen = Generator(ROWS, COLS, resolution, size) vis = Plotter(data, cell_edge, ROWS, COLS) # Display maps until we get one we like while True: data.racetrack = gen.generate_map() data.get_start_line() data.get_finish_line() vis.visualize_racetrack() response = input("Is this map suitable for our test? (y/n): ") if response == 'y': break # Finish setting up RL env = Environment(data, gen, ROWS, COLS) mcc = Monte_Carlo_Control(data, ROWS, COLS) agent = Agent() print("Saving data") # Save arrays data.save_racetrack(directory=directory) data.save_Q_vals(directory=directory) data.save_C_vals(directory=directory) data.save_rewards(directory=directory) data.save_pi(directory=directory) # Train agent train_agent(mcc, env, agent, data, plot=True, image_name=image_name) # Sensitivity analysis settings for epsilon=0.01 image_name = 'epsilon=0.01' epsilon = 0.01 gif_name = 'epsilon=0.01' # Set up RL data2 = Data(ROWS, COLS, load_data=False, epsilon=epsilon) vis = Plotter(data2, cell_edge, ROWS, COLS) # Get old map data2.racetrack = data.racetrack data2.get_start_line() data2.get_finish_line() # Finish setting up RL env = Environment(data2, gen, ROWS, COLS) mcc = Monte_Carlo_Control(data2, ROWS, COLS) print("Saving data") # Save arrays data.save_racetrack(directory=directory) data.save_Q_vals(directory=directory) data.save_C_vals(directory=directory) data.save_rewards(directory=directory) data.save_pi(directory=directory) # Train agent train_agent(mcc, env, agent, data2, plot=True, image_name=image_name) get_gif(gif_name, data2, vis)
class GA_Agent: hparams = {} random_seed = 0 eval_interval = -1 display_moves = False should_log = False random_util = RandomUtil(0) plotter = Plotter('none') base_path = 'none' episode_count = 0 def __init__(self, hyperparameters, should_log=False, eval_interval=-1, display_moves=False): self.hparams = hyperparameters self.random_seed = round(time.time()) self.should_log = should_log self.display_moves = display_moves self.eval_interval = eval_interval self.random_util = RandomUtil(self.random_seed) self.base_path = 'results/GA-' + self.random_util.generate_random_string( 6) + '/' self.plotter = Plotter(self.base_path) self.episode_count = 0 def __evaluate_ga(self, blue_coeffs, red_coeffs, eval_py_env, display_moves=False, no_tests=10): if display_moves: episode_path = 'game-drawn/episode-' + str(self.episode_count) if not os.path.exists(self.base_path + episode_path): os.makedirs(self.base_path + episode_path) time_step = eval_py_env.reset() picture_count = 0 while not time_step.is_last(): observation = time_step.observation self.plotter.plot_state( observation, episode_path + '/' + str(picture_count) + '.jpeg') picture_count += 1 time_step = eval_py_env.step() picture_count = 0 self.episode_count += 1 total_return = 0.0 for _ in range(no_tests): time_step = eval_py_env.reset() episode_return = 0.0 while not time_step.is_last(): #print(time_step.observation.numpy()) time_step = eval_py_env.step() episode_return += time_step.reward total_return += episode_return return total_return / no_tests def __plot_coefs(self, single_organism_coefs, file_name): x = 0.0 x_values = [] function_values = [] def evaluate_function(x): value = 0.0 x_pow = 1 for coef in single_organism_coefs: value += coef * x_pow x_pow *= x return value step = 0.03 while x < 10: function_values.append(evaluate_function(x)) x_values.append(x) x += step self.plotter.plot_simple_values(x=x_values, y=function_values, directory=file_name) def train(self, eval_game_params): hparams = self.hparams max_degree = hparams['max_parameter_degree'] no_blues = hparams['no_blue_organisms'] no_reds = hparams['no_red_organisms'] no_parameters = hparams['no_parameters'] coef_count = np.power(max_degree + 1, no_parameters) self.ga_util = GaUtil(self.random_util, coef_count) blue_coeffs = self.random_util.get_random_matrix( no_blues, coef_count, [-1000, 1000]) red_coeffs = self.random_util.get_random_matrix( no_reds, coef_count, [-1000, 1000]) returns = [] # Train the genetic algorithm no_random = hparams['no_random_start'] * 1.0 random_step = (hparams['no_random_final'] - no_random) / hparams['no_random_anneal_time'] mutation_factor_range = np.array( hparams['mutation_factor_range_start']) mutation_factor_range_final = np.array( hparams['mutation_factor_range_final']) mutation_factor_range_step = (mutation_factor_range_final - mutation_factor_range)\ / hparams['mutation_factor_range_anneal_time'] for generation_number in range(hparams['no_generations']): prev_blue_organisms = [] prev_red_organisms = [] if generation_number > 0: prev_blue_organisms = env.dead_blue_organisms prev_red_organisms = env.dead_red_organisms env = GameEnv(blue_coeffs, red_coeffs, max_degree, hparams['food_count'], hparams['board_size']) #utils.validate_py_environment(py_environment, episodes=5) # Evaluate the GA if self.eval_interval > 0 and (generation_number + 1) % self.eval_interval == 0: eval_blue_coeffs = self.ga_util.get_coeffs_from_best( prev_blue_organisms, eval_game_params['no_blue_organisms'], eval_game_params['no_blue_organisms'], 0, [0, 0]) eval_red_coeffs = self.ga_util.get_coeffs_from_best( prev_red_organisms, eval_game_params['no_red_organisms'], eval_game_params['no_red_organisms'], 0, [0, 0]) eval_py_env = GameEnv(eval_blue_coeffs, eval_red_coeffs, max_degree, eval_game_params['food_count'], eval_game_params['board_size']) avg_return = self.__evaluate_ga(blue_coeffs, red_coeffs, eval_py_env, self.display_moves) returns.append(avg_return) if self.should_log: print(avg_return) # Play the game time_step = env.reset() while not time_step.is_last(): #print(time_step.observation.numpy()) time_step = env.step() # Pick best genomes for the next generation blue_organisms = env.dead_blue_organisms blue_coeffs = self.ga_util.get_coeffs_from_best( blue_organisms, no_blues, hparams['no_best'], round(no_random), mutation_factor_range) red_organisms = env.dead_red_organisms red_coeffs = self.ga_util.get_coeffs_from_best( red_organisms, no_reds, hparams['no_best'], round(no_random), mutation_factor_range) # Reduce the number of random organisms and mutation_factor_range no_random += random_step mutation_factor_range += mutation_factor_range_step self.plotter.plot_simple_values(y=list(returns), directory='score.jpeg') for single_organism_coefs in red_coeffs: self.__plot_coefs(single_organism_coefs, 'red-coeffs.jpeg') for single_organism_coefs in blue_coeffs: self.__plot_coefs(single_organism_coefs, 'blue-coeffs.jpeg') hparams['random_seed'] = self.random_seed self.plotter.dump_to_json(hparams, 'hparams.json')
def create_plotters_from_config(file_path): user_specs = yaml_loader(file_path) # user_specs = OrderedDict(yaml_loader(file_path)) # user_specs = OrderedDict(user_specs) # 2. Grab schema file if user_specs[source_key][capnp_schemas_location]: schemas = user_specs[source_key][capnp_schemas_location] else: sys.exit('You have not specified schema files') # 3. Grab source from which to read data (.stk or live transport) # Checks to see what source is specified (For now cannot be both) if user_specs[source_key].has_key(stk_file_source): stk = user_specs[source_key][stk_file_source] reader = DataReaderFromFile(schemas, stk) elif user_specs[source_key][kb_tranport_settings_key][ kb_transport_type_key]: transport_type = user_specs[source_key][kb_tranport_settings_key][ kb_transport_type_key] hosts = user_specs[source_key][kb_tranport_settings_key][ kb_transport_hosts_key] kb_name = user_specs[source_key][kb_tranport_settings_key][kb_name_key] queue_lenght = None thread_hertz = None read_threads = None transport_settings = user_specs[source_key][kb_tranport_settings_key] #sub_dict = dict(sub_dict) if transport_settings.has_key(queue_lenght_key): queue_lenght = transport_settings[queue_lenght_key] if transport_settings.has_key(read_threads_key): read_threads = transport_settings[read_threads_key] if transport_settings.has_key(thread_hertz_key): thread_hertz = transport_settings[thread_hertz_key] creator = KnowledgeBaseCreator(kb_name, transport_type, hosts, queue_lenght, read_threads, thread_hertz) kb = creator.get_knowledge_base() reader = DataReaderFromKB(schemas, kb) else: print( 'You have not specified a data source to read from or it is incomplete!' ) sys.exit() # TODO: maybe merge types into one? ####################################### Plot Data ############################################### # 1. Create list of KRs for which the user would like to be plotted. # Separate lists for Any and Frame type data if user_specs[subkeys_key][frames_key]: frame_KRs = user_specs[subkeys_key][frames_key] # list else: frame_KRs = {} if user_specs[subkeys_key][any_type_key]: any_KRs = user_specs[subkeys_key][any_type_key] # list else: any_KRs = {} has_other_key = False plot_dict = {} for key in user_specs[subkeys_key]: if not (key == frames_key) and not (key == any_type_key): has_other_key = True value = user_specs[subkeys_key][key] if not (value == None): sub_plot_list = [] for subkey in value.keys(): if 'plot' in subkey: sub_plot_list.append(value[subkey].items()) plot_dict[key] = Plotter(reader, key, subkeys=sub_plot_list) else: plot_dict[key] = Plotter(reader, key) if len(any_KRs) == 0 and len(frame_KRs) == 0 and not (has_other_key): print('You have not specified any knowledge records!') sys.exit() # 2. Frame type data plotting for key, value in frame_KRs.items(): sub_plot_list = [] flagger = [0, 0, 0, 0] try: if '3D' in value.keys() and value['3D']: flagger[0] += 1 if 'points_per_plot' in value.keys(): points = value['points_per_plot'] flagger[1] += 1 if 'reference_frame' in value.keys(): reference_frames = value['reference_frame'] flagger[2] += 1 # just to find if 'plot_' substring is in a value string s = [s for s in value.keys() if 'plot_' in s] if s: # if 'plot_' in value.keys(): flagger[3] += 1 for subkey in value.keys(): if 'plot_' in subkey: sub_plot_list.append(frame_KRs[key][subkey].items()) except: pass if flagger == [0, 0, 0, 0]: plot_dict[key] = Plotter( reader, data_reader_interface.frames_prefix + '.' + key) continue if flagger == [1, 0, 0, 0]: plot_dict[key] = Plotter(reader, data_reader_interface.frames_prefix + '.' + key, plot_to_3d=True) continue if flagger == [0, 1, 0, 0]: plot_dict[key] = Plotter(reader, data_reader_interface.frames_prefix + '.' + key, points_per_plot=points) continue if flagger == [0, 0, 1, 0]: plot_dict[key] = Plotter(reader, data_reader_interface.frames_prefix + '.' + key, frames_of_choice=reference_frames) continue if flagger == [0, 0, 0, 1]: plot_dict[key] = Plotter(reader, data_reader_interface.frames_prefix + '.' + key, subkeys=sub_plot_list) continue if flagger == [1, 1, 0, 0]: plot_dict[key] = Plotter(reader, data_reader_interface.frames_prefix + '.' + key, points_per_plot=points, plot_to_3d=True) continue if flagger == [0, 1, 1, 0]: plot_dict[key] = Plotter(reader, data_reader_interface.frames_prefix + '.' + key, points_per_plot=points, frames_of_choice=reference_frames) continue if flagger == [0, 0, 1, 1]: plot_dict[key] = Plotter(reader, data_reader_interface.frames_prefix + '.' + key, frames_of_choice=reference_frames, subkeys=sub_plot_list) continue if flagger == [1, 0, 1, 0]: plot_dict[key] = Plotter(reader, data_reader_interface.frames_prefix + '.' + key, plot_to_3d=True, frames_of_choice=reference_frames) continue if flagger == [1, 0, 0, 1]: plot_dict[key] = Plotter(reader, data_reader_interface.frames_prefix + '.' + key, plot_to_3d=True, subkeys=sub_plot_list) continue if flagger == [0, 1, 0, 1]: plot_dict[key] = Plotter(reader, data_reader_interface.frames_prefix + '.' + key, points_per_plot=points, subkeys=sub_plot_list) continue if flagger == [1, 1, 1, 0]: plot_dict[key] = Plotter(reader, data_reader_interface.frames_prefix + '.' + key, points_per_plot=points, plot_to_3d=True, frames_of_choice=reference_frames) continue if flagger == [0, 1, 1, 1]: plot_dict[key] = Plotter(reader, data_reader_interface.frames_prefix + '.' + key, points_per_plot=points, subkeys=sub_plot_list, frames_of_choice=reference_frames) continue if flagger == [1, 0, 1, 1]: plot_dict[key] = Plotter(reader, data_reader_interface.frames_prefix + '.' + key, plot_to_3d=True, subkeys=sub_plot_list, frames_of_choice=reference_frames) continue if flagger == [1, 1, 0, 1]: plot_dict[key] = Plotter(reader, data_reader_interface.frames_prefix + '.' + key, plot_to_3d=True, subkeys=sub_plot_list, points_per_plot=points) continue if flagger == [1, 1, 1, 1]: plot_dict[key] = Plotter(reader, data_reader_interface.frames_prefix + '.' + key, points_per_plot=points, plot_to_3d=True, frames_of_choice=reference_frames, subkeys=sub_plot_list) continue # 3. Any type data plotting #TODO: Allow for handling of just specifying key and plotting subkeys automatically for key, value in any_KRs.items(): sub_plot_list = [] flagger = [0, 0, 0] try: if '3D' in value.keys() and value['3D']: flagger[0] += 1 if 'points_per_plot' in value.keys(): points = value['points_per_plot'] flagger[1] += 1 # just to find if 'plot_' substring is in a value string s = [s for s in value.keys() if 'plot_' in s] if s: flagger[2] += 1 for subkey, subvalue in value.items(): if 'plot_' in subkey: try: sub_plot_list.append( sorted(any_KRs[key][subkey].items())) except: sub_plot_list.append(subvalue) except: pass if flagger == [0, 0, 1]: plot_dict[key] = Plotter(reader, key, subkeys=sub_plot_list) continue if flagger == [0, 1, 1]: plot_dict[key] = Plotter(reader, key, subkeys=sub_plot_list, points_per_plot=points) continue if flagger == [1, 0, 1]: plot_dict[key] = Plotter(reader, key, plot_to_3d=True, subkeys=sub_plot_list) continue if flagger == [1, 1, 1]: plot_dict[key] = Plotter(reader, key, plot_to_3d=True, subkeys=sub_plot_list, points_per_plot=points) continue return plot_dict
_summaryPlots = [ _summary, _summaryN, ] _summaryPlotsHp = [ _summaryHp, _summaryNHp, ] _packedCandidatePlots = [ _packedCandidateFlow, _packedCandidateParam1, _packedCandidateParam2, _packedCandidateMomVert, _packedCandidateHits, ] plotter = Plotter() def _appendTrackingPlots(lastDirName, name, algoPlots, onlyForPileup=False): # to keep backward compatibility, this set of plots has empty name plotter.append( name, _trackingFolders(lastDirName), TrackingPlotFolder(*algoPlots, onlyForPileup=onlyForPileup, purpose=PlotPurpose.TrackingIteration)) summaryName = "" if name != "": summaryName += name + "_" summaryName += "summary" plotter.append( summaryName, _trackingFolders(lastDirName),
def main(model=None, output_dir=None, n_iter=20, n_texts=2000, init_tok2vec=None, plot=False): if output_dir is not None: output_dir = Path(output_dir) if not output_dir.exists(): output_dir.mkdir() if model is not None: nlp = spacy.load(model) # load existing spaCy model print("Loaded model '%s'" % model) else: nlp = spacy.blank("en") # create blank Language class print("Created blank 'en' model") # add the text classifier to the pipeline if it doesn't exist # nlp.create_pipe works for built-ins that are registered with spaCy if "textcat" not in nlp.pipe_names: textcat = nlp.create_pipe( "textcat", config={"exclusive_classes": True, "architecture": "simple_cnn"} ) nlp.add_pipe(textcat, last=True) # otherwise, get it, so we can add labels to it else: textcat = nlp.get_pipe("textcat") # add label to text classifier textcat.add_label("POSITIVE") textcat.add_label("NEGATIVE") # load the IMDB dataset print("Loading IMDB data...") (train_texts, train_cats), (dev_texts, dev_cats) = load_data() train_texts = train_texts[:n_texts] train_cats = train_cats[:n_texts] print( "Using {} examples ({} training, {} evaluation)".format( n_texts, len(train_texts), len(dev_texts) ) ) train_data = list(zip(train_texts, [{"cats": cats} for cats in train_cats])) dev_data = list(zip(dev_texts, [{"cats": cats} for cats in dev_cats])) # get names of other pipes to disable them during training pipe_exceptions = ["textcat", "trf_wordpiecer", "trf_tok2vec"] other_pipes = [pipe for pipe in nlp.pipe_names if pipe not in pipe_exceptions] with nlp.disable_pipes(*other_pipes): # only train textcat optimizer = nlp.begin_training() if init_tok2vec is not None: with init_tok2vec.open("rb") as file_: textcat.model.tok2vec.from_bytes(file_.read()) print("Training the model...") print("{:^5}\t{:^5}\t{:^5}\t{:^5}".format("LOSS", "P", "R", "F")) # Set-up the plotter: if plot: plotter = Plotter( title='IMDB Text categorisation training', ylabels=["Train-loss", "Dev-loss", "Precision", "Recall", "F-score"], iterations=n_iter, figsize=(8, 10)) batch_sizes = compounding(4.0, 32.0, 1.001) for i in range(n_iter): losses = {} # batch up the examples using spaCy's minibatch random.shuffle(train_data) batches = minibatch(train_data, size=batch_sizes) for batch in batches: texts, annotations = zip(*batch) nlp.update(texts, annotations, sgd=optimizer, drop=0.2, losses=losses) with textcat.model.use_params(optimizer.averages): # evaluate on the dev data split off in load_data() scores = evaluate(nlp.tokenizer, textcat, dev_texts, dev_cats) dev_losses = {} random.shuffle(dev_data) batches = minibatch(dev_data, size=batch_sizes) for batch in batches: texts, annotations = zip(*batch) nlp.update(texts, annotations, sgd=None, losses=dev_losses) print( "{0:.3f}\t{1:.3f}\t{2:.3f}\t{3:.3f}".format( # print a simple table losses["textcat"], dev_losses["textcat"], scores["textcat_p"], scores["textcat_r"], scores["textcat_f"], ) ) # Update the plot: if plot: plotter.update(y=[ losses["textcat"], dev_losses["textcat"], scores["textcat_p"], scores["textcat_r"], scores["textcat_f"], ]) # test the trained model test_text = "This movie sucked" doc = nlp(test_text) print(test_text, doc.cats) if output_dir is not None: with nlp.use_params(optimizer.averages): nlp.to_disk(output_dir) print("Saved model to", output_dir) # test the saved model print("Loading from", output_dir) nlp2 = spacy.load(output_dir) doc2 = nlp2(test_text) print(test_text, doc2.cats) # Keep showing the plot until the plotting window is closed. if plot: plotter.keep()
import matplotlib.pyplot as plt import seaborn as sns sys.path.insert(1, "../tools") from analysis import Analyzer from plotting import Plotter from training import Trainer if __name__ == "__main__": sns.set() plot_dir = "plots" plot_file = os.path.join(plot_dir, "rdf.png") if not os.path.exists(plot_dir): os.mkdir(plot_dir) anl = Analyzer() plter = Plotter() r_cut = 6.0 r, rdf = anl.calculate_rdf("trajs/training.traj", r_max=r_cut) rdf[np.nonzero(rdf)] /= max(rdf) cutoff = plter.polynomial(r, r_cut, gamma=5.0) plt.plot(r, rdf, label="Radial distribution function") plt.plot(r, cutoff, label="Polynomial cutoff, gamma=5.0") plt.legend() plt.title("Copper radial distribution function") plt.xlabel("Radial distance [Angstrom]") plt.ylabel("Radial distribution function (normalized to 1)") plt.savefig(plot_file)
assert b1 in range(1, 9) b2 = int( input( 'Choose action:\n1)Visualize Bigrams\n2)Visualize Terms\n3)Run model grid param search\n4)Run all ' 'models kfold\n> ')) assert b2 in range(1, 5) b3 = int( input( 'Perform clean file parsing or use pickles?:\n1)Use pickles\n2)Perform clean file parsing\n> ' )) assert b3 in range(1, 3) plotter = Plotter(threads=b1, ignore_pickles=True, strict=(b3 == 2)) while b2 != 5: if b2 == 1: plotter.visualize_bigrams(40) #print(plotter.prep.best_bigram_scores) elif b2 == 2: plotter.visualize_descriptive_terms(200) #print(plotter.prep.selected_words) elif b2 == 3: start = time.time()