def __open_selected(self, widget, path): """Open selected item in either active, or new tab""" # unquote path before giving it to handler if path is not None and '://' in path: data = path.split('://', 1) data[1] = urllib.unquote(data[1]) path = '://'.join(data) # open selected item if self._open_in_new_tab: # create new tab options = Parameters() options.set('path', path) self._application.create_tab( self._object._notebook, self._object.__class__, options ) elif hasattr(self._object, 'change_path'): self._object.change_path(path) # reset values self._open_in_new_tab = False return True
def _change_path(self, widget=None, new_tab=False): """Change to selected path""" selection = self._history_list.get_selection() item_list, selected_iter = selection.get_selected() # if selection is valid, change to selected path if selected_iter is not None: path = item_list.get_value(selected_iter, Column.PATH) if not new_tab: # change path self._parent._handle_history_click(path=path) else: # create a new tab options = Parameters() options.set('path', path) self._application.create_tab( self._parent._notebook, self._parent.__class__, options ) # close dialog self._close()
def countRegions(): # Requires: -the first command line argument is the name of the # image to be segmented # -the second command line argument is the color space being # used, either RGB, HSV, or HLS # Effects: -calls closure with count foreground argument on, returns # count of distinct foreground objects colorSpace = argv[2].lower() if not (colorSpace in ["rgb", "hsv", "hls"]): print "Second argument not one of RGB, HSV, or HLS" print "The first argument should be the name of the image to be segmented" print "Followed by the desire color space representation" exit(1) try: image = Image.open(argv[1]) imageData = colorSpaceConvert(list(image.getdata()), argv[2].lower()) except: print "Invalid or no image name given" print "The first argument should be the name of the image to be segmented" print "Followed by the desire color space representation" exit(1) if colorSpace == "rgb": redMinMax = raw_input("Red min-max, between 0 and 255: ") greenMinMax = raw_input("Green min-max, between 0 and 255: ") blueMinMax = raw_input("Blue min-max, between 0 and 255: ") redMinMax = [float(x) / 255.0 for x in redMinMax.split()] greenMinMax = [float(x) / 255.0 for x in greenMinMax.split()] blueMinMax = [float(x) / 255.0 for x in blueMinMax.split()] colorRanges = [redMinMax, greenMinMax, blueMinMax] elif colorSpace == "hsv": hueMinMax = raw_input("Hue min-max, between 0 and 360: ") satMinMax = raw_input("Saturation min-max, between 0 and 100: ") valMinMax = raw_input("Value min-max, between 0 and 100: ") hueMinMax = [float(x) / 360.0 for x in hueMinMax.split()] satMinMax = [float(x) / 100.0 for x in satMinMax.split()] valMinMax = [float(x) / 100.0 for x in valMinMax.split()] colorRanges = [hueMinMax, satMinMax, valMinMax] else: hueMinMax = raw_input("Hue min-max, between 0 and 360: ") lightMinMax = raw_input("Lightness min-max, between 0 and 100: ") satMinMax = raw_input("Saturation min-max, between 0 and 100: ") hueMinMax = [float(x) / 360.0 for x in hueMinMax.split()] lightMinMax = [float(x) / 100.0 for x in lightMinMax.split()] satMinMax = [float(x) / 100.0 for x in satMinMax.split()] colorRanges = [hueMinMax, lightMinMax, satMinMax] param = Parameters() param.setImageSize(image.size) param.setColorRanges(colorRanges) seg = segmentation.colorSegmenter() mask = seg.segmentImage(imageData, param, True) close = closure.closure() close.segmentRegions(mask, param, 0, True, False)
def _create_file_list(self, widget=None, data=None): """Create file list in parent notebook""" self.__update_path_from_pid() DefaultList = self._parent.plugin_classes['file_list'] options = Parameters() options.set('path', self.path) self._parent.create_tab(self._notebook, DefaultList, options) return True
def _close_tab(self, widget=None, data=None): """Provide additional functionality""" if self._notebook.get_n_pages() == 1: DefaultList = self._parent.plugin_classes['file_list'] options = Parameters() options.set('path', self.path) self._parent.create_tab(self._notebook, DefaultList, options) return Terminal._close_tab(self, widget, data)
def open_file(self, selection, application_info=None, exec_command=None): """Open filename using config file or specified execute command""" if application_info is not None: # get command from config file command = application_info.command_line elif exec_command is not None: # use specified command command = exec_command else: # raise exception, we need at least one argument raise AttributeError('Error opening file. We need command or application to be specified.') exec_string = self.__format_command_string(selection, command) # open selected file(s) split_command = shlex.split(exec_string) test_command = split_command[0] if len(split_command) > 1 else exec_string if is_x_app(test_command): os.system('{0} &'.format(exec_string)) else: active_object = self._application.get_active_object() options = Parameters() options.set('close_with_child', True) options.set('shell_command', split_command[0]) options.set('arguments', split_command) options.set('path', os.path.dirname(selection[0])) self._application.create_terminal_tab(active_object._notebook, options)
def edit_file(self, selection): """Edit selected filename""" section = self._application.options.section('editor') command = section.get('default_editor') exec_string = self.__format_command_string(selection, command) # open selected file(s) split_command = shlex.split(exec_string) test_command = split_command[0] if len(split_command) > 1 else exec_string if (section.get('terminal_command') and section.get('type') == 1) \ or not is_x_app(test_command): active_object = self._application.get_active_object() options = Parameters() options.set('close_with_child', True) options.set('shell_command', split_command[0]) options.set('arguments', split_command) options.set('path', os.path.dirname(selection[0])) self._application.create_terminal_tab(active_object._notebook, options) else: os.system('{0} &'.format(exec_string))
def load_parameters(json_parameters): parameters = Parameters() parameters.clustering_threshold = json_parameters['clustering_threshold'] parameters.size_threshold = json_parameters['size_threshold'] parameters.distance_threshold = json_parameters['distance_threshold'] parameters.hashing_depth = json_parameters['hashing_depth'] parameters.clusterize_using_dcup = json_parameters['clusterize_using_dcup'] parameters.clusterize_using_hash = json_parameters['clusterize_using_hash'] parameters.report_unifiers = json_parameters['report_unifiers'] parameters.force = json_parameters['force'] parameters.use_diff = json_parameters['use_diff']
def create_terminal(self, path, position=None): options = Parameters() options.set('path', path) if position == 'left': notebook = self._application.left_notebook elif position == 'right': notebook = self._application.right_notebook else: notebook = self._application.get_active_notebook() self._application.create_tab(notebook, self._application.plugin_classes['system_terminal'], options)
def scan(language, file_manifest, source_file_names): # Determine the files to scan. If no files are given, use a default manifest. if len(source_file_names) == 0 and file_manifest is None: file_manifest = manifest.default_manifest(language) source_file_names = set(source_file_names) if file_manifest is not None: source_file_names.update(set(manifest.contents(file_manifest))) supplier = ast_suppliers.abstract_syntax_tree_suppliers[language] # TODO: Configuration files! parameters = Parameters() parameters.distance_threshold = supplier.distance_threshold parameters.size_threshold = supplier.size_threshold source_files = [] report = Report(parameters) def parse_file(file_name): try: logging.info('Parsing ' + file_name + '...') source_file = supplier(file_name, parameters) source_file.getTree().propagateCoveredLineNumbers() source_file.getTree().propagateHeight() source_files.append(source_file) report.addFileName(file_name) logging.info('done') except: logging.warn('Can\'t parse "%s" \n: ' % (file_name,) + traceback.format_exc()) for file_name in source_file_names: parse_file(file_name) duplicates = clone_detection_algorithm.findDuplicateCode(source_files, report) n = 1 for duplicate in duplicates: distance = duplicate.calcDistance() summary = CloneSummary( "Clone #"+str(n), [ # TODO: This is a mess! Most of this info should be assembled on the fly and in member functions. Snippet( duplicate[i].getSourceFile()._file_name, duplicate[i].getCoveredLineNumbers(), '\n'.join([line for line in duplicate[i].getSourceLines()]) ) for i in [0, 1]], distance) report.addClone(summary) n += 1 report.sortByCloneSize() save_report(".orphanblack", report)
def create_terminal(self, path, position=None): """Expose method for creating terminal tab.""" options = Parameters() options.set("path", path) if position == "left": notebook = self._application.left_notebook elif position == "right": notebook = self._application.right_notebook else: notebook = self._application.get_active_notebook() self._application.create_tab(notebook, self._application.plugin_classes["system_terminal"], options)
def create_tab(self, path, position=None): """Expose method for creating standard tab.""" options = Parameters() options.set('path', path) if position == 'left': notebook = self._application.left_notebook elif position == 'right': notebook = self._application.right_notebook else: notebook = self._application.get_active_notebook() self._application.create_tab(notebook, self._application.plugin_classes['file_list'], options)
def __init__(self): self.words = list() self.list_number = 2 self.min = dict() self.max = dict() self.first_list = list() self.second_list = list() self.first_list_output = list() self.second_list_output = list() self.minimum = None self.length = 0 self.number_of_same = 0 self.allow = True self.same = list() self.statistics = None self.key_for_differ_feature = "" self.which_higher = None self.p_values = list() self.time_begin = None self.success = True self.first_list_equality_counter = dict() self.second_list_equality_counter = dict() self.should_append_first = dict() self.should_append_second = dict() self.numeric_features = list() self.categorical_features = dict() self.categorical_features_list = list() self.len_of_numeric = 0 self.len_of_categorical = 0 self.parameters = Parameters()
def __init__(self): print 'call me' self.parameters = Parameters() if LBL: graph.output_weights = self.parameters.output_weights graph.output_biases = self.parameters.output_biases graph.score_biases = self.parameters.score_biases else: graph.hidden_weights = self.parameters.hidden_weights graph.hidden_biases = self.parameters.hidden_biases graph.output_weights = self.parameters.output_weights graph.output_biases = self.parameters.output_biases # (self.graph_train, self.graph_predict, self.graph_verbose_predict) = graph.functions(self.parameters) import sets self.train_loss = MovingAverage() self.train_err = MovingAverage() self.train_lossnonzero = MovingAverage() self.train_squashloss = MovingAverage() self.train_unpenalized_loss = MovingAverage() self.train_l1penalty = MovingAverage() self.train_unpenalized_lossnonzero = MovingAverage() self.train_correct_score = MovingAverage() self.train_noise_score = MovingAverage() self.train_cnt = 0
def __init__(self, modelname="", window_size=HYPERPARAMETERS["WINDOW_SIZE"], vocab_size=vocabulary.wordmap().len, embedding_size=HYPERPARAMETERS["EMBEDDING_SIZE"], hidden_size=HYPERPARAMETERS["HIDDEN_SIZE"], seed=miscglobals.RANDOMSEED, initial_embeddings=None, two_hidden_layers=HYPERPARAMETERS["TWO_HIDDEN_LAYERS"]): self.modelname = modelname self.parameters = Parameters(window_size, vocab_size, embedding_size, hidden_size, seed, initial_embeddings, two_hidden_layers) if LBL: graph.output_weights = self.parameters.output_weights graph.output_biases = self.parameters.output_biases graph.score_biases = self.parameters.score_biases else: graph.hidden_weights = self.parameters.hidden_weights graph.hidden_biases = self.parameters.hidden_biases if self.parameters.two_hidden_layers: graph.hidden2_weights = self.parameters.hidden2_weights graph.hidden2_biases = self.parameters.hidden2_biases graph.output_weights = self.parameters.output_weights graph.output_biases = self.parameters.output_biases # (self.graph_train, self.graph_predict, self.graph_verbose_predict) = graph.functions(self.parameters) import sets self.train_loss = MovingAverage() self.train_err = MovingAverage() self.train_lossnonzero = MovingAverage() self.train_squashloss = MovingAverage() self.train_unpenalized_loss = MovingAverage() self.train_l1penalty = MovingAverage() self.train_unpenalized_lossnonzero = MovingAverage() self.train_correct_score = MovingAverage() self.train_noise_score = MovingAverage() self.train_cnt = 0
def main(): application = QApplication(sys.argv) application.quitOnLastWindowClosed = True params = Parameters() if os.path.isfile("default.cfg"): try: params.load_from_file("default.cfg") except: # Even if the operation fails the object should still be usable. pass window = ApplicationWidget(params) window.show() sys.exit(application.exec_())
def main(argv): print(argv) # Create and parse parameters. parameters = Parameters() parameters.parse(argv) # Do we need to show help? if parameters.help or len(parameters.directories) == 0: show_help(True) return # Handle each directory. for directory in parameters.directories: process_directory(directory, parameters.recurse, parameters.preview, parameters.fileType)
def run_cli(): """run_cli runs the pipeline from the command line interface. see -h flag for arguments """ params = Parameters.from_command_line() run(params)
def __init__(self, parameters=None, **kwargs): self.__H = None self.__derivative_ops = {} self.__derivative_ops_default = ['evolution'] self.__named_states = {} # A dictionary of named states for easy recollection self.__named_ensembles = {} # A dictionary of named ensembles for density states self.__named_subspaces = {} # A dictionary of named subspaces for easy identification self.__named_bases = {} # A dictionary of named bases for state representation self.__basis_default = None self.init(**kwargs) # Initialise parameters instance if isinstance(parameters, str): self.p = Parameters.load(parameters, constants=True) elif isinstance(parameters, Parameters): self.p = parameters else: self.p = Parameters(constants=True) if isinstance(parameters, dict): self.p << parameters params = self.init_parameters() if isinstance(params, dict): self.p << params # Initialise Hamiltonian H = self.init_hamiltonian() if isinstance(H, (Operator, OperatorSet)): self.hamiltonian = H # Initialise Bases bases = self.init_bases() if isinstance(bases, dict): for name, basis in bases.items(): self.add_basis(name, basis) # Initialise named states states = self.init_states() if isinstance(states, dict): for name, state in states.items(): self.add_state(name, state) # Initialise derivative operators derivatives = self.init_derivative_ops() if isinstance(derivatives, dict): for name, derivative in derivatives.items(): self.add_derivative_op(name, derivative) # Initialise measurements self.measure = Measurements(self) measurements = self.init_measurements() if isinstance(measurements, dict): for name, measurement in measurements: self.add_measurement(name, measurement)
def _open_selected(self, widget, in_new_tab=False): """Open selected mount""" selection = self._list.get_selection() item_list, selected_iter = selection.get_selected() if selected_iter is not None: uri = item_list.get_value(selected_iter, MountsColumn.URI) active_object = self._application.get_active_object() if not in_new_tab and hasattr(active_object, 'change_path'): active_object.change_path(uri) else: # create new tab options = Parameters() options.set('path', uri) self._application.create_tab( active_object._notebook, active_object.__class__, options ) return True
def spike_detection_job(DatFileNames, ProbeFileName, output_dir, output_name): """ Top level function that starts a data processing job. """ for DatFileName in DatFileNames: if not os.path.exists(DatFileName): raise Exception("Dat file %s does not exist" % DatFileName) DatFileNames = [os.path.abspath(DatFileName) for DatFileName in DatFileNames] probe = probes.Probe(ProbeFileName) n_ch_dat = Parameters['NCHANNELS'] sample_rate = Parameters['SAMPLERATE'] high_frequency_factor = Parameters['F_HIGH_FACTOR'] set_globals_samples(sample_rate, high_frequency_factor) Parameters['CHUNK_OVERLAP'] = int( sample_rate * Parameters['CHUNK_OVERLAP_SECONDS']) Parameters['N_CH'] = probe.num_channels max_spikes = Parameters['MAX_SPIKES'] basename = basenamefolder = output_name # OutDir = join(output_dir, basenamefolder) OutDir = output_dir with indir(OutDir): # Create a log file GlobalVariables['log_fd'] = open(basename + '.log', 'w') if Parameters['DEBUG']: GlobalVariables['debug_fd'] = open(basename + '.debug', 'w') Channels_dat = np.arange(probe.num_channels) # Print Parameters dictionary to .log file log_message("\n".join(["{0:s} = {1:s}".format(key, str(value)) for key, value in sorted(Parameters.iteritems()) if not key.startswith('_')])) spike_detection_from_raw_data(basename, DatFileNames, n_ch_dat, Channels_dat, probe.channel_graph, probe, max_spikes) numwarn = GlobalVariables['warnings'] if numwarn: log_message( 'WARNINGS ENCOUNTERED: ' + str(numwarn) + ', check log file.') # Close the log file at the end. if 'log_fd' in GlobalVariables: GlobalVariables['log_fd'].close()
def execute_file(self, path, provider=None): """Execute specified item properly.""" mime_type = self.get_mime_type(path) terminal_type = self._application.options.section('terminal').get('type') should_execute = False if provider is not None and provider.is_local: # only allow local files which have execute # bit set to be executed locally should_execute = os.access(path, os.X_OK) # if we still don't know content type, try to guess if self.is_mime_type_unknown(mime_type): data = self.get_sample_data(path, provider) mime_type = self.get_mime_type(data=data) if gio.content_type_can_be_executable(mime_type) and should_execute: # file type is executable if is_x_app(path): subprocess.Popen( (path, '&'), cwd=os.path.dirname(path) ) else: # command is console based, create terminal tab and fork it if terminal_type != TerminalType.EXTERNAL: active_object = self._application.get_active_object() options = Parameters() options.set('close_with_child', False) options.set('shell_command', path) options.set('path', os.path.dirname(path)) tab = self._application.create_terminal_tab(active_object._notebook, options) else: # file type is not executable, try to open with default associated application default_application = self.get_default_application_for_type(mime_type) if default_application is not None: self.open_file((path,), default_application) else: # no default application selected, show application selection dialog dialog = ApplicationSelectDialog(self._application, path) result = dialog.get_response() if result[0] == gtk.RESPONSE_OK: self.open_file(selection=(path,), exec_command=result[2])
def open_file(self, selection, application_info=None, exec_command=None): """Open filename using config file or specified execute command""" if application_info is not None: # launch application using GIO API application = self.get_gio_application_by_id(application_info.id) if application is not None: if application.supports_uris(): application.launch_uris(selection) else: application.launch([gio.File(path=path) for path in selection]) elif exec_command is not None: # use specified command command = exec_command selection = map(lambda item: item.replace('"', '\\"'), selection) exec_string = self.__format_command_string(selection, command) # open selected file(s) split_command = shlex.split(exec_string, posix=False) test_command = split_command[0] if len(split_command) > 1 else exec_string if is_x_app(test_command): os.system('{0} &'.format(exec_string)) else: active_object = self._application.get_active_object() options = Parameters() options.set('close_with_child', True) options.set('shell_command', split_command[0]) options.set('arguments', split_command) options.set('path', os.path.dirname(selection[0])) self._application.create_terminal_tab(active_object._notebook, options)
folder = 'quadratic' files = os.listdir(path=f'./data/{folder}/') i = 0 for name in files: print(i) f = gzip.open(f'./data/{folder}/{name}', 'rb') net = pickle.load(f) f.close() params = Parameters(net.layers, -1, -1, -1, -1, -1, parameters=(net.weights, net.biases), training_info="migrated from old class") os.remove(f'./data/{folder}/{name}') f = gzip.open(f'./data/{folder}/{name}', 'wb') pickle.dump(params, f) f.close() i += 1
# include script as parameter so different scripts can be executed #scripts = ["optimize_top_1.py"] #length = len(scripts) #list_of_parameter_dictionaries # #TODO add script #and regularization selection parameters to parameters class, optimize script needs to be adjusted to react all_parameters = [] parameters = Parameters( starname="Wolf294", defaultQ=False, start=11, end=53, chunk_size=5, niter=160, reg_file_star= '../wobble/regularization/Wolf294_star_K0_orders[11,54)_regtest1406.hdf5', reg_file_t= '../wobble/regularization/Wolf294_t_K3_orders[11,54)_regtest1406.hdf5', output_suffix='regtest1406_opt4') parameters.dictionary.update({"script": "optimize_top_4.py"}) all_parameters.append(parameters.dictionary) #test whether new reg wll run on old files #parameters = Parameters(starname = "Wolf294", defaultQ = False, #start = 11, #end = 54, #chunk_size = 16, #reg_file_star = '../wobble/regularization/Wolf294_star_K0_orders[11,54)_regtest1406.hdf5', #reg_file_t = '../wobble/regularization/Wolf294_t_K3_orders[11,54)_regtest1406.hdf5',
import os from parameters import Parameters from scipy.io.matlab.mio import savemat params = Parameters() def saveTensorToMat(x, varName='x', fileName='', save_dir=params.net_save_dir): ''' x: is the variable to be saved name: is the name of the file and name of variable in the mat file ''' if fileName is '': fileName = varName if not os.path.exists(save_dir): os.makedirs(save_dir) savemat(save_dir + fileName + '.mat', {varName: x.cpu().data.numpy()}) def saveArrayToMat(x, varName='x', fileName='', save_dir=params.net_save_dir): ''' x: is the variable to be saved name: is the name of the file and name of variable in the mat file ''' if fileName is '': fileName = varName if not os.path.exists(save_dir): os.makedirs(save_dir)
def __init__(self): self.parameters = Parameters() self.parameter_store = ParameterStore(self.parameters) self._uuid_mapping = {}
def json_to_obj(combination_json: dict): return Combination(combination_id=combination_json['_id'], compiler_name=combination_json['compiler_name'], parameters=Parameters.json_to_obj( combination_json['parameters']))
import combine_results as cr from parameters import Parameters import numpy as np import matplotlib.pyplot as plt import tensorflow as tf import wobble from time import time import h5py import os import sys import ast import yaml #load parameters parameters = Parameters(filename="yaml_temp/optimize_parameters_chunk.yaml") parameter_dict = parameters.dictionary start_order = parameter_dict["start_chunk"] #TODO fix this name mismatch end_order = parameter_dict["end_chunk"] starname = parameter_dict["starname"] K_star = parameter_dict["K_star"] K_t = parameter_dict["K_t"] epochs_list = parameter_dict["epochs_list"] niter = parameter_dict["niter"] defaultQ = parameter_dict["defaultQ"] if True: plots = True epochs = [0, 50] # to plot movies = False
def plot2(): fig, ax = plt.subplots() ax.set_aspect('equal') ax.set_xlim(50, 250) ax.set_ylim(50, 250) # Initialize classes params = Parameters() target = Target(25, 25, 0) agents = [] for i in range(params.nveh): agents.append(Agent(0, 25 * i, 0, params.monSpeed, 0, params, ax=ax)) # Give the target a commanded speed target.send_cmd(3, 0) # Get first plan for i, agent in enumerate(agents): agent.detect_target(target.get_state()) agent.compute_flight_traj(tf=params.tflight + i * params.tmon) agent.plot_arrow() # Plot initial states pts = target.get_state() trgtPlot = ax.plot(pts[0], pts[1], 'r*', markersize=10, label='Target') for i, agent in enumerate(agents): agent.plot_arrow() # Run the simulation for t in np.arange(0, params.tflight + params.nveh * params.tmon + 0.1, 0.1): # Update states target.update(t) for agent in agents: agent.update(t) # Detect target if t % params.detPer < 1e-6: for agent in agents: agent.detect_target(target.get_state()) # Update plots pts = target.get_state() trgtPlot[0].set_data(pts[0], pts[1]) for i, agent in enumerate(agents): agent.plot_arrow() if t >= 1: target.send_cmd(3, np.pi / 2) if t >= 1.5: target.send_cmd(3, 0) plt.pause(0.01) # Plot the inner and outer radii cir1 = Circle(target.get_state()[:2], ls=':', fill=False, ec='r', label='Outer Radius', radius=params.outerR) cir2 = Circle(target.get_state()[:2], ls=':', fill=False, ec='r', label='Inner Radius', radius=params.innerR) cir3 = Circle(target.get_state()[:2], lw=None, fc='gray', alpha=0.5, label='No Fly Zone', radius=params.noflyR) ax.add_artist(cir1) ax.add_artist(cir2) ax.add_artist(cir3) # Draw legend and clean up Agent class ax.legend([trgtPlot[0]] + [agent._arrow for agent in agents], ['Target', 'Agent 1', 'Agent 2', 'Agent 3']) Agent.agentIdx = 0 Agent.trajList = [] Agent.timeList = [] plt.title(f'$t = {t}$') return
def set_parameters(**kwargs): """ Set the values of parameters input from the command line. """ logger.info("Parse arguments ...") ## Create a Parameters object to store parameters. params = Parameters() ## Process arguments. # if kwargs.get(Args.COMPARE_MODEL_NAME): params.compare_model_name = kwargs.get(Args.COMPARE_MODEL_NAME) logger.info("Compare model name: %s" % params.compare_model_name) if kwargs.get(Args.COMPARE_OUTPUT_DIRECTORY): params.compare_output_directory = kwargs.get( Args.COMPARE_OUTPUT_DIRECTORY) if not os.path.exists(params.compare_output_directory): logger.error("The compare output directory '%s' was not found." % params.compare_output_directory) return None logger.info("Compare output directory: '%s'." % params.compare_output_directory) params.data_name = kwargs.get(Args.DATA_NAME) logger.info("Data name: %s" % params.data_name) params.group = kwargs.get(Args.GROUP) logger.info("Group number: %s" % params.group) params.model_name = kwargs.get(Args.MODEL_NAME) logger.info("Model name: %s" % params.model_name) params.output_directory = kwargs.get(Args.OUTPUT_DIRECTORY) if not os.path.exists(params.output_directory): logger.error("The output directory '%s' was not found." % params.output_directory) return None logger.info("Output directory: '%s'." % params.output_directory) params.model_name = kwargs.get(Args.MODEL_NAME) logger.info("Model name: %s" % params.model_name) params.segment = kwargs.get(Args.SEGMENT) logger.info("Segment number: %s" % params.segment) if kwargs.get(Args.TIME_STEP): params.time_step = float(kwargs.get(Args.TIME_STEP)) logger.info("Time step: %g" % params.time_step) return params
import os from time import time import wobble import h5py import numpy as np import shutil import yaml #NOTE WORKS only in Wob_env_2 (wobble_19_03_2019, deprecated in wobble_14_06_2019) #TODO pass results_file_base to chunk if __name__ == "__main__": #NOTE If called via os.system this will trigger even if in queue queue = True #################### parameters if queue: parameter_filename = "yaml_temp/optimize_parameters.yaml" parameters = Parameters(filename=parameter_filename) starname = parameters.dictionary["starname"] K_star = parameters.dictionary["K_star"] K_t = parameters.dictionary["K_t"] niter = parameters.dictionary["niter"] start = parameters.dictionary["start"] end = parameters.dictionary["end"] chunk_size = parameters.dictionary["chunk_size"] defaultQ = parameters.dictionary["defaultQ"] if defaultQ: default_str = "_def" else: default_str = "" parameter_dict = parameters.dictionary
def __init__(self, nbr_of_agents): self.co = ConsoleOutput() self.co.t_print("INITIALISING CHARGING MODEL") self.parameters = Parameters() self.num_agents = nbr_of_agents # time step and time step limit in minutes self.clock = Clock(self.parameters) self.cmm = CarModelManager() self.chm = ChargerManager() self.epm = ElectricityPlanManager(self.parameters, self.clock) self.cpm = CompanyManager(self.parameters, self.clock, self.chm, self.epm) self.lrm = LocationRoadManager(self.cpm) self.wm = WhereaboutsManager(self.lrm, self.clock) self.cp = CalendarPlanner(self.parameters, self.clock, self.lrm) self.hcm = HouseConsumptionManager(self.clock) self.hgm = HouseGenerationManager(self.parameters, self.clock, self.epm, self.cmm) self.schedule_cars = RandomActivation(self) self.schedule_houses = RandomActivation(self) # extra promil is needed as east_west_spread and north_south_spread # are OPEN interval limits and agents can not be placed on this border # point self.space = ContinuousSpace(self.lrm.east_west_spread * 1.001, self.lrm.north_south_spread * 1.001, False) self.extracted_lon = [] self.extracted_lat = [] self.extracted_soc = [] # create agents self.co.t_print("Start to create agents") for agent_uid in range(self.num_agents): residency_location = self.lrm.draw_location_of_residency() house_agent = HouseAgent(agent_uid, self, self.clock, residency_location, self.chm, self.epm, self.hcm, self.hgm) employment_location = residency_location while employment_location == residency_location: employment_location = \ self.lrm.draw_location_of_employment(residency_location) company = self.cpm.add_employee_to_location(employment_location) #TODO reconsider if agents should all start at home cur_location = residency_location pos = self.lrm.relative_location_position(cur_location) car_agent = CarAgent(agent_uid, self, self.clock, cur_location, house_agent, company, self.lrm, self.cmm, self.wm, self.cp, self.parameters) self.schedule_houses.add(house_agent) self.schedule_cars.add(car_agent) self.space.place_agent(car_agent, pos) self.extracted_lon.append([]) self.extracted_lat.append([]) self.extracted_soc.append([]) self.co.t_print("Agent creation complete") self.co.t_print("INITIALISATION COMPLETE") self.co.t_print("") self.co.t_print("COMMENCING STEP CALCULATION")
params_save_path = '/net/data1/ml2017/gpyparams/E_curvelhs_'+ kernel + (str)index + times +'.pickle' params_load_path = '/net/data1/ml2017/gpyparams/E_curvelhs_'+ kernel + (str)index + times +'.pickle' #------------------------------------------------------------------------------------------ # Actual code # Training dimension, do we want to train on energy or only lecs? # This is for generation parameter_dim = LEC_LENGTH energy_as_param = False if energy[1] != energy[0]: parameter_dim += 1 energy_as_param = True # Set up necessary classes) param = Parameters(interval, samples, center_lecs=lec_center) nsopt = NsoptCaller() gauss = Gaussfit() dm = Datamanager(echo=False) gauss.save_fig = save_fig gauss.save_path = save_fig_path @profile def get_observable(energies, lecs): """Wrapper function to measure time with memory profiler.""" return nsopt.get_nsopt_observable(energies,LECM=lecs) @profile def train_gp_model(train_obs, train_lecs): """Populate and optimize GP model.""" gauss.populate_gp_model(train_obs, train_lecs, rescale=rescale_data)
def get_file_params(params_file='sailnet_parameters.txt'): return Parameters('sailnet_parameters.txt')
plus45 = 'C-Bracket_0.50_plus45WithPlate.STL' paramFile = 'Print1.json' fnames = (zero, minus45, plus45) colors = 'rgb' angles = (0, -45, 45) SecAng = namedtuple('SecAng', 'section angle') secAngs = [] meshes = [] with open(path + paramFile, 'r') as fp: data = json.load(fp) params = Parameters(data[0], data[1]) for fname, color, angle in zip(fnames, colors, angles): mesh = trimesh.load_mesh(path + fname) meshes.append(mesh) section = mesh.section(plane_origin=[0, 0, 0.01], plane_normal=[0, 0, 1]) loops = section.discrete for loop in loops: outline = Outline() outline._name = fname outline.addCoordLoop(loop * 1000) # 1000 to convert from meters to mm outline = outline.translate(-50, -12) secAngs.append(SecAng(Section(outline), angle)) plt.plot(loop[:, 0] - 0.050, loop[:, 1] - 0.012, color) params.outline = [tuple(secAngs)]
#!/usr/bin/env python import numpy import os import sys sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import config from parameters import Parameters if __name__ == '__main__': conf = config.get_config() output_folder = os.path.expanduser(conf.get('Output', 'OutputFolder')) n_folds = int(conf.get('CrossValidation', 'NumberOfFolds')) if not os.path.isdir(output_folder): raise StandardError("Experiment folder %s does not exists." % output_folder) print("Evaluating Cross-validation Scores for experiment %s..." % output_folder) parameters_files = ["parameters%i.pkl" % (x + 1) for x in xrange(n_folds)] parameters = [Parameters.load_from_pickle_file(os.path.join(output_folder, x)) for x in parameters_files] test_accuracies = numpy.array([1.0 - x.test_error for x in parameters]) print("Accuracy: %0.4f (+/- %0.4f)" % (test_accuracies.mean(), test_accuracies.std() * 2))
DATASET = 'epsilon.pickle' NUM_EPOCH = 10 print('load dataset') dataset = os.path.join(DATA_DIR, DATASET) with open(dataset, 'rb') as f: X, y = pickle.load(f) n, d = X.shape params = [] params.append( Parameters(name="full-sgd", num_epoch=NUM_EPOCH, lr_type='bottou', initial_lr=10., regularizer=1 / n, estimate='mean')) params.append( Parameters(name="top1", num_epoch=NUM_EPOCH, lr_type='bottou', initial_lr=10., regularizer=1 / n, estimate='mean', take_k=1, take_top=True, with_memory=True)) params.append( Parameters(name="rand1", num_epoch=NUM_EPOCH,
print('Starting experiment n.', str(exp)) run_folder = exp_folder + '/run_' + str(run) + '/' print('Current folder: ', run_folder) if not os.path.exists(run_folder): os.makedirs(run_folder) os.chdir(run_folder) directory_results = run_folder directory_models = run_folder + 'models/' directory_plots = run_folder + 'plots/' directory_plots_gif = run_folder + 'plots/gif/' #os.makedirs(directory_results) os.makedirs(directory_models) os.makedirs(directory_plots) os.makedirs(directory_plots_gif) # create parameters object param = Parameters() param.set('directory_results', run_folder) param.set('directory_models', run_folder + 'models/') param.set('directory_plots', run_folder + 'plots/') param.set('directory_plots_gif', run_folder + 'plots/gif/') param.set('directory_datasets', datasets_folder) dataset_type = doe.loc[exp, 'dataset_train_type'] if dataset_type == 0: param.set('dataset_train_type', 'icub_alone') param.set('directory_datasets_train', datasets_folder + 'icub_alone/') elif dataset_type == 1: param.set('dataset_train_type', 'icub_and_ball') param.set('directory_datasets_train', datasets_folder + 'icub_and_ball/') elif dataset_type == 2:
torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False param.preparedata() param.creatScheduler() param.createOptimizer() param.createLoss() for epoch in range(param.numepoch): #Train loss, acc = param.train(epoch) train_accs.append(acc) train_losses.append(loss) #val loss, acc = param.val(epoch) val_accs.append(acc) val_losses.append(loss) param.scheduler.step() if acc>best_acc: best_acc = acc print("Best model so far") torch.save(param.Net, "model.pth") f=open("log.txt", "a") f.write("RUN: BatchSize"+str(cfg.BatchSizes.BatchSize)+ " BertLayer"+ str(cfg.db.Bertlayer)+str(acc)+"\n") f.close() if __name__ == "__main__": param=Parameters() BertMainFunction()
from glob import glob from parameters import Parameters from os.path import splitext, basename input_dir = '/home/sauvan/Documents/HEP/Projects/CMS/L1CalorimeterTrigger_Phase2HGCal/Misc/FastShower/output/' parameters = Parameters() parameters.input_files = sorted([(splitext(basename(f))[0], f) for f in glob(input_dir + '*large*.root')]) parameters.tree = 'tree' parameters.log_weights = True parameters.output_file = 'output/output_largecells_logweights.root'
def set_parameters(**kwargs): """ Set the values of parameters input from the command line. """ logger.info("Parse arguments ...") ## Create a Parameters object to store parameters. params = Parameters() ## Process arguments. # if kwargs.get(Args.OUTPUT_DIRECTORY): params.output_directory = kwargs.get(Args.OUTPUT_DIRECTORY) if not os.path.exists(params.output_directory): logger.error("The output directory '%s' was not found." % params.output_directory) return None logger.info("Output directory: '%s'." % params.output_directory) if kwargs.get(Args.RESULTS_DIRECTORY): params.results_directory = kwargs.get(Args.RESULTS_DIRECTORY) if not os.path.exists(params.results_directory): logger.error("The results directory '%s' was not found." % params.results_directory) return None logger.info("Results directory: '%s'." % params.results_directory) params.output_file_name = kwargs.get(Args.OUTPUT_FILE) logger.info("Output file name: %s" % params.output_file_name) params.output_format = kwargs.get(Args.OUTPUT_FORMAT) logger.info("Output format: %s" % params.output_format) params.solver_file_name = kwargs.get(Args.SOLVER_FILE) logger.info("Solver file name: %s" % params.solver_file_name) params.data_names = kwargs.get(Args.DATA_NAMES).split(",") logger.info("Data names: %s" % ','.join(params.data_names)) if kwargs.get(Args.OUTLET_SEGMENTS): params.outlet_segments = True logger.info("Outlet segments: %s" % params.outlet_segments) if kwargs.get(Args.ALL_SEGMENTS): params.all_segments = True logger.info("All segments: %s" % params.all_segments) if kwargs.get(Args.SELECT_SEGMENTS): params.select_segment_names = True logger.info("Select segments: %s" % params.select_segment_names) if kwargs.get(Args.DISPLAY_GEOMETRY): params.display_geometry = (kwargs.get(Args.DISPLAY_GEOMETRY) in ["on", "true"]) logger.info("Display geometry: %s" % params.display_geometry) if kwargs.get(Args.NODE_SPHERE_RADIUS): params.node_sphere_radius = float(kwargs.get(Args.NODE_SPHERE_RADIUS)) if kwargs.get(Args.PLOT): params.plot_results = (kwargs.get(Args.PLOT) in ["on", "true"]) logger.info("Plot results: %s" % params.plot_results) if kwargs.get(Args.SEGMENTS): params.segment_names = kwargs.get(Args.SEGMENTS).split(",") logger.info("Segments: %s" % ','.join(params.segment_names)) if kwargs.get(Args.TIME_RANGE): params.time_range = [ float(s) for s in kwargs.get(Args.TIME_RANGE).split(",") ] logger.info("Time range: %s" % ','.join(map(str, params.time_range))) return params
######################################################################### ## ## Some utility for training, data processing, and network. ## ######################################################################### import torch import torch.nn as nn from parameters import Parameters from UNet_2Plus import UNet_2Plus p = Parameters() ###################################################################### ## ## Convolution layer modules ## ###################################################################### class Conv2D_BatchNorm_Relu(nn.Module): def __init__(self, in_channels, n_filters, k_size, padding, stride, bias=True, acti=True, dilation=1): super(Conv2D_BatchNorm_Relu, self).__init__() if acti: self.cbr_unit = nn.Sequential(
""" # Set environment and parameters if args.mux: env = environment.Multiplexer(args) elif args.chk and args.chk_div: env = environment.Checkerboard(args) elif args.mario: env = environment.MarioClassification(args) elif args.csv: env = environment.Dataset(args) else: print('Choose the environment, please.') sys.exit(1) parameters = Parameters(env.num_actions, args) helper = Helper(args, env) print("[ Settings ]") print(" Environment =", env) print(" NumIterations =", env.iteration) if env.condense_iter != 0: print("NumCondenseIter =", env.condense_iter) if env.repr == 'cs': print(" ConditionRepr = CSR") elif env.repr == 'lu': print(" ConditionRepr = OBR") elif env.repr == 'ub': print(" ConditionRepr = UBR") print("\n[ XCSR General Parameters ]")
def optimize_bollinger(self): ##values to test out ##14400 if "14400" in self.test_table_array[0]: self.bb_factor = [2.5] self.stddev_adjust = [True, False] self.avg_period = [40] self.num_past_buy = [0, 1, 2] self.num_past_sell = [2, 3, 4] self.one_op = [False] self.to_carry = [True, False] ##7200 elif "7200" in self.test_table_array[0]: self.bb_factor = [2, 2.5, 3] self.stddev_adjust = [False] self.avg_period = [60, 80, 100, 120] self.num_past_buy = [0, 1] self.num_past_sell = [2, 4, 6, 8] self.one_op = [False] self.to_carry = [False] ##1800 ##self.bb_factor = [2, 2.5, 3] ##self.stddev_adjust = [True, False] ##self.avg_period = [240, 320, 400, 480] ##self.num_past_buy = [0, 8] ##self.num_past_sell = [0 , 8, 16, 24] self.parameters_array = [] for bb in self.bb_factor: for std in self.stddev_adjust: for period in self.avg_period: for num_buy in self.num_past_buy: for num_sell in self.num_past_sell: for oo in self.one_op: for tc in self.to_carry: p = Parameters(bb, std, period, num_buy, num_sell, oo, tc) strat_array = [] for tn in self.test_table_array: strat = BollingerStrategy( tn, bb_factor=bb, stddev_adjust=std, avg_period=period, num_past_buy=num_buy, num_past_sell=num_sell, one_op=oo, to_carry=tc) strat_array.append(strat) trade_sim = TradeSimulator( self.test_table_array, strat_array, to_log=False) print( "*************************************************************************************************************************" ) print(("bb: ", bb, " std: ", std, " period: ", period, " num_buy: ", num_buy, " num_sell: ", num_sell, " one_op: ", oo, " to_carry: ", tc)) trade_sim.run() print( "*************************************************************************************************************************" ) p.set_balance(trade_sim.balance) p.set_percent_profit( trade_sim.profit_percent) self.parameters_array.append(p) self.print_summary([ "bb_factor", "stddev_adjust", "avg_period", "num_past_buy", "num_past_sell", "to_carry", "one_op" ])
#import numpy as np import dataset_readers as data import image_encoders as encoder from parameters import Parameters from nupic.research.spatial_pooler import SpatialPooler #from nupic.encoders import ScalarEncoder from vision_testbench import VisionTestBench # Get training images and convert them to vectors. trainingImages, trainingTags = data.getImagesAndTags(trainingDataset) trainingVectors = encoder.imagesToVectors(trainingImages) # Specify parameter values to search parameters = Parameters() parameters.define("dataSet", ['20.xml']) #parameters.define("dataSet",[ #'1.xml','2.xml', '3.xml', '4.xml', '5.xml', '6.xml', '7.xml', '8.xml', #'9.xml', '10.xml', '11.xml', '12.xml', '13.xml', '14.xml', '15.xml', #'16.xml', '17.xml', '18.xml', '19.xml', '20.xml', '21.xml', '22.xml', #'23.xml', '24.xml', '25.xml', '26.xml', '27.xml', '28.xml', '29.xml', #'30.xml', '31.xml', '32.xml', '33.xml', '34.xml', '35.xml', '36.xml', #'37.xml', '38.xml', '39.xml', '40.xml', '41.xml', '42.xml', '43.xml', #'44.xml', '45.xml', '46.xml', '47.xml', '48.xml', '49.xml', '50.xml', #'51.xml', '52.xml', '53.xml', '54.xml', '55.xml', '56.xml', '57.xml', #'58.xml', '59.xml', '60.xml', '61.xml', '62.xml']) parameters.define("numCols", [16]) #parameters.define("numCols", [256,512,1024,2048]) parameters.define("synPermConn", [0.3]) #parameters.define("synPermConn", [0.9, 0.7, 0.5, 0.3, 0.1])
import dataset_readers as data import image_encoders as encoder from parameters import Parameters from nupic.research.spatial_pooler import SpatialPooler from vision_testbench import VisionTestBench from classifiers import KNNClassifier minAccuracy = 200.0 maxTrainingCycles = 5 if __name__ == "__main__": # Specify parameter values to search parameters = Parameters() parameters.define("dataSet",[ '1.xml','2.xml', '3.xml', '4.xml', '5.xml', '6.xml', '7.xml', '8.xml', '9.xml', '10.xml', '11.xml', '12.xml', '13.xml', '14.xml', '15.xml', '16.xml', '17.xml', '18.xml', '19.xml', '20.xml', '21.xml', '22.xml', '23.xml', '24.xml', '25.xml', '26.xml', '27.xml', '28.xml', '29.xml', '30.xml', '31.xml', '32.xml', '33.xml', '34.xml', '35.xml', '36.xml', '37.xml', '38.xml', '39.xml', '40.xml', '41.xml', '42.xml', '43.xml', '44.xml', '45.xml', '46.xml', '47.xml', '48.xml', '49.xml', '50.xml', '51.xml', '52.xml', '53.xml', '54.xml', '55.xml', '56.xml', '57.xml', '58.xml', '59.xml', '60.xml', '61.xml', '62.xml']) # Run the model until all combinations have been tried while parameters.getNumResults() < parameters.numCombinations:
from classifiers import KNNClassifier trainingDataset = 'DataSets/OCR/characters/cmr_hex.xml' minAccuracy = 100.0 maxTrainingCycles = 5 testingDataset = 'DataSets/OCR/characters/cmr_hex.xml' if __name__ == "__main__": # Get training images and convert them to vectors. trainingImages, trainingTags = data.getImagesAndTags(trainingDataset) trainingVectors = encoder.imagesToVectors(trainingImages) # Specify parameter values to search parameters = Parameters() parameters.define("synPermConn", [0.5]) parameters.define("synPermDecFrac", [1.0, 0.5, 0.1]) parameters.define("synPermIncFrac", [1.0, 0.5, 0.1]) # Run the model until all combinations have been tried while parameters.getNumResults() < parameters.numCombinations: # Pick a combination of parameter values parameters.nextCombination() #parameters.nextRandomCombination() synPermConn = parameters.getValue("synPermConn") synPermDec = synPermConn*parameters.getValue("synPermDecFrac") synPermInc = synPermConn*parameters.getValue("synPermIncFrac")
def plot_multiple_runs(main_path, multiple_experiments_folder, num_experiments, num_runs, save=True): #data_mse_inv = [] #data_mse_fwd = [] #data_slope = [] #data_intercept = [] mean_mse_fwd = [] mean_mse_inv = [] mean_slope = [] mean_intercept = [] stddev_mse_fwd = [] stddev_mse_inv = [] stddev_slope = [] stddev_intercept = [] full_path = main_path + '/' + multiple_experiments_folder + '/' for exp in range(num_experiments): full_path_exp = full_path + 'exp' + str(exp) + '/' data_mse_inv = [] data_mse_fwd = [] data_slope = [] data_intercept = [] for run in range(num_runs): directory = full_path_exp + 'run_' + str(run) + '/' parameters = Parameters() # parameters.set('goal_selection_mode', 'som') parameters.set('run_id', run) parameters.set('directory_main', directory) parameters.set('directory_results', directory + 'results/') parameters.set('directory_plots', directory + 'plots/') data_mse_fwd.append( np.load(parameters.get('directory_results') + 'mse_fwd.npy')) data_mse_inv.append( np.load(parameters.get('directory_results') + 'mse_inv.npy')) regr = np.load( parameters.get('directory_results') + 'im_linregr_mse_vs_raw_mov.npy') data_slope.append(regr[2]) data_intercept.append(regr[4]) mean_mse_fwd.append(np.mean(np.asarray(data_mse_fwd), axis=0)) mean_mse_inv.append(np.mean(np.asarray(data_mse_inv), axis=0)) mean_slope.append(np.mean(np.asarray(data_slope), axis=0)) mean_intercept.append(np.mean(np.asarray(data_intercept), axis=0)) stddev_mse_fwd.append(np.std(np.asarray(data_mse_fwd), axis=0)) stddev_mse_inv.append(np.std(np.asarray(data_mse_inv), axis=0)) stddev_slope.append(np.std(np.asarray(data_slope), axis=0)) stddev_intercept.append(np.std(np.asarray(data_intercept), axis=0)) fig1 = plt.figure(figsize=(10, 10)) plt.title('MSE forward model') plt.ylabel('MSE') plt.xlabel('Time') #plot1, = plt.plot(mean_mse_fwd, color='#CC4F1B', label='db') #plt.fill_between(np.arange(len(mean_mse_fwd)), mean_mse_fwd-stddev_mse_fwd, mean_mse_fwd+stddev_mse_fwd, alpha=0.5, edgecolor='#CC4F1B', facecolor='#FF9848') plt.errorbar(range(len(mean_mse_fwd[-1])), mean_mse_fwd[-1], stddev_mse_fwd[-1], capsize=5, errorevery=499) if save: filename = full_path_exp + 'exp_' + str(exp) + '_mse_fwd.jpg' plt.savefig(filename) plt.close() fig1 = plt.figure(figsize=(5, 5)) plt.title('MSE inverse model') plt.ylabel('MSE') plt.xlabel('Time') #plot1, = plt.plot(mean_mse_inv, color='#CC4F1B', label='db') #plt.fill_between(np.arange(len(mean_mse_inv)), mean_mse_inv-stddev_mse_inv, mean_mse_inv+stddev_mse_inv, alpha=0.5, edgecolor='#CC4F1B', facecolor='#FF9848') plt.errorbar(range(len(mean_mse_inv[-1])), mean_mse_inv[-1], stddev_mse_inv[-1], capsize=5, errorevery=499) if save: filename = full_path_exp + 'exp_' + str(exp) + '_mse_inv.jpg' plt.savefig(filename) plt.close() fig1 = plt.figure(figsize=(5, 5)) plt.title('MSE slopes VS movements') plt.ylabel('Movement amplitude') plt.xlabel('Slope of MSE') for run in range(num_runs): x_vals = np.array(plt.xlim()) y_vals = data_intercept[run] + data_slope[run] * x_vals plt.plot(x_vals, y_vals, '-', color='b', alpha=0.5) x_vals = np.array(plt.xlim()) y_vals = mean_intercept[-1] + mean_slope[-1] * x_vals plt.plot(x_vals, y_vals, '--', color='r') if save: filename = full_path_exp + 'exp_' + str(exp) + '_correlation.jpg' plt.savefig(filename) plt.close() fig1 = plt.figure(figsize=(8, 6)) plt.ylabel('MSE forward model') #plt.ylim(0, 2) plt.xlabel('Time') for i in range(len(mean_mse_fwd)): lab = 'exp_' + str(i) plt.errorbar(range(len(mean_mse_fwd[i])), mean_mse_fwd[i], stddev_mse_fwd[i], label=lab, capsize=5, errorevery=(500 - 20 * i)) plt.legend() if save: filename = full_path + 'multiexp_mse_fwd.jpg' plt.savefig(filename) plt.close() fig1 = plt.figure(figsize=(8, 6)) plt.ylabel('MSE inverse models') #plt.ylim(0, 2) plt.xlabel('Time') for i in range(len(mean_mse_inv)): lab = 'exp_' + str(i) plt.errorbar(range(len(mean_mse_inv[i])), mean_mse_inv[i], stddev_mse_inv[i], label=lab, capsize=5, errorevery=(500 - 25 * i)) plt.legend() if save: filename = full_path + 'multiexp_mse_inv.jpg' plt.savefig(filename) plt.close()
def __init__(self): self.params = Parameters()
#parameters = Parameters(starname = "GJ436", defaultQ = False, #data_suffix = "_vis_drift_shift", #start = 11, #end = 53, #chunk_size = 5, #niter = 160, #reg_file_star = starting_star_reg, #reg_file_t = starting_t_reg, #output_suffix = 'reg_search_test') parameters = Parameters(starname="GJ436", defaultQ=False, data_suffix="_nir_drift_shift_split", start=0, end=56, chunk_size=5, niter=160, reg_file_star=starting_star_reg, reg_file_t=starting_t_reg, output_suffix='reg_search_test') parameter_filename = "yaml_temp/optimize_parameters.yaml" parameters.write(parameter_filename) parameters = Parameters(filename=parameter_filename) starname = parameters.dictionary["starname"] K_star = parameters.dictionary["K_star"] K_t = parameters.dictionary["K_t"] niter = parameters.dictionary["niter"] output_suffix = parameters.dictionary["output_suffix"] start = parameters.dictionary["start"]
class Model: """ A Model can: @type parameters: L{Parameters} @todo: Document """ def __init__(self): print 'call me' self.parameters = Parameters() if LBL: graph.output_weights = self.parameters.output_weights graph.output_biases = self.parameters.output_biases graph.score_biases = self.parameters.score_biases else: graph.hidden_weights = self.parameters.hidden_weights graph.hidden_biases = self.parameters.hidden_biases graph.output_weights = self.parameters.output_weights graph.output_biases = self.parameters.output_biases # (self.graph_train, self.graph_predict, self.graph_verbose_predict) = graph.functions(self.parameters) import sets self.train_loss = MovingAverage() self.train_err = MovingAverage() self.train_lossnonzero = MovingAverage() self.train_squashloss = MovingAverage() self.train_unpenalized_loss = MovingAverage() self.train_l1penalty = MovingAverage() self.train_unpenalized_lossnonzero = MovingAverage() self.train_correct_score = MovingAverage() self.train_noise_score = MovingAverage() self.train_cnt = 0 def __getstate__(self): return (self.parameters, self.train_loss, self.train_err, self.train_lossnonzero, self.train_squashloss, self.train_unpenalized_loss, self.train_l1penalty, self.train_unpenalized_lossnonzero, self.train_correct_score, self.train_noise_score, self.train_cnt) def __setstate__(self, state): (self.parameters, self.train_loss, self.train_err, self.train_lossnonzero, self.train_squashloss, self.train_unpenalized_loss, self.train_l1penalty, self.train_unpenalized_lossnonzero, self.train_correct_score, self.train_noise_score, self.train_cnt) = state if LBL: graph.output_weights = self.parameters.output_weights graph.output_biases = self.parameters.output_biases graph.score_biases = self.parameters.score_biases else: graph.hidden_weights = self.parameters.hidden_weights graph.hidden_biases = self.parameters.hidden_biases graph.output_weights = self.parameters.output_weights graph.output_biases = self.parameters.output_biases # def load(self, filename): # sys.stderr.write("Loading model from: %s\n" % filename) # f = myopen(filename, "rb") # (self.parameters, self.train_loss, self.train_err, self.train_lossnonzero, self.train_squashloss, self.train_unpenalized_loss, self.train_l1penalty, self.train_unpenalized_lossnonzero, self.train_correct_score, self.train_noise_score, self.train_cnt) = pickle.load(f) # if LBL: # graph.output_weights = self.parameters.output_weights # graph.output_biases = self.parameters.output_biases # graph.score_biases = self.parameters.score_biases # else: # graph.hidden_weights = self.parameters.hidden_weights # graph.hidden_biases = self.parameters.hidden_biases # graph.output_weights = self.parameters.output_weights # graph.output_biases = self.parameters.output_biases # # def save(self, filename): # sys.stderr.write("Saving model to: %s\n" % filename) # f = myopen(filename, "wb") # pickle.dump((self.parameters, self.train_loss, self.train_err, self.train_lossnonzero, self.train_squashloss, self.train_unpenalized_loss, self.train_l1penalty, self.train_unpenalized_lossnonzero, self.train_correct_score, self.train_noise_score, self.train_cnt), f) def embed(self, sequence): """ Embed a sequence of vocabulary IDs """ seq = [self.parameters.embeddings[s] for s in sequence] import numpy return [numpy.resize(s, (1, s.size)) for s in seq] # return [self.parameters.embeddings[s] for s in sequence] def embeds(self, sequences): """ Embed sequences of vocabulary IDs. If we are given a list of MINIBATCH lists of SEQLEN items, return a list of SEQLEN matrices of shape (MINIBATCH, EMBSIZE) """ embs = [] for sequence in sequences: embs.append(self.embed(sequence)) for emb in embs: assert len(emb) == len(embs[0]) new_embs = [] for i in range(len(embs[0])): colembs = [embs[j][i] for j in range(len(embs))] import numpy new_embs.append(numpy.vstack(colembs)) assert new_embs[-1].shape == (len(sequences), self.parameters.embedding_size) assert len(new_embs) == len(sequences[0]) return new_embs def corrupt_example(self, e): """ Return a corrupted version of example e, plus the weight of this example. """ from hyperparameters import HYPERPARAMETERS import random import copy e = copy.copy(e) last = e[-1] cnt = 0 while e[-1] == last: if HYPERPARAMETERS["NGRAM_FOR_TRAINING_NOISE"] == 0: e[-1] = random.randint(0, self.parameters.vocab_size-1) pr = 1./self.parameters.vocab_size elif HYPERPARAMETERS["NGRAM_FOR_TRAINING_NOISE"] == 1: import noise from common.myrandom import weighted_sample e[-1], pr = weighted_sample(noise.indexed_weights()) # from vocabulary import wordmap # print wordmap.str(e[-1]), pr else: assert 0 cnt += 1 # Backoff to 0gram smoothing if we fail 10 times to get noise. if cnt > 10: e[-1] = random.randint(0, self.parameters.vocab_size-1) weight = 1./pr return e, weight def corrupt_examples(self, correct_sequences): noise_sequences = [] weights = [] for e in correct_sequences: noise_sequence, weight = self.corrupt_example(e) noise_sequences.append(noise_sequence) weights.append(weight) return noise_sequences, weights def train(self, correct_sequences): from hyperparameters import HYPERPARAMETERS learning_rate = HYPERPARAMETERS["LEARNING_RATE"] if LBL: noise_sequences, weights = self.corrupt_examples(correct_sequences) # All weights must be the same, if we first multiply by the learning rate for w in weights: assert w == weights[0] # REWRITE FOR MINIBATCH assert 0 # noise_repr = noise_sequence[-1] # correct_repr = correct_sequence[-1] noise_repr = noise_sequence[-1:] correct_repr = correct_sequence[-1:] assert noise_repr != correct_repr assert noise_sequence[:-1] == correct_sequence[:-1] sequence = correct_sequence[:-1] # r = graph.train(self.embed(sequence), self.embed([correct_repr])[0], self.embed([noise_repr])[0], self.parameters.score_biases[correct_repr], self.parameters.score_biases[noise_repr]) r = graph.train(self.embed(sequence), self.embed(correct_repr)[0], self.embed(noise_repr)[0], self.parameters.score_biases[correct_repr], self.parameters.score_biases[noise_repr], learning_rate * weight) assert len(noise_repr) == 1 assert len(correct_repr) == 1 noise_repr = noise_repr[0] correct_repr = correct_repr[0] (loss, predictrepr, correct_score, noise_score, dsequence, dcorrect_repr, dnoise_repr, dcorrect_scorebias, dnoise_scorebias) = r # print # print "loss = ", loss # print "predictrepr = ", predictrepr # print "correct_repr = ", correct_repr, self.embed(correct_repr)[0] # print "noise_repr = ", noise_repr, self.embed(noise_repr)[0] # print "correct_score = ", correct_score # print "noise_score = ", noise_score else: noise_sequences, weights = self.corrupt_examples(correct_sequences) # All weights must be the same, if we first multiply by the learning rate for w in weights: assert w == weights[0] #print self.embeds(correct_sequences) #print self.embeds(noise_sequences) #print learning_rate * weights[0] r = graph.train(self.embeds(correct_sequences), self.embeds(noise_sequences), learning_rate * weights[0]) (dcorrect_inputss, dnoise_inputss, losss, unpenalized_losss, l1penaltys, correct_scores, noise_scores) = r # print [d.shape for d in dcorrect_inputss] # print [d.shape for d in dnoise_inputss] # print "losss", losss.shape, losss # print "unpenalized_losss", unpenalized_losss.shape, unpenalized_losss # print "l1penaltys", l1penaltys.shape, l1penaltys # print "correct_scores", correct_scores.shape, correct_scores # print "noise_scores", noise_scores.shape, noise_scores import sets to_normalize = sets.Set() for ecnt in range(len(correct_sequences)): (loss, unpenalized_loss, correct_score, noise_score) = \ (losss[ecnt], unpenalized_losss[ecnt], correct_scores[ecnt], noise_scores[ecnt]) if l1penaltys.shape == (): assert l1penaltys == 0 l1penalty = 0 else: l1penalty = l1penaltys[ecnt] correct_sequence = correct_sequences[ecnt] noise_sequence = noise_sequences[ecnt] dcorrect_inputs = [d[ecnt] for d in dcorrect_inputss] dnoise_inputs = [d[ecnt] for d in dnoise_inputss] # print [d.shape for d in dcorrect_inputs] # print [d.shape for d in dnoise_inputs] # print "loss", loss.shape, loss # print "unpenalized_loss", unpenalized_loss.shape, unpenalized_loss # print "l1penalty", l1penalty.shape, l1penalty # print "correct_score", correct_score.shape, correct_score # print "noise_score", noise_score.shape, noise_score self.train_loss.add(loss) self.train_err.add(correct_score <= noise_score) self.train_lossnonzero.add(loss > 0) squashloss = 1./(1.+math.exp(-loss)) self.train_squashloss.add(squashloss) if not LBL: self.train_unpenalized_loss.add(unpenalized_loss) self.train_l1penalty.add(l1penalty) self.train_unpenalized_lossnonzero.add(unpenalized_loss > 0) self.train_correct_score.add(correct_score) self.train_noise_score.add(noise_score) self.train_cnt += 1 if self.train_cnt % 10000 == 0: # if self.train_cnt % 1000 == 0: # print self.train_cnt # graph.COMPILE_MODE.print_summary() logging.info(("After %d updates, pre-update train loss %s" % (self.train_cnt, self.train_loss))) logging.info(("After %d updates, pre-update train error %s" % (self.train_cnt, self.train_err))) logging.info(("After %d updates, pre-update train Pr(loss != 0) %s" % (self.train_cnt, self.train_lossnonzero))) logging.info(("After %d updates, pre-update train squash(loss) %s" % (self.train_cnt, self.train_squashloss))) if not LBL: logging.info(("After %d updates, pre-update train unpenalized loss %s" % (self.train_cnt, self.train_unpenalized_loss))) logging.info(("After %d updates, pre-update train l1penalty %s" % (self.train_cnt, self.train_l1penalty))) logging.info(("After %d updates, pre-update train Pr(unpenalized loss != 0) %s" % (self.train_cnt, self.train_unpenalized_lossnonzero))) logging.info(("After %d updates, pre-update train correct score %s" % (self.train_cnt, self.train_correct_score))) logging.info(("After %d updates, pre-update train noise score %s" % (self.train_cnt, self.train_noise_score))) if LBL: i = 1. while i < wordmap.len: inti = int(i) str = "word %s, rank %d, score %f" % (wordmap.str(inti), inti, self.parameters.score_biases[inti]) logging.info("After %d updates, score biases: %s" % (self.train_cnt, str)) i *= 3.2 # print(("After %d updates, pre-update train loss %s" % (self.train_cnt, self.train_loss.verbose_string()))) # print(("After %d updates, pre-update train error %s" % (self.train_cnt, self.train_err.verbose_string()))) for w in weights: assert w == weights[0] embedding_learning_rate = HYPERPARAMETERS["EMBEDDING_LEARNING_RATE"] * weights[0] if loss == 0: if LBL: for di in dsequence + [dcorrect_repr, dnoise_repr]: # This tends to trigger if training diverges (NaN) assert (di == 0).all() else: for di in dcorrect_inputs + dnoise_inputs: assert (di == 0).all() if loss != 0: if LBL: val = sequence + [correct_repr, noise_repr] dval = dsequence + [dcorrect_repr, dnoise_repr] # print val for (i, di) in zip(val, dval): # for (i, di) in zip(tuple(sequence + [correct_repr, noise_repr]), tuple(dsequence + [dcorrect_repr, dnoise_repr])): assert di.shape[0] == 1 di.resize(di.size) # print i, di self.parameters.embeddings[i] -= 1.0 * embedding_learning_rate * di if HYPERPARAMETERS["NORMALIZE_EMBEDDINGS"]: to_normalize.add(i) for (i, di) in zip([correct_repr, noise_repr], [dcorrect_scorebias, dnoise_scorebias]): self.parameters.score_biases[i] -= 1.0 * embedding_learning_rate * di # print "REMOVEME", i, self.parameters.score_biases[i] else: for (i, di) in zip(correct_sequence, dcorrect_inputs): # assert di.shape[0] == 1 # di.resize(di.size) # print i, di assert di.shape == (self.parameters.embedding_size,) self.parameters.embeddings[i] -= 1.0 * embedding_learning_rate * di if HYPERPARAMETERS["NORMALIZE_EMBEDDINGS"]: to_normalize.add(i) for (i, di) in zip(noise_sequence, dnoise_inputs): # assert di.shape[0] == 1 # di.resize(di.size) # print i, di assert di.shape == (self.parameters.embedding_size,) self.parameters.embeddings[i] -= 1.0 * embedding_learning_rate * di if HYPERPARAMETERS["NORMALIZE_EMBEDDINGS"]: to_normalize.add(i) # print to_normalize if len(to_normalize) > 0: to_normalize = [i for i in to_normalize] # print "NORMALIZING", to_normalize self.parameters.normalize(to_normalize) def predict(self, sequence): if LBL: targetrepr = sequence[-1:] sequence = sequence[:-1] (predictrepr, score) = graph.predict(self.embed(sequence), self.embed(targetrepr)[0], self.parameters.score_biases[targetrepr], self.parameters) return score else: (score) = graph.predict(self.embed(sequence), self.parameters) return score def verbose_predict(self, sequence): if LBL: assert 0 else: (score, prehidden) = graph.verbose_predict(self.embed(sequence), self.parameters) return score, prehidden def validate(self, sequence): """ Get the rank of this final word, as opposed to all other words in the vocabulary. """ import random r = random.Random() r.seed(0) from hyperparameters import HYPERPARAMETERS import copy corrupt_sequence = copy.copy(sequence) rank = 1 correct_score = self.predict(sequence) # print "CORRECT", correct_score, [wordmap.str(id) for id in sequence] for i in range(self.parameters.vocab_size): if r.random() > HYPERPARAMETERS["PERCENT OF NOISE EXAMPLES FOR VALIDATION LOGRANK"]: continue if i == sequence[-1]: continue corrupt_sequence[-1] = i corrupt_score = self.predict(corrupt_sequence) if correct_score <= corrupt_score: # print " CORRUPT", corrupt_score, [wordmap.str(id) for id in corrupt_sequence] rank += 1 return rank
class Model: """ A Model can: @type parameters: L{Parameters} @todo: Document """ import hyperparameters import miscglobals import vocabulary def __init__(self, modelname="", window_size=HYPERPARAMETERS["WINDOW_SIZE"], vocab_size=vocabulary.wordmap().len, embedding_size=HYPERPARAMETERS["EMBEDDING_SIZE"], hidden_size=HYPERPARAMETERS["HIDDEN_SIZE"], seed=miscglobals.RANDOMSEED, initial_embeddings=None, two_hidden_layers=HYPERPARAMETERS["TWO_HIDDEN_LAYERS"]): self.modelname = modelname self.parameters = Parameters(window_size, vocab_size, embedding_size, hidden_size, seed, initial_embeddings, two_hidden_layers) if LBL: graph.output_weights = self.parameters.output_weights graph.output_biases = self.parameters.output_biases graph.score_biases = self.parameters.score_biases else: graph.hidden_weights = self.parameters.hidden_weights graph.hidden_biases = self.parameters.hidden_biases if self.parameters.two_hidden_layers: graph.hidden2_weights = self.parameters.hidden2_weights graph.hidden2_biases = self.parameters.hidden2_biases graph.output_weights = self.parameters.output_weights graph.output_biases = self.parameters.output_biases # (self.graph_train, self.graph_predict, self.graph_verbose_predict) = graph.functions(self.parameters) import sets self.train_loss = MovingAverage() self.train_err = MovingAverage() self.train_lossnonzero = MovingAverage() self.train_squashloss = MovingAverage() self.train_unpenalized_loss = MovingAverage() self.train_l1penalty = MovingAverage() self.train_unpenalized_lossnonzero = MovingAverage() self.train_correct_score = MovingAverage() self.train_noise_score = MovingAverage() self.train_cnt = 0 def __getstate__(self): return (self.modelname, self.parameters, self.train_loss, self.train_err, self.train_lossnonzero, self.train_squashloss, self.train_unpenalized_loss, self.train_l1penalty, self.train_unpenalized_lossnonzero, self.train_correct_score, self.train_noise_score, self.train_cnt) def __setstate__(self, state): (self.modelname, self.parameters, self.train_loss, self.train_err, self.train_lossnonzero, self.train_squashloss, self.train_unpenalized_loss, self.train_l1penalty, self.train_unpenalized_lossnonzero, self.train_correct_score, self.train_noise_score, self.train_cnt) = state if LBL: graph.output_weights = self.parameters.output_weights graph.output_biases = self.parameters.output_biases graph.score_biases = self.parameters.score_biases else: graph.hidden_weights = self.parameters.hidden_weights graph.hidden_biases = self.parameters.hidden_biases if self.parameters.two_hidden_layers: graph.hidden2_weights = self.parameters.hidden2_weights graph.hidden2_biases = self.parameters.hidden2_biases graph.output_weights = self.parameters.output_weights graph.output_biases = self.parameters.output_biases # def load(self, filename): # sys.stderr.write("Loading model from: %s\n" % filename) # f = myopen(filename, "rb") # (self.parameters, self.train_loss, self.train_err, self.train_lossnonzero, self.train_squashloss, self.train_unpenalized_loss, self.train_l1penalty, self.train_unpenalized_lossnonzero, self.train_correct_score, self.train_noise_score, self.train_cnt) = pickle.load(f) # if LBL: # graph.output_weights = self.parameters.output_weights # graph.output_biases = self.parameters.output_biases # graph.score_biases = self.parameters.score_biases # else: # graph.hidden_weights = self.parameters.hidden_weights # graph.hidden_biases = self.parameters.hidden_biases # graph.output_weights = self.parameters.output_weights # graph.output_biases = self.parameters.output_biases # # def save(self, filename): # sys.stderr.write("Saving model to: %s\n" % filename) # f = myopen(filename, "wb") # pickle.dump((self.parameters, self.train_loss, self.train_err, self.train_lossnonzero, self.train_squashloss, self.train_unpenalized_loss, self.train_l1penalty, self.train_unpenalized_lossnonzero, self.train_correct_score, self.train_noise_score, self.train_cnt), f) def embed(self, sequence): """ Embed a sequence of vocabulary IDs """ seq = [self.parameters.embeddings[s] for s in sequence] import numpy return [numpy.resize(s, (1, s.size)) for s in seq] # return [self.parameters.embeddings[s] for s in sequence] def embeds(self, sequences): """ Embed sequences of vocabulary IDs. If we are given a list of MINIBATCH lists of SEQLEN items, return a list of SEQLEN matrices of shape (MINIBATCH, EMBSIZE) """ embs = [] for sequence in sequences: embs.append(self.embed(sequence)) for emb in embs: assert len(emb) == len(embs[0]) new_embs = [] for i in range(len(embs[0])): colembs = [embs[j][i] for j in range(len(embs))] import numpy new_embs.append(numpy.vstack(colembs)) assert new_embs[-1].shape == (len(sequences), self.parameters.embedding_size) assert len(new_embs) == len(sequences[0]) return new_embs def train(self, correct_sequences, noise_sequences, weights): from hyperparameters import HYPERPARAMETERS learning_rate = HYPERPARAMETERS["LEARNING_RATE"] # All weights must be the same, because of how we use a scalar learning rate assert HYPERPARAMETERS["UNIFORM EXAMPLE WEIGHTS"] if HYPERPARAMETERS["UNIFORM EXAMPLE WEIGHTS"]: for w in weights: assert w == weights[0] if LBL: # REWRITE FOR MINIBATCH assert 0 # noise_repr = noise_sequence[-1] # correct_repr = correct_sequence[-1] noise_repr = noise_sequence[-1:] correct_repr = correct_sequence[-1:] assert noise_repr != correct_repr assert noise_sequence[:-1] == correct_sequence[:-1] sequence = correct_sequence[:-1] # r = graph.train(self.embed(sequence), self.embed([correct_repr])[0], self.embed([noise_repr])[0], self.parameters.score_biases[correct_repr], self.parameters.score_biases[noise_repr]) r = graph.train(self.embed(sequence), self.embed(correct_repr)[0], self.embed(noise_repr)[0], self.parameters.score_biases[correct_repr], self.parameters.score_biases[noise_repr], learning_rate * weight) assert len(noise_repr) == 1 assert len(correct_repr) == 1 noise_repr = noise_repr[0] correct_repr = correct_repr[0] (loss, predictrepr, correct_score, noise_score, dsequence, dcorrect_repr, dnoise_repr, dcorrect_scorebias, dnoise_scorebias) = r # print # print "loss = ", loss # print "predictrepr = ", predictrepr # print "correct_repr = ", correct_repr, self.embed(correct_repr)[0] # print "noise_repr = ", noise_repr, self.embed(noise_repr)[0] # print "correct_score = ", correct_score # print "noise_score = ", noise_score else: r = graph.train(self.embeds(correct_sequences), self.embeds(noise_sequences), learning_rate * weights[0]) if HYPERPARAMETERS["EMBEDDING_LEARNING_RATE"] != 0: (dcorrect_inputss, dnoise_inputss, losss, unpenalized_losss, l1penaltys, correct_scores, noise_scores) = r else: (losss, unpenalized_losss, l1penaltys, correct_scores, noise_scores) = r # print [d.shape for d in dcorrect_inputss] # print [d.shape for d in dnoise_inputss] # print "losss", losss.shape, losss # print "unpenalized_losss", unpenalized_losss.shape, unpenalized_losss # print "l1penaltys", l1penaltys.shape, l1penaltys # print "correct_scores", correct_scores.shape, correct_scores # print "noise_scores", noise_scores.shape, noise_scores import sets to_normalize = sets.Set() for ecnt in range(len(correct_sequences)): (loss, unpenalized_loss, correct_score, noise_score) = \ (losss[ecnt], unpenalized_losss[ecnt], correct_scores[ecnt], noise_scores[ecnt]) if l1penaltys.shape == (): assert l1penaltys == 0 l1penalty = 0 else: l1penalty = l1penaltys[ecnt] correct_sequence = correct_sequences[ecnt] noise_sequence = noise_sequences[ecnt] if HYPERPARAMETERS["EMBEDDING_LEARNING_RATE"] != 0: dcorrect_inputs = [d[ecnt] for d in dcorrect_inputss] dnoise_inputs = [d[ecnt] for d in dnoise_inputss] # print [d.shape for d in dcorrect_inputs] # print [d.shape for d in dnoise_inputs] # print "loss", loss.shape, loss # print "unpenalized_loss", unpenalized_loss.shape, unpenalized_loss # print "l1penalty", l1penalty.shape, l1penalty # print "correct_score", correct_score.shape, correct_score # print "noise_score", noise_score.shape, noise_score self.train_loss.add(loss) self.train_err.add(correct_score <= noise_score) self.train_lossnonzero.add(loss > 0) squashloss = 1./(1.+math.exp(-loss)) self.train_squashloss.add(squashloss) if not LBL: self.train_unpenalized_loss.add(unpenalized_loss) self.train_l1penalty.add(l1penalty) self.train_unpenalized_lossnonzero.add(unpenalized_loss > 0) self.train_correct_score.add(correct_score) self.train_noise_score.add(noise_score) self.train_cnt += 1 if self.train_cnt % 10000 == 0: # if self.train_cnt % 1000 == 0: # print self.train_cnt # graph.COMPILE_MODE.print_summary() logging.info(("After %d updates, pre-update train loss %s" % (self.train_cnt, self.train_loss.verbose_string()))) logging.info(("After %d updates, pre-update train error %s" % (self.train_cnt, self.train_err.verbose_string()))) logging.info(("After %d updates, pre-update train Pr(loss != 0) %s" % (self.train_cnt, self.train_lossnonzero.verbose_string()))) logging.info(("After %d updates, pre-update train squash(loss) %s" % (self.train_cnt, self.train_squashloss.verbose_string()))) if not LBL: logging.info(("After %d updates, pre-update train unpenalized loss %s" % (self.train_cnt, self.train_unpenalized_loss.verbose_string()))) logging.info(("After %d updates, pre-update train l1penalty %s" % (self.train_cnt, self.train_l1penalty.verbose_string()))) logging.info(("After %d updates, pre-update train Pr(unpenalized loss != 0) %s" % (self.train_cnt, self.train_unpenalized_lossnonzero.verbose_string()))) logging.info(("After %d updates, pre-update train correct score %s" % (self.train_cnt, self.train_correct_score.verbose_string()))) logging.info(("After %d updates, pre-update train noise score %s" % (self.train_cnt, self.train_noise_score.verbose_string()))) self.debug_prehidden_values(correct_sequences) if LBL: i = 1. while i < wordmap.len: inti = int(i) str = "word %s, rank %d, score %f" % (wordmap.str(inti), inti, self.parameters.score_biases[inti]) logging.info("After %d updates, score biases: %s" % (self.train_cnt, str)) i *= 3.2 # print(("After %d updates, pre-update train loss %s" % (self.train_cnt, self.train_loss.verbose_string()))) # print(("After %d updates, pre-update train error %s" % (self.train_cnt, self.train_err.verbose_string()))) # All weights must be the same, because of how we use a scalar learning rate assert HYPERPARAMETERS["UNIFORM EXAMPLE WEIGHTS"] if HYPERPARAMETERS["UNIFORM EXAMPLE WEIGHTS"]: for w in weights: assert w == weights[0] embedding_learning_rate = HYPERPARAMETERS["EMBEDDING_LEARNING_RATE"] * weights[0] if loss == 0: if LBL: for di in dsequence + [dcorrect_repr, dnoise_repr]: # This tends to trigger if training diverges (NaN) assert (di == 0).all() # if not (di == 0).all(): # print "WARNING:", di # print "WARNING in ", dsequence + [dcorrect_repr, dnoise_repr] # print "loss = ", loss # print "predictrepr = ", predictrepr # print "correct_repr = ", correct_repr, self.embed(correct_repr)[0] # print "noise_repr = ", noise_repr, self.embed(noise_repr)[0] # print "correct_score = ", correct_score # print "noise_score = ", noise_score else: if HYPERPARAMETERS["EMBEDDING_LEARNING_RATE"] != 0: for di in dcorrect_inputs + dnoise_inputs: assert (di == 0).all() if loss != 0: if LBL: val = sequence + [correct_repr, noise_repr] dval = dsequence + [dcorrect_repr, dnoise_repr] # print val for (i, di) in zip(val, dval): # for (i, di) in zip(tuple(sequence + [correct_repr, noise_repr]), tuple(dsequence + [dcorrect_repr, dnoise_repr])): assert di.shape[0] == 1 di.resize(di.size) # print i, di self.parameters.embeddings[i] -= 1.0 * embedding_learning_rate * di if HYPERPARAMETERS["NORMALIZE_EMBEDDINGS"]: to_normalize.add(i) for (i, di) in zip([correct_repr, noise_repr], [dcorrect_scorebias, dnoise_scorebias]): self.parameters.score_biases[i] -= 1.0 * embedding_learning_rate * di # print "REMOVEME", i, self.parameters.score_biases[i] else: if HYPERPARAMETERS["EMBEDDING_LEARNING_RATE"] != 0: for (i, di) in zip(correct_sequence, dcorrect_inputs): # assert di.shape[0] == 1 # di.resize(di.size) # print i, di assert di.shape == (self.parameters.embedding_size,) self.parameters.embeddings[i] -= 1.0 * embedding_learning_rate * di if HYPERPARAMETERS["NORMALIZE_EMBEDDINGS"]: to_normalize.add(i) for (i, di) in zip(noise_sequence, dnoise_inputs): # assert di.shape[0] == 1 # di.resize(di.size) # print i, di assert di.shape == (self.parameters.embedding_size,) self.parameters.embeddings[i] -= 1.0 * embedding_learning_rate * di if HYPERPARAMETERS["NORMALIZE_EMBEDDINGS"]: to_normalize.add(i) # print to_normalize if len(to_normalize) > 0: to_normalize = [i for i in to_normalize] # print "NORMALIZING", to_normalize self.parameters.normalize(to_normalize) def predict(self, sequence): if LBL: targetrepr = sequence[-1:] sequence = sequence[:-1] (predictrepr, score) = graph.predict(self.embed(sequence), self.embed(targetrepr)[0], self.parameters.score_biases[targetrepr], self.parameters) return score else: (score) = graph.predict(self.embed(sequence), self.parameters) return score def verbose_predict(self, sequence): if LBL: assert 0 else: (score, prehidden) = graph.verbose_predict(self.embed(sequence)) return score, prehidden def debug_prehidden_values(self, sequences): """ Give debug output on pre-squash hidden values. """ import numpy for (i, ve) in enumerate(sequences): (score, prehidden) = self.verbose_predict(ve) abs_prehidden = numpy.abs(prehidden) med = numpy.median(abs_prehidden) abs_prehidden = abs_prehidden.tolist() assert len(abs_prehidden) == 1 abs_prehidden = abs_prehidden[0] abs_prehidden.sort() abs_prehidden.reverse() logging.info("model %s, %s %s %s %s %s" % (self.modelname, self.train_cnt, "abs(pre-squash hidden) median =", med, "max =", abs_prehidden[:3])) if i+1 >= 3: break def validate(self, sequence): """ Get the rank of this final word, as opposed to all other words in the vocabulary. """ import random r = random.Random() r.seed(0) from hyperparameters import HYPERPARAMETERS import copy corrupt_sequence = copy.copy(sequence) rank = 1 correct_score = self.predict(sequence) # print "CORRECT", correct_score, [wordmap.str(id) for id in sequence] for i in range(self.parameters.vocab_size): if r.random() > HYPERPARAMETERS["PERCENT OF NOISE EXAMPLES FOR VALIDATION LOGRANK"]: continue if i == sequence[-1]: continue corrupt_sequence[-1] = i corrupt_score = self.predict(corrupt_sequence) if correct_score <= corrupt_score: # print " CORRUPT", corrupt_score, [wordmap.str(id) for id in corrupt_sequence] rank += 1 return rank def validate_errors(self, correct_sequences, noise_sequences): """ Count the errors in this validation batch. """ # r = graph.train(self.embeds(correct_sequences), self.embeds(noise_sequences), learning_rate * weights[0]) correct_scores = graph.predict(self.embeds(correct_sequences)) noise_scores = graph.predict(self.embeds(noise_sequences)) # print correct_scores # print noise_scores return correct_scores > noise_scores
def output_query_result(res): if res == None: print('None') else: for r in res: print(r) print('---') factory = Factory() factory.register('FsWrapper', FsWrapper()) try: p = Parameters(factory, ['?'], ['db', 'q', 'Q', 'x', 'f'], [], sys.argv[1:]) except ParamError as e: Usage('Param error: ' + e.value) if p.get_switch('?'): print('select * from Collections') print("update Collections set name='Davids' where name = 'pop'") sys.exit(0) if p.get_option('db') == None: Usage('db missing') if not os.path.isfile(p.get_option('db')): Usage('db not found') try: database = SQLiteDatabase(
def main(): """ """ sensor = options['sensor'] mapsets = options['mapsets'] prefix = options['input_prefix'] suffix = options['output_suffix'] metafile = grass.basename(options['metafile']) # 6S parameter names shortened following i.atcorr's manual atm = int(options['atmospheric_model']) # Atmospheric model [index] aer = int(options['aerosols_model']) # Aerosols model [index] vis = options['visibility_range'] # Visibility [km] aod = options['aerosol_optical_depth'] # Aerosol Optical Depth at 550nm xps = options['altitude'] # Mean Target Altitude [negative km] if not xps: msg = "Note, this value will be overwritten if a DEM raster has been "\ "defined as an input!" g.message(msg) elevation_map = options['elevation'] visibility_map = options['visibility'] radiance = flags['r'] if radiance: global rad_flg radiance_flag = 'r' else: radiance_flag = '' # If the scene to be processed was imported via the (custom) python # Landsat import script, then, Mapset name == Scene identifier mapset = grass.gisenv()['MAPSET'] if mapset == 'PERMANENT': grass.fatal(_('Please change to another mapset than the PERMANENT')) # elif 'L' not in mapset: # msg = "Assuming the Landsat scene(s) ha-s/ve been imported using the "\ # "custom python import script, the Mapset's name *should* begin "\ # "with the letter L!" # grass.fatal(_(msg)) else: result = grass.find_file(element='cell_misc', name=metafile, mapset='.') if not result['file']: grass.fatal("The metadata file <%s> is not in GRASS' data base!" % metafile) else: metafile = result['file'] # # Acquisition's metadata # msg = "Acquisition metadata for 6S code (line 2 in Parameters file)\n" # Month, day date = grass.parse_command('i.landsat.toar', flags='p', input='dummy', output='dummy', metfile=metafile, lsatmet='date') mon = int(date['date'][5:7]) # Month of acquisition day = int(date['date'][8:10]) # Day of acquisition # GMT in decimal hours gmt = grass.read_command('i.landsat.toar', flags='p', input='dummy', output='dummy', metfile=metafile, lsatmet='time') gmt = float(gmt.rstrip('\n')) # Scene's center coordinates cll = grass.parse_command('g.region', flags='clg') lon = float(cll['center_long']) # Center Longitude [decimal degrees] lat = float(cll['center_lat']) # Center Latitude [decimal degrees] msg += str(mon) + ' ' + str(day) + ' ' + str(gmt) + ' ' + \ str(lon) + ' ' + str(lat) g.message(msg) # # AOD # if aod: aod = float(options['aerosol_optical_depth']) else: # sane defaults if 4 < mon < 10: aod = float(0.222) # summer else: aod = float(0.111) # winter # # Mapsets are Scenes. Read'em all! # if mapsets == 'all': scenes = grass.mapsets() elif mapsets == 'current': scenes = [mapset] else: scenes = mapsets.split(',') if 'PERMANENT' in scenes: scenes.remove('PERMANENT') # access only to specific mapsets! msg = "\n|* Performing atmospheric correction for scenes: %s" % scenes g.message(msg) for scene in scenes: # ensure access only to *current* mapset run('g.mapsets', mapset='.', operation='set') # scene's basename as in GRASS' db basename = grass.read_command('g.mapset', flags='p') msg = " | Processing scene: %s" % basename g.message(msg) # loop over Landsat bands in question for band in sensors[sensor].keys(): inputband = prefix + str(band) msg = '\n>>> Processing band: {band}'.format(band=inputband) g.message(msg) # Generate 6S parameterization file p6s = Parameters(geo=geo[sensor], mon=mon, day=day, gmt=gmt, lon=lon, lat=lat, atm=atm, aer=aer, vis=vis, aod=aod, xps=xps, xpp=xpp, bnd=sensors[sensor][band]) # # Temporary files # tmpfile = grass.tempfile() tmp = "tmp." + grass.basename(tmpfile) # use its basename tmp_p6s = grass.tempfile() # 6S Parameters ASCII file tmp_atm_cor = "%s_cor_out" % tmp # Atmospherically Corrected Img p6s.export_ascii(tmp_p6s) # Process band-wise atmospheric correction with 6s msg = "6S parameters:\n\n" msg += p6s.parameters g.message(msg) # inform about input's range? input_range = grass.parse_command('r.info', flags='r', map=inputband) input_range['min'] = float(input_range['min']) input_range['max'] = float(input_range['max']) msg = "Input range: %.2f ~ %.2f" % (input_range['min'], input_range['max']) g.message(msg) # # Applying 6S Atmospheric Correction algorithm # run_i_atcorr(radiance_flag, inputband, input_range, elevation_map, visibility_map, tmp_p6s, tmp_atm_cor, (0,1)) # inform about output's range? output_range = grass.parse_command('r.info', flags='r', map=tmp_atm_cor) output_range['min'] = float(output_range['min']) output_range['max'] = float(output_range['max']) msg = "Output range: %.2f ~ %.2f" \ % (output_range['min'], output_range['max']) g.message(msg) # add suffix to basename & rename end product atm_cor_nam = ("%s%s.%s" % (prefix, suffix, band)) run('g.rename', rast=(tmp_atm_cor, atm_cor_nam))
def test_epoch(self): x = np.array([[1, i] for i in range(10)]) y = np.array([2*i + 10 for i in range(10)]) gd = GradientDescent(Parameters(batch_size=4, learning_rate=1, decay=0.5), linear_regression_hypothesis) gd.epoch((x, y), np.zeros(2)) self.assertEqual(gd.parameters.learning_rate, 0.5)
index=True, header=True) # for each row in the design of the experiment table for exp in range(doe.shape[0]): # repeat it for number_of_runs times for run in range(number_of_runs): print('Starting experiment n.', str(exp)) directory = main_path + '/' + multiple_experiments_folder + '/exp' + str( exp) + '/run_' + str(run) + '/' if not os.path.exists(directory): os.makedirs(directory) parameters = Parameters() #parameters.set('goal_selection_mode', 'som') parameters.set('doe_experiment_id', exp) parameters.set('run_id', run) romi_dataset_folder = main_path + '/romi_data/' parameters.set('directory_romi_dataset', romi_dataset_folder) parameters.set('directory_main', directory) parameters.set('directory_models', directory + 'models/') parameters.set('directory_pretrained_models', main_path + '/pretrained_models/') parameters.set('directory_results', directory + 'results/') parameters.set('directory_plots', directory + 'plots/') # design of experiments
MAX_DOCUMENT_LENGTH = 300 EMBEDDING_SIZE = 5 HIDDEN_SIZE1 = 4 HIDDEN_SIZE2 = 4 ATTENTION_SIZE = 2 lr = 1e-4 BATCH_SIZE = 256 KEEP_PROB = 0.5 LAMBDA = 0.0001 MAX_LABEL = 2 epochs = 200 #dbpedia = tf.contrib.learn.datasets.load_dataset('dbpedia') parameters = Parameters() parameters.add_parameter("METHOD", "O-SVM") parameters.add_parameter("MAX_DOCUMENT_LENGTH", MAX_DOCUMENT_LENGTH) parameters.add_parameter("EMBEDDING_SIZE", EMBEDDING_SIZE) parameters.add_parameter("HIDDEN_SIZE1", HIDDEN_SIZE1) parameters.add_parameter("HIDDEN_SIZE2", HIDDEN_SIZE2) parameters.add_parameter("lr", lr) parameters.add_parameter("BATCH_SIZE", BATCH_SIZE) parameters.add_parameter("KEEP_PROB", KEEP_PROB) parameters.add_parameter("LAMBDA", LAMBDA) parameters.add_parameter("MAX_LABEL", MAX_LABEL) parameters.add_parameter("epochs", epochs) # load data x_train, y_train = ( [], []
from nupic.research.spatial_pooler import SpatialPooler from vision_testbench import VisionTestBench from classifiers import KNNClassifier trainingDataset = 'DataSets/OCR/characters/cmr_hex.xml' minAccuracy = 100.0 maxTrainingCycles = 5 testingDataset = 'DataSets/OCR/characters/cmr_hex.xml' if __name__ == "__main__": # Get training images and convert them to vectors. trainingImages, trainingTags = data.getImagesAndTags(trainingDataset) trainingVectors = encoder.imagesToVectors(trainingImages) # Specify parameter values to search parameters = Parameters() parameters.define("synPermConn", [0.5]) parameters.define("synPermDecFrac", [1.0, 0.5, 0.1]) parameters.define("synPermIncFrac", [1.0, 0.5, 0.1]) # Run the model until all combinations have been tried while parameters.getNumResults() < parameters.numCombinations: # Pick a combination of parameter values parameters.nextCombination() #parameters.nextRandomCombination() synPermConn = parameters.getValue("synPermConn") synPermDec = synPermConn * parameters.getValue("synPermDecFrac") synPermInc = synPermConn * parameters.getValue("synPermIncFrac") # Instantiate our spatial pooler
NUM_EPOCH = 3 print('load dataset') dataset = os.path.join(DATA_DIR, DATASET) with open(dataset, 'rb') as f: X, y = pickle.load(f) n, d = X.shape params = [] params.append( Parameters(name="full-sgd", num_epoch=NUM_EPOCH, lr_type='decay', initial_lr=2, tau=d, regularizer=1 / n, estimate='(t+tau)^2')) params.append( Parameters(name="full-sgd-no-shift", num_epoch=NUM_EPOCH, lr_type='decay', initial_lr=2, tau=1, regularizer=1 / n, estimate='(t+tau)^2')) params.append( Parameters(name="top1-no-memory", num_epoch=NUM_EPOCH, lr_type='decay',