def wrapper(self, *args, **kwargs): result = getattr(self, new_method_name)(*args, **kwargs) # pickle biflow filename = (self._snapshot_name_ + "_%d" % self._snapshot_counter_ + PICKLE_EXT) robust_pickle(self._snapshot_path_, filename, self) self._snapshot_counter_ += 1 return result
def show_training(flow, data_iterables, msg_iterables=None, stop_messages=None, path=None, tracer=None, debug=False, show_size=False, open_browser=True, **kwargs): """Perform both the flow training and the training inspection. The return value is the filename of the slideshow HTML file. This function must be used with the untrained flow (no previous call of Flow.train is required, the training happens here). This function is more convenient than inspect_training since it includes all required steps, but it is also less customizable. After everything is complete the inspection slideshow is opened in the browser. flow -- The untrained Flow or BiFlow. After this function has been called the flow will be fully trained. data_iterables, msg_iterables, stop_messages -- Same as for calling train on a flow. path -- Path were both the training snapshots and the inspection slides will be stored. If None (default value) a temporary directory will be used. tracer -- Instance of InspectionHTMLTracer, can be None for default class. debug -- Ignore exception during training and try to complete the slideshow (default value is False). show_size -- Show the approximate memory footprint of all nodes. open_browser -- If True (default value) then the slideshow file is automatically opened in a webbrowser. One can also use string value with the browser name (for webbrowser.get) to request a specific browser. **kwargs -- Additional arguments for flow.train can be specified as keyword arguments. """ if path is None: path = tempfile.mkdtemp(prefix='MDP_') # get first part of data iterators as sample data for inspection # if data_iterables is an array, wrap it up in a list if isinstance(data_iterables, numx.ndarray): data_iterables = [[data_iterables]] * len(flow) x_samples = [] for i, data_iterable in enumerate(data_iterables): if data_iterable is None: x_sample, new_data_iterable = None, None else: x_sample, new_data_iterable = first_iterable_elem(data_iterable) x_samples.append(x_sample) data_iterables[i] = new_data_iterable del x_sample if msg_iterables: msg_samples = [] for i, msg_iterable in enumerate(msg_iterables): if msg_iterable is None: msg_sample, new_msg_iterable = None, None else: msg_sample, new_msg_iterable = first_iterable_elem(msg_iterable) msg_samples.append(msg_sample) msg_iterables[i] = new_msg_iterable del msg_sample else: msg_samples = None # store the data to disk to disk to save memory and safeguard against # any change made to the data during the training robust_pickle(path, "training_data_samples.pckl", (x_samples, msg_samples, stop_messages)) del x_samples del msg_samples # perform the training and gather snapshots prepare_training_inspection(flow=flow, path=path) try: if isinstance(flow, BiFlow): flow.train(data_iterables, msg_iterables, stop_messages, **kwargs) else: flow.train(data_iterables, **kwargs) except Exception: if debug: traceback.print_exc() print ("exception during training, " + "inspecting up to failure point...") # create the last snapshot manually try: # if a normal mdp.Flow instance was given then this fails flow._bi_reset() except Exception: pass filename = (flow._snapshot_name_ + "_%d" % flow._snapshot_counter_ + PICKLE_EXT) robust_pickle(flow._snapshot_path_, filename, flow) else: raise remove_inspection_residues(flow) # reload data samples with open(os.path.join(path, "training_data_samples.pckl"), "rb") as sample_file: x_samples, msg_samples, stop_messages = pickle.load(sample_file) # create slideshow slideshow = inspect_training(snapshot_path=path, inspection_path=path, x_samples=x_samples, msg_samples=msg_samples, stop_messages=stop_messages, tracer=tracer, debug=debug, show_size=show_size, verbose=False) filename = os.path.join(path, "training_inspection.html") title = "Training Inspection" with open(filename, 'w') as html_file: html_file.write('<html>\n<head>\n<title>%s</title>\n' % title) html_file.write('<style type="text/css" media="screen">') html_file.write(standard_css()) html_file.write('</style>\n</head>\n<body>\n') html_file.write('<h3>%s</h3>\n' % title) html_file.write(slideshow) html_file.write('</body>\n</html>') if open_browser: _open_custom_brower(open_browser, os.path.abspath(filename)) return filename
def show_training(flow, data_iterables, msg_iterables=None, stop_messages=None, path=None, tracer=None, debug=False, show_size=False, open_browser=True, **kwargs): """Perform both the flow training and the training inspection. The return value is the filename of the slideshow HTML file. This function must be used with the untrained flow (no previous call of Flow.train is required, the training happens here). This function is more convenient than inspect_training since it includes all required steps, but it is also less customizable. After everything is complete the inspection slideshow is opened in the browser. flow -- The untrained Flow or BiFlow. After this function has been called the flow will be fully trained. data_iterables, msg_iterables, stop_messages -- Same as for calling train on a flow. path -- Path were both the training snapshots and the inspection slides will be stored. If None (default value) a temporary directory will be used. tracer -- Instance of InspectionHTMLTracer, can be None for default class. debug -- Ignore exception during training and try to complete the slideshow (default value is False). show_size -- Show the approximate memory footprint of all nodes. open_browser -- If True (default value) then the slideshow file is automatically opened in a webbrowser. One can also use string value with the browser name (for webbrowser.get) to request a specific browser. **kwargs -- Additional arguments for flow.train can be specified as keyword arguments. """ if path is None: path = tempfile.mkdtemp(prefix='MDP_') # get first part of data iterators as sample data for inspection # if data_iterables is an array, wrap it up in a list if isinstance(data_iterables, numx.ndarray): data_iterables = [[data_iterables]] * len(flow) x_samples = [] for i, data_iterable in enumerate(data_iterables): if data_iterable is None: x_sample, new_data_iterable = None, None else: x_sample, new_data_iterable = first_iterable_elem(data_iterable) x_samples.append(x_sample) data_iterables[i] = new_data_iterable del x_sample if msg_iterables: msg_samples = [] for i, msg_iterable in enumerate(msg_iterables): if msg_iterable is None: msg_sample, new_msg_iterable = None, None else: msg_sample, new_msg_iterable = first_iterable_elem( msg_iterable) msg_samples.append(msg_sample) msg_iterables[i] = new_msg_iterable del msg_sample else: msg_samples = None # store the data to disk to disk to save memory and safeguard against # any change made to the data during the training robust_pickle(path, "training_data_samples.pckl", (x_samples, msg_samples, stop_messages)) del x_samples del msg_samples # perform the training and gather snapshots prepare_training_inspection(flow=flow, path=path) try: if isinstance(flow, BiFlow): flow.train(data_iterables, msg_iterables, stop_messages, **kwargs) else: flow.train(data_iterables, **kwargs) except Exception: if debug: traceback.print_exc() print("exception during training, " + "inspecting up to failure point...") # create the last snapshot manually try: # if a normal mdp.Flow instance was given then this fails flow._bi_reset() except Exception: pass filename = (flow._snapshot_name_ + "_%d" % flow._snapshot_counter_ + PICKLE_EXT) robust_pickle(flow._snapshot_path_, filename, flow) else: raise remove_inspection_residues(flow) # reload data samples with open(os.path.join(path, "training_data_samples.pckl"), "rb") as sample_file: x_samples, msg_samples, stop_messages = pickle.load(sample_file) # create slideshow slideshow = inspect_training(snapshot_path=path, inspection_path=path, x_samples=x_samples, msg_samples=msg_samples, stop_messages=stop_messages, tracer=tracer, debug=debug, show_size=show_size, verbose=False) filename = os.path.join(path, "training_inspection.html") title = "Training Inspection" with open(filename, 'w') as html_file: html_file.write('<html>\n<head>\n<title>%s</title>\n' % title) html_file.write('<style type="text/css" media="screen">') html_file.write(standard_css()) html_file.write('</style>\n</head>\n<body>\n') html_file.write('<h3>%s</h3>\n' % title) html_file.write(slideshow) html_file.write('</body>\n</html>') if open_browser: _open_custom_brower(open_browser, os.path.abspath(filename)) return filename