def interactive_refilt(self): """Launch an interactive prompt to design REGEX filters for batch operation.""" if self._bad_file: new_ph = ParamHandler(params=self.ph["mapping"]) else: new_ph = ParamHandler(in_loc=self.ph["mapping_file"]) regex_filts, _ = new_ph.interactive_refilt(self.in_dir, self.ph["regex_filters"]) neuro_file = open(self.file_loc, "r") temp_file = open("temp.txt", "w") for line in neuro_file: if line.startswith("regex_filters ="): line = "regex_filters = " + str(regex_filts) + "\n" print("Set the regex filters to: " + line) temp_file.write(line) elif line.startswith("interactive ="): line = "interactive = False\n" temp_file.write(line) else: temp_file.write(line) neuro_file.close() temp_file.close() os.remove(self.file_loc) shutil.move("temp.txt", self.file_loc) self.setup()
def setup(self): """Call on initialisation.""" self.ph = ParamHandler( in_loc=self.file_loc, name="params", dirname_replacement=self.dirname_replacement, ) self._bad_file = self.ph["mapping_file"] is None if not self._bad_file: self._bad_file = not os.path.isfile(self.ph["mapping_file"]) if self.ph["mapping"] == {} and self._bad_file: raise ValueError("Please pass either a valid mapping file " + "or a parameter mapping, " + "currently:\n {} and {} respectively\n".format( self.ph["mapping_file"], self.ph["mapping"]))
def test_param_load(): params = {"hello_world": "banana", 0: [1, 10, 14.1], "chans": {"1": "b"}} ph_write = ParamHandler(params=params) ph_write.write("test_simuran_temp.py") ph_read = ParamHandler() ph_read.read("test_simuran_temp.py") assert ph_read.params["hello_world"] == "banana" assert ph_read.params["0"] == [1, 10, 14.1] assert ph_read["chans"]["1"] == "b" os.remove("test_simuran_temp.py")
def _setup_from_dict(self, params, load=True): """ Set up this recording from a dictionary of parameters. Parameters ---------- params : dict The parameters to setup from. load : bool, optional Whether the information should be loaded, by default True Returns ------- None """ self.param_handler = ParamHandler(params=params) self._setup(load=load)
def _setup_from_file(self, param_file, load=True): """ Set up this recording from a source parameter file. Parameters ---------- param_file : str The parameter file to setup from. load : bool, optional Whether the information should be loaded, by default True Returns ------- None """ self.param_handler = ParamHandler(in_loc=param_file) self._setup(load=load)
def clean(): run_dict_ = ParamHandler(in_loc=batch_file, name="params") for run_dict_i in run_dict_["run_list"]: function_loc_ = os.path.abspath(run_dict_i["fn_param_loc"]) to_remove = os.path.join( batch_file, "..", "sim_results", os.path.splitext(os.path.basename(function_loc_))[0], ) if os.path.isdir(to_remove): print("Removing folder {}".format(to_remove)) shutil.rmtree(to_remove) for fname_ in targets: if os.path.isfile(fname_): print("Removing file {}".format(fname_)) os.remove(fname_)
def write_batch_params(self, verbose_params=False, verbose=False): """ Write parameters to the established locations in setup. If verbose_params is True, prints the files that would be written to. Parameters ---------- verbose_params : bool, optional Whether to write the parameters in short or long format, default is False. E.g. if params = [1, 2, 3] * 2, verbose_params set to True would write [1, 2, 3, 1, 2, 3], while false would write params = [1, 2, 3] * 2 verbose : bool, optional Whether to print out extra information during execution, default is False. Returns ------- list A list of directories that were written to. Raises ------ ValueError If a non-existent file is attempted to be written. Can only occur in non verbose_params mode. """ check_only = self.ph["only_check"] or self.only_check overwrite = self.ph["overwrite"] re_filts = self.ph["regex_filters"] delete_old_files = self.ph.get("delete_old_files", False) if delete_old_files and not check_only: BatchSetup.clear_params(self.in_dir, to_remove=self.ph["out_basename"]) if verbose_params and not check_only: if self._bad_file: new_ph = ParamHandler(params=self.ph["mapping"]) else: new_ph = ParamHandler(in_loc=self.ph["mapping_file"]) dirs = new_ph.batch_write( self.in_dir, re_filters=re_filts, fname=self.ph["out_basename"], check_only=check_only, overwrite=overwrite, verbose=verbose, ) else: fname = self.ph["mapping_file"] if self._bad_file: raise ValueError( "Can't copy non-existant file {}".format(fname)) dirs = self.ph.batch_write( self.in_dir, re_filters=re_filts, fname=self.ph["out_basename"], check_only=check_only, overwrite=overwrite, exact_file=fname, verbose=verbose, ) return dirs
class BatchSetup(object): """ Help managing parameters for running batch operations. Parameters ---------- in_dir : str Sets attribute in_dir. fpath : str Either the full path to a parameter file, or just the name of the parameter file relative to in_dir. This should describe how batch_setup will behave. ph : simuran.params.ParamHandler A param handler that holds the configuration. dirname_replacement : str What to replace "__dirname__" by in config files Attributes ---------- in_dir : str The path to the directory to start batch operations in. fpath : str The path to the parameter file describing the behaviour. only_check : bool Whether files should actually be written out, or just have the paths that would be written to printed out. dirname_replacement : str What to replace "__dirname__" by in config files _bad_file : bool True if the file passed is not valid, or not passed. Example ------- .. highlight:: python .. code-block:: python batch_setup = BatchSetup(directory, fpath="params.py") batch_setup.interactive_refilt() print(batch_setup) batch_setup.write_batch_params() """ def __init__(self, in_dir, fpath="simuran_batch_params.py", dirname_replacement=""): """See help(BatchSetup).""" super().__init__() self.in_dir = in_dir if not os.path.isfile(fpath): if os.path.isfile(os.path.join(self.in_dir, fpath)): self.file_loc = os.path.join(self.in_dir, fpath) else: self.file_loc = fpath self.dirname_replacement = dirname_replacement self.setup() self.only_check = False def set_only_check(self, val): """ Set the value of only_check. Parameters ---------- val : bool The value to set. Returns ------- None """ self.only_check = val def setup(self): """Call on initialisation.""" self.ph = ParamHandler( in_loc=self.file_loc, name="params", dirname_replacement=self.dirname_replacement, ) self._bad_file = self.ph["mapping_file"] is None if not self._bad_file: self._bad_file = not os.path.isfile(self.ph["mapping_file"]) if self.ph["mapping"] == {} and self._bad_file: raise ValueError("Please pass either a valid mapping file " + "or a parameter mapping, " + "currently:\n {} and {} respectively\n".format( self.ph["mapping_file"], self.ph["mapping"])) def start(self): """Start the main control function.""" if self.ph["interactive"]: print("Interactive mode selected") self.interactive_refilt() self.write_batch_params(verbose_params=True) def interactive_refilt(self): """Launch an interactive prompt to design REGEX filters for batch operation.""" if self._bad_file: new_ph = ParamHandler(params=self.ph["mapping"]) else: new_ph = ParamHandler(in_loc=self.ph["mapping_file"]) regex_filts, _ = new_ph.interactive_refilt(self.in_dir, self.ph["regex_filters"]) neuro_file = open(self.file_loc, "r") temp_file = open("temp.txt", "w") for line in neuro_file: if line.startswith("regex_filters ="): line = "regex_filters = " + str(regex_filts) + "\n" print("Set the regex filters to: " + line) temp_file.write(line) elif line.startswith("interactive ="): line = "interactive = False\n" temp_file.write(line) else: temp_file.write(line) neuro_file.close() temp_file.close() os.remove(self.file_loc) shutil.move("temp.txt", self.file_loc) self.setup() def write_batch_params(self, verbose_params=False, verbose=False): """ Write parameters to the established locations in setup. If verbose_params is True, prints the files that would be written to. Parameters ---------- verbose_params : bool, optional Whether to write the parameters in short or long format, default is False. E.g. if params = [1, 2, 3] * 2, verbose_params set to True would write [1, 2, 3, 1, 2, 3], while false would write params = [1, 2, 3] * 2 verbose : bool, optional Whether to print out extra information during execution, default is False. Returns ------- list A list of directories that were written to. Raises ------ ValueError If a non-existent file is attempted to be written. Can only occur in non verbose_params mode. """ check_only = self.ph["only_check"] or self.only_check overwrite = self.ph["overwrite"] re_filts = self.ph["regex_filters"] delete_old_files = self.ph.get("delete_old_files", False) if delete_old_files and not check_only: BatchSetup.clear_params(self.in_dir, to_remove=self.ph["out_basename"]) if verbose_params and not check_only: if self._bad_file: new_ph = ParamHandler(params=self.ph["mapping"]) else: new_ph = ParamHandler(in_loc=self.ph["mapping_file"]) dirs = new_ph.batch_write( self.in_dir, re_filters=re_filts, fname=self.ph["out_basename"], check_only=check_only, overwrite=overwrite, verbose=verbose, ) else: fname = self.ph["mapping_file"] if self._bad_file: raise ValueError( "Can't copy non-existant file {}".format(fname)) dirs = self.ph.batch_write( self.in_dir, re_filters=re_filts, fname=self.ph["out_basename"], check_only=check_only, overwrite=overwrite, exact_file=fname, verbose=verbose, ) return dirs @staticmethod def clear_params(start_dir, to_remove="simuran_params.py", recursive=True, verbose=False): """ Remove all files with the name to_remove from start_dir. Parameters ---------- start_dir : str Where to start removing files from to_remove : str, optional The name of the files to remove, by default "simuran_params.py" recursive : bool, optional Whether to recursive through child directories, by default True verbose : bool, optional Whether to print the files that were deleted, by default False Returns ------- None """ fnames = BatchSetup.get_param_locations(start_dir, to_find=to_remove, recursive=recursive) for fname in fnames: if os.path.basename(fname) == to_remove: if verbose: print("Removing {}".format(fname)) os.remove(fname) @staticmethod def get_param_locations(start_dir, to_find="simuran_params.py", recursive=True): """ Find all directories that have to_find in them. Parameters ---------- start_dir : str Where to start the search from. to_find : str, optional What filename to find, by default "simuran_params.py" recursive : bool, optional Whether to recurse through child directories, by default True Returns ------- list Paths to each of the files found. """ fnames = get_all_files_in_dir(start_dir, ext=".py", recursive=recursive) def keep_file(filename): return ("__pycache__" not in filename) and (os.path.basename(filename) == to_find) return [fname for fname in fnames if keep_file(fname)] @staticmethod def get_params_matching_pattern(start_dir, re_filter=".*simuran.*params", recursive=True): """ Find all files matching a regex pattern. Parameters ---------- start_dir : str Where to start the search from. re_filter : str, optional A regular expression pattern, by default ".*simuran.*params" recursive : bool, optional Whether to recurse into subdirectories, by default True Returns ------- list Paths to the files found. """ fnames = get_all_files_in_dir(start_dir, ext=".py", recursive=recursive, re_filter=re_filter) def keep_file(filename): return "__pycache__" not in filename return [fname for fname in fnames if keep_file(fname)] @staticmethod def copy_params(from_dir, to_dir, re_filter=".*simuran.*params", recursive=True, test_only=False): """ Copy all parameters matching a regex between directories. Parameters ---------- from_dir : str Path to the directory to copy from. to_dir : str Path to the directory to copy to. re_filter : str, optional A regular expression pattern, by default ".*simuran.*params" recursive : bool, optional Whether to recurse into subdirectories, by default True test_only : bool, optional If True, it only prints what would be copied, by default False Returns ------- None """ files = BatchSetup.get_params_matching_pattern(from_dir, re_filter=re_filter, recursive=recursive) for f in files: file_without_base = f[len(from_dir + os.sep):] dest = os.path.join(to_dir, file_without_base) if test_only: print("Would copy {} to {}".format(f, dest)) else: os.makedirs(os.path.dirname(dest), exist_ok=True) shutil.copy(f, dest) def __str__(self): """Call on print.""" return "{} from {} with parameters:\n {} ".format( self.__class__.__name__, self.file_loc, pformat(self.ph.params, width=200))
def create_task(batch_file, analysis_functions=[], num_workers=1, dirname=""): """ Create a doit task. Parameters ---------- batch_file : str The path to file to use for batch running SIMURAN. analysis_functions : list, optional List of strings of python files containing dependent functions, by default [] num_workers : int, optional The number of workers to use for the task, by default 1 dirname : str, optional The directory name to replace __dirname__ by in SIMURAN. Returns ------- dict A python doit dictionary. """ modify_path( os.path.abspath( os.path.join(os.path.dirname(batch_file), "..", "analysis")), verbose=False, ) run_dict = ParamHandler(in_loc=batch_file, name="params", dirname_replacement="") dependencies = [] fnames = [] for run_dict in run_dict["run_list"]: batch_param_loc = os.path.abspath(run_dict["batch_param_loc"]) if batch_param_loc not in dependencies: in_dir = os.path.dirname(batch_param_loc) dependencies.append(batch_param_loc) new_params = ParamHandler(in_loc=batch_param_loc, name="params") if new_params["mapping_file"] not in dependencies: dependencies.append(new_params["mapping_file"]) function_loc = os.path.abspath(run_dict["fn_param_loc"]) if function_loc not in dependencies: dependencies.append(function_loc) new_params = ParamHandler(in_loc=function_loc, name="fn_params") functions = new_params["run"] for fn in functions: if not isinstance(fn, (tuple, list)): if fn.__name__ not in fnames: fnames.append(fn.__name__) for fname in analysis_functions: path = os.path.abspath( os.path.join(os.path.dirname(in_dir), "..", "analysis", fname)) if os.path.isfile(path): dependencies.append(path) targets = [ os.path.join( batch_file, "..", "sim_results", "pickles", os.path.splitext(os.path.basename(batch_file))[0] + "_dump.pickle", ) ] if dirname != "": action = "simuran -r -m -o -n {} --dirname {} {}".format( num_workers, dirname, batch_file) else: action = "simuran -r -m -o -n {} {}".format(num_workers, batch_file) def clean(): run_dict_ = ParamHandler(in_loc=batch_file, name="params") for run_dict_i in run_dict_["run_list"]: function_loc_ = os.path.abspath(run_dict_i["fn_param_loc"]) to_remove = os.path.join( batch_file, "..", "sim_results", os.path.splitext(os.path.basename(function_loc_))[0], ) if os.path.isdir(to_remove): print("Removing folder {}".format(to_remove)) shutil.rmtree(to_remove) for fname_ in targets: if os.path.isfile(fname_): print("Removing file {}".format(fname_)) os.remove(fname_) return { "file_dep": dependencies, "targets": targets, "actions": [action], "clean": [clean], "title": title_with_actions, "verbosity": 0, "doc": action, }
class Recording(BaseSimuran): """ Describe a full recording session and holds data. This holds single unit information, spatial information, EEG information, stimulation information, and event information. Note that anything you want to store on this object that has not been accounted for in attributes, then this can be stored on self.info Attributes ---------- signals : list of simuran.base_signal.BaseSignal The signals in the recording. units : list of simuran.single_unit.SingleUnit The single units in the recording. spatial : list of simuran.spatial.Spatial The spatial information in the recording. stimulation : list of TODO The stimulation, events, and stimuli in the recording. available : list of strings Each value in the list should be present if the information is available. E.g. available = ["signals", "spatial"] would indicate that this recording has EEG information and spatial information only. param_handler : simuran.param_handler.ParamHandler Parameters which describe the information in the recording. source_file : str The path to the underlying source file describing the recording. When recordings have many source files, this should be either the directory where they are all located, or a file listing them. source_files : dict A dictionary describing the source files for each attribute. Parameters ---------- params : dict, optional Direct parameters which describe the recording, default is None See simuran.params.simuran_base_params for what can be passed. param_file : str, optional The path to a file which contains parameters, default is None base_file : str, optional Sets the value of self.source_file, default is None load : bool, optional Whether to load the recording on initialisation, default is True See also -------- simuran.base_class.BaseSimuran """ def __init__(self, params=None, param_file=None, base_file=None, load=True): """See help(Recording).""" super().__init__() self.signals = None self.units = None self.spatial = None self.stimulation = None self.available = [] self.param_handler = None self.source_file = base_file self.source_files = {} if param_file is not None: self._setup_from_file(param_file, load=load) elif params is not None: self._setup_from_dict(params, load=load) def load(self, *args, **kwargs): """Load each available attribute.""" for item in self.get_available(): item.load() def get_available(self): """Get the available attributes.""" return [getattr(self, item) for item in self.available] def set_base_file(self, base): """Set the source file of this recording.""" self.source_file = base def get_signal_channels(self, as_idx=False): """ Get the channel of each signal in the recording. Parameters ---------- as_idx : bool, optional If true, just returns [i for i in range(num_signals)] Returns ------- list The channels found. Returns None if no channels are set. """ if self.signals is None: if self.param_handler.get("signals", None) is not None: num_sigs = self.param_handler["signals"]["num_signals"] if as_idx: return [i for i in range(num_sigs)] default_chans = [i + 1 for i in range(num_sigs)] chans = self.param_handler["signals"].get( "channels", default_chans) return chans else: return None else: chans = [] if as_idx: return [i for i in range(len(self.signals))] for s in self.signals: if s.channel is not None: chans.append(s.channel) else: return [i for i in range(len(self.signals))] return chans def get_unit_groups(self): """ Get the groups of the units. For example, the list of tetrodes in the recording. Or the IDs of the sites on a ephys probe. Returns ------- list The found groups. Returns None if none are set yet. """ if self.units is None: if self.param_handler.get("units", None) is not None: groups = self.param_handler["units"]["group"] return groups else: return None else: groups = set([unit.group for unit in self.units]) def get_name_for_save(self, rel_dir=None): """ Get the name of the recording for saving purposes. This is very useful when plotting. For example, if the recording is in foo/bar/foo/pie.py Then passing rel_dir as foo, would return the name as bar--foo--pie Parameters ---------- rel_dir, str, optional The directory to take the path relative to, default is None. If None, it returns the full path with os.sep replaced by -- and with no extension. Returns ------- str The name for saving, it has no extension and os.sep replaced by -- """ if rel_dir is None: base_name_part, _ = os.path.splitext( os.path.basename(self.source_file)) else: name_up_to_rel = self.source_file[len(rel_dir + os.sep):] base_name_part, _ = os.path.splitext(name_up_to_rel) base_name_part = base_name_part.replace(os.sep, "--") return base_name_part def get_available_units(self): """ Get the list of available units. Returns ------- list list of tuple(group, list of units) """ all_units = [] for i, unit in enumerate(self.units): all_units.append([unit.group, unit.get_available_units()]) return all_units def get_set_units(self): """Get the units which are set for analysis.""" return [unit.units_to_use for unit in self.units] def get_set_units_as_dict(self): """Get the units which are set as a dictionary""" groups = [unit.group for unit in self.units] units = self.get_set_units() out_dict = {} for g, u in zip(groups, units): out_dict[g] = u return out_dict def get_signals(self): """Get the signals.""" return self.signals def get_eeg_signals(self, copy=True): """ Get the eeg signals as an EegArray. Parameters ---------- copy : bool, optional Whether to copy the retrieved signals, by default True. Returns ------- simuran.eeg.EegArray The signals as an EegArray. """ inplace = not copy eeg_array = EegArray() _, eeg_idxs = self.signals.group_by_property("channel_type", "eeg") eeg_sigs = self.signals.subsample(idx_list=eeg_idxs, inplace=inplace) if inplace: eeg_sigs = self.signals eeg_array.set_container([Eeg(signal=eeg) for eeg in eeg_sigs]) return eeg_array def get_np_signals(self): """Return a 2D array of signals as a numpy array.""" return np.array([s.samples for s in self.signals], float) def get_unit_signals(self): """Return a 2D array of signals with units.""" return np.array([s.samples.to(u.mV) for s in self.signals], float) * u.mV def _parse_source_files(self): """ Set the value of self.source_files based on the parameters. This only functions for things that have been set as available. """ source_files = {} for item, name in zip(self.get_available(), self.available): if isinstance(item, GenericContainer): source_files[name] = [s.source_file for s in item] else: source_files[name] = item.source_file self.source_files = source_files def _setup_from_file(self, param_file, load=True): """ Set up this recording from a source parameter file. Parameters ---------- param_file : str The parameter file to setup from. load : bool, optional Whether the information should be loaded, by default True Returns ------- None """ self.param_handler = ParamHandler(in_loc=param_file) self._setup(load=load) def _setup_from_dict(self, params, load=True): """ Set up this recording from a dictionary of parameters. Parameters ---------- params : dict The parameters to setup from. load : bool, optional Whether the information should be loaded, by default True Returns ------- None """ self.param_handler = ParamHandler(params=params) self._setup(load=load) def _setup(self, load=True): """ Set up this recording. Parameters ---------- load : bool, optional Whether the information should be loaded, by default True Returns ------- None """ if self.source_file is None: default_base_val = None if self.param_handler.location is not None: default_base_val = os.path.dirname(self.param_handler.location) base = self.param_handler.get("base_fname", default_base_val) if base is None: raise ValueError("Must set a base file in Recording setup") else: base = self.source_file data_loader_cls = loaders_dict.get( self.param_handler.get("loader", None), None) if data_loader_cls is None: raise ValueError("Unrecognised loader {}, options are {}".format( self.param_handler.get("loader", None), list(loaders_dict.keys()))) elif data_loader_cls == "params_only_no_cls": data_loader = None load = False else: data_loader = data_loader_cls(self.param_handler["loader_kwargs"]) chans = self.get_signal_channels() groups = self.get_unit_groups() fnames, base = data_loader.auto_fname_extraction( base, sig_channels=chans, unit_groups=groups) if fnames is None: self.valid = False logging.warning("Invalid recording setup from {}".format(base)) return self.source_file = base # TODO this could possibly have different classes for diff loaders self.signals = GenericContainer(BaseSignal) use_all = (("signals" not in self.param_handler.keys()) and ("units" not in self.param_handler.keys()) and ("spatial" not in self.param_handler.keys())) if "signals" in self.param_handler.keys() or use_all: self.available.append("signals") signal_dict = self.param_handler.get("signals", None) if data_loader_cls == "params_only_no_cls": to_iter = signal_dict["num_signals"] else: to_iter = len(fnames["Signal"]) for i in range(to_iter): if signal_dict is not None: params = split_dict(signal_dict, i) else: params = {} self.signals.append_new(params) if data_loader is not None: self.signals[-1].set_source_file(fnames["Signal"][i]) self.signals[-1].set_loader(data_loader) if "units" in self.param_handler.keys() or use_all: self.units = GenericContainer(SingleUnit) self.available.append("units") units_dict = self.param_handler.get("units", None) if data_loader_cls == "params_only_no_cls": to_iter = units_dict["num_groups"] else: to_iter = len(fnames["Spike"]) for i in range(to_iter): if units_dict is not None: params = split_dict(units_dict, i) else: params = {} self.units.append_new(params) if data_loader is not None: self.units[-1].set_source_file({ "Spike": fnames["Spike"][i], "Clusters": fnames["Clusters"][i] }) self.units[-1].set_loader(data_loader) if "spatial" in self.param_handler.keys() or use_all: self.spatial = Spatial() self.available.append("spatial") if data_loader is not None: self.spatial.set_source_file(fnames["Spatial"]) self.spatial.set_loader(data_loader) self._parse_source_files() if load: self.load() self.valid = True def __str__(self): """Call on print.""" if self.param_handler is not None: return "{} with params {} and source files {}".format( self.__class__.__name__, self.param_handler.params, self.source_files) else: return "{} with no params and source files {}".format( self.__class__.__name__, self.source_files)