def init_logger(logger_name=__name__, output_path=None, level=logging.INFO): """ Initializes a logger for console and file writing. :param logger_name: self-explanatory. :param output_path: directory or file path where the logs should be saved. By default it will not store file logs. :param level: self-explanatory. """ logger = logging.getLogger(logger_name) logger.setLevel(level) formatter = logging.Formatter("%(asctime)s [%(levelname)s]: %(message)s") # adding console output stream_handler = logging.StreamHandler() stream_handler.setFormatter(formatter) logger.addHandler(stream_handler) if output_path: if is_file_path(output_path): safe_mkfdir(output_path) if os.path.exists(output_path): os.remove(output_path) else: safe_mkdir(output_path) # using the default name of the logger default_file_name = "log_" + strftime("%b_%d_%H_%M_%S") + '.txt' output_path = os.path.join(output_path, default_file_name) file_handler = logging.FileHandler(output_path) file_handler.setFormatter(formatter) logger.addHandler(file_handler) return logger
def save(self, file_path, encoding='utf-8'): """Saves hyper-params object as a json file.""" safe_mkfdir(file_path) hparams_to_save = self._get_simple_attrs() f = codecs.open(file_path, encoding=encoding, mode='w') json.dump(hparams_to_save, f, indent=2) logger.debug("Extracted the following hparams: '%s'." "" % " ".join(hparams_to_save.keys())) logger.info("Saved hyper-parameters to '%s'." % file_path)
def delete_attr_from_params(input_fp, output_fp, attr_names, device='cpu'): """Removes a particular attrs from the dictionary of params, saves back.""" model_params = T.load(input_fp, device)[MODEL_PARAMS] for attr_name in attr_names: if attr_name in model_params: del model_params[attr_name] # dumping to the disk safe_mkfdir(output_fp) T.save({MODEL_PARAMS: model_params}, f=output_fp)
def merge_csv_files(input_fps, output_fp, sep="\t"): """ Merges the csv files that have the same header into one file. Does not perform shuffling to avoid problems with the same group entries. """ dfs = [] for fp in input_fps: dfs.append(read_csv(fp, sep=sep, quoting=QUOTE_NONE, encoding='utf-8')) df = concat(dfs, axis=0, ignore_index=True, copy=True) safe_mkfdir(output_fp) df.to_csv(output_fp, sep=sep, index=False, encoding='utf-8', quoting=QUOTE_NONE)
def rename_attrs_in_params(input_fp, output_fp, old_attr_names, new_attr_names, device='cpu'): """Renames a model's parameters, saves them to an output file.""" assert len(old_attr_names) == len(new_attr_names) model_params = T.load(input_fp, device)[MODEL_PARAMS] for old_name, new_name in zip(old_attr_names, new_attr_names): model_params[new_name] = model_params[old_name] del model_params[old_name] # dumping to the disk safe_mkfdir(output_fp) T.save({MODEL_PARAMS: model_params}, f=output_fp)
def write(self, file_path, sep=' ', encoding='utf-8'): """ Writes the vocabulary to a plain text file where each line is of the form: {token}{sep}{count}. Default special symbols are not written. :param file_path: self-explanatory. :param sep: self-explanatory. :param encoding: self-explanatory. """ safe_mkfdir(file_path) with codecs.open(file_path, 'w', encoding=encoding) as f: for symbol in self: token = symbol.token count = str(symbol.count) try: str_entry = sep.join([token, count]) f.write(str_entry) f.write("\n") except Exception: logger.fatal( "Below entry produced a fatal error in write().") logger.fatal(symbol.token) raise ValueError("Could not process a token.") logger.info("Vocabulary is written to: '%s'." % file_path)