def serialize(self, collection, target, labels_formatter, error_on_non_supported=True): assert (isinstance(collection, OpinionCollection)) assert (isinstance(target, str)) assert (isinstance(labels_formatter, StringLabelsFormatter)) assert (isinstance(error_on_non_supported, bool)) def __opinion_key(opinion): assert (isinstance(opinion, Opinion)) return opinion.SourceValue + opinion.TargetValue sorted_ops = sorted(collection, key=__opinion_key) create_dir_if_not_exists(target) with io.open(target, 'w') as f: for o in sorted_ops: str_value = OpinionConverter.try_to_string( opinion=o, labels_formatter=labels_formatter) if str_value is None: if error_on_non_supported: raise Exception( "Opinion label `{label}` is not supported by formatter" .format(label=o.Sentiment)) else: continue f.write(str_value) f.write('\n')
def __get_model_states_dir(self): result_dir = os.path.join(self.__get_model_root(), os.path.join(u'model_states/')) create_dir_if_not_exists(result_dir) return result_dir
def get_path_of_subfolder_in_experiments_dir(subfolder_name): """ Returns subfolder in experiments directory """ assert(isinstance(subfolder_name, unicode)) target_dir = join(get_experiments_dir(), u"{}/".format(subfolder_name)) create_dir_if_not_exists(target_dir) return target_dir
def __save_minibatch_variable_values(self, target, predict_log, var_name): assert (isinstance(predict_log, NetworkInputDependentVariables)) create_dir_if_not_exists(target) id_and_value_pairs = list( predict_log.iter_by_parameter_values(var_name)) id_and_value_pairs = sorted(id_and_value_pairs, key=lambda pair: pair[0]) self.__writer.write(target=target, data=[pair[1] for pair in id_and_value_pairs])
def create_result_opinion_collection_filepath(self, data_type, doc_id, epoch_index): assert (isinstance(epoch_index, int)) model_eval_root = self.__get_eval_root_filepath( data_type=data_type, epoch_index=epoch_index) filepath = os.path.join(model_eval_root, u"{}.opin.txt".format(doc_id)) create_dir_if_not_exists(filepath) return filepath
def __get_eval_root_filepath(self, data_type, epoch_index): assert (isinstance(epoch_index, int)) result_dir = os.path.join( self.__get_model_root(), os.path.join(u"eval/{}/{}/{}".format(data_type, self.__current_cv_index, str(epoch_index)))) create_dir_if_not_exists(result_dir) return result_dir
def __save_model_hidden_values(self, epoch_index): if not self.__key_save_hidden_parameters: return names, values = self.__model.get_hidden_parameters() assert(isinstance(names, list)) assert(isinstance(values, list)) assert(len(names) == len(values)) for i, name in enumerate(names): variable_path = os.path.join(self.__log_dir, self.HiddenParamsTemplate.format(name, epoch_index)) if self.__debug_save_info: print "Save hidden values: {}".format(variable_path) create_dir_if_not_exists(variable_path) np.save(variable_path, values[i])
def __save_minibatch_variable_values(self, data_type, epoch_index, predict_log, var_name): assert(isinstance(predict_log, NetworkInputDependentVariables)) assert(isinstance(var_name, unicode)) if not self.__key_save_hidden_parameters: return vars_path = os.path.join(self.__log_dir, self.InputDependentParamsTemplate.format( '{}-{}'.format(var_name, data_type), epoch_index)) create_dir_if_not_exists(vars_path) if self.__debug_save_info: print "Save input dependent hidden values in a list using np.savez: {}".format(vars_path) id_and_value_pairs = list(predict_log.iter_by_parameter_values(var_name)) id_and_value_pairs = sorted(id_and_value_pairs, key=lambda pair: pair[0]) np.savez(vars_path, [pair[1] for pair in id_and_value_pairs])
def __enter__(self): # Compose filepath for verbose results. eval_verbose_log_filepath = self.__create_verbose_log_filepath() eval_short_log_filepath = join( self.__log_dir, Common.create_log_eval_filename(iter_index=self.__it_index, data_type=self.__data_type)) create_dir_if_not_exists(eval_verbose_log_filepath) create_dir_if_not_exists(eval_short_log_filepath) print u"Saving at: {}".format(eval_short_log_filepath) self.__eval_short_file = open(eval_short_log_filepath, u"w", buffering=0) self.__eval_verbose_file = open(eval_verbose_log_filepath, u"w", buffering=0)
def __open_filepaths_optionally(self): # Compose filepath for verbose results. eval_verbose_log_filepath = join( self.__log_dir, self.__log_eval_iter_verbose_filename.format( iter=self.__it_index, dtype=self.__data_type)) eval_short_log_filepath = self.__create_short_log_filepath() create_dir_if_not_exists(eval_verbose_log_filepath) create_dir_if_not_exists(eval_short_log_filepath) if self.__eval_short_file is None: self.__eval_short_file = open(eval_short_log_filepath, u"w", buffering=0) if self.__eval_verbose_file is None: self.__eval_verbose_file = open(eval_verbose_log_filepath, u"w", buffering=0)
def __enter__(self): assert (self.__log_dir is not None) iter_index_int = self.__get_iter_index() iter_index = str(iter_index_int) for d_type in self.__iter_supported_data_types(): train_log_filepath = join( self.__log_dir, Common.create_log_train_filename(iter_index=iter_index_int, data_type=d_type)) eval_log_filepath = join( self.__log_dir, Common.create_log_eval_filename(iter_index=iter_index_int, data_type=d_type)) eval_verbose_log_filepath = join( self.__log_dir, self.__log_eval_iter_verbose_filename.format(iter=iter_index, dtype=d_type)) create_dir_if_not_exists(train_log_filepath) create_dir_if_not_exists(eval_log_filepath) create_dir_if_not_exists(eval_verbose_log_filepath) self.__train_iter_log_files[d_type] = open(train_log_filepath, u"w", buffering=0) self.__eval_iter_log_files[d_type] = open(eval_log_filepath, u"w", buffering=0) self.__eval_iter_verbose_log_files[d_type] = open( eval_verbose_log_filepath, u"w", buffering=0)
def on_experiment_finished(self): """ Providing results aggregation across all the experiment iterations. """ # Considering that we perform evaluations per every iteration. if not self.__do_eval: return # Opening the related file. log_eval_filepath = join(self.__log_dir, Common.log_test_eval_exp_filename) create_dir_if_not_exists(log_eval_filepath) with open(log_eval_filepath, u'w', buffering=0) as f: iter_messages = chain([ u"Results for model: {}".format( self.__model.IO.get_model_name()) ], create_experiment_eval_msgs(self.__test_results_exp_history), [u'--------------']) for msg in iter_messages: f.write(u"{}\n".format(msg))
def write(self, data, target): assert (isinstance(target, str)) logger.info("Save: {}".format(target)) create_dir_if_not_exists(target) np.savez(target, data)
def __enter__(self): create_dir_if_not_exists(self._target) self.__f = gzip.open(self._target, 'wb') return self
def get_experiments_dir(self): experiments_name = u'rusentrel' target_dir = join(self.get_data_root(), u"./{}/".format(experiments_name)) create_dir_if_not_exists(target_dir) return target_dir
def get_experiments_dir(): target_dir = join(get_data_root(), u"./{}/".format(EXPERIMENTS_NAME)) create_dir_if_not_exists(target_dir) return target_dir
def get_log_root(): log_root_dir = path.join(get_data_root(), u"Log/") create_dir_if_not_exists(log_root_dir) return log_root_dir