def serialize(self,
collection,
target,
labels_formatter,
error_on_non_supported=True):
assert (isinstance(collection, OpinionCollection))
assert (isinstance(target, str))
assert (isinstance(labels_formatter, StringLabelsFormatter))
assert (isinstance(error_on_non_supported, bool))
def __opinion_key(opinion):
assert (isinstance(opinion, Opinion))
return opinion.SourceValue + opinion.TargetValue
sorted_ops = sorted(collection, key=__opinion_key)
create_dir_if_not_exists(target)
with io.open(target, 'w') as f:
for o in sorted_ops:
str_value = OpinionConverter.try_to_string(
opinion=o, labels_formatter=labels_formatter)
if str_value is None:
if error_on_non_supported:
raise Exception(
"Opinion label `{label}` is not supported by formatter"
.format(label=o.Sentiment))
else:
continue
f.write(str_value)
f.write('\n')
def __get_model_states_dir(self):
result_dir = os.path.join(self.__get_model_root(),
os.path.join(u'model_states/'))
create_dir_if_not_exists(result_dir)
return result_dir
def get_path_of_subfolder_in_experiments_dir(subfolder_name):
"""
Returns subfolder in experiments directory
"""
assert(isinstance(subfolder_name, unicode))
target_dir = join(get_experiments_dir(), u"{}/".format(subfolder_name))
create_dir_if_not_exists(target_dir)
return target_dir
def __save_minibatch_variable_values(self, target, predict_log, var_name):
assert (isinstance(predict_log, NetworkInputDependentVariables))
create_dir_if_not_exists(target)
id_and_value_pairs = list(
predict_log.iter_by_parameter_values(var_name))
id_and_value_pairs = sorted(id_and_value_pairs,
key=lambda pair: pair[0])
self.__writer.write(target=target,
data=[pair[1] for pair in id_and_value_pairs])
def create_result_opinion_collection_filepath(self, data_type, doc_id,
epoch_index):
assert (isinstance(epoch_index, int))
model_eval_root = self.__get_eval_root_filepath(
data_type=data_type, epoch_index=epoch_index)
filepath = os.path.join(model_eval_root, u"{}.opin.txt".format(doc_id))
create_dir_if_not_exists(filepath)
return filepath
def __get_eval_root_filepath(self, data_type, epoch_index):
assert (isinstance(epoch_index, int))
result_dir = os.path.join(
self.__get_model_root(),
os.path.join(u"eval/{}/{}/{}".format(data_type,
self.__current_cv_index,
str(epoch_index))))
create_dir_if_not_exists(result_dir)
return result_dir
def __save_model_hidden_values(self, epoch_index):
if not self.__key_save_hidden_parameters:
return
names, values = self.__model.get_hidden_parameters()
assert(isinstance(names, list))
assert(isinstance(values, list))
assert(len(names) == len(values))
for i, name in enumerate(names):
variable_path = os.path.join(self.__log_dir, self.HiddenParamsTemplate.format(name, epoch_index))
if self.__debug_save_info:
print "Save hidden values: {}".format(variable_path)
create_dir_if_not_exists(variable_path)
np.save(variable_path, values[i])
def __save_minibatch_variable_values(self, data_type, epoch_index, predict_log, var_name):
assert(isinstance(predict_log, NetworkInputDependentVariables))
assert(isinstance(var_name, unicode))
if not self.__key_save_hidden_parameters:
return
vars_path = os.path.join(self.__log_dir,
self.InputDependentParamsTemplate.format(
'{}-{}'.format(var_name, data_type),
epoch_index))
create_dir_if_not_exists(vars_path)
if self.__debug_save_info:
print "Save input dependent hidden values in a list using np.savez: {}".format(vars_path)
id_and_value_pairs = list(predict_log.iter_by_parameter_values(var_name))
id_and_value_pairs = sorted(id_and_value_pairs, key=lambda pair: pair[0])
np.savez(vars_path, [pair[1] for pair in id_and_value_pairs])
def __enter__(self):
# Compose filepath for verbose results.
eval_verbose_log_filepath = self.__create_verbose_log_filepath()
eval_short_log_filepath = join(
self.__log_dir,
Common.create_log_eval_filename(iter_index=self.__it_index,
data_type=self.__data_type))
create_dir_if_not_exists(eval_verbose_log_filepath)
create_dir_if_not_exists(eval_short_log_filepath)
print u"Saving at: {}".format(eval_short_log_filepath)
self.__eval_short_file = open(eval_short_log_filepath,
u"w",
buffering=0)
self.__eval_verbose_file = open(eval_verbose_log_filepath,
u"w",
buffering=0)
def __open_filepaths_optionally(self):
# Compose filepath for verbose results.
eval_verbose_log_filepath = join(
self.__log_dir,
self.__log_eval_iter_verbose_filename.format(
iter=self.__it_index, dtype=self.__data_type))
eval_short_log_filepath = self.__create_short_log_filepath()
create_dir_if_not_exists(eval_verbose_log_filepath)
create_dir_if_not_exists(eval_short_log_filepath)
if self.__eval_short_file is None:
self.__eval_short_file = open(eval_short_log_filepath,
u"w",
buffering=0)
if self.__eval_verbose_file is None:
self.__eval_verbose_file = open(eval_verbose_log_filepath,
u"w",
buffering=0)
def __enter__(self):
assert (self.__log_dir is not None)
iter_index_int = self.__get_iter_index()
iter_index = str(iter_index_int)
for d_type in self.__iter_supported_data_types():
train_log_filepath = join(
self.__log_dir,
Common.create_log_train_filename(iter_index=iter_index_int,
data_type=d_type))
eval_log_filepath = join(
self.__log_dir,
Common.create_log_eval_filename(iter_index=iter_index_int,
data_type=d_type))
eval_verbose_log_filepath = join(
self.__log_dir,
self.__log_eval_iter_verbose_filename.format(iter=iter_index,
dtype=d_type))
create_dir_if_not_exists(train_log_filepath)
create_dir_if_not_exists(eval_log_filepath)
create_dir_if_not_exists(eval_verbose_log_filepath)
self.__train_iter_log_files[d_type] = open(train_log_filepath,
u"w",
buffering=0)
self.__eval_iter_log_files[d_type] = open(eval_log_filepath,
u"w",
buffering=0)
self.__eval_iter_verbose_log_files[d_type] = open(
eval_verbose_log_filepath, u"w", buffering=0)
def on_experiment_finished(self):
""" Providing results aggregation across all the experiment iterations.
"""
# Considering that we perform evaluations per every iteration.
if not self.__do_eval:
return
# Opening the related file.
log_eval_filepath = join(self.__log_dir,
Common.log_test_eval_exp_filename)
create_dir_if_not_exists(log_eval_filepath)
with open(log_eval_filepath, u'w', buffering=0) as f:
iter_messages = chain([
u"Results for model: {}".format(
self.__model.IO.get_model_name())
], create_experiment_eval_msgs(self.__test_results_exp_history),
[u'--------------'])
for msg in iter_messages:
f.write(u"{}\n".format(msg))
def write(self, data, target):
assert (isinstance(target, str))
logger.info("Save: {}".format(target))
create_dir_if_not_exists(target)
np.savez(target, data)
def __enter__(self):
create_dir_if_not_exists(self._target)
self.__f = gzip.open(self._target, 'wb')
return self
def get_experiments_dir(self):
experiments_name = u'rusentrel'
target_dir = join(self.get_data_root(),
u"./{}/".format(experiments_name))
create_dir_if_not_exists(target_dir)
return target_dir
def get_experiments_dir():
target_dir = join(get_data_root(), u"./{}/".format(EXPERIMENTS_NAME))
create_dir_if_not_exists(target_dir)
return target_dir
def get_log_root():
log_root_dir = path.join(get_data_root(), u"Log/")
create_dir_if_not_exists(log_root_dir)
return log_root_dir
