def evaluate(self, std_path, test_path, output_path=''):
"""Run evaluation on all files in the given directories"""
std = loadAllStandard(std_path)
test = loadAllTest(test_path)
diff = set([x.name for x in std]).symmetric_difference(
set([y.name for y in test]))
if len(diff) > 0:
print('Warning: missing files :\n {}'.format('\n '.join(diff)))
std = [s for s in std if s.name not in diff]
test = [t for t in test if t.name not in diff]
res = []
for tag in Evaluator.stat_tags:
res.append(Metrics())
for i, s in enumerate(std):
if not s.has_coref:
# do not compare documents without a .coref file
# this is just for convenience
continue
m_tuple = self.evaluateDocument(s, test[i])
self.printReport(s.name, output_path)
for i, val in enumerate(res):
val.add(m_tuple[i])
print(self.buildMetricsTable(res))
def evaluate(self, std_path, test_path, output_path='', is_silent=False):
"""Run evaluation on all files in the given directories.
If output_path is provided, evaluation reports will be written there.
is_silent determines if the result is printed to the output"""
std = loadAllStandard(std_path)
test = loadAllTest(test_path)
diff = set([x.name for x in std
]).symmetric_difference(set([y.name for y in test]))
if len(diff) > 0:
print('Warning: missing files :\n {}'.format('\n '.join(diff)))
std = [s for s in std if s.name not in diff]
test = [t for t in test if t.name not in diff]
res = dict((tag, Metrics()) for tag in Evaluator.stat_tags)
for i, s in enumerate(std):
if not s.has_coref:
# do not compare documents without a .coref file
# this is just for convenience
continue
m_tuple = self.evaluateDocument(s, test[i])
self.printReport(s.name, output_path)
for j, tag in enumerate(Evaluator.stat_tags):
res[tag].add(m_tuple[j])
if not is_silent:
print(self.buildMetricsTable(res))
return res
def evaluate(self, std_path, test_path, output_path=''):
"""Run evaluation on all files in the given directories"""
std = loadAllStandard(std_path)
test = loadAllTest(test_path)
diff = set([x.name for x in std
]).symmetric_difference(set([y.name for y in test]))
if len(diff) > 0:
print('Warning: missing files :\n {}'.format('\n '.join(diff)))
std = [s for s in std if s.name not in diff]
test = [t for t in test if t.name not in diff]
res = []
for tag in Evaluator.stat_tags:
res.append(Metrics())
for i, s in enumerate(std):
if not s.has_coref:
# do not compare documents without a .coref file
# this is just for convenience
continue
m_tuple = self.evaluateDocument(s, test[i])
self.printReport(s.name, output_path)
for i, val in enumerate(res):
val.add(m_tuple[i])
print(self.buildMetricsTable(res))
def evaluate(self, std_path, test_path, output_path='', is_silent=False):
"""Run evaluation on all files in the given directories.
If output_path is provided, evaluation reports will be written there.
is_silent determines if the result is printed to the output"""
std = loadAllStandard(std_path)
test = loadAllTest(test_path)
diff = set([x.name for x in std]).symmetric_difference(
set([y.name for y in test]))
if len(diff) > 0:
print('Warning: missing files :\n {}'.format('\n '.join(diff)))
std = [s for s in std if s.name not in diff]
test = [t for t in test if t.name not in diff]
res = dict((tag, Metrics()) for tag in Evaluator.stat_tags)
for i, s in enumerate(std):
if not s.has_coref:
# do not compare documents without a .coref file
# this is just for convenience
continue
m_tuple = self.evaluateDocument(s, test[i])
self.printReport(s.name, output_path)
for j, tag in enumerate(Evaluator.stat_tags):
res[tag].add(m_tuple[j])
if not is_silent:
print(self.buildMetricsTable(res))
return res
def generate(self, standard_path, test_path):
"""Loads the stnadard markup from standard_path, generates a response and saves
it to the test_path"""
os.makedirs(test_path, exist_ok = True)
std = loadAllStandard(standard_path)
for s in std:
self.generateDoc(s, os.path.join(test_path, s.name + '.task3'))
def generate(self, standard_path, test_path):
"""Loads the stnadard markup from standard_path, generates a response and saves
it to the test_path"""
os.makedirs(test_path, exist_ok = True)
std = loadAllStandard(standard_path)
for s in std:
self.generateDoc(s, os.path.join(test_path, s.name + '.task3'))
def validate(standard_path):
"""Prints various information on standard files in the given directory"""
std = loadAllStandard(standard_path)
no_fact_count = 0
f_attr_count = dict()
f_attr_value_count = dict()
for s in std:
if not s.has_facts:
no_fact_count += 1
for f in s.facts:
for a in f.arguments:
key = f.tag + ' - ' + a.name
if key in f_attr_count:
f_attr_count[key] += 1
else:
f_attr_count[key] = 1
for val in a.values:
if not isinstance(val, StringValue):
if 'span' in val.values:
print('SPAN IN VAL!!!!')
if a.name in ['модальность', 'сложность', 'фаза']:
key = f.tag + ' - ' + a.name + ' - ' + a.values[0].descr
if key in f_attr_value_count:
f_attr_value_count[key] += 1
else:
f_attr_value_count[key] = 1
print('Fact standard statistics for "{}":'.format(standard_path))
print('Total files: {}'.format(len(std)))
print('Files with no fact layer: {}'.format(no_fact_count))
print('Fact-argument pair counts:')
for key in sorted(f_attr_count.keys()):
print(' {}\t{}'.format(key, f_attr_count[key]))
print('Fact-argument-value counts for the auxiliary arguments:')
for key in sorted(f_attr_value_count.keys()):
print(' {}\t{}'.format(key, f_attr_value_count[key]))