def evaluate(self, std_path, test_path, output_path=''): """Run evaluation on all files in the given directories""" std = loadAllStandard(std_path) test = loadAllTest(test_path) diff = set([x.name for x in std]).symmetric_difference( set([y.name for y in test])) if len(diff) > 0: print('Warning: missing files :\n {}'.format('\n '.join(diff))) std = [s for s in std if s.name not in diff] test = [t for t in test if t.name not in diff] res = [] for tag in Evaluator.stat_tags: res.append(Metrics()) for i, s in enumerate(std): if not s.has_coref: # do not compare documents without a .coref file # this is just for convenience continue m_tuple = self.evaluateDocument(s, test[i]) self.printReport(s.name, output_path) for i, val in enumerate(res): val.add(m_tuple[i]) print(self.buildMetricsTable(res))
def evaluate(self, std_path, test_path, output_path='', is_silent=False): """Run evaluation on all files in the given directories. If output_path is provided, evaluation reports will be written there. is_silent determines if the result is printed to the output""" std = loadAllStandard(std_path) test = loadAllTest(test_path) diff = set([x.name for x in std ]).symmetric_difference(set([y.name for y in test])) if len(diff) > 0: print('Warning: missing files :\n {}'.format('\n '.join(diff))) std = [s for s in std if s.name not in diff] test = [t for t in test if t.name not in diff] res = dict((tag, Metrics()) for tag in Evaluator.stat_tags) for i, s in enumerate(std): if not s.has_coref: # do not compare documents without a .coref file # this is just for convenience continue m_tuple = self.evaluateDocument(s, test[i]) self.printReport(s.name, output_path) for j, tag in enumerate(Evaluator.stat_tags): res[tag].add(m_tuple[j]) if not is_silent: print(self.buildMetricsTable(res)) return res
def evaluate(self, std_path, test_path, output_path=''): """Run evaluation on all files in the given directories""" std = loadAllStandard(std_path) test = loadAllTest(test_path) diff = set([x.name for x in std ]).symmetric_difference(set([y.name for y in test])) if len(diff) > 0: print('Warning: missing files :\n {}'.format('\n '.join(diff))) std = [s for s in std if s.name not in diff] test = [t for t in test if t.name not in diff] res = [] for tag in Evaluator.stat_tags: res.append(Metrics()) for i, s in enumerate(std): if not s.has_coref: # do not compare documents without a .coref file # this is just for convenience continue m_tuple = self.evaluateDocument(s, test[i]) self.printReport(s.name, output_path) for i, val in enumerate(res): val.add(m_tuple[i]) print(self.buildMetricsTable(res))
def evaluate(self, std_path, test_path, output_path='', is_silent=False): """Run evaluation on all files in the given directories. If output_path is provided, evaluation reports will be written there. is_silent determines if the result is printed to the output""" std = loadAllStandard(std_path) test = loadAllTest(test_path) diff = set([x.name for x in std]).symmetric_difference( set([y.name for y in test])) if len(diff) > 0: print('Warning: missing files :\n {}'.format('\n '.join(diff))) std = [s for s in std if s.name not in diff] test = [t for t in test if t.name not in diff] res = dict((tag, Metrics()) for tag in Evaluator.stat_tags) for i, s in enumerate(std): if not s.has_coref: # do not compare documents without a .coref file # this is just for convenience continue m_tuple = self.evaluateDocument(s, test[i]) self.printReport(s.name, output_path) for j, tag in enumerate(Evaluator.stat_tags): res[tag].add(m_tuple[j]) if not is_silent: print(self.buildMetricsTable(res)) return res
def generate(self, standard_path, test_path): """Loads the stnadard markup from standard_path, generates a response and saves it to the test_path""" os.makedirs(test_path, exist_ok = True) std = loadAllStandard(standard_path) for s in std: self.generateDoc(s, os.path.join(test_path, s.name + '.task3'))
def validate(standard_path): """Prints various information on standard files in the given directory""" std = loadAllStandard(standard_path) no_fact_count = 0 f_attr_count = dict() f_attr_value_count = dict() for s in std: if not s.has_facts: no_fact_count += 1 for f in s.facts: for a in f.arguments: key = f.tag + ' - ' + a.name if key in f_attr_count: f_attr_count[key] += 1 else: f_attr_count[key] = 1 for val in a.values: if not isinstance(val, StringValue): if 'span' in val.values: print('SPAN IN VAL!!!!') if a.name in ['модальность', 'сложность', 'фаза']: key = f.tag + ' - ' + a.name + ' - ' + a.values[0].descr if key in f_attr_value_count: f_attr_value_count[key] += 1 else: f_attr_value_count[key] = 1 print('Fact standard statistics for "{}":'.format(standard_path)) print('Total files: {}'.format(len(std))) print('Files with no fact layer: {}'.format(no_fact_count)) print('Fact-argument pair counts:') for key in sorted(f_attr_count.keys()): print(' {}\t{}'.format(key, f_attr_count[key])) print('Fact-argument-value counts for the auxiliary arguments:') for key in sorted(f_attr_value_count.keys()): print(' {}\t{}'.format(key, f_attr_value_count[key]))