def do_context_sentence_evaluation_classification():
"""
Experiment evaluating performance of sentences as contexts for
co-occurrence networks in the classification task.
"""
print '> Reading cases..'
path = '../data/tasa/TASA900_text'
texts, labels = data.read_files(path)
print '> Evaluating..'
graphs = []
results = {}
for text in texts:
g = graph_representation.construct_cooccurrence_network(
text, context='sentence')
graphs.append(g)
for metric in graph_representation.get_metrics():
print ' ', metric
vectors = graph_representation.graphs_to_vectors(graphs,
metric,
verbose=True)
score = evaluation.evaluate_classification(vectors, labels)
results[metric + ' (sentence)'] = score
data.pickle_to_file(results, 'output/class_context_sentence')
pp.pprint(results)
return results
def do_context_sentence_evaluation_classification():
"""
Experiment evaluating performance of sentences as contexts for
co-occurrence networks in the classification task.
"""
print '> Reading cases..'
path = '../data/tasa/TASA900_text'
texts, labels = data.read_files(path)
print '> Evaluating..'
graphs = []
results = {}
for text in texts:
g = graph_representation.construct_cooccurrence_network(text, context='sentence')
graphs.append(g)
for metric in graph_representation.get_metrics():
print ' ', metric
vectors = graph_representation.graphs_to_vectors(graphs, metric, verbose=True)
score = evaluation.evaluate_classification(vectors, labels)
results[metric+' (sentence)'] = score
data.pickle_to_file(results, 'output/class_context_sentence')
pp.pprint(results)
return results
def do_retrieval_experiments(
descriptions='air/problem_descriptions',
solutions='air/solutions',
graph_types=['co-occurrence', 'dependency', 'random'],
use_frequency=True):
"""
Experiment used for comparative evaluation of different network
representations on the retrieval task.
Toggle comparison with frequency-based methods using *use_frequency*.
"""
results = {
'_solutions': solutions,
'_descriptions': descriptions,
'_evaluation': 'retrieval'
}
print '> Evaluation type: retrieval'
print '> Reading cases..'
descriptions_path = '../data/' + descriptions
descriptiondata = data.read_data(descriptions_path, graph_types)
solutions_path = '../data/' + solutions + '_preprocessed'
solution_texts, labels = data.read_files(solutions_path)
solution_vectors = freq_representation.text_to_vector(
solution_texts, freq_representation.FrequencyMetrics.TF_IDF)
print '> Evaluating..'
for gtype in graph_types:
print ' ', gtype
docs, labels = descriptiondata[gtype]
graphs = graph_representation.create_graphs(docs, gtype)
results[gtype] = {}
for metric in graph_representation.get_metrics():
print ' -', metric
vectors = graph_representation.graphs_to_vectors(graphs, metric)
results[gtype][metric] = evaluation.evaluate_retrieval(
vectors, solution_vectors)
if use_frequency:
print ' frequency'
results['freq'] = {}
for metric in freq_representation.get_metrics():
print ' -', metric
docs, labels = data.read_files(descriptions_path + '_preprocessed')
vectors = freq_representation.text_to_vector(docs, metric)
results['freq'][metric] = evaluation.evaluate_retrieval(
vectors, solution_vectors)
print
pp.pprint(results)
return results
def do_retrieval_experiments(descriptions='air/problem_descriptions',
solutions='air/solutions',
graph_types=['co-occurrence','dependency','random'],
use_frequency=True):
"""
Experiment used for comparative evaluation of different network
representations on the retrieval task.
Toggle comparison with frequency-based methods using *use_frequency*.
"""
results = {'_solutions':solutions,
'_descriptions':descriptions,
'_evaluation':'retrieval'}
print '> Evaluation type: retrieval'
print '> Reading cases..'
descriptions_path = '../data/'+descriptions
descriptiondata = data.read_data(descriptions_path, graph_types)
solutions_path = '../data/'+solutions+'_preprocessed'
solution_texts, labels = data.read_files(solutions_path)
solution_vectors = freq_representation.text_to_vector(solution_texts, freq_representation.FrequencyMetrics.TF_IDF)
print '> Evaluating..'
for gtype in graph_types:
print ' ',gtype
docs, labels = descriptiondata[gtype]
graphs = graph_representation.create_graphs(docs, gtype)
results[gtype] = {}
for metric in graph_representation.get_metrics():
print ' -', metric
vectors = graph_representation.graphs_to_vectors(graphs, metric)
results[gtype][metric] = evaluation.evaluate_retrieval(vectors, solution_vectors)
if use_frequency:
print ' frequency'
results['freq'] = {}
for metric in freq_representation.get_metrics():
print ' -', metric
docs, labels = data.read_files(descriptions_path+'_preprocessed')
vectors = freq_representation.text_to_vector(docs, metric)
results['freq'][metric] = evaluation.evaluate_retrieval(vectors, solution_vectors)
print
pp.pprint(results)
return results
def do_classification_experiments(
dataset='tasa/TASA900',
graph_types=['co-occurrence', 'dependency', 'random'],
use_frequency=True):
"""
Experiment used for comparative evaluation of different network
representations on classification.
Toggle comparison with frequency-based methods using *use_frequency*.
"""
results = {'_dataset': dataset, '_evaluation': 'classification'}
print '> Evaluation type: classification'
print '> Reading data..', dataset
corpus_path = '../data/' + dataset
docdata = data.read_data(corpus_path, graph_types)
print '> Evaluating..'
for gtype in graph_types:
print ' ', gtype
documents, labels = docdata[gtype]
graphs = graph_representation.create_graphs(documents, gtype)
results[gtype] = {}
for metric in graph_representation.get_metrics():
print ' -', metric
vectors = graph_representation.graphs_to_vectors(graphs, metric)
results[gtype][metric] = evaluation.evaluate_classification(
vectors, labels)
if use_frequency:
print ' frequency'
results['freq'] = {}
for metric in freq_representation.get_metrics():
print ' -', metric
documents, labels = data.read_files(corpus_path + '_preprocessed')
vectors = freq_representation.text_to_vector(documents, metric)
results['freq'][metric] = evaluation.evaluate_classification(
vectors, labels)
print
pp.pprint(results)
return results
def do_classification_experiments(dataset='tasa/TASA900',
graph_types = ['co-occurrence','dependency','random'],
use_frequency = True):
"""
Experiment used for comparative evaluation of different network
representations on classification.
Toggle comparison with frequency-based methods using *use_frequency*.
"""
results = {'_dataset':dataset,
'_evaluation':'classification'}
print '> Evaluation type: classification'
print '> Reading data..', dataset
corpus_path = '../data/'+dataset
docdata = data.read_data(corpus_path, graph_types)
print '> Evaluating..'
for gtype in graph_types:
print ' ',gtype
documents, labels = docdata[gtype]
graphs = graph_representation.create_graphs(documents, gtype)
results[gtype] = {}
for metric in graph_representation.get_metrics():
print ' -', metric
vectors = graph_representation.graphs_to_vectors(graphs, metric)
results[gtype][metric] = evaluation.evaluate_classification(vectors, labels)
if use_frequency:
print ' frequency'
results['freq'] = {}
for metric in freq_representation.get_metrics():
print ' -', metric
documents, labels = data.read_files(corpus_path+'_preprocessed')
vectors = freq_representation.text_to_vector(documents, metric)
results['freq'][metric] = evaluation.evaluate_classification(vectors, labels)
print
pp.pprint(results)
return results