def map_concepts(self):
"""
Concepts are weighted according to the number of annotator summaries
in which they apper.
"""
## make sure gold summaries are loaded
if (not self.problem.annotators) or (not self.problem.training):
sys.stderr.write('\nError: no gold summaries loaded\n')
return
concepts = {}
for annotator in self.problem.annotators:
annotator_concepts = {}
for rawsent in self.problem.training[annotator]:
sent = text.Sentence(rawsent)
units = self.unit_selector(sent.stemmed)
for unit in units:
if not unit in annotator_concepts:
annotator_concepts[unit] = 0
annotator_concepts[unit] += 1
for concept in annotator_concepts:
if not concept in concepts: concepts[concept] = 0
concepts[concept] += 1
self.concepts = concepts
return True
def __init__(self, sentences, gold_sentences, idx=None,
label_sentences=None):
"""Create an instance with multiple input sentences and output
sentences.
"""
TransductionInstance.__init__(self, sentences, idx=idx)
# Add each gold sentence as a MultiSentence (for annotation purposes)
self.gold_sentences = [text.MultiSentence(
[text.Sentence(gold_sentence)])
for gold_sentence in gold_sentences]
self.sum_gold_len = sum(len(gold_sent.tokens)
for gold_sent in self.gold_sentences)
self.avg_gold_len = self.sum_gold_len / len(self.gold_sentences)
# If provided, add additional sentences as well.
if label_sentences is not None:
self.label_sentences = [text.MultiSentence(
[text.Sentence(label_sentence)])
for label_sentence in label_sentences]
def __init__(self, id, title, narr, new_docs, old_docs, old_problems=[]):
self.id = id
self.title = title
self.narr = narr
self.query = text.Sentence(title + ": " + narr)
self.new_docs_paths = new_docs[:]
self.old_docs_paths = old_docs[:]
self.old_problems = old_problems
## variables that might get set later
self.new_docs = None
self.old_docs = None
self.training = {}
self.annotators = set()
def map_concepts(self):
concepts = {}
for annotator in self.problem.annotators:
annotator_concepts = {}
for sent in self.problem.training[annotator]:
sentence = text.Sentence(sent)
units = self.unit_selector(sentence.stemmed)
for unit in units:
if unit not in annotator_concepts: annotator_concepts[unit] = 0
annotator_concepts[unit] += 1
for concept in annotator_concepts:
if concept not in concepts: concepts[concept] = 0
concepts[concept] += 1
self.concept_sets = [concepts]
def get_program_result(program):
# get the selected sentences
selection = []
for id in program.output:
if id.startswith("s") and program.output[id] == 1:
node = program.binary[
id] # gives you back the actual node (which can be a subsentence, or a chunk not removed)
if not program.nodeHasSelectedParent(
node): # only start printing at the topmost nodes
# create a fake sentence to hold the compressed content
sentence = text.Sentence(compression.postProcess(program.getSelectedText(node)), \
node.root.sentence.order, node.root.sentence.source, node.root.sentence.date)
sentence.parsed = str(node)
sentence.original_node = node
selection.append(sentence)
#print node.root.getPrettyCandidates()
return selection
def __init__(self, id, title, narr, new_docs, old_docs):
self.id = id
self.title = title
self.narr = narr
self.query = text.Sentence(title + ": " + narr)
self.new_docs_paths = new_docs[:]
self.old_docs_paths = old_docs[:]
## for checking state
self.loaded_docs = False
self.parsed = False
self.loaded_ir_docs = False
## variables that might get set later
self.new_docs = None
self.old_docs = None
self.training = {}
self.annotators = set()
def by_dendrogram(sentences, concept_weight, problem):
cooccurrences = build_cooccurrence_matrix(problem)
clusters = []
for sentence in sentences:
#concepts = util.get_ngrams(sentence.stemmed, n=1, bounds=False)
#concepts = dict([(x, concept_weight[x]) for x in concepts if x in concept_weight])
concepts = dict([(x, 1.0) for x in sentence.no_stop])
clusters.append(Cluster(concepts, text=sentence))
while len(clusters) > 1:
max_sim = -1
argmax = (None, None)
for i in range(len(clusters)):
for j in range(i + 1, len(clusters)):
#similarity = clusters[i].similarity(clusters[j]) + clusters[j].similarity(clusters[i])
similarity = 0.0
for word in clusters[i].words:
for peer in clusters[j].words:
occurrence = tuple(sorted([word, peer]))
if occurrence in cooccurrences:
similarity += cooccurrences[occurrence]
similarity /= len(clusters[i].words) * len(clusters[j].words)
#print similarity, i, j
if (similarity > max_sim):
max_sim = similarity
argmax = (i, j)
child1 = clusters[argmax[0]]
child2 = clusters[argmax[1]]
#print max_sim, child1.original_order, child2.original_order
#clusters[argmax[0]] = Cluster(child1.words | child2.words, child1, child2)
clusters[argmax[0]] = Cluster(
merge_vectors(child1.words, child2.words), child1, child2)
del clusters[argmax[1]]
#print clusters[0]
fake_query = text.Sentence('')
fake_query.no_stop = {}
for concept in concept_weight:
for gram in concept:
if gram not in text.text_processor._stopwords:
fake_query.no_stop[gram] = 1
return clusters[0].get_ordered(fake_query)
def build_alternative_program(problem,
concept_weight,
length=100,
sentences=None,
longuest_candidate_only=False,
providedAcronyms=None):
if not sentences:
sentences = problem.get_new_sentences()
for sentence in problem.get_new_and_old_sentences():
if not hasattr(sentence, "compression_node"):
sentence.compression_node = compression.TreebankNode(
sentence.parsed)
nounPhraseMapping = compression.generateNounPhraseMapping(
[s.compression_node for s in problem.get_new_and_old_sentences()])
#print "generating acronyms"
acronymMapping = None
if providedAcronyms:
acronymMapping = providedAcronyms
else:
acronymMapping = compression.generateAcronymMapping(
problem.get_new_and_old_sentences())
print problem.id, acronymMapping
compressed_sentences = []
seen_sentences = {}
group_id = 0
for sentence in sentences:
subsentences = sentence.compression_node.getNodesByFilter(
compression.TreebankNode.isSubsentence)
candidates = {}
for node in subsentences:
candidates.update(
node.getCandidates(beam=100,
mapping=nounPhraseMapping,
use_mandatory_removals=True))
if longuest_candidate_only:
max_length = 0
argmax = None
for candidate in candidates:
if len(candidate) > max_length:
max_length = len(candidate)
argmax = candidate
if argmax != None:
candidates = [argmax]
for candidate in candidates:
new_sentence = text.Sentence(compression.postProcess(candidate),
sentence.order, sentence.source,
sentence.date)
if new_sentence.length <= 5: continue # skip short guys
new_sentence.group_id = group_id
compressed_sentences.append(new_sentence)
seen_sentences[new_sentence.original] = 1
group_id += 1
compression.replaceAcronyms(compressed_sentences, acronymMapping)
#log_file = open("%s.log" % problem.id, "w")
#for sentence in compressed_sentences:
# log_file.write("%d %s\n" %( group_id, str(sentence)))
#log_file.close()
# generate ids for acronyms
acronym_id = {}
acronym_length = {}
for definition, acronym in acronymMapping.items():
if acronym not in acronym_id:
acronym_id[acronym] = len(acronym_id)
acronym_length[acronym] = len(definition.strip().split())
# get concepts
relevant_sentences = []
sentence_concepts = []
groups = {}
used_concepts = set()
acronym_index = {}
sent_index = 0
for sentence in compressed_sentences:
units = util.get_ngrams(sentence.stemmed, n=2, bounds=False)
overlapping = set([u for u in units if u in concept_weight])
if len(overlapping) == 0:
continue # get rid of sentences that do not overlap with concepts
relevant_sentences.append(sentence)
sentence_concepts.append(overlapping)
used_concepts.update(overlapping)
if sentence.group_id not in groups: groups[sentence.group_id] = []
groups[sentence.group_id].append(sent_index)
# generate an acronym index
for acronym in acronym_id:
if re.search(r'\b' + acronym + r'\b', sentence.original):
if acronym not in acronym_index: acronym_index[acronym] = []
acronym_index[acronym].append(sent_index)
sent_index += 1
# build inverted index
filtered_concepts = {}
concept_index = {}
index = 0
for concept in used_concepts:
concept_index[concept] = index
filtered_concepts[concept] = concept_weight[concept]
index += 1
relevant_sent_concepts = [[concept_index[c] for c in cs]
for cs in sentence_concepts]
concept_weights = filtered_concepts
curr_concept_sents = {}
for sent_index in range(len(relevant_sentences)):
concepts = relevant_sent_concepts[sent_index]
for concept in concepts:
if not concept in curr_concept_sents:
curr_concept_sents[concept] = []
curr_concept_sents[concept].append(sent_index)
# generate the actual ILP
program = ilp.IntegerLinearProgram()
program.objective["score"] = ' + '.join([
'%f c%d' % (concept_weight[concept], concept_index[concept])
for concept in concept_index
])
s1 = ' + '.join([
'%d s%d' % (relevant_sentences[sent_index].length, sent_index)
for sent_index in range(len(relevant_sentences))
])
# add enough space to fit the definition of each acronym employed in the summary
s_acronyms = ' + '.join([
'%d a%d' % (acronym_length[acronym], acronym_id[acronym])
for acronym in acronym_id
])
if s_acronyms != "":
s_acronyms = " + " + s_acronyms
s2 = ' <= %s\n' % length
program.constraints["length"] = s1 + s_acronyms + s2
for concept, index in concept_index.items():
## at least one sentence containing a selected bigram must be selected
s1 = ' + '.join(
['s%d' % sent_index for sent_index in curr_concept_sents[index]])
s2 = ' - c%d >= 0' % index
program.constraints["presence_%d" % index] = s1 + s2
## if a bigram is not selected then all sentences containing it are deselected
#### this constraint is disabled since it is not necessary when all sentences contain at least one concept
#### it might also be the reason for singlar matrices that crash the solver
#s1 = ' + '.join([ 's%d' %sent_index for sent_index in curr_concept_sents[index]])
#s2 = ' - %d c%d <= 0' %(len(curr_concept_sents[index]), index)
#program.constraints["absence_%d" % index] = s1 + s2
# constraints so that acronyms get selected along with sentences they belong to
for acronym, index in acronym_index.items():
s1 = ' + '.join(['s%d' % sent_index for sent_index in index])
s2 = ' - a%d >= 0' % acronym_id[acronym]
program.constraints["acronym_presence_%d" %
acronym_id[acronym]] = s1 + s2
s1 = ' + '.join(['s%d' % sent_index for sent_index in index])
s2 = ' - %d a%d <= 0' % (len(index), acronym_id[acronym])
program.constraints["acronym_absence_%d" %
acronym_id[acronym]] = s1 + s2
# add sentence compression groups
for group in groups:
if len(groups[group]) > 1:
program.constraints["group_%d" % group] = " + ".join(
["s%d" % sent_index for sent_index in groups[group]]) + " <= 1"
for sent_index in range(len(relevant_sentences)):
program.binary["s%d" % sent_index] = relevant_sentences[sent_index]
for concept, concept_index in concept_index.items():
program.binary["c%d" % concept_index] = 1
for acronym, id in acronym_id.items():
program.binary["a%d" % id] = 1
sys.stderr.write("compression candidates: %d, original: %d\n" %
(len(relevant_sentences), len(sentences)))
program.acronyms = acronymMapping
return program
def run_standard(options, max_sents=10000):
## create output directory
try:
os.popen('rm -rf %s' % options.output)
except:
pass
try:
os.popen('mkdir -p %s' % options.output)
except:
sys.stderr.write('Error: could not create output directory [%s]\n')
sys.exit()
## summarize!
sys.stderr.write('generating summaries for task [%s]\n' % options.task)
sys.stderr.write('length limit [%d]\n' % task.length_limit)
sys.stderr.write('writing output to [%s]\n' % options.output)
map_times, run_times = {}, {}
## sentence compression
if options.compress:
for problem in task.problems:
if not '-A' in problem.id: continue
sys.stderr.write(
"%s %d\n" %
(problem.id,
sum([len(doc.sentences) for doc in problem.new_docs])))
#mapper = concept_mapper.HeuristicMapper(problem, "n2")
mapper = concept_mapper.CheatingMapper(problem, "n2")
mapper.map_concepts()
mapper.choose_sents()
concept_weights = mapper.concept_weights
#print concept_weight
#program = framework.build_program(problem, concept_weight, length=task.length_limit, sentences=mapper.relevant_sent_sets[0])
program = framework.build_alternative_program(
problem,
concept_weights,
length=task.length_limit,
sentences=mapper.relevant_sents,
longuest_candidate_only=False)
# run the program and get the output
program.debug = 0
program.run()
#selection = framework.get_program_result(program)
selection = []
for variable in program.output:
if re.match(r'^s\d+$',
variable) and program.output[variable] == 1:
selection.append(program.binary[variable])
selection = ordering.by_date(selection)
summary = "\n".join(sentence.original for sentence in selection)
#summary = compression.addAcronymDefinitionsToSummary(summary, program.acronyms)
## TAC id convention is annoying
output_id = problem.id
if options.task in ['u09', 'u08']:
output_id = problem.id[:5] + problem.id[6:]
output_file = open('%s/%s' % (options.output, output_id), 'w')
output_file.write(summary)
output_file.close()
elif options.mcd:
for problem in task.problems:
num_problem_sentences = len(problem.get_new_sentences())
if num_problem_sentences < 500: continue
used_sent_count = 0
for sentence in problem.get_new_sentences():
used_sent_count += 1
sentence.set_text(sentence.original)
if used_sent_count < max_sents: sentence.used = True
else: sentence.used = False
problem.query.set_text(problem.query.original)
sys.stdout.write(
"%s %d\n" %
(problem.id,
sum([len(doc.sentences) for doc in problem.new_docs])))
# compute idf values
word_idf = {}
for doc in problem.new_docs:
seen_words = {}
for sentence in doc.sentences:
if not sentence.used: continue
for word in sentence.no_stop_freq:
if word not in seen_words: seen_words[word] = 1
for word in seen_words:
if word not in word_idf: word_idf[word] = 1
else: word_idf[word] += 1
for word in word_idf:
word_idf[word] = 1.0 / word_idf[word]
# compare sentences to centroid and derive McDonald's relevance score
sentences = []
index = 0
for doc in problem.new_docs:
doc_text = " ".join([
sentence.original for sentence in doc.sentences
if sentence.used
])
centroid = text.Sentence(doc_text)
centroid.compute_norm()
problem.query.compute_norm()
for sentence in doc.sentences:
if not sentence.used: continue
sentence.compute_norm()
sentence.rel_score = sentence.sim_cosine(
centroid, word_idf) + 1 / (sentence.order + 1)
#sentence.rel_score = sentence.sim_cosine(centroid, word_idf) + sentence.sim_cosine(problem.query, word_idf)
sentences.append(sentence)
sentence.index = index
index += 1
# apply cutoff
sentences.sort(lambda x, y: 1 if x.rel_score < y.rel_score else -1)
if options.cutoff > 0 and len(sentences) > options.cutoff:
sentences = sentences[0:options.cutoff]
# construct ILP
program = ilp.IntegerLinearProgram(debug=0)
objective = []
length_constraint = []
for sentence in sentences:
objective.append("%+g s%d" %
(sentence.rel_score, sentence.index))
program.binary["s%d" % sentence.index] = sentence
length_constraint.append("%+g s%d" %
(sentence.length, sentence.index))
for peer in sentences:
if sentence == peer: continue
score = sentence.sim_cosine(peer, word_idf)
if score > 0:
objective.append("%+g s%d_%d" %
(-score, sentence.index, peer.index))
program.binary["s%d_%d" %
(sentence.index, peer.index)] = [
sentence, peer
]
program.constraints["c1_%d_%d" % (sentence.index, peer.index)] = \
"s%d_%d - s%d <= 0" % (sentence.index, peer.index, sentence.index)
program.constraints["c2_%d_%d" % (sentence.index, peer.index)] = \
"s%d_%d - s%d <= 0" % (sentence.index, peer.index, peer.index)
program.constraints["c3_%d_%d" % (sentence.index, peer.index)] = \
"s%d + s%d - s%d_%d <= 1" % (sentence.index, peer.index, sentence.index, peer.index)
program.objective["score"] = " ".join(objective)
program.constraints["length"] = " ".join(
length_constraint) + " <= %g" % task.length_limit
run_times[problem.id] = time.time()
program.run()
run_times[problem.id] = time.time() - run_times[problem.id]
selection = []
score = 0
# get solution and check consistency
for variable in program.binary:
if variable in program.output and program.output[variable] == 1:
if type(program.binary[variable]) == type(sentences[0]):
selection.append(program.binary[variable])
score += program.binary[variable].rel_score
for peer in program.output:
if program.output[
peer] == 0 or peer == variable or type(
program.binary[peer]) != type(
sentences[0]):
continue
if program.binary[variable].sim_cosine(
program.binary[peer], word_idf) == 0:
continue
quadratic = "s%d_%d" % (
program.binary[variable].index,
program.binary[peer].index)
if quadratic not in program.output or program.output[
quadratic] != 1:
print "WARNING: %s selected but %s not selected" % (
variable, quadratic)
else:
score -= program.binary[variable][0].sim_cosine(
program.binary[variable][1], word_idf)
if program.output[
"s%d" %
program.binary[variable][0].index] != 1:
print "WARNING: %s selected while s%d not selected" % (
variable, program.binary[variable][0].index)
if program.output[
"s%d" %
program.binary[variable][1].index] != 1:
print "WARNING: %s selected while s%d not selected" % (
variable, program.binary[variable][1].index)
#if math.fabs(program.result["score"] - score) > .1:
# print "WARNING: difference between score = %g and expected = %g" % (program.result["score"], score)
selection = ordering.by_date(selection)
new_id = re.sub(r'.-(.)$', r'-\1', problem.id)
output_file = open("%s/%s" % (options.output, new_id), "w")
for sentence in selection:
output_file.write(sentence.original + "\n")
output_file.close()
else:
hist = prob_util.Counter()
input_sents = []
for problem in task.problems:
num_problem_sentences = len(problem.get_new_sentences())
#if num_problem_sentences < 300: continue
if not '-A' in problem.id: continue
if options.ir:
#docs = [doc for doc, val in problem.ir_docs]
#for doc in docs: doc.get_sentences()
num_overlap = len(
set([d.id for d in problem.ir_docs
]).intersection(set([d.id
for d in problem.new_docs])))
print '%s overlap: %d' % (problem.id, num_overlap)
info_fh.write('%s overlap [%d]\n' % (problem.id, num_overlap))
sys.stderr.write('problem [%s] input sentences [%d]' %
(problem.id, num_problem_sentences))
input_sents.append(num_problem_sentences)
## select a concept mapper
map_times[problem.id] = time.time()
if options.cheat:
mapper = concept_mapper.CheatingMapper(problem, options.units)
else:
mapper = concept_mapper.HeuristicMapperExp(
problem, options.units)
## timing test
mapper.max_sents = max_sents
## map input concepts to weights
success = mapper.map_concepts()
if not success: sys.exit()
## choose a subset of the input sentences based on the mapping
success = mapper.choose_sents()
if not success: sys.exit()
map_times[problem.id] = time.time() - map_times[problem.id]
## testing
#fh = open('concept_matrix', 'w')
for sent in mapper.relevant_sent_concepts:
hist[len(sent)] += 1
#fh.write(''.join(['%d, ' %concept for concept in sent[:-1]]))
#fh.write('%d\n' %sent[-1])
hist[0] += (num_problem_sentences -
len(mapper.relevant_sent_concepts))
#hist.displaySorted(N=100)
#sys.exit()
## end testing
## setup and run the ILP
run_times[problem.id] = time.time()
selection = mapper.run(task.length_limit)
selection = ordering.by_date(selection)
run_times[problem.id] = time.time() - run_times[problem.id]
## TAC id convention is annoying
output_id = problem.id
if options.task in ['u09', 'u08']:
output_id = problem.id[:5] + problem.id[6:]
output_file = open('%s/%s' % (options.output, output_id), 'w')
word_count = 0
for sentence in selection:
output_file.write(sentence.original + '\n')
word_count += len(sentence.original.split())
output_file.close()
curr_time = map_times[problem.id] + run_times[problem.id]
sys.stderr.write(' word count [%d] time [%1.2fs]\n' %
(word_count, curr_time))
def setup_features(problem, unit_selector, train=True):
## for training, get gold concepts
gold_concepts = prob_util.Counter()
if train:
for annotator in problem.annotators:
annotator_concepts = {}
for sent in problem.training[annotator]:
sentence = text.Sentence(sent)
units = unit_selector(sentence.stemmed)
for unit in units:
if unit not in annotator_concepts: annotator_concepts[unit] = 0
annotator_concepts[unit] += 1
for concept in annotator_concepts:
gold_concepts[concept] += 1
## get all sentences and unit frequencies
sents = []
doc_freq = prob_util.Counter()
sent_freq = prob_util.Counter()
raw_freq = prob_util.Counter()
for doc in problem.new_docs:
#if doc.doctype != 'NEWS STORY': continue
doc_counts = prob_util.Counter()
for sent in doc.sentences:
sent_counts = prob_util.Counter()
sents.append(sent)
for unit in unit_selector(sent.stemmed):
doc_counts[unit] += 1
sent_counts[unit] += 1
for unit in sent_counts:
sent_freq[unit] += 1
for unit in doc_counts:
doc_freq[unit] += 1
raw_freq[unit] += doc_counts[unit]
## get features for each concept unit
lines = []
concepts = []
title = text.Sentence(problem.title)
narr = text.Sentence(problem.narr)
for sent in sents:
## sentence features
sentence_sim = sent.sim_basic(problem.query)
sentence_order = sent.order
sentence_source = sent.source
sentence_length = sent.length
units = unit_selector(sent.stemmed)
for unit in units:
## concept features
stopword_ratio = 1 - (1.0*len(text.text_processor.remove_stopwords(unit)) / len(unit))
doc_ratio = 1.0 * doc_freq[unit] / len(problem.new_docs)
sent_ratio = 1.0 * sent_freq[unit] / len(sents)
ngram = ' '.join(unit)
sunit = text.Sentence(ngram)
title_sim = sunit.sim_basic(title)
narr_sim = sunit.sim_basic(narr)
## output format (boostexter)
line = '%s, %1.2f, %1.2f, %1.2f, ' %(ngram, doc_ratio, sent_ratio, stopword_ratio)
line += '%1.2f, %d, %s, %d, ' %(sentence_sim, sentence_order, sentence_source, sentence_length)
line += '%1.2f, %1.2f, ' %(title_sim, narr_sim)
if train: line += '%s' %int(gold_concepts[unit]>0)
else: line += '0'
line += '.'
if stopword_ratio == 1: continue
lines.append(line)
concepts.append(unit)
for rep in range(int(gold_concepts[unit]-1)):
if train:
lines.append(line)
concepts.append(unit)
return lines, concepts, doc_freq
def decode(self, weights, feats, relax=False, display_output=False,
**kwargs):
"""Decode a transduction for this instance.
"""
if not hasattr(self, 'decoder'):
# Looks like we've given up on this one
print "wut?"
return
# Update weights and solve the LP. Don't save the internal LP
# if we don't have much memory left. TODO: handle large LPs gracefully
self.decoder.update(weights, first_call=(len(self.decode_times)==0))
start_moment = time.time()
self.decoder.solve(save=psutil.virtual_memory()[2] < 90,
relax=relax,
**kwargs)
self.decode_times.append(time.time() - start_moment)
if self.decoder.has_solution():
if not relax:
# Recover the current sentence and optionally display it.
# Note that output tokens are stored separately for evaluation.
# TODO: added timing here as well
start_moment2 = time.time()
self.output_idxs = self.decoder.get_solution(self)
self.solution_times.append(time.time() - start_moment2)
self.output_tokens = [self.input_sents[s].tokens[w]
for s, w in self.output_idxs]
if len(self.output_tokens) > 0:
self.output_sent = text.Sentence(self.output_tokens)
if 'dep' in feats.categories:
# Record inferred parse
self.output_sent.outtree = \
self.decoder.get_tree_solution(
self, self.output_idxs)
if 'arity' in feats.categories:
self.decoder.verify_arity(self, self.output_idxs,
active_tree=self.output_sent.outtree)
if 'range' in feats.categories:
self.decoder.verify_range(self, self.output_idxs,
active_tree=self.output_sent.outtree)
if 'frame' in feats.categories or 'fe' in feats.categories:
self.output_sent.outframes = \
self.decoder.get_frame_solution(
self, self.output_idxs)
if display_output:
print self.get_display_string()
print self.output_sent.outtree.to_text(
ext_nodes=self.output_tokens,
attribute='score')
print
# HACK: Before returning the final feature vector, we should
# nullify the fixed feature values so that scores will be
# displayed properly by the learner. For this, we replace
# them with the corresponding gold feature values, if available.
feat_vector = self.decoder.get_solution_feats()
return feats.sanitize_feat_vector(feat_vector, self)
else:
print "Failed on instance", self.idx
self.failed_count += 1
for s, sent in enumerate(self.input_sents):
print str(s) + ':', ' '.join(sent.tokens)
for edge in sent.dparse.edges:
print sent.tokens[edge.src_idx],
print '--[' + edge.label + ']->',
print sent.tokens[edge.tgt_idx]
# If this rarely succeeds, don't bother with this instance in
# the future
if self.failed_count >= 2:
delattr(self, 'decoder')
return None