def testSource(self):
corpus1 = annotationIo.load("data/corpusWithPathsSmall.v0.yaml")
annotation1 = corpus1[0]
esdc1 = annotation1.esdcs[0]
annotation1.setSource(esdc1, "person 1")
self.assertEqual(annotation1.getSource(esdc1), "person 1")
annotationIo.save(corpus1, "data/corpusWithPathsSmall.v1.yaml")
corpus2 = annotationIo.load("data/corpusWithPathsSmall.v1.yaml")
annotation2 = corpus2[0]
esdc2 = annotation2.esdcs[0]
self.assertEqual(annotation1.getSource(esdc1),
annotation2.getSource(esdc2))
def main():
"""
Splits a corpus yaml file into multiple smaller files, for faster
annotation saving and loading.
"""
from optparse import OptionParser
parser = OptionParser()
parser.add_option("--corpus_fname", dest="corpus_fname")
parser.add_option("--page_size", dest="page_size", type="int")
parser.add_option("--result_dir", dest="result_dir", metavar="FILE")
(options, args) = parser.parse_args()
corpus = annotationIo.load(options.corpus_fname)
pages = []
current_page = []
for annotation in corpus:
current_page.append(annotation)
if len(current_page) >= options.page_size:
pages.append(current_page)
current_page = []
if len(current_page) != 0:
pages.append(current_page)
if not os.path.exists(options.result_dir):
os.makedirs(options.result_dir)
basename = os.path.basename(options.corpus_fname)
name = basename[0:-5]
extension = basename[-5:]
assert extension == ".yaml"
for page_i, page in enumerate(pages):
fname = "%s/%s.page_%d.yaml" % (options.result_dir, name, page_i)
annotationIo.save(page, fname)
def testGroundings(self):
corpus = annotationIo.load(SOURCE_FILE)
annotation = corpus[0]
esdc = annotation.flattenedEsdcs[0]
annotation.addGrounding(
esdc,
PhysicalObject(
Prism.from_points_xy(tp([(0, 0), (1, 0), (1, 1), (0, 1)]), 3,
4), ["tire", "pallet"]))
annotation.addGrounding(
esdc,
Place(
Prism.from_points_xy(tp([(0, 0), (1, 0), (1, 1), (0, 1)]), 3,
4)))
annotation.addGrounding(
esdc,
Path.from_xyztheta(timestamps=[0, 1],
points_xyztheta=pts_to_xyzTheta([(0, 0),
(1, 1)])))
yamlCorpus = annotationIo.toYaml(corpus)
print "yaml", yamlCorpus
newCorpus = annotationIo.fromYaml(yamlCorpus)
esdc1 = corpus[0].flattenedEsdcs[0]
esdc2 = newCorpus[0].flattenedEsdcs[0]
null_ids(esdc1)
null_ids(esdc2)
self.assertEqual(esdc1, esdc2)
def load(self, fname=None):
if fname == None:
fname = QFileDialog.getLoadFileName(self, "Open File", self.fname)
if fname != "":
self.fname = fname
annotations = annotationIo.load(self.fname)
self.loadAnnotations(annotations)
def main():
trainer = Trainer()
#dirName = "%s/tools/forklift/dataAnnotation/data" % os.environ["SLU_HOME"]
#positiveFname = dirName + "/forkliftMturkEsdcs.stefie10.yaml"
fname = "%s/tools/forklift/dataAnnotation/data/forkliftMturkEsdcs.stefie10.groundings.yaml" % os.environ["SLU_HOME"]
corpus = annotationIo.load(fname)
trainer.trainAndPlot(corpus)
def evaluate_objects(model, corpus_fname, state_type):
corpus = annotationIo.load(corpus_fname)
state_cls = state_type_from_name(state_type)
from g3.inference import nodeSearch
taskPlanner = nodeSearch.BeamSearch(model)
predictions = []
done = False
phrases = set()
for i, annotation in enumerate(corpus):
start_state = state_cls.from_context(annotation.context)
for esdc in annotation.esdcs:
#if esdc.text != "the pallet of boxes":
# continue
#if esdc.text in phrases:
# continue
isCorrect = annotation.isGroundingCorrect(esdc)
if isCorrect != None:
ggg = ggg_from_esdc(esdc)
groundings = annotation.getGroundings(esdc)
assert len(groundings) == 1
grounding = groundings[0]
#if "generator" not in grounding.tags:
# continue
prob = evaluate_ggg(ggg, grounding, start_state, taskPlanner)
if prob > 0.7:
predicted_class = True
else:
predicted_class = False
predictions.append((predicted_class, isCorrect))
#print "Query: Is object", " ".join(grounding.tags),
#print "'" + esdc.text + "'?"
#print "System: ",
#if predicted_class:
# print "Yes."
#else:
# print "No."
#done = True
phrases.add(esdc.text)
if done:
break
if done:
break
tp = len([(p, l) for p, l in predictions if p and p == l])
fp = len([(p, l) for p, l in predictions if p and p != l])
tn = len([(p, l) for p, l in predictions if not p and p == l])
fn = len([(p, l) for p, l in predictions if not p and p != l])
cm = ConfusionMatrix(tp, fp, tn, fn)
#cm.print_all()
#if len(phrases) > 20:
# phrases = random.sample(phrases, 20)
#for phrase in sorted(phrases):
# print phrase
return cm
def testDuplicateEsdcs(self):
corpus = annotationIo.load(SOURCE_FILE)
annotation = corpus[-1]
self.assertEqual(len(annotation.esdcs), 3)
for i, esdc in enumerate(annotation.esdcs):
groundings = annotation.getGroundings(esdc)
self.assertEqual(len(groundings), 1)
grounding = groundings[0]
print "id", esdc.id
self.assertEqual(grounding.tags, ("trailer%d" % (i + 1), ))
def main():
corpus = annotationIo.load(
"%s/tools/forklift/dataAnnotation/data/negativeEsdcs.yaml" % SLU_HOME)
assignment_id_to_count = collections.defaultdict(lambda: 0)
new_corpus = []
for a in corpus:
assignment_id_to_count[a.id] += 1
if assignment_id_to_count[a.id] <= 2:
new_corpus.append(a)
annotationIo.save(annotationIo.Corpus(new_corpus), "test.yaml")
def main():
from optparse import OptionParser
parser = OptionParser()
parser.add_option("--cost-function-class", dest="cost_function_class")
parser.add_option("--model", dest="model_fname")
parser.add_option("--dataset", dest="dataset_fname", default=None)
parser.add_option("--corpus-fname", dest="corpus_fnames")
parser.add_option("--runid", dest="run_id", type="int", default=-1)
parser.add_option("--userrt", dest="use_rrt", action="store_true", default=False)
parser.add_option("--esdcmulti", dest="esdc_multi", type="string", default="False")
parser.add_option("--state-type",dest="state_type",
help="State or Agent Type", metavar="FILE")
parser.add_option("--esdc-extractor", type="string", dest="esdc_extractor")
parser.add_option("--esdc-extractor-model", type="string",
dest="esdc_extractor_model")
parser.add_option("-m", "--merging", dest="use_merging",
help="Use merging?", metavar="FILE")
(options, args) = parser.parse_args()
cost_function_cls = make_cost_function_class(options.cost_function_class)
model = cost_function_cls(options.model_fname)
if options.dataset_fname != None:
dataset = cPickle.load(open(options.dataset_fname))
else:
dataset = None
runId = options.run_id
multiEsdcs = eval(options.esdc_multi)
use_merging = eval(options.use_merging)
#########semantic map for d8##########
# smap = CarmenSemanticMap("%s/data/directions/direction_floor_8_full/direction_floor_8_full.cmf.gz" % SLU_HOME,
# "%s/data/directions/direction_floor_8_full/tags/df8_full_tags.tag" % SLU_HOME,
# "%s/data/directions/direction_floor_8_full/partitions/d8_full_part.pck" % SLU_HOME)
#state_cls = state_type_from_name("forklift")
state_cls = state_type_from_name(options.state_type)
corpus = annotationIo.load(options.corpus_fnames)
print "corpus is", corpus
extractor_func = extractor_utils.make_extractor_func(options.esdc_extractor,
options.esdc_extractor_model)
print "func", extractor_func
print "multi", multiEsdcs
evaluateDataSet(model, corpus, dataset, extractor_func, state_cls,
useRrt=options.use_rrt, runId=runId,
runMultiEsdcCommands=multiEsdcs, use_merging=use_merging)
def main():
corpus = annotationIo.load(
"dataAnnotation/data/forkliftMturkEsdcs.stefie10.groundings.yaml")
for annotation in corpus:
esdcs = annotation.esdcs
for esdc in esdcs.flattenedEsdcs:
if (not esdc.childIsEmpty("r") and esdc.type == "OBJECT"
or esdc.type == "PLACE"):
rwords = [str(e.text) for e in esdc.r]
if (len(rwords) > 1 and not "right" in rwords
and not "left" in rwords and not "front" in rwords
and not "next" in rwords):
print esdc.text
def main():
corpus = annotationIo.load(
"dataAnnotation/data/spatialRelations.stefie10.yaml")
sr_to_num_positive = collections.defaultdict(lambda: 0)
sr_to_num_negative = collections.defaultdict(lambda: 0)
srs = set()
total = 0
for annotation in corpus:
path_esdc = annotation.esdcs[0]
sr = path_esdc.r[0].text
srs.add(sr)
if annotation.isGroundingCorrect(path_esdc):
sr_to_num_positive[sr] += 1
else:
sr_to_num_negative[sr] += 1
total += 1
for sr in sorted(srs):
print sr, sr_to_num_positive[sr], sr_to_num_negative[sr]
print "total", total
def testReader(self):
corpus = annotationIo.load(SOURCE_FILE)
assignmentIds = set()
for annotation in corpus:
assignmentIds.add(annotation.assignmentId)
self.assertEqual(len(assignmentIds), len(corpus))
yamlCorpus = annotationIo.toYaml(corpus)
newCorpus = annotationIo.fromYaml(yamlCorpus)
esdc1 = corpus[0].esdcs[-1]
esdc2 = newCorpus[0].esdcs[-1]
null_ids(esdc1)
null_ids(esdc2)
self.assertEqual(esdc1, esdc2)
self.assertEqual(corpus[0].esdcToGroundings[esdc1],
newCorpus[0].esdcToGroundings[esdc2])
self.assertEqual(corpus[0], newCorpus[0])
for a1, a2 in zip(corpus, newCorpus):
print "***********"
print "a1"
print[str(e) for e in a1.esdcs]
print "a2"
print[str(e) for e in a2.esdcs]
for esdc1, esdc2 in zip(a1.flattenedEsdcs, a2.flattenedEsdcs):
self.assertEqual(esdc1.text, esdc2.text)
groundings1 = a1.getGroundings(esdc1)
groundings2 = a2.getGroundings(esdc2)
self.assertEqual(len(groundings1), len(groundings2))
for g1, g2 in zip(groundings1, groundings2):
self.assertEqual(g1, g2)
def main(argv):
app = basewindow.makeApp()
from optparse import OptionParser
parser = OptionParser()
parser.add_option("--training_filename",
dest="training_fname",
help="Training Filename",
metavar="FILE")
(options, args) = parser.parse_args()
annotations = annotationIo.load(options.training_fname)
annotation = annotations[0]
state, esdc_to_ggg = annotation_to_ggg_map(annotation)
ggg = esdc_to_ggg[annotation.esdcs[0]]
wnd = MainWindow()
wnd.show()
wnd.load(ggg)
app.exec_()
def main():
corpus = annotationIo.load(
"dataAnnotation/data/forkliftMturkEsdcs.stefie10.groundings.yaml")
wordHist = Histogram()
fieldToHist = {}
count = 0
for annotation in corpus:
for token in annotation.entireText.split():
if not token in stopwords:
wordHist.add(token.lower())
esdcs = annotation.esdcs
for esdc in esdcs:
for fieldName in esdc.fieldNames:
fieldToHist.setdefault(fieldName, Histogram())
for word in esdc.childTokens(fieldName):
text = word.text
if not text in stopwords:
fieldToHist[fieldName].add(text.lower())
count += 1
if count >= 10:
#break
pass
graphStacked({"words": wordHist}, "histogram", "Words", maxCols=10)
for key in ExtendedSdc.fieldNames:
if len(fieldToHist[key].bins) != 0:
graphStacked(
{key: fieldToHist[key]},
"histogram",
ExtendedSdc.fieldNamesToDescriptions[key].capitalize(),
maxCols=10)
mpl.show()
def main():
import sys
fname = sys.argv[1]
assignments = readCorpus.Corpus(
"dataCollection/data/corpusCommandsForVideoSmallFilesOnly/")
corpus = annotationIo.load(fname)
word_cnt = 0
workers = set()
scenarios = set()
for annotation in corpus:
assignment = assignments.assignmentForId(annotation.assignmentId)
word_cnt += len(annotation.entireText.split())
workers.add(assignment.workerId)
scenarios.add(assignment.scenario)
if assignment.scenario.name == "put_tire_pallet_on_loaded_truck":
print "command", assignment.scenario.name, annotation.entireText
print len(scenarios), "scenarios"
print len(workers), "annotators"
print word_cnt, "words"
print len(corpus), "commands"
def main():
extractor = Extractor()
oldCorpus = annotationIo.load("%s/dataAnnotation/data/forkliftMturkEsdcs.stefie10.groundings.withPaths.yaml" % os.environ['FORKLIFT_HOME'])
annotations = []
for i, a in enumerate(oldCorpus):
# if i != 140:
# continue
print "doing", i, a.entireText
automatic_esdcs_groups = extractor.extractTopNEsdcs(a.entireText, n=10)
for automatic_esdc_group in automatic_esdcs_groups:
annotation = annotationIo.Annotation(a.assignmentId,
automatic_esdc_group)
for automatic_esdc in automatic_esdc_group.flattenedEsdcs:
if automatic_esdc in a.flattenedEsdcs:
annotation.setGroundingIsCorrect(automatic_esdc, True)
else:
annotation.setGroundingIsCorrect(automatic_esdc, False)
annotations.append(annotation)
annotationIo.save(annotationIo.Corpus(annotations), "negativeEsdcs.yaml")
def testLoadSave(self):
corpus1 = annotationIo.load("data/corpusWithPathsSmall.v0.yaml")
annotationIo.save(corpus1, "data/corpusWithPathsSmall.v1.yaml")
corpus2 = annotationIo.load("data/corpusWithPathsSmall.v1.yaml")
self.assertEqual(len(corpus1), len(corpus2))
def evaluateParallel(options):
cost_function_cls = make_cost_function_class(options.cost_function_class)
model = cost_function_cls.from_mallet(options.model_fname,
feature_extractor_cls=GGGFeatures,
guiMode=False)
if options.test_set_fname != None:
test_set = cPickle.load(open(options.test_set_fname))
else:
test_set = None
runId = options.run_id
multi_esdcs = eval(options.esdc_multi)
use_merging = eval(options.use_merging)
debug("use_merging: %s" % use_merging)
force_sync = eval(options.force_sync)
resolver_type = options.resolver_type
# qa_corpus_fname_1 = "%s/tools/forklift/dataAnnotation/data/dialog_ambiguous_RSS_12/forklift_ambiguous_larger_corpus_all_questions.yaml" % SLU_HOME
qa_corpus_fname_1 = options.qa_corpus_fname_1
qa_corpus_fname_2 = options.qa_corpus_fname_2
qa_corpora = []
if qa_corpus_fname_1:
qa_corpora.append(annotationIo.load(qa_corpus_fname_1))
# qa_corpus_fname_2 = "%s/tools/forklift/dataAnnotation/data/dialog_ambiguous_RSS_12/forklift_ambiguous_larger_corpus_all_questions_set000.yaml" % SLU_HOME
if qa_corpus_fname_2:
qa_corpus_2 = annotationIo.load(qa_corpus_fname_2)
qa_corpora.append(qa_corpus_2)
if resolver_type == "bag_of_words":
from coreference.bag_of_words_resolver import BagOfWordsResolver
resolver = BagOfWordsResolver(
"%s/tools/coreference/models/coref_1.5.pck" % SLU_HOME)
elif resolver_type == "oracle":
from coreference.oracle_resolver import OracleResolver
resolver = OracleResolver(options.corpus_fname)
else:
resolver = None
state_cls = state_type_from_name(options.state_type)
corpus = annotationIo.load_all(options.corpus_fname)
########### UGLY HACK ##############
# The corpus file is always a command set, so we set its ESDC sources to be
# "command" in every case. This is necessary, because our commands_AAAI_11
# datasets don't have their sources set at all.
for annotation in corpus:
for esdc in annotation.flattenedEsdcs:
annotation.setSource(esdc, 'command')
extractor_func = extractor_utils.make_extractor_func(
options.esdc_extractor, options.esdc_extractor_model)
info("num questions: %d" % options.num_questions)
info("num answers: %d" % options.num_answers)
evaluateDataSet(model,
corpus,
test_set,
extractor_func,
state_cls,
options.entropy_metric,
useRrt=options.use_rrt,
runId=runId,
multi_esdcs=multi_esdcs,
use_merging=use_merging,
resolver=resolver,
qa_corpora=qa_corpora,
num_questions=options.num_questions,
num_answers=options.num_answers,
force_sync=force_sync,
question_type=options.question_type,
random_seed=options.random_seed)
def main():
parser = OptionParser()
parser.add_option("--outfile_training",
dest="training_fname",
help="Training Output Filename")
parser.add_option("--outfile_test",
dest="testing_fname",
help="Test Output Filename")
parser.add_option(
"--infile_positive",
dest="positive_fnames",
action="append",
default=[],
help="Positive Filename; default to True if isGroundingCorrect is None"
)
parser.add_option(
"--infile_negative",
dest="negative_fnames",
action="append",
default=[],
help="Negative Filename; default to False if isGroundingCorrect is None"
)
parser.add_option(
"--infile_labeled",
dest="labeled_fnames",
action="append",
default=[],
help="Labeled examples; skip if isGroundingCorrect is None")
parser.add_option("--infile_unlabeled",
dest="unlabeled_fnames",
action="append",
default=[],
help="unlabeld fnames")
parser.add_option("--feature_extractor",
dest="feature_extractor",
help="Feature Extractor Class")
parser.add_option("--split",
dest="split",
type="string",
help="'random' to split randomly; 'scenario' to split " +
"by scenario.")
parser.add_option(
"--training_examples",
dest="training_examples",
action="append",
help=
"Examples that are in the training set; others go in the test set. Can be passed more than once. "
)
(options, args) = parser.parse_args()
try:
from g3.feature_extractor.esdc_features import EsdcFeatures
from g3.feature_extractor.esdc_flattened_features import EsdcFlattenedFeatures
from g3.feature_extractor.grounded_features import GGGFeatures
from g3.feature_extractor.rl_features import RLFeatures
from g3.feature_extractor.bolt_features import BoltFeatures
from g3.feature_extractor.ikea_features import IkeaFeatures
from g3.feature_extractor.sr_features import SrFeatures
#feature_extractor = semantic_map.esdc_semantic_map2.esdc_semantic_map()
feature_extractor_cls = eval(options.feature_extractor)
feature_extractor = feature_extractor_cls()
except:
print "error doing", options.feature_extractor
raise
observations = list()
for positive_fname in options.positive_fnames:
corpus = annotationIo.load(positive_fname)
new_examples = generate_examples(basename(positive_fname),
corpus,
feature_extractor,
default_class_value=True)
if len(new_examples) == 0:
raise ValueError("No examples from" + ` positive_fname `)
observations.extend(new_examples)
for negative_fname in options.negative_fnames:
corpus = annotationIo.load(negative_fname)
new_examples = generate_examples(basename(negative_fname),
corpus,
feature_extractor,
default_class_value=False)
if len(new_examples) == 0:
raise ValueError("No examples from" + ` negative_fname `)
observations.extend(new_examples)
for labeled_fname in options.labeled_fnames:
corpus = annotationIo.load(labeled_fname, check=False)
new_examples = generate_examples(basename(labeled_fname),
corpus,
feature_extractor,
default_class_value=None)
if len(new_examples) == 0:
raise ValueError("No examples from" + ` labeled_fname `)
observations.extend(new_examples)
for unlabeled_fname in options.unlabeled_fnames:
corpus = annotationIo.load(unlabeled_fname)
new_examples = generate_examples(basename(unlabeled_fname),
corpus,
feature_extractor,
default_class_value=None,
force_default_class_value=True)
if len(new_examples) == 0:
raise ValueError("No examples from" + ` unlabeled_fname `)
observations.extend(new_examples)
if options.split == "scenario":
mturkCorpus = readCorpus.Corpus(
"%s/data/corpusCommandsForVideoSmallFilesOnly/" % SLU_HOME)
scenario_names = list(
set(
mturkCorpus.assignmentForId(
obs.annotation.assignmentId.split("_")[0]).scenario.name
for obs in observations))
random.shuffle(scenario_names)
n_training_scenarios = int(ceil(len(scenario_names) * 0.7))
training_scenarios = scenario_names[:n_training_scenarios]
testing_scenarios = scenario_names[n_training_scenarios:]
training = [
o for o in observations if mturkCorpus.assignmentForId(
o.annotation.assignmentId.split("_")[0]).scenario.name in
training_scenarios
]
testing = [
o for o in observations if mturkCorpus.assignmentForId(
o.annotation.assignmentId.split("_")[0]).scenario.name in
testing_scenarios
]
elif options.split == "annotation":
'''
Splits the examples, grouped by annotation.
If the spatial relations corpus is included,
that data will be in the training set only.
'''
training = []
testing = []
sr_ids = []
ids = []
for o in observations:
aid = o.annotation.id
if ((aid not in ids) and ("sr_" not in aid)):
ids.append(aid)
elif "sr_" in aid:
sr_ids.append(aid)
random.shuffle(ids)
n_training_ids = int(ceil(len(ids) * 0.7))
training_ids = ids[:n_training_ids]
testing_ids = ids[n_training_ids:]
training = [
o for o in observations if o.annotation.id in training_ids
or o.annotation.assignmentId in sr_ids
]
testing = [o for o in observations if o.annotation.id in testing_ids]
elif options.split == "random":
random.shuffle(observations)
n_training = int(ceil(len(observations) * 0.7))
training = observations[0:n_training]
testing = observations[n_training:]
elif options.split == "labeled_annotation":
training_ids = set()
training = []
testing = []
for training_fname in options.training_examples:
ds = pickle_util.load(training_fname)
for ex in ds.observations:
training_ids.add(ex.annotation.id)
training_ids.add(ex.annotation.id.split("_")[0])
print "training", training_ids
for example in observations:
if example.annotation.id in training_ids:
training.append(example)
else:
aid = example.annotation.id.split("_")[0]
if aid in training_ids:
training.append(example)
else:
print "skipping", example.annotation.id, aid
testing.append(example)
print "labeled training", len(training)
print "labeled testing", len(testing)
elif options.split == "labeled_file":
training = []
testing = []
for example in observations:
if "training" in example.annotation.fname:
training.append(example)
elif "testing" in example.annotation.fname:
testing.append(example)
else:
training.append(example)
elif options.split == "labeled":
training_ids = set()
training = []
testing = []
for training_fname in options.training_examples:
ds = pickle_util.load(training_fname)
for ex in ds.observations:
print "id", ex.id
training_ids.add(ex.id)
for example in observations:
print "example", example.id
if example.id in training_ids:
training.append(example)
else:
testing.append(example)
else:
raise ValueError("Unexpected split type: " + ` options.split `)
training_dataset = ContinuousDataset(training, feature_extractor_cls)
testing_dataset = ContinuousDataset(testing, feature_extractor_cls)
print "saving ", len(training), " examples to:", options.training_fname
pickle_util.save(options.training_fname, training_dataset)
print "saving ", len(testing), " examples to:", options.testing_fname
pickle_util.save(options.testing_fname, testing_dataset)