def load(identity):
gae_bingo_identity_record = _GAEBingoIdentityRecord.get_by_key_name(
_GAEBingoIdentityRecord.key_for_identity(identity))
if gae_bingo_identity_record:
return pickle_util.load(gae_bingo_identity_record.pickled)
return None
def get_multi(keys, namespace=""):
''' gets multiple KeyValueCache entries at once. It mirrors the
parameters of memcache.get_multi and its return values (ie. it will
return a dict of the original non-namepsaced keys to their values)
'''
namespaced_keys = KeyValueCache.get_namespaced_keys(keys, namespace)
key_values = KeyValueCache.get_by_key_name(namespaced_keys)
values = {}
for (key, key_value) in zip(keys, key_values):
if key_value and not key_value.is_expired():
# legacy entries in key_value cache that were set with
# persist_across_app_versions=True might still be around with
# .pickled=None. Once we are sure they are all expired we can
# delete the "or key_value.pickled is None:"
# TODO(james): After May 2nd 2012 the default cache time of 25
# days should have ended and the or can be removed.
if key_value.pickled or key_value.pickled is None:
values[key] = pickle_util.load(key_value.value)
else:
values[key] = key_value.value
return values
def test_new_class_to_old_class(self):
class NewOldClass(object):
"""New-style class."""
def __init__(self, x, y):
self.x = x
self.y = y
# A trick so we can pickle this class even though it's nested.
setattr(sys.modules[__name__], 'NewOldClass', NewOldClass)
pickled_new = pickle_util.dump(NewOldClass(5, 11))
# Redefine NewOldClass to be old-style
del NewOldClass
class NewOldClass:
"""Old-style class."""
def __init__(self, x, y):
self.x = x
self.y = y
setattr(sys.modules[__name__], 'NewOldClass', NewOldClass)
# Make sure the unpickling uses pickle, not cpickle
old_cpickle = pickle_util.cPickle
old_unpickler_class = pickle_util.g_unpickler_class
try:
pickle_util.cPickle = pickle_util.pickle
pickle_util.g_unpickler_class = (
pickle_util.RenamedClassUnpicklerForPicklePy)
foo = pickle_util.load(pickled_new)
self.assertEqual(5, foo.x)
self.assertEqual(11, foo.y)
self.assertEqual('Old-style class.', foo.__doc__)
finally:
pickle_util.cPickle = old_cpickle
pickle_util.g_unpickler_class = old_unpickler_class
def main():
training_set = pickle_util.load("training.pck")
arc = annotatedRecipe.Corpus(training_set=training_set)
recipeCorpus = arc.recipes
for i in recipeCorpus:
if i.is_training_set == False:
print i.name
def get_multi(keys, namespace=""):
''' gets multiple KeyValueCache entries at once. It mirrors the
parameters of memcache.get_multi and its return values (ie. it will
return a dict of the original non-namepsaced keys to their values)
'''
namespaced_keys = KeyValueCache.get_namespaced_keys(keys, namespace)
key_values = KeyValueCache.get_by_key_name(namespaced_keys)
values = {}
for (key, key_value) in zip(keys, key_values):
if key_value and not key_value.is_expired():
# legacy entries in key_value cache that were set with
# persist_across_app_versions=True might still be around with
# .pickled=None. Once we are sure they are all expired we can
# delete the "or key_value.pickled is None:"
# TODO(james): After May 2nd 2012 the default cache time of 25
# days should have ended and the or can be removed.
if key_value.pickled or key_value.pickled is None:
values[key] = pickle_util.load(key_value.value)
else:
values[key] = key_value.value
return values
def _set_css_deferred(user_data_key, video_key, status, version):
user_data = user_models.UserData.get(user_data_key)
uvc = UserVideoCss.get_for_user_data(user_data)
css = pickle_util.load(uvc.pickled_dict)
id = '.v%d' % video_key.id()
if status == UserVideoCss.STARTED:
if id in css['completed']:
logging.warn("video [%s] for [%s] went from completed->started. ignoring." %
(video_key, user_data_key))
else:
css['started'].add(id)
else:
css['started'].discard(id)
css['completed'].add(id)
uvc.pickled_dict = pickle_util.dump(css)
uvc.load_pickled()
# if set_css_deferred runs out of order then we bump the version number
# to break the cache
if version < uvc.version:
version = uvc.version + 1
user_data.uservideocss_version += 1
db.put(user_data)
uvc.version = version
db.put(uvc)
def load(identity):
gae_bingo_identity_record = (
_GAEBingoIdentityRecord.get_by_key_name(
_GAEBingoIdentityRecord.key_for_identity(identity)))
if gae_bingo_identity_record:
return pickle_util.load(gae_bingo_identity_record.pickled)
return None
def accumulate_from_stat_shard(stat_shard):
shard_val = pickle_util.load(stat_shard.blob_val)
for k, v in shard_val.items():
if isinstance(v, dict):
dict_update_sum(results[k], v)
else:
results[k] += v
def main():
dataset = pickle_util.load("testing.pck")
lcrf = CRFMallet.load("kitchenModel_1.5.pck")
results = []
for i, obs in enumerate(dataset.observations):
entry = resultsModel.Entry(i, lcrf, obs)
results.append(entry)
cm = make_confusion_matrix(results)
cm.print_all()
def get(request_id):
if request_id:
compressed_pickled = memcache.get(RequestStats.memcache_key(request_id))
if compressed_pickled:
pickled = zlib.decompress(compressed_pickled)
return pickle_util.load(pickled)
return None
def main():
from optparse import OptionParser
parser = OptionParser()
parser.add_option("--dfactor", dest="dfactor", default=36)
(options, args) = parser.parse_args()
original_dataset_fname = args[0]
convert_dataset_fname = args[1]
save_fname = args[2]
original_dataset = pickle_util.load(original_dataset_fname)
convert_dataset = pickle_util.load(convert_dataset_fname)
obs = convert_dataset.observations
discrete_dataset = original_dataset.to_discrete_dataset(
dataset=obs, dfactor=int(options.dfactor))
print "saving to...", save_fname
print "...with ", len(discrete_dataset.fnames), "features"
pickle_util.save(save_fname, discrete_dataset)
print "done"
def main():
"""
Evaluates object groundings for the NIST/BOLT evaluation. Takes a
special input file annotated with correctness for noun phrases in
the test corpus.
"""
from optparse import OptionParser
parser = OptionParser()
parser.add_option("--cost-function-class", dest="cost_function_class")
parser.add_option("--model", dest="model_fname")
parser.add_option("--training_fname", dest="training_fname")
parser.add_option("--corpus-fname", dest="corpus_fname")
parser.add_option("--state-type", dest="state_type")
(options, args) = parser.parse_args()
print "training", options.training_fname
training_set = pickle_util.load(options.training_fname)
print "Training on", len(training_set.observations), "examples"
global evaluator
evaluator = Evaluator(options.model_fname, options.cost_function_class,
training_set, options.corpus_fname,
options.state_type)
results = []
args = []
for i in range(1, 100, 10):
fraction = float(i) / 100
args.append((fraction, ))
pool = pool_27.Pool(processes=2, maxtasksperchild=1)
def callback(result):
num_examples, cms = result
print "***** finished results", num_examples
results.append((num_examples, cms))
#args = args[2:3]
for arg in args:
print "apply"
pool.apply_async(evaluate, arg, callback=callback)
#pool.apply_sync(evaluate, arg, callback=callback)
while len(results) < len(args):
time.sleep(1)
print "closing"
pool.close()
print "joining"
pool.join()
fname = "confusion_matrices.pck"
print "saving", fname
pickle_util.save(fname, results)
def load(self, fname):
self.recipeTable.selectRow(0)
self.instructionTable.selectRow(0)
training_set = pickle_util.load("training.pck")
print "training", training_set
recipes = Corpus(fname, training_set)
self.recipeModel.setData(recipes)
self.recipeTable.sortByColumn(recipeModel.COL_IS_TRAINING_SET,
Qt.AscendingOrder)
self.recipeTable.selectRow(0)
def test_rewritten_class_instance(self):
global OldClass
# Mock out the rename-map.
pickle_util._CLASS_RENAME_MAP = {
('pickle_util_test', 'OldClass'):
('mod.submod1.submod2', 'NewClass')
}
pickled = pickle_util.dump(OldClass())
# Just to make this more fun, delete OldClass
del OldClass
actual = pickle_util.load(pickled)
import mod.submod1.submod2
self.assertTrue(isinstance(actual, mod.submod1.submod2.NewClass))
def dest_viterbi_simple(self):
model_fname = "kitchenModel_1.5.pck"
training_set = pickle_util.load("training.pck")
rm = recipeManager.RecipeManager(model_fname)
ingList = [("1/2 cup flour", kitchenState.Ingredient(contains=["flour"], homogenous=True,
amount="1/2 cup", physicalObject=PhysicalObject(prism_from_point(3, 1, 1, 2),
lcmId = 5, tags=['Flour'])))]
instList = [("Mix flour into mixing bowl.", "pour(flour), mix(flour)"),
("Preheat the oven to 350 F.", "preheat(350)"),
("Bake at 350 F for 30 minutes.", "bake(30)")]
ar = annotatedRecipe.AnnotatedRecipe("Temp Name", "Temp Source", ingList, instList)
rm.find_viterbi_plan(ar.instruction_text, ar.start_state)
def test_rewritten_class_instance(self):
global OldClass
# Mock out the rename-map.
pickle_util._CLASS_RENAME_MAP = {
('pickle_util_test', 'OldClass'):
('mod.submod1.submod2', 'NewClass')
}
pickled = pickle_util.dump(OldClass())
# Just to make this more fun, delete OldClass
del OldClass
actual = pickle_util.load(pickled)
import mod.submod1.submod2
self.assertTrue(isinstance(actual, mod.submod1.submod2.NewClass))
def resultFunction(fname, check=True):
if fname.endswith(".pck"):
return pickle_util.load(fname)
else:
pickleFname = "/tmp/%s_%s.pck" % (getuser(), fname.replace(
"/", "_"))
#os.path.basename(fname))
if (os.path.exists(pickleFname) and
os.path.getmtime(pickleFname) > os.path.getmtime(fname)):
try:
print "loading", pickleFname, "...",
obj = pickle_util.load(pickleFname)
print "done"
return obj
except:
print "ignoring, and reloading from yaml"
traceback.print_exc()
print "loading", fname, "then will dump", pickleFname, "..."
annotations = loadFunction(fname, check=check)
pickle_util.save(pickleFname, annotations)
print 'done'
return annotations
def get_result(self, cache_class=memcache, namespace=""):
'''If the results are stored within this ChunkedResult object it will
decompress, depickle and return it. Otherwise it calls get_multi on the
cache_class for all items in its chunk_list, combines them together and
returns the descompressed and depickled result
'''
if hasattr(self, "chunk_list"):
chunked_results = cache_class.get_multi(self.chunk_list,
namespace=namespace)
self.data = ""
for chunk_key in self.chunk_list:
if chunk_key not in chunked_results:
# if a chunk is missing then it is impossible to depickle
# so target func will need to be re-executed
return None
# It is possible that the results come some from a new value of
# a cached item and some from an old version of the cached item
# as set_multi is not atomic. By adding a random generation
# string to the beginning of each chunk we can be sure they are
# all part of the same set, by checking if that string matches
# the one in the index
chunk_result = chunked_results[chunk_key]
chunk_generation = chunk_result[:CHUNK_GENERATION_LENGTH]
if chunk_generation != self.generation:
logging.warning("invalid chunk: wrong generation string"
" in chunk %s" % chunk_key)
return None
self.data += chunk_result[CHUNK_GENERATION_LENGTH:]
if self.compress:
try:
self.data = zlib.decompress(self.data)
except zlib.error:
# If for some reason the data coming back is corrupted so it
# can't be decompressed, we return None in order to recaclulate
# the target function
logging.warning(
"could not decompress ChunkedResult from cache")
return None
try:
return pickle_util.load(self.data)
except Exception:
# If for some reason the data coming back is corrupted so it can't
# be depickled, we will return None in order to recaclulate the
# target function
logging.warning("could not depickle ChunkedResult from cache")
return None
def get_result(self, cache_class=memcache, namespace=""):
'''If the results are stored within this ChunkedResult object it will
decompress, depickle and return it. Otherwise it calls get_multi on the
cache_class for all items in its chunk_list, combines them together and
returns the descompressed and depickled result
'''
if hasattr(self, "chunk_list"):
chunked_results = cache_class.get_multi(self.chunk_list,
namespace=namespace)
self.data = ""
for chunk_key in self.chunk_list:
if chunk_key not in chunked_results:
# if a chunk is missing then it is impossible to depickle
# so target func will need to be re-executed
return None
# It is possible that the results come some from a new value of
# a cached item and some from an old version of the cached item
# as set_multi is not atomic. By adding a random generation
# string to the beginning of each chunk we can be sure they are
# all part of the same set, by checking if that string matches
# the one in the index
chunk_result = chunked_results[chunk_key]
chunk_generation = chunk_result[:CHUNK_GENERATION_LENGTH]
if chunk_generation != self.generation:
logging.warning("invalid chunk: wrong generation string"
" in chunk %s" % chunk_key)
return None
self.data += chunk_result[CHUNK_GENERATION_LENGTH:]
if self.compress:
try:
self.data = zlib.decompress(self.data)
except zlib.error:
# If for some reason the data coming back is corrupted so it
# can't be decompressed, we return None in order to recaclulate
# the target function
logging.warning("could not decompress ChunkedResult from cache")
return None
try:
return pickle_util.load(self.data)
except Exception:
# If for some reason the data coming back is corrupted so it can't
# be depickled, we will return None in order to recaclulate the
# target function
logging.warning("could not depickle ChunkedResult from cache")
return None
def main(argv):
app = basewindow.makeApp()
from sys import argv
model_fname = argv[1]
dataset_fname = argv[2]
crf = CRFMallet.load(model_fname)
dataset = pickle_util.load(dataset_fname)
wnd = MainWindow()
wnd.load(crf, dataset, title=dataset_fname)
wnd.show()
app.exec_()
def load_pickled(self):
max_selectors = 20
css_list = []
css = pickle_util.load(self.pickled_dict)
started_css = '{background-image:url(/images/video-indicator-started.png);padding-left:14px;}'
complete_css = '{background-image:url(/images/video-indicator-complete.png);padding-left:14px;}'
for id in UserVideoCss._chunker(list(css['started']), max_selectors):
css_list.append(','.join(id))
css_list.append(started_css)
for id in UserVideoCss._chunker(list(css['completed']), max_selectors):
css_list.append(','.join(id))
css_list.append(complete_css)
self.video_css = ''.join(css_list)
def evaluateEndToEnd(self, targetRecipe, useBeam=True):
#A list of (planningLanguage, is_correct) tuples. Used mainly for GUI
completePath = []
self.totalCount += 1
model_fname = "kitchenModel_1.5.pck"
training_set = pickle_util.load("training.pck")
rm = recipeManager.RecipeManager(model_fname)
pl = planningLanguage.PlanningLanguage()
print "\nEvaluating (end-to-end):", targetRecipe.name
recipeText = targetRecipe.instruction_text
initialState = targetRecipe.start_state
if useBeam:
inferredPlan = rm.find_beam_plan(recipeText, initialState)
else:
inferredPlan = rm.find_plan(recipeText, initialState)
print "\ninferred", inferredPlan
actualEndState = inferredPlan[-1][1][-1][1]
print "\ndesired states", targetRecipe.states
desiredEndState = targetRecipe.states[-1][-1][1]
plInferredPath = ""
for i in inferredPlan:
plInferredPath = plInferredPath + " | " + planningLanguage.decompile(
i[1])
print "\nPL inferred:", plInferredPath
plActual = ""
for i in targetRecipe.instructions:
plActual = plActual + " | " + i[1]
print "\nPL Desired:", plActual, "\n"
#print desiredEndState
#print "end state", actualEndState
if desiredEndState == actualEndState:
self.successCount += 1
print "\n\nResults for the End-to-End evaluation for :", targetRecipe.name
print "Success"
else:
print "\nResults for the End-to-End evaluation for :", targetRecipe.name
print "Failure"
return 0
def main():
print "loading"
confusion_matrices = pickle_util.load("confusion_matrices.pck")
print "loaded", confusion_matrices
confusion_matrices.sort(key=lambda x: x[0])
X = []
Y = []
Yerr = []
for num_results, cms in confusion_matrices:
print "num_results", num_results
f_scores = [cm.fscore for cm in cms]
print na.mean(f_scores)
error = scipy.stats.sem(f_scores)
X.append(num_results)
Y.append(na.mean(f_scores))
Yerr.append(error)
mpl.errorbar(X, Y, yerr=Yerr)
mpl.xlabel("Number of Training Examples")
mpl.ylabel("F-Score")
mpl.show()
def main():
parser = OptionParser()
parser.add_option("--training_filename",dest="training_fname",
help="Training Filename", metavar="FILE")
parser.add_option("--save_filename", dest="save_fname",
help="Save Filename", metavar="FILE")
(options, args) = parser.parse_args()
print "opening positive dataset"
dataset_all = pickle_util.load(options.training_fname)
print "dataset", dataset_all
feature_cache = FeatureCache(dataset_all)
feature_cache.cache_feature_counts()
print "saving to...", options.save_fname
print "...with ", len(feature_cache), "features"
FeatureCache.save(feature_cache, options.save_fname)
print "done"
def main():
training_set = pickle_util.load("training.pck")
totalRecipes = 0
totalIngredients = 0
ingredientsList = []
arc = annotatedRecipe.Corpus(training_set=training_set)
recipeCorpus = arc.recipes
for i in recipeCorpus:
totalRecipes += 1
for j in i.ingredients:
if j[1].contains[0] == "onion":
print i.name
if j[1].contains[0] not in ingredientsList:
ingredientsList.append(j[1].contains[0])
totalIngredients += 1
ingredientsList.sort()
print "\n\n\n"
print "Total Recipes:", totalRecipes
print "Total Ingredients", totalIngredients
print "Ingredient List:", ingredientsList
print "\n\n\n"
return 1
def runInstructionEvaluation(runTestSet=True):
training_set = pickle_util.load("training.pck")
totalRecipes = 0
arc = annotatedRecipe.Corpus(training_set=training_set)
recipeCorpus = arc.recipes
ev = Evaluator()
for i in recipeCorpus:
if i.is_training_set == runTestSet:
continue
totalRecipes += 1
ev.evaluateInstructions(i)
print "\n\nOverall results for the entire instruction-level evaluation."
print "Total Recipes:", totalRecipes
print "Total Instructions:", ev.totalCount, "\nSuccess:", ev.successCount
print "Noop Success:", ev.noopSuccessCount
print "Failures:", ev.totalCount - (ev.successCount + ev.noopSuccessCount)
right = ev.successCount + ev.noopSuccessCount
print "%.3f%% (%d/%d)" % (float(right) / ev.totalCount * 100, right,
ev.totalCount)
print "\n\n"
return (ev.totalCount, ev.successCount, ev.noopSuccessCount)
def test_new_class_to_old_class(self):
class NewOldClass(object):
"""New-style class."""
def __init__(self, x, y):
self.x = x
self.y = y
# A trick so we can pickle this class even though it's nested.
setattr(sys.modules[__name__], 'NewOldClass', NewOldClass)
pickled_new = pickle_util.dump(NewOldClass(5, 11))
# Redefine NewOldClass to be old-style
del NewOldClass
class NewOldClass:
"""Old-style class."""
def __init__(self, x, y):
self.x = x
self.y = y
setattr(sys.modules[__name__], 'NewOldClass', NewOldClass)
# Make sure the unpickling uses pickle, not cpickle
old_cpickle = pickle_util.cPickle
old_unpickler_class = pickle_util.g_unpickler_class
try:
pickle_util.cPickle = pickle_util.pickle
pickle_util.g_unpickler_class = (
pickle_util.RenamedClassUnpicklerForPicklePy)
foo = pickle_util.load(pickled_new)
self.assertEqual(5, foo.x)
self.assertEqual(11, foo.y)
self.assertEqual('Old-style class.', foo.__doc__)
finally:
pickle_util.cPickle = old_cpickle
pickle_util.g_unpickler_class = old_unpickler_class
def runEndToEndEvaluation(runTestSet=True):
training_set = pickle_util.load("training.pck")
totalRecipes = 0
arc = annotatedRecipe.Corpus(training_set=training_set)
recipeCorpus = arc.recipes
ev = Evaluator()
for i in recipeCorpus:
if i.is_training_set == runTestSet:
continue
totalRecipes += 1
result = ev.evaluateEndToEnd(i)
if result != 0:
print "Failure 1"
return 1
print "\n\nOverall results for the entire end-to-end evaluation."
print "Total Recipes:", totalRecipes
print "Success:", ev.successCount
print "Failures:", ev.totalCount - ev.successCount
right = ev.successCount + ev.noopSuccessCount
print "%.3f%% (%d/%d)" % (float(right) / ev.totalCount * 100, right,
ev.totalCount)
print "\n\n"
return (totalRecipes, ev.successCount)
def make_value_from_datastore(self, value):
value = pickle_util.load(str(value))
return super(ObjectProperty, self).make_value_from_datastore(value)
def test_simple_class(self):
"""Test pickling and unpickling a class and class instance."""
expected = (OldClass, OldClass())
actual = pickle_util.load(pickle_util.dump(expected))
self.assertEqual(expected[0], actual[0])
self.assertEqual(type(expected[1]), type(actual[1]))
def test_simple(self):
expected = 'i am a simple type'
actual = pickle_util.load(pickle_util.dump(expected))
self.assertEqual(expected, actual)
def decompress(data):
"""Decompress value from its compressed memcache state."""
pickled = zlib.decompress(data)
return pickle_util.load(pickled)
def test_unpickling_data_pickled_with_cpickle_vhigh(self):
expected = 'This is a test string'
actual = pickle_util.load(cPickle.dumps(expected,
cPickle.HIGHEST_PROTOCOL))
self.assertEqual(expected, actual)
def test_unpickling_data_pickled_with_cpickle_vhigh(self):
expected = 'This is a test string'
actual = pickle_util.load(
cPickle.dumps(expected, cPickle.HIGHEST_PROTOCOL))
self.assertEqual(expected, actual)
def main():
parser = OptionParser()
parser.add_option("--training_filename",
dest="training_fname",
help="Training Filename",
metavar="FILE")
parser.add_option("--crf_filename",
dest="crf_fname",
help="CRF Filename",
metavar="FILE")
parser.add_option("-s",
"--sigma",
dest="sigma",
type="float",
help="Sigma")
parser.add_option(
"--iterative-train-and-save-observations",
dest="iterative_train_and_save_observations",
default=None,
help=
"trains the model with frist 1 observation, then 2, then 3 .. etc until all. Saves each model after each train"
)
parser.add_option("--iterative-train-and-save-crf-iterations",
dest="iterative_train_and_save_train_iterations",
default=None)
(options, args) = parser.parse_args()
print "opening positive dataset"
dataset_all = pickle_util.load(options.training_fname)
print "dataset", dataset_all
print len([
e for e in dataset_all.observations if e.sdcs[0].type == "EVENT"
]), "events"
if options.iterative_train_and_save_observations is not None:
## ok, let's build up the model names and the training dataset
for i in range(1, len(dataset_all.observations)):
print "*!*!*!*!*!*!!* Taking ", i, " Observations"
output_model_name = options.crf_fname.replace(
".", "_numobs=" + str(i) + ".")
current_dataset = dataset.DiscreteDataset(
dataset_all.observations[0:i],
dataset_all.discretization_factor, dataset_all.f_min_max,
dataset_all.feature_extractor_cls)
train_lccrf(current_dataset, output_model_name, options.sigma)
elif options.iterative_train_and_save_train_iterations is not None:
## let's train with different iterations all the data
for i in range(1, 500, 1):
output_model_name = options.crf_fname.replace(
".", "_trainiters=" + str(i) + ".")
dataset_all = pickle_util.load(options.training_fname)
train_lccrf(dataset_all, output_model_name, options.sigma, i)
else:
train_lccrf(dataset_all, options.crf_fname, options.sigma)
def histogram(self):
return pickle_util.load(self.blob_val)
def unpickled(*args, **kwargs):
return pickle_util.load(func(*args, **kwargs))
def test_simple_class(self):
"""Test pickling and unpickling a class and class instance."""
expected = (OldClass, OldClass())
actual = pickle_util.load(pickle_util.dump(expected))
self.assertEqual(expected[0], actual[0])
self.assertEqual(type(expected[1]), type(actual[1]))
def test_simple(self):
expected = 'i am a simple type'
actual = pickle_util.load(pickle_util.dump(expected))
self.assertEqual(expected, actual)
def loadState(self):
fname = QFileDialog.getOpenFileName(self)
state = pickle_util.load(fname)
self.setState(state)
def topictree_import_task(version_id, topic_id, publish, tree_json_compressed):
try:
tree_json = pickle_util.load(zlib.decompress(tree_json_compressed))
logging.info("starting import")
version = topic_models.TopicVersion.get_by_id(version_id)
parent = topic_models.Topic.get_by_id(topic_id, version)
topics = topic_models.Topic.get_all_topics(version, True)
logging.info("got all topics")
topic_dict = dict((topic.id, topic) for topic in topics)
topic_keys_dict = dict((topic.key(), topic) for topic in topics)
videos = video_models.Video.get_all()
logging.info("got all videos")
video_dict = dict((video.readable_id, video) for video in videos)
exercises = exercise_models.Exercise.get_all_use_cache()
logging.info("got all exercises")
exercise_dict = dict((exercise.name, exercise)
for exercise in exercises)
all_entities_dict = {}
new_content_keys = []
# on dev server dont record new items in ContentVersionChanges
if app.App.is_dev_server:
put_change = False
else:
put_change = True
# delete all subtopics of node we are copying over the same topic
if tree_json["id"] == parent.id:
parent.delete_descendants()
# adds key to each entity in json tree, if the node is not in
# the tree then add it
def add_keys_json_tree(tree, parent, do_exercises, i=0, prefix=None):
pos = ((prefix + ".") if prefix else "") + str(i)
if not do_exercises and tree["kind"] == "Topic":
if tree["id"] in topic_dict:
topic = topic_dict[tree["id"]]
tree["key"] = topic.key()
else:
kwargs = dict((str(key), value)
for key, value in tree.iteritems()
if key in ['standalone_title', 'description',
'tags'])
kwargs["version"] = version
topic = topic_models.Topic.insert(title=tree['title'],
parent=parent, **kwargs)
logging.info("%s: added topic %s" % (pos, topic.title))
tree["key"] = topic.key()
topic_dict[tree["id"]] = topic
# if this topic is not the parent topic (ie. its not
# root, nor the topic_id you are updating)
if (parent.key() != topic.key() and
# and this topic is not in the new parent
topic.key() not in parent.child_keys and
# if it already exists in a topic
len(topic.parent_keys) and
# and that topic is not the parent topic
topic.parent_keys[0] != parent.key()):
# move it from that old parent topic, its position
# in the new parent does not matter as child_keys
# will get written over later. move_child is
# needed only to make sure that the parent_keys
# and ancestor_keys will all match up correctly
old_parent = topic_keys_dict[topic.parent_keys[0]]
logging.info("moving topic %s from %s to %s" % (topic.id,
old_parent.id, parent.id))
old_parent.move_child(topic, parent, 0)
all_entities_dict[tree["key"]] = topic
elif not do_exercises and tree["kind"] == "Video":
if tree["readable_id"] in video_dict:
video = video_dict[tree["readable_id"]]
tree["key"] = video.key()
else:
changeable_props = ["youtube_id", "url", "title",
"description", "keywords", "duration",
"readable_id", "views"]
video = topic_models.VersionContentChange.add_new_content(
video_models.Video,
version,
tree,
changeable_props,
put_change)
logging.info("%s: added video %s" % (pos, video.title))
new_content_keys.append(video.key())
tree["key"] = video.key()
video_dict[tree["readable_id"]] = video
all_entities_dict[tree["key"]] = video
elif do_exercises and tree["kind"] == "Exercise":
if tree["name"] in exercise_dict:
tree["key"] = (exercise_dict[tree["name"]].key()
if tree["name"] in exercise_dict else None)
else:
if "related_video_readable_ids" in tree:
# adding keys to entity tree so we don't need to look
# it up again when creating the video in
# add_new_content
tree["related_video_keys"] = []
for readable_id in tree["related_video_readable_ids"]:
video = video_dict[readable_id]
tree["related_video_keys"].append(video.key())
exercise = exercise_save_data(version,
tree,
None,
put_change)
logging.info("%s: added Exercise %s" %
(pos, exercise.name))
new_content_keys.append(exercise.key())
tree["key"] = exercise.key()
exercise_dict[tree["name"]] = exercise
all_entities_dict[tree["key"]] = exercise_dict[tree["name"]]
elif not do_exercises and tree["kind"] == "Url":
if tree["id"] in url_dict:
url = url_dict[tree["id"]]
tree["key"] = url.key()
else:
changeable_props = ["tags", "title", "url"]
url = topic_models.VersionContentChange.add_new_content(
url_model.Url,
version,
tree,
changeable_props,
put_change)
logging.info("%s: added Url %s" % (pos, url.title))
new_content_keys.append(url.key())
tree["key"] = url.key()
url_dict[tree["id"]] = url
all_entities_dict[tree["key"]] = url
i = 0
# recurse through the tree's children
if "children" in tree:
for child in tree["children"]:
add_keys_json_tree(child,
topic_dict[tree["id"]],
do_exercises, i, pos)
i += 1
add_keys_json_tree(tree_json, parent, do_exercises=False)
# add related_video_readable_ids prop to exercises
evs = exercise_video_model.ExerciseVideo.all().fetch(10000)
exercise_key_dict = dict((e.key(), e) for e in exercises)
video_key_dict = dict((v.key(), v) for v in video_dict.values())
exercise_models.Exercise.add_related_video_readable_ids_prop(
exercise_key_dict,
evs,
video_key_dict)
# exercises need to be done after, because if they reference
# ExerciseVideos those Videos have to already exist
add_keys_json_tree(tree_json, parent, do_exercises=True)
logging.info("added keys to nodes")
def add_child_keys_json_tree(tree):
if tree["kind"] == "Topic":
tree["child_keys"] = []
if "children" in tree:
for child in tree["children"]:
tree["child_keys"].append(child["key"])
add_child_keys_json_tree(child)
add_child_keys_json_tree(tree_json)
logging.info("added children keys")
def extract_nodes(tree, nodes):
if "children" in tree:
for child in tree["children"]:
nodes.update(extract_nodes(child, nodes))
del(tree["children"])
nodes[tree["key"]] = tree
return nodes
nodes = extract_nodes(tree_json, {})
logging.info("extracted %i nodes" % len(nodes))
changed_nodes = []
i = 0
# now loop through all the nodes
for key, node in nodes.iteritems():
if node["kind"] == "Topic":
topic = all_entities_dict[node["key"]]
logging.info("%i/%i Updating any change to Topic %s" %
(i, len(nodes), topic.title))
kwargs = (dict((str(key), value)
for key, value in node.iteritems()
if key in ['id', 'title', 'standalone_title',
'description', 'tags', 'hide',
'child_keys']))
kwargs["version"] = version
kwargs["put"] = False
if topic.update(**kwargs):
changed_nodes.append(topic)
elif (node["kind"] == "Video" and
node["key"] not in new_content_keys):
video = all_entities_dict[node["key"]]
logging.info("%i/%i Updating any change to Video %s" %
(i, len(nodes), video.title))
change = topic_models.VersionContentChange.add_content_change(
video,
version,
node,
["readable_id", "title", "youtube_id", "description",
"keywords"])
if change:
logging.info("changed")
elif (node["kind"] == "Exercise" and
node["key"] not in new_content_keys):
exercise = all_entities_dict[node["key"]]
logging.info("%i/%i Updating any changes to Exercise %s" %
(i, len(nodes), exercise.name))
change = exercise_save_data(version, node, exercise)
if change:
logging.info("changed")
elif node["kind"] == "Url" and node["key"] not in new_content_keys:
url = all_entities_dict[node["key"]]
logging.info("%i/%i Updating any changes to Url %s" %
(i, len(nodes), url.title))
changeable_props = ["tags", "title", "url"]
change = topic_models.VersionContentChange.add_content_change(
url,
version,
node,
changeable_props)
if change:
logging.info("changed")
i += 1
logging.info("about to put %i topic nodes" % len(changed_nodes))
setting_model.Setting.cached_content_add_date(datetime.datetime.now())
db.put(changed_nodes)
logging.info("done with import")
if publish:
version.set_default_version()
except Exception, e:
fp = StringIO.StringIO()
traceback.print_exc(file=fp)
logging.error(fp.getvalue())
logging.error("Topic import failed with %s", e)
raise deferred.PermanentTaskFailure
def main(argv):
app = basewindow.makeApp()
parser = OptionParser()
parser.add_option("--model-filename",
dest="model_fname",
help="CRF Filename",
metavar="FILE")
parser.add_option("--use-rrt", dest="use_rrt", help="Use RRT?")
parser.add_option("--limits",
dest="limits",
help="Browser axis limits",
metavar="FILE")
parser.add_option("--state-type",
dest="state_type",
help="State or Agent Type",
metavar="FILE")
parser.add_option("--merging-mode",
dest="merging_mode",
default="merge_none")
parser.add_option("--start-command",
dest="start_command",
help="Initial Command",
default=None)
parser.add_option("--start-state-file",
dest="start_state_fname",
help="Initial state file",
default=None)
parser.add_option("--start-context-file",
dest="start_context_fname",
help="Initial context file",
default=None)
parser.add_option("--search-depth",
dest="search_depth",
type="int",
default=2)
parser.add_option("--beam-width", dest="beam_width", type="int", default=2)
parser.add_option("--beam-width-event",
dest="beam_width_event",
type="int",
default=2)
(options, args) = parser.parse_args()
useRrt = False
cf = CostFnCrf.from_mallet(options.model_fname)
#cf_constant = cost_function_constant.CostFunction(cf.lccrf.dataset)
taskPlanner = nodeSearch.BeamSearch(cf, useRrt=useRrt)
limits = [float(x) for x in options.limits.split(",")]
wnd = MainWindow(taskPlanner=taskPlanner,
limits=limits,
merging_mode=options.merging_mode,
start_command=options.start_command)
wnd.searchDepthBox.setValue(options.search_depth)
wnd.beamWidthBox.setValue(options.beam_width)
wnd.beamWidthEventBox.setValue(options.beam_width_event)
state_type = state_type_from_name(options.state_type)
if options.start_state_fname != None:
start_state = pickle_util.load(options.start_state_fname)
elif options.start_context_fname:
start_context = Context.fromYaml(
yaml.load(open(options.start_context_fname)))
start_state = state_type.from_context(start_context)
else:
start_state = state_type.init_state()
wnd.setState(start_state)
wnd.show()
wnd.selectEsdcExtractor()
wnd.followCommand()
app.exec_()
def test_unpickling_data_pickled_with_cpickle(self):
expected = 'This is a test string'
actual = pickle_util.load(cPickle.dumps(expected))
self.assertEqual(expected, actual)
def load(fname):
python_fname, java_fname = fnames_from_root(fname)
m = pickle_util.load(python_fname)
return m
def test_unpickling_data_pickled_with_cpickle(self):
expected = 'This is a test string'
actual = pickle_util.load(cPickle.dumps(expected))
self.assertEqual(expected, actual)
def short_circuit_content(self):
if self.short_circuit_pickled_content:
return pickle_util.load(self.short_circuit_pickled_content)
else:
return None
def short_circuit_content(self):
return pickle_util.load(self.short_circuit_pickled_content)
def emotions(self):
"""Return unpickled list of emotions tied to these notes."""
if self.pickled_emotions:
return pickle_util.load(self.pickled_emotions)
else:
return None
def content(self):
return pickle_util.load(self.pickled_content)