def post(request):
#today=datetime.date.today()
topic = None
nick = None
channel = None
date = None
retval = {}
if request.method == "POST":
logging.debug('raw_post_data %s' % (request.raw_post_data))
try:
data = simplejson.loads(request.raw_post_data)
logging.debug('data %s' % (data))
topic = data.get('topic')
channel = data.get('channel')
nick = data.get('nick')
date = datetime.datetime.strptime(data.get('date'), '%Y%m%d%H%M')
logging.debug('date: %s, nick: %s, channel, %s, topic: %s' %
(date, nick, channel, topic))
except Exception, e:
logging.warning('TOPIC/POST: Somewhere near %s' % (e))
retval = {'id': 0, 'topic': e}
else:
try:
t = Topic(channel=channel, nick=nick, topic=topic, date=date)
t.put()
except Exception, e:
logging.warning('TOPIC/POST/NEW: Somewhere near %s' % (e))
retval = {'id': 0, 'topic': e}
else:
def index(request):
topics = Topic.query().order(-Topic.date)
template_values = {
'topics': topics,
}
return render_to_response('topic.html', template_values)
def show_video(handler, readable_id, topic_id,
redirect_to_canonical_url=False):
topic = None
query_string = ''
if topic_id is not None and len(topic_id) > 0:
topic = Topic.get_by_id(topic_id)
key_id = 0 if not topic else topic.key().id()
# If a topic_id wasn't specified or the specified topic wasn't found
# use the first topic for the requested video.
if topic is None:
# Get video by readable_id to get the first topic for the video
video = Video.get_for_readable_id(readable_id)
if video is None:
raise MissingVideoException("Missing video '%s'" %
readable_id)
topic = video.first_topic()
if not topic:
raise MissingVideoException("No topic has video '%s'" %
readable_id)
if handler.request.query_string:
query_string = '?' + handler.request.query_string
redirect_to_canonical_url = True
if redirect_to_canonical_url:
url = "/%s/v/%s%s" % (topic.get_extended_slug(),
urllib.quote(readable_id),
query_string)
logging.info("Redirecting to %s" % url)
handler.redirect(url, True)
return None
# Note: Bingo conversions are tracked on the client now,
# so they have been removed here. (tomyedwab)
topic_data = topic.get_play_data()
discussion_options = qa.add_template_values({}, handler.request)
video_data = Video.get_play_data(readable_id, topic,
discussion_options)
if video_data is None:
raise MissingVideoException("Missing video '%s'" % readable_id)
template_values = {
"topic_data": topic_data,
"topic_data_json": api.jsonify.jsonify(topic_data),
"video": video_data,
"video_data_json": api.jsonify.jsonify(video_data),
"selected_nav_link": 'watch',
}
return template_values
def show_video(handler, readable_id, topic_id,
redirect_to_canonical_url=False):
topic = None
query_string = ''
if topic_id is not None and len(topic_id) > 0:
topic = Topic.get_by_id(topic_id)
key_id = 0 if not topic else topic.key().id()
# If a topic_id wasn't specified or the specified topic wasn't found
# use the first topic for the requested video.
if topic is None:
# Get video by readable_id to get the first topic for the video
video = Video.get_for_readable_id(readable_id)
if video is None:
raise MissingVideoException("Missing video '%s'" %
readable_id)
topic = video.first_topic()
if not topic:
raise MissingVideoException("No topic has video '%s'" %
readable_id)
if handler.request.query_string:
query_string = '?' + handler.request.query_string
redirect_to_canonical_url = True
if redirect_to_canonical_url:
url = "/%s/v/%s%s" % (topic.get_extended_slug(),
urllib.quote(readable_id),
query_string)
logging.info("Redirecting to %s" % url)
handler.redirect(url, True)
return None
# Note: Bingo conversions are tracked on the client now,
# so they have been removed here. (tomyedwab)
topic_data = topic.get_play_data()
discussion_options = qa.add_template_values({}, handler.request)
video_data = Video.get_play_data(readable_id, topic,
discussion_options)
if video_data is None:
raise MissingVideoException("Missing video '%s'" % readable_id)
template_values = {
"topic_data": topic_data,
"topic_data_json": api.jsonify.jsonify(topic_data),
"video": video_data,
"video_data_json": api.jsonify.jsonify(video_data),
"selected_nav_link": 'watch',
}
return template_values
def get(self):
version_name = self.request.get('version', 'edit')
edit_version = TopicVersion.get_by_id(version_name)
if edit_version is None:
default_version = TopicVersion.get_default_version()
if default_version is None:
# Create empty default version
default_version = TopicVersion.create_new_version()
root = create_root(default_version)
default_version.default = True
default_version.edit = False
default_version.put()
map_layout = layout.MapLayout.get_for_version(default_version)
if not map_layout.has_layout:
# TODO(KNL): Load map layout from location
map_layout.layout = json.loads('{"polylines":[],"topics":{"Getallen":{"icon_url":"\x2Fimages\x2Fpower-mode\x2Fbadges\x2Fdefault-40x40.png","id":"getallen","standalone_title":"Getallen","x":0,"y":4},"Verhoudingen":{"icon_url":"\x2Fimages\x2Fpower-mode\x2Fbadges\x2Fdefault-40x40.png","id":"verhoudingen","standalone_title":"Verhoudingen","x":0,"y":7},"Meten en Meetkunde":{"icon_url":"\x2Fimages\x2Fpower-mode\x2Fbadges\x2Fdefault-40x40.png","id":"meetkunde","standalone_title":"Meten en Meetkunde","x":3,"y":4},"Verbanden":{"icon_url":"\x2Fimages\x2Fpower-mode\x2Fbadges\x2Fdefault-40x40.png","id":"verbanden","standalone_title":"Verbanden","x":3,"y":7}}}');
map_layout.put()
# Create empty edit version
old_tree = root.make_tree(types=["Topics"], include_hidden=True)
edit_version = TopicVersion.create_new_version()
edit_version.copied_from = default_version
edit_version.edit = True
TopicVersion.copy_tree(root, edit_version)
edit_version.put()
else:
raise Exception("Wait for setting default version to finish making an edit version.")
if self.request.get('autoupdate', False):
self.render_jinja2_template('autoupdate_in_progress.html', {"edit_version": edit_version})
return
if self.request.get('autoupdate_begin', False):
return self.topic_update_from_live(edit_version)
if self.request.get('migrate', False):
return self.topic_migration()
if self.request.get('fixdupes', False):
return self.fix_duplicates()
root = Topic.get_root(edit_version)
data = root.get_visible_data()
tree_nodes = [data]
template_values = {
'edit_version': jsonify(edit_version),
'tree_nodes': jsonify(tree_nodes)
}
self.render_jinja2_template('topics-admin.html', template_values)
return
def get(self):
version_name = self.request.get('version', 'edit')
from_version_name = self.request.get('from_version_name', None)
edit_version = TopicVersion.get_by_id(version_name)
if from_version_name:
from_version = TopicVersion.get_by_id(from_version_name)
edit_version = TopicVersion.create_edit_version(from_version)
elif edit_version is None:
default_version = TopicVersion.get_default_version()
if default_version == None:
#invi: should only run when topic tree is empty
edit_version = TopicVersion.create_new_version()
edit_version.edit = True
edit_version.put()
create_root(edit_version)
edit_version.put()
edit_version.edit = False
edit_version.default = True
edit_version.put()
edit_version = TopicVersion.create_new_version()
edit_version.edit = True
edit_version.put()
create_root(edit_version)
edit_version.put()
else:
edit_version = TopicVersion.create_edit_version()
if self.request.get('autoupdate', False):
self.render_jinja2_template('autoupdate_in_progress.html', {"edit_version": edit_version})
return
if self.request.get('autoupdate_begin', False):
return self.topic_update_from_live(edit_version)
if self.request.get('migrate', False):
return self.topic_migration()
if self.request.get('fixdupes', False):
return self.fix_duplicates()
root = Topic.get_root(edit_version)
data = root.get_visible_data()
tree_nodes = [data]
template_values = {
'edit_version': jsonify(edit_version),
'tree_nodes': jsonify(tree_nodes)
}
self.render_jinja2_template('topics-admin.html', template_values)
return
def get(self):
version_name = self.request.get('version', 'edit')
edit_version = TopicVersion.get_by_id(version_name)
if edit_version is None:
default_version = TopicVersion.get_default_version()
if default_version is None:
# Assuming this is dev, there is an empty datastore and we need an import
edit_version = TopicVersion.create_new_version()
edit_version.edit = True
edit_version.put()
create_root(edit_version)
else:
raise Exception("Wait for setting default version to finish making an edit version.")
if self.request.get('autoupdate', False):
self.render_jinja2_template('autoupdate_in_progress.html', {"edit_version": edit_version})
return
if self.request.get('autoupdate_begin', False):
return self.topic_update_from_live(edit_version)
if self.request.get('migrate', False):
return self.topic_migration()
if self.request.get('fixdupes', False):
return self.fix_duplicates()
root = Topic.get_root(edit_version)
data = root.get_visible_data()
tree_nodes = [data]
template_values = {
'edit_version': jsonify(edit_version),
'tree_nodes': jsonify(tree_nodes)
}
self.render_jinja2_template('topics-admin.html', template_values)
return
def get(self):
version_name = self.request.get('version', 'edit')
edit_version = TopicVersion.get_by_id(version_name)
if edit_version is None:
default_version = TopicVersion.get_default_version()
if default_version is None:
# Assuming this is dev, there is an empty datastore and we need an import
edit_version = TopicVersion.create_new_version()
edit_version.edit = True
edit_version.put()
create_root(edit_version)
else:
raise Exception("Wait for setting default version to finish making an edit version.")
if self.request.get('autoupdate', False):
self.render_jinja2_template('autoupdate_in_progress.html', {"edit_version": edit_version})
return
if self.request.get('autoupdate_begin', False):
return self.topic_update_from_live(edit_version)
if self.request.get('migrate', False):
return self.topic_migration()
if self.request.get('fixdupes', False):
return self.fix_duplicates()
root = Topic.get_root(edit_version)
data = root.get_visible_data()
tree_nodes = [data]
template_values = {
'edit_version': jsonify(edit_version),
'tree_nodes': jsonify(tree_nodes)
}
self.render_jinja2_template('topics-admin.html', template_values)
return
def importIntoVersion(version):
logging.info("comparing to version number %i" % version.number)
topic = Topic.get_by_id("art-history", version)
if not topic:
parent = Topic.get_by_id("humanities---other", version)
if not parent:
raise Exception("Could not find the Humanities & Other topic to put art history into")
topic = Topic.insert(title="Art History",
parent=parent,
id="art-history",
standalone_title="Art History",
description="Spontaneous conversations about works of art where the speakers are not afraid to disagree with each other or art history orthodoxy. Videos are made by Dr. Beth Harris and Dr. Steven Zucker along with other contributors.")
urls = topic.get_urls(include_descendants=True)
href_to_key_dict = dict((url.url, url.key()) for url in urls)
videos = topic.get_videos(include_descendants=True)
video_dict = dict((v.youtube_id, v) for v in videos)
content = getSmartHistoryContent()
if content is None:
raise Exception("Aborting import, could not read from smarthistory")
subtopics = topic.get_child_topics()
subtopic_dict = dict((t.title, t) for t in subtopics)
subtopic_child_keys = {}
new_subtopic_keys = []
child_keys = []
i = 0
for link in content:
href = link["href"]
title = link["title"]
parent_title = link["parent"]
content = link["content"]
youtube_id = link["youtube_id"] if "youtube_id" in link else None
extra_properties = {"original_url": href}
if parent_title not in subtopic_dict:
subtopic = Topic.insert(title=parent_title,
parent=topic,
standalone_title="Art History: %s"
% parent_title,
description="")
subtopic_dict[parent_title] = subtopic
else:
subtopic = subtopic_dict[parent_title]
if subtopic.key() not in new_subtopic_keys:
new_subtopic_keys.append(subtopic.key())
if parent_title not in subtopic_child_keys:
subtopic_child_keys[parent_title] = []
if youtube_id:
if youtube_id not in video_dict:
# make sure it didn't get imported before, but never put
# into a topic
query = video_models.Video.all()
video = query.filter("youtube_id =", youtube_id).get()
if video is None:
logging.info("adding youtube video %i %s %s %s to %s" %
(i, youtube_id, href, title, parent_title))
video_data = youtube_get_video_data_dict(youtube_id)
# use the title from the webpage not from the youtube
# page
video = None
if video_data:
video_data["title"] = title
video_data["extra_properties"] = extra_properties
video = topic_models.VersionContentChange.add_new_content(
video_models.Video,
version,
video_data)
else:
logging.error(("Could not import youtube_id %s " +
"for %s %s") % (youtube_id, href, title))
raise Exception(("Could not import youtube_id %s " +
" for %s %s") % (youtube_id, href,
title))
else:
video = video_dict[youtube_id]
if video.extra_properties != extra_properties:
logging.info(("changing extra properties of %i %s %s " +
"from %s to %s") % (i, href, title,
video.extra_properties, extra_properties))
video.extra_properties = extra_properties
video.put()
if video:
subtopic_child_keys[parent_title].append(video.key())
elif href not in href_to_key_dict:
logging.info("adding %i %s %s to %s" %
(i, href, title, parent_title))
topic_models.VersionContentChange.add_new_content(
Url,
version,
{"title": title,
"url": href
},
["title", "url"])
url = Url(url=href,
title=title,
id=id)
url.put()
subtopic_child_keys[parent_title].append(url.key())
else:
subtopic_child_keys[parent_title].append(href_to_key_dict[href])
i += 1
logging.info("updating child_keys")
change = False
for parent_title, child_keys in subtopic_child_keys.iteritems():
subtopic = subtopic_dict[parent_title]
if subtopic.child_keys != subtopic_child_keys[parent_title]:
change = True
subtopic.update(child_keys=subtopic_child_keys[parent_title])
if topic.child_keys != new_subtopic_keys:
change = True
topic.update(child_keys=new_subtopic_keys)
if change:
logging.info("finished updating version number %i" % version.number)
else:
logging.info("nothing changed")
return change
def create_root(version):
root = Topic.insert(title="The Root of All Knowledge",
description="All concepts fit into the root of all knowledge",
id="root",
version=version)
def get_old(self):
""" Deprecated old version of search, so we can Gandalf in the new one.
If new search is working, this should be taken out by May 31, 2012.
"""
show_update = False
if App.is_dev_server and user_util.is_current_user_admin():
update = self.request_bool("update", False)
if update:
self.update()
version_number = layer_cache.KeyValueCache.get(
"last_dev_topic_vesion_indexed")
default_version = topic_models.TopicVersion.get_default_version()
if version_number != default_version.number:
show_update = True
query = self.request.get('page_search_query')
template_values = {'page_search_query': query}
query = query.strip()
if len(query) < search.SEARCH_PHRASE_MIN_LENGTH:
if len(query) > 0:
template_values.update(
{'query_too_short': search.SEARCH_PHRASE_MIN_LENGTH})
self.render_jinja2_template("searchresults.html", template_values)
return
searched_phrases = []
# Do an async query for all ExerciseVideos, since this may be slow
exvids_query = ExerciseVideo.all()
exvids_future = util.async_queries([exvids_query])
# One full (non-partial) search, then sort by kind
all_text_keys = Topic.full_text_search(
query,
limit=50,
kind=None,
stemming=Topic.INDEX_STEMMING,
multi_word_literal=Topic.INDEX_MULTI_WORD,
searched_phrases_out=searched_phrases)
# Quick title-only partial search
topic_partial_results = filter(
lambda topic_dict: query in topic_dict["title"].lower(),
autocomplete.topic_title_dicts())
video_partial_results = filter(
lambda video_dict: query in video_dict["title"].lower(),
autocomplete.video_title_dicts())
url_partial_results = filter(
lambda url_dict: query in url_dict["title"].lower(),
autocomplete.url_title_dicts())
# Combine results & do one big get!
all_keys = [str(key_and_title[0]) for key_and_title in all_text_keys]
all_keys.extend([result["key"] for result in topic_partial_results])
all_keys.extend([result["key"] for result in video_partial_results])
all_keys.extend([result["key"] for result in url_partial_results])
all_keys = list(set(all_keys))
# Filter out anything that isn't a Topic, Url or Video
all_keys = [
key for key in all_keys
if db.Key(key).kind() in ["Topic", "Url", "Video"]
]
# Get all the entities
all_entities = db.get(all_keys)
# Group results by type
topics = []
videos = []
for entity in all_entities:
if isinstance(entity, Topic):
topics.append(entity)
elif isinstance(entity, Video):
videos.append(entity)
elif isinstance(entity, Url):
videos.append(entity)
elif entity:
logging.info("Found unknown object " + repr(entity))
# Get topics for videos not in matching topics
filtered_videos = []
filtered_videos_by_key = {}
for video in videos:
if [(str(topic.key()) in video.topic_string_keys)
for topic in topics].count(True) == 0:
video_topic = video.first_topic()
if video_topic != None:
topics.append(video_topic)
filtered_videos.append(video)
filtered_videos_by_key[str(video.key())] = []
else:
filtered_videos.append(video)
filtered_videos_by_key[str(video.key())] = []
video_count = len(filtered_videos)
# Get the related exercises
all_exercise_videos = exvids_future[0].get_result()
exercise_keys = []
for exvid in all_exercise_videos:
video_key = str(ExerciseVideo.video.get_value_for_datastore(exvid))
if video_key in filtered_videos_by_key:
exercise_key = ExerciseVideo.exercise.get_value_for_datastore(
exvid)
video_exercise_keys = filtered_videos_by_key[video_key]
video_exercise_keys.append(exercise_key)
exercise_keys.append(exercise_key)
exercises = db.get(exercise_keys)
# Sort exercises with videos
video_exercises = {}
for video_key, exercise_keys in filtered_videos_by_key.iteritems():
video_exercises[video_key] = map(
lambda exkey: [
exercise for exercise in exercises
if exercise.key() == exkey
][0], exercise_keys)
# Count number of videos in each topic and sort descending
topic_count = 0
matching_topic_count = 0
if topics:
if len(filtered_videos) > 0:
for topic in topics:
topic.match_count = [
(str(topic.key()) in video.topic_string_keys)
for video in filtered_videos
].count(True)
if topic.match_count > 0:
topic_count += 1
topics = sorted(topics,
key=lambda topic: topic.match_count,
reverse=True)
else:
for topic in topics:
topic.match_count = 0
for topic in topics:
if topic.title.lower() == query:
topic.matches = True
matching_topic_count += 1
child_topics = topic.get_child_topics(
include_descendants=True)
topic.child_topics = [
t for t in child_topics if t.has_content()
]
template_values.update({
'show_update': show_update,
'topics': topics,
'videos': filtered_videos,
'video_exercises': video_exercises,
'search_string': query,
'video_count': video_count,
'topic_count': topic_count,
'matching_topic_count': matching_topic_count
})
self.render_jinja2_template("searchresults.html", template_values)
def get_old(self):
""" Deprecated old version of search, so we can Gandalf in the new one.
If new search is working, this should be taken out by May 31, 2012.
"""
show_update = False
if App.is_dev_server and user_util.is_current_user_admin():
update = self.request_bool("update", False)
if update:
self.update()
version_number = layer_cache.KeyValueCache.get(
"last_dev_topic_vesion_indexed")
default_version = topic_models.TopicVersion.get_default_version()
if version_number != default_version.number:
show_update = True
query = self.request.get('page_search_query')
template_values = {'page_search_query': query}
query = query.strip()
if len(query) < search.SEARCH_PHRASE_MIN_LENGTH:
if len(query) > 0:
template_values.update({
'query_too_short': search.SEARCH_PHRASE_MIN_LENGTH
})
self.render_jinja2_template("searchresults.html", template_values)
return
searched_phrases = []
# Do an async query for all ExerciseVideos, since this may be slow
exvids_query = ExerciseVideo.all()
exvids_future = util.async_queries([exvids_query])
# One full (non-partial) search, then sort by kind
all_text_keys = Topic.full_text_search(
query, limit=50, kind=None,
stemming=Topic.INDEX_STEMMING,
multi_word_literal=Topic.INDEX_MULTI_WORD,
searched_phrases_out=searched_phrases)
# Quick title-only partial search
topic_partial_results = filter(
lambda topic_dict: query in topic_dict["title"].lower(),
autocomplete.topic_title_dicts())
video_partial_results = filter(
lambda video_dict: query in video_dict["title"].lower(),
autocomplete.video_title_dicts())
url_partial_results = filter(
lambda url_dict: query in url_dict["title"].lower(),
autocomplete.url_title_dicts())
# Combine results & do one big get!
all_keys = [str(key_and_title[0]) for key_and_title in all_text_keys]
all_keys.extend([result["key"] for result in topic_partial_results])
all_keys.extend([result["key"] for result in video_partial_results])
all_keys.extend([result["key"] for result in url_partial_results])
all_keys = list(set(all_keys))
# Filter out anything that isn't a Topic, Url or Video
all_keys = [key for key in all_keys
if db.Key(key).kind() in ["Topic", "Url", "Video"]]
# Get all the entities
all_entities = db.get(all_keys)
# Group results by type
topics = []
videos = []
for entity in all_entities:
if isinstance(entity, Topic):
topics.append(entity)
elif isinstance(entity, Video):
videos.append(entity)
elif isinstance(entity, Url):
videos.append(entity)
elif entity:
logging.info("Found unknown object " + repr(entity))
# Get topics for videos not in matching topics
filtered_videos = []
filtered_videos_by_key = {}
for video in videos:
if [(str(topic.key()) in video.topic_string_keys)
for topic in topics].count(True) == 0:
video_topic = video.first_topic()
if video_topic != None:
topics.append(video_topic)
filtered_videos.append(video)
filtered_videos_by_key[str(video.key())] = []
else:
filtered_videos.append(video)
filtered_videos_by_key[str(video.key())] = []
video_count = len(filtered_videos)
# Get the related exercises
all_exercise_videos = exvids_future[0].get_result()
exercise_keys = []
for exvid in all_exercise_videos:
video_key = str(ExerciseVideo.video.get_value_for_datastore(exvid))
if video_key in filtered_videos_by_key:
exercise_key = ExerciseVideo.exercise.get_value_for_datastore(
exvid)
video_exercise_keys = filtered_videos_by_key[video_key]
video_exercise_keys.append(exercise_key)
exercise_keys.append(exercise_key)
exercises = db.get(exercise_keys)
# Sort exercises with videos
video_exercises = {}
for video_key, exercise_keys in filtered_videos_by_key.iteritems():
video_exercises[video_key] = map(
lambda exkey: [exercise for exercise in exercises
if exercise.key() == exkey][0], exercise_keys)
# Count number of videos in each topic and sort descending
topic_count = 0
matching_topic_count = 0
if topics:
if len(filtered_videos) > 0:
for topic in topics:
topic.match_count = [
(str(topic.key()) in video.topic_string_keys)
for video in filtered_videos].count(True)
if topic.match_count > 0:
topic_count += 1
topics = sorted(topics,
key=lambda topic: topic.match_count,
reverse=True)
else:
for topic in topics:
topic.match_count = 0
for topic in topics:
if topic.title.lower() == query:
topic.matches = True
matching_topic_count += 1
child_topics = topic.get_child_topics(
include_descendants=True)
topic.child_topics = [t for t in child_topics
if t.has_content()]
template_values.update({
'show_update': show_update,
'topics': topics,
'videos': filtered_videos,
'video_exercises': video_exercises,
'search_string': query,
'video_count': video_count,
'topic_count': topic_count,
'matching_topic_count': matching_topic_count
})
self.render_jinja2_template("searchresults.html", template_values)
def importIntoVersion(version):
logging.info("comparing to version number %i" % version.number)
topic = Topic.get_by_id("art-history", version)
if not topic:
parent = Topic.get_by_id("humanities---other", version)
if not parent:
raise Exception("Could not find the Humanities & Other topic to put art history into")
topic = Topic.insert(title="Art History",
parent=parent,
id="art-history",
standalone_title="Art History",
description="Spontaneous conversations about works of art where the speakers are not afraid to disagree with each other or art history orthodoxy. Videos are made by Dr. Beth Harris and Dr. Steven Zucker along with other contributors.")
urls = topic.get_urls(include_descendants=True)
href_to_key_dict = dict((url.url, url.key()) for url in urls)
videos = topic.get_videos(include_descendants=True)
video_dict = dict((v.youtube_id, v) for v in videos)
content = getSmartHistoryContent()
if content is None:
raise Exception("Aborting import, could not read from smarthistory")
subtopics = topic.get_child_topics()
subtopic_dict = dict((t.title, t) for t in subtopics)
subtopic_child_keys = {}
new_subtopic_keys = []
child_keys = []
i = 0
for link in content:
href = link["href"]
title = link["title"]
parent_title = link["parent"]
content = link["content"]
youtube_id = link["youtube_id"] if "youtube_id" in link else None
extra_properties = {"original_url": href}
if parent_title not in subtopic_dict:
subtopic = Topic.insert(title=parent_title,
parent=topic,
standalone_title="Art History: %s"
% parent_title,
description="")
subtopic_dict[parent_title] = subtopic
else:
subtopic = subtopic_dict[parent_title]
if subtopic.key() not in new_subtopic_keys:
new_subtopic_keys.append(subtopic.key())
if parent_title not in subtopic_child_keys:
subtopic_child_keys[parent_title] = []
if youtube_id:
if youtube_id not in video_dict:
# make sure it didn't get imported before, but never put
# into a topic
query = video_models.Video.all()
video = query.filter("youtube_id =", youtube_id).get()
if video is None:
logging.info("adding youtube video %i %s %s %s to %s" %
(i, youtube_id, href, title, parent_title))
video_data = youtube_get_video_data_dict(youtube_id)
# use the title from the webpage not from the youtube
# page
video = None
if video_data:
video_data["title"] = title
video_data["extra_properties"] = extra_properties
video = topic_models.VersionContentChange.add_new_content(
video_models.Video,
version,
video_data)
else:
logging.error(("Could not import youtube_id %s " +
"for %s %s") % (youtube_id, href, title))
raise Exception(("Could not import youtube_id %s " +
" for %s %s") % (youtube_id, href,
title))
else:
video = video_dict[youtube_id]
if video.extra_properties != extra_properties:
logging.info(("changing extra properties of %i %s %s " +
"from %s to %s") % (i, href, title,
video.extra_properties, extra_properties))
video.extra_properties = extra_properties
video.put()
if video:
subtopic_child_keys[parent_title].append(video.key())
elif href not in href_to_key_dict:
logging.info("adding %i %s %s to %s" %
(i, href, title, parent_title))
topic_models.VersionContentChange.add_new_content(
Url,
version,
{"title": title,
"url": href
},
["title", "url"])
url = Url(url=href,
title=title,
id=id)
url.put()
subtopic_child_keys[parent_title].append(url.key())
else:
subtopic_child_keys[parent_title].append(href_to_key_dict[href])
i += 1
logging.info("updating child_keys")
change = False
for parent_title, child_keys in subtopic_child_keys.iteritems():
subtopic = subtopic_dict[parent_title]
if subtopic.child_keys != subtopic_child_keys[parent_title]:
change = True
subtopic.update(child_keys=subtopic_child_keys[parent_title])
if topic.child_keys != new_subtopic_keys:
change = True
topic.update(child_keys=new_subtopic_keys)
if change:
logging.info("finished updating version number %i" % version.number)
else:
logging.info("nothing changed")
return change
def create_root(version):
root = Topic.insert(title="The Root of All Knowledge",
description="All concepts fit into the root of all knowledge",
id="root",
version=version)