def handle(self, *args, **options): if len(args) != 0: raise CommandError("Unknown argument list: %s" % args) # TODO(bcipolli) # Make remove_unknown_exercises and force_icons into command-line arguments topic_tree = rebuild_topictree( remove_unknown_exercises=not options["keep_new_exercises"]) node_cache = topic_tools.generate_node_cache(topic_tree) slug2id_map = topic_tools.generate_slug_to_video_id_map(node_cache) # Disabled until we revamp it based on the current KA API. # h_position and v_position are available on each exercise now. # If not on the topic_tree, then here: http://api-explorer.khanacademy.org/api/v1/playlists/topic_slug/exercises rebuild_knowledge_map(topic_tree, node_cache, force_icons=options["force_icons"]) scrub_topic_tree(node_cache=node_cache) validate_data(topic_tree, node_cache, slug2id_map) save_topic_tree(topic_tree) sys.stdout.write( "Downloaded topic_tree data for %d topics, %d videos, %d exercises\n" % ( len(node_cache["Topic"]), len(node_cache["Video"]), len(node_cache["Exercise"]), ))
def handle(self, *args, **options): if len(args) != 0: raise CommandError("Unknown argument list: %s" % args) # TODO(bcipolli) # Make remove_unknown_exercises and force_icons into command-line arguments topic_tree = rebuild_topictree(remove_unknown_exercises=not options["keep_new_exercises"]) node_cache = topic_tools.generate_node_cache(topic_tree) slug2id_map = topic_tools.generate_slug_to_video_id_map(node_cache) # Disabled until we revamp it based on the current KA API. # h_position and v_position are available on each exercise now. # If not on the topic_tree, then here: http://api-explorer.khanacademy.org/api/v1/playlists/topic_slug/exercises rebuild_knowledge_map(topic_tree, node_cache, force_icons=options["force_icons"]) scrub_topic_tree(node_cache=node_cache) validate_data(topic_tree, node_cache, slug2id_map) save_topic_tree(topic_tree) sys.stdout.write("Downloaded topic_tree data for %d topics, %d videos, %d exercises\n" % ( len(node_cache["Topic"]), len(node_cache["Video"]), len(node_cache["Exercise"]), ))
def handle(self, *args, **options): channel_name = channel = options["channel"] if options["import_files"]: channel = "import_channel" channel_tools = importlib.import_module( "kalite.contentload.management.commands.channels.{channel}".format(channel=channel) ) if options["import_files"]: channel_tools.path = options["import_files"] if not channel_name or channel_name == "khan": channel_name = os.path.basename(options["import_files"]) if options["channel_data"]: channel_tools.channel_data_path = options["channel_data"] channel_path = os.path.join(settings.CONTENT_DATA_PATH, slugify(unicode(channel_name))) ensure_dir(channel_path) channel_id = hashlib.md5(channel_name).hexdigest() channel_dict = {"id": channel_id, "name": channel_name, "path": channel_path} topic_tree, exercises, assessment_items, content = channel_tools.rebuild_topictree(channel=channel_dict) exercise_cache = channel_tools.build_full_cache(exercises, id_key=channel_tools.id_key["Exercise"]) assessment_item_cache = channel_tools.build_full_cache(assessment_items) content_cache = channel_tools.build_full_cache(content) node_cache = topic_tools.generate_node_cache(topic_tree) node_cache["Exercise"] = exercise_cache node_cache["Content"] = content_cache node_cache["AssessmentItem"] = assessment_item_cache slug2id_map = topic_tools.generate_slug_to_video_id_map(node_cache) if channel_tools.channel_data["temp_ok_atts"]: scrub_topic_tree(node_cache=node_cache, channel_data=channel_tools.channel_data) validate_data(topic_tree, node_cache, slug2id_map) # The reason why we catch all errors was that this thing takes # around 6 hours to run, and having them error out in the end # is kind of a bummer. Since this is meant to just run # occasionally on our side (as a hidden feature of KA Lite), # it's safe to have this code smell for now. try: save_cache_file("Topic", cache_object=topic_tree, data_path=channel_path) save_cache_file("Exercise", cache_object=exercise_cache, data_path=channel_path) save_cache_file("AssessmentItem", cache_object=assessment_item_cache, data_path=channel_path) save_cache_file("Content", cache_object=content_cache, data_path=channel_path) except Exception as e: import IPython IPython.embed() if hasattr(channel_tools, "channel_data_files"): channel_tools.channel_data_files(dest=channel_path) sys.stdout.write( """Downloaded topic_tree data for {topics} topics {exercises} exercises {contents} content files {assessments} assessment items """.format( topics=len(node_cache["Topic"]), exercises=len(node_cache["Exercise"]), contents=len(node_cache["Content"]), assessments=len(node_cache["AssessmentItem"]), ) )
def handle(self, *args, **options): channel_name = channel = options["channel"] if options["import_files"]: channel = "import_channel" channel_tools = importlib.import_module( "kalite.contentload.management.commands.channels.{channel}".format( channel=channel)) if options["import_files"]: channel_tools.path = options["import_files"] if not channel_name or channel_name == "khan": channel_name = os.path.basename(options["import_files"]) if options["channel_data"]: channel_tools.channel_data_path = options["channel_data"] channel_path = os.path.join(settings.CONTENT_DATA_PATH, slugify(unicode(channel_name))) ensure_dir(channel_path) channel_id = hashlib.md5(channel_name).hexdigest() channel_dict = { "id": channel_id, "name": channel_name, "path": channel_path, } topic_tree, exercises, assessment_items, content = channel_tools.rebuild_topictree( channel=channel_dict) exercise_cache = channel_tools.build_full_cache( exercises, id_key=channel_tools.id_key["Exercise"]) assessment_item_cache = channel_tools.build_full_cache( assessment_items) content_cache = channel_tools.build_full_cache(content) node_cache = topic_tools.generate_node_cache(topic_tree) node_cache["Exercise"] = exercise_cache node_cache["Content"] = content_cache node_cache["AssessmentItem"] = assessment_item_cache slug2id_map = topic_tools.generate_slug_to_video_id_map(node_cache) if channel_tools.channel_data["temp_ok_atts"]: scrub_topic_tree(node_cache=node_cache, channel_data=channel_tools.channel_data) validate_data(topic_tree, node_cache, slug2id_map) # The reason why we catch all errors was that this thing takes # around 6 hours to run, and having them error out in the end # is kind of a bummer. Since this is meant to just run # occasionally on our side (as a hidden feature of KA Lite), # it's safe to have this code smell for now. try: save_cache_file("Topic", cache_object=topic_tree, data_path=channel_path) save_cache_file("Exercise", cache_object=exercise_cache, data_path=channel_path) save_cache_file("AssessmentItem", cache_object=assessment_item_cache, data_path=channel_path) save_cache_file("Content", cache_object=content_cache, data_path=channel_path) except: import IPython IPython.embed() if hasattr(channel_tools, "channel_data_files"): channel_tools.channel_data_files(dest=channel_path) sys.stdout.write("""Downloaded topic_tree data for {topics} topics {exercises} exercises {contents} content files {assessments} assessment items """.format(topics=len(node_cache["Topic"]), exercises=len(node_cache["Exercise"]), contents=len(node_cache["Content"]), assessments=len(node_cache["AssessmentItem"], )))