def set_content_fixture(self, db_path_mock):
_, self.content_db_path = tempfile.mkstemp(suffix=".sqlite3")
db_path_mock.return_value = self.content_db_path
self.content_engine = create_engine("sqlite:///" +
self.content_db_path,
convert_unicode=True)
with open(SCHEMA_PATH_TEMPLATE.format(name=self.schema_name),
"rb") as f:
metadata = pickle.load(f)
data_path = DATA_PATH_TEMPLATE.format(name=self.data_name)
with io.open(data_path, mode="r", encoding="utf-8") as f:
data = json.load(f)
metadata.bind = self.content_engine
metadata.create_all()
conn = self.content_engine.connect()
# Write data for each fixture into the table
for table in metadata.sorted_tables:
if data[table.name]:
conn.execute(table.insert(), data[table.name])
conn.close()
with patch(
"kolibri.core.content.utils.sqlalchemybridge.get_engine",
new=self.get_engine,
):
import_channel_from_local_db("6199dde695db4ee4ab392222d5af1e5c")
update_content_metadata("6199dde695db4ee4ab392222d5af1e5c")
def import_external_content_dbs():
"""
If we are potentially moving from a version of Kolibri that did not import its content data,
scan through the content database folder for all channel content databases,
and pull the data from each database if we have not already imported it.
"""
channel_ids = get_channel_ids_for_content_database_dir(
get_content_database_dir_path()
)
for channel_id in channel_ids:
if not ChannelMetadata.objects.filter(id=channel_id).exists():
try:
import_channel_from_local_db(channel_id)
set_content_visibility_from_disk(channel_id)
except (InvalidSchemaVersionError, FutureSchemaError):
logger.warning(
"Tried to import channel {channel_id}, but database file was incompatible".format(
channel_id=channel_id
)
)
except DatabaseError:
logger.warning(
"Tried to import channel {channel_id}, but database file was corrupted.".format(
channel_id=channel_id
)
)
def fix_multiple_trees_with_tree_id1():
# Do a check for improperly imported ContentNode trees
# These trees have been naively imported, and so there are multiple trees
# with tree_ids set to 1. Just check the root nodes to reduce the query size.
tree_id_one_channel_ids = ContentNode.objects.filter(
parent=None, tree_id=1
).values_list("channel_id", flat=True)
if len(tree_id_one_channel_ids) > 1:
logger.warning("Improperly imported channels discovered")
# There is more than one channel with a tree_id of 1
# Find which channel has the most content nodes, and then delete and reimport the rest.
channel_sizes = {}
for channel_id in tree_id_one_channel_ids:
channel_sizes[channel_id] = ContentNode.objects.filter(
channel_id=channel_id
).count()
# Get sorted list of ids by increasing number of nodes
sorted_channel_ids = sorted(channel_sizes, key=channel_sizes.get)
# Loop through all but the largest channel, delete and reimport
count = 0
for channel_id in sorted_channel_ids[:-1]:
# Double check that we have a content db to import from before deleting any metadata
if os.path.exists(get_content_database_file_path(channel_id)):
logger.warning(
"Deleting and reimporting channel metadata for {channel_id}".format(
channel_id=channel_id
)
)
ChannelMetadata.objects.get(
id=channel_id
).delete_content_tree_and_files()
import_channel_from_local_db(channel_id)
logger.info(
"Successfully reimported channel metadata for {channel_id}".format(
channel_id=channel_id
)
)
count += 1
else:
logger.warning(
"Attempted to reimport channel metadata for channel {channel_id} but no content database found".format(
channel_id=channel_id
)
)
if count:
logger.info(
"Successfully reimported channel metadata for {count} channels".format(
count=count
)
)
failed_count = len(sorted_channel_ids) - 1 - count
if failed_count:
logger.warning(
"Failed to reimport channel metadata for {count} channels".format(
count=failed_count
)
)
def test_non_existent_root_node(self, initialize_manager_mock):
import_mock = MagicMock()
initialize_manager_mock.return_value = import_mock
channel_id = "6199dde695db4ee4ab392222d5af1e5c"
def create_channel():
ChannelMetadata.objects.create(id=channel_id,
name="test",
min_schema_version="1",
root_id=channel_id)
import_mock.import_channel_data.side_effect = create_channel
import_channel_from_local_db(channel_id)
try:
channel = ChannelMetadata.objects.get(id=channel_id)
assert channel.root
except ContentNode.DoesNotExist:
self.fail("Channel imported without a valid root node")
