def test_set_two_files_one_exists(self, path_mock):
LocalFile.objects.filter(id=self.file_id_2).update(available=True)
set_local_file_availability_from_disk(
checksums=[self.file_id_1, self.file_id_2])
self.assertEqual(LocalFile.objects.filter(available=True).count(), 1)
self.assertTrue(LocalFile.objects.get(id=self.file_id_1).available)
self.assertFalse(LocalFile.objects.get(id=self.file_id_2).available)
def test_set_bad_filenames(self):
local_files = list(LocalFile.objects.all())
LocalFile.objects.all().delete()
for i, lf in enumerate(local_files):
lf.id = "bananas" + str(i)
lf.save()
set_local_file_availability_from_disk()
self.assertEqual(LocalFile.objects.filter(available=True).count(), 0)
def diff_stats(channel_id, method, drive_id=None, baseurl=None):
"""
Download the channel database to an upgraded path.
Annotate the local file availability of the upgraded channel db.
Calculate diff stats comparing default db and annotated channel db.
"""
# upgraded content database path
source_path = paths.get_upgrade_content_database_file_path(channel_id)
# annotated db to be used for calculating diff stats
destination_path = paths.get_annotated_content_database_file_path(channel_id)
try:
if method == "network":
call_command(
"importchannel", "network", channel_id, baseurl=baseurl, no_upgrade=True
)
elif method == "disk":
drive = get_mounted_drive_by_id(drive_id)
call_command(
"importchannel", "disk", channel_id, drive.datafolder, no_upgrade=True
)
# create all fields/tables at the annotated destination db, based on the current schema version
bridge = Bridge(
sqlite_file_path=destination_path, schema_version=CURRENT_SCHEMA_VERSION
)
bridge.Base.metadata.create_all(bridge.engine)
# initialize import manager based on annotated destination path, pulling from source db path
import_manager = channel_import.initialize_import_manager(
channel_id,
cancel_check=False,
source=source_path,
destination=destination_path,
)
# import channel data from source db path
import_manager.import_channel_data()
import_manager.end()
# annotate file availability on destination db
annotation.set_local_file_availability_from_disk(destination=destination_path)
# get the diff count between whats on the default db and the annotated db
new_resources_count = count_new_resources_available_for_import(
destination_path, channel_id
)
# get the count for leaf nodes which are in the default db, but not in the annotated db
resources_to_be_deleted_count = count_removed_resources(
destination_path, channel_id
)
# get the ids of leaf nodes which are now incomplete due to missing local files
updated_resources_ids = automatically_updated_resource_ids(
destination_path, channel_id
)
# remove the annotated database
try:
os.remove(destination_path)
except OSError as e:
logger.info(
"Tried to remove {}, but exception {} occurred.".format(
destination_path, e
)
)
# annotate job metadata with diff stats
job = get_current_job()
if job:
job.extra_metadata["new_resources_count"] = new_resources_count
job.extra_metadata[
"deleted_resources_count"
] = resources_to_be_deleted_count
job.extra_metadata["updated_node_ids"] = updated_resources_ids
job.save_meta()
except UserCancelledError:
# remove the annotated database
try:
os.remove(destination_path)
except OSError:
pass
raise
def test_set_all_files_two_exist(self, path_mock):
set_local_file_availability_from_disk()
self.assertEqual(LocalFile.objects.filter(available=True).count(), 2)
self.assertEqual(LocalFile.objects.exclude(available=True).count(), 3)
def test_set_all_files_all_exist(self, path_mock):
LocalFile.objects.update(available=False)
set_local_file_availability_from_disk()
self.assertEqual(LocalFile.objects.exclude(available=True).count(), 0)
def test_set_one_file_not_list_not_exist(self, path_mock):
set_local_file_availability_from_disk(checksums=self.file_id_1)
self.assertEqual(LocalFile.objects.filter(available=True).count(), 0)
self.assertFalse(LocalFile.objects.get(id=self.file_id_1).available)
