def test_set_two_files_one_exists(self, path_mock): LocalFile.objects.filter(id=self.file_id_2).update(available=True) set_local_file_availability_from_disk( checksums=[self.file_id_1, self.file_id_2]) self.assertEqual(LocalFile.objects.filter(available=True).count(), 1) self.assertTrue(LocalFile.objects.get(id=self.file_id_1).available) self.assertFalse(LocalFile.objects.get(id=self.file_id_2).available)
def test_set_bad_filenames(self): local_files = list(LocalFile.objects.all()) LocalFile.objects.all().delete() for i, lf in enumerate(local_files): lf.id = "bananas" + str(i) lf.save() set_local_file_availability_from_disk() self.assertEqual(LocalFile.objects.filter(available=True).count(), 0)
def diff_stats(channel_id, method, drive_id=None, baseurl=None): """ Download the channel database to an upgraded path. Annotate the local file availability of the upgraded channel db. Calculate diff stats comparing default db and annotated channel db. """ # upgraded content database path source_path = paths.get_upgrade_content_database_file_path(channel_id) # annotated db to be used for calculating diff stats destination_path = paths.get_annotated_content_database_file_path(channel_id) try: if method == "network": call_command( "importchannel", "network", channel_id, baseurl=baseurl, no_upgrade=True ) elif method == "disk": drive = get_mounted_drive_by_id(drive_id) call_command( "importchannel", "disk", channel_id, drive.datafolder, no_upgrade=True ) # create all fields/tables at the annotated destination db, based on the current schema version bridge = Bridge( sqlite_file_path=destination_path, schema_version=CURRENT_SCHEMA_VERSION ) bridge.Base.metadata.create_all(bridge.engine) # initialize import manager based on annotated destination path, pulling from source db path import_manager = channel_import.initialize_import_manager( channel_id, cancel_check=False, source=source_path, destination=destination_path, ) # import channel data from source db path import_manager.import_channel_data() import_manager.end() # annotate file availability on destination db annotation.set_local_file_availability_from_disk(destination=destination_path) # get the diff count between whats on the default db and the annotated db new_resources_count = count_new_resources_available_for_import( destination_path, channel_id ) # get the count for leaf nodes which are in the default db, but not in the annotated db resources_to_be_deleted_count = count_removed_resources( destination_path, channel_id ) # get the ids of leaf nodes which are now incomplete due to missing local files updated_resources_ids = automatically_updated_resource_ids( destination_path, channel_id ) # remove the annotated database try: os.remove(destination_path) except OSError as e: logger.info( "Tried to remove {}, but exception {} occurred.".format( destination_path, e ) ) # annotate job metadata with diff stats job = get_current_job() if job: job.extra_metadata["new_resources_count"] = new_resources_count job.extra_metadata[ "deleted_resources_count" ] = resources_to_be_deleted_count job.extra_metadata["updated_node_ids"] = updated_resources_ids job.save_meta() except UserCancelledError: # remove the annotated database try: os.remove(destination_path) except OSError: pass raise
def test_set_all_files_two_exist(self, path_mock): set_local_file_availability_from_disk() self.assertEqual(LocalFile.objects.filter(available=True).count(), 2) self.assertEqual(LocalFile.objects.exclude(available=True).count(), 3)
def test_set_all_files_all_exist(self, path_mock): LocalFile.objects.update(available=False) set_local_file_availability_from_disk() self.assertEqual(LocalFile.objects.exclude(available=True).count(), 0)
def test_set_one_file_not_list_not_exist(self, path_mock): set_local_file_availability_from_disk(checksums=self.file_id_1) self.assertEqual(LocalFile.objects.filter(available=True).count(), 0) self.assertFalse(LocalFile.objects.get(id=self.file_id_1).available)