def test_no_nodes_present_peer(self, channel_stats_mock): ContentNode.objects.update(available=False) LocalFile.objects.update(available=False) stats = {} channel_stats_mock.return_value = stats _, files_to_transfer, _ = get_import_export_data( self.the_channel_id, [], [], False, renderable_only=False, peer_id="1" ) self.assertEqual(len(files_to_transfer), 0)
def test_one_node_present_peer(self, channel_stats_mock): ContentNode.objects.update(available=False) LocalFile.objects.update(available=False) obj = ContentNode.objects.get(title="c2c1") stats = {obj.id: {}} channel_stats_mock.return_value = stats _, files_to_transfer, _ = get_import_export_data( self.the_channel_id, [], [], False, renderable_only=False, peer_id="1" ) self.assertEqual(len(files_to_transfer), obj.files.count())
def test_all_nodes_present_peer(self, channel_stats_mock): ContentNode.objects.update(available=False) LocalFile.objects.update(available=False) stats = { key: {} for key in ContentNode.objects.all().values_list("id", flat=True) } channel_stats_mock.return_value = stats _, files_to_transfer, _ = get_import_export_data( self.the_channel_id, [], [], False, renderable_only=False, peer_id="1" ) self.assertEqual( len(files_to_transfer), LocalFile.objects.filter(available=False).count() )
def post(self, request): try: channel_id = self.request.data["channel_id"] except KeyError: raise ValidationError( "channel_id is required for calculating file size and resource counts" ) drive_id = self.request.data.get("drive_id") peer_id = self.request.data.get("peer_id") for_export = self.request.data.get("export") node_ids = self.request.data.get("node_ids") exclude_node_ids = self.request.data.get("exclude_node_ids") flag_count = sum( int(bool(flag)) for flag in (drive_id, peer_id, for_export)) if flag_count > 1: raise ValidationError( "Must specify at most one of drive_id, peer_id, and export") # By default filter to unavailable files available = False if for_export: available = True try: ( total_resource_count, files_to_download, total_bytes_to_transfer, ) = get_import_export_data( channel_id, node_ids, exclude_node_ids, available, drive_id=drive_id, peer_id=peer_id, ) except LocationError: if drive_id: raise ValidationError( "The external drive with given drive id {} does not exist." .format(drive_id)) if peer_id: raise ValidationError( "The network location with the id {} does not exist". format(peer_id)) return Response({ "resource_count": total_resource_count, "file_size": total_bytes_to_transfer, })
def delete_metadata(channel, node_ids, exclude_node_ids, force_delete): # Only delete all metadata if we are not doing selective deletion delete_all_metadata = not (node_ids or exclude_node_ids) if node_ids or exclude_node_ids: # If we have been passed node ids do not do a full deletion pass set_content_invisible(channel.id, node_ids, exclude_node_ids) # If everything has been made invisible, delete all the metadata delete_all_metadata = not channel.root.available if force_delete: # Do this before we delete all the metadata, as otherwise we lose # track of which local files were associated with the channel we # just deleted. _, unused_files, _ = get_import_export_data( channel.id, node_ids, exclude_node_ids, # Don't filter by availability as we have set nodes invisible # above, but the localfiles we are trying to delete are still # available None, renderable_only=False, topic_thumbnails=False, ) with db_lock(): propagate_forced_localfile_removal(unused_files) # Separate these operations as running the SQLAlchemy code in the latter # seems to cause the Django ORM interactions in the former to roll back # Not quite sure what is causing it, but presumably due to transaction # scopes. reannotate_all_channels() if delete_all_metadata: logger.info("Deleting all channel metadata") with db_lock(): channel.delete_content_tree_and_files() # Clear any previously set channel availability stats for this channel clear_channel_stats(channel.id) return delete_all_metadata
def test_no_exclude_duplicate_files(self): """ Test that including a node id in exclude_node_ids does not exclude a shared file that is also used an in included node """ root_node = ContentNode.objects.get(parent__isnull=True) node = ContentNode.objects.filter( parent=root_node, kind=content_kinds.TOPIC ).first() node1 = ContentNode.objects.create( title="test1", id=uuid.uuid4().hex, content_id=uuid.uuid4().hex, channel_id=root_node.channel_id, parent=node, kind=content_kinds.VIDEO, available=False, ) node2 = ContentNode.objects.create( title="test2", id=uuid.uuid4().hex, content_id=uuid.uuid4().hex, channel_id=root_node.channel_id, parent=node, kind=content_kinds.VIDEO, available=False, ) local_file = LocalFile.objects.create( id=uuid.uuid4().hex, extension="mp4", available=False, file_size=10 ) File.objects.create( id=uuid.uuid4().hex, local_file=local_file, contentnode=node1 ) File.objects.create( id=uuid.uuid4().hex, local_file=local_file, contentnode=node2 ) _, files_to_transfer, _ = get_import_export_data( root_node.channel_id, [node1.id], [node2.id], False, renderable_only=False ) self.assertEqual( len(list(filter(lambda x: x.id == local_file.id, files_to_transfer))), 1 )
def handle_async(self, *args, **options): if paths.using_remote_storage(): raise CommandError( "Cannot export files when using remote file storage") channel_id = options["channel_id"] data_dir = os.path.realpath(options["destination"]) node_ids = options["node_ids"] exclude_node_ids = options["exclude_node_ids"] logger.info("Exporting content for channel id {} to {}".format( channel_id, data_dir)) ( total_resource_count, files, total_bytes_to_transfer, ) = get_import_export_data(channel_id, node_ids, exclude_node_ids, True) self.update_job_metadata(total_bytes_to_transfer, total_resource_count) exported_files = [] with self.start_progress( total=total_bytes_to_transfer) as overall_progress_update: for f in files: if self.is_cancelled(): break dest = self.export_file(f, data_dir, overall_progress_update) if dest: exported_files.append(dest) if self.is_cancelled(): # Cancelled, clean up any already downloading files. for dest in exported_files: os.remove(dest) self.cancel()
def delete_metadata(channel, node_ids, exclude_node_ids, force_delete): # Only delete all metadata if we are not doing selective deletion delete_all_metadata = not (node_ids or exclude_node_ids) if node_ids or exclude_node_ids: # If we have been passed node ids do not do a full deletion pass with db_task_write_lock: set_content_invisible(channel.id, node_ids, exclude_node_ids) # If everything has been made invisible, delete all the metadata delete_all_metadata = not channel.root.available if force_delete: # Do this before we delete all the metadata, as otherwise we lose # track of which local files were associated with the channel we # just deleted. _, unused_files, _ = get_import_export_data( channel.id, node_ids, exclude_node_ids, # Don't filter by availability as we have set nodes invisible # above, but the localfiles we are trying to delete are still # available None, renderable_only=False, topic_thumbnails=False, ) with db_task_write_lock: propagate_forced_localfile_removal(unused_files) if delete_all_metadata: logger.info("Deleting all channel metadata") with db_task_write_lock: channel.delete_content_tree_and_files() # Clear any previously set channel availability stats for this channel clear_channel_stats(channel.id) return delete_all_metadata
def _transfer( # noqa: max-complexity=16 self, method, channel_id, path=None, drive_id=None, node_ids=None, exclude_node_ids=None, baseurl=None, peer_id=None, renderable_only=True, import_updates=False, ): try: if not import_updates: ( total_resource_count, files_to_download, total_bytes_to_transfer, ) = get_import_export_data( channel_id, node_ids, exclude_node_ids, False, renderable_only=renderable_only, drive_id=drive_id, peer_id=peer_id, ) else: ( total_resource_count, files_to_download, total_bytes_to_transfer, ) = get_import_data_for_update( channel_id, renderable_only=renderable_only, drive_id=drive_id, peer_id=peer_id, ) except LocationError: if drive_id: raise CommandError( "The external drive with given drive id {} does not exist.".format( drive_id ) ) if peer_id: raise CommandError( "The network location with the id {} does not exist".format(peer_id) ) except ValueError: if import_updates: raise CommandError( "Tried to perform an channel update import when update data was not available" ) raise job = get_current_job() if job: job.extra_metadata["file_size"] = total_bytes_to_transfer job.extra_metadata["total_resources"] = total_resource_count job.save_meta() number_of_skipped_files = 0 transferred_file_size = 0 file_checksums_to_annotate = [] public = None # If we're downloading, check listing status if method == DOWNLOAD_METHOD: public = lookup_channel_listing_status( channel_id=channel_id, baseurl=baseurl ) resources_before_transfer = ( ContentNode.objects.filter(channel_id=channel_id, available=True) .exclude(kind=content_kinds.TOPIC) .values("content_id") .distinct() .count() ) dummy_bytes_for_annotation = annotation.calculate_dummy_progress_for_annotation( node_ids, exclude_node_ids, total_bytes_to_transfer ) with self.start_progress( total=total_bytes_to_transfer + dummy_bytes_for_annotation ) as overall_progress_update: if method == DOWNLOAD_METHOD: session = requests.Session() file_transfers = [] for f in files_to_download: if self.is_cancelled(): break filename = f.get_filename() try: dest = paths.get_content_storage_file_path(filename) except InvalidStorageFilenameError: # If the destination file name is malformed, just stop now. overall_progress_update(f.file_size) continue # if the file already exists, or we are using remote storage, add its size to our overall progress, and skip if paths.using_remote_storage() or ( os.path.isfile(dest) and os.path.getsize(dest) == f.file_size ): overall_progress_update(f.file_size) file_checksums_to_annotate.append(f.id) transferred_file_size += f.file_size continue # determine where we're downloading/copying from, and create appropriate transfer object if method == DOWNLOAD_METHOD: url = paths.get_content_storage_remote_url( filename, baseurl=baseurl ) filetransfer = transfer.FileDownload( url, dest, session=session, cancel_check=self.is_cancelled ) file_transfers.append((f, filetransfer)) elif method == COPY_METHOD: try: srcpath = paths.get_content_storage_file_path( filename, datafolder=path ) except InvalidStorageFilenameError: # If the source file name is malformed, just stop now. overall_progress_update(f.file_size) continue filetransfer = transfer.FileCopy( srcpath, dest, cancel_check=self.is_cancelled ) file_transfers.append((f, filetransfer)) with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: batch_size = 100 # ThreadPoolExecutor allows us to download files concurrently, # greatly reducing download time in most cases. However, loading # all the downloads into the pool requires considerable memory, # so we divide the downloads into batches to keep memory usage down. # In batches of 100, total RAM usage doesn't exceed 250MB in testing. while len(file_transfers) > 0: future_file_transfers = {} for i in range(batch_size): if len(file_transfers) > 0: f, filetransfer = file_transfers.pop() future = executor.submit( self._start_file_transfer, f, filetransfer ) future_file_transfers[future] = (f, filetransfer) for future in concurrent.futures.as_completed( future_file_transfers ): f, filetransfer = future_file_transfers[future] try: status, data_transferred = future.result() overall_progress_update(data_transferred) if self.is_cancelled(): break if status == FILE_SKIPPED: number_of_skipped_files += 1 else: file_checksums_to_annotate.append(f.id) transferred_file_size += f.file_size except transfer.TransferCanceled: break except Exception as e: logger.error( "An error occurred during content import: {}".format(e) ) if ( isinstance(e, requests.exceptions.HTTPError) and e.response.status_code == 404 ) or (isinstance(e, OSError) and e.errno == 2): # Continue file import when the current file is not found from the source and is skipped. overall_progress_update(f.file_size) number_of_skipped_files += 1 continue else: self.exception = e break annotation.set_content_visibility( channel_id, file_checksums_to_annotate, node_ids=node_ids, exclude_node_ids=exclude_node_ids, public=public, ) resources_after_transfer = ( ContentNode.objects.filter(channel_id=channel_id, available=True) .exclude(kind=content_kinds.TOPIC) .values("content_id") .distinct() .count() ) if job: job.extra_metadata["transferred_file_size"] = transferred_file_size job.extra_metadata["transferred_resources"] = ( resources_after_transfer - resources_before_transfer ) job.save_meta() if number_of_skipped_files > 0: logger.warning( "{} files are skipped, because errors occurred during the import.".format( number_of_skipped_files ) ) overall_progress_update(dummy_bytes_for_annotation) if self.exception: raise self.exception if self.is_cancelled(): self.cancel()