def create(self, request, repository_pk): """ Queues a task that creates a new RepositoryVersion by adding and removing content units """ add_content_units = [] remove_content_units = [] repository = self.get_parent_object() if 'base_version' in request.data: base_version_pk = self.get_resource(request.data['base_version'], RepositoryVersion).pk else: base_version_pk = None if 'add_content_units' in request.data: for url in request.data['add_content_units']: content = self.get_resource(url, Content) add_content_units.append(content.pk) if 'remove_content_units' in request.data: for url in request.data['remove_content_units']: content = self.get_resource(url, Content) remove_content_units.append(content.pk) result = enqueue_with_reservation( tasks.repository.add_and_remove, [repository], kwargs={ 'repository_pk': repository_pk, 'base_version_pk': base_version_pk, 'add_content_units': add_content_units, 'remove_content_units': remove_content_units } ) return OperationPostponedResponse(result, request)
def delete(self, request, format=None): """ Cleans up all the Content and Artifact orphans in the system """ async_result = enqueue_with_reservation(orphan_cleanup, []) return OperationPostponedResponse(async_result, request)
def destroy(self, request, repository_pk, number): """ Queues a task to handle deletion of a RepositoryVersion """ version = self.get_object() async_result = enqueue_with_reservation( tasks.repository.delete_version, [version.repository], kwargs={'pk': version.pk} ) return OperationPostponedResponse(async_result, request)
def destroy(self, request, pk): """ Generates a Task to delete a Repository """ repo = self.get_object() async_result = enqueue_with_reservation( tasks.repository.delete, [repo], kwargs={'repo_id': repo.id} ) return OperationPostponedResponse(async_result, request)
def delete(self, request, pk, *args, **kwargs): """ Dispatches a task with reservation for deleting a distribution. """ self.get_object() async_result = enqueue_with_reservation( tasks.distribution.delete, "/api/v3/distributions/", args=(pk,) ) return OperationPostponedResponse(async_result, request)
def destroy(self, request, pk, **kwargs): """ Delete a model instance """ instance = self.get_object() serializer = self.get_serializer(instance) app_label = instance._meta.app_label async_result = enqueue_with_reservation( tasks.base.general_delete, [instance], args=(pk, app_label, serializer.__class__.__name__)) return OperationPostponedResponse(async_result, request)
def create(self, request, *args, **kwargs): """ Dispatches a task with reservation for creating a distribution. """ serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) async_result = enqueue_with_reservation( tasks.distribution.create, "/api/v3/distributions/", kwargs={'data': request.data} ) return OperationPostponedResponse(async_result, request)
def update(self, request, pk, **kwargs): partial = kwargs.pop('partial', False) instance = self.get_object() serializer = self.get_serializer(instance, data=request.data, partial=partial) serializer.is_valid(raise_exception=True) app_label = instance._meta.app_label async_result = enqueue_with_reservation( tasks.base.general_update, [instance], args=(pk, app_label, serializer.__class__.__name__), kwargs={'data': request.data, 'partial': partial} ) return OperationPostponedResponse(async_result, request)
def repair(self, request, repository_pk, number): """ Queues a task to repair currupted artifacts corresponding to a RepositoryVersion """ version = self.get_object() async_result = enqueue_with_reservation( tasks.repository.repair_version, [version.repository], kwargs={"repository_version_pk": version.pk}, ) return OperationPostponedResponse(async_result, request)
def post(self, request): """ Repair artifacts. """ serializer = RepairSerializer(data=request.data) serializer.is_valid() verify_checksums = serializer.validated_data["verify_checksums"] async_result = enqueue_with_reservation(repair_all_artifacts, [], args=[verify_checksums]) return OperationPostponedResponse(async_result, request)
def destroy(self, request, pk, **kwargs): """ Delete a model instance """ instance = self.get_object() serializer = self.get_serializer(instance) app_label = instance._meta.app_label async_result = enqueue_with_reservation( tasks.base.general_delete, [instance], args=(pk, app_label, serializer.__class__.__name__) ) return OperationPostponedResponse(async_result, request)
def update(self, request, pk, partial=False): """ Generates a Task to update a Repository """ instance = self.get_object() serializer = self.get_serializer(instance, data=request.data, partial=partial) serializer.is_valid(raise_exception=True) async_result = enqueue_with_reservation( tasks.repository.update, [instance], args=(instance.pk, ), kwargs={'data': request.data, 'partial': partial} ) return OperationPostponedResponse(async_result, request)
def update(self, request, pk, partial=False): """ Generates a Task to update a Repository """ instance = self.get_object() serializer = self.get_serializer(instance, data=request.data, partial=partial) serializer.is_valid(raise_exception=True) async_result = enqueue_with_reservation( tasks.repository.update, [instance], args=(instance.id, ), kwargs={'data': request.data, 'partial': partial} ) return OperationPostponedResponse(async_result, request)
def destroy(self, request, repository_pk, number): """ Queues a task to handle deletion of a RepositoryVersion """ version = self.get_object() if version.number == 0: raise serializers.ValidationError(detail=_("Cannot delete repository version 0.")) async_result = enqueue_with_reservation( tasks.repository.delete_version, [version.repository], kwargs={"pk": version.pk} ) return OperationPostponedResponse(async_result, request)
def commit(self, request, pk): """ Queues a Task that creates an Artifact, and the Upload gets deleted and cannot be re-used. """ serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) sha256 = serializer.validated_data["sha256"] upload = self.get_object() async_result = enqueue_with_reservation(tasks.upload.commit, [upload], args=(upload.pk, sha256)) return OperationPostponedResponse(async_result, request)
def create(self, request, *args, **kwargs): """ Dispatches a task with reservation for creating an instance. """ serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) app_label = self.queryset.model._meta.app_label async_result = enqueue_with_reservation( tasks.base.general_create, self.async_reserved_resources(None), args=(app_label, serializer.__class__.__name__), kwargs={'data': request.data}) return OperationPostponedResponse(async_result, request)
def update(self, request, pk, *args, **kwargs): """ Dispatches a task with reservation for updating a distribution. """ partial = kwargs.pop('partial', False) instance = self.get_object() serializer = self.get_serializer(instance, data=request.data, partial=partial) serializer.is_valid(raise_exception=True) async_result = enqueue_with_reservation( tasks.distribution.update, "/api/v3/distributions/", args=(pk,), kwargs={'data': request.data, 'partial': partial} ) return OperationPostponedResponse(async_result, request)
def repair(self, request, repository_pk, number): """ Queues a task to repair corrupted artifacts corresponding to a RepositoryVersion """ version = self.get_object() serializer = RepairSerializer(data=request.data) serializer.is_valid() verify_checksums = serializer.validated_data["verify_checksums"] async_result = enqueue_with_reservation( tasks.repository.repair_version, [version.repository], args=[version.pk, verify_checksums], ) return OperationPostponedResponse(async_result, request)
def modify(self, request, pk): """ Queues a task that creates a new RepositoryVersion by adding and removing content units """ add_content_units = [] remove_content_units = [] repository = Repository.objects.get(pk=pk) serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) if 'base_version' in request.data: base_version_pk = self.get_resource(request.data['base_version'], RepositoryVersion).pk else: base_version_pk = None if 'add_content_units' in request.data: for url in request.data['add_content_units']: content = self.get_resource(url, Content) add_content_units.append(content.pk) if 'remove_content_units' in request.data: for url in request.data['remove_content_units']: if url == '*': remove_content_units = [url] break else: content = self.get_resource(url, Content) remove_content_units.append(content.pk) result = enqueue_with_reservation(tasks.repository.add_and_remove, [repository], kwargs={ 'repository_pk': pk, 'base_version_pk': base_version_pk, 'add_content_units': add_content_units, 'remove_content_units': remove_content_units }) return OperationPostponedResponse(result, request)
def post(self, request, *args, **kwargs): """ Queues a task that creates a new Collection from an uploaded artifact. """ serializer = GalaxyCollectionUploadSerializer( data=request.data, context={'request': request}) serializer.is_valid(raise_exception=True) expected_digests = {'sha256': serializer.validated_data['sha256']} artifact = Artifact.init_and_validate( serializer.validated_data['file'], expected_digests=expected_digests) artifact.save() async_result = enqueue_with_reservation(import_collection, [str(artifact.pk)], kwargs={ 'artifact_pk': artifact.pk, }) return OperationPostponedResponse(async_result, request)
def modify(self, request, pk): """ Queues a task that creates a new RepositoryVersion by adding and removing content units """ add_content_units = [] remove_content_units = [] repository = self.get_object() serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) if "base_version" in request.data: base_version_pk = self.get_resource(request.data["base_version"], RepositoryVersion).pk else: base_version_pk = None if "add_content_units" in request.data: for url in request.data["add_content_units"]: content = self.get_resource(url, Content) add_content_units.append(content.pk) if "remove_content_units" in request.data: for url in request.data["remove_content_units"]: if url == "*": remove_content_units = [url] break else: content = self.get_resource(url, Content) remove_content_units.append(content.pk) result = enqueue_with_reservation( tasks.repository.add_and_remove, [repository], kwargs={ "repository_pk": pk, "base_version_pk": base_version_pk, "add_content_units": add_content_units, "remove_content_units": remove_content_units, }, ) return OperationPostponedResponse(result, request)
def create(self, request, importer_pk): """Import a Pulp export into Pulp.""" try: importer = PulpImporter.objects.get(pk=importer_pk) except PulpImporter.DoesNotExist: raise Http404 serializer = PulpImportSerializer(data=request.data, context={"request": request}) serializer.is_valid(raise_exception=True) path = serializer.validated_data.get("path") # resources = [pulp_importer] + result = enqueue_with_reservation( pulp_import, [importer], kwargs={ "importer_pk": importer.pk, "path": path }, ) return OperationPostponedResponse(result, request)
def post(self, request, path): """ Queues a task that creates a new Collection from an uploaded artifact. """ distro = get_object_or_404(AnsibleDistribution, base_path=path) serializer = GalaxyCollectionUploadSerializer( data=request.data, context={"request": request}) serializer.is_valid(raise_exception=True) artifact = Artifact.init_and_validate( serializer.validated_data["file"]) artifact.save() locks = [str(artifact.pk)] kwargs = {"artifact_pk": artifact.pk} if distro.repository: locks.append(distro.repository) kwargs["repository_pk"] = distro.repository.pk async_result = enqueue_with_reservation(import_collection, locks, kwargs=kwargs) return OperationPostponedResponse(async_result, request)
def create(self, request, repository_pk): """ Queues a task that creates a new RepositoryVersion by adding and removing content units """ add_content_units = [] remove_content_units = [] repository = self.get_parent_object() if 'base_version' in request.data: base_version_pk = self.get_resource(request.data['base_version'], RepositoryVersion).pk else: base_version_pk = None if 'add_content_units' in request.data: for url in request.data['add_content_units']: content = self.get_resource(url, Content) add_content_units.append(content.pk) if 'remove_content_units' in request.data: for url in request.data['remove_content_units']: content = self.get_resource(url, Content) remove_content_units.append(content.pk) result = enqueue_with_reservation(tasks.repository.add_and_remove, [repository], kwargs={ 'repository_pk': repository_pk, 'base_version_pk': base_version_pk, 'add_content_units': add_content_units, 'remove_content_units': remove_content_units }) return OperationPostponedResponse(result, request)
def pulp_import(importer_pk, path): """ Import a Pulp export into Pulp. Args: importer_pk (str): Primary key of PulpImporter to do the import path (str): Path to the export to be imported """ def destination_repo(source_repo_name): """Find the destination repository based on source repo's name.""" if importer.repo_mapping and importer.repo_mapping.get( source_repo_name): dest_repo_name = importer.repo_mapping[source_repo_name] else: dest_repo_name = source_repo_name return Repository.objects.get(name=dest_repo_name) log.info(_("Importing {}.").format(path)) importer = PulpImporter.objects.get(pk=importer_pk) the_import = PulpImport.objects.create(importer=importer, task=Task.current(), params={"path": path}) CreatedResource.objects.create(content_object=the_import) task_group = TaskGroup.objects.create(description=f"Import of {path}") CreatedResource.objects.create(content_object=task_group) with tempfile.TemporaryDirectory() as temp_dir: with tarfile.open(path, "r:gz") as tar: tar.extractall(path=temp_dir) # Check version info with open(os.path.join(temp_dir, VERSIONS_FILE)) as version_file: version_json = json.load(version_file) _check_versions(version_json) # Artifacts ar_result = _import_file(os.path.join(temp_dir, ARTIFACT_FILE), ArtifactResource) for row in ar_result.rows: artifact = Artifact.objects.get(pk=row.object_id) base_path = os.path.join("artifact", artifact.sha256[0:2], artifact.sha256[2:]) src = os.path.join(temp_dir, base_path) dest = os.path.join(settings.MEDIA_ROOT, base_path) if not default_storage.exists(dest): with open(src, "rb") as f: default_storage.save(dest, f) with open(os.path.join(temp_dir, REPO_FILE), "r") as repo_data_file: data = json.load(repo_data_file) for src_repo in data: try: dest_repo = destination_repo(src_repo["name"]) except Repository.DoesNotExist: log.warn( _("Could not find destination repo for {}. " "Skipping.").format(src_repo["name"])) continue enqueue_with_reservation( import_repository_version, [dest_repo], args=[dest_repo.pk, src_repo["name"], path], task_group=task_group, ) task_group.finish()
def pulp_import(importer_pk, path, toc): """ Import a Pulp export into Pulp. Args: importer_pk (str): Primary key of PulpImporter to do the import path (str): Path to the export to be imported """ def _compute_hash(filename): sha256_hash = hashlib.sha256() with open(filename, "rb") as f: # Read and update hash string value in blocks of 4K for byte_block in iter(lambda: f.read(4096), b""): sha256_hash.update(byte_block) return sha256_hash.hexdigest() def validate_toc(toc_filename): """ Check validity of table-of-contents file. table-of-contents must: * exist * be valid JSON * point to chunked-export-files that exist 'next to' the 'toc' file * point to chunks whose checksums match the checksums stored in the 'toc' file Args: toc_filename (str): The user-provided toc-file-path to be validated. Raises: ValidationError: If toc is not a valid JSON table-of-contents file, or when toc points to chunked-export-files that can't be found in the same directory as the toc-file, or the checksums of the chunks do not match the checksums stored in toc. """ with open(toc_filename) as json_file: # Valid JSON? the_toc = json.load(json_file) if not the_toc.get("files", None) or not the_toc.get("meta", None): raise ValidationError(_("Missing 'files' or 'meta' keys in table-of-contents!")) base_dir = os.path.dirname(toc_filename) # Points at chunks that exist? missing_files = [] for f in sorted(the_toc["files"].keys()): if not os.path.isfile(os.path.join(base_dir, f)): missing_files.append(f) if missing_files: raise ValidationError( _( "Missing import-chunks named in table-of-contents: {}.".format( str(missing_files) ) ) ) errs = [] # validate the sha256 of the toc-entries # gather errors for reporting at the end chunks = sorted(the_toc["files"].keys()) data = dict(message="Validating Chunks", code="validate.chunks", total=len(chunks)) with ProgressReport(**data) as pb: for chunk in pb.iter(chunks): a_hash = _compute_hash(os.path.join(base_dir, chunk)) if not a_hash == the_toc["files"][chunk]: err_str = "File {} expected checksum : {}, computed checksum : {}".format( chunk, the_toc["files"][chunk], a_hash ) errs.append(err_str) # if there are any errors, report and fail if errs: raise ValidationError(_("Import chunk hash mismatch: {}).").format(str(errs))) return the_toc def validate_and_assemble(toc_filename): """Validate checksums of, and reassemble, chunks in table-of-contents file.""" the_toc = validate_toc(toc_filename) toc_dir = os.path.dirname(toc_filename) result_file = os.path.join(toc_dir, the_toc["meta"]["file"]) # if we have only one entry in "files", it must be the full .tar.gz - return it if len(the_toc["files"]) == 1: return os.path.join(toc_dir, list(the_toc["files"].keys())[0]) # We have multiple chunks. # reassemble into one file 'next to' the toc and return the resulting full-path chunk_size = int(the_toc["meta"]["chunk_size"]) offset = 0 block_size = 1024 blocks_per_chunk = int(chunk_size / block_size) # sorting-by-filename is REALLY IMPORTANT here # keys are of the form <base-export-name>.00..<base-export-name>.NN, # and must be reassembled IN ORDER the_chunk_files = sorted(the_toc["files"].keys()) data = dict( message="Recombining Chunks", code="recombine.chunks", total=len(the_chunk_files) ) with ProgressReport(**data) as pb: for chunk in pb.iter(the_chunk_files): # For each chunk, add it to the reconstituted tar.gz, picking up where the previous # chunk left off subprocess.run( [ "dd", "if={}".format(os.path.join(toc_dir, chunk)), "of={}".format(result_file), "bs={}".format(str(block_size)), "seek={}".format(str(offset)), ], ) offset += blocks_per_chunk # To keep from taking up All The Disk, we delete each chunk after it has been added # to the recombined file. try: subprocess.run(["rm", "-f", os.path.join(toc_dir, chunk)]) except OSError: log.warning( _("Failed to remove chunk {} after recombining. Continuing.").format( os.path.join(toc_dir, chunk) ), exc_info=True, ) combined_hash = _compute_hash(result_file) if combined_hash != the_toc["meta"]["global_hash"]: raise ValidationError( _("Mismatch between combined .tar.gz checksum [{}] and originating [{}]).").format( combined_hash, the_toc["meta"]["global_hash"] ) ) # if we get this far, then: the chunk-files all existed, they all pass checksum validation, # and there exists a combined .tar.gz, which *also* passes checksum-validation. # Let the rest of the import process do its thing on the new combined-file. return result_file if toc: log.info(_("Validating TOC {}.").format(toc)) path = validate_and_assemble(toc) log.info(_("Importing {}.").format(path)) current_task = Task.current() importer = PulpImporter.objects.get(pk=importer_pk) the_import = PulpImport.objects.create( importer=importer, task=current_task, params={"path": path} ) CreatedResource.objects.create(content_object=the_import) task_group = TaskGroup.objects.create(description=f"Import of {path}") current_task.task_group = task_group current_task.save() CreatedResource.objects.create(content_object=task_group) with tempfile.TemporaryDirectory() as temp_dir: with tarfile.open(path, "r:gz") as tar: tar.extractall(path=temp_dir) # Check version info with open(os.path.join(temp_dir, VERSIONS_FILE)) as version_file: version_json = json.load(version_file) _check_versions(version_json) # Artifacts ar_result = _import_file(os.path.join(temp_dir, ARTIFACT_FILE), ArtifactResource) data = dict( message="Importing Artifacts", code="import.artifacts", total=len(ar_result.rows) ) with ProgressReport(**data) as pb: for row in pb.iter(ar_result.rows): artifact = Artifact.objects.get(pk=row.object_id) base_path = os.path.join("artifact", artifact.sha256[0:2], artifact.sha256[2:]) src = os.path.join(temp_dir, base_path) dest = os.path.join(settings.MEDIA_ROOT, base_path) if not default_storage.exists(dest): with open(src, "rb") as f: default_storage.save(dest, f) with open(os.path.join(temp_dir, REPO_FILE), "r") as repo_data_file: data = json.load(repo_data_file) gpr = GroupProgressReport( message="Importing repository versions", code="import.repo.versions", total=len(data), done=0, task_group=task_group, ) gpr.save() for src_repo in data: try: dest_repo = _destination_repo(importer, src_repo["name"]) except Repository.DoesNotExist: log.warning( _("Could not find destination repo for {}. Skipping.").format( src_repo["name"] ) ) continue enqueue_with_reservation( import_repository_version, [dest_repo], args=[importer.pk, dest_repo.pk, src_repo["name"], path], task_group=task_group, ) task_group.finish()