def general_update(instance_id, app_label, serializer_name, *args, **kwargs): """ Update a model The model instance is identified using the app_label, id, and serializer name. The serializer is used to perform validation. Args: id (str): the id of the model app_label (str): the Django app label of the plugin that provides the model serializer_name (str): name of the serializer class for the model data (dict): dictionary whose keys represent the fields of the model and their corresponding values. partial (bool): When true, only the fields specified in the data dictionary are updated. When false, any fields missing from the data dictionary are assumed to be None and their values are updated as such. Raises: :class:`rest_framework.exceptions.ValidationError`: When serializer instance can't be saved due to validation error. This theoretically should never occur since validation is performed before the task is dispatched. """ data = kwargs.pop('data', None) partial = kwargs.pop('partial', False) serializer_class = get_plugin_config(app_label).named_serializers[serializer_name] instance = serializer_class.Meta.model.objects.get(pk=instance_id).cast() serializer = serializer_class(instance, data=data, partial=partial) serializer.is_valid(raise_exception=True) serializer.save()
def general_update(instance_id, app_label, serializer_name, *args, **kwargs): """ Update a model The model instance is identified using the app_label, id, and serializer name. The serializer is used to perform validation. Args: id (str): the id of the model app_label (str): the Django app label of the plugin that provides the model serializer_name (str): name of the serializer class for the model data (dict): dictionary whose keys represent the fields of the model and their corresponding values. partial (bool): When true, only the fields specified in the data dictionary are updated. When false, any fields missing from the data dictionary are assumed to be None and their values are updated as such. Raises: :class:`rest_framework.exceptions.ValidationError`: When serializer instance can't be saved due to validation error. This theoretically should never occur since validation is performed before the task is dispatched. """ data = kwargs.pop('data', None) partial = kwargs.pop('partial', False) serializer_class = get_plugin_config(app_label).named_serializers[serializer_name] instance = serializer_class.Meta.model.objects.get(pk=instance_id).cast() serializer = serializer_class(instance, data=data, partial=partial) serializer.is_valid(raise_exception=True) serializer.save()
def export_content(export, repository_version): """ Export db-content, and the db-content of the owning repositories Args: export (django.db.models.PulpExport): export instance that's doing the export repository_version (django.db.models.RepositoryVersion): RepositoryVersion being exported last_export (django.db.models.PulpExport): previous export of the 'owning' Exporter """ dest_dir = os.path.join( "repository-{}_{}".format( str(repository_version.repository.pulp_id), repository_version.number ) ) # export the resources pulpcore is responsible for resource = ContentResource(repository_version) _write_export(export.tarfile, resource, dest_dir) resource = ContentArtifactResource(repository_version) _write_export(export.tarfile, resource, dest_dir) # find and export any ModelResource found in pulp_<repo-type>.app.modelresource plugin_name = repository_version.repository.pulp_type.split(".")[0] cfg = get_plugin_config(plugin_name) if cfg.exportable_classes: for cls in cfg.exportable_classes: _write_export(export.tarfile, cls(repository_version), dest_dir)
def import_repository_version(destination_repo_pk, source_repo_pk, tar_path): """ Import a repository version from a Pulp export. Args: destination_repo_pk (str): Primary key of Repository to import into. source_repo_pk (str): Primary key of the Repository in the export. tar_path (str): A path to export tar. """ dest_repo = Repository.objects.get(pk=destination_repo_pk) with tempfile.TemporaryDirectory() as temp_dir: # Extract the repo file for the repo info with tarfile.open(tar_path, "r:gz") as tar: tar.extract(REPO_FILE, path=temp_dir) with open(os.path.join(temp_dir, REPO_FILE), "r") as repo_data_file: data = json.load(repo_data_file) src_repo = next(repo for repo in data if repo["pulp_id"] == source_repo_pk) rv_path = os.path.join(temp_dir, _repo_version_path(src_repo)) if dest_repo.pulp_type != src_repo["pulp_type"]: raise ValidationError( _( "Repository type mismatch: {src_repo} ({src_type}) vs {dest_repo} " "({dest_type})." ).format( src_repo=src_repo["name"], src_type=src_repo["pulp_type"], dest_repo=dest_repo.name, dest_type=dest_repo.pulp_type, ) ) # Extract the repo version files with tarfile.open(tar_path, "r:gz") as tar: for mem in tar.getmembers(): if re.match(fr"^{_repo_version_path(src_repo)}/.+", mem.name): tar.extract(mem, path=temp_dir) # Untyped Content content_path = os.path.join(rv_path, CONTENT_FILE) c_result = _import_file(content_path, ContentResource) content = Content.objects.filter(pk__in=[r.object_id for r in c_result.rows]) # Content Artifacts ca_path = os.path.join(rv_path, CA_FILE) _import_file(ca_path, ContentArtifactResource) # Content plugin_name = src_repo["pulp_type"].split(".")[0] cfg = get_plugin_config(plugin_name) for res_class in cfg.exportable_classes: filename = f"{res_class.__module__}.{res_class.__name__}.json" _import_file(os.path.join(rv_path, filename), res_class) # Create the repo version with dest_repo.new_version() as new_version: new_version.set_content(content)
def export_content(export, repository_version): """ Export db-content, and the db-content of the owning repositories Args: export (django.db.models.PulpExport): export instance that's doing the export repository_version (django.db.models.RepositoryVersion): RepositoryVersion being exported """ def _combine_content_mappings(map1, map2): """Combine two content mapping dicts into one by combining ids for for each key.""" result = {} for key in map1.keys() | map2.keys(): result[key] = list(set(map1.get(key, []) + map2.get(key, []))) return result dest_dir = os.path.join("repository-{}_{}".format( str(repository_version.repository.name), repository_version.number)) # Export the connection between content and artifacts resource = ContentArtifactResource(repository_version) _write_export(export.tarfile, resource, dest_dir) # content mapping is used by repo versions with subrepos (eg distribution tree repos) content_mapping = {} # find and export any ModelResource found in pulp_<repo-type>.app.modelresource plugin_name = repository_version.repository.pulp_type.split(".")[0] cfg = get_plugin_config(plugin_name) if cfg.exportable_classes: for cls in cfg.exportable_classes: resource = cls(repository_version) _write_export(export.tarfile, resource, dest_dir) if hasattr(resource, "content_mapping") and resource.content_mapping: content_mapping = _combine_content_mappings( content_mapping, resource.content_mapping) msg = ( f"Exporting content for {plugin_name} " f"repository-version {repository_version.repository.name}/{repository_version.number}" ) content_count = repository_version.content.count() data = dict( message=msg, code="export.repo.version.content", total=content_count, done=content_count, state=TASK_STATES.COMPLETED, ) pb = ProgressReport(**data) pb.save() if content_mapping: # write the content mapping to tarfile cm_json = json.dumps(content_mapping).encode("utf8") info = tarfile.TarInfo(name=f"{dest_dir}/content_mapping.json") info.size = len(cm_json) export.tarfile.addfile(info, io.BytesIO(cm_json))
def get_plugin_storage_path(plugin_app_label): """ Returns the path to the plugin's storage An interface for finding the path to a plugin's persistent storage location. It is designed to be used by plugins that need to store more than just :class:`~pulpcore.plugin.models.Artifact` models. Args: plugin_app_label (str): Django app label of the pulp plugin Returns: String containing the absolute path to the plugin's storage on the filesystem. Raises: :class:`~pulpcore.exceptions.plugin.MissingPlugin`: When plugin with the requested app label is not installed. """ get_plugin_config(plugin_app_label) return os.path.join("/var/lib/pulp/shared", plugin_app_label, "")
def get_plugin_storage_path(plugin_app_label): """ Returns the path to the plugin's storage An interface for finding the path to a plugin's persistent storage location. It is designed to be used by plugins that need to store more than just :class:`~pulpcore.plugin.models.Artifact` models. Args: plugin_app_label (str): Django app label of the pulp plugin Returns: String containing the absolute path to the plugin's storage on the filesystem. Raises: :class:`~pulpcore.exceptions.plugin.MissingPlugin`: When plugin with the requested app label is not installed. """ get_plugin_config(plugin_app_label) return os.path.join('/var/lib/pulp/shared', plugin_app_label, '')
def general_delete(instance_id, app_label, serializer_name): """ Delete a model The model instance is identified using the app_label, id, and serializer name. Args: id (str): the id of the model app_label (str): the Django app label of the plugin that provides the model serializer_name (str): name of the serializer class for the model """ serializer_class = get_plugin_config(app_label).named_serializers[serializer_name] instance = serializer_class.Meta.model.objects.get(pk=instance_id).cast() instance.delete()
def general_delete(instance_id, app_label, serializer_name): """ Delete a model The model instance is identified using the app_label, id, and serializer name. Args: id (str): the id of the model app_label (str): the Django app label of the plugin that provides the model serializer_name (str): name of the serializer class for the model """ serializer_class = get_plugin_config(app_label).named_serializers[serializer_name] instance = serializer_class.Meta.model.objects.get(pk=instance_id).cast() instance.delete()
def general_create(app_label, serializer_name, *args, **kwargs): """ Create a model instance. Raises: ValidationError: If the serializer is not valid """ data = kwargs.pop('data', None) serializer_class = get_plugin_config(app_label).named_serializers[serializer_name] serializer = serializer_class(data=data) serializer.is_valid(raise_exception=True) serializer.save() resource = CreatedResource(content_object=serializer.instance) resource.save()
def general_multi_delete(instance_ids): """ Delete a list of model instances in a transaction The model instances are identified using the id, app_label, and serializer_name. Args: instance_ids (list): List of tupels of id, app_label, serializer_name """ instances = [] for instance_id, app_label, serializer_name in instance_ids: serializer_class = get_plugin_config( app_label).named_serializers[serializer_name] instances.append( serializer_class.Meta.model.objects.get(pk=instance_id).cast()) with transaction.atomic(): for instance in instances: instance.delete()
def general_create(app_label, serializer_name, *args, **kwargs): """ Create a model instance. Raises: ValidationError: If the serializer is not valid """ data = kwargs.pop("data", None) context = kwargs.pop("context", {}) serializer_class = get_plugin_config( app_label).named_serializers[serializer_name] serializer = serializer_class(data=data, context=context) serializer.is_valid(raise_exception=True) serializer.save() instance = serializer_class.Meta.model.objects.get( pk=serializer.instance.pk).cast() resource = CreatedResource(content_object=instance) resource.save()
def general_create_from_temp_file(app_label, serializer_name, *args, **kwargs): """ Create a model instance from contents stored in a temporary Artifact. A caller should always pass the dictionary "data", as a keyword argument, containing the href to the temporary Artifact. Otherwise, the function does nothing. This function calls the function general_create() to create a model instance. Data passed to that function already contains a serialized artifact converted to PulpTemporaryUploadFile that will be deleted afterwards. """ data = kwargs.pop("data", None) if data and "artifact" in data: named_model_view_set = get_plugin_config( app_label).viewsets_module.NamedModelViewSet artifact = named_model_view_set.get_resource(data.pop("artifact"), Artifact) data["file"] = PulpTemporaryUploadedFile.from_file(artifact.file) general_create(app_label, serializer_name, data=data, *args, **kwargs) artifact.delete()
def import_repository_version(importer_pk, destination_repo_pk, source_repo_name, tar_path): """ Import a repository version from a Pulp export. Args: importer_pk (str): Importer we are working with destination_repo_pk (str): Primary key of Repository to import into. source_repo_name (str): Name of the Repository in the export. tar_path (str): A path to export tar. """ dest_repo = Repository.objects.get(pk=destination_repo_pk) importer = PulpImporter.objects.get(pk=importer_pk) pb = ProgressReport( message=f"Importing content for {dest_repo.name}", code="import.repo.version.content", state=TASK_STATES.RUNNING, ) pb.save() with tempfile.TemporaryDirectory() as temp_dir: # Extract the repo file for the repo info with tarfile.open(tar_path, "r:gz") as tar: tar.extract(REPO_FILE, path=temp_dir) with open(os.path.join(temp_dir, REPO_FILE), "r") as repo_data_file: data = json.load(repo_data_file) src_repo = next(repo for repo in data if repo["name"] == source_repo_name) if dest_repo.pulp_type != src_repo["pulp_type"]: raise ValidationError( _("Repository type mismatch: {src_repo} ({src_type}) vs {dest_repo} " "({dest_type}).").format( src_repo=src_repo["name"], src_type=src_repo["pulp_type"], dest_repo=dest_repo.name, dest_type=dest_repo.pulp_type, )) rv_name = "" # Extract the repo version files with tarfile.open(tar_path, "r:gz") as tar: for mem in tar.getmembers(): match = re.search( fr"(^repository-{source_repo_name}_[0-9]+)/.+", mem.name) if match: rv_name = match.group(1) tar.extract(mem, path=temp_dir) if not rv_name: raise ValidationError( _("No RepositoryVersion found for {}").format(rv_name)) rv_path = os.path.join(temp_dir, rv_name) # Content plugin_name = src_repo["pulp_type"].split(".")[0] cfg = get_plugin_config(plugin_name) resulting_content_ids = [] for res_class in cfg.exportable_classes: filename = f"{res_class.__module__}.{res_class.__name__}.json" a_result = _import_file(os.path.join(rv_path, filename), res_class, do_raise=False) # django import-export can have a problem with concurrent-imports that are # importing the same 'thing' (e.g., a Package that exists in two different # repo-versions that are being imported at the same time). We will try an import # that will simply record errors as they happen (rather than failing with an exception) # first. If errors happen, we'll do one retry before we give up on this repo-version's # import. if a_result.has_errors(): log.info( _("...{} import-errors encountered importing {} from {}, retrying" ).format(a_result.totals["error"], filename, rv_name)) # Second attempt, we allow to raise an exception on any problem. # This will either succeed, or log a fatal error and fail. try: a_result = _import_file(os.path.join(rv_path, filename), res_class) except Exception as e: # noqa log on ANY exception and then re-raise log.error( _("FATAL import-failure importing {} from {}").format( filename, rv_name)) raise resulting_content_ids.extend(row.object_id for row in a_result.rows if row.import_type in ("new", "update")) # Once all content exists, create the ContentArtifact links ca_path = os.path.join(rv_path, CA_FILE) _import_file(ca_path, ContentArtifactResource) # see if we have a content mapping mapping_path = f"{rv_name}/{CONTENT_MAPPING_FILE}" mapping = {} with tarfile.open(tar_path, "r:gz") as tar: if mapping_path in tar.getnames(): tar.extract(mapping_path, path=temp_dir) with open(os.path.join(temp_dir, mapping_path), "r") as mapping_file: mapping = json.load(mapping_file) if mapping: # use the content mapping to map content to repos for repo_name, content_ids in mapping.items(): repo = _destination_repo(importer, repo_name) content = Content.objects.filter(upstream_id__in=content_ids) with repo.new_version() as new_version: new_version.set_content(content) else: # just map all the content to our destination repo content = Content.objects.filter(pk__in=resulting_content_ids) with dest_repo.new_version() as new_version: new_version.set_content(content) content_count = content.count() pb.total = content_count pb.done = content_count pb.state = TASK_STATES.COMPLETED pb.save() gpr = TaskGroup.current().group_progress_reports.filter( code="import.repo.versions") gpr.update(done=F("done") + 1)
def import_repository_version(importer_pk, destination_repo_pk, source_repo_name, tar_path): """ Import a repository version from a Pulp export. Args: importer_pk (str): Importer we are working with destination_repo_pk (str): Primary key of Repository to import into. source_repo_name (str): Name of the Repository in the export. tar_path (str): A path to export tar. """ dest_repo = Repository.objects.get(pk=destination_repo_pk) importer = PulpImporter.objects.get(pk=importer_pk) pb = ProgressReport( message=f"Importing content for {dest_repo.name}", code="import.repo.version.content", state=TASK_STATES.RUNNING, ) pb.save() with tempfile.TemporaryDirectory() as temp_dir: # Extract the repo file for the repo info with tarfile.open(tar_path, "r:gz") as tar: tar.extract(REPO_FILE, path=temp_dir) with open(os.path.join(temp_dir, REPO_FILE), "r") as repo_data_file: data = json.load(repo_data_file) src_repo = next(repo for repo in data if repo["name"] == source_repo_name) rv_path = os.path.join(temp_dir, _repo_version_path(src_repo)) if dest_repo.pulp_type != src_repo["pulp_type"]: raise ValidationError( _( "Repository type mismatch: {src_repo} ({src_type}) vs {dest_repo} " "({dest_type})." ).format( src_repo=src_repo["name"], src_type=src_repo["pulp_type"], dest_repo=dest_repo.name, dest_type=dest_repo.pulp_type, ) ) # Extract the repo version files with tarfile.open(tar_path, "r:gz") as tar: for mem in tar.getmembers(): if re.match(fr"^{_repo_version_path(src_repo)}/.+", mem.name): tar.extract(mem, path=temp_dir) # Content plugin_name = src_repo["pulp_type"].split(".")[0] cfg = get_plugin_config(plugin_name) resulting_content_ids = [] for res_class in cfg.exportable_classes: filename = f"{res_class.__module__}.{res_class.__name__}.json" a_result = _import_file(os.path.join(rv_path, filename), res_class) resulting_content_ids.extend( row.object_id for row in a_result.rows if row.import_type in ("new", "update") ) # Once all content exists, create the ContentArtifact links ca_path = os.path.join(rv_path, CA_FILE) _import_file(ca_path, ContentArtifactResource) # see if we have a content mapping mapping_path = f"{_repo_version_path(src_repo)}/{CONTENT_MAPPING_FILE}" mapping = {} with tarfile.open(tar_path, "r:gz") as tar: if mapping_path in tar.getnames(): tar.extract(mapping_path, path=temp_dir) with open(os.path.join(temp_dir, mapping_path), "r") as mapping_file: mapping = json.load(mapping_file) if mapping: # use the content mapping to map content to repos for repo_name, content_ids in mapping.items(): repo = _destination_repo(importer, repo_name) content = Content.objects.filter(upstream_id__in=content_ids) with repo.new_version() as new_version: new_version.set_content(content) else: # just map all the content to our destination repo content = Content.objects.filter(pk__in=resulting_content_ids) with dest_repo.new_version() as new_version: new_version.set_content(content) content_count = content.count() pb.total = content_count pb.done = content_count pb.state = TASK_STATES.COMPLETED pb.save() gpr = TaskGroup.current().group_progress_reports.filter(code="import.repo.versions") gpr.update(done=F("done") + 1)
def pulp_import(importer_pk, path): """ Import a Pulp export into Pulp. Args: importer_pk (str): Primary key of PulpImporter to do the import path (str): Path to the export to be imported """ def import_file(fpath, resource_class): log.info(_("Importing file {}.").format(fpath)) with open(fpath, "r") as json_file: data = Dataset().load(json_file.read(), format="json") resource = resource_class() return resource.import_data(data, raise_errors=True) def destination_repo(source_repo_name): """Find the destination repository based on source repo's name.""" if importer.repo_mapping and importer.repo_mapping.get( source_repo_name): dest_repo_name = importer.repo_mapping[source_repo_name] else: dest_repo_name = source_repo_name return Repository.objects.get(name=dest_repo_name) def repo_version_path(temp_dir, src_repo): """Find the repo version path in the export based on src_repo json.""" src_repo_version = int(src_repo["next_version"]) - 1 return os.path.join( temp_dir, f"repository-{src_repo['pulp_id']}_{src_repo_version}") log.info(_("Importing {}.").format(path)) importer = PulpImporter.objects.get(pk=importer_pk) pulp_import = PulpImport.objects.create(importer=importer, task=Task.current(), params={"path": path}) CreatedResource.objects.create(content_object=pulp_import) with tempfile.TemporaryDirectory() as temp_dir: with tarfile.open(path, "r|gz") as tar: tar.extractall(path=temp_dir) # Artifacts ar_result = import_file(os.path.join(temp_dir, ARTIFACT_FILE), ArtifactResource) for row in ar_result.rows: artifact = Artifact.objects.get(pk=row.object_id) base_path = os.path.join('artifact', artifact.sha256[0:2], artifact.sha256[2:]) src = os.path.join(temp_dir, base_path) dest = os.path.join(settings.MEDIA_ROOT, base_path) if not default_storage.exists(dest): with open(src, 'rb') as f: default_storage.save(dest, f) # Repo Versions with open(os.path.join(temp_dir, REPO_FILE), "r") as repo_data_file: data = json.load(repo_data_file) for src_repo in data: try: dest_repo = destination_repo(src_repo["name"]) except Repository.DoesNotExist: log.warn( _("Could not find destination repo for {}. " "Skipping.").format(src_repo["name"])) continue rv_path = repo_version_path(temp_dir, src_repo) # Untyped Content content_path = os.path.join(rv_path, CONTENT_FILE) c_result = import_file(content_path, ContentResource) content = Content.objects.filter( pk__in=[r.object_id for r in c_result.rows]) # Content Artifacts ca_path = os.path.join(rv_path, CA_FILE) import_file(ca_path, ContentArtifactResource) # Content plugin_name = src_repo["pulp_type"].split('.')[0] cfg = get_plugin_config(plugin_name) for res_class in cfg.exportable_classes: filename = f"{res_class.__module__}.{res_class.__name__}.json" import_file(os.path.join(rv_path, filename), res_class) # Create the repo version with dest_repo.new_version() as new_version: new_version.set_content(content) return importer