def _publish_distribution_files(self, distribution_unit): """ For a given AssociatedUnit for a distribution. Create all the links back to the content units that are referenced within the 'files' metadata section of the unit. :param distribution_unit: The unit for the distribution from which the list of files to be published should be pulled from. :type distribution_unit: pulp_rpm.plugins.db.models.Distribution """ if not distribution_unit.files: msg = "No distribution files found for unit %s" % distribution_unit logger.warning(msg) return distro_files = distribution_unit.files total_files = len(distro_files) logger.debug("Found %s distribution files to symlink" % total_files) source_path_dir = distribution_unit._storage_path symlink_dir = self.get_working_dir() for dfile in distro_files: if dfile['relativepath'].startswith('repodata/'): continue source_path = os.path.join(source_path_dir, dfile['relativepath']) symlink_path = os.path.join(symlink_dir, dfile['relativepath']) plugin_misc.create_symlink(source_path, symlink_path)
def process_main(self, item): """ Process the v2 manifests. :param item: The manifest to process :type item: pulp_docker.plugins.models.Tag """ if item.schema_version != 2: return name_to_tags = self.parent.manifest_to_imgname_to_tags.get(item.digest) if not name_to_tags: # We have no image name for this manifest. We cannot publish it return for img_name, tags in sorted(name_to_tags.items()): # This image can be fetched by digest too self.parent.imgname_to_mfid.setdefault(img_name, set()).add(item.digest) refs = [item.digest] refs.extend(x[0] for x in tags) for ref in refs: dest_name = os.path.join(self.get_working_dir(), img_name, "manifests", str(item.schema_version), ref) misc.create_symlink(item._storage_path, dest_name) layer_to_mf = self.parent.layer_to_manifests for layer in item.fs_layers: layer_to_mf.setdefault(layer.blob_sum, set()).add(item.digest) if item.config_layer: layer_to_mf.setdefault(item.config_layer, set()).add(item.digest)
def _publish_distribution_treeinfo(self, distribution_unit): """ For a given AssociatedUnit for a distribution. Create the links for the treeinfo file back to the treeinfo in the content. :param distribution_unit: The unit for the distribution from which the list of files to be published should be pulled from. :type distribution_unit: pulp_rpm.plugins.db.models.Distribution """ distribution_unit_storage_path = distribution_unit._storage_path src_treeinfo_path = None treeinfo_file_name = None for treeinfo in constants.TREE_INFO_LIST: test_treeinfo_path = os.path.join(distribution_unit_storage_path, treeinfo) if os.path.exists(test_treeinfo_path): # we found the treeinfo file src_treeinfo_path = test_treeinfo_path treeinfo_file_name = treeinfo break if src_treeinfo_path is not None: # create a symlink from content location to repo location. symlink_treeinfo_path = os.path.join(self.get_working_dir(), treeinfo_file_name) logger.debug("creating treeinfo symlink from %s to %s" % (src_treeinfo_path, symlink_treeinfo_path)) plugin_misc.create_symlink(src_treeinfo_path, symlink_treeinfo_path)
def process_main(self, item=None): """ Link the unit to the content directory and the package_dir :param unit: The unit to process :type unit: pulp.server.db.model.NonMetadataPackage """ unit = item source_path = unit._storage_path relative_path = unit.filename destination_path = os.path.join(self.get_working_dir(), relative_path) plugin_misc.create_symlink(source_path, destination_path) filename = unit.name + '-' + unit.version + '-' + unit.release + '.' + unit.arch + '.json' path = os.path.join(self.get_working_dir(), filename) metadata_dict = unit.create_legacy_metadata_dict() # The repodata is large, and can get re-generated during upload, so we leave it out here. metadata_dict.pop('repodata', None) dict_to_write = { 'unit_key': unit.unit_key, 'unit_metadata': metadata_dict } with open(path, 'w') as f: json.dump(dict_to_write, f)
def test_create_symlink(self): source_path = os.path.join(self.working_dir, "source") link_path = os.path.join(self.published_dir, "link") touch(source_path) self.assertFalse(os.path.exists(link_path)) misc.create_symlink(source_path, link_path) self.assertTrue(os.path.exists(link_path))
def test_create_symlink(self): source_path = os.path.join(self.working_dir, 'source') link_path = os.path.join(self.published_dir, 'link') touch(source_path) self.assertFalse(os.path.exists(link_path)) misc.create_symlink(source_path, link_path) self.assertTrue(os.path.exists(link_path))
def process_main(self, item): """ Link the item to the Blob file. :param item: The Blob to process :type item: pulp_docker.plugins.models.Blob """ misc.create_symlink(item._storage_path, os.path.join(self.get_blobs_directory(), item.unit_key['digest']))
def test_create_symlink_no_source(self): """Assert links are created, even if the source doesn't exist.""" source_path = os.path.join(self.working_dir, 'source') link_path = os.path.join(self.published_dir, 'link') self.assertFalse(os.path.exists(source_path)) self.assertFalse(os.path.exists(link_path)) misc.create_symlink(source_path, link_path) self.assertTrue(os.path.lexists(link_path)) self.assertFalse(os.path.exists(source_path))
def test_create_symlink_no_source(self): """Assert links are created, even if the source doesn't exist.""" source_path = os.path.join(self.working_dir, "source") link_path = os.path.join(self.published_dir, "link") self.assertFalse(os.path.exists(source_path)) self.assertFalse(os.path.exists(link_path)) misc.create_symlink(source_path, link_path) self.assertTrue(os.path.lexists(link_path)) self.assertFalse(os.path.exists(source_path))
def test_create_symlink_no_link_parent_with_permissions(self, mock_makedirs, mock_symlink): source_path = os.path.join(self.working_dir, 'source') link_path = os.path.join(self.published_dir, 'foo/bar/baz/link') touch(source_path) self.assertFalse(os.path.exists(os.path.dirname(link_path))) misc.create_symlink(source_path, link_path, directory_permissions=0700) mock_makedirs.assert_called_once_with(os.path.dirname(link_path), mode=0700) mock_symlink.assert_called_once_with(source_path, link_path)
def test_create_symlink_no_link_parent(self, mock_makedirs, mock_symlink): source_path = os.path.join(self.working_dir, 'source') link_path = os.path.join(self.published_dir, 'foo/bar/baz/link') touch(source_path) self.assertFalse(os.path.exists(os.path.dirname(link_path))) misc.create_symlink(source_path, link_path) mock_makedirs.assert_called_once_with(os.path.dirname(link_path), mode=0770) mock_symlink.assert_called_once_with(source_path, link_path)
def test_create_symlink_no_link_parent_with_permissions(self, mock_makedirs, mock_symlink): source_path = os.path.join(self.working_dir, "source") link_path = os.path.join(self.published_dir, "foo/bar/baz/link") touch(source_path) self.assertFalse(os.path.exists(os.path.dirname(link_path))) misc.create_symlink(source_path, link_path, directory_permissions=0700) mock_makedirs.assert_called_once_with(os.path.dirname(link_path), mode=0700) mock_symlink.assert_called_once_with(source_path, link_path)
def process_main(self, item): """ Link the item to the Manifest file. :param item: The Manifest to process :type item: pulp_docker.plugins.models.Manifest """ misc.create_symlink(item._storage_path, os.path.join(self.get_manifests_directory(), str(item.schema_version), item.unit_key['digest'])) self.parent.redirect_data[item.schema_version].add(item.unit_key['digest'])
def process_main(self, item): """ Create the manifest tag links. :param item: The tag to process :type item: pulp_docker.plugins.models.Tag """ manifest = models.Manifest.objects.get(digest=item.manifest_digest) misc.create_symlink( manifest._storage_path, os.path.join(self.parent.publish_manifests_step.get_manifests_directory(), item.name)) self._tag_names.add(item.name)
def test_create_symlink_link_exists_and_is_correct(self): new_source_path = os.path.join(self.working_dir, 'new_source') link_path = os.path.join(self.published_dir, 'link') touch(new_source_path) os.symlink(new_source_path, link_path) self.assertEqual(os.readlink(link_path), new_source_path) misc.create_symlink(new_source_path, link_path) self.assertEqual(os.readlink(link_path), new_source_path)
def process_main(self, item): """ Link the unit to the image content directory and the package_dir :param item: The Image to process :type item: pulp_docker.common.models.Image """ self.redirect_context.add_unit_metadata(item) target_base = os.path.join(self.get_web_directory(), item.unit_key['image_id']) files = ['ancestry', 'json', 'layer'] for file_name in files: misc.create_symlink(os.path.join(item.storage_path, file_name), os.path.join(target_base, file_name))
def test_create_symlink_link_exists_and_is_correct(self): new_source_path = os.path.join(self.working_dir, "new_source") link_path = os.path.join(self.published_dir, "link") touch(new_source_path) os.symlink(new_source_path, link_path) self.assertEqual(os.readlink(link_path), new_source_path) misc.create_symlink(new_source_path, link_path) self.assertEqual(os.readlink(link_path), new_source_path)
def process_main(self, item=None): """ Link the unit to the image content directory and the package_dir :param item: The unit to process :type item: pulp_docker.plugins.db.models.DockerImage """ self.redirect_context.add_unit_metadata(item) target_base = os.path.join(self.get_web_directory(), item.image_id) files = ['ancestry', 'json', 'layer'] for file_name in files: plugin_utils.create_symlink(os.path.join(item.storage_path, file_name), os.path.join(target_base, file_name))
def process_main(self, item): """ Create the manifest tag links. :param item: The tag to process :type item: pulp_docker.plugins.models.Tag """ manifest = models.Manifest.objects.get(digest=item.manifest_digest) misc.create_symlink( manifest._storage_path, os.path.join( self.parent.publish_manifests_step.get_manifests_directory(), item.name)) self._tag_names.add(item.name)
def process_main(self, item): """ Link the item to the Blob file. :param item: The Blob to process :type item: pulp_docker.plugins.models.Blob """ manifest_ids = self.parent.layer_to_manifests.get(item.digest) if manifest_ids is None: return for mf_id in manifest_ids: name_to_tags = self.parent.manifest_to_imgname_to_tags[mf_id] for img_name in name_to_tags: dest_name = os.path.join(self.get_working_dir(), img_name, "blobs", item.digest) misc.create_symlink(item._storage_path, dest_name)
def test_create_symlink_link_exists(self): old_source_path = os.path.join(self.working_dir, 'old_source') new_source_path = os.path.join(self.working_dir, 'new_source') link_path = os.path.join(self.published_dir, 'link') touch(old_source_path) touch(new_source_path) os.symlink(old_source_path, link_path) self.assertEqual(os.readlink(link_path), old_source_path) link_path_with_slash = link_path + '/' misc.create_symlink(new_source_path, link_path_with_slash) self.assertEqual(os.readlink(link_path), new_source_path)
def process_main(self, item=None): """ Link the unit to the content directory and the package_dir :param item: The item to process or none if this get_iterator is not defined :type item: pulp_rpm.plugins.db.models.RPM or pulp_rpm.plugins.db.models.SRPM """ unit = item source_path = unit._storage_path destination_path = os.path.join(self.get_working_dir(), unit.filename) plugin_misc.create_symlink(source_path, destination_path) for package_dir in self.dist_step.package_dirs: destination_path = os.path.join(package_dir, unit.filename) plugin_misc.create_symlink(source_path, destination_path) for context in (self.file_lists_context, self.other_context, self.primary_context): context.add_unit_metadata(unit)
def test_create_symlink_link_exists(self): old_source_path = os.path.join(self.working_dir, "old_source") new_source_path = os.path.join(self.working_dir, "new_source") link_path = os.path.join(self.published_dir, "link") touch(old_source_path) touch(new_source_path) os.symlink(old_source_path, link_path) self.assertEqual(os.readlink(link_path), old_source_path) link_path_with_slash = link_path + "/" misc.create_symlink(new_source_path, link_path_with_slash) self.assertEqual(os.readlink(link_path), new_source_path)
def _create_symlink(source_path, link_path): """ Create a symlink from the link path to the source path. If the link_path points to a directory that does not exist the directory will be created first. If we are overriding a current symlink with a new target - a debug message will be logged If a file already exists at the location specified by link_path an exception will be raised :param source_path: path of the source to link to :type source_path: str :param link_path: path of the link :type link_path: str """ misc.create_symlink(source_path, link_path)
def process_main(self, item=None): """ Link the unit to the drpm content directory and update the prestodelta metadata file. :param item: The unit to process :type item: pulp.server.db.model.ContentUnit """ unit = item source_path = unit._storage_path unit_filename = os.path.basename(unit.filename) relative_path = os.path.join('drpms', unit_filename) destination_path = os.path.join(self.get_working_dir(), relative_path) plugin_misc.create_symlink(source_path, destination_path) for package_dir in self.dist_step.package_dirs: destination_path = os.path.join(package_dir, relative_path) plugin_misc.create_symlink(source_path, destination_path) self.context.add_unit_metadata(unit)
def process_main(self, item): """ Create the manifest tag links. :param item: The tag to process :type item: pulp_docker.plugins.models.Tag """ try: manifest = models.Manifest.objects.get(digest=item.manifest_digest) schema_version = manifest.schema_version except mongoengine.DoesNotExist: manifest = models.ManifestList.objects.get(digest=item.manifest_digest) schema_version = constants.MANIFEST_LIST_TYPE misc.create_symlink( manifest._storage_path, os.path.join(self.parent.publish_manifests_step.get_manifests_directory(), str(schema_version), item.name)) self._tag_names.add(item.name) self.parent.redirect_data[schema_version].add(item.name)
def process_main(self, item=None): """ Copy the metadata file into place and add it tot he repomd file. :param item: The unit to process :type item: pulp.server.db.model.ContentUnit """ unit = item # Copy the file to the location on disk where the published repo is built publish_location_relative_path = os.path.join(self.get_working_dir(), REPO_DATA_DIR_NAME) metadata_file_name = os.path.basename(unit._storage_path) link_path = os.path.join(publish_location_relative_path, metadata_file_name) plugin_misc.create_symlink(unit._storage_path, link_path) # Add the proper relative reference to the metadata file to repomd self.parent.repomd_file_context.\ add_metadata_file_metadata(unit.data_type, link_path)
def _publish_distribution_files(self, distribution_unit): """ For a given AssociatedUnit for a distribution. Create all the links back to the content units that are referenced within the 'files' metadata section of the unit. :param distribution_unit: The unit for the distribution from which the list of files to be published should be pulled from. :type distribution_unit: pulp_rpm.plugins.db.models.Distribution """ if not distribution_unit.files: msg = "No distribution files found for unit %s" % distribution_unit logger.warning(msg) return # The files from `treeinfo` and `PULP_DISTRIBUTION.xml` are mashed into the # files list on the unit. This is a hacky work-around to unmash them, filter # out anything that is in the main repodata directory (which is potentially not # safe, but was happening before I got here and we don't have time to fix this # properly right now), and generate a new `PULP_DISTRIBUTION.xml` that doesn't # reference files that don't exist in this publish. pulp_distribution_file = False distro_files = [f['relativepath'] for f in distribution_unit.files] if constants.DISTRIBUTION_XML in distro_files: distro_files.remove(constants.DISTRIBUTION_XML) pulp_distribution_file = True distro_files = filter(lambda f: not f.startswith('repodata/'), distro_files) total_files = len(distro_files) logger.debug("Found %s distribution files to symlink" % total_files) source_path_dir = distribution_unit._storage_path symlink_dir = self.get_working_dir() for dfile in distro_files: source_path = os.path.join(source_path_dir, dfile) symlink_path = os.path.join(symlink_dir, dfile) plugin_misc.create_symlink(source_path, symlink_path) # Not all repositories have this file so this is only done if the upstream repo # had the file. if pulp_distribution_file: xml_file_path = os.path.join(source_path_dir, constants.DISTRIBUTION_XML) self._write_pulp_distribution_file(distro_files, xml_file_path)
def process_main(self, item): """ Link the item to the Manifest List file. :param item: The Manifest List to process :type item: pulp_docker.plugins.models.ManifestList """ misc.create_symlink(item._storage_path, os.path.join(self.get_manifests_directory(), constants.MANIFEST_LIST_TYPE, item.unit_key['digest'])) redirect_data = self.parent.redirect_data redirect_data[constants.MANIFEST_LIST_TYPE].add(item.unit_key['digest']) if item.amd64_digest: # we query the tag collection because the manifest list model does not contain # the tag field anymore # Manifest list can have several tags tags = models.Tag.objects.filter(manifest_digest=item.digest, repo_id=self.get_repo().id) for tag in tags: redirect_data['amd64'][tag.name] = (item.amd64_digest, item.amd64_schema_version)
def process_main(self, item=None): """ Link the unit to the content directory and the package_dir :param unit: The unit to process :type unit: pulp.server.db.model.NonMetadataPackage """ unit = item source_path = unit._storage_path relative_path = unit.filename destination_path = os.path.join(self.get_working_dir(), relative_path) plugin_misc.create_symlink(source_path, destination_path) filename = unit.name + '-' + unit.version + '-' + unit.release + '.' + unit.arch + '.json' path = os.path.join(self.get_working_dir(), filename) metadata_dict = unit.create_legacy_metadata_dict() # The repodata is large, and can get re-generated during upload, so we leave it out here. metadata_dict.pop('repodata', None) dict_to_write = {'unit_key': unit.unit_key, 'unit_metadata': metadata_dict} with open(path, 'w') as f: json.dump(dict_to_write, f)
def process_main(self, item=None): unit = item self.units.append(unit) dest_path = os.path.join(self.get_working_dir(), unit.filename) misc.create_symlink(unit.storage_path, dest_path)