def _url_and_destination(self, base_url, unit): """ Get the download URL and download destination. :param base_url: The base URL. :type base_url: str :param unit: A content unit. :type unit: dict :return: (url, destination) :rtype: tuple(2) """ storage_path = unit[constants.STORAGE_PATH] tar_path = unit.get(constants.TARBALL_PATH) if not tar_path: # The pulp/nodes/content endpoint provides all content. # This replaced the publishing of individual links for each unit. parsed = urlparse(base_url) relative_path = unit[constants.RELATIVE_PATH] path = pathlib.join(constants.CONTENT_PATH, pathlib.quote(relative_path)) base_url = ParseResult( scheme=parsed.scheme, netloc=parsed.netloc, path=path, params=parsed.params, query=parsed.query, fragment=parsed.fragment) return base_url.geturl(), storage_path else: return pathlib.url_join(base_url, pathlib.quote(tar_path)),\ pathlib.join(os.path.dirname(storage_path), os.path.basename(tar_path))
def publish(self, units): """ Publish the specified units. Writes the units.json file and symlinks each of the files associated to the unit.storage_path. Publishing is staged in a temporary directory and must use commit() to make the publishing permanent. :param units: A list of units to publish. :type units: iterable :return: The absolute path to the manifest. :rtype: str """ pathlib.mkdir(self.publish_dir) self.tmp_dir = mkdtemp(dir=self.publish_dir) units_path = pathlib.join(self.tmp_dir, UNITS_FILE_NAME) manifest_path = pathlib.join(self.tmp_dir, MANIFEST_FILE_NAME) with UnitWriter(units_path) as writer: for unit in units: self.publish_unit(unit) writer.add(unit) manifest_id = str(uuid4()) manifest = Manifest(manifest_id) manifest.set_units(writer) manifest_path = manifest.write(manifest_path) self.staged = True return manifest_path
def test_publisher(self): # setup units = self.populate() # test # publish repo_id = 'test_repo' base_url = 'file://' publish_dir = os.path.join(self.tmpdir, 'nodes/repos') virtual_host = (publish_dir, publish_dir) with HttpPublisher(base_url, virtual_host, repo_id) as p: p.publish(units) p.commit() # verify conf = DownloaderConfig() downloader = LocalFileDownloader(conf) manifest_path = p.manifest_path() working_dir = os.path.join(self.tmpdir, 'working_dir') os.makedirs(working_dir) url = pathlib.url_join(base_url, manifest_path) manifest = RemoteManifest(url, downloader, working_dir) manifest.fetch() manifest.fetch_units() self.assertTrue(manifest.has_valid_units()) units = manifest.get_units() n = 0 for unit, ref in units: self.assertEqual(manifest.publishing_details[constants.BASE_URL], pathlib.url_join(base_url, publish_dir, repo_id)) if n == 0: # TARBALL path = pathlib.join(publish_dir, repo_id, unit[constants.TARBALL_PATH]) self.assertTrue(os.path.isfile(path)) else: path = pathlib.join(publish_dir, repo_id, unit[constants.RELATIVE_PATH]) self.assertTrue(os.path.islink(path)) self.assertEqual(unit[constants.FILE_SIZE], os.path.getsize(path)) if n == 0: # TARBALL path = pathlib.join(publish_dir, repo_id, unit[constants.TARBALL_PATH]) tb = tarfile.open(path) try: files = sorted(tb.getnames()) finally: tb.close() self.assertEqual(len(files), self.NUM_TARED_FILES) else: path = pathlib.join(publish_dir, repo_id, unit[constants.RELATIVE_PATH]) with open(path, 'rb') as fp: unit_content = fp.read() self.assertEqual(unit_content, unit_content) self.assertEqual(unit['unit_key']['n'], n) n += 1
def fetch_units(self): """ Fetch the units file referenced in the manifest. :raise HTTPError: on URL errors. - :raise ValueError: on json decoding errors """ base_url = self.url.rsplit('/', 1)[0] url = pathlib.join(base_url, UNITS_FILE_NAME) destination = pathlib.join(os.path.dirname(self.path), UNITS_FILE_NAME) request = DownloadRequest(str(url), destination) self.downloader.download([request])
def test_publisher(self): # setup units = self.populate() # test # publish repo_id = 'test_repo' base_url = 'file://' publish_dir = os.path.join(self.tmpdir, 'nodes/repos') repo_publish_dir = os.path.join(publish_dir, repo_id) virtual_host = (publish_dir, publish_dir) with HttpPublisher(base_url, virtual_host, repo_id, repo_publish_dir) as p: p.publish(units) p.commit() # verify conf = DownloaderConfig() downloader = LocalFileDownloader(conf) manifest_path = p.manifest_path() working_dir = os.path.join(self.tmpdir, 'working_dir') os.makedirs(working_dir) url = pathlib.url_join(base_url, manifest_path) manifest = RemoteManifest(url, downloader, working_dir) manifest.fetch() manifest.fetch_units() self.assertTrue(manifest.has_valid_units()) units = manifest.get_units() n = 0 for unit, ref in units: self.assertEqual( manifest.publishing_details[constants.BASE_URL], pathlib.url_join(base_url, publish_dir, repo_id)) if n == 0: # TARBALL path = pathlib.join(publish_dir, repo_id, unit[constants.TARBALL_PATH]) self.assertTrue(os.path.isfile(path)) else: path = pathlib.join(publish_dir, repo_id, unit[constants.RELATIVE_PATH]) self.assertTrue(os.path.islink(path)) self.assertEqual(unit[constants.FILE_SIZE], os.path.getsize(path)) if n == 0: # TARBALL path = pathlib.join(publish_dir, repo_id, unit[constants.TARBALL_PATH]) tb = tarfile.open(path) try: files = sorted(tb.getnames()) finally: tb.close() self.assertEqual(len(files), self.NUM_TARED_FILES) else: path = pathlib.join(publish_dir, repo_id, unit[constants.RELATIVE_PATH]) with open(path, 'rb') as fp: unit_content = fp.read() self.assertEqual(unit_content, unit_content) self.assertEqual(unit['unit_key']['n'], n) n += 1
def manifest_path(self): """ Get the relative URL path to the manifest. :return: The path component of the URL. :rtype: str """ return pathlib.join(self.alias[0], self.repo_id, MANIFEST_FILE_NAME)
def units_path(self): """ Get the absolute path to the associated units file. The path is """ return self.units[UNITS_PATH] or pathlib.join( os.path.dirname(self.path), UNITS_FILE_NAME)
def fetch_units(self): """ Fetch the units file referenced in the manifest. :raise ManifestDownloadError: on downloading errors. :raise HTTPError: on URL errors. :raise ValueError: on json decoding errors """ base_url = self.url.rsplit('/', 1)[0] url = pathlib.join(base_url, UNITS_FILE_NAME) destination = pathlib.join(os.path.dirname(self.path), UNITS_FILE_NAME) request = DownloadRequest(str(url), destination) listener = AggregatingEventListener() self.downloader.event_listener = listener self.downloader.download([request]) if listener.failed_reports: report = listener.failed_reports[0] raise ManifestDownloadError(self.url, report.error_msg)
def commit(self): """ Commit publishing. Move the tmp_dir to the publish_dir. """ if not self.staged: # nothing to commit return dir_path = pathlib.join(self.publish_dir, self.repo_id) rmtree(dir_path, ignore_errors=True) os.rename(self.tmp_dir, dir_path) self.staged = False
def commit(self): """ Commit publishing. Move the tmp_dir to the publish_dir. """ if not self.staged: # nothing to commit return dir_path = pathlib.join(self.publish_dir, self.repo_id) os.system('rm -rf %s' % dir_path) os.rename(self.tmp_dir, dir_path) self.staged = False
def __init__(self, path): """ :param path: The absolute path to a file or directory. When a directory is specified, the standard file name is appended. :type path: str :raise IOError: on I/O errors """ if os.path.isdir(path): path = pathlib.join(path, UNITS_FILE_NAME) self.path = path self.fp = gzip.open(path, 'wb') self.total_units = 0 self.bytes_written = 0
def fetch(self, url, dir_path, downloader): """ Fetch the manifest file using the specified URL. :param url: The URL to the manifest. :type url: str :param dir_path: The absolute path to a directory for the downloaded manifest. :type dir_path: str :param downloader: The nectar downloader to be used. :type downloader: nectar.downloaders.base.Downloader :raise HTTPError: on URL errors. - :raise ValueError: on json decoding errors """ destination = pathlib.join(dir_path, MANIFEST_FILE_NAME) request = DownloadRequest(str(url), destination) request_list = [request] downloader.download(request_list) if compressed(destination): destination = decompress(destination) with open(destination) as fp: manifest = json.load(fp) self.__dict__.update(manifest) self.units_path = pathlib.join(dir_path, os.path.basename(self.units_path))
def get_units(self): """ Get the content units referenced in the manifest. :return: An iterator used to read downloaded content units. :rtype: iterable :raise IOError: on I/O errors. - :raise ValueError: json decoding errors """ if self.total_units: path = pathlib.join(os.path.dirname(self.path), UNITS_FILE_NAME) return UnitIterator(path, self.total_units) else: return []
def _update_storage_path(self, unit): """ Update the unit's storage_path using the storage_dir defined in server.conf and the relative_path injected when the unit was published. :param unit: A published unit. :type unit: dict """ storage_path = unit.get(constants.STORAGE_PATH) if not storage_path: return storage_dir = pulp_conf.get("server", "storage_dir") relative_path = unit[constants.RELATIVE_PATH] storage_path = pathlib.join(storage_dir, relative_path) unit[constants.STORAGE_PATH] = storage_path
def _update_storage_path(self, unit): """ Update the unit's storage_path using the storage_dir defined in server.conf and the relative_path injected when the unit was published. :param unit: A published unit. :type unit: dict """ storage_path = unit.get(constants.STORAGE_PATH) if not storage_path: return storage_dir = pulp_conf.get('server', 'storage_dir') relative_path = unit[constants.RELATIVE_PATH] storage_path = pathlib.join(storage_dir, relative_path) unit[constants.STORAGE_PATH] = storage_path
def has_valid_units(self): """ Validate the associated units file by comparing the size of the units file to units_size in the manifest. :return: True if valid. :rtype: bool """ try: path = pathlib.join(os.path.dirname(self.path), UNITS_FILE_NAME) size = os.path.getsize(path) return size == self.units_size except OSError, e: if e.errno != errno.ENOENT: raise
def __init__(self, path, manifest_id=None): """ :param path: The path to either a directory containing the standard named manifest or the absolute path to the manifest. :type path: str :param manifest_id: An optional manifest ID. :type manifest_id: str """ self.id = manifest_id self.version = MANIFEST_VERSION self.units = {UNITS_PATH: None, UNITS_TOTAL: 0, UNITS_SIZE: 0} self.publishing_details = {} if os.path.isdir(path): path = pathlib.join(path, MANIFEST_FILE_NAME) self.path = path
def _path_and_destination(self, unit): """ Get the path component of the download URL and download destination. :param unit: A content unit. :type unit: dict :return: (path, destination) :rtype: tuple(2) """ storage_path = unit[constants.STORAGE_PATH] tar_path = unit.get(constants.TARBALL_PATH) if not tar_path: relative_path = unit[constants.RELATIVE_PATH] return pathlib.quote(relative_path), storage_path else: return pathlib.quote(tar_path), pathlib.join(os.path.dirname(storage_path), os.path.basename(tar_path))
def __init__(self, url, downloader, destination): """ :param url: The URL to the remote manifest. :type url: str :param downloader: The downloader used for fetch methods. :type downloader: nectar.downloaders.base.Downloader :param destination: An absolute path to a file or directory. :type destination: str """ if os.path.isdir(destination): destination = pathlib.join(destination, MANIFEST_FILE_NAME) Manifest.__init__(self, destination) self.url = str(url) self.downloader = downloader self.destination = destination
def test_publisher(self): # setup units = self.populate() # test # publish repo_id = 'test_repo' base_url = 'file://' publish_dir = os.path.join(self.tmpdir, 'nodes/repos') virtual_host = (publish_dir, publish_dir) with HttpPublisher(base_url, virtual_host, repo_id) as p: p.publish(units) p.commit() # verify conf = DownloaderConfig() downloader = HTTPSCurlDownloader(conf) manifest_path = p.manifest_path() working_dir = os.path.join(self.tmpdir, 'working_dir') os.makedirs(working_dir) manifest = Manifest() url = pathlib.url_join(base_url, manifest_path) manifest.fetch(url, working_dir, downloader) manifest.fetch_units(url, downloader) units = manifest.get_units() n = 0 for unit, ref in units: if n == 0: self.assertTrue(unit[constants.PUBLISHED_AS_TARBALL]) else: self.assertFalse( unit.get(constants.PUBLISHED_AS_TARBALL, False)) path = pathlib.join(publish_dir, repo_id, unit[constants.RELATIVE_PATH]) self.assertEqual(manifest.publishing_details[constants.BASE_URL], pathlib.url_join(base_url, publish_dir, repo_id)) if n == 0: self.assertTrue(os.path.isfile(path)) else: self.assertTrue(os.path.islink(path)) if n == 0: with tarfile.open(path) as tb: files = sorted(tb.getnames()) self.assertEqual(len(files), self.NUM_TARED_FILES + 1) else: with open(path, 'rb') as fp: unit_content = fp.read() self.assertEqual(unit_content, unit_content) self.assertEqual(unit['unit_key']['n'], n) n += 1
def _path_and_destination(self, unit): """ Get the path component of the download URL and download destination. :param unit: A content unit. :type unit: dict :return: (path, destination) :rtype: tuple(2) """ storage_path = unit[constants.STORAGE_PATH] tar_path = unit.get(constants.TARBALL_PATH) if not tar_path: relative_path = unit[constants.RELATIVE_PATH] return pathlib.quote(relative_path), storage_path else: return pathlib.quote(tar_path),\ pathlib.join(os.path.dirname(storage_path), os.path.basename(tar_path))
def _reset_storage_path(self, unit): """ Reset the storage_path using the storage_dir defined in server.conf and the relative_path injected when the unit was published. :param unit: A published unit. :type unit: dict :return: The re-oriented storage path. :rtype: str """ storage_path = unit.get(constants.STORAGE_PATH) if not storage_path: return storage_dir = pulp_conf.get('server', 'storage_dir') relative_path = unit[constants.RELATIVE_PATH] storage_path = pathlib.join(storage_dir, relative_path) unit[constants.STORAGE_PATH] = storage_path
def test_publisher(self): # setup units = self.populate() # test # publish repo_id = 'test_repo' base_url = 'file://' publish_dir = os.path.join(self.tmpdir, 'nodes/repos') virtual_host = (publish_dir, publish_dir) with HttpPublisher(base_url, virtual_host, repo_id) as p: p.publish(units) p.commit() # verify conf = DownloaderConfig() downloader = HTTPSCurlDownloader(conf) manifest_path = p.manifest_path() working_dir = os.path.join(self.tmpdir, 'working_dir') os.makedirs(working_dir) manifest = Manifest() url = pathlib.url_join(base_url, manifest_path) manifest.fetch(url, working_dir, downloader) manifest.fetch_units(url, downloader) units = manifest.get_units() n = 0 for unit, ref in units: if n == 0: self.assertTrue(unit[constants.PUBLISHED_AS_TARBALL]) else: self.assertFalse(unit.get(constants.PUBLISHED_AS_TARBALL, False)) path = pathlib.join(publish_dir, repo_id, unit[constants.RELATIVE_PATH]) self.assertEqual( manifest.publishing_details[constants.BASE_URL], pathlib.url_join(base_url, publish_dir, repo_id)) if n == 0: self.assertTrue(os.path.isfile(path)) else: self.assertTrue(os.path.islink(path)) if n == 0: with tarfile.open(path) as tb: files = sorted(tb.getnames()) self.assertEqual(len(files), self.NUM_TARED_FILES + 1) else: with open(path, 'rb') as fp: unit_content = fp.read() self.assertEqual(unit_content, unit_content) self.assertEqual(unit['unit_key']['n'], n) n += 1
def publish_unit(self, unit): """ Publish the file associated with the unit into the publish directory. :param unit: A content unit. :type unit: dict """ storage_path = unit.get(constants.STORAGE_PATH) if not storage_path: # not all units have associated files. return unit, None relative_path = unit[constants.RELATIVE_PATH] published_path = pathlib.join(self.tmp_dir, relative_path) pathlib.mkdir(os.path.dirname(published_path)) unit[constants.FILE_SIZE] = os.path.getsize(storage_path) if os.path.isdir(storage_path): tar_dir(storage_path, tar_path(published_path)) unit[constants.TARBALL_PATH] = tar_path(relative_path) else: os.symlink(storage_path, published_path)
def publish_unit(self, unit): """ Publish the file associated with the unit into the publish directory. :param unit: A content unit. :type unit: dict """ storage_path = unit.get('storage_path') if not storage_path: # not all units have associated files. return unit, None relative_path = unit['relative_path'] published_path = pathlib.join(self.tmp_dir, relative_path) pathlib.mkdir(os.path.dirname(published_path)) if os.path.isdir(storage_path): self.tar_dir(storage_path, published_path) unit[constants.PUBLISHED_AS_TARBALL] = True else: os.symlink(storage_path, published_path) unit[constants.PUBLISHED_AS_FILE] = True
def units_path(self): """ Get the absolute path to the associated units file. The path is """ return self.units[UNITS_PATH] or pathlib.join(os.path.dirname(self.path), UNITS_FILE_NAME)