def _url_and_destination(self, base_url, unit):
"""
Get the download URL and download destination.
:param base_url: The base URL.
:type base_url: str
:param unit: A content unit.
:type unit: dict
:return: (url, destination)
:rtype: tuple(2)
"""
storage_path = unit[constants.STORAGE_PATH]
tar_path = unit.get(constants.TARBALL_PATH)
if not tar_path:
# The pulp/nodes/content endpoint provides all content.
# This replaced the publishing of individual links for each unit.
parsed = urlparse(base_url)
relative_path = unit[constants.RELATIVE_PATH]
path = pathlib.join(constants.CONTENT_PATH, pathlib.quote(relative_path))
base_url = ParseResult(
scheme=parsed.scheme,
netloc=parsed.netloc,
path=path,
params=parsed.params,
query=parsed.query,
fragment=parsed.fragment)
return base_url.geturl(), storage_path
else:
return pathlib.url_join(base_url, pathlib.quote(tar_path)),\
pathlib.join(os.path.dirname(storage_path), os.path.basename(tar_path))
def publish(self, units):
"""
Publish the specified units.
Writes the units.json file and symlinks each of the files associated
to the unit.storage_path. Publishing is staged in a temporary directory and
must use commit() to make the publishing permanent.
:param units: A list of units to publish.
:type units: iterable
:return: The absolute path to the manifest.
:rtype: str
"""
pathlib.mkdir(self.publish_dir)
self.tmp_dir = mkdtemp(dir=self.publish_dir)
units_path = pathlib.join(self.tmp_dir, UNITS_FILE_NAME)
manifest_path = pathlib.join(self.tmp_dir, MANIFEST_FILE_NAME)
with UnitWriter(units_path) as writer:
for unit in units:
self.publish_unit(unit)
writer.add(unit)
manifest_id = str(uuid4())
manifest = Manifest(manifest_id)
manifest.set_units(writer)
manifest_path = manifest.write(manifest_path)
self.staged = True
return manifest_path
def publish(self, units):
"""
Publish the specified units.
Writes the units.json file and symlinks each of the files associated
to the unit.storage_path. Publishing is staged in a temporary directory and
must use commit() to make the publishing permanent.
:param units: A list of units to publish.
:type units: iterable
:return: The absolute path to the manifest.
:rtype: str
"""
pathlib.mkdir(self.publish_dir)
self.tmp_dir = mkdtemp(dir=self.publish_dir)
units_path = pathlib.join(self.tmp_dir, UNITS_FILE_NAME)
manifest_path = pathlib.join(self.tmp_dir, MANIFEST_FILE_NAME)
with UnitWriter(units_path) as writer:
for unit in units:
self.publish_unit(unit)
writer.add(unit)
manifest_id = str(uuid4())
manifest = Manifest(manifest_id)
manifest.set_units(writer)
manifest_path = manifest.write(manifest_path)
self.staged = True
return manifest_path
def test_publisher(self):
# setup
units = self.populate()
# test
# publish
repo_id = 'test_repo'
base_url = 'file://'
publish_dir = os.path.join(self.tmpdir, 'nodes/repos')
virtual_host = (publish_dir, publish_dir)
with HttpPublisher(base_url, virtual_host, repo_id) as p:
p.publish(units)
p.commit()
# verify
conf = DownloaderConfig()
downloader = LocalFileDownloader(conf)
manifest_path = p.manifest_path()
working_dir = os.path.join(self.tmpdir, 'working_dir')
os.makedirs(working_dir)
url = pathlib.url_join(base_url, manifest_path)
manifest = RemoteManifest(url, downloader, working_dir)
manifest.fetch()
manifest.fetch_units()
self.assertTrue(manifest.has_valid_units())
units = manifest.get_units()
n = 0
for unit, ref in units:
self.assertEqual(manifest.publishing_details[constants.BASE_URL],
pathlib.url_join(base_url, publish_dir, repo_id))
if n == 0: # TARBALL
path = pathlib.join(publish_dir, repo_id,
unit[constants.TARBALL_PATH])
self.assertTrue(os.path.isfile(path))
else:
path = pathlib.join(publish_dir, repo_id,
unit[constants.RELATIVE_PATH])
self.assertTrue(os.path.islink(path))
self.assertEqual(unit[constants.FILE_SIZE],
os.path.getsize(path))
if n == 0: # TARBALL
path = pathlib.join(publish_dir, repo_id,
unit[constants.TARBALL_PATH])
tb = tarfile.open(path)
try:
files = sorted(tb.getnames())
finally:
tb.close()
self.assertEqual(len(files), self.NUM_TARED_FILES)
else:
path = pathlib.join(publish_dir, repo_id,
unit[constants.RELATIVE_PATH])
with open(path, 'rb') as fp:
unit_content = fp.read()
self.assertEqual(unit_content, unit_content)
self.assertEqual(unit['unit_key']['n'], n)
n += 1
def fetch_units(self):
"""
Fetch the units file referenced in the manifest.
:raise HTTPError: on URL errors.
- :raise ValueError: on json decoding errors
"""
base_url = self.url.rsplit('/', 1)[0]
url = pathlib.join(base_url, UNITS_FILE_NAME)
destination = pathlib.join(os.path.dirname(self.path), UNITS_FILE_NAME)
request = DownloadRequest(str(url), destination)
self.downloader.download([request])
def test_publisher(self):
# setup
units = self.populate()
# test
# publish
repo_id = 'test_repo'
base_url = 'file://'
publish_dir = os.path.join(self.tmpdir, 'nodes/repos')
repo_publish_dir = os.path.join(publish_dir, repo_id)
virtual_host = (publish_dir, publish_dir)
with HttpPublisher(base_url, virtual_host, repo_id, repo_publish_dir) as p:
p.publish(units)
p.commit()
# verify
conf = DownloaderConfig()
downloader = LocalFileDownloader(conf)
manifest_path = p.manifest_path()
working_dir = os.path.join(self.tmpdir, 'working_dir')
os.makedirs(working_dir)
url = pathlib.url_join(base_url, manifest_path)
manifest = RemoteManifest(url, downloader, working_dir)
manifest.fetch()
manifest.fetch_units()
self.assertTrue(manifest.has_valid_units())
units = manifest.get_units()
n = 0
for unit, ref in units:
self.assertEqual(
manifest.publishing_details[constants.BASE_URL],
pathlib.url_join(base_url, publish_dir, repo_id))
if n == 0: # TARBALL
path = pathlib.join(publish_dir, repo_id, unit[constants.TARBALL_PATH])
self.assertTrue(os.path.isfile(path))
else:
path = pathlib.join(publish_dir, repo_id, unit[constants.RELATIVE_PATH])
self.assertTrue(os.path.islink(path))
self.assertEqual(unit[constants.FILE_SIZE], os.path.getsize(path))
if n == 0: # TARBALL
path = pathlib.join(publish_dir, repo_id, unit[constants.TARBALL_PATH])
tb = tarfile.open(path)
try:
files = sorted(tb.getnames())
finally:
tb.close()
self.assertEqual(len(files), self.NUM_TARED_FILES)
else:
path = pathlib.join(publish_dir, repo_id, unit[constants.RELATIVE_PATH])
with open(path, 'rb') as fp:
unit_content = fp.read()
self.assertEqual(unit_content, unit_content)
self.assertEqual(unit['unit_key']['n'], n)
n += 1
def manifest_path(self):
"""
Get the relative URL path to the manifest.
:return: The path component of the URL.
:rtype: str
"""
return pathlib.join(self.alias[0], self.repo_id, MANIFEST_FILE_NAME)
def units_path(self):
"""
Get the absolute path to the associated units file.
The path is
"""
return self.units[UNITS_PATH] or pathlib.join(
os.path.dirname(self.path), UNITS_FILE_NAME)
def manifest_path(self):
"""
Get the relative URL path to the manifest.
:return: The path component of the URL.
:rtype: str
"""
return pathlib.join(self.alias[0], self.repo_id, MANIFEST_FILE_NAME)
def fetch_units(self):
"""
Fetch the units file referenced in the manifest.
:raise ManifestDownloadError: on downloading errors.
:raise HTTPError: on URL errors.
:raise ValueError: on json decoding errors
"""
base_url = self.url.rsplit('/', 1)[0]
url = pathlib.join(base_url, UNITS_FILE_NAME)
destination = pathlib.join(os.path.dirname(self.path), UNITS_FILE_NAME)
request = DownloadRequest(str(url), destination)
listener = AggregatingEventListener()
self.downloader.event_listener = listener
self.downloader.download([request])
if listener.failed_reports:
report = listener.failed_reports[0]
raise ManifestDownloadError(self.url, report.error_msg)
def fetch_units(self):
"""
Fetch the units file referenced in the manifest.
:raise ManifestDownloadError: on downloading errors.
:raise HTTPError: on URL errors.
:raise ValueError: on json decoding errors
"""
base_url = self.url.rsplit('/', 1)[0]
url = pathlib.join(base_url, UNITS_FILE_NAME)
destination = pathlib.join(os.path.dirname(self.path), UNITS_FILE_NAME)
request = DownloadRequest(str(url), destination)
listener = AggregatingEventListener()
self.downloader.event_listener = listener
self.downloader.download([request])
if listener.failed_reports:
report = listener.failed_reports[0]
raise ManifestDownloadError(self.url, report.error_msg)
def commit(self):
"""
Commit publishing.
Move the tmp_dir to the publish_dir.
"""
if not self.staged:
# nothing to commit
return
dir_path = pathlib.join(self.publish_dir, self.repo_id)
rmtree(dir_path, ignore_errors=True)
os.rename(self.tmp_dir, dir_path)
self.staged = False
def commit(self):
"""
Commit publishing.
Move the tmp_dir to the publish_dir.
"""
if not self.staged:
# nothing to commit
return
dir_path = pathlib.join(self.publish_dir, self.repo_id)
rmtree(dir_path, ignore_errors=True)
os.rename(self.tmp_dir, dir_path)
self.staged = False
def commit(self):
"""
Commit publishing.
Move the tmp_dir to the publish_dir.
"""
if not self.staged:
# nothing to commit
return
dir_path = pathlib.join(self.publish_dir, self.repo_id)
os.system('rm -rf %s' % dir_path)
os.rename(self.tmp_dir, dir_path)
self.staged = False
def commit(self):
"""
Commit publishing.
Move the tmp_dir to the publish_dir.
"""
if not self.staged:
# nothing to commit
return
dir_path = pathlib.join(self.publish_dir, self.repo_id)
os.system('rm -rf %s' % dir_path)
os.rename(self.tmp_dir, dir_path)
self.staged = False
def __init__(self, path):
"""
:param path: The absolute path to a file or directory.
When a directory is specified, the standard file name is appended.
:type path: str
:raise IOError: on I/O errors
"""
if os.path.isdir(path):
path = pathlib.join(path, UNITS_FILE_NAME)
self.path = path
self.fp = gzip.open(path, 'wb')
self.total_units = 0
self.bytes_written = 0
def fetch(self, url, dir_path, downloader):
"""
Fetch the manifest file using the specified URL.
:param url: The URL to the manifest.
:type url: str
:param dir_path: The absolute path to a directory for the downloaded manifest.
:type dir_path: str
:param downloader: The nectar downloader to be used.
:type downloader: nectar.downloaders.base.Downloader
:raise HTTPError: on URL errors.
- :raise ValueError: on json decoding errors
"""
destination = pathlib.join(dir_path, MANIFEST_FILE_NAME)
request = DownloadRequest(str(url), destination)
request_list = [request]
downloader.download(request_list)
if compressed(destination):
destination = decompress(destination)
with open(destination) as fp:
manifest = json.load(fp)
self.__dict__.update(manifest)
self.units_path = pathlib.join(dir_path, os.path.basename(self.units_path))
def __init__(self, path):
"""
:param path: The absolute path to a file or directory.
When a directory is specified, the standard file name is appended.
:type path: str
:raise IOError: on I/O errors
"""
if os.path.isdir(path):
path = pathlib.join(path, UNITS_FILE_NAME)
self.path = path
self.fp = gzip.open(path, 'wb')
self.total_units = 0
self.bytes_written = 0
def get_units(self):
"""
Get the content units referenced in the manifest.
:return: An iterator used to read downloaded content units.
:rtype: iterable
:raise IOError: on I/O errors.
- :raise ValueError: json decoding errors
"""
if self.total_units:
path = pathlib.join(os.path.dirname(self.path), UNITS_FILE_NAME)
return UnitIterator(path, self.total_units)
else:
return []
def _update_storage_path(self, unit):
"""
Update the unit's storage_path using the storage_dir defined in
server.conf and the relative_path injected when the unit was published.
:param unit: A published unit.
:type unit: dict
"""
storage_path = unit.get(constants.STORAGE_PATH)
if not storage_path:
return
storage_dir = pulp_conf.get("server", "storage_dir")
relative_path = unit[constants.RELATIVE_PATH]
storage_path = pathlib.join(storage_dir, relative_path)
unit[constants.STORAGE_PATH] = storage_path
def _update_storage_path(self, unit):
"""
Update the unit's storage_path using the storage_dir defined in
server.conf and the relative_path injected when the unit was published.
:param unit: A published unit.
:type unit: dict
"""
storage_path = unit.get(constants.STORAGE_PATH)
if not storage_path:
return
storage_dir = pulp_conf.get('server', 'storage_dir')
relative_path = unit[constants.RELATIVE_PATH]
storage_path = pathlib.join(storage_dir, relative_path)
unit[constants.STORAGE_PATH] = storage_path
def has_valid_units(self):
"""
Validate the associated units file by comparing the size of the
units file to units_size in the manifest.
:return: True if valid.
:rtype: bool
"""
try:
path = pathlib.join(os.path.dirname(self.path), UNITS_FILE_NAME)
size = os.path.getsize(path)
return size == self.units_size
except OSError, e:
if e.errno != errno.ENOENT:
raise
def fetch(self, url, dir_path, downloader):
"""
Fetch the manifest file using the specified URL.
:param url: The URL to the manifest.
:type url: str
:param dir_path: The absolute path to a directory for the downloaded manifest.
:type dir_path: str
:param downloader: The nectar downloader to be used.
:type downloader: nectar.downloaders.base.Downloader
:raise HTTPError: on URL errors.
- :raise ValueError: on json decoding errors
"""
destination = pathlib.join(dir_path, MANIFEST_FILE_NAME)
request = DownloadRequest(str(url), destination)
request_list = [request]
downloader.download(request_list)
if compressed(destination):
destination = decompress(destination)
with open(destination) as fp:
manifest = json.load(fp)
self.__dict__.update(manifest)
self.units_path = pathlib.join(dir_path,
os.path.basename(self.units_path))
def __init__(self, path, manifest_id=None):
"""
:param path: The path to either a directory containing the standard
named manifest or the absolute path to the manifest.
:type path: str
:param manifest_id: An optional manifest ID.
:type manifest_id: str
"""
self.id = manifest_id
self.version = MANIFEST_VERSION
self.units = {UNITS_PATH: None, UNITS_TOTAL: 0, UNITS_SIZE: 0}
self.publishing_details = {}
if os.path.isdir(path):
path = pathlib.join(path, MANIFEST_FILE_NAME)
self.path = path
def _path_and_destination(self, unit):
"""
Get the path component of the download URL and download destination.
:param unit: A content unit.
:type unit: dict
:return: (path, destination)
:rtype: tuple(2)
"""
storage_path = unit[constants.STORAGE_PATH]
tar_path = unit.get(constants.TARBALL_PATH)
if not tar_path:
relative_path = unit[constants.RELATIVE_PATH]
return pathlib.quote(relative_path), storage_path
else:
return pathlib.quote(tar_path), pathlib.join(os.path.dirname(storage_path), os.path.basename(tar_path))
def __init__(self, url, downloader, destination):
"""
:param url: The URL to the remote manifest.
:type url: str
:param downloader: The downloader used for fetch methods.
:type downloader: nectar.downloaders.base.Downloader
:param destination: An absolute path to a file or directory.
:type destination: str
"""
if os.path.isdir(destination):
destination = pathlib.join(destination, MANIFEST_FILE_NAME)
Manifest.__init__(self, destination)
self.url = str(url)
self.downloader = downloader
self.destination = destination
def __init__(self, path, manifest_id=None):
"""
:param path: The path to either a directory containing the standard
named manifest or the absolute path to the manifest.
:type path: str
:param manifest_id: An optional manifest ID.
:type manifest_id: str
"""
self.id = manifest_id
self.version = MANIFEST_VERSION
self.units = {UNITS_PATH: None, UNITS_TOTAL: 0, UNITS_SIZE: 0}
self.publishing_details = {}
if os.path.isdir(path):
path = pathlib.join(path, MANIFEST_FILE_NAME)
self.path = path
def test_publisher(self):
# setup
units = self.populate()
# test
# publish
repo_id = 'test_repo'
base_url = 'file://'
publish_dir = os.path.join(self.tmpdir, 'nodes/repos')
virtual_host = (publish_dir, publish_dir)
with HttpPublisher(base_url, virtual_host, repo_id) as p:
p.publish(units)
p.commit()
# verify
conf = DownloaderConfig()
downloader = HTTPSCurlDownloader(conf)
manifest_path = p.manifest_path()
working_dir = os.path.join(self.tmpdir, 'working_dir')
os.makedirs(working_dir)
manifest = Manifest()
url = pathlib.url_join(base_url, manifest_path)
manifest.fetch(url, working_dir, downloader)
manifest.fetch_units(url, downloader)
units = manifest.get_units()
n = 0
for unit, ref in units:
if n == 0:
self.assertTrue(unit[constants.PUBLISHED_AS_TARBALL])
else:
self.assertFalse(
unit.get(constants.PUBLISHED_AS_TARBALL, False))
path = pathlib.join(publish_dir, repo_id,
unit[constants.RELATIVE_PATH])
self.assertEqual(manifest.publishing_details[constants.BASE_URL],
pathlib.url_join(base_url, publish_dir, repo_id))
if n == 0:
self.assertTrue(os.path.isfile(path))
else:
self.assertTrue(os.path.islink(path))
if n == 0:
with tarfile.open(path) as tb:
files = sorted(tb.getnames())
self.assertEqual(len(files), self.NUM_TARED_FILES + 1)
else:
with open(path, 'rb') as fp:
unit_content = fp.read()
self.assertEqual(unit_content, unit_content)
self.assertEqual(unit['unit_key']['n'], n)
n += 1
def _path_and_destination(self, unit):
"""
Get the path component of the download URL and download destination.
:param unit: A content unit.
:type unit: dict
:return: (path, destination)
:rtype: tuple(2)
"""
storage_path = unit[constants.STORAGE_PATH]
tar_path = unit.get(constants.TARBALL_PATH)
if not tar_path:
relative_path = unit[constants.RELATIVE_PATH]
return pathlib.quote(relative_path), storage_path
else:
return pathlib.quote(tar_path),\
pathlib.join(os.path.dirname(storage_path), os.path.basename(tar_path))
def _reset_storage_path(self, unit):
"""
Reset the storage_path using the storage_dir defined in
server.conf and the relative_path injected when the unit was published.
:param unit: A published unit.
:type unit: dict
:return: The re-oriented storage path.
:rtype: str
"""
storage_path = unit.get(constants.STORAGE_PATH)
if not storage_path:
return
storage_dir = pulp_conf.get('server', 'storage_dir')
relative_path = unit[constants.RELATIVE_PATH]
storage_path = pathlib.join(storage_dir, relative_path)
unit[constants.STORAGE_PATH] = storage_path
def _reset_storage_path(self, unit):
"""
Reset the storage_path using the storage_dir defined in
server.conf and the relative_path injected when the unit was published.
:param unit: A published unit.
:type unit: dict
:return: The re-oriented storage path.
:rtype: str
"""
storage_path = unit.get(constants.STORAGE_PATH)
if not storage_path:
return
storage_dir = pulp_conf.get('server', 'storage_dir')
relative_path = unit[constants.RELATIVE_PATH]
storage_path = pathlib.join(storage_dir, relative_path)
unit[constants.STORAGE_PATH] = storage_path
def test_publisher(self):
# setup
units = self.populate()
# test
# publish
repo_id = 'test_repo'
base_url = 'file://'
publish_dir = os.path.join(self.tmpdir, 'nodes/repos')
virtual_host = (publish_dir, publish_dir)
with HttpPublisher(base_url, virtual_host, repo_id) as p:
p.publish(units)
p.commit()
# verify
conf = DownloaderConfig()
downloader = HTTPSCurlDownloader(conf)
manifest_path = p.manifest_path()
working_dir = os.path.join(self.tmpdir, 'working_dir')
os.makedirs(working_dir)
manifest = Manifest()
url = pathlib.url_join(base_url, manifest_path)
manifest.fetch(url, working_dir, downloader)
manifest.fetch_units(url, downloader)
units = manifest.get_units()
n = 0
for unit, ref in units:
if n == 0:
self.assertTrue(unit[constants.PUBLISHED_AS_TARBALL])
else:
self.assertFalse(unit.get(constants.PUBLISHED_AS_TARBALL, False))
path = pathlib.join(publish_dir, repo_id, unit[constants.RELATIVE_PATH])
self.assertEqual(
manifest.publishing_details[constants.BASE_URL],
pathlib.url_join(base_url, publish_dir, repo_id))
if n == 0:
self.assertTrue(os.path.isfile(path))
else:
self.assertTrue(os.path.islink(path))
if n == 0:
with tarfile.open(path) as tb:
files = sorted(tb.getnames())
self.assertEqual(len(files), self.NUM_TARED_FILES + 1)
else:
with open(path, 'rb') as fp:
unit_content = fp.read()
self.assertEqual(unit_content, unit_content)
self.assertEqual(unit['unit_key']['n'], n)
n += 1
def publish_unit(self, unit):
"""
Publish the file associated with the unit into the publish directory.
:param unit: A content unit.
:type unit: dict
"""
storage_path = unit.get(constants.STORAGE_PATH)
if not storage_path:
# not all units have associated files.
return unit, None
relative_path = unit[constants.RELATIVE_PATH]
published_path = pathlib.join(self.tmp_dir, relative_path)
pathlib.mkdir(os.path.dirname(published_path))
unit[constants.FILE_SIZE] = os.path.getsize(storage_path)
if os.path.isdir(storage_path):
tar_dir(storage_path, tar_path(published_path))
unit[constants.TARBALL_PATH] = tar_path(relative_path)
else:
os.symlink(storage_path, published_path)
def publish_unit(self, unit):
"""
Publish the file associated with the unit into the publish directory.
:param unit: A content unit.
:type unit: dict
"""
storage_path = unit.get('storage_path')
if not storage_path:
# not all units have associated files.
return unit, None
relative_path = unit['relative_path']
published_path = pathlib.join(self.tmp_dir, relative_path)
pathlib.mkdir(os.path.dirname(published_path))
if os.path.isdir(storage_path):
self.tar_dir(storage_path, published_path)
unit[constants.PUBLISHED_AS_TARBALL] = True
else:
os.symlink(storage_path, published_path)
unit[constants.PUBLISHED_AS_FILE] = True
def publish_unit(self, unit):
"""
Publish the file associated with the unit into the publish directory.
:param unit: A content unit.
:type unit: dict
"""
storage_path = unit.get(constants.STORAGE_PATH)
if not storage_path:
# not all units have associated files.
return unit, None
relative_path = unit[constants.RELATIVE_PATH]
published_path = pathlib.join(self.tmp_dir, relative_path)
pathlib.mkdir(os.path.dirname(published_path))
unit[constants.FILE_SIZE] = os.path.getsize(storage_path)
if os.path.isdir(storage_path):
tar_dir(storage_path, tar_path(published_path))
unit[constants.TARBALL_PATH] = tar_path(relative_path)
else:
os.symlink(storage_path, published_path)
def publish_unit(self, unit):
"""
Publish the file associated with the unit into the publish directory.
:param unit: A content unit.
:type unit: dict
"""
storage_path = unit.get('storage_path')
if not storage_path:
# not all units have associated files.
return unit, None
relative_path = unit['relative_path']
published_path = pathlib.join(self.tmp_dir, relative_path)
pathlib.mkdir(os.path.dirname(published_path))
if os.path.isdir(storage_path):
self.tar_dir(storage_path, published_path)
unit[constants.PUBLISHED_AS_TARBALL] = True
else:
os.symlink(storage_path, published_path)
unit[constants.PUBLISHED_AS_FILE] = True
def units_path(self):
"""
Get the absolute path to the associated units file.
The path is
"""
return self.units[UNITS_PATH] or pathlib.join(os.path.dirname(self.path), UNITS_FILE_NAME)
