def _url_and_destination(self, base_url, unit):
"""
Get the download URL and download destination.
:param base_url: The base URL.
:type base_url: str
:param unit: A content unit.
:type unit: dict
:return: (url, destination)
:rtype: tuple(2)
"""
storage_path = unit[constants.STORAGE_PATH]
tar_path = unit.get(constants.TARBALL_PATH)
if not tar_path:
# The pulp/nodes/content endpoint provides all content.
# This replaced the publishing of individual links for each unit.
parsed = urlparse(base_url)
relative_path = unit[constants.RELATIVE_PATH]
path = pathlib.join(constants.CONTENT_PATH, pathlib.quote(relative_path))
base_url = ParseResult(
scheme=parsed.scheme,
netloc=parsed.netloc,
path=path,
params=parsed.params,
query=parsed.query,
fragment=parsed.fragment)
return base_url.geturl(), storage_path
else:
return pathlib.url_join(base_url, pathlib.quote(tar_path)),\
pathlib.join(os.path.dirname(storage_path), os.path.basename(tar_path))
def _path_and_destination(self, unit):
"""
Get the path component of the download URL and download destination.
:param unit: A content unit.
:type unit: dict
:return: (path, destination)
:rtype: tuple(2)
"""
storage_path = unit[constants.STORAGE_PATH]
tar_path = unit.get(constants.TARBALL_PATH)
if not tar_path:
relative_path = unit[constants.RELATIVE_PATH]
return pathlib.quote(relative_path), storage_path
else:
return pathlib.quote(tar_path), pathlib.join(os.path.dirname(storage_path), os.path.basename(tar_path))
def _path_and_destination(self, unit):
"""
Get the path component of the download URL and download destination.
:param unit: A content unit.
:type unit: dict
:return: (path, destination)
:rtype: tuple(2)
"""
storage_path = unit[constants.STORAGE_PATH]
tar_path = unit.get(constants.TARBALL_PATH)
if not tar_path:
relative_path = unit[constants.RELATIVE_PATH]
return pathlib.quote(relative_path), storage_path
else:
return pathlib.quote(tar_path),\
pathlib.join(os.path.dirname(storage_path), os.path.basename(tar_path))
def _add_units(self, request, unit_inventory):
"""
Determine the list of units contained in the parent inventory
but are not contained in the child inventory and add them.
For each unit, this is performed in the following steps:
1. Download the file (if defined) associated with the unit.
2. Add the unit to the child inventory.
3. Associate the unit to the repository.
The unit is added only:
1. If no file is associated with unit.
2. The file associated with the unit is successfully downloaded.
For units with files, the unit is added to the inventory as part of the
unit download manager callback.
:param request: A synchronization request.
:type request: SyncRequest
:param unit_inventory: The inventory of both parent and child content units.
:type unit_inventory: UnitInventory
"""
download_list = []
units = unit_inventory.units_on_parent_only()
request.progress.begin_adding_units(len(units))
manager = UnitDownloadManager(self, request)
publishing_details = unit_inventory.manifest.publishing_details
for unit, unit_ref in units:
if request.cancelled():
return
self._update_storage_path(unit)
if not self._needs_download(unit):
# unit has no file associated
self.add_unit(request, unit_ref.fetch())
continue
url = pathlib.url_join(
publishing_details[constants.BASE_URL],
pathlib.quote(unit[constants.RELATIVE_PATH]))
storage_path = unit[constants.STORAGE_PATH]
_request = manager.create_request(url, storage_path, unit_ref)
download_list.append(_request)
if request.cancelled():
return
request.downloader.event_listener = manager
request.downloader.download(download_list)
request.summary.errors.extend(manager.error_list())
def _add_units(self, request, unit_inventory):
"""
Determine the list of units contained in the parent inventory
but are not contained in the child inventory and add them.
For each unit, this is performed in the following steps:
1. Download the file (if defined) associated with the unit.
2. Add the unit to the child inventory.
3. Associate the unit to the repository.
The unit is added only:
1. If no file is associated with unit.
2. The file associated with the unit is successfully downloaded.
For units with files, the unit is added to the inventory as part of the
unit download manager callback.
:param request: A synchronization request.
:type request: SyncRequest
:param unit_inventory: The inventory of both parent and child content units.
:type unit_inventory: UnitInventory
"""
download_list = []
units = unit_inventory.units_on_parent_only()
request.progress.begin_adding_units(len(units))
manager = UnitDownloadManager(self, request)
publishing_details = unit_inventory.manifest.publishing_details
for unit, unit_ref in units:
if request.cancelled():
return
self._update_storage_path(unit)
if not self._needs_download(unit):
# unit has no file associated
self.add_unit(request, unit_ref.fetch())
continue
url = pathlib.url_join(publishing_details[constants.BASE_URL], pathlib.quote(unit[constants.RELATIVE_PATH]))
storage_path = unit[constants.STORAGE_PATH]
_request = manager.create_request(url, storage_path, unit_ref)
download_list.append(_request)
if request.cancelled():
return
request.downloader.event_listener = manager
request.downloader.download(download_list)
request.summary.errors.extend(manager.error_list())
