def validate(self, data):
"""
Validate collection remote data.
- url: to ensure it does not end with slashes.
- requirements_file: to ensure it is a valid yaml file.
Args:
data (dict): User data to validate
Returns:
dict: Validated data
Raises:
rest_framework.serializers.ValidationError: If the url or requirements_file is invalid
"""
data = super().validate(data)
if data.get("requirements_file"):
parse_collections_requirements_file(data["requirements_file"])
if data["url"].endswith("/"):
raise serializers.ValidationError(_("url should not end with '/'"))
return data
def __init__(self, remote, repository, optimize):
"""
The first stage of a pulp_ansible sync pipeline.
Args:
remote (CollectionRemote): The remote data to be used when syncing
repository (AnsibleRepository): The repository being syncedself.
optimize (boolean): Whether to optimize sync or not.
"""
super().__init__()
msg = _("Parsing CollectionVersion Metadata")
self.parsing_metadata_progress_bar = ProgressReport(
message=msg, code="parsing.metadata")
self.remote = remote
self.repository = repository
self.optimize = optimize
self.collection_info = parse_collections_requirements_file(
remote.requirements_file)
self.deprecations = Q()
self.add_dependents = self.collection_info and self.remote.sync_dependencies
self.already_synced = set()
self._unpaginated_collection_metadata = None
self._unpaginated_collection_version_metadata = None
self.last_synced_metadata_time = None
# Interpret download policy
self.deferred_download = self.remote.policy != Remote.IMMEDIATE
async def _download_unpaginated_metadata(self):
root_endpoint, api_version = await self._get_root_api(self.remote.url)
self._api_version = api_version
if api_version > 2:
loop = asyncio.get_event_loop()
collection_endpoint = f"{root_endpoint}/collections/all/"
excludes_endpoint = f"{root_endpoint}/excludes/"
col_downloader = self.remote.get_downloader(
url=collection_endpoint,
silence_errors_for_response_status_codes={404})
exc_downloader = self.remote.get_downloader(
url=excludes_endpoint,
silence_errors_for_response_status_codes={404})
tasks = [
loop.create_task(col_downloader.run()),
loop.create_task(exc_downloader.run())
]
col_results, exc_results = await asyncio.gather(
*tasks, return_exceptions=True)
if not isinstance(exc_results, FileNotFoundError):
excludes_response = parse_metadata(exc_results)
if excludes_response:
try:
excludes_list = parse_collections_requirements_file(
excludes_response)
except ValidationError:
pass
else:
excludes = {
r.name: parse_requirements_entry(r)
for r in excludes_list
}
self.exclude_info.update(excludes)
if not isinstance(col_results, FileNotFoundError):
collection_metadata_list = parse_metadata(col_results)
self._unpaginated_collection_metadata = defaultdict(dict)
for collection in collection_metadata_list:
namespace = collection["namespace"]
name = collection["name"]
self._unpaginated_collection_metadata[namespace][
name] = collection
collection_version_endpoint = f"{root_endpoint}/collection_versions/all/"
downloader = self.remote.get_downloader(
url=collection_version_endpoint)
collection_version_metadata_list = parse_metadata(
await downloader.run())
self._unpaginated_collection_version_metadata = defaultdict(
lambda: defaultdict(list))
for collection_version_metadata in collection_version_metadata_list:
namespace = collection_version_metadata["namespace"][
"name"]
name = collection_version_metadata["name"]
self._unpaginated_collection_version_metadata[namespace][
name].append(collection_version_metadata)
def __init__(self, remote, repository, is_repo_remote,
deprecation_before_sync, optimize):
"""
The first stage of a pulp_ansible sync pipeline.
Args:
remote (CollectionRemote): The remote data to be used when syncing
repository (AnsibleRepository): The repository being syncedself.
is_repo_remote (bool): True if the remote is the repository's remote.
deprecation_before_sync (set): Set of deprecations before the sync.
optimize (boolean): Whether to optimize sync or not.
"""
super().__init__()
self.remote = remote
self.repository = repository
self.is_repo_remote = is_repo_remote
self.deprecation_before_sync = deprecation_before_sync
self.deprecation_after_sync = set()
self.collection_info = parse_collections_requirements_file(
remote.requirements_file)
self.exclude_info = {}
self.add_dependents = self.collection_info and self.remote.sync_dependencies
self.signed_only = self.remote.signed_only
self.already_synced = set()
self._unpaginated_collection_metadata = None
self._unpaginated_collection_version_metadata = None
self.last_synced_metadata_time = None
# Interpret download policy
self.deferred_download = self.remote.policy != Remote.IMMEDIATE
# Check if we should sync
self.should_sync = not optimize or asyncio.get_event_loop(
).run_until_complete(self._should_we_sync())
def load_requirements_file(self, requirements_file):
"""Loads and validates the requirements file
If the requirements_file is an absolute path for a file it is
opened and read.
"""
if os.path.exists(requirements_file) and requirements_file.endswith(
(".yaml", ".yml")):
with open(requirements_file) as file_obj:
requirements_file = file_obj.read()
try:
parse_collections_requirements_file(requirements_file)
except ValidationError as e:
raise CommandError("Error parsing requirements_file {}".format(
str(e)))
else:
self.status_messages.append("requirements_file loaded and parsed")
return requirements_file
def __init__(self, remote):
"""
The first stage of a pulp_ansible sync pipeline.
Args:
remote (CollectionRemote): The remote data to be used when syncing
"""
super().__init__()
self.remote = remote
self.collection_info = parse_collections_requirements_file(remote.requirements_file)
# Interpret download policy
self.deferred_download = self.remote.policy != Remote.IMMEDIATE
def validate(self, data):
"""
Validate collection remote data.
- url: to ensure it ends with slashes.
- requirements_file: to ensure it is a valid yaml file.
Args:
data (dict): User data to validate
Returns:
dict: Validated data
Raises:
rest_framework.serializers.ValidationError: If the url or requirements_file is invalid
"""
def _validate_url(url):
"""Based on https://git.io/JTMAA."""
if not url.endswith("/") and url != "https://galaxy.ansible.com":
raise serializers.ValidationError(
_("Invalid URL {url}. Ensure the URL ends '/'.").format(url=url)
)
data = super().validate(data)
if data.get("url"):
_validate_url(data["url"])
if data.get("requirements_file"):
collections = parse_collections_requirements_file(data["requirements_file"])
for collection in collections:
if collection[2]:
_validate_url(collection[2])
has_token = "token" in data or getattr(self.instance, "token", False)
if data.get("auth_url") and not has_token:
raise serializers.ValidationError(
_("When specifying 'auth_url' you must also specify 'token'.")
)
return data
def validate(self, data):
"""
Validate collection remote data.
- url: to ensure it does not end with slashes.
- requirements_file: to ensure it is a valid yaml file.
Args:
data (dict): User data to validate
Returns:
dict: Validated data
Raises:
rest_framework.serializers.ValidationError: If the url or requirements_file is invalid
"""
data = super().validate(data)
if data.get("requirements_file"):
parsed_file = parse_collections_requirements_file(
data["requirements_file"])
urls_to_validate = [data["url"]]
for name, version, source in parsed_file:
if source:
urls_to_validate.append(source)
for url in urls_to_validate:
if not url.endswith("/v2/collections") and not url.endswith(
"/v3/collections"):
raise serializers.ValidationError(
_("Invalid URL {url}. ".format(url=data["url"]) +
"Ensure the URL ends in either '/v2/collections' or '/v3/collections'"
))
if data["url"].endswith("/"):
raise serializers.ValidationError(_("url should not end with '/'"))
return data
def validate(self, data):
"""
Validate collection remote data.
- url: to ensure it ends with slashes.
- requirements_file: to ensure it is a valid yaml file.
Args:
data (dict): User data to validate
Returns:
dict: Validated data
Raises:
rest_framework.serializers.ValidationError: If the url or requirements_file is invalid
"""
def _validate_url(url):
"""Based on https://git.io/JTMAA."""
if not url.endswith("/") and url != "https://galaxy.ansible.com":
raise serializers.ValidationError(
_("Invalid URL {url}. Ensure the URL ends '/'.").format(
url=data["url"]))
data = super().validate(data)
_validate_url(data["url"])
if data.get("requirements_file"):
collections = parse_collections_requirements_file(
data["requirements_file"])
for collection in collections:
if collection[2]:
_validate_url(collection[2])
return data
