def validate(self, data): """ Validate collection remote data. - url: to ensure it does not end with slashes. - requirements_file: to ensure it is a valid yaml file. Args: data (dict): User data to validate Returns: dict: Validated data Raises: rest_framework.serializers.ValidationError: If the url or requirements_file is invalid """ data = super().validate(data) if data.get("requirements_file"): parse_collections_requirements_file(data["requirements_file"]) if data["url"].endswith("/"): raise serializers.ValidationError(_("url should not end with '/'")) return data
def __init__(self, remote, repository, optimize): """ The first stage of a pulp_ansible sync pipeline. Args: remote (CollectionRemote): The remote data to be used when syncing repository (AnsibleRepository): The repository being syncedself. optimize (boolean): Whether to optimize sync or not. """ super().__init__() msg = _("Parsing CollectionVersion Metadata") self.parsing_metadata_progress_bar = ProgressReport( message=msg, code="parsing.metadata") self.remote = remote self.repository = repository self.optimize = optimize self.collection_info = parse_collections_requirements_file( remote.requirements_file) self.deprecations = Q() self.add_dependents = self.collection_info and self.remote.sync_dependencies self.already_synced = set() self._unpaginated_collection_metadata = None self._unpaginated_collection_version_metadata = None self.last_synced_metadata_time = None # Interpret download policy self.deferred_download = self.remote.policy != Remote.IMMEDIATE
async def _download_unpaginated_metadata(self): root_endpoint, api_version = await self._get_root_api(self.remote.url) self._api_version = api_version if api_version > 2: loop = asyncio.get_event_loop() collection_endpoint = f"{root_endpoint}/collections/all/" excludes_endpoint = f"{root_endpoint}/excludes/" col_downloader = self.remote.get_downloader( url=collection_endpoint, silence_errors_for_response_status_codes={404}) exc_downloader = self.remote.get_downloader( url=excludes_endpoint, silence_errors_for_response_status_codes={404}) tasks = [ loop.create_task(col_downloader.run()), loop.create_task(exc_downloader.run()) ] col_results, exc_results = await asyncio.gather( *tasks, return_exceptions=True) if not isinstance(exc_results, FileNotFoundError): excludes_response = parse_metadata(exc_results) if excludes_response: try: excludes_list = parse_collections_requirements_file( excludes_response) except ValidationError: pass else: excludes = { r.name: parse_requirements_entry(r) for r in excludes_list } self.exclude_info.update(excludes) if not isinstance(col_results, FileNotFoundError): collection_metadata_list = parse_metadata(col_results) self._unpaginated_collection_metadata = defaultdict(dict) for collection in collection_metadata_list: namespace = collection["namespace"] name = collection["name"] self._unpaginated_collection_metadata[namespace][ name] = collection collection_version_endpoint = f"{root_endpoint}/collection_versions/all/" downloader = self.remote.get_downloader( url=collection_version_endpoint) collection_version_metadata_list = parse_metadata( await downloader.run()) self._unpaginated_collection_version_metadata = defaultdict( lambda: defaultdict(list)) for collection_version_metadata in collection_version_metadata_list: namespace = collection_version_metadata["namespace"][ "name"] name = collection_version_metadata["name"] self._unpaginated_collection_version_metadata[namespace][ name].append(collection_version_metadata)
def __init__(self, remote, repository, is_repo_remote, deprecation_before_sync, optimize): """ The first stage of a pulp_ansible sync pipeline. Args: remote (CollectionRemote): The remote data to be used when syncing repository (AnsibleRepository): The repository being syncedself. is_repo_remote (bool): True if the remote is the repository's remote. deprecation_before_sync (set): Set of deprecations before the sync. optimize (boolean): Whether to optimize sync or not. """ super().__init__() self.remote = remote self.repository = repository self.is_repo_remote = is_repo_remote self.deprecation_before_sync = deprecation_before_sync self.deprecation_after_sync = set() self.collection_info = parse_collections_requirements_file( remote.requirements_file) self.exclude_info = {} self.add_dependents = self.collection_info and self.remote.sync_dependencies self.signed_only = self.remote.signed_only self.already_synced = set() self._unpaginated_collection_metadata = None self._unpaginated_collection_version_metadata = None self.last_synced_metadata_time = None # Interpret download policy self.deferred_download = self.remote.policy != Remote.IMMEDIATE # Check if we should sync self.should_sync = not optimize or asyncio.get_event_loop( ).run_until_complete(self._should_we_sync())
def load_requirements_file(self, requirements_file): """Loads and validates the requirements file If the requirements_file is an absolute path for a file it is opened and read. """ if os.path.exists(requirements_file) and requirements_file.endswith( (".yaml", ".yml")): with open(requirements_file) as file_obj: requirements_file = file_obj.read() try: parse_collections_requirements_file(requirements_file) except ValidationError as e: raise CommandError("Error parsing requirements_file {}".format( str(e))) else: self.status_messages.append("requirements_file loaded and parsed") return requirements_file
def __init__(self, remote): """ The first stage of a pulp_ansible sync pipeline. Args: remote (CollectionRemote): The remote data to be used when syncing """ super().__init__() self.remote = remote self.collection_info = parse_collections_requirements_file(remote.requirements_file) # Interpret download policy self.deferred_download = self.remote.policy != Remote.IMMEDIATE
def validate(self, data): """ Validate collection remote data. - url: to ensure it ends with slashes. - requirements_file: to ensure it is a valid yaml file. Args: data (dict): User data to validate Returns: dict: Validated data Raises: rest_framework.serializers.ValidationError: If the url or requirements_file is invalid """ def _validate_url(url): """Based on https://git.io/JTMAA.""" if not url.endswith("/") and url != "https://galaxy.ansible.com": raise serializers.ValidationError( _("Invalid URL {url}. Ensure the URL ends '/'.").format(url=url) ) data = super().validate(data) if data.get("url"): _validate_url(data["url"]) if data.get("requirements_file"): collections = parse_collections_requirements_file(data["requirements_file"]) for collection in collections: if collection[2]: _validate_url(collection[2]) has_token = "token" in data or getattr(self.instance, "token", False) if data.get("auth_url") and not has_token: raise serializers.ValidationError( _("When specifying 'auth_url' you must also specify 'token'.") ) return data
def validate(self, data): """ Validate collection remote data. - url: to ensure it does not end with slashes. - requirements_file: to ensure it is a valid yaml file. Args: data (dict): User data to validate Returns: dict: Validated data Raises: rest_framework.serializers.ValidationError: If the url or requirements_file is invalid """ data = super().validate(data) if data.get("requirements_file"): parsed_file = parse_collections_requirements_file( data["requirements_file"]) urls_to_validate = [data["url"]] for name, version, source in parsed_file: if source: urls_to_validate.append(source) for url in urls_to_validate: if not url.endswith("/v2/collections") and not url.endswith( "/v3/collections"): raise serializers.ValidationError( _("Invalid URL {url}. ".format(url=data["url"]) + "Ensure the URL ends in either '/v2/collections' or '/v3/collections'" )) if data["url"].endswith("/"): raise serializers.ValidationError(_("url should not end with '/'")) return data
def validate(self, data): """ Validate collection remote data. - url: to ensure it ends with slashes. - requirements_file: to ensure it is a valid yaml file. Args: data (dict): User data to validate Returns: dict: Validated data Raises: rest_framework.serializers.ValidationError: If the url or requirements_file is invalid """ def _validate_url(url): """Based on https://git.io/JTMAA.""" if not url.endswith("/") and url != "https://galaxy.ansible.com": raise serializers.ValidationError( _("Invalid URL {url}. Ensure the URL ends '/'.").format( url=data["url"])) data = super().validate(data) _validate_url(data["url"]) if data.get("requirements_file"): collections = parse_collections_requirements_file( data["requirements_file"]) for collection in collections: if collection[2]: _validate_url(collection[2]) return data