def download_sources(self, sources, insecure=False, download_dir=SRPMS_DOWNLOAD_DIR): """Download sources content Download content in the given URLs into a new temporary directory and return a list with each downloaded artifact's path. :param sources: list, dicts with URLs to download :param insecure: bool, whether to perform TLS checks of urls :param download_dir: str, directory where to download content :return: str, paths to directory with downloaded sources """ dest_dir: Path = self.workflow.build_dir.source_container_sources_dir / download_dir dest_dir.mkdir(parents=True, exist_ok=True) req_session = get_retrying_requests_session() for source in sources: subdir: Path = dest_dir / source.get('subdir', '') subdir.mkdir(parents=True, exist_ok=True) checksums = source.get('checksums', {}) download_url(source['url'], subdir, insecure=insecure, session=req_session, dest_filename=source.get('dest'), expected_checksums=checksums) return str(dest_dir)
def download_files(self, downloads): artifacts_path = os.path.join(self.workdir, self.DOWNLOAD_DIR) koji_config = get_koji(self.workflow) insecure = koji_config.get('insecure_download', False) self.log.debug('%d files to download', len(downloads)) session = util.get_retrying_requests_session() for index, download in enumerate(downloads): dest_path = os.path.join(artifacts_path, download.dest) dest_dir = dest_path.rsplit('/', 1)[0] dest_filename = dest_path.rsplit('/', 1)[-1] if not os.path.exists(dest_dir): os.makedirs(dest_dir) self.log.debug('%d/%d downloading %s', index + 1, len(downloads), download.url) download_url(url=download.url, dest_dir=dest_dir, insecure=insecure, session=session, dest_filename=dest_filename, expected_checksums=download.checksums)
def download_sources(self, sources, insecure=False, download_dir=SRPMS_DOWNLOAD_DIR): """Download sources content Download content in the given URLs into a new temporary directory and return a list with each downloaded artifact's path. :param sources: list, dicts with URLs to download :param insecure: bool, whether to perform TLS checks of urls :param download_dir: str, directory where to download content :return: str, paths to directory with downloaded sources """ workdir = tempfile.mkdtemp() dest_dir = os.path.join(workdir, download_dir) if not os.path.exists(dest_dir): os.makedirs(dest_dir) req_session = get_retrying_requests_session() for source in sources: download_url(source['url'], dest_dir, insecure=insecure, session=req_session, dest_filename=source.get('dest')) return dest_dir
def download_files(self, downloads: Sequence[DownloadRequest], build_dir: BuildDir) -> Iterator[Path]: """Download maven artifacts to a build dir.""" artifacts_path = build_dir.path / self.DOWNLOAD_DIR koji_config = self.workflow.conf.koji insecure = koji_config.get('insecure_download', False) self.log.debug('%d files to download', len(downloads)) session = util.get_retrying_requests_session() for index, download in enumerate(downloads): dest_path = artifacts_path / download.dest dest_dir = dest_path.parent dest_filename = dest_path.name if not dest_dir.exists(): dest_dir.mkdir(parents=True) self.log.debug('%d/%d downloading %s', index + 1, len(downloads), download.url) download_url(url=download.url, dest_dir=dest_dir, insecure=insecure, session=session, dest_filename=dest_filename, expected_checksums=download.checksums) yield dest_path
def test_connection_failure(self): url = 'https://example.com/path/file' dest_dir = tempfile.mkdtemp() session = get_retrying_requests_session() (flexmock(session).should_receive('get').and_raise( requests.exceptions.RetryError)) with pytest.raises(requests.exceptions.RetryError): download_url(url, dest_dir, session=session)
def get_remote_source_files( self, download_queue: Sequence[DownloadRequest]) -> List[Dict[str, Any]]: remote_source_files = [] downloads_path = self.workflow.build_dir.any_platform.path / self.DOWNLOAD_DIR session = util.get_retrying_requests_session() self.log.debug('%d url source files to download', len(download_queue)) koji_config = self.workflow.conf.koji insecure = koji_config.get('insecure_download', False) for index, download in enumerate(download_queue): dest_filename = download.dest if not re.fullmatch(r'^[\w\-.]+$', dest_filename): dest_filename = session.head(download.url).headers.get( "Content-disposition").split("filename=")[1].replace( '"', '') dest_path = os.path.join(downloads_path, dest_filename) dest_dir = os.path.dirname(dest_path) if not os.path.exists(dest_dir): os.makedirs(dest_dir) self.log.debug('%d/%d downloading %s', index + 1, len(download_queue), download.url) download_url(url=download.url, dest_dir=dest_dir, insecure=insecure, session=session, dest_filename=dest_filename, expected_checksums=download.checksums) checksum_type = list(download.checksums.keys())[0] remote_source_files.append({ 'file': dest_path, 'metadata': { 'type': KOJI_BTYPE_REMOTE_SOURCE_FILE, 'checksum_type': checksum_type, 'checksum': download.checksums[checksum_type], 'filename': dest_filename, 'filesize': os.path.getsize(dest_path), 'extra': { 'source-url': download.url, 'artifacts': self.source_url_to_artifacts[download.url], 'typeinfo': { KOJI_BTYPE_REMOTE_SOURCE_FILE: {} }, }, } }) return remote_source_files
def run(self): """ Run the plugin. """ if not self.url: self.log.info('No remote source url to download, skipping plugin') return # Download the source code archive archive = download_url(self.url, self.workflow.source.workdir) # Unpack the source code archive into a dedicated dir in container build workdir dest_dir = os.path.join(self.workflow.builder.df_dir, self.REMOTE_SOURCE) if not os.path.exists(dest_dir): os.makedirs(dest_dir) else: raise RuntimeError('Conflicting path {} already exists in the dist-git repository' .format(self.REMOTE_SOURCE)) with tarfile.open(archive) as tf: tf.extractall(dest_dir) # Set build args self.workflow.builder.buildargs.update(self.buildargs) # To copy the sources into the build image, Dockerfile should contain # COPY $REMOTE_SOURCE $REMOTE_SOURCE_DIR args_for_dockerfile_to_add = { 'REMOTE_SOURCE': self.REMOTE_SOURCE, 'REMOTE_SOURCE_DIR': REMOTE_SOURCE_DIR, } self.workflow.builder.buildargs.update(args_for_dockerfile_to_add) return archive
def test_streaming_failure(self): url = 'https://example.com/path/file' dest_dir = tempfile.mkdtemp() session = get_retrying_requests_session() # get response shows successful connection response = flexmock() (response.should_receive('raise_for_status')) # but streaming from the response fails (response.should_receive('iter_content').and_raise( requests.exceptions.RequestException)) # get on the session should return our mock response (flexmock(session).should_receive('get').and_return(response)) # Speed through the retries (flexmock(time).should_receive('sleep')) with pytest.raises(requests.exceptions.RequestException): download_url(url, dest_dir, session=session)
def download_sources(self, urls, insecure=False): """Download sources content Download content in the given URLs into a new temporary directory and return a list with each downloaded artifact's path. :param urls: int, Koji build id of the container image we want SRPMs for :param insecure: bool, whether to perform TLS checks of urls :return: str, paths to directory with downloaded sources """ workdir = tempfile.mkdtemp() dest_dir = os.path.join(workdir, self.DOWNLOAD_DIR) if not os.path.exists(dest_dir): os.makedirs(dest_dir) req_session = get_retrying_requests_session() for url in urls: download_url(url, dest_dir, insecure=insecure, session=req_session) return dest_dir
def test_happy_path(self): url = 'https://example.com/path/file' dest_dir = tempfile.mkdtemp() content = b'abc' reader = BufferedReader(BytesIO(content), buffer_size=1) responses.add(responses.GET, url, body=reader) result = download_url(url, dest_dir) assert os.path.basename(result) == 'file' with open(result, 'rb') as f: assert f.read() == content
def run(self): """ Run the plugin. """ # Download the source code archive archive = download_url(self.url, self.workflow.source.workdir) # Unpack the source code archive into the workdir with tarfile.open(archive) as tf: tf.extractall(self.workflow.source.workdir) # Set build args self.workflow.builder.buildargs.update(self.buildargs) return archive
def download_sources(self, request, dest_dir='.', dest_filename=REMOTE_SOURCE_TARBALL_FILENAME): """Download the sources from a Cachito request :param request: int or dict, either the Cachito request ID or a dict with 'id' key :param dest_dir: str, existing directory to create file in :param dest_filename: str, optional filename for downloaded file """ request_id = self._get_request_id(request) logger.debug('Downloading sources bundle from request %ds', request_id) url = self.assemble_download_url(request_id) dest_path = download_url( url, dest_dir=dest_dir, insecure=not self.session.verify, session=self.session, dest_filename=dest_filename) logger.debug('Sources bundle for request %d downloaded to %s', request_id, dest_path) return dest_path
def run(self): """ Run the plugin. """ if not self.remote_sources: self.log.info('Missing remote_sources parameters, skipping plugin') return session = get_retrying_requests_session() archives = [] cachito_config = get_cachito(self.workflow) insecure_ssl_conn = cachito_config.get('insecure', False) for remote_source in self.remote_sources: parsed_url = urlparse(remote_source['url']) dest_filename = os.path.basename(parsed_url.path) # prepend remote source name to destination filename, so multiple source archives # don't have name collision if self.multiple_remote_sources: dest_filename = "{}_{}".format(remote_source['name'], dest_filename) # Download the source code archive archive = download_url(remote_source['url'], self.workflow.source.workdir, session=session, insecure=insecure_ssl_conn, dest_filename=dest_filename) archives.append(archive) # Unpack the source code archive into a dedicated dir in container build workdir dest_dir = os.path.join(self.workflow.builder.df_dir, self.REMOTE_SOURCE) sub_path = self.REMOTE_SOURCE if self.multiple_remote_sources: dest_dir = os.path.join(dest_dir, remote_source['name']) sub_path = os.path.join(sub_path, remote_source['name']) if not os.path.exists(dest_dir): os.makedirs(dest_dir) else: raise RuntimeError( 'Conflicting path {} already exists in the dist-git repository' .format(sub_path)) with tarfile.open(archive) as tf: tf.extractall(dest_dir) config_files = (self.get_remote_source_config( session, remote_source["configs"], insecure_ssl_conn)) self.generate_cachito_config_files(dest_dir, config_files) # Set build args if not self.multiple_remote_sources: self.workflow.builder.buildargs.update( remote_source['build_args']) # Create cachito.env file with environment variables received from cachito request self.generate_cachito_env_file(dest_dir, remote_source['build_args']) self.add_general_buildargs() return archives
def run(self): """ Run the plugin. """ if not self.url: self.log.info('No remote source url to download, skipping plugin') return session = get_retrying_requests_session() # Download the source code archive cachito_config = get_cachito(self.workflow) insecure_ssl_conn = cachito_config.get('insecure', False) archive = download_url(self.url, self.workflow.source.workdir, session=session, insecure=insecure_ssl_conn) # Unpack the source code archive into a dedicated dir in container build workdir dest_dir = os.path.join(self.workflow.builder.df_dir, self.REMOTE_SOURCE) if not os.path.exists(dest_dir): os.makedirs(dest_dir) else: raise RuntimeError( 'Conflicting path {} already exists in the dist-git repository' .format(self.REMOTE_SOURCE)) with tarfile.open(archive) as tf: tf.extractall(dest_dir) config_files = (self.get_remote_source_config( session, self.remote_source_conf_url, insecure_ssl_conn) if self.remote_source_conf_url else []) # Inject cachito provided configuration files for config in config_files: config_path = os.path.join(dest_dir, config['path']) if config['type'] == CFG_TYPE_B64: data = base64.b64decode(config['content']) with open(config_path, 'wb') as f: f.write(data) else: err_msg = "Unknown cachito configuration file data type '{}'".format( config['type']) raise ValueError(err_msg) os.chmod(config_path, 0o444) # Set build args self.workflow.builder.buildargs.update(self.buildargs) # Create cachito.env file with environment variables received from cachito request self.generate_cachito_env_file() # To copy the sources into the build image, Dockerfile should contain # COPY $REMOTE_SOURCE $REMOTE_SOURCE_DIR args_for_dockerfile_to_add = { 'REMOTE_SOURCE': self.REMOTE_SOURCE, 'REMOTE_SOURCE_DIR': REMOTE_SOURCE_DIR, } self.workflow.builder.buildargs.update(args_for_dockerfile_to_add) return archive