def download_sources(self,
sources,
insecure=False,
download_dir=SRPMS_DOWNLOAD_DIR):
"""Download sources content
Download content in the given URLs into a new temporary directory and
return a list with each downloaded artifact's path.
:param sources: list, dicts with URLs to download
:param insecure: bool, whether to perform TLS checks of urls
:param download_dir: str, directory where to download content
:return: str, paths to directory with downloaded sources
"""
dest_dir: Path = self.workflow.build_dir.source_container_sources_dir / download_dir
dest_dir.mkdir(parents=True, exist_ok=True)
req_session = get_retrying_requests_session()
for source in sources:
subdir: Path = dest_dir / source.get('subdir', '')
subdir.mkdir(parents=True, exist_ok=True)
checksums = source.get('checksums', {})
download_url(source['url'],
subdir,
insecure=insecure,
session=req_session,
dest_filename=source.get('dest'),
expected_checksums=checksums)
return str(dest_dir)
def download_files(self, downloads):
artifacts_path = os.path.join(self.workdir, self.DOWNLOAD_DIR)
koji_config = get_koji(self.workflow)
insecure = koji_config.get('insecure_download', False)
self.log.debug('%d files to download', len(downloads))
session = util.get_retrying_requests_session()
for index, download in enumerate(downloads):
dest_path = os.path.join(artifacts_path, download.dest)
dest_dir = dest_path.rsplit('/', 1)[0]
dest_filename = dest_path.rsplit('/', 1)[-1]
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
self.log.debug('%d/%d downloading %s', index + 1, len(downloads),
download.url)
download_url(url=download.url,
dest_dir=dest_dir,
insecure=insecure,
session=session,
dest_filename=dest_filename,
expected_checksums=download.checksums)
def download_sources(self,
sources,
insecure=False,
download_dir=SRPMS_DOWNLOAD_DIR):
"""Download sources content
Download content in the given URLs into a new temporary directory and
return a list with each downloaded artifact's path.
:param sources: list, dicts with URLs to download
:param insecure: bool, whether to perform TLS checks of urls
:param download_dir: str, directory where to download content
:return: str, paths to directory with downloaded sources
"""
workdir = tempfile.mkdtemp()
dest_dir = os.path.join(workdir, download_dir)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
req_session = get_retrying_requests_session()
for source in sources:
download_url(source['url'],
dest_dir,
insecure=insecure,
session=req_session,
dest_filename=source.get('dest'))
return dest_dir
def download_files(self, downloads: Sequence[DownloadRequest],
build_dir: BuildDir) -> Iterator[Path]:
"""Download maven artifacts to a build dir."""
artifacts_path = build_dir.path / self.DOWNLOAD_DIR
koji_config = self.workflow.conf.koji
insecure = koji_config.get('insecure_download', False)
self.log.debug('%d files to download', len(downloads))
session = util.get_retrying_requests_session()
for index, download in enumerate(downloads):
dest_path = artifacts_path / download.dest
dest_dir = dest_path.parent
dest_filename = dest_path.name
if not dest_dir.exists():
dest_dir.mkdir(parents=True)
self.log.debug('%d/%d downloading %s', index + 1, len(downloads),
download.url)
download_url(url=download.url,
dest_dir=dest_dir,
insecure=insecure,
session=session,
dest_filename=dest_filename,
expected_checksums=download.checksums)
yield dest_path
def test_connection_failure(self):
url = 'https://example.com/path/file'
dest_dir = tempfile.mkdtemp()
session = get_retrying_requests_session()
(flexmock(session).should_receive('get').and_raise(
requests.exceptions.RetryError))
with pytest.raises(requests.exceptions.RetryError):
download_url(url, dest_dir, session=session)
def get_remote_source_files(
self,
download_queue: Sequence[DownloadRequest]) -> List[Dict[str, Any]]:
remote_source_files = []
downloads_path = self.workflow.build_dir.any_platform.path / self.DOWNLOAD_DIR
session = util.get_retrying_requests_session()
self.log.debug('%d url source files to download', len(download_queue))
koji_config = self.workflow.conf.koji
insecure = koji_config.get('insecure_download', False)
for index, download in enumerate(download_queue):
dest_filename = download.dest
if not re.fullmatch(r'^[\w\-.]+$', dest_filename):
dest_filename = session.head(download.url).headers.get(
"Content-disposition").split("filename=")[1].replace(
'"', '')
dest_path = os.path.join(downloads_path, dest_filename)
dest_dir = os.path.dirname(dest_path)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
self.log.debug('%d/%d downloading %s', index + 1,
len(download_queue), download.url)
download_url(url=download.url,
dest_dir=dest_dir,
insecure=insecure,
session=session,
dest_filename=dest_filename,
expected_checksums=download.checksums)
checksum_type = list(download.checksums.keys())[0]
remote_source_files.append({
'file': dest_path,
'metadata': {
'type': KOJI_BTYPE_REMOTE_SOURCE_FILE,
'checksum_type': checksum_type,
'checksum': download.checksums[checksum_type],
'filename': dest_filename,
'filesize': os.path.getsize(dest_path),
'extra': {
'source-url': download.url,
'artifacts':
self.source_url_to_artifacts[download.url],
'typeinfo': {
KOJI_BTYPE_REMOTE_SOURCE_FILE: {}
},
},
}
})
return remote_source_files
def run(self):
"""
Run the plugin.
"""
if not self.url:
self.log.info('No remote source url to download, skipping plugin')
return
# Download the source code archive
archive = download_url(self.url, self.workflow.source.workdir)
# Unpack the source code archive into a dedicated dir in container build workdir
dest_dir = os.path.join(self.workflow.builder.df_dir, self.REMOTE_SOURCE)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
else:
raise RuntimeError('Conflicting path {} already exists in the dist-git repository'
.format(self.REMOTE_SOURCE))
with tarfile.open(archive) as tf:
tf.extractall(dest_dir)
# Set build args
self.workflow.builder.buildargs.update(self.buildargs)
# To copy the sources into the build image, Dockerfile should contain
# COPY $REMOTE_SOURCE $REMOTE_SOURCE_DIR
args_for_dockerfile_to_add = {
'REMOTE_SOURCE': self.REMOTE_SOURCE,
'REMOTE_SOURCE_DIR': REMOTE_SOURCE_DIR,
}
self.workflow.builder.buildargs.update(args_for_dockerfile_to_add)
return archive
def test_streaming_failure(self):
url = 'https://example.com/path/file'
dest_dir = tempfile.mkdtemp()
session = get_retrying_requests_session()
# get response shows successful connection
response = flexmock()
(response.should_receive('raise_for_status'))
# but streaming from the response fails
(response.should_receive('iter_content').and_raise(
requests.exceptions.RequestException))
# get on the session should return our mock response
(flexmock(session).should_receive('get').and_return(response))
# Speed through the retries
(flexmock(time).should_receive('sleep'))
with pytest.raises(requests.exceptions.RequestException):
download_url(url, dest_dir, session=session)
def download_sources(self, urls, insecure=False):
"""Download sources content
Download content in the given URLs into a new temporary directory and
return a list with each downloaded artifact's path.
:param urls: int, Koji build id of the container image we want SRPMs for
:param insecure: bool, whether to perform TLS checks of urls
:return: str, paths to directory with downloaded sources
"""
workdir = tempfile.mkdtemp()
dest_dir = os.path.join(workdir, self.DOWNLOAD_DIR)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
req_session = get_retrying_requests_session()
for url in urls:
download_url(url, dest_dir, insecure=insecure, session=req_session)
return dest_dir
def test_happy_path(self):
url = 'https://example.com/path/file'
dest_dir = tempfile.mkdtemp()
content = b'abc'
reader = BufferedReader(BytesIO(content), buffer_size=1)
responses.add(responses.GET, url, body=reader)
result = download_url(url, dest_dir)
assert os.path.basename(result) == 'file'
with open(result, 'rb') as f:
assert f.read() == content
def run(self):
"""
Run the plugin.
"""
# Download the source code archive
archive = download_url(self.url, self.workflow.source.workdir)
# Unpack the source code archive into the workdir
with tarfile.open(archive) as tf:
tf.extractall(self.workflow.source.workdir)
# Set build args
self.workflow.builder.buildargs.update(self.buildargs)
return archive
def download_sources(self, request, dest_dir='.', dest_filename=REMOTE_SOURCE_TARBALL_FILENAME):
"""Download the sources from a Cachito request
:param request: int or dict, either the Cachito request ID or a dict with 'id' key
:param dest_dir: str, existing directory to create file in
:param dest_filename: str, optional filename for downloaded file
"""
request_id = self._get_request_id(request)
logger.debug('Downloading sources bundle from request %ds', request_id)
url = self.assemble_download_url(request_id)
dest_path = download_url(
url, dest_dir=dest_dir, insecure=not self.session.verify, session=self.session,
dest_filename=dest_filename)
logger.debug('Sources bundle for request %d downloaded to %s', request_id, dest_path)
return dest_path
def run(self):
"""
Run the plugin.
"""
if not self.remote_sources:
self.log.info('Missing remote_sources parameters, skipping plugin')
return
session = get_retrying_requests_session()
archives = []
cachito_config = get_cachito(self.workflow)
insecure_ssl_conn = cachito_config.get('insecure', False)
for remote_source in self.remote_sources:
parsed_url = urlparse(remote_source['url'])
dest_filename = os.path.basename(parsed_url.path)
# prepend remote source name to destination filename, so multiple source archives
# don't have name collision
if self.multiple_remote_sources:
dest_filename = "{}_{}".format(remote_source['name'],
dest_filename)
# Download the source code archive
archive = download_url(remote_source['url'],
self.workflow.source.workdir,
session=session,
insecure=insecure_ssl_conn,
dest_filename=dest_filename)
archives.append(archive)
# Unpack the source code archive into a dedicated dir in container build workdir
dest_dir = os.path.join(self.workflow.builder.df_dir,
self.REMOTE_SOURCE)
sub_path = self.REMOTE_SOURCE
if self.multiple_remote_sources:
dest_dir = os.path.join(dest_dir, remote_source['name'])
sub_path = os.path.join(sub_path, remote_source['name'])
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
else:
raise RuntimeError(
'Conflicting path {} already exists in the dist-git repository'
.format(sub_path))
with tarfile.open(archive) as tf:
tf.extractall(dest_dir)
config_files = (self.get_remote_source_config(
session, remote_source["configs"], insecure_ssl_conn))
self.generate_cachito_config_files(dest_dir, config_files)
# Set build args
if not self.multiple_remote_sources:
self.workflow.builder.buildargs.update(
remote_source['build_args'])
# Create cachito.env file with environment variables received from cachito request
self.generate_cachito_env_file(dest_dir,
remote_source['build_args'])
self.add_general_buildargs()
return archives
def run(self):
"""
Run the plugin.
"""
if not self.url:
self.log.info('No remote source url to download, skipping plugin')
return
session = get_retrying_requests_session()
# Download the source code archive
cachito_config = get_cachito(self.workflow)
insecure_ssl_conn = cachito_config.get('insecure', False)
archive = download_url(self.url,
self.workflow.source.workdir,
session=session,
insecure=insecure_ssl_conn)
# Unpack the source code archive into a dedicated dir in container build workdir
dest_dir = os.path.join(self.workflow.builder.df_dir,
self.REMOTE_SOURCE)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
else:
raise RuntimeError(
'Conflicting path {} already exists in the dist-git repository'
.format(self.REMOTE_SOURCE))
with tarfile.open(archive) as tf:
tf.extractall(dest_dir)
config_files = (self.get_remote_source_config(
session, self.remote_source_conf_url, insecure_ssl_conn)
if self.remote_source_conf_url else [])
# Inject cachito provided configuration files
for config in config_files:
config_path = os.path.join(dest_dir, config['path'])
if config['type'] == CFG_TYPE_B64:
data = base64.b64decode(config['content'])
with open(config_path, 'wb') as f:
f.write(data)
else:
err_msg = "Unknown cachito configuration file data type '{}'".format(
config['type'])
raise ValueError(err_msg)
os.chmod(config_path, 0o444)
# Set build args
self.workflow.builder.buildargs.update(self.buildargs)
# Create cachito.env file with environment variables received from cachito request
self.generate_cachito_env_file()
# To copy the sources into the build image, Dockerfile should contain
# COPY $REMOTE_SOURCE $REMOTE_SOURCE_DIR
args_for_dockerfile_to_add = {
'REMOTE_SOURCE': self.REMOTE_SOURCE,
'REMOTE_SOURCE_DIR': REMOTE_SOURCE_DIR,
}
self.workflow.builder.buildargs.update(args_for_dockerfile_to_add)
return archive
