def _get_lookaside_sources(self) -> List[Dict[str, str]]:
"""
Read "sources" file from the dist-git repo and return a list of dicts
with path and url to sources stored in the lookaside cache
"""
pkg_tool = self.pkg_tool or self.config.pkg_tool
try:
config = LookasideCacheHelper._read_config(pkg_tool)
base_url = config["lookaside"]
except (configparser.Error, KeyError) as e:
raise LookasideCacheError(
"Failed to read rpkg configuration") from e
package = self.pkg_name
basepath = self.dist_git.working_dir
sources = []
for source in LookasideCacheHelper._read_sources(basepath):
url = "{0}/rpms/{1}/{2}/{3}/{4}/{2}".format(
base_url,
package,
source["filename"],
source["hashtype"],
source["hash"],
)
path = source["filename"]
sources.append({"path": path, "url": url})
return sources
def download_remote_sources(self):
"""
Method that iterates over all sources and downloads ones, which contain URL instead of just a file.
:return: None
"""
try:
# try to download old sources from Fedora lookaside cache
LookasideCacheHelper.download(self.lookaside_cache_preset, os.path.dirname(self.path), self.header.name,
self.sources_location)
except LookasideCacheError as e:
logger.verbose("Downloading sources from lookaside cache failed. "
"Reason: %s.", str(e))
# filter out only sources with URL
remote_files = [source for source in self.sources if bool(urllib.parse.urlparse(source).scheme)]
# download any sources that are not yet downloaded
for remote_file in remote_files:
local_file = os.path.join(self.sources_location, os.path.basename(remote_file))
if not os.path.isfile(local_file):
logger.verbose("File '%s' doesn't exist locally, downloading it.", local_file)
try:
DownloadHelper.download_file(remote_file, local_file)
except DownloadError as e:
raise RebaseHelperError("Failed to download file from URL {}. "
"Reason: '{}'. ".format(remote_file, str(e))) from e
def test_upload(self, filename, hashtype, hsh):
# pylint: disable=protected-access
LookasideCacheHelper._upload_source('https://integration:4430/pkgs',
'test',
filename,
hashtype,
hsh,
None)
def test_upload(self, filename, hashtype, hsh):
# pylint: disable=protected-access
LookasideCacheHelper._upload_source('https://integration:4430/pkgs',
'test',
'',
filename,
hashtype,
hsh)
def test_download(self, package, filename, hashtype, hsh):
# pylint: disable=protected-access
target = os.path.basename(filename)
LookasideCacheHelper._download_source('fedpkg',
'https://integration:4430/pkgs',
package, filename, hashtype, hsh,
target)
assert os.path.isfile(target)
assert LookasideCacheHelper._hash(target, hashtype) == hsh
def test_download(self, package, filename, hashtype, hsh):
# pylint: disable=protected-access
target = os.path.basename(filename)
LookasideCacheHelper._download_source('fedpkg',
'https://integration:4430/pkgs',
package,
filename,
hashtype,
hsh,
target)
assert os.path.isfile(target)
assert LookasideCacheHelper._hash(target, hashtype) == hsh
def __init__(self, cli_conf, execution_dir, results_dir, debug_log_file):
"""
Initialize the application
:param cli_conf: CLI object with configuration gathered from commandline
:return:
"""
results_store.clear()
self.conf = cli_conf
self.execution_dir = execution_dir
self.rebased_sources_dir = os.path.join(results_dir, 'rebased-sources')
self.debug_log_file = debug_log_file
self.kwargs.update(self.conf.config)
# Temporary workspace for Builder, checks, ...
self.kwargs['workspace_dir'] = self.workspace_dir = os.path.join(self.execution_dir, constants.WORKSPACE_DIR)
# Directory where results should be put
self.kwargs['results_dir'] = self.results_dir = results_dir
# Directory contaning only those files, which are relevant for the new rebased version
self.kwargs['rebased_sources_dir'] = self.rebased_sources_dir
logger.verbose("Rebase-helper version: %s", VERSION)
if self.conf.build_tasks is None:
# check the workspace dir
if not self.conf.cont:
self._check_workspace_dir()
self._get_spec_file()
self._prepare_spec_objects()
# verify all sources for the new version are present
missing_sources = [os.path.basename(s) for s in self.rebase_spec_file.sources
if not os.path.isfile(os.path.basename(s))]
if missing_sources:
raise RebaseHelperError('The following sources are missing: {}'.format(','.join(missing_sources)))
if self.conf.update_sources:
sources = [os.path.basename(s) for s in self.spec_file.sources]
rebased_sources = [os.path.basename(s) for s in self.rebase_spec_file.sources]
uploaded = LookasideCacheHelper.update_sources('fedpkg', self.rebased_sources_dir,
self.rebase_spec_file.get_package_name(),
sources, rebased_sources,
upload=not self.conf.skip_upload)
self._update_gitignore(uploaded, self.rebased_sources_dir)
# TODO: Remove the value from kwargs and use only CLI attribute!
self.kwargs['continue'] = self.conf.cont
self._initialize_data()
if self.conf.cont or self.conf.build_only:
self._delete_old_builds()
def download_remote_sources(self):
"""
Download the sources from the URL in the configuration (if the path in
the configuration match to the URL basename from SourceX) or from the one
from SourceX in specfile.
"""
# Fetch all sources defined in packit.yaml -> sources
for source in self.package_config.sources:
source_path = self.specfile.sourcedir.joinpath(source.path)
if not source_path.is_file():
logger.info(f"Downloading source {source.path!r}.")
DownloadHelper.download_file(
source.url,
str(source_path),
)
# Try to download sources defined in "sources" file from Fedora lookaside cache
try:
LookasideCacheHelper.download(
"fedpkg",
self.specfile.path.parent,
self.specfile.expanded_name,
self.specfile.sourcedir,
)
except LookasideCacheError as e:
logger.debug(
f"Downloading sources from lookaside cache failed: {e}.")
# Fetch all remote sources defined in the spec file
with self.specfile.sources() as sources, self.specfile.patches(
) as patches:
for source in sources + patches:
if source.remote:
source_path = self.specfile.sourcedir.joinpath(
source.expanded_filename)
if not source_path.is_file():
logger.info(f"Downloading source {source.filename!r}.")
DownloadHelper.download_file(
source.expanded_location,
str(source_path),
)
def __init__(self, cli_conf, execution_dir, results_dir, debug_log_file):
"""
Initialize the application
:param cli_conf: CLI object with configuration gathered from commandline
:return:
"""
results_store.clear()
self.conf = cli_conf
self.execution_dir = execution_dir
self.rebased_sources_dir = os.path.join(results_dir, 'rebased-sources')
self.debug_log_file = debug_log_file
# Temporary workspace for Builder, checks, ...
self.kwargs['workspace_dir'] = self.workspace_dir = os.path.join(self.execution_dir, constants.WORKSPACE_DIR)
# Directory where results should be put
self.kwargs['results_dir'] = self.results_dir = results_dir
# Directory contaning only those files, which are relevant for the new rebased version
self.kwargs['rebased_sources_dir'] = self.rebased_sources_dir
self.kwargs['non_interactive'] = self.conf.non_interactive
self.kwargs['favor_on_conflict'] = self.conf.favor_on_conflict
self.kwargs['changelog_entry'] = self.conf.changelog_entry
self.kwargs['spec_hook_blacklist'] = self.conf.spec_hook_blacklist
self.kwargs['build_log_hook_blacklist'] = self.conf.build_log_hook_blacklist
self.kwargs['force_build_log_hooks'] = self.conf.force_build_log_hooks
logger.verbose("Rebase-helper version: %s", VERSION)
if self.conf.build_tasks is None:
# check the workspace dir
if not self.conf.cont:
self._check_workspace_dir()
self._get_spec_file()
self._prepare_spec_objects()
# verify all sources for the new version are present
missing_sources = [os.path.basename(s) for s in self.rebase_spec_file.sources
if not os.path.isfile(os.path.basename(s))]
if missing_sources:
raise RebaseHelperError('The following sources are missing: {}'.format(','.join(missing_sources)))
if self.conf.update_sources:
sources = [os.path.basename(s) for s in self.spec_file.sources]
rebased_sources = [os.path.basename(s) for s in self.rebase_spec_file.sources]
uploaded = LookasideCacheHelper.update_sources('fedpkg', self.rebased_sources_dir,
self.rebase_spec_file.get_package_name(),
sources, rebased_sources,
upload=not self.conf.skip_upload)
self._update_gitignore(uploaded, self.rebased_sources_dir)
# TODO: Remove the value from kwargs and use only CLI attribute!
self.kwargs['continue'] = self.conf.cont
self._initialize_data()
if self.conf.cont or self.conf.build_only:
self._delete_old_builds()
def __init__(self,
cli_conf: Config,
start_dir: str,
execution_dir: str,
results_dir: str,
create_logs: bool = True) -> None:
"""Initializes the application.
Args:
cli_conf: Application configuration.
start_dir: Directory where rebase-helper was started.
execution_dir: Working directory.
results_dir: Location of rebase results.
create_logs: Whether to create default logging file handlers.
"""
results_store.clear()
# Initialize instance attributes
self.old_sources = ''
self.new_sources = ''
self.old_rest_sources: List[str] = []
self.new_rest_sources: List[str] = []
self.rebased_patches: Dict[str, List[str]] = {}
self.rebased_repo: Optional[git.Repo] = None
self.handlers = LoggerHelper.create_file_handlers(
results_dir) if create_logs else []
self.conf = cli_conf
self.start_dir = start_dir
self.execution_dir = execution_dir
self.rebased_sources_dir = os.path.join(results_dir, 'rebased-sources')
self.kwargs: Dict[str, Any] = {}
self.kwargs.update(self.conf.config)
# Temporary workspace for Builder, checks, ...
workspace_location = os.path.abspath(
cli_conf.workspace_dir
) if cli_conf.workspace_dir else self.execution_dir
self.kwargs['workspace_dir'] = self.workspace_dir = os.path.join(
workspace_location, constants.WORKSPACE_DIR)
# Directory where results should be put
self.kwargs['results_dir'] = self.results_dir = results_dir
# Directory contaning only those files, which are relevant for the new rebased version
self.kwargs['rebased_sources_dir'] = self.rebased_sources_dir
self.spec_file_path = self._find_spec_file()
self._prepare_spec_objects()
if self.conf.build_tasks is None:
# check the workspace dir
self._check_workspace_dir()
# verify all sources for the new version are present
missing_sources = [
os.path.basename(s) for s in self.rebase_spec_file.sources
if not os.path.isfile(os.path.basename(s))
]
if missing_sources:
raise RebaseHelperError(
'The following sources are missing: {}'.format(
','.join(missing_sources)))
if self.conf.update_sources:
sources = [os.path.basename(s) for s in self.spec_file.sources]
rebased_sources = [
os.path.basename(s) for s in self.rebase_spec_file.sources
]
uploaded = LookasideCacheHelper.update_sources(
'fedpkg',
self.rebased_sources_dir,
self.rebase_spec_file.header.name,
sources,
rebased_sources,
upload=not self.conf.skip_upload)
self._update_gitignore(uploaded, self.rebased_sources_dir)
self._initialize_data()