def extract_archive(self, path=None): """ Extracts the archive into the given path :param path: Path where to extract the archive to. :return: """ if path is None: TypeError("Expected argument 'path' (pos 1) is missing") logger.verbose("Extracting '%s' into '%s'", self._filename, path) try: LZMAError = lzma.LZMAError except AttributeError: LZMAError = lzma.error try: archive = self._archive_type.open(self._filename) except (tarfile.ReadError, LZMAError) as e: raise IOError(six.text_type(e)) self._archive_type.extract(archive, self._filename, path) try: archive.close() except AttributeError: # pseudo archive types don't return real file-like object pass
def build(cls, spec, results_dir, srpm, **kwargs): """ Builds the RPMs using rpmbuild :param spec: SpecFile object :param results_dir: absolute path to DIR where results should be stored :param srpm: absolute path to SRPM :return: dict with: 'rpm' -> list with absolute paths to RPMs 'logs' -> list with absolute paths to build_logs """ cls.logs = [] rpm_results_dir = os.path.join(results_dir, "RPM") sources = spec.get_sources() patches = [p.get_path() for p in spec.get_patches()] with RpmbuildTemporaryEnvironment(sources, patches, spec.get_path(), rpm_results_dir) as tmp_env: env = tmp_env.env() tmp_dir = tmp_env.path() tmp_results_dir = env.get(RpmbuildTemporaryEnvironment.TEMPDIR_RESULTS) rpms = cls._build_rpm(srpm, tmp_dir, tmp_results_dir, rpm_results_dir, builder_options=cls.get_builder_options(**kwargs)) logger.info("Building RPMs finished successfully") # RPMs paths in results_dir rpms = [os.path.join(rpm_results_dir, os.path.basename(f)) for f in rpms] logger.verbose("Successfully built RPMs: '%s'", str(rpms)) # gather logs cls.logs.extend([l for l in PathHelper.find_all_files(rpm_results_dir, '*.log')]) logger.verbose("logs: '%s'", str(cls.logs)) return dict(rpm=rpms, logs=cls.logs)
def build(cls, spec, results_dir, **kwargs): """ Build SRPM with chosen SRPM Build Tool :param spec: SpecFile object :param results_dir: absolute path to DIR where results should be stored :return: absolute path to SRPM, list with absolute paths to logs """ srpm_results_dir = os.path.join(results_dir, "SRPM") sources = spec.get_sources() patches = [p.get_path() for p in spec.get_patches()] with RpmbuildTemporaryEnvironment(sources, patches, spec.get_path(), srpm_results_dir) as tmp_env: srpm_builder_options = cls.get_srpm_builder_options(**kwargs) env = tmp_env.env() tmp_dir = tmp_env.path() tmp_spec = env.get(RpmbuildTemporaryEnvironment.TEMPDIR_SPEC) tmp_results_dir = env.get( RpmbuildTemporaryEnvironment.TEMPDIR_RESULTS) srpm = cls._build_srpm(tmp_spec, tmp_dir, tmp_results_dir, srpm_results_dir, srpm_builder_options=srpm_builder_options) logger.info("Building SRPM finished successfully") # srpm path in results_dir srpm = os.path.join(srpm_results_dir, os.path.basename(srpm)) logger.verbose("Successfully built SRPM: '%s'", str(srpm)) # gather logs logs = [l for l in PathHelper.find_all_files(srpm_results_dir, '*.log')] logger.verbose("logs: '%s'", str(logs)) return dict(srpm=srpm, logs=logs)
def build(cls, spec, results_dir, srpm, **kwargs): """ Builds the RPMs using rpmbuild :param spec: SpecFile object :param results_dir: absolute path to DIR where results should be stored :param srpm: absolute path to SRPM :return: dict with: 'rpm' -> list with absolute paths to RPMs 'logs' -> list with absolute paths to build_logs """ cls.logs = [] rpm_results_dir = os.path.join(results_dir, "RPM") sources = spec.get_sources() patches = [p.get_path() for p in spec.get_patches()] with RpmbuildTemporaryEnvironment(sources, patches, spec.get_path(), rpm_results_dir) as tmp_env: env = tmp_env.env() tmp_dir = tmp_env.path() tmp_results_dir = env.get(RpmbuildTemporaryEnvironment.TEMPDIR_RESULTS) rpms = cls._build_rpm(srpm, tmp_dir, tmp_results_dir, rpm_results_dir, builder_options=cls.get_builder_options(**kwargs)) logger.info("Building RPMs finished successfully") # RPMs paths in results_dir rpms = [os.path.join(rpm_results_dir, os.path.basename(f)) for f in rpms] logger.verbose("Successfully built RPMs: '%s'", str(rpms)) # gather logs cls.logs.extend(l for l in PathHelper.find_all_files(rpm_results_dir, '*.log')) logger.verbose("logs: '%s'", str(cls.logs)) return dict(rpm=rpms, logs=cls.logs)
def build(cls, spec, results_dir, **kwargs): """ Build SRPM with chosen SRPM Build Tool :param spec: SpecFile object :param results_dir: absolute path to DIR where results should be stored :return: absolute path to SRPM, list with absolute paths to logs """ srpm_results_dir = os.path.join(results_dir, "SRPM") sources = spec.get_sources() patches = [p.get_path() for p in spec.get_patches()] with MockTemporaryEnvironment(sources, patches, spec.get_path(), srpm_results_dir) as tmp_env: srpm_builder_options = cls.get_srpm_builder_options(**kwargs) env = tmp_env.env() tmp_dir = tmp_env.path() tmp_spec = env.get(MockTemporaryEnvironment.TEMPDIR_SPEC) tmp_results_dir = env.get( MockTemporaryEnvironment.TEMPDIR_RESULTS) srpm = cls._build_srpm(tmp_spec, tmp_dir, tmp_results_dir, srpm_results_dir, srpm_builder_options=srpm_builder_options) logger.info("Building SRPM finished successfully") # srpm path in results_dir srpm = os.path.join(srpm_results_dir, os.path.basename(srpm)) logger.verbose("Successfully built SRPM: '%s'", str(srpm)) # gather logs logs = [l for l in PathHelper.find_all_files(srpm_results_dir, '*.log')] logger.verbose("logs: '%s'", str(logs)) return dict(srpm=srpm, logs=logs)
def _prepare_spec_objects(self): """ Prepare spec files and initialize objects :return: """ self.rebase_spec_file_path = get_rebase_name(self.rebased_sources_dir, self.spec_file_path) self.spec_file = SpecFile(self.spec_file_path, self.conf.changelog_entry, self.execution_dir, download=not self.conf.not_download_sources) # Check whether test suite is enabled at build time if not self.spec_file.is_test_suite_enabled(): results_store.set_info_text('WARNING', 'Test suite is not enabled at build time.') # create an object representing the rebased SPEC file self.rebase_spec_file = self.spec_file.copy(self.rebase_spec_file_path) if not self.conf.sources: self.conf.sources = versioneers_runner.run(self.conf.versioneer, self.spec_file.get_package_name(), self.spec_file.category, self.conf.versioneer_blacklist) if self.conf.sources: logger.info("Determined latest upstream version '%s'", self.conf.sources) else: raise RebaseHelperError('Could not determine latest upstream version ' 'and no SOURCES argument specified!') # Prepare rebased_sources_dir self.rebased_repo = self._prepare_rebased_repository(self.spec_file.patches, self.execution_dir, self.rebased_sources_dir) # check if argument passed as new source is a file or just a version if [True for ext in Archive.get_supported_archives() if self.conf.sources.endswith(ext)]: logger.verbose("argument passed as a new source is a file") self.rebase_spec_file.set_version_using_archive(self.conf.sources) else: logger.verbose("argument passed as a new source is a version") version, extra_version, separator = SpecFile.split_version_string(self.conf.sources) self.rebase_spec_file.set_version(version) self.rebase_spec_file.set_extra_version_separator(separator) self.rebase_spec_file.set_extra_version(extra_version) if not self.conf.skip_version_check and parse_version(self.rebase_spec_file.get_version()) \ <= parse_version(self.spec_file.get_version()): raise RebaseHelperError("Current version is equal to or newer than the requested version, nothing to do.") # run spec hooks spec_hooks_runner.run_spec_hooks(self.spec_file, self.rebase_spec_file, **self.kwargs) # spec file object has been sanitized downloading can proceed for spec_file in [self.spec_file, self.rebase_spec_file]: if spec_file.download: spec_file.download_remote_sources() # parse spec again with sources downloaded to properly expand %prep section spec_file._update_data() # pylint: disable=protected-access
def _delete_workspace_dir(self): """ Deletes workspace directory and loggs message :return: """ logger.verbose("Removing the workspace directory '%s'", self.workspace_dir) if os.path.isdir(self.workspace_dir): shutil.rmtree(self.workspace_dir)
def _delete_workspace_dir(self): """ Deletes workspace directory and loggs message :return: """ logger.verbose("Removing the workspace directory '%s'", self.workspace_dir) if os.path.isdir(self.workspace_dir): shutil.rmtree(self.workspace_dir)
def _prepare_spec_objects(self): """ Prepare spec files and initialize objects :return: """ self.rebase_spec_file_path = get_rebase_name(self.rebased_sources_dir, self.spec_file_path) self.spec_file = SpecFile(self.spec_file_path, self.execution_dir) # Check whether test suite is enabled at build time if not self.spec_file.is_test_suite_enabled(): results_store.set_info_text('WARNING', 'Test suite is not enabled at build time.') # create an object representing the rebased SPEC file self.rebase_spec_file = self.spec_file.copy(self.rebase_spec_file_path) if not self.conf.sources: self.conf.sources = plugin_manager.versioneers.run(self.conf.versioneer, self.spec_file.get_package_name(), self.spec_file.category, self.conf.versioneer_blacklist) if self.conf.sources: logger.info("Determined latest upstream version '%s'", self.conf.sources) else: raise RebaseHelperError('Could not determine latest upstream version ' 'and no SOURCES argument specified!') # Prepare rebased_sources_dir self.rebased_repo = self._prepare_rebased_repository(self.spec_file.patches, self.execution_dir, self.rebased_sources_dir) # check if argument passed as new source is a file or just a version if [True for ext in Archive.get_supported_archives() if self.conf.sources.endswith(ext)]: logger.verbose("argument passed as a new source is a file") self.rebase_spec_file.set_version_using_archive(self.conf.sources) else: logger.verbose("argument passed as a new source is a version") version, extra_version, separator = SpecFile.split_version_string(self.conf.sources) self.rebase_spec_file.set_version(version) self.rebase_spec_file.set_extra_version_separator(separator) self.rebase_spec_file.set_extra_version(extra_version) if not self.conf.skip_version_check and parse_version(self.rebase_spec_file.get_version()) \ <= parse_version(self.spec_file.get_version()): raise RebaseHelperError("Current version is equal to or newer than the requested version, nothing to do.") self.rebase_spec_file.update_changelog(self.conf.changelog_entry) # run spec hooks plugin_manager.spec_hooks.run(self.spec_file, self.rebase_spec_file, **self.kwargs) # spec file object has been sanitized downloading can proceed if not self.conf.not_download_sources: for spec_file in [self.spec_file, self.rebase_spec_file]: spec_file.download_remote_sources() # parse spec again with sources downloaded to properly expand %prep section spec_file._update_data() # pylint: disable=protected-access
def download_file(url, destination_path, blocksize=8192): """Downloads a file from HTTP, HTTPS or FTP URL. Args: url (str): URL to be downloaded. destination_path (str): Path to where the downloaded file will be stored. blocksize (int): Block size in bytes. """ r = DownloadHelper.request(url, stream=True) if r is None: raise DownloadError( "An unexpected error occurred during the download.") if not 200 <= r.status_code < 300: raise DownloadError(r.reason) file_size = int(r.headers.get('content-length', -1)) # file exists, check the size if os.path.exists(destination_path): if file_size < 0 or file_size != os.path.getsize(destination_path): logger.verbose( "The destination file '%s' exists, but sizes don't match! Removing it.", destination_path) os.remove(destination_path) else: logger.verbose( "The destination file '%s' exists, and the size is correct! Skipping download.", destination_path) return try: with open(destination_path, 'wb') as local_file: logger.info('Downloading file from URL %s', url) download_start = time.time() downloaded = 0 # report progress DownloadHelper.progress(file_size, downloaded, download_start) # do the actual download for chunk in r.iter_content(chunk_size=blocksize): downloaded += len(chunk) local_file.write(chunk) # report progress DownloadHelper.progress(file_size, downloaded, download_start) sys.stdout.write('\n') sys.stdout.flush() except KeyboardInterrupt as e: os.remove(destination_path) raise e
def __init__(self, cli_conf, execution_dir, results_dir, debug_log_file): """ Initialize the application :param cli_conf: CLI object with configuration gathered from commandline :return: """ results_store.clear() self.conf = cli_conf self.execution_dir = execution_dir self.rebased_sources_dir = os.path.join(results_dir, 'rebased-sources') self.debug_log_file = debug_log_file self.kwargs.update(self.conf.config) # Temporary workspace for Builder, checks, ... self.kwargs['workspace_dir'] = self.workspace_dir = os.path.join(self.execution_dir, constants.WORKSPACE_DIR) # Directory where results should be put self.kwargs['results_dir'] = self.results_dir = results_dir # Directory contaning only those files, which are relevant for the new rebased version self.kwargs['rebased_sources_dir'] = self.rebased_sources_dir logger.verbose("Rebase-helper version: %s", VERSION) if self.conf.build_tasks is None: # check the workspace dir if not self.conf.cont: self._check_workspace_dir() self._get_spec_file() self._prepare_spec_objects() # verify all sources for the new version are present missing_sources = [os.path.basename(s) for s in self.rebase_spec_file.sources if not os.path.isfile(os.path.basename(s))] if missing_sources: raise RebaseHelperError('The following sources are missing: {}'.format(','.join(missing_sources))) if self.conf.update_sources: sources = [os.path.basename(s) for s in self.spec_file.sources] rebased_sources = [os.path.basename(s) for s in self.rebase_spec_file.sources] uploaded = LookasideCacheHelper.update_sources('fedpkg', self.rebased_sources_dir, self.rebase_spec_file.get_package_name(), sources, rebased_sources, upload=not self.conf.skip_upload) self._update_gitignore(uploaded, self.rebased_sources_dir) # TODO: Remove the value from kwargs and use only CLI attribute! self.kwargs['continue'] = self.conf.cont self._initialize_data() if self.conf.cont or self.conf.build_only: self._delete_old_builds()
def run_check(cls, results_dir, **kwargs): """Compares old and new RPMs using abipkgdiff""" # Check if ABI changes occured cls.abi_changes = None cls.results_dir = os.path.join(results_dir, cls.name) os.makedirs(cls.results_dir) debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff( results_store.get_build('old')) debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff( results_store.get_build('new')) cmd = [cls.CMD] reports = {} for pkg in rest_pkgs_old: command = list(cmd) debug = cls._find_debuginfo(debug_old, pkg) if debug: command.append('--d1') command.append(debug) old_name = RpmHelper.split_nevra(os.path.basename(pkg))['name'] find = [ x for x in rest_pkgs_new if RpmHelper.split_nevra( os.path.basename(x))['name'] == old_name ] if not find: logger.warning('New version of package %s was not found!', old_name) continue new_pkg = find[0] debug = cls._find_debuginfo(debug_new, new_pkg) if debug: command.append('--d2') command.append(debug) command.append(pkg) command.append(new_pkg) logger.verbose('Package name for ABI comparison %s', old_name) output = os.path.join(cls.results_dir, old_name + '.txt') try: ret_code = ProcessHelper.run_subprocess(command, output_file=output) except OSError: raise CheckerNotFoundError( "Checker '{}' was not found or installed.".format( cls.name)) if int(ret_code) & cls.ABIDIFF_ERROR and int( ret_code) & cls.ABIDIFF_USAGE_ERROR: raise RebaseHelperError( 'Execution of {} failed.\nCommand line is: {}'.format( cls.CMD, cmd)) reports[old_name] = int(ret_code) return dict(packages=cls.parse_abi_logs(reports), abi_changes=cls.abi_changes, path=cls.get_checker_output_dir_short())
def download_file(url, destination_path, blocksize=8192): """Downloads a file from HTTP, HTTPS or FTP URL. Args: url (str): URL to be downloaded. destination_path (str): Path to where the downloaded file will be stored. blocksize (int): Block size in bytes. """ r = DownloadHelper.request(url, stream=True) if r is None: raise DownloadError("An unexpected error occurred during the download.") if not 200 <= r.status_code < 300: raise DownloadError(r.reason) file_size = int(r.headers.get('content-length', -1)) # file exists, check the size if os.path.exists(destination_path): if file_size < 0 or file_size != os.path.getsize(destination_path): logger.verbose("The destination file '%s' exists, but sizes don't match! Removing it.", destination_path) os.remove(destination_path) else: logger.verbose("The destination file '%s' exists, and the size is correct! Skipping download.", destination_path) return try: with open(destination_path, 'wb') as local_file: logger.info('Downloading file from URL %s', url) download_start = time.time() downloaded = 0 # report progress DownloadHelper.progress(file_size, downloaded, download_start) # do the actual download for chunk in r.iter_content(chunk_size=blocksize): downloaded += len(chunk) local_file.write(chunk) # report progress DownloadHelper.progress(file_size, downloaded, download_start) sys.stdout.write('\n') sys.stdout.flush() except KeyboardInterrupt as e: os.remove(destination_path) raise e
def patch(self, old_dir, new_dir, rest_sources, patches, **kwargs): """ Apply patches and generate rebased patches if needed :param old_dir: path to dir with old patches :param new_dir: path to dir with new patches :param patches: old patches :param rebased_patches: rebased patches :param kwargs: -- :return: """ logger.verbose("Patching source by patch tool %s", self._patch_tool_name) return self._tool.run_patch(old_dir, new_dir, rest_sources, patches, **kwargs)
def fill_dictionary(cls, result_dir, old_version=None, new_version=None): """ Parsed files.xml and symbols.xml and fill dictionary :param result_dir: where should be stored file for pkgdiff :param old_version: old version of package :param new_version: new version of package :return: """ XML_FILES = ['files.xml', 'symbols.xml'] if old_version is None: old_version = results_store.get_old_build().get('version') if old_version is '': old_version = cls._get_rpm_info( 'version', results_store.get_old_build()['rpm']) if new_version is None: new_version = results_store.get_new_build().get('version') if new_version is '': new_version = cls._get_rpm_info( 'version', results_store.get_new_build()['rpm']) for tag in cls.CHECKER_TAGS: cls.results_dict[tag] = [] for file_name in [os.path.join(result_dir, x) for x in XML_FILES]: logger.verbose('Processing %s file.', file_name) try: with open(file_name, "r") as f: lines = ['<pkgdiff>'] lines.extend(f.readlines()) lines.append('</pkgdiff>') pkgdiff_tree = ElementTree.fromstringlist(lines) for tag in cls.CHECKER_TAGS: for pkgdiff in pkgdiff_tree.findall('.//' + tag): files = [ x.strip() for x in pkgdiff.text.strip().split('\n') ] files = [ x.replace(old_version, '*') for x in files ] files = [ x.replace(new_version, '*') for x in files ] cls.results_dict[tag].extend(files) except IOError: continue
def parse_spec(cls, path, flags=None): with open(path, 'rb') as orig: with tempfile.NamedTemporaryFile() as tmp: # remove BuildArch to workaround rpm bug tmp.write(b''.join([ l for l in orig.readlines() if not l.startswith(b'BuildArch') ])) tmp.flush() with ConsoleHelper.Capturer(stderr=True) as capturer: result = rpm.spec( tmp.name, flags) if flags is not None else rpm.spec( tmp.name) for line in capturer.stderr.split('\n'): if line: logger.verbose('rpm: %s', line) return result
def build(cls, spec, results_dir, srpm, **kwargs): """ Builds the RPMs using mock :param spec: SpecFile object :param results_dir: absolute path to directory where results will be stored :param srpm: absolute path to SRPM :param root: mock root used for building :param arch: architecture to build the RPM for :return: dict with: 'rpm' -> list with absolute paths to RPMs 'logs' -> list with absolute paths to logs """ cls.logs = [] rpm_results_dir = os.path.join(results_dir, "RPM") sources = spec.get_sources() patches = [p.get_path() for p in spec.get_patches()] with MockTemporaryEnvironment(sources, patches, spec.get_path(), rpm_results_dir) as tmp_env: env = tmp_env.env() tmp_results_dir = env.get(MockTemporaryEnvironment.TEMPDIR_RESULTS) rpms = cls._build_rpm( srpm, tmp_results_dir, rpm_results_dir, builder_options=cls.get_builder_options(**kwargs)) # remove SRPM - side product of building RPM tmp_srpm = PathHelper.find_first_file(tmp_results_dir, "*.src.rpm") if tmp_srpm is not None: os.unlink(tmp_srpm) logger.info("Building RPMs finished successfully") rpms = [ os.path.join(rpm_results_dir, os.path.basename(f)) for f in rpms ] logger.verbose("Successfully built RPMs: '%s'", str(rpms)) # gather logs cls.logs.extend( [l for l in PathHelper.find_all_files(rpm_results_dir, '*.log')]) logger.verbose("logs: '%s'", str(cls.logs)) return dict(rpm=rpms, logs=cls.logs)
def _analyze_logs(cls, output, results_dict): removed_things = ['.build-id', '.dwz', 'PROVIDE', 'REQUIRES'] for line in output: if [x for x in removed_things if x in line]: continue fields = line.strip().split() logger.verbose(fields) if line.startswith('removed'): results_dict['removed'].append(fields[1]) continue if line.startswith('added'): results_dict['added'].append(fields[1]) continue if re.match(r'(S..|..5)........', fields[0]): # size or checksum changed results_dict['changed'].append(fields[1]) return results_dict
def run_check(cls, results_dir, **kwargs): """Compares old and new RPMs using abipkgdiff""" # Check if ABI changes occured cls.abi_changes = False cls.results_dir = os.path.join(results_dir, cls.name) os.makedirs(cls.results_dir) debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(results_store.get_build('old')) debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(results_store.get_build('new')) cmd = [cls.CMD] reports = {} for pkg in rest_pkgs_old: command = list(cmd) debug = cls._find_debuginfo(debug_old, pkg) if debug: command.append('--d1') command.append(debug) old_name = RpmHelper.split_nevra(os.path.basename(pkg))['name'] find = [x for x in rest_pkgs_new if RpmHelper.split_nevra(os.path.basename(x))['name'] == old_name] if not find: logger.warning('New version of package %s was not found!', old_name) continue new_pkg = find[0] debug = cls._find_debuginfo(debug_new, new_pkg) if debug: command.append('--d2') command.append(debug) command.append(pkg) command.append(new_pkg) logger.verbose('Package name for ABI comparison %s', old_name) output = os.path.join(cls.results_dir, old_name + '.txt') try: ret_code = ProcessHelper.run_subprocess(command, output_file=output) except OSError: raise CheckerNotFoundError("Checker '{}' was not found or installed.".format(cls.name)) if int(ret_code) & cls.ABIDIFF_ERROR and int(ret_code) & cls.ABIDIFF_USAGE_ERROR: raise RebaseHelperError('Execution of {} failed.\nCommand line is: {}'.format(cls.CMD, cmd)) reports[old_name] = int(ret_code) return dict(packages=cls.parse_abi_logs(reports), abi_changes=cls.abi_changes, path=cls.get_checker_output_dir_short())
def apply_patch(cls, repo, patch_object): """ Function applies patches to old sources It tries apply patch with am command and if it fails then with command --apply """ logger.verbose('Applying patch with git-am') patch_name = patch_object.get_path() patch_strip = patch_object.get_strip() try: repo.git.am(patch_name) commit = repo.head.commit except git.GitCommandError: logger.verbose('Applying patch with git-am failed.') try: repo.git.am(abort=True) except git.GitCommandError: pass logger.verbose('Applying patch with git-apply') try: repo.git.apply(patch_name, p=patch_strip) except git.GitCommandError: repo.git.apply(patch_name, p=patch_strip, reject=True, whitespace='fix') repo.git.add(all=True) commit = repo.index.commit(cls.decorate_patch_name( os.path.basename(patch_name)), skip_hooks=True) repo.git.commit(amend=True, m=cls.insert_patch_name(commit.message, os.path.basename(patch_name)))
def extract_archive(self, path=None): """ Extracts the archive into the given path :param path: Path where to extract the archive to. :return: """ if path is None: TypeError("Expected argument 'path' (pos 1) is missing") logger.verbose("Extracting '%s' into '%s'", self._filename, path) try: archive = self._archive_type.open(self._filename) except (EOFError, tarfile.ReadError, lzma.LZMAError) as e: raise IOError(str(e)) self._archive_type.extract(archive, self._filename, path) try: archive.close() except AttributeError: # pseudo archive types don't return real file-like object pass
def build(cls, spec, results_dir, srpm, **kwargs): """ Builds the RPMs using mock :param spec: SpecFile object :param results_dir: absolute path to directory where results will be stored :param srpm: absolute path to SRPM :param root: mock root used for building :param arch: architecture to build the RPM for :return: dict with: 'rpm' -> list with absolute paths to RPMs 'logs' -> list with absolute paths to logs """ cls.logs = [] rpm_results_dir = os.path.join(results_dir, "RPM") sources = spec.get_sources() patches = [p.get_path() for p in spec.get_patches()] with MockTemporaryEnvironment(sources, patches, spec.get_path(), rpm_results_dir) as tmp_env: env = tmp_env.env() tmp_results_dir = env.get(MockTemporaryEnvironment.TEMPDIR_RESULTS) rpms = cls._build_rpm(srpm, tmp_results_dir, rpm_results_dir, builder_options=cls.get_builder_options(**kwargs)) # remove SRPM - side product of building RPM tmp_srpm = PathHelper.find_first_file(tmp_results_dir, "*.src.rpm") if tmp_srpm is not None: os.unlink(tmp_srpm) logger.info("Building RPMs finished successfully") rpms = [os.path.join(rpm_results_dir, os.path.basename(f)) for f in rpms] logger.verbose("Successfully built RPMs: '%s'", str(rpms)) # gather logs cls.logs.extend(l for l in PathHelper.find_all_files(rpm_results_dir, '*.log')) logger.verbose("logs: '%s'", str(cls.logs)) return dict(rpm=rpms, logs=cls.logs)
def fill_dictionary(cls, result_dir, old_version=None, new_version=None): """ Parsed files.xml and symbols.xml and fill dictionary :param result_dir: where should be stored file for pkgdiff :param old_version: old version of package :param new_version: new version of package :return: """ XML_FILES = ['files.xml', 'symbols.xml'] if old_version is None: old_version = results_store.get_old_build().get('version') if old_version is '': old_version = cls._get_rpm_info('version', results_store.get_old_build()['rpm']) if new_version is None: new_version = results_store.get_new_build().get('version') if new_version is '': new_version = cls._get_rpm_info('version', results_store.get_new_build()['rpm']) for tag in cls.CHECKER_TAGS: cls.results_dict[tag] = [] for file_name in [os.path.join(result_dir, x) for x in XML_FILES]: logger.verbose('Processing %s file.', file_name) try: with open(file_name, "r") as f: lines = ['<pkgdiff>'] lines.extend(f.readlines()) lines.append('</pkgdiff>') pkgdiff_tree = ElementTree.fromstringlist(lines) for tag in cls.CHECKER_TAGS: for pkgdiff in pkgdiff_tree.findall('.//' + tag): files = [x.strip() for x in pkgdiff.text.strip().split('\n')] files = [x.replace(old_version, '*') for x in files] files = [x.replace(new_version, '*') for x in files] cls.results_dict[tag].extend(files) except IOError: continue
def _git_rebase(cls): """Function performs git rebase between old and new sources""" def compare_commits(a, b): # compare commit diffs disregarding differences in blob hashes attributes = ('a_mode', 'b_mode', 'a_rawpath', 'b_rawpath', 'new_file', 'deleted_file', 'raw_rename_from', 'raw_rename_to', 'diff', 'change_type', 'score') diff1 = a.diff(a.parents[0], create_patch=True) diff2 = b.diff(b.parents[0], create_patch=True) if len(diff1) != len(diff2): return False for d1, d2 in zip(diff1, diff2): for attr in attributes: if getattr(d1, attr) != getattr(d2, attr): return False return True # in old_sources do: # 1) git remote add new_sources <path_to_new_sources> # 2) git fetch new_sources # 3) git rebase --onto new_sources/master <root_commit_old_sources> <last_commit_old_sources> if not cls.cont: logger.info('git-rebase operation to %s is ongoing...', os.path.basename(cls.new_sources)) upstream = 'new_upstream' cls.old_repo.create_remote(upstream, url=cls.new_sources).fetch() root_commit = cls.old_repo.git.rev_list('HEAD', max_parents=0) last_commit = cls.old_repo.commit('HEAD') if cls.favor_on_conflict == 'upstream': strategy_option = 'ours' elif cls.favor_on_conflict == 'downstream': strategy_option = 'theirs' else: strategy_option = False try: cls.output_data = cls.old_repo.git.rebase( root_commit, last_commit, strategy_option=strategy_option, onto='{}/master'.format(upstream), stdout_as_string=six.PY3) except git.GitCommandError as e: ret_code = e.status cls.output_data = e.stdout else: ret_code = 0 else: logger.info('git-rebase operation continues...') try: cls.output_data = cls.old_repo.git.rebase( '--continue', stdout_as_string=six.PY3) except git.GitCommandError as e: ret_code = e.status cls.output_data = e.stdout else: ret_code = 0 logger.verbose(cls.output_data) patch_dictionary = {} modified_patches = [] inapplicable_patches = [] while ret_code != 0: if not cls.old_repo.index.unmerged_blobs( ) and not cls.old_repo.index.diff(cls.old_repo.commit()): # empty commit - conflict has been automatically resolved - skip try: cls.output_data = cls.old_repo.git.rebase( skip=True, stdout_as_string=six.PY3) except git.GitCommandError as e: ret_code = e.status cls.output_data = e.stdout continue else: break try: with open( os.path.join(cls.old_sources, '.git', 'rebase-apply', 'next')) as f: next_index = int(f.readline()) with open( os.path.join(cls.old_sources, '.git', 'rebase-apply', 'last')) as f: last_index = int(f.readline()) except (FileNotFoundError, IOError): raise RuntimeError( 'Git rebase failed with unknown reason. Please check log file' ) patch_name = cls.patches[next_index - 1].get_patch_name() inapplicable = False if cls.non_interactive: inapplicable = True else: logger.info('Failed to auto-merge patch %s', patch_name) unmerged = cls.old_repo.index.unmerged_blobs() GitHelper.run_mergetool(cls.old_repo) if cls.old_repo.index.unmerged_blobs(): if InputHelper.get_message( 'There are still unmerged entries. Do you want to skip this patch', default_yes=False): inapplicable = True else: continue if not inapplicable: # check for unresolved conflicts unresolved = [] for file in unmerged: with open(os.path.join(cls.old_sources, file)) as f: if [l for l in f.readlines() if '<<<<<<<' in l]: unresolved.append(file) if unresolved: if InputHelper.get_message( 'There are still unresolved conflicts. ' 'Do you want to skip this patch', default_yes=False): inapplicable = True else: cls.old_repo.index.reset(paths=unresolved) unresolved.insert(0, '--') cls.old_repo.git.checkout(*unresolved, conflict='diff3') continue if inapplicable: inapplicable_patches.append(patch_name) try: cls.output_data = cls.old_repo.git.rebase( skip=True, stdout_as_string=six.PY3) except git.GitCommandError as e: ret_code = e.status cls.output_data = e.stdout continue else: break diff = cls.old_repo.index.diff(cls.old_repo.commit()) if diff: modified_patches.append(patch_name) if next_index < last_index: if not InputHelper.get_message( 'Do you want to continue with another patch'): raise KeyboardInterrupt try: if diff: cls.output_data = cls.old_repo.git.rebase( '--continue', stdout_as_string=six.PY3) else: cls.output_data = cls.old_repo.git.rebase( skip=True, stdout_as_string=six.PY3) except git.GitCommandError as e: ret_code = e.status cls.output_data = e.stdout else: break original_commits = list( cls.old_repo.iter_commits(rev=cls.old_repo.branches.master)) commits = list(cls.old_repo.iter_commits()) untouched_patches = [] deleted_patches = [] for patch in cls.patches: patch_name = patch.get_patch_name() original_commit = [ c for c in original_commits if cls.extract_patch_name(c.message) == patch_name ] commit = [ c for c in commits if cls.extract_patch_name(c.message) == patch_name ] if original_commit and commit: if patch_name not in modified_patches and compare_commits( original_commit[0], commit[0]): untouched_patches.append(patch_name) else: base_name = os.path.join(cls.kwargs['rebased_sources_dir'], patch_name) if commit[0].summary == cls.decorate_patch_name( patch_name): diff = cls.old_repo.git.diff(commit[0].parents[0], commit[0], stdout_as_string=False) else: diff = cls.old_repo.git.format_patch( commit[0], '-1', stdout=True, no_numbered=True, no_attach=True, stdout_as_string=False) diff = cls.strip_patch_name(diff, patch_name) with open(base_name, 'wb') as f: f.write(diff) f.write(b'\n') if patch_name not in modified_patches: modified_patches.append(patch_name) elif patch_name not in inapplicable_patches: deleted_patches.append(patch_name) if deleted_patches: patch_dictionary['deleted'] = deleted_patches if modified_patches: patch_dictionary['modified'] = modified_patches if inapplicable_patches: patch_dictionary['inapplicable'] = inapplicable_patches if untouched_patches: patch_dictionary['untouched'] = untouched_patches return patch_dictionary
def __init__(self, cli_conf, execution_dir, results_dir, debug_log_file): """ Initialize the application :param cli_conf: CLI object with configuration gathered from commandline :return: """ results_store.clear() self.conf = cli_conf self.execution_dir = execution_dir self.rebased_sources_dir = os.path.join(results_dir, 'rebased-sources') self.debug_log_file = debug_log_file # Temporary workspace for Builder, checks, ... self.kwargs['workspace_dir'] = self.workspace_dir = os.path.join(self.execution_dir, constants.WORKSPACE_DIR) # Directory where results should be put self.kwargs['results_dir'] = self.results_dir = results_dir # Directory contaning only those files, which are relevant for the new rebased version self.kwargs['rebased_sources_dir'] = self.rebased_sources_dir self.kwargs['non_interactive'] = self.conf.non_interactive self.kwargs['favor_on_conflict'] = self.conf.favor_on_conflict self.kwargs['changelog_entry'] = self.conf.changelog_entry self.kwargs['spec_hook_blacklist'] = self.conf.spec_hook_blacklist self.kwargs['build_log_hook_blacklist'] = self.conf.build_log_hook_blacklist self.kwargs['force_build_log_hooks'] = self.conf.force_build_log_hooks logger.verbose("Rebase-helper version: %s", VERSION) if self.conf.build_tasks is None: # check the workspace dir if not self.conf.cont: self._check_workspace_dir() self._get_spec_file() self._prepare_spec_objects() # verify all sources for the new version are present missing_sources = [os.path.basename(s) for s in self.rebase_spec_file.sources if not os.path.isfile(os.path.basename(s))] if missing_sources: raise RebaseHelperError('The following sources are missing: {}'.format(','.join(missing_sources))) if self.conf.update_sources: sources = [os.path.basename(s) for s in self.spec_file.sources] rebased_sources = [os.path.basename(s) for s in self.rebase_spec_file.sources] uploaded = LookasideCacheHelper.update_sources('fedpkg', self.rebased_sources_dir, self.rebase_spec_file.get_package_name(), sources, rebased_sources, upload=not self.conf.skip_upload) self._update_gitignore(uploaded, self.rebased_sources_dir) # TODO: Remove the value from kwargs and use only CLI attribute! self.kwargs['continue'] = self.conf.cont self._initialize_data() if self.conf.cont or self.conf.build_only: self._delete_old_builds()