def main(args_list=None):
"""CLI entrypoint"""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-b', '--base-bundle', metavar='NAME', action=NewBaseBundleAction,
required=True, help='The base bundle to use')
parser.add_argument('-p', '--pruning', metavar='PATH', type=Path, required=True,
help='The path to store pruning.list')
parser.add_argument('-d', '--domain-substitution', metavar='PATH', type=Path, required=True,
help='The path to store domain_substitution.list')
parser.add_argument('--tree', metavar='PATH', type=Path, default=BUILDSPACE_TREE,
help=('The path to the buildspace tree to create. '
'If it is not empty, the source will not be unpacked. '
'Default: %s') % BUILDSPACE_TREE)
parser.add_argument('--downloads', metavar='PATH', type=Path, default=BUILDSPACE_DOWNLOADS,
help=('The path to the buildspace downloads directory. '
'It must already exist. Default: %s') % BUILDSPACE_DOWNLOADS)
args = parser.parse_args(args_list)
try:
if args.tree.exists() and not dir_empty(args.tree):
get_logger().info('Using existing buildspace tree at %s', args.tree)
else:
source_retrieval.retrieve_and_extract(
args.base_bundle, args.downloads, args.tree, prune_binaries=False)
get_logger().info('Computing lists...')
pruning_list, domain_substitution_list = compute_lists(
args.tree, args.base_bundle.domain_regex.search_regex)
except BuildkitAbort:
exit(1)
with args.pruning.open('w', encoding=ENCODING) as file_obj:
file_obj.writelines('%s\n' % line for line in pruning_list)
with args.domain_substitution.open('w', encoding=ENCODING) as file_obj:
file_obj.writelines('%s\n' % line for line in domain_substitution_list)
def main(arg_list=None):
"""CLI entrypoint"""
parser = argparse.ArgumentParser(description=__doc__)
config_group = parser.add_mutually_exclusive_group()
config_group.add_argument(
'-b',
'--base-bundle',
metavar='NAME',
dest='bundle',
action=NewBaseBundleAction,
help=
('The base config bundle name to use (located in resources/config_bundles). '
'Mutually exclusive with --user-bundle-path. '))
config_group.add_argument(
'-u',
'--user-bundle',
metavar='PATH',
dest='bundle',
type=lambda x: ConfigBundle(Path(x)),
help=('The path to a user bundle to use. '
'Mutually exclusive with --base-bundle-name. '))
args = parser.parse_args(args=arg_list)
try:
domain_substitution.process_bundle_patches(args.bundle, invert=True)
except ValueError:
get_logger().exception('A regex pair is not invertible')
parser.exit(status=1)
def _get_gitiles_commit_before_date(repo_url, target_branch, target_datetime):
"""Returns the hexadecimal hash of the closest commit before target_datetime"""
json_log_url = '{repo}/+log/{branch}?format=JSON'.format(
repo=repo_url, branch=target_branch)
response = requests.get(json_log_url)
response.raise_for_status()
git_log = json.loads(
response.text[5:]) # Trim closing delimiters for various structures
assert len(git_log) == 2 # 'log' and 'next' entries
assert 'log' in git_log
assert git_log['log']
git_log = git_log['log']
# Check boundary conditions
if _get_gitiles_git_log_date(git_log[0]) < target_datetime:
# Newest commit is older than target datetime
return git_log[0]['commit']
if _get_gitiles_git_log_date(git_log[-1]) > target_datetime:
# Oldest commit is newer than the target datetime; assume oldest is close enough.
get_logger().warning(
'Oldest entry in gitiles log for repo "%s" is newer than target; '
'continuing with oldest entry...')
return git_log[-1]['commit']
# Do binary search
low_index = 0
high_index = len(git_log) - 1
mid_index = high_index
while low_index != high_index:
mid_index = low_index + (high_index - low_index) // 2
if _get_gitiles_git_log_date(git_log[mid_index]) > target_datetime:
low_index = mid_index + 1
else:
high_index = mid_index
return git_log[mid_index]['commit']
def compute_lists(source_tree, search_regex):
"""
Compute the binary pruning and domain substitution lists of the source tree.
Returns a tuple of two items in the following order:
1. The sorted binary pruning list
2. The sorted domain substitution list
source_tree is a pathlib.Path to the source tree
search_regex is a compiled regex object to search for domain names
"""
pruning_set = set()
domain_substitution_set = set()
deferred_symlinks = dict(
) # POSIX resolved path -> set of POSIX symlink paths
source_tree = source_tree.resolve()
unused_patterns = UnusedPatterns()
for path in source_tree.rglob('*'):
if not path.is_file():
# NOTE: Path.rglob() does not traverse symlink dirs; no need for special handling
continue
relative_path = path.relative_to(source_tree)
if path.is_symlink():
try:
resolved_relative_posix = path.resolve().relative_to(
source_tree).as_posix()
except ValueError:
# Symlink leads out of the source tree
continue
if resolved_relative_posix in pruning_set:
pruning_set.add(relative_path.as_posix())
else:
symlink_set = deferred_symlinks.get(resolved_relative_posix,
None)
if symlink_set is None:
symlink_set = set()
deferred_symlinks[resolved_relative_posix] = symlink_set
symlink_set.add(relative_path.as_posix())
# Path has finished processing because...
# Pruning: either symlink has been added or removal determination has been deferred
# Domain substitution: Only the real paths can be added, not symlinks
continue
try:
if should_prune(path, relative_path, unused_patterns):
relative_posix_path = relative_path.as_posix()
pruning_set.add(relative_posix_path)
symlink_set = deferred_symlinks.pop(relative_posix_path,
tuple())
if symlink_set:
pruning_set.update(symlink_set)
elif should_domain_substitute(path, relative_path, search_regex,
unused_patterns):
domain_substitution_set.add(relative_path.as_posix())
except:
get_logger().exception('Unhandled exception while processing %s',
relative_path)
raise BuildkitAbort()
return sorted(pruning_set), sorted(
domain_substitution_set), unused_patterns
def _validate_deps(deps_text):
"""Returns True if the DEPS file passes validation; False otherwise"""
try:
_DepsNodeVisitor().visit(ast.parse(deps_text))
except _UnexpectedSyntaxError as exc:
get_logger().error('%s', exc)
return False
return True
def sleep_for_retry(self, response=None):
"""Sleeps for Retry-After, and logs the sleep time"""
if response:
retry_after = self.get_retry_after(response)
if retry_after:
get_logger().info(
'Got HTTP status %s with Retry-After header. Retrying after %s seconds...',
response.status, retry_after)
else:
get_logger().info(
'Could not find Retry-After header for HTTP response %s. Status reason: %s',
response.status, response.reason)
return super().sleep_for_retry(response)
def _download_googlesource_file(download_session, repo_url, version, relative_path):
"""
Returns the contents of the text file with path within the given
googlesource.com repo as a string.
"""
if 'googlesource.com' not in repo_url:
raise ValueError('Repository URL is not a googlesource.com URL: {}'.format(repo_url))
full_url = repo_url + '/+/{}/{}?format=TEXT'.format(version, str(relative_path))
get_logger(prepend_timestamp=False, log_init=False).debug('Downloading: %s', full_url)
response = download_session.get(full_url)
response.raise_for_status()
# Assume all files that need patching are compatible with UTF-8
return base64.b64decode(response.text, validate=True).decode('UTF-8')
def log_unused(self):
"""
Logs unused patterns and prefixes
Returns True if there are unused patterns or prefixes; False otherwise
"""
have_unused = False
for name in self._all_names:
current_set = getattr(self, name, None)
if current_set:
get_logger().error('Unused from %s: %s', name.upper(), current_set)
have_unused = True
return have_unused
def _retrieve_remote_files(file_iter):
"""
Retrieves all file paths in file_iter from Google
file_iter is an iterable of strings that are relative UNIX paths to
files in the Chromium source.
Returns a dict of relative UNIX path strings to a list of lines in the file as strings
"""
files = dict()
root_deps_tree = _initialize_deps_tree()
try:
total_files = len(file_iter)
except TypeError:
total_files = None
logger = get_logger()
if total_files is None:
logger.info('Downloading remote files...')
else:
logger.info('Downloading %d remote files...', total_files)
last_progress = 0
file_count = 0
fallback_repo_manager = _FallbackRepoManager()
with requests.Session() as download_session:
download_session.stream = False # To ensure connection to Google can be reused
for file_path in file_iter:
if total_files:
file_count += 1
current_progress = file_count * 100 // total_files // 5 * 5
if current_progress != last_progress:
last_progress = current_progress
logger.info('%d%% downloaded', current_progress)
else:
current_progress = file_count // 20 * 20
if current_progress != last_progress:
last_progress = current_progress
logger.info('%d files downloaded', current_progress)
try:
files[file_path] = _download_source_file(
download_session, root_deps_tree, fallback_repo_manager,
file_path).split('\n')
except _NotInRepoError:
get_logger().warning(
'Could not find "%s" remotely. Skipping...', file_path)
return files
def _get_current_commit():
"""
Returns a string of the current commit hash.
It assumes "git" is in PATH, and that buildkit is run within a git repository.
Raises BuildkitAbort if invoking git fails.
"""
result = subprocess.run(['git', 'rev-parse', '--verify', 'HEAD'],
stdout=subprocess.PIPE,
universal_newlines=True,
cwd=str(Path(__file__).resolve().parent))
if result.returncode:
get_logger().error('Unexpected return code %s', result.returncode)
get_logger().error('Command output: %s', result.stdout)
raise BuildkitAbort()
return result.stdout.strip('\n')
def _retrieve_local_files(file_iter, source_dir):
"""
Retrieves all file paths in file_iter from the local source tree
file_iter is an iterable of strings that are relative UNIX paths to
files in the Chromium source.
Returns a dict of relative UNIX path strings to a list of lines in the file as strings
"""
files = dict()
for file_path in file_iter:
try:
files[file_path] = (source_dir / file_path).read_text().split('\n')
except FileNotFoundError:
get_logger().warning('Missing file from patches: %s', file_path)
if not files:
get_logger().error('All files used by patches are missing!')
return files
def get_fallback(self, current_relative_path, current_node, root_deps_tree):
"""
Helper for _download_source_file
It returns a new (repo_url, version, new_relative_path) to attempt a file download with
"""
assert len(current_node) == 3
# GN special processing
try:
new_relative_path = current_relative_path.relative_to('tools/gn')
except ValueError:
pass
else:
if current_node is root_deps_tree[Path('src')]:
get_logger().info('Redirecting to GN repo version %s for path: %s', self.gn_version,
current_relative_path)
return (self._GN_REPO_URL, self.gn_version, new_relative_path)
return None, None, None
def _download_source_file(download_session, root_deps_tree, fallback_repo_manager, target_file):
"""
Downloads the source tree file from googlesource.com
download_session is an active requests.Session() object
deps_dir is a pathlib.Path to the directory containing a DEPS file.
"""
current_node, current_relative_path = _get_target_file_deps_node(download_session,
root_deps_tree, target_file)
# Attempt download with potential fallback logic
repo_url, version, _ = current_node
try:
# Download with DEPS-provided repo
return _download_googlesource_file(download_session, repo_url, version,
current_relative_path)
except _NotInRepoError:
pass
get_logger().debug(
'Path "%s" (relative: "%s") not found using DEPS tree; finding fallback repo...',
target_file, current_relative_path)
repo_url, version, current_relative_path = fallback_repo_manager.get_fallback(
current_relative_path, current_node, root_deps_tree)
if not repo_url:
get_logger().error('No fallback repo found for "%s" (relative: "%s")', target_file,
current_relative_path)
raise _NotInRepoError()
try:
# Download with fallback repo
return _download_googlesource_file(download_session, repo_url, version,
current_relative_path)
except _NotInRepoError:
pass
get_logger().error('File "%s" (relative: "%s") not found in fallback repo "%s", version "%s"',
target_file, current_relative_path, repo_url, version)
raise _NotInRepoError()
def _test_patches(patch_trie, bundle_cache, patch_cache, orig_files):
"""
Tests the patches with DFS in the trie of config bundles
Returns a boolean indicating if any of the patches have failed
"""
# Stack of iterables over each node's children
# First, insert iterable over root node's children
node_iter_stack = [iter(patch_trie.items())]
# Stack of files at each node differing from the parent
# The root node thus contains all the files to be patched
file_layers = collections.ChainMap(orig_files)
# Whether any branch had failed validation
had_failure = False
while node_iter_stack:
try:
child_path, grandchildren = next(node_iter_stack[-1])
except StopIteration:
# Finished exploring all children of this node
node_iter_stack.pop()
del file_layers.maps[0]
continue
# Add storage for child's patched files
file_layers = file_layers.new_child()
# Apply children's patches
get_logger().info('Verifying at depth %s: %s ...',
len(node_iter_stack), child_path.name)
# Potential optimization: Use interval tree data structure instead of copying
# the entire array to track only diffs
had_failure, branch_validation_failed = _apply_child_bundle_patches(
child_path, had_failure, file_layers, patch_cache, bundle_cache)
if branch_validation_failed:
# Add blank children to force stack to move onto the next branch
node_iter_stack.append(iter(tuple()))
else:
# Explore this child's children
node_iter_stack.append(iter(grandchildren.items()))
return had_failure
def _apply_child_bundle_patches(child_path, had_failure, file_layers,
patch_cache, bundle_cache):
"""Helper for _test_patches"""
# Whether the curent patch trie branch failed validation
branch_validation_failed = False
assert child_path in bundle_cache
try:
child_patch_order = bundle_cache[child_path].patch_order
except KeyError:
# No patches in the bundle
pass
else:
patches_outdated = bundle_cache[child_path].bundlemeta.patches_outdated
for patch_path_str in child_patch_order:
for patched_file in patch_cache[patch_path_str]:
try:
_apply_file_unidiff(patched_file, file_layers.maps[0],
file_layers.parents)
except _PatchValidationError as exc:
# Branch failed validation; abort
get_logger().error(
"Validation failure for file '%s' from patch '%s': %s",
patched_file.path, patch_path_str, str(exc))
branch_validation_failed = True
had_failure = had_failure or not patches_outdated
break
except BaseException:
# Branch failed validation; abort
get_logger().exception(
"Error processing file '%s' from patch '%s'",
patched_file.path, patch_path_str)
branch_validation_failed = True
had_failure = had_failure or not patches_outdated
break
if branch_validation_failed:
if patches_outdated:
get_logger().warning(
'%s is marked with outdated patches. Ignoring failure...',
child_path.name)
break
if branch_validation_failed != patches_outdated:
# Metadata for patch validity is out-of-date
get_logger().error(
'%s patch validity is inconsistent with patches_outdated marking in bundlemeta',
child_path.name)
had_failure = True
return had_failure, branch_validation_failed
def main():
"""CLI Entrypoint"""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
'-b',
'--bundle',
action='append',
type=Path,
metavar='DIRECTORY',
help=('Verify patches for a config bundle. Specify multiple times to '
'verify multiple bundles. Without specifying, all bundles will be verified.'))
parser.add_argument(
'-v', '--verbose', action='store_true', help='Log more information to stdout/stderr')
file_source_group = parser.add_mutually_exclusive_group(required=True)
file_source_group.add_argument(
'-l', '--local', type=Path, metavar='DIRECTORY', help='Use a local source tree')
file_source_group.add_argument(
'-r',
'--remote',
action='store_true',
help='Download the required source tree files from Google')
file_source_group.add_argument(
'-c',
'--cache-remote',
type=Path,
metavar='DIRECTORY',
help='(For debugging) Store the required remote files in an empty local directory')
args = parser.parse_args()
if args.cache_remote and not args.cache_remote.exists():
if args.cache_remote.parent.exists():
args.cache_remote.mkdir()
else:
parser.error('Parent of cache path {} does not exist'.format(args.cache_remote))
if args.verbose:
get_logger(initial_level=logging.DEBUG, prepend_timestamp=False, log_init=False)
else:
get_logger(initial_level=logging.INFO, prepend_timestamp=False, log_init=False)
if args.bundle:
for bundle_path in args.bundle:
if not bundle_path.exists():
parser.error('Could not find config bundle at: {}'.format(bundle_path))
# Path to bundle -> ConfigBundle without dependencies
bundle_cache = dict(
map(lambda x: (x, ConfigBundle(x, load_depends=False)), _CONFIG_BUNDLES_PATH.iterdir()))
patch_trie = _get_patch_trie(bundle_cache, args.bundle)
patch_cache = _load_all_patches(bundle_cache.values())
required_files = _get_required_files(patch_cache)
orig_files = _get_orig_files(args, required_files, parser)
had_failure = _test_patches(patch_trie, bundle_cache, patch_cache, orig_files)
if had_failure:
if not args.verbose:
get_logger().info('(For more error details, re-run with the "-v" flag)')
parser.exit(status=1)
def _get_current_commit_or_tag():
"""
Returns a string of the current commit hash, or the tag name based
on version.ini if the script is not in a git repo.
It assumes "git" is in PATH, and that buildkit is run within a git repository.
Raises BuildkitAbort if invoking git fails.
"""
# Use presence of .git directory to determine if using git or not
# since git will be aggressive in finding a git repository otherwise
if not (Path(__file__).parent / '.git').exists(): #pylint: disable=no-member
# Probably not in a git checkout; extrapolate tag name based on
# version.ini
return '{}-{}'.format(get_chromium_version(), get_release_revision())
result = subprocess.run(['git', 'rev-parse', '--verify', 'HEAD'],
stdout=subprocess.PIPE,
universal_newlines=True,
cwd=str(Path(__file__).resolve().parent))
if result.returncode:
get_logger().error('Unexpected return code %s', result.returncode)
get_logger().error('Command output: %s', result.stdout)
raise BuildkitAbort()
return result.stdout.strip('\n')
def _retrieve_remote_files(file_iter):
"""
Retrieves all file paths in file_iter from Google
file_iter is an iterable of strings that are relative UNIX paths to
files in the Chromium source.
Returns a dict of relative UNIX path strings to a list of lines in the file as strings
"""
# Load requests here so it isn't a dependency for local file reading
import requests
files = dict()
deps_tree = _initialize_deps_tree()
try:
total_files = len(file_iter)
except TypeError:
total_files = None
logger = get_logger()
if total_files is None:
logger.info('Downloading remote files...')
else:
logger.info('Downloading %d remote files...', total_files)
last_progress = 0
file_count = 0
with requests.Session() as download_session:
download_session.stream = False # To ensure connection to Google can be reused
for file_path in file_iter:
if total_files:
file_count += 1
current_progress = file_count * 100 // total_files // 5 * 5
if current_progress != last_progress:
last_progress = current_progress
logger.info('%d%% downloaded', current_progress)
else:
current_progress = file_count // 20 * 20
if current_progress != last_progress:
last_progress = current_progress
logger.info('%d files downloaded', current_progress)
files[file_path] = _download_source_file(download_session,
deps_tree,
file_path).split('\n')
return files
def main(arg_list=None):
"""CLI entrypoint"""
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
config_group = parser.add_mutually_exclusive_group()
config_group.add_argument(
'-b',
'--base-bundle',
metavar='NAME',
dest='bundle',
action=NewBaseBundleAction,
help=
('The base config bundle name to use (located in resources/config_bundles). '
'Mutually exclusive with --user-bundle-path. '))
config_group.add_argument(
'-u',
'--user-bundle',
metavar='PATH',
dest='bundle',
type=lambda x: ConfigBundle(Path(x)),
help=('The path to a user bundle to use. '
'Mutually exclusive with --base-bundle-name. '))
args = parser.parse_args(args=arg_list)
logger = get_logger()
search_regex = re.compile('|'.join(
map(lambda x: x[0].pattern,
args.bundle.domain_regex.get_pairs(invert=True))))
for patch_path in args.bundle.patches.patch_iter():
if patch_path.exists():
with patch_path.open(encoding=ENCODING) as file_obj:
try:
patchset = unidiff.PatchSet(file_obj.read())
except unidiff.errors.UnidiffParseError:
logger.exception('Could not parse patch: %s', patch_path)
continue
if _check_substituted_domains(patchset, search_regex):
logger.warning('Patch has substituted domains: %s',
patch_path)
else:
logger.warning('Patch not found: %s', patch_path)
def main():
"""CLI entrypoint"""
logger = get_logger(prepend_timestamp=False, log_init=False)
warnings = True
patches_dir = get_resources_dir() / PATCHES_DIR
config_bundles_dir = get_resources_dir() / CONFIG_BUNDLES_DIR
journal = ExplorationJournal(
# base bundles not explored yet
unexplored_set=set(map(
lambda x: x.name,
config_bundles_dir.iterdir())),
# base bundle name -> namedtuple(leaves=set(), gn_flags=set())
results=dict(),
# dependency -> set of dependents
dependents=dict(),
# patches unused by patch orders
unused_patches=set(map(
lambda x: str(x.relative_to(patches_dir)), patches_dir.rglob('*.patch')))
)
try:
# Explore and validate base bundles
while journal.unexplored_set:
warnings = _explore_base_bundle(
next(iter(journal.unexplored_set)), journal, logger) or warnings
# Check for config file entries that should be merged into dependencies
warnings = _check_mergability((
('GN flags', lambda x: journal.results[x].gn_flags),
('patches', lambda x: journal.results[x].patches),
), journal.dependents, logger) or warnings
# Check for patch files not referenced in patch_orders
if journal.unused_patches:
logger.warning('Unused patches found: %s', journal.unused_patches)
warnings = True
except BuildkitAbort:
exit(2)
if warnings:
exit(1)
def main(args_list=None):
"""CLI entrypoint"""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
'-a', '--auto-download', action='store_true',
help='If specified, it will download the source code and dependencies '
'for the --base-bundle given. Otherwise, only an existing '
'buildspace tree will be used.')
parser.add_argument(
'-b', '--base-bundle', metavar='NAME', type=get_basebundle_verbosely,
default='common', help='The base bundle to use. Default: %(default)s')
parser.add_argument(
'-p', '--pruning', metavar='PATH', type=Path,
default='resources/config_bundles/common/pruning.list',
help='The path to store pruning.list. Default: %(default)s')
parser.add_argument(
'-d', '--domain-substitution', metavar='PATH', type=Path,
default='resources/config_bundles/common/domain_substitution.list',
help='The path to store domain_substitution.list. Default: %(default)s')
parser.add_argument(
'--tree', metavar='PATH', type=Path, default=BUILDSPACE_TREE,
help=('The path to the buildspace tree to create. '
'If it is not empty, the source will not be unpacked. '
'Default: %(default)s'))
parser.add_argument(
'--downloads', metavar='PATH', type=Path, default=BUILDSPACE_DOWNLOADS,
help=('The path to the buildspace downloads directory. '
'It must already exist. Default: %(default)s'))
try:
args = parser.parse_args(args_list)
if args.tree.exists() and not dir_empty(args.tree):
get_logger().info('Using existing buildspace tree at %s', args.tree)
elif args.auto_download:
source_retrieval.retrieve_and_extract(
args.base_bundle, args.downloads, args.tree, prune_binaries=False)
else:
get_logger().error('No buildspace tree found and --auto-download '
'is not specified. Aborting.')
raise BuildkitAbort()
get_logger().info('Computing lists...')
pruning_list, domain_substitution_list = compute_lists(
args.tree, args.base_bundle.domain_regex.search_regex)
except BuildkitAbort:
exit(1)
with args.pruning.open('w', encoding=ENCODING) as file_obj:
file_obj.writelines('%s\n' % line for line in pruning_list)
with args.domain_substitution.open('w', encoding=ENCODING) as file_obj:
file_obj.writelines('%s\n' % line for line in domain_substitution_list)
def main(args_list=None):
"""CLI entrypoint"""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
'-a',
'--auto-download',
action='store_true',
help='If specified, it will download the source code and dependencies '
'for the --bundle given. Otherwise, only an existing '
'source tree will be used.')
parser.add_argument('-b',
'--bundle',
metavar='PATH',
type=Path,
default='config_bundles/common',
help='The bundle to use. Default: %(default)s')
parser.add_argument(
'--pruning',
metavar='PATH',
type=Path,
default='config_bundles/common/pruning.list',
help='The path to store pruning.list. Default: %(default)s')
parser.add_argument(
'--domain-substitution',
metavar='PATH',
type=Path,
default='config_bundles/common/domain_substitution.list',
help='The path to store domain_substitution.list. Default: %(default)s'
)
parser.add_argument(
'-t',
'--tree',
metavar='PATH',
type=Path,
required=True,
help=('The path to the source tree to create. '
'If it is not empty, the source will not be unpacked.'))
parser.add_argument('-c',
'--cache',
metavar='PATH',
type=Path,
help='The path to the downloads cache.')
try:
args = parser.parse_args(args_list)
try:
bundle = ConfigBundle(args.bundle)
except BaseException:
get_logger().exception('Error loading config bundle')
raise BuildkitAbort()
if args.tree.exists() and not dir_empty(args.tree):
get_logger().info('Using existing source tree at %s', args.tree)
elif args.auto_download:
if not args.cache:
get_logger().error('--cache is required with --auto-download')
raise BuildkitAbort()
downloads.retrieve_downloads(bundle, args.cache, True)
downloads.check_downloads(bundle, args.cache)
downloads.unpack_downloads(bundle, args.cache, args.tree)
else:
get_logger().error('No source tree found and --auto-download '
'is not specified. Aborting.')
raise BuildkitAbort()
get_logger().info('Computing lists...')
pruning_list, domain_substitution_list = compute_lists(
args.tree, bundle.domain_regex.search_regex)
except BuildkitAbort:
exit(1)
with args.pruning.open('w', encoding=ENCODING) as file_obj:
file_obj.writelines('%s\n' % line for line in pruning_list)
with args.domain_substitution.open('w', encoding=ENCODING) as file_obj:
file_obj.writelines('%s\n' % line for line in domain_substitution_list)
import argparse
import os
import subprocess
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from buildkit.common import get_logger
from buildkit.config import ConfigBundle
sys.path.pop(0)
_CONFIG_BUNDLES_PATH = Path(__file__).parent.parent / 'config_bundles'
_PATCHES_PATH = Path(__file__).parent.parent / 'patches'
_LOGGER = get_logger(prepend_timestamp=False, log_init=False)
def _get_run_quilt(source_dir, series_path, patches_dir):
"""Create a function to run quilt with proper settings"""
def _run_quilt(*args, log_stderr=True, **kwargs):
result = subprocess.run(
('quilt', '--quiltrc', '-', *args),
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=str(source_dir),
env={
'QUILT_PATCHES':
str(patches_dir.resolve()),
'QUILT_SERIES':
def _apply_child_bundle_patches(child_path, had_failure, file_layers,
patch_cache, bundle_cache):
"""Helper for _test_patches"""
# Whether the curent patch trie branch failed validation
branch_validation_failed = False
assert child_path in bundle_cache
try:
child_patch_order = bundle_cache[child_path].patch_order
except KeyError:
# No patches in the bundle
pass
else:
patches_outdated = bundle_cache[child_path].bundlemeta.patches_outdated
for patch_path_str in child_patch_order:
for patched_file in patch_cache[patch_path_str]:
try:
_apply_file_unidiff(patched_file, file_layers.maps[0],
file_layers.parents)
except _PatchValidationError as exc:
# Branch failed validation; abort
get_logger().warning('Patch failed validation: %s',
patch_path_str)
get_logger().debug(
'Specifically, file "%s" failed validation: %s',
patched_file.path, exc)
branch_validation_failed = True
had_failure = had_failure or not patches_outdated
break
except BaseException:
# Branch failed validation; abort
get_logger().warning('Patch failed validation: %s',
patch_path_str)
get_logger().debug(
'Specifically, file "%s" caused exception while applying:',
patched_file.path,
exc_info=True)
branch_validation_failed = True
had_failure = had_failure or not patches_outdated
break
if branch_validation_failed:
if patches_outdated:
get_logger().warning(
'%s is marked with outdated patches. Ignoring failure...',
child_path.name)
break
if branch_validation_failed != patches_outdated:
# Metadata for patch validity is out-of-date
if branch_validation_failed:
get_logger().error((
"%s patches have become outdated. "
"Please add 'patches_outdated = true' to its bundlemeta.ini"
), child_path.name)
else:
get_logger().error((
'"%s" is no longer out-of-date! '
'Please remove the patches_outdated marking from its bundlemeta.ini'
), child_path.name)
had_failure = True
return had_failure, branch_validation_failed
def main():
"""CLI Entrypoint"""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
'--downloads-cache',
type=Path,
metavar='PATH',
default='../../downloads_cache',
help='The path to the downloads cache')
parser.add_argument(
'--disable-ssl-verification',
action='store_true',
help='Disables SSL verification for downloading')
parser.add_argument(
'--7z-path',
dest='sevenz_path',
default=SEVENZIP_USE_REGISTRY,
help=('Command or path to 7-Zip\'s "7z" binary. If "_use_registry" is '
'specified, determine the path from the registry. Default: %(default)s'))
args = parser.parse_args()
# Set common variables
bundle_path = Path(__file__).parent / 'config_bundles/windows'
bundle = buildkit.config.ConfigBundle(bundle_path)
source_tree = Path(__file__).resolve().parent.parent
domsubcache = Path(__file__).parent / 'domsubcache.tar.gz'
# Test environment
_test_python2(parser.error)
# Setup environment
if not args.downloads_cache.exists():
args.downloads_cache.mkdir()
_make_tmp_paths()
# Retrieve downloads
get_logger().info('Downloading required files...')
buildkit.downloads.retrieve_downloads(bundle, args.downloads_cache, True,
args.disable_ssl_verification)
try:
buildkit.downloads.check_downloads(bundle, args.downloads_cache)
except buildkit.downloads.HashMismatchError as exc:
get_logger().error('File checksum does not match: %s', exc)
parser.exit(1)
# Unpack downloads
extractors = {
ExtractorEnum.SEVENZIP: args.sevenz_path,
}
get_logger().info('Unpacking downloads...')
buildkit.downloads.unpack_downloads(bundle, args.downloads_cache, source_tree, extractors)
# Prune binaries
unremovable_files = buildkit.extraction.prune_dir(source_tree, bundle.pruning)
if unremovable_files:
get_logger().error('Files could not be pruned: %s', unremovable_files)
parser.exit(1)
# Apply patches
buildkit.patches.apply_patches(
buildkit.patches.patch_paths_by_bundle(bundle), source_tree, patch_bin_path=None)
# Substitute domains
buildkit.domain_substitution.apply_substitution(bundle, source_tree, domsubcache)
# Output args.gn
(source_tree / 'out/Default').mkdir(parents=True)
(source_tree / 'out/Default/args.gn').write_text(str(bundle.gn_flags), encoding=ENCODING)
# Run GN bootstrap
_run_build_process(
shutil.which('python'), 'tools\\gn\\bootstrap\\bootstrap.py', '-o', 'out\\Default\\gn.exe',
'--skip-generate-buildfiles')
# Run gn gen
_run_build_process('out\\Default\\gn.exe', 'gen', 'out\\Default', '--fail-on-unused-args')
# Run ninja
_run_build_process('third_party\\ninja\\ninja.exe', '-C', 'out\\Default', 'chrome',
'chromedriver')
def _test_patches(patch_trie, bundle_cache, patch_cache, orig_files):
"""
Tests the patches with DFS in the trie of config bundles
Returns a boolean indicating if any of the patches have failed
"""
# Stack of iterables over each node's children
# First, insert iterable over root node's children
node_iter_stack = [iter(patch_trie.items())]
# Stack of files at each node differing from the parent
# The root node thus contains all the files to be patched
file_layers = collections.ChainMap(orig_files)
# Whether any branch had failed validation
had_failure = False
while node_iter_stack:
try:
child_path, grandchildren = next(node_iter_stack[-1])
except StopIteration:
# Finished exploring all children of this node
node_iter_stack.pop()
del file_layers.maps[0]
continue
# Add storage for child's patched files
file_layers = file_layers.new_child()
# Apply children's patches
get_logger(
prepend_timestamp=False, log_init=False).info('Verifying at depth %s: %s',
len(node_iter_stack), child_path.name)
# Potential optimization: Use interval tree data structure instead of copying
# the entire array to track only diffs
# Whether the curent patch trie branch failed validation
branch_validation_failed = False
assert child_path in bundle_cache
try:
child_patch_order = bundle_cache[child_path].patch_order
except KeyError:
# No patches in the bundle
pass
else:
for patch_path_str in child_patch_order:
for patched_file in patch_cache[patch_path_str]:
try:
_apply_file_unidiff(patched_file, file_layers.maps[0], file_layers.parents)
except _PatchValidationError as exc:
# Branch failed validation; abort
get_logger(
prepend_timestamp=False, log_init=False).error(
"Error processing file '%s' from patch '%s': %s", patched_file.path,
patch_path_str, str(exc))
branch_validation_failed = True
had_failure = True
break
except BaseException:
# Branch failed validation; abort
get_logger(
prepend_timestamp=False, log_init=False).exception(
"Error processing file '%s' from patch '%s'", patched_file.path,
patch_path_str)
branch_validation_failed = True
had_failure = True
break
if branch_validation_failed:
break
if branch_validation_failed:
# Add blank children to force stack to move onto the next branch
node_iter_stack.append(iter(tuple()))
else:
# Explore this child's children
node_iter_stack.append(iter(grandchildren.items()))
return had_failure
def _sleep_backoff(self):
"""Log info about backoff sleep"""
get_logger().info('Running HTTP request sleep backoff')
super()._sleep_backoff()
