def test_no_extra_vars(self):
deps_file = '\n'.join([
'vars = {',
' "foo": "bar",',
'}',
'deps = {',
' "a_dep": "a{baz}b",',
'}',
])
with self.assertRaises(ValueError) as cm:
gclient_eval.Parse(deps_file,
expand_vars=True,
validate_syntax=True,
filename='<unknown>',
vars_override={'baz': 'lalala'})
self.assertIn('baz was used as a variable, but was not declared',
str(cm.exception))
with self.assertRaises(KeyError) as cm:
gclient_eval.Parse(deps_file,
expand_vars=True,
validate_syntax=False,
filename='<unknown>',
vars_override={'baz': 'lalala'})
self.assertIn('baz', str(cm.exception))
def test_merges_hooks_os(self):
for validate_syntax in True, False:
local_scope = gclient_eval.Parse(
'\n'.join([
'hooks = [',
' {',
' "action": ["a", "action"],',
' },',
']',
'hooks_os = {',
' "mac": [',
' {',
' "action": ["b", "action"]',
' },',
' ]',
'}',
]), validate_syntax, '<unknown>')
self.assertEqual(
{
"hooks": [{
"action": ["a", "action"]
}, {
"action": ["b", "action"],
"condition": "checkout_mac"
}],
}, local_scope)
def test_merges_deps_os_extra_dep(self):
for validate_syntax in True, False:
local_scope = gclient_eval.Parse(
'\n'.join([
'deps = {',
' "a_dep": "a_url@a_rev",',
'}',
'deps_os = {',
' "mac": {',
' "b_dep": "b_url@b_rev"',
' },',
'}',
]), validate_syntax, '<unknown>')
self.assertEqual(
{
'deps': {
'a_dep': {
'url': 'a_url@a_rev',
'dep_type': 'git'
},
'b_dep': {
'url': 'b_url@b_rev',
'dep_type': 'git',
'condition': 'checkout_mac'
}
},
}, local_scope)
def test_declaring_builtin_var_has_no_effect(self):
builtin_vars = {'builtin_var': 'foo'}
deps_file = '\n'.join([
'vars = {',
' "builtin_var": "bar",',
'}',
'deps = {',
' "a_dep": "a{builtin_var}b",',
'}',
])
for validate_syntax in False, True:
local_scope = gclient_eval.Parse(deps_file, validate_syntax,
'<unknown>', None, builtin_vars)
self.assertEqual(
{
'vars': {
'builtin_var': 'bar'
},
'deps': {
'a_dep': {
'url': 'afoob',
'dep_type': 'git'
}
},
}, local_scope)
def get_cipd_dependency_rev(self):
"""Return CTS CIPD revision in the checkout's DEPS file."""
deps_file = os.path.join(self._root_dir, DEPS_FILE)
with open(deps_file) as f:
contents = f.read()
dd = gclient_eval.Parse(contents, False, deps_file)
return gclient_eval.GetCIPD(dd, CTS_DEP_NAME, CTS_DEP_PACKAGE)
def test_merges_deps_os_existing_dep_with_condition(self):
for validate_syntax in True, False:
local_scope = gclient_eval.Parse(
'\n'.join([
'deps = {',
' "a_dep": {',
' "url": "a_url@a_rev",',
' "condition": "some_condition",',
' },',
'}',
'deps_os = {',
' "mac": {',
' "a_dep": "a_url@a_rev"',
' },',
'}',
]), validate_syntax, '<unknown>')
self.assertEqual(
{
'deps': {
'a_dep': {
'url': 'a_url@a_rev',
'dep_type': 'git',
'condition': '(checkout_mac) or (some_condition)'
},
},
}, local_scope)
def test_supports_vars_inside_vars(self):
deps_file = '\n'.join([
'vars = {',
' "foo": "bar",',
' "baz": "\\"{foo}\\" == \\"bar\\"",',
'}',
'deps = {',
' "src/baz": {',
' "url": "baz_url",',
' "condition": "baz",',
' },',
'}',
])
for validate_syntax in False, True:
local_scope = gclient_eval.Parse(deps_file, validate_syntax,
'<unknown>', None)
self.assertEqual(
{
'vars': {
'foo': 'bar',
'baz': '"bar" == "bar"'
},
'deps': {
'src/baz': {
'url': 'baz_url',
'dep_type': 'git',
'condition': 'baz'
}
},
}, local_scope)
def callParse(self, validate_syntax=True, vars_override=None):
return gclient_eval.Parse('\n'.join([
'vars = {',
' "foo": "bar",',
'}',
'deps = {',
' "a_dep": "a{foo}b",',
'}',
]), validate_syntax, '<unknown>', vars_override)
def test_no_extra_vars(self):
deps_file = '\n'.join([
'vars = {',
' "foo": "bar",',
'}',
'deps = {',
' "a_dep": "a{baz}b",',
'}',
])
with self.assertRaises(KeyError) as cm:
gclient_eval.Parse(deps_file, True, '<unknown>', {'baz': 'lalala'})
self.assertIn('baz was used as a variable, but was not declared',
str(cm.exception))
with self.assertRaises(KeyError) as cm:
gclient_eval.Parse(deps_file, False, '<unknown>',
{'baz': 'lalala'})
self.assertIn('baz', str(cm.exception))
def test_standardizes_deps_string_dep(self):
local_scope = gclient_eval.Parse('\n'.join([
'deps = {',
' "a_dep": "a_url@a_rev",',
'}',
]), '<unknown>')
self.assertEqual({
'deps': {'a_dep': {'url': 'a_url@a_rev',
'dep_type': 'git'}},
}, local_scope)
def test_has_builtin_vars(self):
builtin_vars = {'builtin_var': 'foo'}
deps_file = '\n'.join([
'deps = {',
' "a_dep": "a{builtin_var}b",',
'}',
])
local_scope = gclient_eval.Parse(deps_file, '<unknown>', None, builtin_vars)
self.assertEqual({
'deps': {'a_dep': {'url': 'afoob',
'dep_type': 'git'}},
}, local_scope)
def test_override_builtin_var(self):
builtin_vars = {'builtin_var': 'foo'}
vars_override = {'builtin_var': 'override'}
deps_file = '\n'.join([
'deps = {',
' "a_dep": "a{builtin_var}b",',
'}',
])
local_scope = gclient_eval.Parse(
deps_file, '<unknown>', vars_override, builtin_vars)
self.assertEqual({
'deps': {'a_dep': {'url': 'aoverrideb',
'dep_type': 'git'}},
}, local_scope, str(local_scope))
def test_standardizes_deps_dict_dep(self):
local_scope = gclient_eval.Parse('\n'.join([
'deps = {',
' "a_dep": {',
' "url": "a_url@a_rev",',
' "condition": "checkout_android",',
' },',
'}',
]), '<unknown>')
self.assertEqual({
'deps': {'a_dep': {'url': 'a_url@a_rev',
'dep_type': 'git',
'condition': 'checkout_android'}},
}, local_scope)
def test_merges_deps_os_existing_dep_with_no_condition(self):
local_scope = gclient_eval.Parse('\n'.join([
'deps = {',
' "a_dep": "a_url@a_rev",',
'}',
'deps_os = {',
' "mac": {',
' "a_dep": "a_url@a_rev"',
' },',
'}',
]), '<unknown>')
self.assertEqual({
'deps': {'a_dep': {'url': 'a_url@a_rev',
'dep_type': 'git'}},
}, local_scope)
def test_doesnt_allow_duplicate_deps(self):
with self.assertRaises(ValueError) as cm:
gclient_eval.Parse('\n'.join([
'deps = {',
' "a_dep": {',
' "url": "a_url@a_rev",',
' "condition": "foo",',
' },',
' "a_dep": {',
' "url": "a_url@another_rev",',
' "condition": "not foo",',
' }',
'}',
]), '<unknown>')
self.assertIn('duplicate key in dictionary: a_dep', str(cm.exception))
def test_ignores_none_in_deps_os(self):
local_scope = gclient_eval.Parse('\n'.join([
'deps = {',
' "a_dep": "a_url@a_rev",',
'}',
'deps_os = {',
' "mac": {',
' "a_dep": None,',
' },',
'}',
]), '<unknown>')
self.assertEqual({
'deps': {'a_dep': {'url': 'a_url@a_rev',
'dep_type': 'git'}},
}, local_scope)
def test_fails_to_merge_same_dep_with_different_revisions(self):
with self.assertRaises(gclient_eval.gclient_utils.Error) as cm:
gclient_eval.Parse('\n'.join([
'deps = {',
' "a_dep": {',
' "url": "a_url@a_rev",',
' "condition": "some_condition",',
' },',
'}',
'deps_os = {',
' "mac": {',
' "a_dep": "a_url@b_rev"',
' },',
'}',
]), '<unknown>')
self.assertIn('conflicts with existing deps', str(cm.exception))
def test_merges_deps_os_multiple_os(self):
local_scope = gclient_eval.Parse('\n'.join([
'deps_os = {',
' "win": {'
' "a_dep": "a_url@a_rev"',
' },',
' "mac": {',
' "a_dep": "a_url@a_rev"',
' },',
'}',
]), '<unknown>')
self.assertEqual({
'deps': {
'a_dep': {'url': 'a_url@a_rev',
'dep_type': 'git',
'condition': '(checkout_mac) or (checkout_win)'},
},
}, local_scope)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
'--ignore-dirty-tree', action='store_true',
help='Roll anyways, even if there is a diff.')
parser.add_argument(
'-r', '--reviewer',
help='To specify multiple reviewers, use comma separated list, e.g. '
'-r joe,jane,john. Defaults to @chromium.org')
parser.add_argument('-b', '--bug', help='Associate a bug number to the roll')
# It is important that --no-log continues to work, as it is used by
# internal -> external rollers. Please do not remove or break it.
parser.add_argument(
'--no-log', action='store_true',
help='Do not include the short log in the commit message')
parser.add_argument(
'--log-limit', type=int, default=100,
help='Trim log after N commits (default: %(default)s)')
parser.add_argument(
'--roll-to', default='origin/master',
help='Specify the new commit to roll to (default: %(default)s)')
parser.add_argument(
'--key', action='append', default=[],
help='Regex(es) for dependency in DEPS file')
parser.add_argument('dep_path', nargs='+', help='Path(s) to dependency')
args = parser.parse_args()
if len(args.dep_path) > 1:
if args.roll_to != 'origin/master':
parser.error(
'Can\'t use multiple paths to roll simultaneously and --roll-to')
if args.key:
parser.error(
'Can\'t use multiple paths to roll simultaneously and --key')
reviewers = None
if args.reviewer:
reviewers = args.reviewer.split(',')
for i, r in enumerate(reviewers):
if not '@' in r:
reviewers[i] = r + '@chromium.org'
gclient_root = get_gclient_root()
current_dir = os.getcwd()
dependencies = sorted(d.rstrip('/').rstrip('\\') for d in args.dep_path)
cmdline = 'roll-dep ' + ' '.join(dependencies) + ''.join(
' --key ' + k for k in args.key)
try:
if not args.ignore_dirty_tree and not is_pristine(current_dir):
raise Error(
'Ensure %s is clean first (no non-merged commits).' % current_dir)
# First gather all the information without modifying anything, except for a
# git fetch.
deps_path, deps_content = get_deps(current_dir)
gclient_dict = gclient_eval.Parse(deps_content, True, True, deps_path)
is_relative = gclient_dict.get('use_relative_paths', False)
root_dir = current_dir if is_relative else gclient_root
rolls = {}
for dependency in dependencies:
full_dir = os.path.normpath(os.path.join(root_dir, dependency))
if not os.path.isdir(full_dir):
raise Error('Directory not found: %s (%s)' % (dependency, full_dir))
head, roll_to = calculate_roll(
full_dir, dependency, gclient_dict, args.roll_to)
if roll_to == head:
if len(dependencies) == 1:
raise AlreadyRolledError('No revision to roll!')
print('%s: Already at latest commit %s' % (dependency, roll_to))
else:
print(
'%s: Rolling from %s to %s' % (dependency, head[:10], roll_to[:10]))
rolls[dependency] = (head, roll_to)
logs = []
for dependency, (head, roll_to) in sorted(rolls.iteritems()):
full_dir = os.path.normpath(os.path.join(root_dir, dependency))
log = generate_commit_message(
full_dir, dependency, head, roll_to, args.no_log, args.log_limit)
logs.append(log)
gclient_eval.SetRevision(gclient_dict, dependency, roll_to)
deps_content = gclient_eval.RenderDEPSFile(gclient_dict)
commit_msg = gen_commit_msg(logs, cmdline, rolls, reviewers, args.bug)
finalize(commit_msg, deps_path, deps_content, rolls, is_relative, root_dir)
except Error as e:
sys.stderr.write('error: %s\n' % e)
return 2 if isinstance(e, AlreadyRolledError) else 1
print('')
if not reviewers:
print('You forgot to pass -r, make sure to insert a [email protected] line')
print('to the commit description before emailing.')
print('')
print('Run:')
print(' git cl upload --send-mail')
return 0
def make_vendor_file(chromium_version, target_os):
topdir = os.path.join(BASEDIR, chromium_version)
if not os.path.isdir(topdir):
os.makedirs(topdir)
# first checkout depot_tools for gclient.py which will help to produce list of deps
if not os.path.isdir(os.path.join(topdir, "depot_tools")):
checkout_git("https://chromium.googlesource.com/chromium/tools/depot_tools",
"fcde3ba0a657dd3d5cac15ab8a1b6361e293c2fe",
os.path.join(topdir, "depot_tools"))
# Import gclient_eval from the just fetched sources
sys.path.append(os.path.join(topdir, "depot_tools"))
import gclient_eval
# Not setting target_cpu, as it's just used to run script fetching sysroot, which we don't use anyway
target_cpu = []
# Normally set in depot_tools/gclient.py
builtin_vars={
'checkout_android': 'android' in target_os,
'checkout_chromeos': 'chromeos' in target_os,
'checkout_fuchsia': 'fuchsia' in target_os,
'checkout_ios': 'ios' in target_os,
'checkout_linux': 'unix' in target_os,
'checkout_mac': 'mac' in target_os,
'checkout_win': 'win' in target_os,
'checkout_arm': 'arm' in target_cpu,
'checkout_arm64': 'arm64' in target_cpu,
'checkout_x86': 'x86' in target_cpu,
'checkout_mips': 'mips' in target_cpu,
'checkout_mips64': 'mips64' in target_cpu,
'checkout_ppc': 'ppc' in target_cpu,
'checkout_s390': 's390' in target_cpu,
'checkout_x64': 'x64' in target_cpu,
'host_os': 'linux', # See _PLATFORM_MARPPING in depot_tools/gclient.py
'host_cpu': 'x64', # See depot_tools/detect_host_arch.py. Luckily this variable is not currently used in DEPS for anything we care about
}
# like checkout() but do not delete .git (gclient expects it) and do not compute hash
# this subdirectory must have "src" name for 'gclient.py' recognises it
src_dir = os.path.join(topdir, "src")
if not os.path.isdir(src_dir):
os.makedirs(src_dir)
subprocess.check_call(["git", "init"], cwd=src_dir)
subprocess.check_call(["git", "remote", "add", "origin", "https://chromium.googlesource.com/chromium/src.git"], cwd=src_dir)
subprocess.check_call(["git", "fetch", "--progress", "--depth", "1", "origin", "+" + chromium_version], cwd=src_dir)
subprocess.check_call(["git", "checkout", "FETCH_HEAD"], cwd=src_dir)
else:
# restore topdir into virgin state
if ("tag '%s' of" % chromium_version) in open(os.path.join(src_dir, ".git/FETCH_HEAD")).read():
print("already at", chromium_version)
else:
print('git fetch --progress --depth 1 origin "+%s"' % chromium_version)
subprocess.check_call(["git", "fetch", "--progress", "--depth", "1", "origin", "+%s" % chromium_version], cwd=src_dir)
subprocess.check_call(["git", "checkout", "FETCH_HEAD"], cwd=src_dir)
# and remove all symlinks to subprojects, so their DEPS files won;t be included
subprocess.check_call(["find", ".", "-name", ".gitignore", "-delete"], cwd=src_dir)
os.system("cd %s; git status -u -s | grep -v '^ D ' | cut -c4- | xargs --delimiter='\\n' rm" % src_dir);
subprocess.check_call(["git", "checkout", "-f", "HEAD"], cwd=src_dir)
deps = {}
need_another_iteration = True
while need_another_iteration:
need_another_iteration = False
subprocess.check_call(["python2", "depot_tools/gclient.py", "config", "https://chromium.googlesource.com/chromium/src.git"], cwd=topdir)
flat = subprocess.check_output(["python2", "depot_tools/gclient.py", "flatten", "--pin-all-deps"], cwd=topdir)
content = gclient_eval.Parse(flat, validate_syntax=True, filename='DEPS',
vars_override={}, builtin_vars=builtin_vars)
merged_vars = dict(content['vars'])
merged_vars.update(builtin_vars)
for path, fields in content['deps'].iteritems():
# Skip these
if path in SKIP_DEPS:
continue
# Skip dependency if its condition evaluates to False
if 'condition' in fields and not gclient_eval.EvaluateCondition(fields['condition'], merged_vars):
continue
if not path in deps:
if fields['dep_type'] == "git":
url, rev = fields['url'].split('@')
wholepath = os.path.join(topdir, path)
memoized_path = os.path.join(BASEDIR, rev)
if os.path.exists(memoized_path + ".sha256"): # memoize hash
sha256 = open(memoized_path + ".sha256").read()
else:
shutil.rmtree(memoized_path, ignore_errors=True)
sha256 = checkout_git(url, rev, memoized_path)
open(memoized_path + ".sha256", "w").write(sha256)
if path != "src":
shutil.rmtree(wholepath, ignore_errors=True)
if not os.path.isdir(os.path.dirname(wholepath)):
os.makedirs(os.path.dirname(wholepath))
#shutil.copytree(memoized_path, wholepath, copy_function=os.link) # copy_function isn't available in python 2
subprocess.check_call(["cp", "-al", memoized_path, wholepath])
if os.path.exists(os.path.join(memoized_path, "DEPS")): # Need to recurse
need_another_iteration = True
deps[path] = {
"url": url,
"rev": rev,
"sha256": sha256,
"dep_type": "git",
}
elif fields['dep_type'] == "cipd":
packages = []
for p in fields['packages']:
package, version = p['package'], p['version']
dirname = (package + '_' + version).replace('/', '_').replace(':', '') # TODO: Better path normalization
memoized_path = os.path.join(BASEDIR, dirname)
if os.path.exists(memoized_path + ".sha256"): # memoize hash
sha256 = open(memoized_path + ".sha256").read()
else:
shutil.rmtree(memoized_path, ignore_errors=True)
sha256 = checkout_cipd(package, version, memoized_path)
open(memoized_path + ".sha256", "w").write(sha256)
packages.append({
"package": package,
"version": version,
"sha256": sha256,
})
deps[path] = {
"packages": packages,
"dep_type": "cipd",
}
else:
raise ValueError("Unrecognized dep_type", fields['dep_type'])
with open('vendor-%s.nix' % chromium_version, 'w') as vendor_nix:
vendor_nix.write("# GENERATED BY 'mk-vendor-file.py %s' for %s\n" % (chromium_version, ", ".join(target_os)))
vendor_nix.write("{fetchgit, fetchcipd, fetchurl, runCommand, symlinkJoin}:\n");
vendor_nix.write("{\n");
for path, dep in sorted(deps.iteritems()):
if dep['dep_type'] == "git":
vendor_nix.write(nix_str_git(path, dep))
if dep['dep_type'] == "cipd":
vendor_nix.write(nix_str_cipd(path, dep))
# Some additional non-git/cipd sources
for path, name in [("src/third_party/node/node_modules", "chromium-nodejs"),
("src/third_party/test_fonts/test_fonts", "chromium-fonts")]:
sha1 = open(os.path.join(topdir, path + ".tar.gz.sha1")).read().strip()
vendor_nix.write(
'''
"%(path)s" = runCommand "download_from_google_storage" {} ''
mkdir $out
tar xf ${fetchurl {
url = "https://commondatastorage.googleapis.com/%(name)s/%(sha1)s";
sha1 = "%(sha1)s";
}} --strip-components=1 -C $out
'';
''' % { "path": path, "name": name, "sha1": sha1 })
# condition: checkout_android or checkout_linux
# TODO: Memoize
gs_url = open(os.path.join(topdir, 'src/chrome/android/profiles/newest.txt')).read().strip()
GS_HTTP_URL = "https://storage.googleapis.com/"
gz_prefix = "gs://"
if gs_url.startswith(gz_prefix):
url = GS_HTTP_URL + gs_url[len(gz_prefix):]
else:
url = GS_HTTP_URL + "chromeos-prebuilt/afdo-job/llvm/" + gs_url
sha256 = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", url]).strip()
path = "src/chrome/android/profiles/afdo.prof"
vendor_nix.write(
'''
"%(path)s" = runCommand "download_afdo_profile" {} ''
bzip2 -d -c ${fetchurl {
url = "%(url)s";
sha256 = "%(sha256)s";
}} > $out
'';
''' % { "path": path, "url": url, "sha256": sha256 })
local_scope = {}
global_scope = {"__file__": "update.py"}
exec(open(os.path.join(topdir, "src/tools/clang/scripts/update.py")).read(), local_scope, global_scope) # TODO: Safety?
url = '%s/Linux_x64/clang-%s.tgz' % (global_scope['CDS_URL'], global_scope['PACKAGE_VERSION'])
sha256 = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", url]).strip()
path = "src/third_party/llvm-build/Release+Asserts"
vendor_nix.write(
'''
"%(path)s" = runCommand "download_upstream_clang" {} ''
mkdir $out
tar xf ${fetchurl {
url = "%(url)s";
sha256 = "%(sha256)s";
}} -C $out
'';
''' % { "path": path, "url": url, "sha256": sha256 })
vendor_nix.write("}\n")
