def test_tuple_presence(self):
self.assertTrue(
gclient_eval.EvaluateCondition('foo in ("bar", "baz")',
{'foo': 'bar'}))
self.assertFalse(
gclient_eval.EvaluateCondition('foo in ("bar", "baz")',
{'foo': 'not_bar'}))
def test_triple_and_and_or(self):
self.assertTrue(gclient_eval.EvaluateCondition(
'a and b and c or d or e',
{'a': 'False', 'b': 'False', 'c': 'False', 'd': 'False', 'e': 'True'}))
self.assertFalse(gclient_eval.EvaluateCondition(
'a and b and c or d or e',
{'a': 'True', 'b': 'True', 'c': 'False', 'd': 'False', 'e': 'False'}))
def test_unsupported_tuple_operation(self):
with self.assertRaises(ValueError) as cm:
gclient_eval.EvaluateCondition('foo == ("bar", "baz")', {'foo': 'bar'})
self.assertIn('unexpected AST node', str(cm.exception))
with self.assertRaises(ValueError) as cm:
gclient_eval.EvaluateCondition('(foo,) == "bar"', {'foo': 'bar'})
self.assertIn('unexpected AST node', str(cm.exception))
def test_str_in_condition(self):
Str = gclient_eval.ConstantString
self.assertTrue(gclient_eval.EvaluateCondition(
's_var == "foo"',
{'s_var': Str("foo")}))
self.assertFalse(gclient_eval.EvaluateCondition(
's_var in ("baz", "quux")',
{'s_var': Str("foo")}))
def test_triple_or(self):
self.assertTrue(
gclient_eval.EvaluateCondition('a or b or c', {
'a': 'False',
'b': 'False',
'c': 'True'
}))
self.assertFalse(
gclient_eval.EvaluateCondition('a or b or c', {
'a': 'False',
'b': 'False',
'c': 'False'
}))
def test_expansion(self):
self.assertTrue(
gclient_eval.EvaluateCondition('a or b', {
'a': 'b and c',
'b': 'not c',
'c': 'False'
}))
def test_operators(self):
self.assertFalse(
gclient_eval.EvaluateCondition('a and not (b or c)', {
'a': 'True',
'b': 'False',
'c': 'True'
}))
def test_string_bool_typo(self):
with self.assertRaises(ValueError) as cm:
gclient_eval.EvaluateCondition('false_var_str and true_var', {
'false_str_var': 'False',
'true_var': True
})
self.assertIn(
'invalid "and" operand \'false_var_str\' '
'(inside \'false_var_str and true_var\')', str(cm.exception))
def test_non_bool_in_and(self):
with self.assertRaises(ValueError) as cm:
gclient_eval.EvaluateCondition('string_var and true_var', {
'string_var': 'Kittens',
'true_var': True
})
self.assertIn(
'invalid "and" operand \'Kittens\' '
'(inside \'string_var and true_var\')', str(cm.exception))
def main(args):
# Need to be in the root directory
os.chdir(os.path.join(scriptDir, os.pardir, os.pardir))
scope = {}
execfile('.gclient', scope)
for sln in scope['solutions']:
dep_info = loadDepInfo(sln)
hooks = dep_info['hooks']
for hook in hooks:
cond = hook.get('condition', 'True')
if not gclient_eval.EvaluateCondition(cond,
get_vars(dep_info['vars'])):
continue
cmd = hook['action']
if hook['name'] == 'gyp':
cmd.extend(args)
rc = execInShell(cmd)
if 0 != rc:
return rc
return 0
def test_string_bool(self):
self.assertFalse(
gclient_eval.EvaluateCondition('false_str_var and true_var', {
'false_str_var': 'False',
'true_var': True
}))
def test_string_inequality(self):
self.assertTrue(
gclient_eval.EvaluateCondition('foo != "bar"', {'foo': '"baz"'}))
self.assertFalse(
gclient_eval.EvaluateCondition('foo != "baz"', {'foo': '"baz"'}))
def test_variable_cyclic_reference(self):
with self.assertRaises(ValueError) as cm:
self.assertTrue(
gclient_eval.EvaluateCondition('bar', {'bar': 'bar'}))
self.assertIn('invalid cyclic reference to \'bar\' (inside \'bar\')',
str(cm.exception))
def test_variable(self):
self.assertFalse(
gclient_eval.EvaluateCondition('foo', {'foo': 'False'}))
def test_true(self):
self.assertTrue(gclient_eval.EvaluateCondition('True', {}))
def make_vendor_file(chromium_version, target_os):
topdir = os.path.join(BASEDIR, chromium_version)
if not os.path.isdir(topdir):
os.makedirs(topdir)
# first checkout depot_tools for gclient.py which will help to produce list of deps
if not os.path.isdir(os.path.join(topdir, "depot_tools")):
checkout_git("https://chromium.googlesource.com/chromium/tools/depot_tools",
"fcde3ba0a657dd3d5cac15ab8a1b6361e293c2fe",
os.path.join(topdir, "depot_tools"))
# Import gclient_eval from the just fetched sources
sys.path.append(os.path.join(topdir, "depot_tools"))
import gclient_eval
# Not setting target_cpu, as it's just used to run script fetching sysroot, which we don't use anyway
target_cpu = []
# Normally set in depot_tools/gclient.py
builtin_vars={
'checkout_android': 'android' in target_os,
'checkout_chromeos': 'chromeos' in target_os,
'checkout_fuchsia': 'fuchsia' in target_os,
'checkout_ios': 'ios' in target_os,
'checkout_linux': 'unix' in target_os,
'checkout_mac': 'mac' in target_os,
'checkout_win': 'win' in target_os,
'checkout_arm': 'arm' in target_cpu,
'checkout_arm64': 'arm64' in target_cpu,
'checkout_x86': 'x86' in target_cpu,
'checkout_mips': 'mips' in target_cpu,
'checkout_mips64': 'mips64' in target_cpu,
'checkout_ppc': 'ppc' in target_cpu,
'checkout_s390': 's390' in target_cpu,
'checkout_x64': 'x64' in target_cpu,
'host_os': 'linux', # See _PLATFORM_MARPPING in depot_tools/gclient.py
'host_cpu': 'x64', # See depot_tools/detect_host_arch.py. Luckily this variable is not currently used in DEPS for anything we care about
}
# like checkout() but do not delete .git (gclient expects it) and do not compute hash
# this subdirectory must have "src" name for 'gclient.py' recognises it
src_dir = os.path.join(topdir, "src")
if not os.path.isdir(src_dir):
os.makedirs(src_dir)
subprocess.check_call(["git", "init"], cwd=src_dir)
subprocess.check_call(["git", "remote", "add", "origin", "https://chromium.googlesource.com/chromium/src.git"], cwd=src_dir)
subprocess.check_call(["git", "fetch", "--progress", "--depth", "1", "origin", "+" + chromium_version], cwd=src_dir)
subprocess.check_call(["git", "checkout", "FETCH_HEAD"], cwd=src_dir)
else:
# restore topdir into virgin state
if ("tag '%s' of" % chromium_version) in open(os.path.join(src_dir, ".git/FETCH_HEAD")).read():
print("already at", chromium_version)
else:
print('git fetch --progress --depth 1 origin "+%s"' % chromium_version)
subprocess.check_call(["git", "fetch", "--progress", "--depth", "1", "origin", "+%s" % chromium_version], cwd=src_dir)
subprocess.check_call(["git", "checkout", "FETCH_HEAD"], cwd=src_dir)
# and remove all symlinks to subprojects, so their DEPS files won;t be included
subprocess.check_call(["find", ".", "-name", ".gitignore", "-delete"], cwd=src_dir)
os.system("cd %s; git status -u -s | grep -v '^ D ' | cut -c4- | xargs --delimiter='\\n' rm" % src_dir);
subprocess.check_call(["git", "checkout", "-f", "HEAD"], cwd=src_dir)
deps = {}
need_another_iteration = True
while need_another_iteration:
need_another_iteration = False
subprocess.check_call(["python2", "depot_tools/gclient.py", "config", "https://chromium.googlesource.com/chromium/src.git"], cwd=topdir)
flat = subprocess.check_output(["python2", "depot_tools/gclient.py", "flatten", "--pin-all-deps"], cwd=topdir)
content = gclient_eval.Parse(flat, validate_syntax=True, filename='DEPS',
vars_override={}, builtin_vars=builtin_vars)
merged_vars = dict(content['vars'])
merged_vars.update(builtin_vars)
for path, fields in content['deps'].iteritems():
# Skip these
if path in SKIP_DEPS:
continue
# Skip dependency if its condition evaluates to False
if 'condition' in fields and not gclient_eval.EvaluateCondition(fields['condition'], merged_vars):
continue
if not path in deps:
if fields['dep_type'] == "git":
url, rev = fields['url'].split('@')
wholepath = os.path.join(topdir, path)
memoized_path = os.path.join(BASEDIR, rev)
if os.path.exists(memoized_path + ".sha256"): # memoize hash
sha256 = open(memoized_path + ".sha256").read()
else:
shutil.rmtree(memoized_path, ignore_errors=True)
sha256 = checkout_git(url, rev, memoized_path)
open(memoized_path + ".sha256", "w").write(sha256)
if path != "src":
shutil.rmtree(wholepath, ignore_errors=True)
if not os.path.isdir(os.path.dirname(wholepath)):
os.makedirs(os.path.dirname(wholepath))
#shutil.copytree(memoized_path, wholepath, copy_function=os.link) # copy_function isn't available in python 2
subprocess.check_call(["cp", "-al", memoized_path, wholepath])
if os.path.exists(os.path.join(memoized_path, "DEPS")): # Need to recurse
need_another_iteration = True
deps[path] = {
"url": url,
"rev": rev,
"sha256": sha256,
"dep_type": "git",
}
elif fields['dep_type'] == "cipd":
packages = []
for p in fields['packages']:
package, version = p['package'], p['version']
dirname = (package + '_' + version).replace('/', '_').replace(':', '') # TODO: Better path normalization
memoized_path = os.path.join(BASEDIR, dirname)
if os.path.exists(memoized_path + ".sha256"): # memoize hash
sha256 = open(memoized_path + ".sha256").read()
else:
shutil.rmtree(memoized_path, ignore_errors=True)
sha256 = checkout_cipd(package, version, memoized_path)
open(memoized_path + ".sha256", "w").write(sha256)
packages.append({
"package": package,
"version": version,
"sha256": sha256,
})
deps[path] = {
"packages": packages,
"dep_type": "cipd",
}
else:
raise ValueError("Unrecognized dep_type", fields['dep_type'])
with open('vendor-%s.nix' % chromium_version, 'w') as vendor_nix:
vendor_nix.write("# GENERATED BY 'mk-vendor-file.py %s' for %s\n" % (chromium_version, ", ".join(target_os)))
vendor_nix.write("{fetchgit, fetchcipd, fetchurl, runCommand, symlinkJoin}:\n");
vendor_nix.write("{\n");
for path, dep in sorted(deps.iteritems()):
if dep['dep_type'] == "git":
vendor_nix.write(nix_str_git(path, dep))
if dep['dep_type'] == "cipd":
vendor_nix.write(nix_str_cipd(path, dep))
# Some additional non-git/cipd sources
for path, name in [("src/third_party/node/node_modules", "chromium-nodejs"),
("src/third_party/test_fonts/test_fonts", "chromium-fonts")]:
sha1 = open(os.path.join(topdir, path + ".tar.gz.sha1")).read().strip()
vendor_nix.write(
'''
"%(path)s" = runCommand "download_from_google_storage" {} ''
mkdir $out
tar xf ${fetchurl {
url = "https://commondatastorage.googleapis.com/%(name)s/%(sha1)s";
sha1 = "%(sha1)s";
}} --strip-components=1 -C $out
'';
''' % { "path": path, "name": name, "sha1": sha1 })
# condition: checkout_android or checkout_linux
# TODO: Memoize
gs_url = open(os.path.join(topdir, 'src/chrome/android/profiles/newest.txt')).read().strip()
GS_HTTP_URL = "https://storage.googleapis.com/"
gz_prefix = "gs://"
if gs_url.startswith(gz_prefix):
url = GS_HTTP_URL + gs_url[len(gz_prefix):]
else:
url = GS_HTTP_URL + "chromeos-prebuilt/afdo-job/llvm/" + gs_url
sha256 = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", url]).strip()
path = "src/chrome/android/profiles/afdo.prof"
vendor_nix.write(
'''
"%(path)s" = runCommand "download_afdo_profile" {} ''
bzip2 -d -c ${fetchurl {
url = "%(url)s";
sha256 = "%(sha256)s";
}} > $out
'';
''' % { "path": path, "url": url, "sha256": sha256 })
local_scope = {}
global_scope = {"__file__": "update.py"}
exec(open(os.path.join(topdir, "src/tools/clang/scripts/update.py")).read(), local_scope, global_scope) # TODO: Safety?
url = '%s/Linux_x64/clang-%s.tgz' % (global_scope['CDS_URL'], global_scope['PACKAGE_VERSION'])
sha256 = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", url]).strip()
path = "src/third_party/llvm-build/Release+Asserts"
vendor_nix.write(
'''
"%(path)s" = runCommand "download_upstream_clang" {} ''
mkdir $out
tar xf ${fetchurl {
url = "%(url)s";
sha256 = "%(sha256)s";
}} -C $out
'';
''' % { "path": path, "url": url, "sha256": sha256 })
vendor_nix.write("}\n")
