def test_native_case_symlink_right_case(self):
actual = file_path.get_native_path_case(os.path.join(BASE_DIR, "trace_inputs"))
self.assertEqual("trace_inputs", os.path.basename(actual))
# Make sure the symlink is not resolved.
actual = file_path.get_native_path_case(os.path.join(BASE_DIR, "trace_inputs", "files2"))
self.assertEqual("files2", os.path.basename(actual))
def test_native_case_non_existing(self):
# Make sure it doesn't throw on non-existing files.
non_existing = 'trace_input_test_this_file_should_not_exist'
path = os.path.expanduser('~/' + non_existing)
self.assertFalse(os.path.exists(path))
path = file_path.get_native_path_case(ROOT_DIR) + os.path.sep
self.assertEqual(file_path.get_native_path_case(path), path)
def test_native_case_alternate_datastream(self):
# Create the file manually, since tempfile doesn't support ADS.
tempdir = unicode(tempfile.mkdtemp(prefix=u'trace_inputs'))
try:
tempdir = file_path.get_native_path_case(tempdir)
basename = 'foo.txt'
filename = basename + ':Zone.Identifier'
filepath = os.path.join(tempdir, filename)
open(filepath, 'w').close()
self.assertEqual(filepath,
file_path.get_native_path_case(filepath))
data_suffix = ':$DATA'
self.assertEqual(
filepath + data_suffix,
file_path.get_native_path_case(filepath + data_suffix))
open(filepath + '$DATA', 'w').close()
self.assertEqual(
filepath + data_suffix,
file_path.get_native_path_case(filepath + data_suffix))
# Ensure the ADS weren't created as separate file. You love NTFS, don't
# you?
self.assertEqual([basename], fs.listdir(tempdir))
finally:
file_path.rmtree(tempdir)
def process_isolate_options(parser, options, cwd=None, require_isolated=True):
"""Handles options added with 'add_isolate_options'.
Mutates |options| in place, by normalizing path to isolate file, values of
variables, etc.
"""
cwd = file_path.get_native_path_case(unicode(cwd or os.getcwd()))
# Parse --isolated option.
if options.isolated:
options.isolated = os.path.normpath(os.path.join(cwd, options.isolated.replace("/", os.path.sep)))
if require_isolated and not options.isolated:
parser.error("--isolated is required.")
if options.isolated and not options.isolated.endswith(".isolated"):
parser.error("--isolated value must end with '.isolated'")
# Processes all the --<foo>-variable flags.
def try_make_int(s):
"""Converts a value to int if possible, converts to unicode otherwise."""
try:
return int(s)
except ValueError:
return s.decode("utf-8")
options.config_variables = dict((k, try_make_int(v)) for k, v in options.config_variables)
options.path_variables = dict(options.path_variables)
options.extra_variables = dict(options.extra_variables)
# Normalize the path in --isolate.
if options.isolate:
# TODO(maruel): Work with non-ASCII.
# The path must be in native path case for tracing purposes.
options.isolate = unicode(options.isolate).replace("/", os.path.sep)
options.isolate = os.path.normpath(os.path.join(cwd, options.isolate))
options.isolate = file_path.get_native_path_case(options.isolate)
def test_native_case_non_existing(self):
# Make sure it doesn't throw on non-existing files.
non_existing = 'trace_input_test_this_file_should_not_exist'
path = os.path.expanduser('~/' + non_existing)
self.assertFalse(os.path.exists(path))
path = file_path.get_native_path_case(ROOT_DIR) + os.path.sep
self.assertEqual(file_path.get_native_path_case(path), path)
def test_native_case_symlink_right_case(self):
actual = file_path.get_native_path_case(
os.path.join(BASE_DIR, 'trace_inputs'))
self.assertEqual('trace_inputs', os.path.basename(actual))
# Make sure the symlink is not resolved.
actual = file_path.get_native_path_case(
os.path.join(BASE_DIR, 'trace_inputs', 'files2'))
self.assertEqual('files2', os.path.basename(actual))
def test_subdir(self):
# The resulting .isolated file will be missing ../../isolate.py. It is
# because this file is outside the --subdir parameter.
isolate_file = os.path.join(
ROOT_DIR, 'tests', 'isolate', 'touch_root.isolate')
options = self._get_option(isolate_file)
complete_state = isolate.load_complete_state(
options, self.cwd, os.path.join('tests', 'isolate'), False)
actual_isolated = complete_state.saved_state.to_isolated()
actual_saved_state = complete_state.saved_state.flatten()
expected_isolated = {
'algo': 'sha-1',
'command': ['python', 'touch_root.py'],
'files': {
os.path.join(u'tests', 'isolate', 'touch_root.py'): {
'm': 488,
'h': hash_file('tests', 'isolate', 'touch_root.py'),
's': _size('tests', 'isolate', 'touch_root.py'),
},
},
'relative_cwd': os.path.join(u'tests', 'isolate'),
'version': isolate.isolateserver.ISOLATED_FILE_VERSION,
}
self._cleanup_isolated(expected_isolated)
self.assertEqual(expected_isolated, actual_isolated)
expected_saved_state = {
'OS': sys.platform,
'algo': 'sha-1',
'child_isolated_files': [],
'command': ['python', 'touch_root.py'],
'config_variables': {
'OS': 'linux',
'chromeos': 1,
},
'extra_variables': {
'foo': 'bar',
},
'files': {
os.path.join(u'tests', 'isolate', 'touch_root.py'): {
'm': 488,
'h': hash_file('tests', 'isolate', 'touch_root.py'),
's': _size('tests', 'isolate', 'touch_root.py'),
},
},
'isolate_file': file_path.safe_relpath(
file_path.get_native_path_case(isolate_file),
os.path.dirname(options.isolated)),
'path_variables': {},
'relative_cwd': os.path.join(u'tests', 'isolate'),
'root_dir': file_path.get_native_path_case(ROOT_DIR),
'version': isolate.SavedState.EXPECTED_VERSION,
}
self._cleanup_isolated(expected_saved_state)
self._cleanup_saved_state(actual_saved_state)
self.assertEqual(expected_saved_state, actual_saved_state)
def test_subdir(self):
# The resulting .isolated file will be missing ../../isolate.py. It is
# because this file is outside the --subdir parameter.
isolate_file = os.path.join(
ROOT_DIR, 'tests', 'isolate', 'touch_root.isolate')
options = self._get_option(isolate_file)
complete_state = isolate.load_complete_state(
options, self.cwd, os.path.join('tests', 'isolate'), False)
actual_isolated = complete_state.saved_state.to_isolated()
actual_saved_state = complete_state.saved_state.flatten()
expected_isolated = {
'algo': 'sha-1',
'command': ['python', 'touch_root.py'],
'files': {
os.path.join(u'tests', 'isolate', 'touch_root.py'): {
'm': 488,
'h': hash_file('tests', 'isolate', 'touch_root.py'),
's': _size('tests', 'isolate', 'touch_root.py'),
},
},
'relative_cwd': os.path.join(u'tests', 'isolate'),
'version': isolate.isolateserver.ISOLATED_FILE_VERSION,
}
self._cleanup_isolated(expected_isolated)
self.assertEqual(expected_isolated, actual_isolated)
expected_saved_state = {
'OS': sys.platform,
'algo': 'sha-1',
'child_isolated_files': [],
'command': ['python', 'touch_root.py'],
'config_variables': {
'OS': 'linux',
'chromeos': 1,
},
'extra_variables': {
'foo': 'bar',
},
'files': {
os.path.join(u'tests', 'isolate', 'touch_root.py'): {
'm': 488,
'h': hash_file('tests', 'isolate', 'touch_root.py'),
's': _size('tests', 'isolate', 'touch_root.py'),
},
},
'isolate_file': file_path.safe_relpath(
file_path.get_native_path_case(isolate_file),
os.path.dirname(options.isolated)),
'path_variables': {},
'relative_cwd': os.path.join(u'tests', 'isolate'),
'root_dir': file_path.get_native_path_case(ROOT_DIR),
'version': isolate.SavedState.EXPECTED_VERSION,
}
self._cleanup_isolated(expected_saved_state)
self._cleanup_saved_state(actual_saved_state)
self.assertEqual(expected_saved_state, actual_saved_state)
def test_native_case_not_sensitive_non_existent(self):
# This test also ensures that the output is independent on the input
# string case.
non_existing = os.path.join("trace_input_test_this_dir_should_not_exist", "really not", "")
path = os.path.expanduser(os.path.join(u"~", non_existing))
path = path.replace("/", os.path.sep)
self.assertFalse(os.path.exists(path))
lower = file_path.get_native_path_case(path.lower())
upper = file_path.get_native_path_case(path.upper())
# Make sure non-existing element is not modified:
self.assertTrue(lower.endswith(non_existing.lower()))
self.assertTrue(upper.endswith(non_existing.upper()))
self.assertEqual(lower[: -len(non_existing)], upper[: -len(non_existing)])
def test_native_case_not_sensitive_non_existent(self):
# This test also ensures that the output is independent on the input
# string case.
non_existing = os.path.join(
'trace_input_test_this_dir_should_not_exist', 'really not', '')
path = os.path.expanduser(os.path.join(u'~', non_existing))
path = path.replace('/', os.path.sep)
self.assertFalse(fs.exists(path))
lower = file_path.get_native_path_case(path.lower())
upper = file_path.get_native_path_case(path.upper())
# Make sure non-existing element is not modified:
self.assertTrue(lower.endswith(non_existing.lower()))
self.assertTrue(upper.endswith(non_existing.upper()))
self.assertEqual(lower[:-len(non_existing)], upper[:-len(non_existing)])
def test_native_case_not_sensitive(self):
# The home directory is almost guaranteed to have mixed upper/lower case
# letters on both Windows and OSX.
# This test also ensures that the output is independent on the input
# string case.
path = os.path.expanduser(u"~")
self.assertTrue(os.path.isdir(path))
path = path.replace("/", os.path.sep)
if sys.platform == "win32":
# Make sure the drive letter is upper case for consistency.
path = path[0].upper() + path[1:]
# This test assumes the variable is in the native path case on disk, this
# should be the case. Verify this assumption:
self.assertEqual(path, file_path.get_native_path_case(path))
self.assertEqual(file_path.get_native_path_case(path.lower()), file_path.get_native_path_case(path.upper()))
def _expected_saved_state(
self, args, read_only, empty_file, extra_vars, root_dir):
expected = {
u'OS': unicode(sys.platform),
u'algo': u'sha-1',
u'child_isolated_files': [],
u'command': [],
u'config_variables': {
u'OS': u'mac',
u'chromeos': 0,
},
u'extra_variables': {
u'EXECUTABLE_SUFFIX': u'.exe' if sys.platform == 'win32' else u'',
},
u'files': self._gen_files(read_only, empty_file, True),
u'isolate_file': file_path.safe_relpath(
file_path.get_native_path_case(unicode(self.filename())),
unicode(os.path.dirname(self.isolated))),
u'path_variables': {},
u'relative_cwd': unicode(RELATIVE_CWD[self.case()]),
u'root_dir': unicode(root_dir or os.path.dirname(self.filename())),
u'version': unicode(isolate.SavedState.EXPECTED_VERSION),
}
if args:
expected[u'command'] = [u'python'] + [unicode(x) for x in args]
expected['extra_variables'].update(extra_vars or {})
with open(self.saved_state(), 'r') as f:
self.assertEqual(expected, json.load(f))
def test_native_case_not_sensitive(self):
# The home directory is almost guaranteed to have mixed upper/lower case
# letters on both Windows and OSX.
# This test also ensures that the output is independent on the input
# string case.
path = os.path.expanduser(u'~')
self.assertTrue(os.path.isdir(path))
path = path.replace('/', os.path.sep)
if sys.platform == 'win32':
# Make sure the drive letter is upper case for consistency.
path = path[0].upper() + path[1:]
# This test assumes the variable is in the native path case on disk, this
# should be the case. Verify this assumption:
self.assertEqual(path, file_path.get_native_path_case(path))
self.assertEqual(file_path.get_native_path_case(path.lower()),
file_path.get_native_path_case(path.upper()))
def _expected_saved_state(self, args, read_only, empty_file, extra_vars):
flavor = isolate.get_flavor()
chromeos_value = int(flavor == 'linux')
expected = {
u'algo': u'sha-1',
u'child_isolated_files': [],
u'command': [],
u'config_variables': {
u'OS': unicode(flavor),
u'chromeos': chromeos_value,
},
u'extra_variables': {
u'EXECUTABLE_SUFFIX': u'.exe' if flavor == 'win' else u'',
},
u'files': self._gen_files(read_only, empty_file, True),
u'isolate_file': file_path.safe_relpath(
file_path.get_native_path_case(unicode(self.filename())),
unicode(os.path.dirname(self.isolated))),
u'relative_cwd': unicode(RELATIVE_CWD[self.case()]),
u'path_variables': {},
u'version': unicode(isolate.isolateserver.ISOLATED_FILE_VERSION),
}
if args:
expected[u'command'] = [u'python'] + [unicode(x) for x in args]
expected['extra_variables'].update(extra_vars or {})
self.assertEqual(expected, json.load(open(self.saved_state(), 'r')))
def setUp(self):
super(IsolateTempdirBase, self).setUp()
self.tempdir = file_path.get_native_path_case(
unicode(tempfile.mkdtemp(prefix=u'isolate_smoke_')))
self.isolated = os.path.join(self.tempdir,
'isolate_smoke_test.isolated')
self.isolate_dir = os.path.join(self.tempdir, 'isolate')
def _expected_saved_state(self, args, read_only, empty_file, extra_vars,
root_dir):
expected = {
u'OS':
unicode(sys.platform),
u'algo':
u'sha-1',
u'child_isolated_files': [],
u'command': [],
u'config_variables': {
u'OS': u'mac',
u'chromeos': 0,
},
u'extra_variables': {
u'EXECUTABLE_SUFFIX':
u'.exe' if sys.platform == 'win32' else u'',
},
u'files':
self._gen_files(read_only, empty_file, True),
u'isolate_file':
file_path.safe_relpath(
file_path.get_native_path_case(unicode(self.filename())),
unicode(os.path.dirname(self.isolated))),
u'path_variables': {},
u'relative_cwd':
unicode(RELATIVE_CWD[self.case()]),
u'root_dir':
unicode(root_dir or os.path.dirname(self.filename())),
u'version':
unicode(isolate.SavedState.EXPECTED_VERSION),
}
def _expected_saved_state(self, args, read_only, empty_file, root_dir):
expected = {
u'OS':
six.text_type(sys.platform),
u'algo':
u'sha-1',
u'child_isolated_files': [],
u'command': [],
u'config_variables': {
u'OS': u'mac',
u'chromeos': 0,
},
u'files':
self._gen_files(read_only, empty_file),
u'isolate_file':
file_path.safe_relpath(
file_path.get_native_path_case(six.ensure_text(
self.filename())),
six.text_type(os.path.dirname(self.isolated))),
u'path_variables': {},
u'relative_cwd':
six.text_type(RELATIVE_CWD[self.case()]),
u'root_dir':
six.text_type(root_dir or os.path.dirname(self.filename())),
u'version':
six.text_type(isolate.SavedState.EXPECTED_VERSION),
}
def file_to_metadata(filepath, collapse_symlinks):
"""Processes an input file, a dependency, and return meta data about it.
Behaviors:
- Retrieves the file mode, file size, file timestamp, file link
destination if it is a file link and calcultate the SHA-1 of the file's
content if the path points to a file and not a symlink.
Arguments:
filepath: File to act on.
collapse_symlinks: True if symlinked files should be treated like they were
the normal underlying file.
Returns:
The necessary dict to create a entry in the 'files' section of an .isolated
file *except* 'h' for files.
"""
out = {}
# Always check the file stat and check if it is a link.
try:
if collapse_symlinks:
# os.stat follows symbolic links
filestats = fs.stat(filepath)
else:
# os.lstat does not follow symbolic links, and thus preserves them.
filestats = fs.lstat(filepath)
except OSError:
# The file is not present.
raise MappingError('%s is missing' % filepath)
is_link = stat.S_ISLNK(filestats.st_mode)
if sys.platform != 'win32':
# Ignore file mode on Windows since it's not really useful there.
filemode = stat.S_IMODE(filestats.st_mode)
# Remove write access for group and all access to 'others'.
filemode &= ~(stat.S_IWGRP | stat.S_IRWXO)
if filemode & (stat.S_IXUSR|stat.S_IRGRP) == (stat.S_IXUSR|stat.S_IRGRP):
# Only keep x group bit if both x user bit and group read bit are set.
filemode |= stat.S_IXGRP
else:
filemode &= ~stat.S_IXGRP
if not is_link:
out['m'] = filemode
if not is_link:
out['s'] = filestats.st_size
else:
# The link could be in an incorrect path case. In practice, this only
# happens on macOS on case insensitive HFS.
# TODO(maruel): It'd be better if it was only done once, in
# expand_directory_and_symlink(), so it would not be necessary to do again
# here.
symlink_value = fs.readlink(filepath) # pylint: disable=no-member
filedir = file_path.get_native_path_case(os.path.dirname(filepath))
native_dest = file_path.fix_native_path_case(filedir, symlink_value)
out['l'] = os.path.relpath(native_dest, filedir)
return out
def test_native_case_symlink_wrong_case(self):
base_dir = file_path.get_native_path_case(BASE_DIR)
trace_inputs_dir = os.path.join(base_dir, "trace_inputs")
actual = file_path.get_native_path_case(trace_inputs_dir)
self.assertEqual(trace_inputs_dir, actual)
# Make sure the symlink is not resolved.
data = os.path.join(trace_inputs_dir, "Files2")
actual = file_path.get_native_path_case(data)
self.assertEqual(os.path.join(trace_inputs_dir, "files2"), actual)
data = os.path.join(trace_inputs_dir, "Files2", "")
actual = file_path.get_native_path_case(data)
self.assertEqual(os.path.join(trace_inputs_dir, "files2", ""), actual)
data = os.path.join(trace_inputs_dir, "Files2", "Child1.py")
actual = file_path.get_native_path_case(data)
# TODO(maruel): Should be child1.py.
self.assertEqual(os.path.join(trace_inputs_dir, "files2", "Child1.py"), actual)
def normalize_path_variables(cwd, path_variables, relative_base_dir):
"""Processes path variables as a special case and returns a copy of the dict.
For each 'path' variable: first normalizes it based on |cwd|, verifies it
exists then sets it as relative to relative_base_dir.
"""
logging.info("normalize_path_variables(%s, %s, %s)", cwd, path_variables, relative_base_dir)
assert isinstance(cwd, unicode), cwd
assert isinstance(relative_base_dir, unicode), relative_base_dir
relative_base_dir = file_path.get_native_path_case(relative_base_dir)
return dict((k, _normalize_path_variable(cwd, relative_base_dir, k, v)) for k, v in path_variables.iteritems())
def test_native_case_alternate_datastream(self):
# Create the file manually, since tempfile doesn't support ADS.
tempdir = unicode(tempfile.mkdtemp(prefix="trace_inputs"))
try:
tempdir = file_path.get_native_path_case(tempdir)
basename = "foo.txt"
filename = basename + ":Zone.Identifier"
filepath = os.path.join(tempdir, filename)
open(filepath, "w").close()
self.assertEqual(filepath, file_path.get_native_path_case(filepath))
data_suffix = ":$DATA"
self.assertEqual(filepath + data_suffix, file_path.get_native_path_case(filepath + data_suffix))
open(filepath + "$DATA", "w").close()
self.assertEqual(filepath + data_suffix, file_path.get_native_path_case(filepath + data_suffix))
# Ensure the ADS weren't created as separate file. You love NTFS, don't
# you?
self.assertEqual([basename], os.listdir(tempdir))
finally:
shutil.rmtree(tempdir)
def _check_merge(self, filename):
filepath = file_path.get_native_path_case(
os.path.join(unicode(ROOT_DIR), 'tests', 'isolate', filename))
expected = 'Updating %s\n' % isolate.safe_relpath(filepath, self.tempdir)
with open(filepath, 'rb') as f:
old_content = f.read()
out = self._execute('merge', filename, [], True) or ''
self.assertEqual(expected, out)
with open(filepath, 'rb') as f:
new_content = f.read()
self.assertEqual(old_content, new_content)
def _check_merge(self, filename):
filepath = file_path.get_native_path_case(
os.path.join(unicode(ROOT_DIR), 'tests', 'isolate', filename))
expected = 'Updating %s\n' % isolate.safe_relpath(
filepath, self.tempdir)
with open(filepath, 'rb') as f:
old_content = f.read()
out = self._execute('merge', filename, [], True) or ''
self.assertEqual(expected, out)
with open(filepath, 'rb') as f:
new_content = f.read()
self.assertEqual(old_content, new_content)
def test_native_case_symlink_wrong_case(self):
base_dir = file_path.get_native_path_case(BASE_DIR)
trace_inputs_dir = os.path.join(base_dir, 'trace_inputs')
actual = file_path.get_native_path_case(trace_inputs_dir)
self.assertEqual(trace_inputs_dir, actual)
# Make sure the symlink is not resolved.
data = os.path.join(trace_inputs_dir, 'Files2')
actual = file_path.get_native_path_case(data)
self.assertEqual(os.path.join(trace_inputs_dir, 'files2'), actual)
data = os.path.join(trace_inputs_dir, 'Files2', '')
actual = file_path.get_native_path_case(data)
self.assertEqual(os.path.join(trace_inputs_dir, 'files2', ''),
actual)
data = os.path.join(trace_inputs_dir, 'Files2', 'Child1.py')
actual = file_path.get_native_path_case(data)
# TODO(maruel): Should be child1.py.
self.assertEqual(
os.path.join(trace_inputs_dir, 'files2', 'Child1.py'), actual)
def _test_missing_trailing_slash(self, mode):
try:
self._execute(mode, 'missing_trailing_slash.isolate', [])
self.fail()
except subprocess.CalledProcessError as e:
self.assertEqual('', e.output)
out = e.stderr
self._expect_no_result()
root = file_path.get_native_path_case(unicode(self.isolate_dir))
expected = ('Input directory %s must have a trailing slash' %
os.path.join(root, 'tests', 'isolate', 'files1'))
self.assertIn(expected, out)
def process_isolate_options(parser, options, cwd=None, require_isolated=True):
"""Handles options added with 'add_isolate_options'.
Mutates |options| in place, by normalizing path to isolate file, values of
variables, etc.
"""
cwd = file_path.get_native_path_case(unicode(cwd or os.getcwd()))
# Parse --isolated option.
if options.isolated:
options.isolated = os.path.abspath(
os.path.join(cwd, unicode(options.isolated).replace('/', os.path.sep)))
if require_isolated and not options.isolated:
parser.error('--isolated is required.')
if options.isolated and not options.isolated.endswith('.isolated'):
parser.error('--isolated value must end with \'.isolated\'')
# Processes all the --<foo>-variable flags.
def try_make_int(s):
"""Converts a value to int if possible, converts to unicode otherwise."""
try:
return int(s)
except ValueError:
return s.decode('utf-8')
options.config_variables = dict(
(k, try_make_int(v)) for k, v in options.config_variables)
options.path_variables = dict(options.path_variables)
options.extra_variables = dict(options.extra_variables)
# Account for default EXECUTABLE_SUFFIX.
if (options.config_variables or options.path_variables or
len(options.extra_variables) > 1):
sys.stderr.write(_VARIABLE_WARNING)
# Normalize the path in --isolate.
if options.isolate:
# TODO(maruel): Work with non-ASCII.
# The path must be in native path case for tracing purposes.
options.isolate = unicode(options.isolate).replace('/', os.path.sep)
options.isolate = os.path.abspath(os.path.join(cwd, options.isolate))
options.isolate = file_path.get_native_path_case(options.isolate)
def _test_non_existent(self, mode):
try:
self._execute(mode, 'non_existent.isolate', [])
self.fail()
except subprocess.CalledProcessError as e:
self.assertEqual('', e.output)
out = e.stderr
self._expect_no_result()
root = file_path.get_native_path_case(unicode(self.isolate_dir))
expected = (
'Input file %s doesn\'t exist' %
os.path.join(root, 'tests', 'isolate', 'A_file_that_do_not_exist'))
self.assertIn(expected, out)
def normalize_path_variables(cwd, path_variables, relative_base_dir):
"""Processes path variables as a special case and returns a copy of the dict.
For each 'path' variable: first normalizes it based on |cwd|, verifies it
exists then sets it as relative to relative_base_dir.
"""
logging.info('normalize_path_variables(%s, %s, %s)', cwd, path_variables,
relative_base_dir)
assert isinstance(cwd, unicode), cwd
assert isinstance(relative_base_dir, unicode), relative_base_dir
relative_base_dir = file_path.get_native_path_case(relative_base_dir)
return dict((k, _normalize_path_variable(cwd, relative_base_dir, k, v))
for k, v in path_variables.iteritems())
def setUp(self):
self.tempdir = None
self.trace_inputs_path = os.path.join(test_env.CLIENT_DIR,
'trace_inputs.py')
# Wraps up all the differences between OSes here.
# - Windows doesn't track initial_cwd.
# - OSX replaces /usr/bin/python with /usr/bin/python2.7.
self.cwd = test_env.TESTS_DIR
self.initial_cwd = self.cwd
self.expected_cwd = test_env.CLIENT_DIR
if sys.platform == 'win32':
# Not supported on Windows.
self.initial_cwd = None
self.expected_cwd = None
# There's 3 kinds of references to python, self.executable,
# self.real_executable and self.naked_executable. It depends how python was
# started.
self.executable = sys.executable
if sys.platform == 'darwin':
# /usr/bin/python is a thunk executable that decides which version of
# python gets executed.
suffix = '.'.join(map(str, sys.version_info[0:2]))
if os.access(self.executable + suffix, os.X_OK):
# So it'll look like /usr/bin/python2.7
self.executable += suffix
self.real_executable = file_path.get_native_path_case(
unicode(self.executable))
self.tempdir = file_path.get_native_path_case(
unicode(tempfile.mkdtemp(prefix=u'trace_smoke_test')))
self.log = os.path.join(self.tempdir, 'log')
# self.naked_executable will only be naked on Windows.
self.naked_executable = unicode(sys.executable)
if sys.platform == 'win32':
self.naked_executable = os.path.basename(sys.executable)
def _test_missing_trailing_slash(self, mode):
try:
self._execute(mode, 'missing_trailing_slash.isolate', [])
self.fail()
except subprocess.CalledProcessError as e:
self.assertEqual('', e.output)
out = e.stderr
self._expect_no_result()
root = file_path.get_native_path_case(unicode(self.isolate_dir))
expected = (
'Input directory %s must have a trailing slash' %
os.path.join(root, 'tests', 'isolate', 'files1')
)
self.assertIn(expected, out)
def _test_non_existent(self, mode):
try:
self._execute(mode, 'non_existent.isolate', [])
self.fail()
except subprocess.CalledProcessError as e:
self.assertEqual('', e.output)
out = e.stderr
self._expect_no_result()
root = file_path.get_native_path_case(unicode(self.isolate_dir))
expected = (
'Input file %s doesn\'t exist' %
os.path.join(root, 'tests', 'isolate', 'A_file_that_do_not_exist')
)
self.assertIn(expected, out)
def test_variable_not_exist(self):
isolate_file = os.path.join(
ROOT_DIR, 'tests', 'isolate', 'touch_root.isolate')
options = self._get_option(isolate_file)
options.path_variables['PRODUCT_DIR'] = os.path.join(u'tests', u'isolate')
native_cwd = file_path.get_native_path_case(unicode(self.cwd))
try:
isolate.load_complete_state(options, self.cwd, None, False)
self.fail()
except isolate.ExecutionError, e:
self.assertEqual(
'PRODUCT_DIR=%s is not a directory' %
os.path.join(native_cwd, 'tests', 'isolate'),
e.args[0])
def test_variable_not_exist(self):
isolate_file = os.path.join(
ROOT_DIR, 'tests', 'isolate', 'touch_root.isolate')
options = self._get_option(isolate_file)
options.path_variables['PRODUCT_DIR'] = os.path.join(u'tests', u'isolate')
native_cwd = file_path.get_native_path_case(unicode(self.cwd))
try:
isolate.load_complete_state(options, self.cwd, None, False)
self.fail()
except isolate.ExecutionError, e:
self.assertEqual(
'PRODUCT_DIR=%s is not a directory' %
os.path.join(native_cwd, 'tests', 'isolate'),
e.args[0])
def _normalize_path_variable(cwd, relative_base_dir, key, value):
"""Normalizes a path variable into a relative directory.
"""
# Variables could contain / or \ on windows. Always normalize to
# os.path.sep.
x = os.path.join(cwd, value.strip().replace("/", os.path.sep))
normalized = file_path.get_native_path_case(os.path.normpath(x))
if not os.path.isdir(normalized):
raise ExecutionError("%s=%s is not a directory" % (key, normalized))
# All variables are relative to the .isolate file.
normalized = os.path.relpath(normalized, relative_base_dir)
logging.debug("Translated variable %s from %s to %s", key, value, normalized)
return normalized
def setUp(self):
self.tempdir = None
self.trace_inputs_path = os.path.join(ROOT_DIR, 'trace_inputs.py')
# Wraps up all the differences between OSes here.
# - Windows doesn't track initial_cwd.
# - OSX replaces /usr/bin/python with /usr/bin/python2.7.
self.cwd = os.path.join(ROOT_DIR, u'tests')
self.initial_cwd = unicode(self.cwd)
self.expected_cwd = unicode(ROOT_DIR)
if sys.platform == 'win32':
# Not supported on Windows.
self.initial_cwd = None
self.expected_cwd = None
# There's 3 kinds of references to python, self.executable,
# self.real_executable and self.naked_executable. It depends how python was
# started.
self.executable = sys.executable
if sys.platform == 'darwin':
# /usr/bin/python is a thunk executable that decides which version of
# python gets executed.
suffix = '.'.join(map(str, sys.version_info[0:2]))
if os.access(self.executable + suffix, os.X_OK):
# So it'll look like /usr/bin/python2.7
self.executable += suffix
self.real_executable = file_path.get_native_path_case(
unicode(self.executable))
self.tempdir = file_path.get_native_path_case(
unicode(tempfile.mkdtemp(prefix='trace_smoke_test')))
self.log = os.path.join(self.tempdir, 'log')
# self.naked_executable will only be naked on Windows.
self.naked_executable = unicode(sys.executable)
if sys.platform == 'win32':
self.naked_executable = os.path.basename(sys.executable)
def _normalize_path_variable(cwd, relative_base_dir, key, value):
"""Normalizes a path variable into a relative directory.
"""
# Variables could contain / or \ on windows. Always normalize to
# os.path.sep.
x = os.path.join(cwd, value.strip().replace('/', os.path.sep))
normalized = file_path.get_native_path_case(os.path.normpath(x))
if not os.path.isdir(normalized):
raise ExecutionError('%s=%s is not a directory' % (key, normalized))
# All variables are relative to the .isolate file.
normalized = os.path.relpath(normalized, relative_base_dir)
logging.debug('Translated variable %s from %s to %s', key, value,
normalized)
return normalized
def test_trace(self):
expected = self._gen_dict_full_gyp()
results = self._execute_trace(self.get_child_command(True))
actual = results.flatten()
self.assertTrue(actual['root'].pop('pid'))
self.assertTrue(actual['root']['children'][0].pop('pid'))
self.assertEqual(expected, actual)
files = [
u'tests/trace_inputs/child1.py'.replace('/', os.path.sep),
u'tests/trace_inputs/child2.py'.replace('/', os.path.sep),
u'tests/trace_inputs/files1/'.replace('/', os.path.sep),
u'tests/trace_inputs/test_file.txt'.replace('/', os.path.sep),
u'tests/trace_inputs_smoke_test.py'.replace('/', os.path.sep),
u'trace_inputs.py',
]
def blacklist(f):
return f.endswith(('.pyc', 'do_not_care.txt', '.git', '.svn'))
simplified = trace_inputs.extract_directories(
file_path.get_native_path_case(unicode(ROOT_DIR)),
results.files,
blacklist)
self.assertEqual(files, [f.path for f in simplified])
def test_trace(self):
expected = self._gen_dict_full_gyp()
results = self._execute_trace(self.get_child_command(True))
actual = results.flatten()
self.assertTrue(actual['root'].pop('pid'))
self.assertTrue(actual['root']['children'][0].pop('pid'))
self.assertEqual(expected, actual)
files = [
u'tests/trace_inputs/child1.py'.replace('/', os.path.sep),
u'tests/trace_inputs/child2.py'.replace('/', os.path.sep),
u'tests/trace_inputs/files1/'.replace('/', os.path.sep),
u'tests/trace_inputs/test_file.txt'.replace('/', os.path.sep),
u'tests/trace_inputs_smoke_test.py'.replace('/', os.path.sep),
u'trace_inputs.py',
]
def blacklist(f):
return f.endswith(('.pyc', 'do_not_care.txt', '.git', '.svn'))
simplified = trace_inputs.extract_directories(
file_path.get_native_path_case(unicode(ROOT_DIR)),
results.files,
blacklist)
self.assertEqual(files, [f.path for f in simplified])
def setUp(self):
self.temp_file = None
self.initial_cwd = GOOGLETEST_DIR
if sys.platform == 'win32':
# Windows has no kernel mode concept of current working directory.
self.initial_cwd = None
# There's 2 kinds of references to python, self.executable,
# self.real_executable. It depends how python was started and on which OS.
self.executable = unicode(sys.executable)
if sys.platform == 'darwin':
# /usr/bin/python is a thunk executable that decides which version of
# python gets executed.
suffix = '.'.join(map(str, sys.version_info[0:2]))
if os.access(self.executable + suffix, os.X_OK):
# So it'll look like /usr/bin/python2.7
self.executable += suffix
self.real_executable = file_path.get_native_path_case(self.executable)
# Make sure there's no environment variable that could do side effects.
os.environ.pop('GTEST_SHARD_INDEX', '')
os.environ.pop('GTEST_TOTAL_SHARDS', '')
def setUp(self):
self.temp_file = None
self.initial_cwd = GOOGLETEST_DIR
if sys.platform == 'win32':
# Windows has no kernel mode concept of current working directory.
self.initial_cwd = None
# There's 2 kinds of references to python, self.executable,
# self.real_executable. It depends how python was started and on which OS.
self.executable = unicode(sys.executable)
if sys.platform == 'darwin':
# /usr/bin/python is a thunk executable that decides which version of
# python gets executed.
suffix = '.'.join(map(str, sys.version_info[0:2]))
if os.access(self.executable + suffix, os.X_OK):
# So it'll look like /usr/bin/python2.7
self.executable += suffix
self.real_executable = file_path.get_native_path_case(self.executable)
# Make sure there's no environment variable that could do side effects.
os.environ.pop('GTEST_SHARD_INDEX', '')
os.environ.pop('GTEST_TOTAL_SHARDS', '')
def _expected_saved_state(self, args, read_only, empty_file, extra_vars):
flavor = isolate.get_flavor()
chromeos_value = int(flavor == 'linux')
expected = {
u'algo':
u'sha-1',
u'child_isolated_files': [],
u'command': [],
u'files':
self._gen_files(read_only, empty_file, True),
u'isolate_file':
isolate.safe_relpath(
file_path.get_native_path_case(unicode(self.filename())),
unicode(os.path.dirname(self.isolated))),
u'relative_cwd':
unicode(RELATIVE_CWD[self.case()]),
u'variables': {
u'EXECUTABLE_SUFFIX': u'.exe' if flavor == 'win' else u'',
u'OS': unicode(flavor),
u'chromeos': chromeos_value,
},
u'version':
u'1.0',
}
def file_to_metadata(filepath, prevdict, read_only, algo):
"""Processes an input file, a dependency, and return meta data about it.
Behaviors:
- Retrieves the file mode, file size, file timestamp, file link
destination if it is a file link and calcultate the SHA-1 of the file's
content if the path points to a file and not a symlink.
Arguments:
filepath: File to act on.
prevdict: the previous dictionary. It is used to retrieve the cached sha-1
to skip recalculating the hash. Optional.
read_only: If 1 or 2, the file mode is manipulated. In practice, only save
one of 4 modes: 0755 (rwx), 0644 (rw), 0555 (rx), 0444 (r). On
windows, mode is not set since all files are 'executable' by
default.
algo: Hashing algorithm used.
Returns:
The necessary dict to create a entry in the 'files' section of an .isolated
file.
"""
# TODO(maruel): None is not a valid value.
assert read_only in (None, 0, 1, 2), read_only
out = {}
# Always check the file stat and check if it is a link. The timestamp is used
# to know if the file's content/symlink destination should be looked into.
# E.g. only reuse from prevdict if the timestamp hasn't changed.
# There is the risk of the file's timestamp being reset to its last value
# manually while its content changed. We don't protect against that use case.
try:
filestats = os.lstat(filepath)
except OSError:
# The file is not present.
raise MappingError('%s is missing' % filepath)
is_link = stat.S_ISLNK(filestats.st_mode)
if sys.platform != 'win32':
# Ignore file mode on Windows since it's not really useful there.
filemode = stat.S_IMODE(filestats.st_mode)
# Remove write access for group and all access to 'others'.
filemode &= ~(stat.S_IWGRP | stat.S_IRWXO)
if read_only:
filemode &= ~stat.S_IWUSR
if filemode & (stat.S_IXUSR|stat.S_IRGRP) == (stat.S_IXUSR|stat.S_IRGRP):
# Only keep x group bit if both x user bit and group read bit are set.
filemode |= stat.S_IXGRP
else:
filemode &= ~stat.S_IXGRP
if not is_link:
out['m'] = filemode
# Used to skip recalculating the hash or link destination. Use the most recent
# update time.
out['t'] = int(round(filestats.st_mtime))
if not is_link:
out['s'] = filestats.st_size
# If the timestamp wasn't updated and the file size is still the same, carry
# on the sha-1.
if (prevdict.get('t') == out['t'] and
prevdict.get('s') == out['s']):
# Reuse the previous hash if available.
out['h'] = prevdict.get('h')
if not out.get('h'):
out['h'] = hash_file(filepath, algo)
else:
# If the timestamp wasn't updated, carry on the link destination.
if prevdict.get('t') == out['t']:
# Reuse the previous link destination if available.
out['l'] = prevdict.get('l')
if out.get('l') is None:
# The link could be in an incorrect path case. In practice, this only
# happen on OSX on case insensitive HFS.
# TODO(maruel): It'd be better if it was only done once, in
# expand_directory_and_symlink(), so it would not be necessary to do again
# here.
symlink_value = os.readlink(filepath) # pylint: disable=E1101
filedir = file_path.get_native_path_case(os.path.dirname(filepath))
native_dest = file_path.fix_native_path_case(filedir, symlink_value)
out['l'] = os.path.relpath(native_dest, filedir)
return out
def expand_directory_and_symlink(indir, relfile, blacklist, follow_symlinks):
"""Expands a single input. It can result in multiple outputs.
This function is recursive when relfile is a directory.
Note: this code doesn't properly handle recursive symlink like one created
with:
ln -s .. foo
"""
if os.path.isabs(relfile):
raise MappingError('Can\'t map absolute path %s' % relfile)
infile = file_path.normpath(os.path.join(indir, relfile))
if not infile.startswith(indir):
raise MappingError('Can\'t map file %s outside %s' % (infile, indir))
filepath = os.path.join(indir, relfile)
native_filepath = file_path.get_native_path_case(filepath)
if filepath != native_filepath:
# Special case './'.
if filepath != native_filepath + '.' + os.path.sep:
# While it'd be nice to enforce path casing on Windows, it's impractical.
# Also give up enforcing strict path case on OSX. Really, it's that sad.
# The case where it happens is very specific and hard to reproduce:
# get_native_path_case(
# u'Foo.framework/Versions/A/Resources/Something.nib') will return
# u'Foo.framework/Versions/A/resources/Something.nib', e.g. lowercase 'r'.
#
# Note that this is really something deep in OSX because running
# ls Foo.framework/Versions/A
# will print out 'Resources', while file_path.get_native_path_case()
# returns a lower case 'r'.
#
# So *something* is happening under the hood resulting in the command 'ls'
# and Carbon.File.FSPathMakeRef('path').FSRefMakePath() to disagree. We
# have no idea why.
if sys.platform not in ('darwin', 'win32'):
raise MappingError(
'File path doesn\'t equal native file path\n%s != %s' %
(filepath, native_filepath))
symlinks = []
if follow_symlinks:
try:
relfile, symlinks = expand_symlinks(indir, relfile)
except OSError:
# The file doesn't exist, it will throw below.
pass
if relfile.endswith(os.path.sep):
if not os.path.isdir(infile):
raise MappingError(
'%s is not a directory but ends with "%s"' % (infile, os.path.sep))
# Special case './'.
if relfile.startswith('.' + os.path.sep):
relfile = relfile[2:]
outfiles = symlinks
try:
for filename in fs.listdir(infile):
inner_relfile = os.path.join(relfile, filename)
if blacklist and blacklist(inner_relfile):
continue
if os.path.isdir(os.path.join(indir, inner_relfile)):
inner_relfile += os.path.sep
outfiles.extend(
expand_directory_and_symlink(indir, inner_relfile, blacklist,
follow_symlinks))
return outfiles
except OSError as e:
raise MappingError(
'Unable to iterate over directory %s.\n%s' % (infile, e))
else:
# Always add individual files even if they were blacklisted.
if os.path.isdir(infile):
raise MappingError(
'Input directory %s must have a trailing slash' % infile)
if not os.path.isfile(infile):
raise MappingError('Input file %s doesn\'t exist' % infile)
return symlinks + [relfile]
def expand_symlinks(indir, relfile):
"""Follows symlinks in |relfile|, but treating symlinks that point outside the
build tree as if they were ordinary directories/files. Returns the final
symlink-free target and a list of paths to symlinks encountered in the
process.
The rule about symlinks outside the build tree is for the benefit of the
Chromium OS ebuild, which symlinks the output directory to an unrelated path
in the chroot.
Fails when a directory loop is detected, although in theory we could support
that case.
"""
is_directory = relfile.endswith(os.path.sep)
done = indir
todo = relfile.strip(os.path.sep)
symlinks = []
while todo:
pre_symlink, symlink, post_symlink = file_path.split_at_symlink(done, todo)
if not symlink:
todo = file_path.fix_native_path_case(done, todo)
done = os.path.join(done, todo)
break
symlink_path = os.path.join(done, pre_symlink, symlink)
post_symlink = post_symlink.lstrip(os.path.sep)
# readlink doesn't exist on Windows.
# pylint: disable=E1101
target = os.path.normpath(os.path.join(done, pre_symlink))
symlink_target = os.readlink(symlink_path)
if os.path.isabs(symlink_target):
# Absolute path are considered a normal directories. The use case is
# generally someone who puts the output directory on a separate drive.
target = symlink_target
else:
# The symlink itself could be using the wrong path case.
target = file_path.fix_native_path_case(target, symlink_target)
if not os.path.exists(target):
raise MappingError(
'Symlink target doesn\'t exist: %s -> %s' % (symlink_path, target))
target = file_path.get_native_path_case(target)
if not file_path.path_starts_with(indir, target):
done = symlink_path
todo = post_symlink
continue
if file_path.path_starts_with(target, symlink_path):
raise MappingError(
'Can\'t map recursive symlink reference %s -> %s' %
(symlink_path, target))
logging.info('Found symlink: %s -> %s', symlink_path, target)
symlinks.append(os.path.relpath(symlink_path, indir))
# Treat the common prefix of the old and new paths as done, and start
# scanning again.
target = target.split(os.path.sep)
symlink_path = symlink_path.split(os.path.sep)
prefix_length = 0
for target_piece, symlink_path_piece in zip(target, symlink_path):
if target_piece == symlink_path_piece:
prefix_length += 1
else:
break
done = os.path.sep.join(target[:prefix_length])
todo = os.path.join(
os.path.sep.join(target[prefix_length:]), post_symlink)
relfile = os.path.relpath(done, indir)
relfile = relfile.rstrip(os.path.sep) + is_directory * os.path.sep
return relfile, symlinks
def file_to_metadata(filepath, prevdict, read_only, algo):
"""Processes an input file, a dependency, and return meta data about it.
Behaviors:
- Retrieves the file mode, file size, file timestamp, file link
destination if it is a file link and calcultate the SHA-1 of the file's
content if the path points to a file and not a symlink.
Arguments:
filepath: File to act on.
prevdict: the previous dictionary. It is used to retrieve the cached sha-1
to skip recalculating the hash. Optional.
read_only: If 1 or 2, the file mode is manipulated. In practice, only save
one of 4 modes: 0755 (rwx), 0644 (rw), 0555 (rx), 0444 (r). On
windows, mode is not set since all files are 'executable' by
default.
algo: Hashing algorithm used.
Returns:
The necessary dict to create a entry in the 'files' section of an .isolated
file.
"""
# TODO(maruel): None is not a valid value.
assert read_only in (None, 0, 1, 2), read_only
out = {}
# Always check the file stat and check if it is a link. The timestamp is used
# to know if the file's content/symlink destination should be looked into.
# E.g. only reuse from prevdict if the timestamp hasn't changed.
# There is the risk of the file's timestamp being reset to its last value
# manually while its content changed. We don't protect against that use case.
try:
filestats = os.lstat(filepath)
except OSError:
# The file is not present.
raise MappingError('%s is missing' % filepath)
is_link = stat.S_ISLNK(filestats.st_mode)
if sys.platform != 'win32':
# Ignore file mode on Windows since it's not really useful there.
filemode = stat.S_IMODE(filestats.st_mode)
# Remove write access for group and all access to 'others'.
filemode &= ~(stat.S_IWGRP | stat.S_IRWXO)
if read_only:
filemode &= ~stat.S_IWUSR
if filemode & (stat.S_IXUSR|stat.S_IRGRP) == (stat.S_IXUSR|stat.S_IRGRP):
# Only keep x group bit if both x user bit and group read bit are set.
filemode |= stat.S_IXGRP
else:
filemode &= ~stat.S_IXGRP
if not is_link:
out['m'] = filemode
# Used to skip recalculating the hash or link destination. Use the most recent
# update time.
out['t'] = int(round(filestats.st_mtime))
if not is_link:
out['s'] = filestats.st_size
# If the timestamp wasn't updated and the file size is still the same, carry
# on the sha-1.
if (prevdict.get('t') == out['t'] and
prevdict.get('s') == out['s']):
# Reuse the previous hash if available.
out['h'] = prevdict.get('h')
if not out.get('h'):
out['h'] = hash_file(filepath, algo)
else:
# If the timestamp wasn't updated, carry on the link destination.
if prevdict.get('t') == out['t']:
# Reuse the previous link destination if available.
out['l'] = prevdict.get('l')
if out.get('l') is None:
# The link could be in an incorrect path case. In practice, this only
# happen on OSX on case insensitive HFS.
# TODO(maruel): It'd be better if it was only done once, in
# expand_directory_and_symlink(), so it would not be necessary to do again
# here.
symlink_value = os.readlink(filepath) # pylint: disable=E1101
filedir = file_path.get_native_path_case(os.path.dirname(filepath))
native_dest = file_path.fix_native_path_case(filedir, symlink_value)
out['l'] = os.path.relpath(native_dest, filedir)
return out
def expand_directory_and_symlink(indir, relfile, blacklist, follow_symlinks):
"""Expands a single input. It can result in multiple outputs.
This function is recursive when relfile is a directory.
Note: this code doesn't properly handle recursive symlink like one created
with:
ln -s .. foo
"""
if os.path.isabs(relfile):
raise MappingError('Can\'t map absolute path %s' % relfile)
infile = file_path.normpath(os.path.join(indir, relfile))
if not infile.startswith(indir):
raise MappingError('Can\'t map file %s outside %s' % (infile, indir))
filepath = os.path.join(indir, relfile)
native_filepath = file_path.get_native_path_case(filepath)
if filepath != native_filepath:
# Special case './'.
if filepath != native_filepath + '.' + os.path.sep:
# While it'd be nice to enforce path casing on Windows, it's impractical.
# Also give up enforcing strict path case on OSX. Really, it's that sad.
# The case where it happens is very specific and hard to reproduce:
# get_native_path_case(
# u'Foo.framework/Versions/A/Resources/Something.nib') will return
# u'Foo.framework/Versions/A/resources/Something.nib', e.g. lowercase 'r'.
#
# Note that this is really something deep in OSX because running
# ls Foo.framework/Versions/A
# will print out 'Resources', while file_path.get_native_path_case()
# returns a lower case 'r'.
#
# So *something* is happening under the hood resulting in the command 'ls'
# and Carbon.File.FSPathMakeRef('path').FSRefMakePath() to disagree. We
# have no idea why.
if sys.platform not in ('darwin', 'win32'):
raise MappingError(
'File path doesn\'t equal native file path\n%s != %s' %
(filepath, native_filepath))
symlinks = []
if follow_symlinks:
try:
relfile, symlinks = expand_symlinks(indir, relfile)
except OSError:
# The file doesn't exist, it will throw below.
pass
if relfile.endswith(os.path.sep):
if not os.path.isdir(infile):
raise MappingError(
'%s is not a directory but ends with "%s"' % (infile, os.path.sep))
# Special case './'.
if relfile.startswith('.' + os.path.sep):
relfile = relfile[2:]
outfiles = symlinks
try:
for filename in fs.listdir(infile):
inner_relfile = os.path.join(relfile, filename)
if blacklist and blacklist(inner_relfile):
continue
if os.path.isdir(os.path.join(indir, inner_relfile)):
inner_relfile += os.path.sep
outfiles.extend(
expand_directory_and_symlink(indir, inner_relfile, blacklist,
follow_symlinks))
return outfiles
except OSError as e:
raise MappingError(
'Unable to iterate over directory %s.\n%s' % (infile, e))
else:
# Always add individual files even if they were blacklisted.
if os.path.isdir(infile):
raise MappingError(
'Input directory %s must have a trailing slash' % infile)
if not os.path.isfile(infile):
raise MappingError('Input file %s doesn\'t exist' % infile)
return symlinks + [relfile]
def expand_symlinks(indir, relfile):
"""Follows symlinks in |relfile|, but treating symlinks that point outside the
build tree as if they were ordinary directories/files. Returns the final
symlink-free target and a list of paths to symlinks encountered in the
process.
The rule about symlinks outside the build tree is for the benefit of the
Chromium OS ebuild, which symlinks the output directory to an unrelated path
in the chroot.
Fails when a directory loop is detected, although in theory we could support
that case.
"""
is_directory = relfile.endswith(os.path.sep)
done = indir
todo = relfile.strip(os.path.sep)
symlinks = []
while todo:
pre_symlink, symlink, post_symlink = file_path.split_at_symlink(done, todo)
if not symlink:
todo = file_path.fix_native_path_case(done, todo)
done = os.path.join(done, todo)
break
symlink_path = os.path.join(done, pre_symlink, symlink)
post_symlink = post_symlink.lstrip(os.path.sep)
# readlink doesn't exist on Windows.
# pylint: disable=E1101
target = os.path.normpath(os.path.join(done, pre_symlink))
symlink_target = os.readlink(symlink_path)
if os.path.isabs(symlink_target):
# Absolute path are considered a normal directories. The use case is
# generally someone who puts the output directory on a separate drive.
target = symlink_target
else:
# The symlink itself could be using the wrong path case.
target = file_path.fix_native_path_case(target, symlink_target)
if not os.path.exists(target):
raise MappingError(
'Symlink target doesn\'t exist: %s -> %s' % (symlink_path, target))
target = file_path.get_native_path_case(target)
if not file_path.path_starts_with(indir, target):
done = symlink_path
todo = post_symlink
continue
if file_path.path_starts_with(target, symlink_path):
raise MappingError(
'Can\'t map recursive symlink reference %s -> %s' %
(symlink_path, target))
logging.info('Found symlink: %s -> %s', symlink_path, target)
symlinks.append(os.path.relpath(symlink_path, indir))
# Treat the common prefix of the old and new paths as done, and start
# scanning again.
target = target.split(os.path.sep)
symlink_path = symlink_path.split(os.path.sep)
prefix_length = 0
for target_piece, symlink_path_piece in zip(target, symlink_path):
if target_piece == symlink_path_piece:
prefix_length += 1
else:
break
done = os.path.sep.join(target[:prefix_length])
todo = os.path.join(
os.path.sep.join(target[prefix_length:]), post_symlink)
relfile = os.path.relpath(done, indir)
relfile = relfile.rstrip(os.path.sep) + is_directory * os.path.sep
return relfile, symlinks
def test_chromium_split(self):
# Create an .isolate file and a tree of random stuff.
isolate_file = os.path.join(
ROOT_DIR, 'tests', 'isolate', 'split.isolate')
options = self._get_option(isolate_file)
options.path_variables = {
'DEPTH': '.',
'PRODUCT_DIR': os.path.join('files1'),
}
options.config_variables = {
'OS': 'linux',
}
complete_state = isolate.load_complete_state(
options, os.path.join(ROOT_DIR, 'tests', 'isolate'), None, False)
# By saving the files, it forces splitting the data up.
complete_state.save_files()
actual_isolated_master = tools.read_json(
os.path.join(self.directory, 'foo.isolated'))
expected_isolated_master = {
u'algo': u'sha-1',
u'command': [u'python', u'split.py'],
u'files': {
u'split.py': {
u'm': 488,
u'h': unicode(hash_file('tests', 'isolate', 'split.py')),
u's': _size('tests', 'isolate', 'split.py'),
},
},
u'includes': [
unicode(hash_file(os.path.join(self.directory, 'foo.0.isolated'))),
unicode(hash_file(os.path.join(self.directory, 'foo.1.isolated'))),
],
u'relative_cwd': u'.',
u'version': unicode(isolate.isolateserver.ISOLATED_FILE_VERSION),
}
self._cleanup_isolated(expected_isolated_master)
self.assertEqual(expected_isolated_master, actual_isolated_master)
actual_isolated_0 = tools.read_json(
os.path.join(self.directory, 'foo.0.isolated'))
expected_isolated_0 = {
u'algo': u'sha-1',
u'files': {
os.path.join(u'test', 'data', 'foo.txt'): {
u'm': 416,
u'h': unicode(
hash_file('tests', 'isolate', 'test', 'data', 'foo.txt')),
u's': _size('tests', 'isolate', 'test', 'data', 'foo.txt'),
},
},
u'version': unicode(isolate.isolateserver.ISOLATED_FILE_VERSION),
}
self._cleanup_isolated(expected_isolated_0)
self.assertEqual(expected_isolated_0, actual_isolated_0)
actual_isolated_1 = tools.read_json(
os.path.join(self.directory, 'foo.1.isolated'))
expected_isolated_1 = {
u'algo': u'sha-1',
u'files': {
os.path.join(u'files1', 'subdir', '42.txt'): {
u'm': 416,
u'h': unicode(
hash_file('tests', 'isolate', 'files1', 'subdir', '42.txt')),
u's': _size('tests', 'isolate', 'files1', 'subdir', '42.txt'),
},
},
u'version': unicode(isolate.isolateserver.ISOLATED_FILE_VERSION),
}
self._cleanup_isolated(expected_isolated_1)
self.assertEqual(expected_isolated_1, actual_isolated_1)
actual_saved_state = tools.read_json(
isolate.isolatedfile_to_state(options.isolated))
isolated_base = unicode(os.path.basename(options.isolated))
expected_saved_state = {
u'OS': unicode(sys.platform),
u'algo': u'sha-1',
u'child_isolated_files': [
isolated_base[:-len('.isolated')] + '.0.isolated',
isolated_base[:-len('.isolated')] + '.1.isolated',
],
u'command': [u'python', u'split.py'],
u'config_variables': {
u'OS': u'linux',
},
u'extra_variables': {
u'foo': u'bar',
},
u'files': {
os.path.join(u'files1', 'subdir', '42.txt'): {
u'm': 416,
u'h': unicode(
hash_file('tests', 'isolate', 'files1', 'subdir', '42.txt')),
u's': _size('tests', 'isolate', 'files1', 'subdir', '42.txt'),
},
u'split.py': {
u'm': 488,
u'h': unicode(hash_file('tests', 'isolate', 'split.py')),
u's': _size('tests', 'isolate', 'split.py'),
},
os.path.join(u'test', 'data', 'foo.txt'): {
u'm': 416,
u'h': unicode(
hash_file('tests', 'isolate', 'test', 'data', 'foo.txt')),
u's': _size('tests', 'isolate', 'test', 'data', 'foo.txt'),
},
},
u'isolate_file': file_path.safe_relpath(
file_path.get_native_path_case(isolate_file),
unicode(os.path.dirname(options.isolated))),
u'path_variables': {
u'DEPTH': u'.',
u'PRODUCT_DIR': u'files1',
},
u'relative_cwd': u'.',
u'root_dir': file_path.get_native_path_case(
os.path.dirname(isolate_file)),
u'version': unicode(isolate.SavedState.EXPECTED_VERSION),
}
self._cleanup_isolated(expected_saved_state)
self._cleanup_saved_state(actual_saved_state)
self.assertEqual(expected_saved_state, actual_saved_state)
self.assertEqual(
[
'foo.0.isolated', 'foo.1.isolated',
'foo.isolated', 'foo.isolated.state',
],
sorted(os.listdir(self.directory)))
def test_subdir_variable(self):
# the resulting .isolated file will be missing ../../isolate.py. it is
# because this file is outside the --subdir parameter.
isolate_file = os.path.join(
ROOT_DIR, 'tests', 'isolate', 'touch_root.isolate')
options = self._get_option(isolate_file)
# Path variables are keyed on the directory containing the .isolate file.
options.path_variables['TEST_ISOLATE'] = '.'
# Note that options.isolated is in self.directory, which is a temporary
# directory.
complete_state = isolate.load_complete_state(
options, os.path.join(ROOT_DIR, 'tests', 'isolate'),
'<(TEST_ISOLATE)', False)
actual_isolated = complete_state.saved_state.to_isolated()
actual_saved_state = complete_state.saved_state.flatten()
expected_isolated = {
'algo': 'sha-1',
'command': ['python', 'touch_root.py'],
'files': {
os.path.join('tests', 'isolate', 'touch_root.py'): {
'm': 488,
'h': hash_file('tests', 'isolate', 'touch_root.py'),
's': _size('tests', 'isolate', 'touch_root.py'),
},
},
'relative_cwd': os.path.join(u'tests', 'isolate'),
'version': isolate.isolateserver.ISOLATED_FILE_VERSION,
}
self._cleanup_isolated(expected_isolated)
self.assertEqual(expected_isolated, actual_isolated)
# It is important to note:
# - the root directory is ROOT_DIR.
# - relative_cwd is tests/isolate.
# - TEST_ISOLATE is based of relative_cwd, so it represents tests/isolate.
# - anything outside TEST_ISOLATE was not included in the 'files' section.
expected_saved_state = {
'OS': sys.platform,
'algo': 'sha-1',
'child_isolated_files': [],
'command': ['python', 'touch_root.py'],
'config_variables': {
'OS': 'linux',
'chromeos': 1,
},
'extra_variables': {
'foo': 'bar',
},
'files': {
os.path.join(u'tests', 'isolate', 'touch_root.py'): {
'm': 488,
'h': hash_file('tests', 'isolate', 'touch_root.py'),
's': _size('tests', 'isolate', 'touch_root.py'),
},
},
'isolate_file': file_path.safe_relpath(
file_path.get_native_path_case(isolate_file),
os.path.dirname(options.isolated)),
'path_variables': {
'TEST_ISOLATE': '.',
},
'relative_cwd': os.path.join(u'tests', 'isolate'),
'root_dir': file_path.get_native_path_case(ROOT_DIR),
'version': isolate.SavedState.EXPECTED_VERSION,
}
self._cleanup_isolated(expected_saved_state)
self._cleanup_saved_state(actual_saved_state)
self.assertEqual(expected_saved_state, actual_saved_state)
def test_simple(self):
file_handle, self.temp_file = tempfile.mkstemp(
prefix='trace_test_cases_test')
os.close(file_handle)
cmd = [
sys.executable,
os.path.join(GOOGLETEST_DIR, 'trace_test_cases.py'),
# Forces 4 parallel jobs.
'--jobs',
'4',
'--out',
self.temp_file,
]
if VERBOSE:
cmd.extend(['-v'] * 3)
cmd.append(TARGET_PATH)
logging.debug(' '.join(cmd))
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
cwd=GOOGLETEST_DIR)
out, err = proc.communicate() or ('', '') # pylint is confused.
self.assertEqual(0, proc.returncode, (out, err))
lines = out.splitlines()
expected_out_re = [
r'Tracing\.\.\.',
r'\[0/4\] +\d+\.\d\ds ',
r'\[1/4\] +\d+\.\d\ds .+',
r'\[2/4\] +\d+\.\d\ds .+',
r'\[3/4\] +\d+\.\d\ds .+',
r'\[4/4\] +\d+\.\d\ds .+',
r'Reading trace logs\.\.\.',
]
self.assertEqual(len(expected_out_re), len(lines), lines)
for index in range(len(expected_out_re)):
self.assertTrue(
re.match('^%s$' % expected_out_re[index],
lines[index]), '%d: %s\n%r\n%s' %
(index, expected_out_re[index], lines[index], out))
# Junk is printed on win32.
if sys.platform != 'win32' and not VERBOSE:
self.assertEqual('', err)
with open(self.temp_file, 'r') as f:
content = f.read()
try:
result = json.loads(content)
except:
print repr(content)
raise
test_cases = {
'Baz.Fail': 1,
'Foo.Bar1': 0,
'Foo.Bar2': 0,
'Foo.Bar3': 0,
}
self.assertEqual(dict, result.__class__)
self.assertEqual(sorted(test_cases), sorted(result))
for index, test_case in enumerate(sorted(result)):
actual = result[test_case]
self.assertEqual([u'duration', u'output', u'returncode', u'trace'],
sorted(actual))
self.assertGreater(actual['duration'], 0.0000001)
self.assertEqual(test_cases[test_case], actual['returncode'])
expected_output = ('Note: Google Test filter = %s\n' % test_case +
'\n' + gtest_fake_base.get_test_output(
test_case, 'Fail' in test_case) + '\n' +
gtest_fake_base.get_footer(1, 1) + '\n')
# On Windows, actual['output'] is unprocessed so it will contain CRLF.
output = actual['output']
if sys.platform == 'win32':
output = output.replace('\r\n', '\n')
self.assertEqual(expected_output, output, repr(output))
expected_trace = {
u'root': {
u'children': [],
u'command': [
self.executable,
TARGET_PATH,
'--gtest_filter=' + test_case,
],
u'executable':
file_path.get_native_path_case(unicode(self.executable)),
u'initial_cwd':
GOOGLETEST_DIR,
},
}
if sys.platform == 'win32':
expected_trace['root']['initial_cwd'] = None
self.assertGreater(actual['trace']['root'].pop('pid'), 1)
self.assertGreater(len(actual['trace']['root'].pop('files')), 10)
self.assertEqual(expected_trace, actual['trace'])
def test_root_dir_because_of_variable(self):
# Ensures that load_isolate() works even when path variables have deep root
# dirs. The end result is similar to touch_root.isolate, except that
# no_run.isolate doesn't reference '..' at all.
#
# A real world example would be PRODUCT_DIR=../../out/Release but nothing in
# this directory is mapped.
#
# Imagine base/base_unittests.isolate would not map anything in
# PRODUCT_DIR. In that case, the automatically determined root dir is
# src/base, since nothing outside this directory is mapped.
isolate_file = os.path.join(
ROOT_DIR, 'tests', 'isolate', 'no_run.isolate')
options = self._get_option(isolate_file)
# Any directory outside ROOT_DIR/tests/isolate.
options.path_variables['PRODUCT_DIR'] = os.path.join('third_party')
complete_state = isolate.load_complete_state(options, ROOT_DIR, None, False)
actual_isolated = complete_state.saved_state.to_isolated()
actual_saved_state = complete_state.saved_state.flatten()
expected_isolated = {
'algo': 'sha-1',
'files': {
os.path.join(u'tests', 'isolate', 'files1', 'subdir', '42.txt'): {
'm': 416,
'h': hash_file('tests', 'isolate', 'files1', 'subdir', '42.txt'),
's': _size('tests', 'isolate', 'files1', 'subdir', '42.txt'),
},
os.path.join(u'tests', 'isolate', 'files1', 'test_file1.txt'): {
'm': 416,
'h': hash_file('tests', 'isolate', 'files1', 'test_file1.txt'),
's': _size('tests', 'isolate', 'files1', 'test_file1.txt'),
},
os.path.join(u'tests', 'isolate', 'files1', 'test_file2.txt'): {
'm': 416,
'h': hash_file('tests', 'isolate', 'files1', 'test_file2.txt'),
's': _size('tests', 'isolate', 'files1', 'test_file2.txt'),
},
os.path.join(u'tests', 'isolate', 'no_run.isolate'): {
'm': 416,
'h': hash_file('tests', 'isolate', 'no_run.isolate'),
's': _size('tests', 'isolate', 'no_run.isolate'),
},
},
'relative_cwd': os.path.join(u'tests', 'isolate'),
'version': isolate.isolateserver.ISOLATED_FILE_VERSION,
}
self._cleanup_isolated(expected_isolated)
self.assertEqual(expected_isolated, actual_isolated)
expected_saved_state = {
'OS': sys.platform,
'algo': 'sha-1',
'child_isolated_files': [],
'command': [],
'config_variables': {
'OS': 'linux',
'chromeos': 1,
},
'extra_variables': {
'foo': 'bar',
},
'files': {
os.path.join(u'tests', 'isolate', 'files1', 'subdir', '42.txt'): {
'm': 416,
'h': hash_file('tests', 'isolate', 'files1', 'subdir', '42.txt'),
's': _size('tests', 'isolate', 'files1', 'subdir', '42.txt'),
},
os.path.join(u'tests', 'isolate', 'files1', 'test_file1.txt'): {
'm': 416,
'h': hash_file('tests', 'isolate', 'files1', 'test_file1.txt'),
's': _size('tests', 'isolate', 'files1', 'test_file1.txt'),
},
os.path.join(u'tests', 'isolate', 'files1', 'test_file2.txt'): {
'm': 416,
'h': hash_file('tests', 'isolate', 'files1', 'test_file2.txt'),
's': _size('tests', 'isolate', 'files1', 'test_file2.txt'),
},
os.path.join(u'tests', 'isolate', 'no_run.isolate'): {
'm': 416,
'h': hash_file('tests', 'isolate', 'no_run.isolate'),
's': _size('tests', 'isolate', 'no_run.isolate'),
},
},
'isolate_file': file_path.safe_relpath(
file_path.get_native_path_case(isolate_file),
os.path.dirname(options.isolated)),
'path_variables': {
'PRODUCT_DIR': os.path.join(u'..', '..', 'third_party'),
},
'relative_cwd': os.path.join(u'tests', 'isolate'),
'root_dir': file_path.get_native_path_case(ROOT_DIR),
'version': isolate.SavedState.EXPECTED_VERSION,
}
self._cleanup_isolated(expected_saved_state)
self._cleanup_saved_state(actual_saved_state)
self.assertEqual(expected_saved_state, actual_saved_state)
self.assertEqual([], os.listdir(self.directory))
def test_native_case_end_with_os_path_sep(self): # Make sure the trailing os.path.sep is kept. path = file_path.get_native_path_case(ROOT_DIR) + os.path.sep self.assertEqual(file_path.get_native_path_case(path), path)
def test_native_case_end_with_os_path_sep(self): # Make sure the trailing os.path.sep is kept. path = file_path.get_native_path_case(ROOT_DIR) + os.path.sep self.assertEqual(file_path.get_native_path_case(path), path)
def load_isolate(self, cwd, isolate_file, path_variables, config_variables,
extra_variables, blacklist, ignore_broken_items,
collapse_symlinks):
"""Updates self.isolated and self.saved_state with information loaded from a
.isolate file.
Processes the loaded data, deduce root_dir, relative_cwd.
"""
# Make sure to not depend on os.getcwd().
assert os.path.isabs(isolate_file), isolate_file
isolate_file = file_path.get_native_path_case(isolate_file)
logging.info('CompleteState.load_isolate(%s, %s, %s, %s, %s, %s, %s)',
cwd, isolate_file, path_variables, config_variables,
extra_variables, ignore_broken_items, collapse_symlinks)
# Config variables are not affected by the paths and must be used to
# retrieve the paths, so update them first.
self.saved_state.update_config(config_variables)
with fs.open(isolate_file, 'r') as f:
# At that point, variables are not replaced yet in command and infiles.
# infiles may contain directory entries and is in posix style.
command, infiles, read_only, isolate_cmd_dir = (
isolate_format.load_isolate_for_config(
os.path.dirname(isolate_file), f.read(),
self.saved_state.config_variables))
# Processes the variables with the new found relative root. Note that 'cwd'
# is used when path variables are used.
path_variables = normalize_path_variables(cwd, path_variables,
isolate_cmd_dir)
# Update the rest of the saved state.
self.saved_state.update(isolate_file, path_variables, extra_variables)
total_variables = self.saved_state.path_variables.copy()
total_variables.update(self.saved_state.config_variables)
total_variables.update(self.saved_state.extra_variables)
command = [
isolate_format.eval_variables(i, total_variables) for i in command
]
total_variables = self.saved_state.path_variables.copy()
total_variables.update(self.saved_state.extra_variables)
infiles = [
isolate_format.eval_variables(f, total_variables) for f in infiles
]
# root_dir is automatically determined by the deepest root accessed with the
# form '../../foo/bar'. Note that path variables must be taken in account
# too, add them as if they were input files.
self.saved_state.root_dir = isolate_format.determine_root_dir(
isolate_cmd_dir,
infiles + self.saved_state.path_variables.values())
# The relative directory is automatically determined by the relative path
# between root_dir and the directory containing the .isolate file,
# isolate_base_dir.
relative_cwd = os.path.relpath(isolate_cmd_dir,
self.saved_state.root_dir)
# Now that we know where the root is, check that the path_variables point
# inside it.
for k, v in self.saved_state.path_variables.iteritems():
dest = os.path.join(isolate_cmd_dir, relative_cwd, v)
if not file_path.path_starts_with(self.saved_state.root_dir, dest):
raise isolated_format.MappingError(
'Path variable %s=%r points outside the inferred root directory '
'%s; %s' % (k, v, self.saved_state.root_dir, dest))
# Normalize the files based to self.saved_state.root_dir. It is important to
# keep the trailing os.path.sep at that step.
infiles = [
file_path.relpath(
file_path.normpath(os.path.join(isolate_cmd_dir, f)),
self.saved_state.root_dir) for f in infiles
]
follow_symlinks = False
if not collapse_symlinks:
follow_symlinks = sys.platform != 'win32'
# Expand the directories by listing each file inside. Up to now, trailing
# os.path.sep must be kept.
infiles = isolated_format.expand_directories_and_symlinks(
self.saved_state.root_dir, infiles, tools.gen_blacklist(blacklist),
follow_symlinks, ignore_broken_items)
# Finally, update the new data to be able to generate the foo.isolated file,
# the file that is used by run_isolated.py.
self.saved_state.update_isolated(command, infiles, read_only,
relative_cwd)
logging.debug(self)
def test_native_case_end_with_dot_os_path_sep(self):
path = file_path.get_native_path_case(ROOT_DIR + os.path.sep)
self.assertEqual(
file_path.get_native_path_case(path + '.' + os.path.sep),
path)
def load_complete_state(options, cwd, subdir, skip_update):
"""Loads a CompleteState.
This includes data from .isolate and .isolated.state files. Never reads the
.isolated file.
Arguments:
options: Options instance generated with process_isolate_options. For either
options.isolate and options.isolated, if the value is set, it is an
absolute path.
cwd: base directory to be used when loading the .isolate file.
subdir: optional argument to only process file in the subdirectory, relative
to CompleteState.root_dir.
skip_update: Skip trying to load the .isolate file and processing the
dependencies. It is useful when not needed, like when tracing.
"""
assert not options.isolate or os.path.isabs(options.isolate)
assert not options.isolated or os.path.isabs(options.isolated)
cwd = file_path.get_native_path_case(unicode(cwd))
if options.isolated:
# Load the previous state if it was present. Namely, "foo.isolated.state".
# Note: this call doesn't load the .isolate file.
complete_state = CompleteState.load_files(options.isolated)
else:
# Constructs a dummy object that cannot be saved. Useful for temporary
# commands like 'run'. There is no directory containing a .isolated file so
# specify the current working directory as a valid directory.
complete_state = CompleteState(None, SavedState(os.getcwd()))
if not options.isolate:
if not complete_state.saved_state.isolate_file:
if not skip_update:
raise ExecutionError('A .isolate file is required.')
isolate = None
else:
isolate = complete_state.saved_state.isolate_filepath
else:
isolate = options.isolate
if complete_state.saved_state.isolate_file:
rel_isolate = file_path.safe_relpath(
options.isolate, complete_state.saved_state.isolated_basedir)
if rel_isolate != complete_state.saved_state.isolate_file:
# This happens if the .isolate file was moved for example. In this case,
# discard the saved state.
logging.warning(
'--isolated %s != %s as saved in %s. Discarding saved state',
rel_isolate, complete_state.saved_state.isolate_file,
isolatedfile_to_state(options.isolated))
complete_state = CompleteState(
options.isolated,
SavedState(complete_state.saved_state.isolated_basedir))
if not skip_update:
# Then load the .isolate and expands directories.
complete_state.load_isolate(cwd, isolate, options.path_variables,
options.config_variables,
options.extra_variables, options.blacklist,
options.ignore_broken_items,
options.collapse_symlinks)
# Regenerate complete_state.saved_state.files.
if subdir:
subdir = unicode(subdir)
# This is tricky here. If it is a path, take it from the root_dir. If
# it is a variable, it must be keyed from the directory containing the
# .isolate file. So translate all variables first.
translated_path_variables = dict(
(k,
os.path.normpath(
os.path.join(complete_state.saved_state.relative_cwd, v)))
for k, v in complete_state.saved_state.path_variables.iteritems())
subdir = isolate_format.eval_variables(subdir,
translated_path_variables)
subdir = subdir.replace('/', os.path.sep)
if not skip_update:
complete_state.files_to_metadata(subdir, options.collapse_symlinks)
return complete_state
def test_variable(self):
isolate_file = os.path.join(
ROOT_DIR, 'tests', 'isolate', 'touch_root.isolate')
options = self._get_option(isolate_file)
options.path_variables['PRODUCT_DIR'] = os.path.join('tests', 'isolate')
complete_state = isolate.load_complete_state(options, ROOT_DIR, None, False)
actual_isolated = complete_state.saved_state.to_isolated()
actual_saved_state = complete_state.saved_state.flatten()
expected_isolated = {
'algo': 'sha-1',
'command': ['python', 'touch_root.py'],
'files': {
u'isolate.py': {
'm': 488,
'h': hash_file('isolate.py'),
's': _size('isolate.py'),
},
os.path.join(u'tests', 'isolate', 'touch_root.py'): {
'm': 488,
'h': hash_file('tests', 'isolate', 'touch_root.py'),
's': _size('tests', 'isolate', 'touch_root.py'),
},
},
'relative_cwd': os.path.join(u'tests', 'isolate'),
'version': isolate.isolateserver.ISOLATED_FILE_VERSION,
}
self._cleanup_isolated(expected_isolated)
self.assertEqual(expected_isolated, actual_isolated)
expected_saved_state = {
'OS': sys.platform,
'algo': 'sha-1',
'child_isolated_files': [],
'command': ['python', 'touch_root.py'],
'config_variables': {
'OS': 'linux',
'chromeos': 1,
},
'extra_variables': {
'foo': 'bar',
},
'files': {
u'isolate.py': {
'm': 488,
'h': hash_file('isolate.py'),
's': _size('isolate.py'),
},
os.path.join(u'tests', 'isolate', 'touch_root.py'): {
'm': 488,
'h': hash_file('tests', 'isolate', 'touch_root.py'),
's': _size('tests', 'isolate', 'touch_root.py'),
},
},
'isolate_file': file_path.safe_relpath(
file_path.get_native_path_case(isolate_file),
os.path.dirname(options.isolated)),
'path_variables': {
'PRODUCT_DIR': '.',
},
'relative_cwd': os.path.join(u'tests', 'isolate'),
'root_dir': file_path.get_native_path_case(ROOT_DIR),
'version': isolate.SavedState.EXPECTED_VERSION,
}
self._cleanup_isolated(expected_saved_state)
self._cleanup_saved_state(actual_saved_state)
self.assertEqual(expected_saved_state, actual_saved_state)
self.assertEqual([], os.listdir(self.directory))
def load_isolate(
self, cwd, isolate_file, path_variables, config_variables, extra_variables, blacklist, ignore_broken_items
):
"""Updates self.isolated and self.saved_state with information loaded from a
.isolate file.
Processes the loaded data, deduce root_dir, relative_cwd.
"""
# Make sure to not depend on os.getcwd().
assert os.path.isabs(isolate_file), isolate_file
isolate_file = file_path.get_native_path_case(isolate_file)
logging.info(
"CompleteState.load_isolate(%s, %s, %s, %s, %s, %s)",
cwd,
isolate_file,
path_variables,
config_variables,
extra_variables,
ignore_broken_items,
)
# Config variables are not affected by the paths and must be used to
# retrieve the paths, so update them first.
self.saved_state.update_config(config_variables)
with open(isolate_file, "r") as f:
# At that point, variables are not replaced yet in command and infiles.
# infiles may contain directory entries and is in posix style.
command, infiles, read_only, isolate_cmd_dir = isolate_format.load_isolate_for_config(
os.path.dirname(isolate_file), f.read(), self.saved_state.config_variables
)
# Processes the variables with the new found relative root. Note that 'cwd'
# is used when path variables are used.
path_variables = normalize_path_variables(cwd, path_variables, isolate_cmd_dir)
# Update the rest of the saved state.
self.saved_state.update(isolate_file, path_variables, extra_variables)
total_variables = self.saved_state.path_variables.copy()
total_variables.update(self.saved_state.config_variables)
total_variables.update(self.saved_state.extra_variables)
command = [isolate_format.eval_variables(i, total_variables) for i in command]
total_variables = self.saved_state.path_variables.copy()
total_variables.update(self.saved_state.extra_variables)
infiles = [isolate_format.eval_variables(f, total_variables) for f in infiles]
# root_dir is automatically determined by the deepest root accessed with the
# form '../../foo/bar'. Note that path variables must be taken in account
# too, add them as if they were input files.
self.saved_state.root_dir = isolate_format.determine_root_dir(
isolate_cmd_dir, infiles + self.saved_state.path_variables.values()
)
# The relative directory is automatically determined by the relative path
# between root_dir and the directory containing the .isolate file,
# isolate_base_dir.
relative_cwd = os.path.relpath(isolate_cmd_dir, self.saved_state.root_dir)
# Now that we know where the root is, check that the path_variables point
# inside it.
for k, v in self.saved_state.path_variables.iteritems():
dest = os.path.join(isolate_cmd_dir, relative_cwd, v)
if not file_path.path_starts_with(self.saved_state.root_dir, dest):
raise isolated_format.MappingError(
"Path variable %s=%r points outside the inferred root directory "
"%s; %s" % (k, v, self.saved_state.root_dir, dest)
)
# Normalize the files based to self.saved_state.root_dir. It is important to
# keep the trailing os.path.sep at that step.
infiles = [
file_path.relpath(file_path.normpath(os.path.join(isolate_cmd_dir, f)), self.saved_state.root_dir)
for f in infiles
]
follow_symlinks = sys.platform != "win32"
# Expand the directories by listing each file inside. Up to now, trailing
# os.path.sep must be kept.
infiles = isolated_format.expand_directories_and_symlinks(
self.saved_state.root_dir, infiles, tools.gen_blacklist(blacklist), follow_symlinks, ignore_broken_items
)
# Finally, update the new data to be able to generate the foo.isolated file,
# the file that is used by run_isolated.py.
self.saved_state.update_isolated(command, infiles, read_only, relative_cwd)
logging.debug(self)
def load_complete_state(options, cwd, subdir, skip_update):
"""Loads a CompleteState.
This includes data from .isolate and .isolated.state files. Never reads the
.isolated file.
Arguments:
options: Options instance generated with process_isolate_options. For either
options.isolate and options.isolated, if the value is set, it is an
absolute path.
cwd: base directory to be used when loading the .isolate file.
subdir: optional argument to only process file in the subdirectory, relative
to CompleteState.root_dir.
skip_update: Skip trying to load the .isolate file and processing the
dependencies. It is useful when not needed, like when tracing.
"""
assert not options.isolate or os.path.isabs(options.isolate)
assert not options.isolated or os.path.isabs(options.isolated)
cwd = file_path.get_native_path_case(unicode(cwd))
if options.isolated:
# Load the previous state if it was present. Namely, "foo.isolated.state".
# Note: this call doesn't load the .isolate file.
complete_state = CompleteState.load_files(options.isolated)
else:
# Constructs a dummy object that cannot be saved. Useful for temporary
# commands like 'run'. There is no directory containing a .isolated file so
# specify the current working directory as a valid directory.
complete_state = CompleteState(None, SavedState(os.getcwd()))
if not options.isolate:
if not complete_state.saved_state.isolate_file:
if not skip_update:
raise ExecutionError("A .isolate file is required.")
isolate = None
else:
isolate = complete_state.saved_state.isolate_filepath
else:
isolate = options.isolate
if complete_state.saved_state.isolate_file:
rel_isolate = file_path.safe_relpath(options.isolate, complete_state.saved_state.isolated_basedir)
if rel_isolate != complete_state.saved_state.isolate_file:
# This happens if the .isolate file was moved for example. In this case,
# discard the saved state.
logging.warning(
"--isolated %s != %s as saved in %s. Discarding saved state",
rel_isolate,
complete_state.saved_state.isolate_file,
isolatedfile_to_state(options.isolated),
)
complete_state = CompleteState(
options.isolated, SavedState(complete_state.saved_state.isolated_basedir)
)
if not skip_update:
# Then load the .isolate and expands directories.
complete_state.load_isolate(
cwd,
isolate,
options.path_variables,
options.config_variables,
options.extra_variables,
options.blacklist,
options.ignore_broken_items,
)
# Regenerate complete_state.saved_state.files.
if subdir:
subdir = unicode(subdir)
# This is tricky here. If it is a path, take it from the root_dir. If
# it is a variable, it must be keyed from the directory containing the
# .isolate file. So translate all variables first.
translated_path_variables = dict(
(k, os.path.normpath(os.path.join(complete_state.saved_state.relative_cwd, v)))
for k, v in complete_state.saved_state.path_variables.iteritems()
)
subdir = isolate_format.eval_variables(subdir, translated_path_variables)
subdir = subdir.replace("/", os.path.sep)
if not skip_update:
complete_state.files_to_metadata(subdir)
return complete_state
def test_native_case_end_with_dot_os_path_sep(self):
path = file_path.get_native_path_case(ROOT_DIR + os.path.sep)
self.assertEqual(
file_path.get_native_path_case(path + '.' + os.path.sep),
path)
