def load_complete_state(options, cwd, subdir, skip_update):
"""Loads a CompleteState.
This includes data from .isolate and .isolated.state files. Never reads the
.isolated file.
Arguments:
options: Options instance generated with process_isolate_options. For either
options.isolate and options.isolated, if the value is set, it is an
absolute path.
cwd: base directory to be used when loading the .isolate file.
subdir: optional argument to only process file in the subdirectory, relative
to CompleteState.root_dir.
skip_update: Skip trying to load the .isolate file and processing the
dependencies. It is useful when not needed, like when tracing.
"""
assert not options.isolate or os.path.isabs(options.isolate)
assert not options.isolated or os.path.isabs(options.isolated)
cwd = file_path.get_native_path_case(unicode(cwd))
if options.isolated:
# Load the previous state if it was present. Namely, "foo.isolated.state".
# Note: this call doesn't load the .isolate file.
complete_state = CompleteState.load_files(options.isolated)
else:
# Constructs a dummy object that cannot be saved. Useful for temporary
# commands like 'run'. There is no directory containing a .isolated file so
# specify the current working directory as a valid directory.
complete_state = CompleteState(None, SavedState(os.getcwd()))
if not options.isolate:
if not complete_state.saved_state.isolate_file:
if not skip_update:
raise ExecutionError("A .isolate file is required.")
isolate = None
else:
isolate = complete_state.saved_state.isolate_filepath
else:
isolate = options.isolate
if complete_state.saved_state.isolate_file:
rel_isolate = file_path.safe_relpath(options.isolate, complete_state.saved_state.isolated_basedir)
if rel_isolate != complete_state.saved_state.isolate_file:
# This happens if the .isolate file was moved for example. In this case,
# discard the saved state.
logging.warning(
"--isolated %s != %s as saved in %s. Discarding saved state",
rel_isolate,
complete_state.saved_state.isolate_file,
isolatedfile_to_state(options.isolated),
)
complete_state = CompleteState(
options.isolated, SavedState(complete_state.saved_state.isolated_basedir)
)
if not skip_update:
# Then load the .isolate and expands directories.
complete_state.load_isolate(
cwd,
isolate,
options.path_variables,
options.config_variables,
options.extra_variables,
options.blacklist,
options.ignore_broken_items,
)
# Regenerate complete_state.saved_state.files.
if subdir:
subdir = unicode(subdir)
# This is tricky here. If it is a path, take it from the root_dir. If
# it is a variable, it must be keyed from the directory containing the
# .isolate file. So translate all variables first.
translated_path_variables = dict(
(k, os.path.normpath(os.path.join(complete_state.saved_state.relative_cwd, v)))
for k, v in complete_state.saved_state.path_variables.iteritems()
)
subdir = isolate_format.eval_variables(subdir, translated_path_variables)
subdir = subdir.replace("/", os.path.sep)
if not skip_update:
complete_state.files_to_metadata(subdir)
return complete_state
def load_isolate(
self, cwd, isolate_file, path_variables, config_variables, extra_variables, blacklist, ignore_broken_items
):
"""Updates self.isolated and self.saved_state with information loaded from a
.isolate file.
Processes the loaded data, deduce root_dir, relative_cwd.
"""
# Make sure to not depend on os.getcwd().
assert os.path.isabs(isolate_file), isolate_file
isolate_file = file_path.get_native_path_case(isolate_file)
logging.info(
"CompleteState.load_isolate(%s, %s, %s, %s, %s, %s)",
cwd,
isolate_file,
path_variables,
config_variables,
extra_variables,
ignore_broken_items,
)
# Config variables are not affected by the paths and must be used to
# retrieve the paths, so update them first.
self.saved_state.update_config(config_variables)
with open(isolate_file, "r") as f:
# At that point, variables are not replaced yet in command and infiles.
# infiles may contain directory entries and is in posix style.
command, infiles, read_only, isolate_cmd_dir = isolate_format.load_isolate_for_config(
os.path.dirname(isolate_file), f.read(), self.saved_state.config_variables
)
# Processes the variables with the new found relative root. Note that 'cwd'
# is used when path variables are used.
path_variables = normalize_path_variables(cwd, path_variables, isolate_cmd_dir)
# Update the rest of the saved state.
self.saved_state.update(isolate_file, path_variables, extra_variables)
total_variables = self.saved_state.path_variables.copy()
total_variables.update(self.saved_state.config_variables)
total_variables.update(self.saved_state.extra_variables)
command = [isolate_format.eval_variables(i, total_variables) for i in command]
total_variables = self.saved_state.path_variables.copy()
total_variables.update(self.saved_state.extra_variables)
infiles = [isolate_format.eval_variables(f, total_variables) for f in infiles]
# root_dir is automatically determined by the deepest root accessed with the
# form '../../foo/bar'. Note that path variables must be taken in account
# too, add them as if they were input files.
self.saved_state.root_dir = isolate_format.determine_root_dir(
isolate_cmd_dir, infiles + self.saved_state.path_variables.values()
)
# The relative directory is automatically determined by the relative path
# between root_dir and the directory containing the .isolate file,
# isolate_base_dir.
relative_cwd = os.path.relpath(isolate_cmd_dir, self.saved_state.root_dir)
# Now that we know where the root is, check that the path_variables point
# inside it.
for k, v in self.saved_state.path_variables.iteritems():
dest = os.path.join(isolate_cmd_dir, relative_cwd, v)
if not file_path.path_starts_with(self.saved_state.root_dir, dest):
raise isolated_format.MappingError(
"Path variable %s=%r points outside the inferred root directory "
"%s; %s" % (k, v, self.saved_state.root_dir, dest)
)
# Normalize the files based to self.saved_state.root_dir. It is important to
# keep the trailing os.path.sep at that step.
infiles = [
file_path.relpath(file_path.normpath(os.path.join(isolate_cmd_dir, f)), self.saved_state.root_dir)
for f in infiles
]
follow_symlinks = sys.platform != "win32"
# Expand the directories by listing each file inside. Up to now, trailing
# os.path.sep must be kept.
infiles = isolated_format.expand_directories_and_symlinks(
self.saved_state.root_dir, infiles, tools.gen_blacklist(blacklist), follow_symlinks, ignore_broken_items
)
# Finally, update the new data to be able to generate the foo.isolated file,
# the file that is used by run_isolated.py.
self.saved_state.update_isolated(command, infiles, read_only, relative_cwd)
logging.debug(self)
def load_isolate(self, cwd, isolate_file, path_variables, config_variables,
extra_variables, blacklist, ignore_broken_items,
collapse_symlinks):
"""Updates self.isolated and self.saved_state with information loaded from a
.isolate file.
Processes the loaded data, deduce root_dir, relative_cwd.
"""
# Make sure to not depend on os.getcwd().
assert os.path.isabs(isolate_file), isolate_file
isolate_file = file_path.get_native_path_case(isolate_file)
logging.info('CompleteState.load_isolate(%s, %s, %s, %s, %s, %s, %s)',
cwd, isolate_file, path_variables, config_variables,
extra_variables, ignore_broken_items, collapse_symlinks)
# Config variables are not affected by the paths and must be used to
# retrieve the paths, so update them first.
self.saved_state.update_config(config_variables)
with fs.open(isolate_file, 'r') as f:
# At that point, variables are not replaced yet in command and infiles.
# infiles may contain directory entries and is in posix style.
command, infiles, read_only, isolate_cmd_dir = (
isolate_format.load_isolate_for_config(
os.path.dirname(isolate_file), f.read(),
self.saved_state.config_variables))
# Processes the variables with the new found relative root. Note that 'cwd'
# is used when path variables are used.
path_variables = normalize_path_variables(cwd, path_variables,
isolate_cmd_dir)
# Update the rest of the saved state.
self.saved_state.update(isolate_file, path_variables, extra_variables)
total_variables = self.saved_state.path_variables.copy()
total_variables.update(self.saved_state.config_variables)
total_variables.update(self.saved_state.extra_variables)
command = [
isolate_format.eval_variables(i, total_variables) for i in command
]
total_variables = self.saved_state.path_variables.copy()
total_variables.update(self.saved_state.extra_variables)
infiles = [
isolate_format.eval_variables(f, total_variables) for f in infiles
]
# root_dir is automatically determined by the deepest root accessed with the
# form '../../foo/bar'. Note that path variables must be taken in account
# too, add them as if they were input files.
self.saved_state.root_dir = isolate_format.determine_root_dir(
isolate_cmd_dir,
infiles + self.saved_state.path_variables.values())
# The relative directory is automatically determined by the relative path
# between root_dir and the directory containing the .isolate file,
# isolate_base_dir.
relative_cwd = os.path.relpath(isolate_cmd_dir,
self.saved_state.root_dir)
# Now that we know where the root is, check that the path_variables point
# inside it.
for k, v in self.saved_state.path_variables.iteritems():
dest = os.path.join(isolate_cmd_dir, relative_cwd, v)
if not file_path.path_starts_with(self.saved_state.root_dir, dest):
raise isolated_format.MappingError(
'Path variable %s=%r points outside the inferred root directory '
'%s; %s' % (k, v, self.saved_state.root_dir, dest))
# Normalize the files based to self.saved_state.root_dir. It is important to
# keep the trailing os.path.sep at that step.
infiles = [
file_path.relpath(
file_path.normpath(os.path.join(isolate_cmd_dir, f)),
self.saved_state.root_dir) for f in infiles
]
follow_symlinks = False
if not collapse_symlinks:
follow_symlinks = sys.platform != 'win32'
# Expand the directories by listing each file inside. Up to now, trailing
# os.path.sep must be kept.
infiles = isolated_format.expand_directories_and_symlinks(
self.saved_state.root_dir, infiles, tools.gen_blacklist(blacklist),
follow_symlinks, ignore_broken_items)
# Finally, update the new data to be able to generate the foo.isolated file,
# the file that is used by run_isolated.py.
self.saved_state.update_isolated(command, infiles, read_only,
relative_cwd)
logging.debug(self)
def load_complete_state(options, cwd, subdir, skip_update):
"""Loads a CompleteState.
This includes data from .isolate and .isolated.state files. Never reads the
.isolated file.
Arguments:
options: Options instance generated with process_isolate_options. For either
options.isolate and options.isolated, if the value is set, it is an
absolute path.
cwd: base directory to be used when loading the .isolate file.
subdir: optional argument to only process file in the subdirectory, relative
to CompleteState.root_dir.
skip_update: Skip trying to load the .isolate file and processing the
dependencies. It is useful when not needed, like when tracing.
"""
assert not options.isolate or os.path.isabs(options.isolate)
assert not options.isolated or os.path.isabs(options.isolated)
cwd = file_path.get_native_path_case(unicode(cwd))
if options.isolated:
# Load the previous state if it was present. Namely, "foo.isolated.state".
# Note: this call doesn't load the .isolate file.
complete_state = CompleteState.load_files(options.isolated)
else:
# Constructs a dummy object that cannot be saved. Useful for temporary
# commands like 'run'. There is no directory containing a .isolated file so
# specify the current working directory as a valid directory.
complete_state = CompleteState(None, SavedState(os.getcwd()))
if not options.isolate:
if not complete_state.saved_state.isolate_file:
if not skip_update:
raise ExecutionError('A .isolate file is required.')
isolate = None
else:
isolate = complete_state.saved_state.isolate_filepath
else:
isolate = options.isolate
if complete_state.saved_state.isolate_file:
rel_isolate = file_path.safe_relpath(
options.isolate, complete_state.saved_state.isolated_basedir)
if rel_isolate != complete_state.saved_state.isolate_file:
# This happens if the .isolate file was moved for example. In this case,
# discard the saved state.
logging.warning(
'--isolated %s != %s as saved in %s. Discarding saved state',
rel_isolate, complete_state.saved_state.isolate_file,
isolatedfile_to_state(options.isolated))
complete_state = CompleteState(
options.isolated,
SavedState(complete_state.saved_state.isolated_basedir))
if not skip_update:
# Then load the .isolate and expands directories.
complete_state.load_isolate(cwd, isolate, options.path_variables,
options.config_variables,
options.extra_variables, options.blacklist,
options.ignore_broken_items,
options.collapse_symlinks)
# Regenerate complete_state.saved_state.files.
if subdir:
subdir = unicode(subdir)
# This is tricky here. If it is a path, take it from the root_dir. If
# it is a variable, it must be keyed from the directory containing the
# .isolate file. So translate all variables first.
translated_path_variables = dict(
(k,
os.path.normpath(
os.path.join(complete_state.saved_state.relative_cwd, v)))
for k, v in complete_state.saved_state.path_variables.iteritems())
subdir = isolate_format.eval_variables(subdir,
translated_path_variables)
subdir = subdir.replace('/', os.path.sep)
if not skip_update:
complete_state.files_to_metadata(subdir, options.collapse_symlinks)
return complete_state
def isolate_format_eval_variables(args):
s = json.load(sys.stdin)
return isolate_format.eval_variables(args[0], s)
