def contains(self, items):
# Ensure all items were initialized with 'prepare' call. Storage does that.
assert all(i.digest is not None and i.size is not None for i in items)
# Request body is a json encoded list of dicts.
body = {
'items': [{
'digest': item.digest,
'is_isolated': bool(item.high_priority),
'size': item.size,
} for item in items],
'namespace':
self._namespace_dict,
}
query_url = '%s/api/isolateservice/v1/preupload' % self._base_url
# Response body is a list of push_urls (or null if file is already present).
response = None
try:
response = net.url_read_json(url=query_url, data=body)
if response is None:
raise isolated_format.MappingError(
'Failed to execute preupload query')
except ValueError as err:
raise isolated_format.MappingError(
'Invalid response from server: %s, body is %s' %
(err, response))
# Pick Items that are missing, attach _PushState to them.
missing_items = {}
for preupload_status in response.get('items', []):
assert 'upload_ticket' in preupload_status, (
preupload_status,
'/preupload did not generate an upload ticket')
index = int(preupload_status['index'])
missing_items[items[index]] = _IsolateServerPushState(
preupload_status, items[index].size)
logging.info('Queried %d files, %d cache hit', len(items),
len(items) - len(missing_items))
return missing_items
def recreate_tree(outdir, indir, infiles, action, as_hash):
"""Creates a new tree with only the input files in it.
Arguments:
outdir: Output directory to create the files in.
indir: Root directory the infiles are based in.
infiles: dict of files to map from |indir| to |outdir|.
action: One of accepted action of file_path.link_file().
as_hash: Output filename is the hash instead of relfile.
"""
logging.info(
'recreate_tree(outdir=%s, indir=%s, files=%d, action=%s, as_hash=%s)' %
(outdir, indir, len(infiles), action, as_hash))
assert os.path.isabs(outdir) and outdir == os.path.normpath(outdir), outdir
if not os.path.isdir(outdir):
logging.info('Creating %s' % outdir)
os.makedirs(outdir)
for relfile, metadata in infiles.iteritems():
infile = os.path.join(indir, relfile)
if as_hash:
# Do the hashtable specific checks.
if 'l' in metadata:
# Skip links when storing a hashtable.
continue
outfile = os.path.join(outdir, metadata['h'])
if os.path.isfile(outfile):
# Just do a quick check that the file size matches. No need to stat()
# again the input file, grab the value from the dict.
if not 's' in metadata:
raise isolated_format.MappingError(
'Misconfigured item %s: %s' % (relfile, metadata))
if metadata['s'] == os.stat(outfile).st_size:
continue
else:
logging.warn('Overwritting %s' % metadata['h'])
os.remove(outfile)
else:
outfile = os.path.join(outdir, relfile)
outsubdir = os.path.dirname(outfile)
if not os.path.isdir(outsubdir):
os.makedirs(outsubdir)
if 'l' in metadata:
pointed = metadata['l']
logging.debug('Symlink: %s -> %s' % (outfile, pointed))
# symlink doesn't exist on Windows.
os.symlink(pointed, outfile) # pylint: disable=E1101
else:
file_path.link_file(outfile, infile, action)
def load_isolate(self, cwd, isolate_file, path_variables, config_variables,
extra_variables, blacklist, ignore_broken_items,
collapse_symlinks):
"""Updates self.isolated and self.saved_state with information loaded from a
.isolate file.
Processes the loaded data, deduce root_dir, relative_cwd.
"""
# Make sure to not depend on os.getcwd().
assert os.path.isabs(isolate_file), isolate_file
isolate_file = file_path.get_native_path_case(isolate_file)
logging.info('CompleteState.load_isolate(%s, %s, %s, %s, %s, %s, %s)',
cwd, isolate_file, path_variables, config_variables,
extra_variables, ignore_broken_items, collapse_symlinks)
# Config variables are not affected by the paths and must be used to
# retrieve the paths, so update them first.
self.saved_state.update_config(config_variables)
with fs.open(isolate_file, 'r') as f:
# At that point, variables are not replaced yet in command and infiles.
# infiles may contain directory entries and is in posix style.
command, infiles, read_only, isolate_cmd_dir = (
isolate_format.load_isolate_for_config(
os.path.dirname(isolate_file), f.read(),
self.saved_state.config_variables))
# Processes the variables with the new found relative root. Note that 'cwd'
# is used when path variables are used.
path_variables = normalize_path_variables(cwd, path_variables,
isolate_cmd_dir)
# Update the rest of the saved state.
self.saved_state.update(isolate_file, path_variables, extra_variables)
total_variables = self.saved_state.path_variables.copy()
total_variables.update(self.saved_state.config_variables)
total_variables.update(self.saved_state.extra_variables)
command = [
isolate_format.eval_variables(i, total_variables) for i in command
]
total_variables = self.saved_state.path_variables.copy()
total_variables.update(self.saved_state.extra_variables)
infiles = [
isolate_format.eval_variables(f, total_variables) for f in infiles
]
# root_dir is automatically determined by the deepest root accessed with the
# form '../../foo/bar'. Note that path variables must be taken in account
# too, add them as if they were input files.
self.saved_state.root_dir = isolate_format.determine_root_dir(
isolate_cmd_dir,
infiles + self.saved_state.path_variables.values())
# The relative directory is automatically determined by the relative path
# between root_dir and the directory containing the .isolate file,
# isolate_base_dir.
relative_cwd = os.path.relpath(isolate_cmd_dir,
self.saved_state.root_dir)
# Now that we know where the root is, check that the path_variables point
# inside it.
for k, v in self.saved_state.path_variables.iteritems():
dest = os.path.join(isolate_cmd_dir, relative_cwd, v)
if not file_path.path_starts_with(self.saved_state.root_dir, dest):
raise isolated_format.MappingError(
'Path variable %s=%r points outside the inferred root directory '
'%s; %s' % (k, v, self.saved_state.root_dir, dest))
# Normalize the files based to self.saved_state.root_dir. It is important to
# keep the trailing os.path.sep at that step.
infiles = [
file_path.relpath(
file_path.normpath(os.path.join(isolate_cmd_dir, f)),
self.saved_state.root_dir) for f in infiles
]
follow_symlinks = False
if not collapse_symlinks:
follow_symlinks = sys.platform != 'win32'
# Expand the directories by listing each file inside. Up to now, trailing
# os.path.sep must be kept.
infiles = isolated_format.expand_directories_and_symlinks(
self.saved_state.root_dir, infiles, tools.gen_blacklist(blacklist),
follow_symlinks, ignore_broken_items)
# Finally, update the new data to be able to generate the foo.isolated file,
# the file that is used by run_isolated.py.
self.saved_state.update_isolated(command, infiles, read_only,
relative_cwd)
logging.debug(self)
