def contains(self, items): # Ensure all items were initialized with 'prepare' call. Storage does that. assert all(i.digest is not None and i.size is not None for i in items) # Request body is a json encoded list of dicts. body = { 'items': [{ 'digest': item.digest, 'is_isolated': bool(item.high_priority), 'size': item.size, } for item in items], 'namespace': self._namespace_dict, } query_url = '%s/api/isolateservice/v1/preupload' % self._base_url # Response body is a list of push_urls (or null if file is already present). response = None try: response = net.url_read_json(url=query_url, data=body) if response is None: raise isolated_format.MappingError( 'Failed to execute preupload query') except ValueError as err: raise isolated_format.MappingError( 'Invalid response from server: %s, body is %s' % (err, response)) # Pick Items that are missing, attach _PushState to them. missing_items = {} for preupload_status in response.get('items', []): assert 'upload_ticket' in preupload_status, ( preupload_status, '/preupload did not generate an upload ticket') index = int(preupload_status['index']) missing_items[items[index]] = _IsolateServerPushState( preupload_status, items[index].size) logging.info('Queried %d files, %d cache hit', len(items), len(items) - len(missing_items)) return missing_items
def recreate_tree(outdir, indir, infiles, action, as_hash): """Creates a new tree with only the input files in it. Arguments: outdir: Output directory to create the files in. indir: Root directory the infiles are based in. infiles: dict of files to map from |indir| to |outdir|. action: One of accepted action of file_path.link_file(). as_hash: Output filename is the hash instead of relfile. """ logging.info( 'recreate_tree(outdir=%s, indir=%s, files=%d, action=%s, as_hash=%s)' % (outdir, indir, len(infiles), action, as_hash)) assert os.path.isabs(outdir) and outdir == os.path.normpath(outdir), outdir if not os.path.isdir(outdir): logging.info('Creating %s' % outdir) os.makedirs(outdir) for relfile, metadata in infiles.iteritems(): infile = os.path.join(indir, relfile) if as_hash: # Do the hashtable specific checks. if 'l' in metadata: # Skip links when storing a hashtable. continue outfile = os.path.join(outdir, metadata['h']) if os.path.isfile(outfile): # Just do a quick check that the file size matches. No need to stat() # again the input file, grab the value from the dict. if not 's' in metadata: raise isolated_format.MappingError( 'Misconfigured item %s: %s' % (relfile, metadata)) if metadata['s'] == os.stat(outfile).st_size: continue else: logging.warn('Overwritting %s' % metadata['h']) os.remove(outfile) else: outfile = os.path.join(outdir, relfile) outsubdir = os.path.dirname(outfile) if not os.path.isdir(outsubdir): os.makedirs(outsubdir) if 'l' in metadata: pointed = metadata['l'] logging.debug('Symlink: %s -> %s' % (outfile, pointed)) # symlink doesn't exist on Windows. os.symlink(pointed, outfile) # pylint: disable=E1101 else: file_path.link_file(outfile, infile, action)
def load_isolate(self, cwd, isolate_file, path_variables, config_variables, extra_variables, blacklist, ignore_broken_items, collapse_symlinks): """Updates self.isolated and self.saved_state with information loaded from a .isolate file. Processes the loaded data, deduce root_dir, relative_cwd. """ # Make sure to not depend on os.getcwd(). assert os.path.isabs(isolate_file), isolate_file isolate_file = file_path.get_native_path_case(isolate_file) logging.info('CompleteState.load_isolate(%s, %s, %s, %s, %s, %s, %s)', cwd, isolate_file, path_variables, config_variables, extra_variables, ignore_broken_items, collapse_symlinks) # Config variables are not affected by the paths and must be used to # retrieve the paths, so update them first. self.saved_state.update_config(config_variables) with fs.open(isolate_file, 'r') as f: # At that point, variables are not replaced yet in command and infiles. # infiles may contain directory entries and is in posix style. command, infiles, read_only, isolate_cmd_dir = ( isolate_format.load_isolate_for_config( os.path.dirname(isolate_file), f.read(), self.saved_state.config_variables)) # Processes the variables with the new found relative root. Note that 'cwd' # is used when path variables are used. path_variables = normalize_path_variables(cwd, path_variables, isolate_cmd_dir) # Update the rest of the saved state. self.saved_state.update(isolate_file, path_variables, extra_variables) total_variables = self.saved_state.path_variables.copy() total_variables.update(self.saved_state.config_variables) total_variables.update(self.saved_state.extra_variables) command = [ isolate_format.eval_variables(i, total_variables) for i in command ] total_variables = self.saved_state.path_variables.copy() total_variables.update(self.saved_state.extra_variables) infiles = [ isolate_format.eval_variables(f, total_variables) for f in infiles ] # root_dir is automatically determined by the deepest root accessed with the # form '../../foo/bar'. Note that path variables must be taken in account # too, add them as if they were input files. self.saved_state.root_dir = isolate_format.determine_root_dir( isolate_cmd_dir, infiles + self.saved_state.path_variables.values()) # The relative directory is automatically determined by the relative path # between root_dir and the directory containing the .isolate file, # isolate_base_dir. relative_cwd = os.path.relpath(isolate_cmd_dir, self.saved_state.root_dir) # Now that we know where the root is, check that the path_variables point # inside it. for k, v in self.saved_state.path_variables.iteritems(): dest = os.path.join(isolate_cmd_dir, relative_cwd, v) if not file_path.path_starts_with(self.saved_state.root_dir, dest): raise isolated_format.MappingError( 'Path variable %s=%r points outside the inferred root directory ' '%s; %s' % (k, v, self.saved_state.root_dir, dest)) # Normalize the files based to self.saved_state.root_dir. It is important to # keep the trailing os.path.sep at that step. infiles = [ file_path.relpath( file_path.normpath(os.path.join(isolate_cmd_dir, f)), self.saved_state.root_dir) for f in infiles ] follow_symlinks = False if not collapse_symlinks: follow_symlinks = sys.platform != 'win32' # Expand the directories by listing each file inside. Up to now, trailing # os.path.sep must be kept. infiles = isolated_format.expand_directories_and_symlinks( self.saved_state.root_dir, infiles, tools.gen_blacklist(blacklist), follow_symlinks, ignore_broken_items) # Finally, update the new data to be able to generate the foo.isolated file, # the file that is used by run_isolated.py. self.saved_state.update_isolated(command, infiles, read_only, relative_cwd) logging.debug(self)