def delete_and_upload(storage, out_dir, leak_temp_dir): """Deletes the temporary run directory and uploads results back. Returns: tuple(outputs_ref, success, cold, hot) - outputs_ref: a dict referring to the results archived back to the isolated server, if applicable. - success: False if something occurred that means that the task must forcibly be considered a failure, e.g. zombie processes were left behind. - cold: list of size of cold items, they had to be uploaded. - hot: list of size of hot items, they didn't have to be uploaded. """ # Upload out_dir and generate a .isolated file out of this directory. It is # only done if files were written in the directory. outputs_ref = None cold = [] hot = [] if fs.isdir(out_dir) and fs.listdir(out_dir): with tools.Profiler('ArchiveOutput'): try: results, f_cold, f_hot = isolateserver.archive_files_to_storage( storage, [out_dir], None) outputs_ref = { 'isolated': results[0][0], 'isolatedserver': storage.location, 'namespace': storage.namespace, } cold = sorted(i.size for i in f_cold) hot = sorted(i.size for i in f_hot) except isolateserver.Aborted: # This happens when a signal SIGTERM was received while uploading data. # There is 2 causes: # - The task was too slow and was about to be killed anyway due to # exceeding the hard timeout. # - The amount of data uploaded back is very large and took too much # time to archive. sys.stderr.write('Received SIGTERM while uploading') # Re-raise, so it will be treated as an internal failure. raise try: if (not leak_temp_dir and fs.isdir(out_dir) and not file_path.rmtree(out_dir)): logging.error('Had difficulties removing out_dir %s', out_dir) return outputs_ref, False, cold, hot except OSError as e: # When this happens, it means there's a process error. logging.exception('Had difficulties removing out_dir %s: %s', out_dir, e) return outputs_ref, False, cold, hot return outputs_ref, True, cold, hot
def CMDremap(parser, args): """Creates a directory with all the dependencies mapped into it. Useful to test manually why a test is failing. The target executable is not run. """ add_isolate_options(parser) add_outdir_options(parser) add_skip_refresh_option(parser) options, args = parser.parse_args(args) if args: parser.error('Unsupported argument: %s' % args) cwd = os.getcwd() process_isolate_options(parser, options, cwd, require_isolated=False) process_outdir_options(parser, options, cwd) complete_state = load_complete_state(options, cwd, None, options.skip_refresh) if not fs.isdir(options.outdir): fs.makedirs(options.outdir) print('Remapping into %s' % options.outdir) if fs.listdir(options.outdir): raise ExecutionError('Can\'t remap in a non-empty directory') create_isolate_tree( options.outdir, complete_state.root_dir, complete_state.saved_state.files, complete_state.saved_state.relative_cwd, complete_state.saved_state.read_only) if complete_state.isolated_filepath: complete_state.save_files() return 0
def create_isolate_tree(outdir, root_dir, files, relative_cwd, read_only): """Creates a isolated tree usable for test execution. Returns the current working directory where the isolated command should be started in. """ # Forcibly copy when the tree has to be read only. Otherwise the inode is # modified, and this cause real problems because the user's source tree # becomes read only. On the other hand, the cost of doing file copy is huge. if read_only not in (0, None): action = file_path.COPY else: action = file_path.HARDLINK_WITH_FALLBACK recreate_tree( outdir=outdir, indir=root_dir, infiles=files, action=action, as_hash=False) cwd = os.path.normpath(os.path.join(outdir, relative_cwd)) if not fs.isdir(cwd): # It can happen when no files are mapped from the directory containing the # .isolate file. But the directory must exist to be the current working # directory. fs.makedirs(cwd) run_isolated.change_tree_read_only(outdir, read_only) return cwd
def read_tree(path): """Returns a dict with {filepath: content}.""" if not fs.isdir(path): return None out = {} for root, _, filenames in fs.walk(path): for filename in filenames: p = os.path.join(root, filename) out[os.path.relpath(p, path)] = read_file(p) return out
def trim(self): evicted = [] with self._lock: if not fs.isdir(self.cache_dir): return evicted # Trim according to maximum number of items. if self._policies.max_items: while len(self._lru) > self._policies.max_items: name, size = self._remove_lru_item() evicted.append(size) logging.info( 'NamedCache.trim(): Removed %r(%d) due to max_items(%d)', name, size, self._policies.max_items) # Trim according to maximum age. if self._policies.max_age_secs: cutoff = self._lru.time_fn() - self._policies.max_age_secs while self._lru: _name, (_data, ts) = self._lru.get_oldest() if ts >= cutoff: break name, size = self._remove_lru_item() evicted.append(size) logging.info( 'NamedCache.trim(): Removed %r(%d) due to max_age_secs(%d)', name, size, self._policies.max_age_secs) # Trim according to minimum free space. if self._policies.min_free_space: while self._lru: free_space = file_path.get_free_space(self.cache_dir) if free_space >= self._policies.min_free_space: break name, size = self._remove_lru_item() evicted.append(size) logging.info( 'NamedCache.trim(): Removed %r(%d) due to min_free_space(%d)', name, size, self._policies.min_free_space) # Trim according to maximum total size. if self._policies.max_cache_size: while self._lru: total = sum(size for _rel_cache, size in self._lru.values()) if total <= self._policies.max_cache_size: break name, size = self._remove_lru_item() evicted.append(size) logging.info( 'NamedCache.trim(): Removed %r(%d) due to max_cache_size(%d)', name, size, self._policies.max_cache_size) self._save() return evicted
def _save(self): """Saves the LRU ordering.""" self._lock.assert_locked() if sys.platform != 'win32': d = os.path.dirname(self.state_file) if fs.isdir(d): # Necessary otherwise the file can't be created. file_path.set_read_only(d, False) if fs.isfile(self.state_file): file_path.set_read_only(self.state_file, False) self._lru.save(self.state_file)
def make_tree(out, contents): for relpath, content in sorted(contents.iteritems()): filepath = os.path.join(out, relpath.replace('/', os.path.sep)) dirpath = os.path.dirname(filepath) if not fs.isdir(dirpath): fs.makedirs(dirpath, 0700) if isinstance(content, SymLink): fs.symlink(content, filepath) else: mode = 0700 if relpath.endswith('.py') else 0600 flags = os.O_WRONLY | os.O_CREAT if sys.platform == 'win32': # pylint: disable=no-member flags |= os.O_BINARY with os.fdopen(os.open(filepath, flags, mode), 'wb') as f: f.write(content)
def __init__(self, task_output_dir, shard_count): """Initializes TaskOutputCollector, ensures |task_output_dir| exists. Args: task_output_dir: (optional) local directory to put fetched files to. shard_count: expected number of task shards. """ self.task_output_dir = ( unicode(os.path.abspath(task_output_dir)) if task_output_dir else task_output_dir) self.shard_count = shard_count self._lock = threading.Lock() self._per_shard_results = {} self._storage = None if self.task_output_dir and not fs.isdir(self.task_output_dir): fs.makedirs(self.task_output_dir)
def cleanup(self): """Cleans up the cache directory. Ensures there is no unknown files in cache_dir. Ensures the read-only bits are set correctly. At that point, the cache was already loaded, trimmed to respect cache policies. """ with self._lock: fs.chmod(self.cache_dir, 0700) # Ensure that all files listed in the state still exist and add new ones. previous = set(self._lru) # It'd be faster if there were a readdir() function. for filename in fs.listdir(self.cache_dir): if filename == self.STATE_FILE: fs.chmod(os.path.join(self.cache_dir, filename), 0600) continue if filename in previous: fs.chmod(os.path.join(self.cache_dir, filename), 0400) previous.remove(filename) continue # An untracked file. Delete it. logging.warning('Removing unknown file %s from cache', filename) p = self._path(filename) if fs.isdir(p): try: file_path.rmtree(p) except OSError: pass else: file_path.try_remove(p) continue if previous: # Filter out entries that were not found. logging.warning('Removed %d lost files', len(previous)) for filename in previous: self._lru.pop(filename) self._save()
def _remove(self, name): """Removes a cache directory and entry. Returns: Number of caches deleted. """ self._lock.assert_locked() # First try to remove the alias if it exists. named_dir = self._get_named_path(name) if fs.islink(named_dir): fs.unlink(named_dir) # Then remove the actual data. if name not in self._lru: return rel_path, _size = self._lru.get(name) abs_path = os.path.join(self.cache_dir, rel_path) if fs.isdir(abs_path): file_path.rmtree(abs_path) self._lru.pop(name)
def _load(self, trim, time_fn): """Loads state of the cache from json file. If cache_dir does not exist on disk, it is created. """ self._lock.assert_locked() if not fs.isfile(self.state_file): if not fs.isdir(self.cache_dir): fs.makedirs(self.cache_dir) else: # Load state of the cache. try: self._lru = lru.LRUDict.load(self.state_file) except ValueError as err: logging.error('Failed to load cache state: %s' % (err, )) # Don't want to keep broken state file. file_path.try_remove(self.state_file) if time_fn: self._lru.time_fn = time_fn if trim: self._trim()
def __init__(self, cache_dir, policies, time_fn=None): """Initializes NamedCaches. Arguments: - cache_dir is a directory for persistent cache storage. - policies is a CachePolicies instance. - time_fn is a function that returns timestamp (float) and used to take timestamps when new caches are requested. Used in unit tests. """ super(NamedCache, self).__init__(cache_dir) self._policies = policies # LRU {cache_name -> tuple(cache_location, size)} self.state_file = os.path.join(cache_dir, self.STATE_FILE) self._lru = lru.LRUDict() if not fs.isdir(self.cache_dir): fs.makedirs(self.cache_dir) elif fs.isfile(self.state_file): try: self._lru = lru.LRUDict.load(self.state_file) for _, size in self._lru.values(): if not isinstance(size, six.integer_types): with open(self.state_file, 'r') as f: logging.info('named cache state file: %s\n%s', self.state_file, f.read()) raise ValueError("size is not integer: %s" % size) except ValueError: logging.exception( 'NamedCache: failed to load named cache state file; obliterating' ) file_path.rmtree(self.cache_dir) fs.makedirs(self.cache_dir) self._lru = lru.LRUDict() with self._lock: self._try_upgrade() if time_fn: self._lru.time_fn = time_fn
def map_and_run(command, isolated_hash, storage, isolate_cache, outputs, init_named_caches, leak_temp_dir, root_dir, hard_timeout, grace_period, bot_file, install_packages_fn, use_symlinks, constant_run_path): """Runs a command with optional isolated input/output. See run_tha_test for argument documentation. Returns metadata about the result. """ assert isinstance(command, list), command assert root_dir or root_dir is None result = { 'duration': None, 'exit_code': None, 'had_hard_timeout': False, 'internal_failure': None, 'stats': { # 'isolated': { # 'cipd': { # 'duration': 0., # 'get_client_duration': 0., # }, # 'download': { # 'duration': 0., # 'initial_number_items': 0, # 'initial_size': 0, # 'items_cold': '<large.pack()>', # 'items_hot': '<large.pack()>', # }, # 'upload': { # 'duration': 0., # 'items_cold': '<large.pack()>', # 'items_hot': '<large.pack()>', # }, # }, }, # 'cipd_pins': { # 'packages': [ # {'package_name': ..., 'version': ..., 'path': ...}, # ... # ], # 'client_package': {'package_name': ..., 'version': ...}, # }, 'outputs_ref': None, 'version': 5, } if root_dir: file_path.ensure_tree(root_dir, 0700) elif isolate_cache.cache_dir: root_dir = os.path.dirname(isolate_cache.cache_dir) # See comment for these constants. # If root_dir is not specified, it is not constant. # TODO(maruel): This is not obvious. Change this to become an error once we # make the constant_run_path an exposed flag. if constant_run_path and root_dir: run_dir = os.path.join(root_dir, ISOLATED_RUN_DIR) os.mkdir(run_dir) else: run_dir = make_temp_dir(ISOLATED_RUN_DIR, root_dir) # storage should be normally set but don't crash if it is not. This can happen # as Swarming task can run without an isolate server. out_dir = make_temp_dir(ISOLATED_OUT_DIR, root_dir) if storage else None tmp_dir = make_temp_dir(ISOLATED_TMP_DIR, root_dir) cwd = run_dir try: with install_packages_fn(run_dir) as cipd_info: if cipd_info: result['stats']['cipd'] = cipd_info.stats result['cipd_pins'] = cipd_info.pins if isolated_hash: isolated_stats = result['stats'].setdefault('isolated', {}) bundle, isolated_stats['download'] = fetch_and_map( isolated_hash=isolated_hash, storage=storage, cache=isolate_cache, outdir=run_dir, use_symlinks=use_symlinks) change_tree_read_only(run_dir, bundle.read_only) cwd = os.path.normpath(os.path.join(cwd, bundle.relative_cwd)) # Inject the command if bundle.command: command = bundle.command + command if not command: # Handle this as a task failure, not an internal failure. sys.stderr.write( '<No command was specified!>\n' '<Please secify a command when triggering your Swarming task>\n' ) result['exit_code'] = 1 return result # If we have an explicit list of files to return, make sure their # directories exist now. if storage and outputs: isolateserver.create_directories(run_dir, outputs) command = tools.fix_python_path(command) command = process_command(command, out_dir, bot_file) file_path.ensure_command_has_abs_path(command, cwd) with init_named_caches(run_dir): sys.stdout.flush() start = time.time() try: result['exit_code'], result[ 'had_hard_timeout'] = run_command( command, cwd, get_command_env(tmp_dir, cipd_info), hard_timeout, grace_period) finally: result['duration'] = max(time.time() - start, 0) except Exception as e: # An internal error occurred. Report accordingly so the swarming task will # be retried automatically. logging.exception('internal failure: %s', e) result['internal_failure'] = str(e) on_error.report(None) # Clean up finally: try: # Try to link files to the output directory, if specified. if out_dir: link_outputs_to_outdir(run_dir, out_dir, outputs) success = False if leak_temp_dir: success = True logging.warning( 'Deliberately leaking %s for later examination', run_dir) else: # On Windows rmtree(run_dir) call above has a synchronization effect: it # finishes only when all task child processes terminate (since a running # process locks *.exe file). Examine out_dir only after that call # completes (since child processes may write to out_dir too and we need # to wait for them to finish). if fs.isdir(run_dir): try: success = file_path.rmtree(run_dir) except OSError as e: logging.error('Failure with %s', e) success = False if not success: print >> sys.stderr, ( 'Failed to delete the run directory, thus failing the task.\n' 'This may be due to a subprocess outliving the main task\n' 'process, holding on to resources. Please fix the task so\n' 'that it releases resources and cleans up subprocesses.' ) if result['exit_code'] == 0: result['exit_code'] = 1 if fs.isdir(tmp_dir): try: success = file_path.rmtree(tmp_dir) except OSError as e: logging.error('Failure with %s', e) success = False if not success: print >> sys.stderr, ( 'Failed to delete the temp directory, thus failing the task.\n' 'This may be due to a subprocess outliving the main task\n' 'process, holding on to resources. Please fix the task so\n' 'that it releases resources and cleans up subprocesses.' ) if result['exit_code'] == 0: result['exit_code'] = 1 # This deletes out_dir if leak_temp_dir is not set. if out_dir: isolated_stats = result['stats'].setdefault('isolated', {}) result['outputs_ref'], success, isolated_stats['upload'] = ( delete_and_upload(storage, out_dir, leak_temp_dir)) if not success and result['exit_code'] == 0: result['exit_code'] = 1 except Exception as e: # Swallow any exception in the main finally clause. if out_dir: logging.exception('Leaking out_dir %s: %s', out_dir, e) result['internal_failure'] = str(e) return result
def run_tha_test(isolated_hash, storage, cache, leak_temp_dir, result_json, root_dir, hard_timeout, grace_period, extra_args): """Downloads the dependencies in the cache, hardlinks them into a temporary directory and runs the executable from there. A temporary directory is created to hold the output files. The content inside this directory will be uploaded back to |storage| packaged as a .isolated file. Arguments: isolated_hash: the SHA-1 of the .isolated file that must be retrieved to recreate the tree of files to run the target executable. storage: an isolateserver.Storage object to retrieve remote objects. This object has a reference to an isolateserver.StorageApi, which does the actual I/O. cache: an isolateserver.LocalCache to keep from retrieving the same objects constantly by caching the objects retrieved. Can be on-disk or in-memory. leak_temp_dir: if true, the temporary directory will be deliberately leaked for later examination. result_json: file path to dump result metadata into. If set, the process exit code is always 0 unless an internal error occured. root_dir: directory to the path to use to create the temporary directory. If not specified, a random temporary directory is created. hard_timeout: kills the process if it lasts more than this amount of seconds. grace_period: number of seconds to wait between SIGTERM and SIGKILL. extra_args: optional arguments to add to the command stated in the .isolate file. Returns: Process exit code that should be used. """ if result_json: # Write a json output file right away in case we get killed. result = { 'exit_code': None, 'had_hard_timeout': False, 'internal_failure': 'Was terminated before completion', 'outputs_ref': None, 'version': 2, } tools.write_json(result_json, result, dense=True) # run_isolated exit code. Depends on if result_json is used or not. result = map_and_run(isolated_hash, storage, cache, leak_temp_dir, root_dir, hard_timeout, grace_period, extra_args) logging.info('Result:\n%s', tools.format_json(result, dense=True)) if result_json: # We've found tests to delete 'work' when quitting, causing an exception # here. Try to recreate the directory if necessary. work_dir = os.path.dirname(result_json) if not fs.isdir(work_dir): fs.mkdir(work_dir) tools.write_json(result_json, result, dense=True) # Only return 1 if there was an internal error. return int(bool(result['internal_failure'])) # Marshall into old-style inline output. if result['outputs_ref']: data = { 'hash': result['outputs_ref']['isolated'], 'namespace': result['outputs_ref']['namespace'], 'storage': result['outputs_ref']['isolatedserver'], } sys.stdout.flush() print('[run_isolated_out_hack]%s[/run_isolated_out_hack]' % tools.format_json(data, dense=True)) sys.stdout.flush() return result['exit_code'] or int(bool(result['internal_failure']))
def map_and_run(isolated_hash, storage, cache, leak_temp_dir, root_dir, hard_timeout, grace_period, extra_args): """Maps and run the command. Returns metadata about the result.""" # TODO(maruel): Include performance statistics. result = { 'exit_code': None, 'had_hard_timeout': False, 'internal_failure': None, 'outputs_ref': None, 'version': 2, } if root_dir: if not fs.isdir(root_dir): fs.makedirs(root_dir, 0700) prefix = u'' else: root_dir = os.path.dirname( cache.cache_dir) if cache.cache_dir else None prefix = u'isolated_' run_dir = make_temp_dir(prefix + u'run', root_dir) out_dir = make_temp_dir(prefix + u'out', root_dir) tmp_dir = make_temp_dir(prefix + u'tmp', root_dir) try: try: bundle = isolateserver.fetch_isolated(isolated_hash=isolated_hash, storage=storage, cache=cache, outdir=run_dir, require_command=True) except isolateserver.IsolatedErrorNoCommand: # Handle this as a task failure, not an internal failure. sys.stderr.write( '<The .isolated doesn\'t declare any command to run!>\n' '<Check your .isolate for missing \'command\' variable>\n') if os.environ.get('SWARMING_TASK_ID'): # Give an additional hint when running as a swarming task. sys.stderr.write('<This occurs at the \'isolate\' step>\n') result['exit_code'] = 1 return result change_tree_read_only(run_dir, bundle.read_only) cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd)) command = bundle.command + extra_args file_path.ensure_command_has_abs_path(command, cwd) result['exit_code'], result['had_hard_timeout'] = run_command( process_command(command, out_dir), cwd, tmp_dir, hard_timeout, grace_period) except Exception as e: # An internal error occured. Report accordingly so the swarming task will be # retried automatically. logging.exception('internal failure: %s', e) result['internal_failure'] = str(e) on_error.report(None) finally: try: if leak_temp_dir: logging.warning( 'Deliberately leaking %s for later examination', run_dir) else: # On Windows rmtree(run_dir) call above has a synchronization effect: it # finishes only when all task child processes terminate (since a running # process locks *.exe file). Examine out_dir only after that call # completes (since child processes may write to out_dir too and we need # to wait for them to finish). if fs.isdir(run_dir): try: success = file_path.rmtree(run_dir) except OSError as e: logging.error('Failure with %s', e) success = False if not success: print >> sys.stderr, ( 'Failed to delete the run directory, forcibly failing\n' 'the task because of it. No zombie process can outlive a\n' 'successful task run and still be marked as successful.\n' 'Fix your stuff.') if result['exit_code'] == 0: result['exit_code'] = 1 if fs.isdir(tmp_dir): try: success = file_path.rmtree(tmp_dir) except OSError as e: logging.error('Failure with %s', e) success = False if not success: print >> sys.stderr, ( 'Failed to delete the temporary directory, forcibly failing\n' 'the task because of it. No zombie process can outlive a\n' 'successful task run and still be marked as successful.\n' 'Fix your stuff.') if result['exit_code'] == 0: result['exit_code'] = 1 # This deletes out_dir if leak_temp_dir is not set. result['outputs_ref'], success = delete_and_upload( storage, out_dir, leak_temp_dir) if not success and result['exit_code'] == 0: result['exit_code'] = 1 except Exception as e: # Swallow any exception in the main finally clause. logging.exception('Leaking out_dir %s: %s', out_dir, e) result['internal_failure'] = str(e) return result
def CMDbatcharchive(parser, args): """Archives multiple isolated trees at once. Using single command instead of multiple sequential invocations allows to cut redundant work when isolated trees share common files (e.g. file hashes are checked only once, their presence on the server is checked only once, and so on). Takes a list of paths to *.isolated.gen.json files that describe what trees to isolate. Format of files is: { "version": 1, "dir": <absolute path to a directory all other paths are relative to>, "args": [list of command line arguments for single 'archive' command] } """ isolateserver.add_isolate_server_options(parser) isolateserver.add_archive_options(parser) auth.add_auth_options(parser) parser.add_option( '--dump-json', metavar='FILE', help='Write isolated hashes of archived trees to this file as JSON') options, args = parser.parse_args(args) auth.process_auth_options(parser, options) isolateserver.process_isolate_server_options(parser, options, True, True) # Validate all incoming options, prepare what needs to be archived as a list # of tuples (archival options, working directory). work_units = [] for gen_json_path in args: # Validate JSON format of a *.isolated.gen.json file. try: data = tools.read_json(gen_json_path) except IOError as e: parser.error('Failed to open %s: %s' % (gen_json_path, e)) if data.get('version') != ISOLATED_GEN_JSON_VERSION: parser.error('Invalid version in %s' % gen_json_path) cwd = data.get('dir') if not isinstance(cwd, unicode) or not fs.isdir(cwd): parser.error('Invalid dir in %s' % gen_json_path) args = data.get('args') if (not isinstance(args, list) or not all(isinstance(x, unicode) for x in args)): parser.error('Invalid args in %s' % gen_json_path) # Convert command line (embedded in JSON) to Options object. work_units.append((parse_archive_command_line(args, cwd), cwd)) # Perform the archival, all at once. isolated_hashes = isolate_and_archive(work_units, options.isolate_server, options.namespace) # TODO(vadimsh): isolate_and_archive returns None on upload failure, there's # no way currently to figure out what *.isolated file from a batch were # successfully uploaded, so consider them all failed (and emit empty dict # as JSON result). if options.dump_json: tools.write_json(options.dump_json, isolated_hashes or {}, False) if isolated_hashes is None: return EXIT_CODE_UPLOAD_ERROR # isolated_hashes[x] is None if 'x.isolate' contains a error. if not all(isolated_hashes.itervalues()): return EXIT_CODE_ISOLATE_ERROR return 0
def cleanup(self): """Removes unknown directories. Does not recalculate the cache size since it's surprisingly slow on some OSes. """ success = True with self._lock: try: actual = set(fs.listdir(self.cache_dir)) actual.discard(self.NAMED_DIR) actual.discard(self.STATE_FILE) expected = {v[0]: k for k, v in self._lru.iteritems()} # First, handle the actual cache content. # Remove missing entries. for missing in (set(expected) - actual): self._lru.pop(expected[missing]) # Remove unexpected items. for unexpected in (actual - set(expected)): try: p = os.path.join(self.cache_dir, unexpected) if fs.isdir(p) and not fs.islink(p): file_path.rmtree(p) else: fs.remove(p) except (IOError, OSError) as e: logging.error('Failed to remove %s: %s', unexpected, e) success = False # Second, fix named cache links. named = os.path.join(self.cache_dir, self.NAMED_DIR) if os.path.isdir(named): actual = set(fs.listdir(named)) expected = set(self._lru) # Confirm entries. Do not add missing ones for now. for name in expected.intersection(actual): p = os.path.join(self.cache_dir, self.NAMED_DIR, name) expected_link = os.path.join(self.cache_dir, self._lru[name][0]) if fs.islink(p): if sys.platform == 'win32': # TODO(maruel): Implement readlink() on Windows in fs.py, then # remove this condition. # https://crbug.com/853721 continue link = fs.readlink(p) if expected_link == link: continue logging.warning( 'Unexpected symlink for cache %s: %s, expected %s', name, link, expected_link) else: logging.warning( 'Unexpected non symlink for cache %s', name) if fs.isdir(p) and not fs.islink(p): file_path.rmtree(p) else: fs.remove(p) # Remove unexpected items. for unexpected in (actual - expected): try: p = os.path.join(self.cache_dir, self.NAMED_DIR, unexpected) if fs.isdir(p): file_path.rmtree(p) else: fs.remove(p) except (IOError, OSError) as e: logging.error('Failed to remove %s: %s', unexpected, e) success = False finally: self._save() return success
def uninstall(self, src, name): """Moves the cache directory back into the named cache hive for an eventual reuse. The opposite of install(). src must be absolute and unicode. Its content is moved back into the local named caches cache. Returns the named cache size in bytes. Raises NamedCacheError if cannot uninstall the cache. """ logging.info('NamedCache.uninstall(%r, %r)', src, name) with self._lock: try: if not fs.isdir(src): logging.warning( 'NamedCache: Directory %r does not exist anymore. Cache lost.', src) return if name in self._lru: # This shouldn't happen but just remove the preexisting one and move # on. logging.error('- overwriting existing cache!') self._remove(name) # Calculate the size of the named cache to keep. It's important because # if size is zero (it's empty), we do not want to add it back to the # named caches cache. size = _get_recursive_size(src) logging.info('- Size is %d', size) if not size: # Do not save empty named cache. return size # Move the dir and create an entry for the named cache. rel_cache = self._allocate_dir() abs_cache = os.path.join(self.cache_dir, rel_cache) logging.info('- Moving to %r', rel_cache) file_path.ensure_tree(os.path.dirname(abs_cache)) fs.rename(src, abs_cache) self._lru.add(name, (rel_cache, size)) self._added.append(size) # Create symlink <cache_dir>/<named>/<name> -> <cache_dir>/<short name> # for user convenience. named_path = self._get_named_path(name) if fs.exists(named_path): file_path.remove(named_path) else: file_path.ensure_tree(os.path.dirname(named_path)) try: fs.symlink(os.path.join(u'..', rel_cache), named_path) logging.info('NamedCache: Created symlink %r to %r', named_path, abs_cache) except OSError: # Ignore on Windows. It happens when running as a normal user or when # UAC is enabled and the user is a filtered administrator account. if sys.platform != 'win32': raise return size except (IOError, OSError) as ex: # Raise using the original traceback. exc = NamedCacheError( 'cannot uninstall cache named %r at %r: %s' % (name, src, ex)) six.reraise(exc, None, sys.exc_info()[2]) finally: # Call save() at every uninstall. The assumptions are: # - The total the number of named caches is low, so the state.json file # is small, so the time it takes to write it to disk is short. # - The number of mapped named caches per task is low, so the number of # times save() is called on tear-down isn't high enough to be # significant. # - uninstall() sometimes throws due to file locking on Windows or # access rights on Linux. We want to keep as many as possible. self._save()
def map_and_run( isolated_hash, storage, cache, leak_temp_dir, root_dir, hard_timeout, grace_period, extra_args): """Maps and run the command. Returns metadata about the result.""" # TODO(maruel): Include performance statistics. result = { 'exit_code': None, 'had_hard_timeout': False, 'internal_failure': None, 'outputs_ref': None, 'version': 2, } if root_dir: if not fs.isdir(root_dir): fs.makedirs(root_dir, 0700) prefix = u'' else: root_dir = os.path.dirname(cache.cache_dir) if cache.cache_dir else None prefix = u'isolated_' run_dir = make_temp_dir(prefix + u'run', root_dir) out_dir = make_temp_dir(prefix + u'out', root_dir) tmp_dir = make_temp_dir(prefix + u'tmp', root_dir) try: bundle = isolateserver.fetch_isolated( isolated_hash=isolated_hash, storage=storage, cache=cache, outdir=run_dir, require_command=True) change_tree_read_only(run_dir, bundle.read_only) cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd)) command = bundle.command + extra_args file_path.ensure_command_has_abs_path(command, cwd) result['exit_code'], result['had_hard_timeout'] = run_command( process_command(command, out_dir), cwd, tmp_dir, hard_timeout, grace_period) except Exception as e: # An internal error occured. Report accordingly so the swarming task will be # retried automatically. logging.exception('internal failure: %s', e) result['internal_failure'] = str(e) on_error.report(None) finally: try: if leak_temp_dir: logging.warning( 'Deliberately leaking %s for later examination', run_dir) else: # On Windows rmtree(run_dir) call above has a synchronization effect: it # finishes only when all task child processes terminate (since a running # process locks *.exe file). Examine out_dir only after that call # completes (since child processes may write to out_dir too and we need # to wait for them to finish). if fs.isdir(run_dir): try: success = file_path.rmtree(run_dir) except OSError as e: logging.error('Failure with %s', e) success = False if not success: print >> sys.stderr, ( 'Failed to delete the run directory, forcibly failing\n' 'the task because of it. No zombie process can outlive a\n' 'successful task run and still be marked as successful.\n' 'Fix your stuff.') if result['exit_code'] == 0: result['exit_code'] = 1 if fs.isdir(tmp_dir): try: success = file_path.rmtree(tmp_dir) except OSError as e: logging.error('Failure with %s', e) success = False if not success: print >> sys.stderr, ( 'Failed to delete the temporary directory, forcibly failing\n' 'the task because of it. No zombie process can outlive a\n' 'successful task run and still be marked as successful.\n' 'Fix your stuff.') if result['exit_code'] == 0: result['exit_code'] = 1 # This deletes out_dir if leak_temp_dir is not set. result['outputs_ref'], success = delete_and_upload( storage, out_dir, leak_temp_dir) if not success and result['exit_code'] == 0: result['exit_code'] = 1 except Exception as e: # Swallow any exception in the main finally clause. logging.exception('Leaking out_dir %s: %s', out_dir, e) result['internal_failure'] = str(e) return result
def map_and_run( isolated_hash, storage, cache, leak_temp_dir, root_dir, hard_timeout, grace_period, extra_args): """Maps and run the command. Returns metadata about the result.""" result = { 'duration': None, 'exit_code': None, 'had_hard_timeout': False, 'internal_failure': None, 'stats': { # 'download': { # 'duration': 0., # 'initial_number_items': 0, # 'initial_size': 0, # 'items_cold': '<large.pack()>', # 'items_hot': '<large.pack()>', # }, # 'upload': { # 'duration': 0., # 'items_cold': '<large.pack()>', # 'items_hot': '<large.pack()>', # }, }, 'outputs_ref': None, 'version': 3, } if root_dir: if not fs.isdir(root_dir): fs.makedirs(root_dir, 0700) prefix = u'' else: root_dir = os.path.dirname(cache.cache_dir) if cache.cache_dir else None prefix = u'isolated_' run_dir = make_temp_dir(prefix + u'run', root_dir) out_dir = make_temp_dir(prefix + u'out', root_dir) tmp_dir = make_temp_dir(prefix + u'tmp', root_dir) try: start = time.time() bundle = isolateserver.fetch_isolated( isolated_hash=isolated_hash, storage=storage, cache=cache, outdir=run_dir) if not bundle.command: # Handle this as a task failure, not an internal failure. sys.stderr.write( '<The .isolated doesn\'t declare any command to run!>\n' '<Check your .isolate for missing \'command\' variable>\n') if os.environ.get('SWARMING_TASK_ID'): # Give an additional hint when running as a swarming task. sys.stderr.write('<This occurs at the \'isolate\' step>\n') result['exit_code'] = 1 return result result['stats']['download'] = { 'duration': time.time() - start, 'initial_number_items': cache.initial_number_items, 'initial_size': cache.initial_size, 'items_cold': base64.b64encode(large.pack(sorted(cache.added))), 'items_hot': base64.b64encode( large.pack(sorted(set(cache.linked) - set(cache.added)))), } change_tree_read_only(run_dir, bundle.read_only) cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd)) command = bundle.command + extra_args file_path.ensure_command_has_abs_path(command, cwd) sys.stdout.flush() start = time.time() try: result['exit_code'], result['had_hard_timeout'] = run_command( process_command(command, out_dir), cwd, tmp_dir, hard_timeout, grace_period) finally: result['duration'] = max(time.time() - start, 0) except Exception as e: # An internal error occured. Report accordingly so the swarming task will be # retried automatically. logging.exception('internal failure: %s', e) result['internal_failure'] = str(e) on_error.report(None) finally: try: if leak_temp_dir: logging.warning( 'Deliberately leaking %s for later examination', run_dir) else: # On Windows rmtree(run_dir) call above has a synchronization effect: it # finishes only when all task child processes terminate (since a running # process locks *.exe file). Examine out_dir only after that call # completes (since child processes may write to out_dir too and we need # to wait for them to finish). if fs.isdir(run_dir): try: success = file_path.rmtree(run_dir) except OSError as e: logging.error('Failure with %s', e) success = False if not success: print >> sys.stderr, ( 'Failed to delete the run directory, forcibly failing\n' 'the task because of it. No zombie process can outlive a\n' 'successful task run and still be marked as successful.\n' 'Fix your stuff.') if result['exit_code'] == 0: result['exit_code'] = 1 if fs.isdir(tmp_dir): try: success = file_path.rmtree(tmp_dir) except OSError as e: logging.error('Failure with %s', e) success = False if not success: print >> sys.stderr, ( 'Failed to delete the temporary directory, forcibly failing\n' 'the task because of it. No zombie process can outlive a\n' 'successful task run and still be marked as successful.\n' 'Fix your stuff.') if result['exit_code'] == 0: result['exit_code'] = 1 # This deletes out_dir if leak_temp_dir is not set. start = time.time() result['outputs_ref'], success, cold, hot = delete_and_upload( storage, out_dir, leak_temp_dir) result['stats']['upload'] = { 'duration': time.time() - start, 'items_cold': base64.b64encode(large.pack(cold)), 'items_hot': base64.b64encode(large.pack(hot)), } if not success and result['exit_code'] == 0: result['exit_code'] = 1 except Exception as e: # Swallow any exception in the main finally clause. logging.exception('Leaking out_dir %s: %s', out_dir, e) result['internal_failure'] = str(e) return result
def CMDreproduce(parser, args): """Runs a task locally that was triggered on the server. This running locally the same commands that have been run on the bot. The data downloaded will be in a subdirectory named 'work' of the current working directory. You can pass further additional arguments to the target command by passing them after --. """ parser.add_option( '--output-dir', metavar='DIR', default='', help='Directory that will have results stored into') options, args = parser.parse_args(args) extra_args = [] if not args: parser.error('Must specify exactly one task id.') if len(args) > 1: if args[1] == '--': if len(args) > 2: extra_args = args[2:] else: extra_args = args[1:] url = options.swarming + '/_ah/api/swarming/v1/task/%s/request' % args[0] request = net.url_read_json(url) if not request: print >> sys.stderr, 'Failed to retrieve request data for the task' return 1 workdir = unicode(os.path.abspath('work')) if fs.isdir(workdir): parser.error('Please delete the directory \'work\' first') fs.mkdir(workdir) properties = request['properties'] env = None if properties.get('env'): env = os.environ.copy() logging.info('env: %r', properties['env']) for i in properties['env']: key = i['key'].encode('utf-8') if not i['value']: env.pop(key, None) else: env[key] = i['value'].encode('utf-8') if properties.get('inputs_ref'): # Create the tree. with isolateserver.get_storage( properties['inputs_ref']['isolatedserver'], properties['inputs_ref']['namespace']) as storage: bundle = isolateserver.fetch_isolated( properties['inputs_ref']['isolated'], storage, isolateserver.MemoryCache(file_mode_mask=0700), workdir) command = bundle.command if bundle.relative_cwd: workdir = os.path.join(workdir, bundle.relative_cwd) command.extend(properties.get('extra_args') or []) # https://github.com/luci/luci-py/blob/master/appengine/swarming/doc/Magic-Values.md new_command = run_isolated.process_command(command, options.output_dir) if not options.output_dir and new_command != command: parser.error('The task has outputs, you must use --output-dir') command = new_command else: command = properties['command'] try: return subprocess.call(command + extra_args, env=env, cwd=workdir) except OSError as e: print >> sys.stderr, 'Failed to run: %s' % ' '.join(command) print >> sys.stderr, str(e) return 1
def CMDbatcharchive(parser, args): """Archives multiple isolated trees at once. Using single command instead of multiple sequential invocations allows to cut redundant work when isolated trees share common files (e.g. file hashes are checked only once, their presence on the server is checked only once, and so on). Takes a list of paths to *.isolated.gen.json files that describe what trees to isolate. Format of files is: { "version": 1, "dir": <absolute path to a directory all other paths are relative to>, "args": [list of command line arguments for single 'archive' command] } """ isolateserver.add_isolate_server_options(parser) isolateserver.add_archive_options(parser) auth.add_auth_options(parser) parser.add_option( '--dump-json', metavar='FILE', help='Write isolated hashes of archived trees to this file as JSON') options, args = parser.parse_args(args) auth.process_auth_options(parser, options) isolateserver.process_isolate_server_options(parser, options, True) # Validate all incoming options, prepare what needs to be archived as a list # of tuples (archival options, working directory). work_units = [] for gen_json_path in args: # Validate JSON format of a *.isolated.gen.json file. try: data = tools.read_json(gen_json_path) except IOError as e: parser.error('Failed to open %s: %s' % (gen_json_path, e)) if data.get('version') != ISOLATED_GEN_JSON_VERSION: parser.error('Invalid version in %s' % gen_json_path) cwd = data.get('dir') if not isinstance(cwd, unicode) or not fs.isdir(cwd): parser.error('Invalid dir in %s' % gen_json_path) args = data.get('args') if (not isinstance(args, list) or not all(isinstance(x, unicode) for x in args)): parser.error('Invalid args in %s' % gen_json_path) # Convert command line (embedded in JSON) to Options object. work_units.append((parse_archive_command_line(args, cwd), cwd)) # Perform the archival, all at once. isolated_hashes = isolate_and_archive( work_units, options.isolate_server, options.namespace) # TODO(vadimsh): isolate_and_archive returns None on upload failure, there's # no way currently to figure out what *.isolated file from a batch were # successfully uploaded, so consider them all failed (and emit empty dict # as JSON result). if options.dump_json: tools.write_json(options.dump_json, isolated_hashes or {}, False) if isolated_hashes is None: return EXIT_CODE_UPLOAD_ERROR # isolated_hashes[x] is None if 'x.isolate' contains a error. if not all(isolated_hashes.itervalues()): return EXIT_CODE_ISOLATE_ERROR return 0
def map_and_run(data, constant_run_path): """Runs a command with optional isolated input/output. Arguments: - data: TaskData instance. - constant_run_path: TODO Returns metadata about the result. """ result = { 'duration': None, 'exit_code': None, 'had_hard_timeout': False, 'internal_failure': 'run_isolated did not complete properly', 'stats': { # 'isolated': { # 'cipd': { # 'duration': 0., # 'get_client_duration': 0., # }, # 'download': { # 'duration': 0., # 'initial_number_items': 0, # 'initial_size': 0, # 'items_cold': '<large.pack()>', # 'items_hot': '<large.pack()>', # }, # 'upload': { # 'duration': 0., # 'items_cold': '<large.pack()>', # 'items_hot': '<large.pack()>', # }, # }, }, # 'cipd_pins': { # 'packages': [ # {'package_name': ..., 'version': ..., 'path': ...}, # ... # ], # 'client_package': {'package_name': ..., 'version': ...}, # }, 'outputs_ref': None, 'version': 5, } if data.root_dir: file_path.ensure_tree(data.root_dir, 0700) elif data.isolate_cache.cache_dir: data = data._replace( root_dir=os.path.dirname(data.isolate_cache.cache_dir)) # See comment for these constants. # If root_dir is not specified, it is not constant. # TODO(maruel): This is not obvious. Change this to become an error once we # make the constant_run_path an exposed flag. if constant_run_path and data.root_dir: run_dir = os.path.join(data.root_dir, ISOLATED_RUN_DIR) if os.path.isdir(run_dir): file_path.rmtree(run_dir) os.mkdir(run_dir, 0700) else: run_dir = make_temp_dir(ISOLATED_RUN_DIR, data.root_dir) # storage should be normally set but don't crash if it is not. This can happen # as Swarming task can run without an isolate server. out_dir = make_temp_dir( ISOLATED_OUT_DIR, data.root_dir) if data.storage else None tmp_dir = make_temp_dir(ISOLATED_TMP_DIR, data.root_dir) cwd = run_dir if data.relative_cwd: cwd = os.path.normpath(os.path.join(cwd, data.relative_cwd)) command = data.command try: with data.install_packages_fn(run_dir) as cipd_info: if cipd_info: result['stats']['cipd'] = cipd_info.stats result['cipd_pins'] = cipd_info.pins if data.isolated_hash: isolated_stats = result['stats'].setdefault('isolated', {}) bundle, isolated_stats['download'] = fetch_and_map( isolated_hash=data.isolated_hash, storage=data.storage, cache=data.isolate_cache, outdir=run_dir, use_symlinks=data.use_symlinks) change_tree_read_only(run_dir, bundle.read_only) # Inject the command if not command and bundle.command: command = bundle.command + data.extra_args # Only set the relative directory if the isolated file specified a # command, and no raw command was specified. if bundle.relative_cwd: cwd = os.path.normpath(os.path.join(cwd, bundle.relative_cwd)) if not command: # Handle this as a task failure, not an internal failure. sys.stderr.write( '<No command was specified!>\n' '<Please secify a command when triggering your Swarming task>\n') result['exit_code'] = 1 return result if not cwd.startswith(run_dir): # Handle this as a task failure, not an internal failure. This is a # 'last chance' way to gate against directory escape. sys.stderr.write('<Relative CWD is outside of run directory!>\n') result['exit_code'] = 1 return result if not os.path.isdir(cwd): # Accepts relative_cwd that does not exist. os.makedirs(cwd, 0700) # If we have an explicit list of files to return, make sure their # directories exist now. if data.storage and data.outputs: isolateserver.create_directories(run_dir, data.outputs) command = tools.fix_python_path(command) command = process_command(command, out_dir, data.bot_file) file_path.ensure_command_has_abs_path(command, cwd) with data.install_named_caches(run_dir): sys.stdout.flush() start = time.time() try: # Need to switch the default account before 'get_command_env' call, # so it can grab correct value of LUCI_CONTEXT env var. with set_luci_context_account(data.switch_to_account, tmp_dir): env = get_command_env( tmp_dir, cipd_info, run_dir, data.env, data.env_prefix) result['exit_code'], result['had_hard_timeout'] = run_command( command, cwd, env, data.hard_timeout, data.grace_period) finally: result['duration'] = max(time.time() - start, 0) # We successfully ran the command, set internal_failure back to # None (even if the command failed, it's not an internal error). result['internal_failure'] = None except Exception as e: # An internal error occurred. Report accordingly so the swarming task will # be retried automatically. logging.exception('internal failure: %s', e) result['internal_failure'] = str(e) on_error.report(None) # Clean up finally: try: # Try to link files to the output directory, if specified. if out_dir: link_outputs_to_outdir(run_dir, out_dir, data.outputs) success = False if data.leak_temp_dir: success = True logging.warning( 'Deliberately leaking %s for later examination', run_dir) else: # On Windows rmtree(run_dir) call above has a synchronization effect: it # finishes only when all task child processes terminate (since a running # process locks *.exe file). Examine out_dir only after that call # completes (since child processes may write to out_dir too and we need # to wait for them to finish). if fs.isdir(run_dir): try: success = file_path.rmtree(run_dir) except OSError as e: logging.error('Failure with %s', e) success = False if not success: sys.stderr.write(OUTLIVING_ZOMBIE_MSG % ('run', data.grace_period)) if result['exit_code'] == 0: result['exit_code'] = 1 if fs.isdir(tmp_dir): try: success = file_path.rmtree(tmp_dir) except OSError as e: logging.error('Failure with %s', e) success = False if not success: sys.stderr.write(OUTLIVING_ZOMBIE_MSG % ('temp', data.grace_period)) if result['exit_code'] == 0: result['exit_code'] = 1 # This deletes out_dir if leak_temp_dir is not set. if out_dir: isolated_stats = result['stats'].setdefault('isolated', {}) result['outputs_ref'], success, isolated_stats['upload'] = ( delete_and_upload(data.storage, out_dir, data.leak_temp_dir)) if not success and result['exit_code'] == 0: result['exit_code'] = 1 except Exception as e: # Swallow any exception in the main finally clause. if out_dir: logging.exception('Leaking out_dir %s: %s', out_dir, e) result['internal_failure'] = str(e) return result
def expand_directory_and_symlink(indir, relfile, blacklist, follow_symlinks): """Expands a single input. It can result in multiple outputs. This function is recursive when relfile is a directory. Note: this code doesn't properly handle recursive symlink like one created with: ln -s .. foo Yields: tuple(Relative path, bool is_symlink) to files and symlinks inside |indir|. """ if os.path.isabs(relfile): raise MappingError(u'Can\'t map absolute path %s' % relfile) infile = file_path.normpath(os.path.join(indir, relfile)) if not infile.startswith(indir): raise MappingError(u'Can\'t map file %s outside %s' % (infile, indir)) filepath = os.path.join(indir, relfile) native_filepath = file_path.get_native_path_case(filepath) if filepath != native_filepath: # Special case './'. if filepath != native_filepath + u'.' + os.path.sep: # While it'd be nice to enforce path casing on Windows, it's impractical. # Also give up enforcing strict path case on OSX. Really, it's that sad. # The case where it happens is very specific and hard to reproduce: # get_native_path_case( # u'Foo.framework/Versions/A/Resources/Something.nib') will return # u'Foo.framework/Versions/A/resources/Something.nib', e.g. lowercase 'r'. # # Note that this is really something deep in OSX because running # ls Foo.framework/Versions/A # will print out 'Resources', while file_path.get_native_path_case() # returns a lower case 'r'. # # So *something* is happening under the hood resulting in the command 'ls' # and Carbon.File.FSPathMakeRef('path').FSRefMakePath() to disagree. We # have no idea why. if sys.platform not in ('darwin', 'win32'): raise MappingError( u'File path doesn\'t equal native file path\n%s != %s' % (filepath, native_filepath)) symlinks = [] if follow_symlinks: try: relfile, symlinks = _expand_symlinks(indir, relfile) except OSError: # The file doesn't exist, it will throw below. pass # The symlinks need to be mapped in. for s in symlinks: yield s, True if relfile.endswith(os.path.sep): if not fs.isdir(infile): raise MappingError(u'%s is not a directory but ends with "%s"' % (infile, os.path.sep)) # Special case './'. if relfile.startswith(u'.' + os.path.sep): relfile = relfile[2:] try: for filename in fs.listdir(infile): inner_relfile = os.path.join(relfile, filename) if blacklist and blacklist(inner_relfile): continue if fs.isdir(os.path.join(indir, inner_relfile)): inner_relfile += os.path.sep # Apply recursively. for i, is_symlink in expand_directory_and_symlink( indir, inner_relfile, blacklist, follow_symlinks): yield i, is_symlink except OSError as e: raise MappingError(u'Unable to iterate over directory %s.\n%s' % (infile, e)) else: # Always add individual files even if they were blacklisted. if fs.isdir(infile): raise MappingError( u'Input directory %s must have a trailing slash' % infile) if not fs.isfile(infile): raise MappingError(u'Input file %s doesn\'t exist' % infile) yield relfile, False
def cleanup(self): """Cleans up the cache directory. Ensures there is no unknown files in cache_dir. Ensures the read-only bits are set correctly. At that point, the cache was already loaded, trimmed to respect cache policies. """ with self._lock: fs.chmod(self.cache_dir, 0o700) # Ensure that all files listed in the state still exist and add new ones. previous = set(self._lru) # It'd be faster if there were a readdir() function. for filename in fs.listdir(self.cache_dir): if filename == self.STATE_FILE: fs.chmod(os.path.join(self.cache_dir, filename), 0o600) continue if filename in previous: fs.chmod(os.path.join(self.cache_dir, filename), 0o400) previous.remove(filename) continue # An untracked file. Delete it. logging.warning('Removing unknown file %s from cache', filename) p = self._path(filename) if fs.isdir(p): try: file_path.rmtree(p) except OSError: pass else: file_path.try_remove(p) continue if previous: # Filter out entries that were not found. logging.warning('Removed %d lost files', len(previous)) for filename in previous: self._lru.pop(filename) self._save() # Verify hash of every single item to detect corruption. the corrupted # files will be evicted. with self._lock: for digest, (_, timestamp) in list(self._lru._items.items()): # verify only if the mtime is grather than the timestamp in state.json # to avoid take too long time. if self._get_mtime(digest) <= timestamp: continue logging.warning('Item has been modified. item: %s', digest) if self._is_valid_hash(digest): # Update timestamp in state.json self._lru.touch(digest) continue # remove corrupted file from LRU and file system self._lru.pop(digest) self._delete_file(digest, UNKNOWN_FILE_SIZE) logging.error('Deleted corrupted item: %s', digest) self._save()
def run_tha_test( isolated_hash, storage, cache, leak_temp_dir, result_json, root_dir, hard_timeout, grace_period, extra_args): """Downloads the dependencies in the cache, hardlinks them into a temporary directory and runs the executable from there. A temporary directory is created to hold the output files. The content inside this directory will be uploaded back to |storage| packaged as a .isolated file. Arguments: isolated_hash: the SHA-1 of the .isolated file that must be retrieved to recreate the tree of files to run the target executable. storage: an isolateserver.Storage object to retrieve remote objects. This object has a reference to an isolateserver.StorageApi, which does the actual I/O. cache: an isolateserver.LocalCache to keep from retrieving the same objects constantly by caching the objects retrieved. Can be on-disk or in-memory. leak_temp_dir: if true, the temporary directory will be deliberately leaked for later examination. result_json: file path to dump result metadata into. If set, the process exit code is always 0 unless an internal error occured. root_dir: directory to the path to use to create the temporary directory. If not specified, a random temporary directory is created. hard_timeout: kills the process if it lasts more than this amount of seconds. grace_period: number of seconds to wait between SIGTERM and SIGKILL. extra_args: optional arguments to add to the command stated in the .isolate file. Returns: Process exit code that should be used. """ # run_isolated exit code. Depends on if result_json is used or not. result = map_and_run( isolated_hash, storage, cache, leak_temp_dir, root_dir, hard_timeout, grace_period, extra_args) logging.info('Result:\n%s', tools.format_json(result, dense=True)) if result_json: # We've found tests to delete 'work' when quitting, causing an exception # here. Try to recreate the directory if necessary. work_dir = os.path.dirname(result_json) if not fs.isdir(work_dir): fs.mkdir(work_dir) tools.write_json(result_json, result, dense=True) # Only return 1 if there was an internal error. return int(bool(result['internal_failure'])) # Marshall into old-style inline output. if result['outputs_ref']: data = { 'hash': result['outputs_ref']['isolated'], 'namespace': result['outputs_ref']['namespace'], 'storage': result['outputs_ref']['isolatedserver'], } sys.stdout.flush() print( '[run_isolated_out_hack]%s[/run_isolated_out_hack]' % tools.format_json(data, dense=True)) sys.stdout.flush() return result['exit_code'] or int(bool(result['internal_failure']))
def CMDreproduce(parser, args): """Runs a task locally that was triggered on the server. This running locally the same commands that have been run on the bot. The data downloaded will be in a subdirectory named 'work' of the current working directory. You can pass further additional arguments to the target command by passing them after --. """ options, args = parser.parse_args(args) extra_args = [] if not args: parser.error('Must specify exactly one task id.') if len(args) > 1: if args[1] == '--': if len(args) > 2: extra_args = args[2:] else: extra_args = args[1:] url = options.swarming + '/_ah/api/swarming/v1/task/%s/request' % args[0] request = net.url_read_json(url) if not request: print >> sys.stderr, 'Failed to retrieve request data for the task' return 1 workdir = unicode(os.path.abspath('work')) if not fs.isdir(workdir): fs.mkdir(workdir) properties = request['properties'] env = None if properties.get('env'): env = os.environ.copy() logging.info('env: %r', properties['env']) for i in properties['env']: key = i['key'].encode('utf-8') if not i['value']: env.pop(key, None) else: env[key] = i['value'].encode('utf-8') if properties.get('inputs_ref'): # Create the tree. with isolateserver.get_storage( properties['inputs_ref']['isolatedserver'], properties['inputs_ref']['namespace']) as storage: bundle = isolateserver.fetch_isolated( properties['inputs_ref']['isolated'], storage, isolateserver.MemoryCache(file_mode_mask=0700), workdir, False) command = bundle.command if bundle.relative_cwd: workdir = os.path.join(workdir, bundle.relative_cwd) else: command = properties['command'] try: return subprocess.call(command + extra_args, env=env, cwd=workdir) except OSError as e: print >> sys.stderr, 'Failed to run: %s' % ' '.join(command) print >> sys.stderr, str(e) return 1
def ensure_tree(path, perm=0777): """Ensures a directory exists.""" if not fs.isdir(path): fs.makedirs(path, perm)
def map_and_run(command, isolated_hash, storage, isolate_cache, outputs, init_name_caches, leak_temp_dir, root_dir, hard_timeout, grace_period, bot_file, extra_args, install_packages_fn, use_symlinks): """Runs a command with optional isolated input/output. See run_tha_test for argument documentation. Returns metadata about the result. """ assert root_dir or root_dir is None assert bool(command) ^ bool(isolated_hash) result = { 'duration': None, 'exit_code': None, 'had_hard_timeout': False, 'internal_failure': None, 'stats': { # 'isolated': { # 'cipd': { # 'duration': 0., # 'get_client_duration': 0., # }, # 'download': { # 'duration': 0., # 'initial_number_items': 0, # 'initial_size': 0, # 'items_cold': '<large.pack()>', # 'items_hot': '<large.pack()>', # }, # 'upload': { # 'duration': 0., # 'items_cold': '<large.pack()>', # 'items_hot': '<large.pack()>', # }, # }, }, # 'cipd_pins': { # 'packages': [ # {'package_name': ..., 'version': ..., 'path': ...}, # ... # ], # 'client_package': {'package_name': ..., 'version': ...}, # }, 'outputs_ref': None, 'version': 5, } if root_dir: file_path.ensure_tree(root_dir, 0700) elif isolate_cache.cache_dir: root_dir = os.path.dirname(isolate_cache.cache_dir) # See comment for these constants. run_dir = make_temp_dir(ISOLATED_RUN_DIR, root_dir) # storage should be normally set but don't crash if it is not. This can happen # as Swarming task can run without an isolate server. out_dir = make_temp_dir(ISOLATED_OUT_DIR, root_dir) if storage else None tmp_dir = make_temp_dir(ISOLATED_TMP_DIR, root_dir) cwd = run_dir try: cipd_info = install_packages_fn(run_dir) if cipd_info: result['stats']['cipd'] = cipd_info['stats'] result['cipd_pins'] = cipd_info['cipd_pins'] if isolated_hash: isolated_stats = result['stats'].setdefault('isolated', {}) bundle, isolated_stats['download'] = fetch_and_map( isolated_hash=isolated_hash, storage=storage, cache=isolate_cache, outdir=run_dir, use_symlinks=use_symlinks) if not bundle.command: # Handle this as a task failure, not an internal failure. sys.stderr.write( '<The .isolated doesn\'t declare any command to run!>\n' '<Check your .isolate for missing \'command\' variable>\n') if os.environ.get('SWARMING_TASK_ID'): # Give an additional hint when running as a swarming task. sys.stderr.write('<This occurs at the \'isolate\' step>\n') result['exit_code'] = 1 return result change_tree_read_only(run_dir, bundle.read_only) cwd = os.path.normpath(os.path.join(cwd, bundle.relative_cwd)) command = bundle.command + extra_args # If we have an explicit list of files to return, make sure their # directories exist now. if storage and outputs: isolateserver.create_directories(run_dir, outputs) command = tools.fix_python_path(command) command = process_command(command, out_dir, bot_file) file_path.ensure_command_has_abs_path(command, cwd) init_name_caches(run_dir) sys.stdout.flush() start = time.time() try: result['exit_code'], result['had_hard_timeout'] = run_command( command, cwd, tmp_dir, hard_timeout, grace_period) finally: result['duration'] = max(time.time() - start, 0) except Exception as e: # An internal error occurred. Report accordingly so the swarming task will # be retried automatically. logging.exception('internal failure: %s', e) result['internal_failure'] = str(e) on_error.report(None) # Clean up finally: try: # Try to link files to the output directory, if specified. if out_dir: link_outputs_to_outdir(run_dir, out_dir, outputs) success = False if leak_temp_dir: success = True logging.warning( 'Deliberately leaking %s for later examination', run_dir) else: # On Windows rmtree(run_dir) call above has a synchronization effect: it # finishes only when all task child processes terminate (since a running # process locks *.exe file). Examine out_dir only after that call # completes (since child processes may write to out_dir too and we need # to wait for them to finish). if fs.isdir(run_dir): try: success = file_path.rmtree(run_dir) except OSError as e: logging.error('Failure with %s', e) success = False if not success: print >> sys.stderr, ( 'Failed to delete the run directory, forcibly failing\n' 'the task because of it. No zombie process can outlive a\n' 'successful task run and still be marked as successful.\n' 'Fix your stuff.') if result['exit_code'] == 0: result['exit_code'] = 1 if fs.isdir(tmp_dir): try: success = file_path.rmtree(tmp_dir) except OSError as e: logging.error('Failure with %s', e) success = False if not success: print >> sys.stderr, ( 'Failed to delete the temporary directory, forcibly failing\n' 'the task because of it. No zombie process can outlive a\n' 'successful task run and still be marked as successful.\n' 'Fix your stuff.') if result['exit_code'] == 0: result['exit_code'] = 1 # This deletes out_dir if leak_temp_dir is not set. if out_dir: isolated_stats = result['stats'].setdefault('isolated', {}) result['outputs_ref'], success, isolated_stats['upload'] = ( delete_and_upload(storage, out_dir, leak_temp_dir)) if not success and result['exit_code'] == 0: result['exit_code'] = 1 except Exception as e: # Swallow any exception in the main finally clause. if out_dir: logging.exception('Leaking out_dir %s: %s', out_dir, e) result['internal_failure'] = str(e) return result