def load(cls, data, isolated_basedir): # pylint: disable=W0221 """Special case loading to disallow different OS. It is not possible to load a .isolated.state files from a different OS, this file is saved in OS-specific format. """ out = super(SavedState, cls).load(data, isolated_basedir) if data.get('OS') != sys.platform: raise isolated_format.IsolatedError('Unexpected OS %s', data.get('OS')) # Converts human readable form back into the proper class type. algo = data.get('algo') if not algo in isolated_format.SUPPORTED_ALGOS: raise isolated_format.IsolatedError('Unknown algo \'%s\'' % out.algo) out.algo = isolated_format.SUPPORTED_ALGOS[algo] # Refuse the load non-exact version, even minor difference. This is unlike # isolateserver.load_isolated(). This is because .isolated.state could have # changed significantly even in minor version difference. if out.version != cls.EXPECTED_VERSION: raise isolated_format.IsolatedError( 'Unsupported version \'%s\'' % out.version) # The .isolate file must be valid. If it is not present anymore, zap the # value as if it was not noted, so .isolate_file can safely be overriden # later. if out.isolate_file and not fs.isfile(out.isolate_filepath): out.isolate_file = None if out.isolate_file: # It could be absolute on Windows if the drive containing the .isolate and # the drive containing the .isolated files differ, .e.g .isolate is on # C:\\ and .isolated is on D:\\ . assert not os.path.isabs(out.isolate_file) or sys.platform == 'win32' assert fs.isfile(out.isolate_filepath), out.isolate_filepath return out
def link_file(outfile, infile, action): """Links a file. The type of link depends on |action|. Returns: True if the action was caried on, False if fallback was used. """ if action not in (HARDLINK, HARDLINK_WITH_FALLBACK, SYMLINK, COPY): raise ValueError('Unknown mapping action %s' % action) if not fs.isfile(infile): raise OSError('%s is missing' % infile) if fs.isfile(outfile): raise OSError( '%s already exist; insize:%d; outsize:%d' % (outfile, fs.stat(infile).st_size, fs.stat(outfile).st_size)) if action == COPY: readable_copy(outfile, infile) elif action == SYMLINK and sys.platform != 'win32': # On windows, symlink are converted to hardlink and fails over to copy. fs.symlink(infile, outfile) # pylint: disable=E1101 else: # HARDLINK or HARDLINK_WITH_FALLBACK. try: hardlink(infile, outfile) except OSError as e: if action == HARDLINK: raise OSError('Failed to hardlink %s to %s: %s' % (infile, outfile, e)) # Probably a different file system. logging.warning( 'Failed to hardlink, failing back to copy %s to %s' % ( infile, outfile)) readable_copy(outfile, infile) # Signal caller that fallback copy was used. return False return True
def test_main_naked_with_cipd_client_no_packages(self): cipd_cache = os.path.join(self.tempdir, 'cipd_cache') cmd = [ '--no-log', '--cache', os.path.join(self.tempdir, 'isolated_cache'), '--cipd-enabled', '--cipd-client-version', 'git:wowza', '--cipd-server', self.cipd_server.url, '--cipd-cache', cipd_cache, '--named-cache-root', os.path.join(self.tempdir, 'named_cache'), '--raw-cmd', '--relative-cwd', 'a', '--', 'bin/echo${EXECUTABLE_SUFFIX}', 'hello', 'world', ] self.capture_popen_env = True ret = run_isolated.main(cmd) self.assertEqual(0, ret) # The CIPD client was bootstrapped and hardlinked (or copied on Win). client_binary_file = unicode( os.path.join(cipd_cache, 'clients', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) self.assertTrue(fs.isfile(client_binary_file)) client_binary_link = unicode( os.path.join(cipd_cache, 'bin', 'cipd' + cipd.EXECUTABLE_SUFFIX)) self.assertTrue(fs.isfile(client_binary_link)) # 'cipd ensure' was NOT called (only 'echo hello world' was). env = self.popen_calls[0][1].pop('env') self.assertEqual([ ([self.ir_dir(u'a', 'bin', 'echo'), u'hello', u'world'], { 'cwd': self.ir_dir('a'), 'detached': True }), ], self.popen_calls) # Directory with cipd client is in front of PATH. path = env['PATH'].split(os.pathsep) self.assertEqual(os.path.join(cipd_cache, 'bin'), path[0]) # CIPD_CACHE_DIR is set. self.assertEqual(os.path.join(cipd_cache, 'cache'), env['CIPD_CACHE_DIR'])
def link_file(outfile, infile, action): """Links a file. The type of link depends on |action|. Returns: True if the action was carried on, False if fallback was used. """ if action < 1 or action > COPY: raise ValueError('Unknown mapping action %s' % action) # TODO(maruel): Skip these checks. if not fs.isfile(infile): raise OSError('%s is missing' % infile) if fs.isfile(outfile): raise OSError( '%s already exist; insize:%d; outsize:%d' % (outfile, fs.stat(infile).st_size, fs.stat(outfile).st_size)) if action == COPY: readable_copy(outfile, infile) return True if action in (SYMLINK, SYMLINK_WITH_FALLBACK): try: fs.symlink(infile, outfile) # pylint: disable=E1101 return True except OSError: if action == SYMLINK: raise logging.warning( 'Failed to symlink, falling back to copy %s to %s' % ( infile, outfile)) # Signal caller that fallback copy was used. readable_copy(outfile, infile) return False # HARDLINK or HARDLINK_WITH_FALLBACK. try: hardlink(infile, outfile) return True except OSError as e: if action == HARDLINK: raise OSError('Failed to hardlink %s to %s: %s' % (infile, outfile, e)) # Probably a different file system. logging.warning( 'Failed to hardlink, falling back to copy %s to %s' % ( infile, outfile)) readable_copy(outfile, infile) # Signal caller that fallback copy was used. return False
def _load(self, trim, time_fn): """Loads state of the cache from json file. If cache_dir does not exist on disk, it is created. """ self._lock.assert_locked() if not fs.isfile(self.state_file): if not fs.isdir(self.cache_dir): fs.makedirs(self.cache_dir) else: # Load state of the cache. try: self._lru = lru.LRUDict.load(self.state_file) except ValueError as err: logging.error('Failed to load cache state: %s' % (err, )) # Don't want to keep broken state file. file_path.try_remove(self.state_file) if time_fn: self._lru.time_fn = time_fn if trim: self._trim() # We want the initial cache size after trimming, i.e. what is readily # avaiable. self._initial_number_items = len(self._lru) self._initial_size = sum(self._lru.itervalues()) if self._evicted: logging.info('Trimming evicted items with the following sizes: %s', sorted(self._evicted))
def __init__(self, cache_dir, policies, time_fn=None): """Initializes NamedCaches. Arguments: - cache_dir is a directory for persistent cache storage. - policies is a CachePolicies instance. - time_fn is a function that returns timestamp (float) and used to take timestamps when new caches are requested. Used in unit tests. """ super(NamedCache, self).__init__(cache_dir) self._policies = policies # LRU {cache_name -> tuple(cache_location, size)} self.state_file = os.path.join(cache_dir, self.STATE_FILE) self._lru = lru.LRUDict() if not fs.isdir(self.cache_dir): fs.makedirs(self.cache_dir) elif fs.isfile(self.state_file): try: self._lru = lru.LRUDict.load(self.state_file) except ValueError: logging.exception( 'NamedCache: failed to load named cache state file; obliterating' ) file_path.rmtree(self.cache_dir) with self._lock: self._try_upgrade() if time_fn: self._lru.time_fn = time_fn
def _save(self): """Saves the LRU ordering.""" self._lock.assert_locked() if sys.platform != 'win32': d = os.path.dirname(self.state_file) if fs.isdir(d): # Necessary otherwise the file can't be created. file_path.set_read_only(d, False) if fs.isfile(self.state_file): file_path.set_read_only(self.state_file, False) self._lru.save(self.state_file)
def is_valid_file(path, size): """Returns if the given files appears valid. Currently it just checks the file exists and its size matches the expectation. """ if size == UNKNOWN_FILE_SIZE: return fs.isfile(path) try: actual_size = fs.stat(path).st_size except OSError as e: logging.warning('Can\'t read item %s, assuming it\'s invalid: %s', os.path.basename(path), e) return False if size != actual_size: logging.warning('Found invalid item %s; %d != %d', os.path.basename(path), actual_size, size) return False return True
def test_rmtree_win(self): # Mock our sleep for faster test case execution. sleeps = [] self.mock(time, 'sleep', sleeps.append) self.mock(sys, 'stderr', StringIO.StringIO()) # Open a child process, so the file is locked. subdir = os.path.join(self.tempdir, 'to_be_deleted') fs.mkdir(subdir) script = 'import time; open(\'a\', \'w\'); time.sleep(60)' proc = subprocess.Popen([sys.executable, '-c', script], cwd=subdir) try: # Wait until the file exist. while not fs.isfile(os.path.join(subdir, 'a')): self.assertEqual(None, proc.poll()) file_path.rmtree(subdir) self.assertEqual([2, 4, 2], sleeps) # sys.stderr.getvalue() would return a fair amount of output but it is # not completely deterministic so we're not testing it here. finally: proc.wait()
def _load(self, trim, time_fn): """Loads state of the cache from json file. If cache_dir does not exist on disk, it is created. """ self._lock.assert_locked() if not fs.isfile(self.state_file): if not fs.isdir(self.cache_dir): fs.makedirs(self.cache_dir) else: # Load state of the cache. try: self._lru = lru.LRUDict.load(self.state_file) except ValueError as err: logging.error('Failed to load cache state: %s' % (err, )) # Don't want to keep broken state file. file_path.try_remove(self.state_file) if time_fn: self._lru.time_fn = time_fn if trim: self._trim()
def copy_recursively(src, dst): """Efficiently copies a file or directory from src_dir to dst_dir. `item` may be a file, directory, or a symlink to a file or directory. All symlinks are replaced with their targets, so the resulting directory structure in dst_dir will never have any symlinks. To increase speed, copy_recursively hardlinks individual files into the (newly created) directory structure if possible, unlike Python's shutil.copytree(). """ orig_src = src try: # Replace symlinks with their final target. while fs.islink(src): res = fs.readlink(src) src = os.path.join(os.path.dirname(src), res) # TODO(sadafm): Explicitly handle cyclic symlinks. # Note that fs.isfile (which is a wrapper around os.path.isfile) throws # an exception if src does not exist. A warning will be logged in that case. if fs.isfile(src): file_path.link_file(dst, src, file_path.HARDLINK_WITH_FALLBACK) return if not fs.exists(dst): os.makedirs(dst) for child in fs.listdir(src): copy_recursively(os.path.join(src, child), os.path.join(dst, child)) except OSError as e: if e.errno == errno.ENOENT: logging.warning('Path %s does not exist or %s is a broken symlink', src, orig_src) else: logging.info("Couldn't collect output file %s: %s", src, e)
def get_client(service_url, package_template, version, cache_dir, timeout=None): """Returns a context manager that yields a CipdClient. A blocking call. Upon exit from the context manager, the client binary may be deleted (if the internal cache is full). Args: service_url (str): URL of the CIPD backend. package_template (str): package name template of the CIPD client. version (str): version of CIPD client package. cache_dir: directory to store instance cache, version cache and a hardlink to the client binary. timeout (int): if not None, timeout in seconds for this function. Yields: CipdClient. Raises: Error if CIPD client version cannot be resolved or client cannot be fetched. """ timeoutfn = tools.sliding_timeout(timeout) # Package names are always lower case. # TODO(maruel): Assert instead? package_name = package_template.lower().replace('${platform}', get_platform()) # Resolve version to instance id. # Is it an instance id already? They look like HEX SHA1. if isolated_format.is_valid_hash(version, hashlib.sha1): instance_id = version elif ':' in version: # it's an immutable tag, cache the resolved version # version_cache is {hash(package_name, tag) -> instance id} mapping. # It does not take a lot of disk space. version_cache = isolateserver.DiskCache( unicode(os.path.join(cache_dir, 'versions')), isolateserver.CachePolicies(0, 0, 300), hashlib.sha1, trim=True) with version_cache: version_cache.cleanup() # Convert (package_name, version) to a string that may be used as a # filename in disk cache by hashing it. version_digest = hashlib.sha1('%s\n%s' % (package_name, version)).hexdigest() try: with version_cache.getfileobj(version_digest) as f: instance_id = f.read() except isolateserver.CacheMiss: instance_id = resolve_version(service_url, package_name, version, timeout=timeoutfn()) version_cache.write(version_digest, instance_id) else: # it's a ref, hit the backend instance_id = resolve_version(service_url, package_name, version, timeout=timeoutfn()) # instance_cache is {instance_id -> client binary} mapping. # It is bounded by 5 client versions. instance_cache = isolateserver.DiskCache( unicode(os.path.join(cache_dir, 'clients')), isolateserver.CachePolicies(0, 0, 5), hashlib.sha1, trim=True) with instance_cache: instance_cache.cleanup() if instance_id not in instance_cache: logging.info('Fetching CIPD client %s:%s', package_name, instance_id) fetch_url = get_client_fetch_url(service_url, package_name, instance_id, timeout=timeoutfn()) _fetch_cipd_client(instance_cache, instance_id, fetch_url, timeoutfn) # A single host can run multiple swarming bots, but ATM they do not share # same root bot directory. Thus, it is safe to use the same name for the # binary. cipd_bin_dir = unicode(os.path.join(cache_dir, 'bin')) binary_path = os.path.join(cipd_bin_dir, 'cipd' + EXECUTABLE_SUFFIX) if fs.isfile(binary_path): file_path.remove(binary_path) else: file_path.ensure_tree(cipd_bin_dir) with instance_cache.getfileobj(instance_id) as f: isolateserver.putfile(f, binary_path, 0511) # -r-x--x--x _ensure_batfile(binary_path) yield CipdClient(binary_path, package_name=package_name, instance_id=instance_id, service_url=service_url)
def expand_directory_and_symlink(indir, relfile, blacklist, follow_symlinks): """Expands a single input. It can result in multiple outputs. This function is recursive when relfile is a directory. Note: this code doesn't properly handle recursive symlink like one created with: ln -s .. foo Yields: tuple(Relative path, bool is_symlink) to files and symlinks inside |indir|. """ if os.path.isabs(relfile): raise MappingError(u'Can\'t map absolute path %s' % relfile) infile = file_path.normpath(os.path.join(indir, relfile)) if not infile.startswith(indir): raise MappingError(u'Can\'t map file %s outside %s' % (infile, indir)) filepath = os.path.join(indir, relfile) native_filepath = file_path.get_native_path_case(filepath) if filepath != native_filepath: # Special case './'. if filepath != native_filepath + u'.' + os.path.sep: # While it'd be nice to enforce path casing on Windows, it's impractical. # Also give up enforcing strict path case on OSX. Really, it's that sad. # The case where it happens is very specific and hard to reproduce: # get_native_path_case( # u'Foo.framework/Versions/A/Resources/Something.nib') will return # u'Foo.framework/Versions/A/resources/Something.nib', e.g. lowercase 'r'. # # Note that this is really something deep in OSX because running # ls Foo.framework/Versions/A # will print out 'Resources', while file_path.get_native_path_case() # returns a lower case 'r'. # # So *something* is happening under the hood resulting in the command 'ls' # and Carbon.File.FSPathMakeRef('path').FSRefMakePath() to disagree. We # have no idea why. if sys.platform not in ('darwin', 'win32'): raise MappingError( u'File path doesn\'t equal native file path\n%s != %s' % (filepath, native_filepath)) symlinks = [] if follow_symlinks: try: relfile, symlinks = _expand_symlinks(indir, relfile) except OSError: # The file doesn't exist, it will throw below. pass # The symlinks need to be mapped in. for s in symlinks: yield s, True if relfile.endswith(os.path.sep): if not fs.isdir(infile): raise MappingError(u'%s is not a directory but ends with "%s"' % (infile, os.path.sep)) # Special case './'. if relfile.startswith(u'.' + os.path.sep): relfile = relfile[2:] try: for filename in fs.listdir(infile): inner_relfile = os.path.join(relfile, filename) if blacklist and blacklist(inner_relfile): continue if fs.isdir(os.path.join(indir, inner_relfile)): inner_relfile += os.path.sep # Apply recursively. for i, is_symlink in expand_directory_and_symlink( indir, inner_relfile, blacklist, follow_symlinks): yield i, is_symlink except OSError as e: raise MappingError(u'Unable to iterate over directory %s.\n%s' % (infile, e)) else: # Always add individual files even if they were blacklisted. if fs.isdir(infile): raise MappingError( u'Input directory %s must have a trailing slash' % infile) if not fs.isfile(infile): raise MappingError(u'Input file %s doesn\'t exist' % infile) yield relfile, False
def get_client(service_url, package_template, version, cache_dir, timeout=None): """Returns a context manager that yields a CipdClient. A blocking call. Upon exit from the context manager, the client binary may be deleted (if the internal cache is full). Args: service_url (str): URL of the CIPD backend. package_template (str): package name template of the CIPD client. version (str): version of CIPD client package. cache_dir: directory to store instance cache, version cache and a hardlink to the client binary. timeout (int): if not None, timeout in seconds for this function. Yields: CipdClient. Raises: Error if CIPD client version cannot be resolved or client cannot be fetched. """ timeoutfn = tools.sliding_timeout(timeout) # Package names are always lower case. # TODO(maruel): Assert instead? package_name = package_template.lower().replace('${platform}', get_platform()) # Resolve version to instance id. # Is it an instance id already? They look like HEX SHA1. if isolated_format.is_valid_hash(version, hashlib.sha1): instance_id = version elif ':' in version: # it's an immutable tag, cache the resolved version # version_cache is {hash(package_name, tag) -> instance id} mapping. # It does not take a lot of disk space. version_cache = local_caching.DiskContentAddressedCache( six.text_type(os.path.join(cache_dir, 'versions')), local_caching.CachePolicies( # 1GiB. max_cache_size=1024 * 1024 * 1024, min_free_space=0, max_items=300, # 3 weeks. max_age_secs=21 * 24 * 60 * 60), trim=True) # Convert (package_name, version) to a string that may be used as a # filename in disk cache by hashing it. version_digest = hashlib.sha1('%s\n%s' % (package_name, version)).hexdigest() try: with version_cache.getfileobj(version_digest) as f: instance_id = f.read() except local_caching.CacheMiss: instance_id = resolve_version(service_url, package_name, version, timeout=timeoutfn()) version_cache.write(version_digest, instance_id) version_cache.trim() else: # it's a ref, hit the backend instance_id = resolve_version(service_url, package_name, version, timeout=timeoutfn()) # instance_cache is {instance_id -> client binary} mapping. # It is bounded by 5 client versions. instance_cache = local_caching.DiskContentAddressedCache( six.text_type(os.path.join(cache_dir, 'clients')), local_caching.CachePolicies( # 1GiB. max_cache_size=1024 * 1024 * 1024, min_free_space=0, max_items=10, # 3 weeks. max_age_secs=21 * 24 * 60 * 60), trim=True) if instance_id not in instance_cache: logging.info('Fetching CIPD client %s:%s', package_name, instance_id) fetch_url = get_client_fetch_url(service_url, package_name, instance_id, timeout=timeoutfn()) _fetch_cipd_client(instance_cache, instance_id, fetch_url, timeoutfn) # A single host can run multiple swarming bots, but they cannot share same # root bot directory. Thus, it is safe to use the same name for the binary. cipd_bin_dir = six.text_type(os.path.join(cache_dir, 'bin')) binary_path = os.path.join(cipd_bin_dir, 'cipd' + EXECUTABLE_SUFFIX) if fs.isfile(binary_path): # TODO(maruel): Do not unconditionally remove the binary. try: file_path.remove(binary_path) except WindowsError: # pylint: disable=undefined-variable # See whether cipd.exe is running for crbug.com/1028781 ret = subprocess42.call(['tasklist.exe']) if ret: logging.error('tasklist returns non-zero: %d', ret) raise else: file_path.ensure_tree(cipd_bin_dir) with instance_cache.getfileobj(instance_id) as f: isolateserver.putfile(f, binary_path, 0o511) # -r-x--x--x _ensure_batfile(binary_path) yield CipdClient(binary_path, package_name=package_name, instance_id=instance_id, service_url=service_url) instance_cache.trim()
def test_main_naked_with_packages(self): self.mock(cipd, 'get_platform', lambda: 'linux-amd64') pins = { '': [ ('infra/data/x', 'badc0fee' * 5), ('infra/data/y', 'cafebabe' * 5), ], 'bin': [ ('infra/tools/echo/linux-amd64', 'deadbeef' * 5), ], } def fake_ensure(args, **_kwargs): if (args[0].endswith('/cipd') and args[1] == 'ensure' and '-json-output' in args): idx = args.index('-json-output') with open(args[idx + 1], 'w') as json_out: json.dump( { 'result': { subdir: [{ 'package': pkg, 'instance_id': ver } for pkg, ver in packages] for subdir, packages in pins.iteritems() } }, json_out) return 0 self.popen_mocks.append(fake_ensure) cipd_cache = os.path.join(self.tempdir, 'cipd_cache') cmd = [ '--no-log', '--cache', os.path.join(self.tempdir, 'isolated_cache'), '--cipd-client-version', 'git:wowza', '--cipd-package', 'bin:infra/tools/echo/${platform}:latest', '--cipd-package', '.:infra/data/x:latest', '--cipd-package', '.:infra/data/y:canary', '--cipd-server', self.cipd_server.url, '--cipd-cache', cipd_cache, '--named-cache-root', os.path.join(self.tempdir, 'named_cache'), '--raw-cmd', '--', 'bin/echo${EXECUTABLE_SUFFIX}', 'hello', 'world', ] ret = run_isolated.main(cmd) self.assertEqual(0, ret) self.assertEqual(2, len(self.popen_calls)) # Test cipd-ensure command for installing packages. cipd_ensure_cmd, _ = self.popen_calls[0] self.assertEqual(cipd_ensure_cmd[:2], [ os.path.join(cipd_cache, 'bin', 'cipd' + cipd.EXECUTABLE_SUFFIX), 'ensure', ]) cache_dir_index = cipd_ensure_cmd.index('-cache-dir') self.assertEqual(cipd_ensure_cmd[cache_dir_index + 1], os.path.join(cipd_cache, 'cache')) # Test cipd client cache. `git:wowza` was a tag and so is cacheable. self.assertEqual(len(os.listdir(os.path.join(cipd_cache, 'versions'))), 2) version_file = unicode( os.path.join(cipd_cache, 'versions', '765a0de4c618f91faf923cb68a47bb564aed412d')) self.assertTrue(fs.isfile(version_file)) with open(version_file) as f: self.assertEqual(f.read(), 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') client_binary_file = unicode( os.path.join(cipd_cache, 'clients', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) self.assertTrue(fs.isfile(client_binary_file)) # Test echo call. echo_cmd, _ = self.popen_calls[1] self.assertTrue( echo_cmd[0].endswith(os.path.sep + 'bin' + os.path.sep + 'echo' + cipd.EXECUTABLE_SUFFIX), echo_cmd[0]) self.assertEqual(echo_cmd[1:], [u'hello', u'world'])
def test_main_naked_with_packages(self): pin_idx_ref = [0] pins = [ [ ('infra/data/x', 'badc0fee'*5), ('infra/data/y', 'cafebabe'*5), ], [ ('infra/tools/echo/linux-amd64', 'deadbeef'*5), ], ] def fake_ensure(args, **_kwargs): if (args[0].endswith('/cipd') and args[1] == 'ensure' and '-json-output' in args): idx = args.index('-json-output') with open(args[idx+1], 'w') as json_out: json.dump({ 'result': [ {'package': pkg, 'instance_id': ver} for pkg, ver in pins[pin_idx_ref[0]] ], }, json_out) pin_idx_ref[0] += 1 return 0 self.popen_mocks.append(fake_ensure) cipd_cache = os.path.join(self.tempdir, 'cipd_cache') cmd = [ '--no-log', '--cache', os.path.join(self.tempdir, 'cache'), '--cipd-client-version', 'git:wowza', '--cipd-package', 'bin:infra/tools/echo/${platform}:latest', '--cipd-package', '.:infra/data/x:latest', '--cipd-package', '.:infra/data/y:canary', '--cipd-server', self.cipd_server.url, '--cipd-cache', cipd_cache, '--named-cache-root', os.path.join(self.tempdir, 'c'), 'bin/echo${EXECUTABLE_SUFFIX}', 'hello', 'world', ] ret = run_isolated.main(cmd) self.assertEqual(0, ret) self.assertEqual(3, len(self.popen_calls)) # Test cipd-ensure command for installing packages. for cipd_ensure_cmd, _ in self.popen_calls[0:2]: self.assertEqual(cipd_ensure_cmd[:2], [ os.path.join(cipd_cache, 'cipd' + cipd.EXECUTABLE_SUFFIX), 'ensure', ]) cache_dir_index = cipd_ensure_cmd.index('-cache-dir') self.assertEqual( cipd_ensure_cmd[cache_dir_index+1], os.path.join(cipd_cache, 'cipd_internal')) # Test cipd client cache. `git:wowza` was a tag and so is cacheable. self.assertEqual(len(os.listdir(os.path.join(cipd_cache, 'versions'))), 2) version_file = unicode(os.path.join( cipd_cache, 'versions', '633d2aa4119cc66803f1600f9c4d85ce0e0581b5')) self.assertTrue(fs.isfile(version_file)) with open(version_file) as f: self.assertEqual(f.read(), 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') client_binary_file = unicode(os.path.join( cipd_cache, 'clients', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) self.assertTrue(fs.isfile(client_binary_file)) # Test echo call. echo_cmd, _ = self.popen_calls[2] self.assertTrue(echo_cmd[0].endswith( os.path.sep + 'bin' + os.path.sep + 'echo' + cipd.EXECUTABLE_SUFFIX), echo_cmd[0]) self.assertEqual(echo_cmd[1:], ['hello', 'world'])