def run_push_and_fetch_test(self, namespace): storage = isolateserver.get_storage(self.server.url, namespace) # Upload items. items = [isolateserver.BufferItem('item %d' % i) for i in xrange(10)] uploaded = storage.upload_items(items) self.assertEqual(set(items), set(uploaded)) # Fetch them all back into local memory cache. cache = isolateserver.MemoryCache() queue = isolateserver.FetchQueue(storage, cache) # Start fetching. pending = set() for item in items: pending.add(item.digest) queue.add(item.digest) # Wait for fetch to complete. while pending: fetched = queue.wait(pending) pending.discard(fetched) # Ensure fetched same data as was pushed. self.assertEqual([i.buffer for i in items], [cache.read(i.digest) for i in items])
def _run_tha_test(self, isolated_hash=None, files=None, command=None): files = files or {} make_tree_call = [] def add(i, _): make_tree_call.append(i) for i in ('make_tree_read_only', 'make_tree_files_read_only', 'make_tree_deleteable', 'make_tree_writeable'): self.mock(file_path, i, functools.partial(add, i)) ret = run_isolated.run_tha_test( command or [], isolated_hash, StorageFake(files), isolateserver.MemoryCache(), None, init_named_caches_stub, False, None, None, None, None, None, run_isolated.noop_install_packages, False) self.assertEqual(0, ret) return make_tree_call
def _run_tha_test(self, isolated_hash=None, files=None, command=None): files = files or {} make_tree_call = [] def add(i, _): make_tree_call.append(i) for i in ('make_tree_read_only', 'make_tree_files_read_only', 'make_tree_deleteable', 'make_tree_writeable'): self.mock(file_path, i, functools.partial(add, i)) data = run_isolated.TaskData( command=command or [], relative_cwd=None, extra_args=[], isolated_hash=isolated_hash, storage=StorageFake(files), isolate_cache=isolateserver.MemoryCache(), outputs=None, install_named_caches=init_named_caches_stub, leak_temp_dir=False, root_dir=None, hard_timeout=60, grace_period=30, bot_file=None, switch_to_account=False, install_packages_fn=run_isolated.noop_install_packages, use_symlinks=False, env={}, env_prefix={}) ret = run_isolated.run_tha_test(data, None) self.assertEqual(0, ret) return make_tree_call
class RunIsolatedTestOutputFiles(RunIsolatedTestBase): def _run_test(self, isolated, command): # Starts a full isolate server mock and have run_tha_test() uploads results # back after the task completed. server = isolateserver_mock.MockIsolateServer() try: script = ('import sys\n' 'open(sys.argv[1], "w").write("bar")\n' 'open(sys.argv[2], "w").write("baz")\n') script_hash = isolateserver_mock.hash_content(script) isolated['files']['cmd.py'] = { 'h': script_hash, 'm': 0700, 's': len(script), } if sys.platform == 'win32': isolated['files']['cmd.py'].pop('m') isolated_data = json_dumps(isolated) isolated_hash = isolateserver_mock.hash_content(isolated_data) server.add_content('default-store', script) server.add_content('default-store', isolated_data) store = isolateserver.get_storage(server.url, 'default-store') self.mock(sys, 'stdout', StringIO.StringIO()) ret = run_isolated.run_tha_test(command, isolated_hash, store, isolateserver.MemoryCache(), ['foo', 'foodir/foo2'], init_named_caches_stub, False, None, None, None, None, None, run_isolated.noop_install_packages, False) self.assertEqual(0, ret) # It uploaded back. Assert the store has a new item containing foo. hashes = {isolated_hash, script_hash} foo_output_hash = isolateserver_mock.hash_content('bar') foo2_output_hash = isolateserver_mock.hash_content('baz') hashes.add(foo_output_hash) hashes.add(foo2_output_hash) isolated = { 'algo': 'sha-1', 'files': { 'foo': { 'h': foo_output_hash, # TODO(maruel): Handle umask. 'm': 0640, 's': 3, }, 'foodir/foo2': { 'h': foo2_output_hash, # TODO(maruel): Handle umask. 'm': 0640, 's': 3, }, }, 'version': isolated_format.ISOLATED_FILE_VERSION, }
def _run_tha_test(self, isolated_hash, files): make_tree_call = [] def add(i, _): make_tree_call.append(i) for i in ('make_tree_read_only', 'make_tree_files_read_only', 'make_tree_deleteable', 'make_tree_writeable'): self.mock(file_path, i, functools.partial(add, i)) ret = run_isolated.run_tha_test(isolated_hash, StorageFake(files), isolateserver.MemoryCache(), False, None, None, None, None, []) self.assertEqual(0, ret) return make_tree_call
def send_and_receive(random_pool, storage, progress, size): """Sends a random file and gets it back. # TODO(maruel): Add a batching argument of value [1, 500] to batch requests. Returns (delay, size) """ # Create a file out of the pool. start = time.time() batch = 1 items = [ isolateserver.BufferItem(random_pool.gen(size), False) for _ in xrange(batch) ] try: # len(_uploaded) may be < len(items) happen if the items is not random # enough or value of --mid-size is very low compared to --items. _uploaded = storage.upload_items(items) start = time.time() cache = isolateserver.MemoryCache() queue = isolateserver.FetchQueue(storage, cache) for i in items: queue.add(i.digest, i.size) waiting = [i.digest for i in items] while waiting: waiting.remove(queue.wait(waiting)) expected = {i.digest: ''.join(i.content()) for i in items} for d in cache.cached_set(): actual = cache.read(d) assert expected.pop(d) == actual assert not expected, expected duration = max(0, time.time() - start) except isolateserver.MappingError as e: duration = str(e) if isinstance(duration, float): progress.update_item('', index=1, data=size) else: progress.update_item('', index=1) return (duration, size)
def process_shard_result(self, result): """Stores results of a single task shard, fetches output files if necessary. Called concurrently from multiple threads. """ # We are going to put |shard_index| into a file path. Make sure it is int. shard_index = result['config_instance_index'] if not isinstance(shard_index, int): raise ValueError('Shard index should be an int: %r' % (shard_index, )) # Sanity check index is in expected range. if shard_index < 0 or shard_index >= self.shard_count: logging.warning( 'Shard index %d is outside of expected range: [0; %d]', shard_index, self.shard_count - 1) return # Store result dict of that shard, ignore results we've already seen. with self._lock: if shard_index in self._per_shard_results: logging.warning('Ignoring duplicate shard index %d', shard_index) return self._per_shard_results[shard_index] = result # Fetch output files if necessary. isolated_files_location = extract_output_files_location( result['output']) if isolated_files_location: isolate_server, namespace, isolated_hash = isolated_files_location storage = self._get_storage(isolate_server, namespace) if storage: # Output files are supposed to be small and they are not reused across # tasks. So use MemoryCache for them instead of on-disk cache. Make # files writable, so that calling script can delete them. isolateserver.fetch_isolated( isolated_hash, storage, isolateserver.MemoryCache(file_mode_mask=0700), os.path.join(self.task_output_dir, str(shard_index)), False)
class RunIsolatedTestOutputFiles(RunIsolatedTestBase): def _run_test(self, isolated, command): # Starts a full isolate server mock and have run_tha_test() uploads results # back after the task completed. server = isolateserver_mock.MockIsolateServer() try: # Output two files. If we're on Linux, we'll try to make one of them a # symlink to ensure that we correctly follow symlinks. Note that this only # tests file symlinks, not directory symlinks. # TODO(aludwin): follow directory symlinks script = ( 'import os\n' 'import sys\n' 'open(sys.argv[1], "w").write("bar")\n' 'if sys.platform.startswith("linux"):\n' ' realpath = os.path.abspath("contents_of_symlink")\n' ' open(realpath, "w").write("baz")\n' ' os.symlink(realpath, sys.argv[2])\n' 'else:\n' ' open(sys.argv[2], "w").write("baz")\n') script_hash = isolateserver_mock.hash_content(script) isolated['files']['cmd.py'] = { 'h': script_hash, 'm': 0700, 's': len(script), } if sys.platform == 'win32': isolated['files']['cmd.py'].pop('m') isolated_data = json_dumps(isolated) isolated_hash = isolateserver_mock.hash_content(isolated_data) server.add_content('default-store', script) server.add_content('default-store', isolated_data) store = isolateserver.get_storage(server.url, 'default-store') self.mock(sys, 'stdout', StringIO.StringIO()) ret = run_isolated.run_tha_test( command, isolated_hash, store, isolateserver.MemoryCache(), ['foo', 'foodir/foo2'], init_named_caches_stub, False, None, None, None, None, None, run_isolated.noop_install_packages, False) self.assertEqual(0, ret) # It uploaded back. Assert the store has a new item containing foo. hashes = {isolated_hash, script_hash} foo_output_hash = isolateserver_mock.hash_content('bar') foo2_output_hash = isolateserver_mock.hash_content('baz') hashes.add(foo_output_hash) hashes.add(foo2_output_hash) isolated = { 'algo': 'sha-1', 'files': { 'foo': { 'h': foo_output_hash, # TODO(maruel): Handle umask. 'm': 0640, 's': 3, }, 'foodir/foo2': { 'h': foo2_output_hash, # TODO(maruel): Handle umask. 'm': 0640, 's': 3, }, }, 'version': isolated_format.ISOLATED_FILE_VERSION, }
class RunIsolatedTestOutputFiles(RunIsolatedTestBase): # Like RunIsolatedTestRun, but ensures that specific output files # (as opposed to anything in $(ISOLATED_OUTDIR)) are returned. def _run_test(self, isolated, command, extra_args): # Starts a full isolate server mock and have run_tha_test() uploads results # back after the task completed. server = isolateserver_mock.MockIsolateServer() try: # Output two files. If we're on Linux, we'll try to make one of them a # symlink to ensure that we correctly follow symlinks. Note that this only # tests file symlinks, not directory symlinks. # TODO(aludwin): follow directory symlinks script = ( 'import os\n' 'import sys\n' 'open(sys.argv[1], "w").write("bar")\n' 'if sys.platform.startswith("linux"):\n' ' realpath = os.path.abspath("contents_of_symlink")\n' ' open(realpath, "w").write("baz")\n' ' os.symlink(realpath, sys.argv[2])\n' 'else:\n' ' open(sys.argv[2], "w").write("baz")\n') script_hash = isolateserver_mock.hash_content(script) isolated['files']['cmd.py'] = { 'h': script_hash, 'm': 0700, 's': len(script), } if sys.platform == 'win32': isolated['files']['cmd.py'].pop('m') isolated_data = json_dumps(isolated) isolated_hash = isolateserver_mock.hash_content(isolated_data) server.add_content('default-store', script) server.add_content('default-store', isolated_data) store = isolateserver.get_storage(server.url, 'default-store') self.mock(sys, 'stdout', StringIO.StringIO()) data = run_isolated.TaskData( command=command, relative_cwd=None, extra_args=extra_args, isolated_hash=isolated_hash, storage=store, isolate_cache=isolateserver.MemoryCache(), outputs=['foo', 'foodir/foo2'], install_named_caches=init_named_caches_stub, leak_temp_dir=False, root_dir=None, hard_timeout=60, grace_period=30, bot_file=None, switch_to_account=False, install_packages_fn=run_isolated.noop_install_packages, use_symlinks=False, env={}, env_prefix={}) ret = run_isolated.run_tha_test(data, None) self.assertEqual(0, ret) # It uploaded back. Assert the store has a new item containing foo. hashes = {isolated_hash, script_hash} foo_output_hash = isolateserver_mock.hash_content('bar') foo2_output_hash = isolateserver_mock.hash_content('baz') hashes.add(foo_output_hash) hashes.add(foo2_output_hash) isolated = { u'algo': u'sha-1', u'files': { u'foo': { u'h': foo_output_hash, # TODO(maruel): Handle umask. u'm': 0640, u's': 3, }, u'foodir/foo2': { u'h': foo2_output_hash, # TODO(maruel): Handle umask. u'm': 0640, u's': 3, }, }, u'version': isolated_format.ISOLATED_FILE_VERSION, }
class RunIsolatedTestRun(RunIsolatedTestBase): # Runs the actual command requested. def test_output(self): # Starts a full isolate server mock and have run_tha_test() uploads results # back after the task completed. server = isolateserver_mock.MockIsolateServer() try: script = ( 'import sys\n' 'open(sys.argv[1], "w").write("bar")\n') script_hash = isolateserver_mock.hash_content(script) isolated = { u'algo': u'sha-1', u'command': [u'cmd.py', u'${ISOLATED_OUTDIR}/foo'], u'files': { u'cmd.py': { u'h': script_hash, u'm': 0700, u's': len(script), }, }, u'version': isolated_format.ISOLATED_FILE_VERSION, } if sys.platform == 'win32': isolated[u'files'][u'cmd.py'].pop(u'm') isolated_data = json_dumps(isolated) isolated_hash = isolateserver_mock.hash_content(isolated_data) server.add_content('default-store', script) server.add_content('default-store', isolated_data) store = isolateserver.get_storage(server.url, 'default-store') self.mock(sys, 'stdout', StringIO.StringIO()) data = run_isolated.TaskData( command=[], relative_cwd=None, extra_args=[], isolated_hash=isolated_hash, storage=store, isolate_cache=isolateserver.MemoryCache(), outputs=None, install_named_caches=init_named_caches_stub, leak_temp_dir=False, root_dir=None, hard_timeout=60, grace_period=30, bot_file=None, switch_to_account=False, install_packages_fn=run_isolated.noop_install_packages, use_symlinks=False, env={}, env_prefix={}) ret = run_isolated.run_tha_test(data, None) self.assertEqual(0, ret) # It uploaded back. Assert the store has a new item containing foo. hashes = {isolated_hash, script_hash} output_hash = isolateserver_mock.hash_content('bar') hashes.add(output_hash) isolated = { u'algo': u'sha-1', u'files': { u'foo': { u'h': output_hash, # TODO(maruel): Handle umask. u'm': 0640, u's': 3, }, }, u'version': isolated_format.ISOLATED_FILE_VERSION, }