def _run_test(self, isolated, command): # Starts a full isolate server mock and have run_tha_test() uploads results # back after the task completed. server = isolateserver_mock.MockIsolateServer() try: # Output two files. If we're on Linux, we'll try to make one of them a # symlink to ensure that we correctly follow symlinks. Note that this only # tests file symlinks, not directory symlinks. # TODO(aludwin): follow directory symlinks script = ( 'import os\n' 'import sys\n' 'open(sys.argv[1], "w").write("bar")\n' 'if sys.platform.startswith("linux"):\n' ' realpath = os.path.abspath("contents_of_symlink")\n' ' open(realpath, "w").write("baz")\n' ' os.symlink(realpath, sys.argv[2])\n' 'else:\n' ' open(sys.argv[2], "w").write("baz")\n') script_hash = isolateserver_mock.hash_content(script) isolated['files']['cmd.py'] = { 'h': script_hash, 'm': 0700, 's': len(script), } if sys.platform == 'win32':
def _run_test(self, isolated, command, extra_args): # Starts a full isolate server mock and have run_tha_test() uploads results # back after the task completed. server = isolateserver_mock.MockIsolateServer() try: # Output the following structure: # # foo1 # foodir --> foo2_sl (symlink to "foo2_content" file) # bardir --> bar1 # # Create the symlinks only on Linux. script = ('import os\n' 'import sys\n' 'open(sys.argv[1], "w").write("foo1")\n' 'bar1_path = os.path.join(sys.argv[3], "bar1")\n' 'open(bar1_path, "w").write("bar1")\n' 'if sys.platform.startswith("linux"):\n' ' foo_realpath = os.path.abspath("foo2_content")\n' ' open(foo_realpath, "w").write("foo2")\n' ' os.symlink(foo_realpath, sys.argv[2])\n' 'else:\n' ' open(sys.argv[2], "w").write("foo2")\n') script_hash = isolateserver_mock.hash_content(script) isolated['files']['cmd.py'] = { 'h': script_hash, 'm': 0700, 's': len(script), } if sys.platform == 'win32':
def setUp(self): super(RunIsolatedTest, self).setUp() self.tempdir = run_isolated.make_temp_dir(u'run_isolated_smoke_test', ROOT_DIR) logging.debug(self.tempdir) # run_isolated.zip executable package. self.run_isolated_zip = os.path.join(self.tempdir, 'run_isolated.zip') run_isolated.get_as_zip_package().zip_into_file(self.run_isolated_zip, compress=False) # The run_isolated local cache. self.cache = os.path.join(self.tempdir, 'cache') self.server = isolateserver_mock.MockIsolateServer()
def _run_test(self, isolated, command): # Starts a full isolate server mock and have run_tha_test() uploads results # back after the task completed. server = isolateserver_mock.MockIsolateServer() try: script = ('import sys\n' 'open(sys.argv[1], "w").write("bar")\n' 'open(sys.argv[2], "w").write("baz")\n') script_hash = isolateserver_mock.hash_content(script) isolated['files']['cmd.py'] = { 'h': script_hash, 'm': 0700, 's': len(script), } if sys.platform == 'win32':
def test_output(self): # Starts a full isolate server mock and have run_tha_test() uploads results # back after the task completed. server = isolateserver_mock.MockIsolateServer() try: script = ('import sys\n' 'open(sys.argv[1], "w").write("bar")\n') script_hash = isolateserver_mock.hash_content(script) isolated = { u'algo': u'sha-1', u'command': [u'cmd.py', u'${ISOLATED_OUTDIR}/foo'], u'files': { u'cmd.py': { u'h': script_hash, u'm': 0700, u's': len(script), }, }, u'version': isolated_format.ISOLATED_FILE_VERSION, } if sys.platform == 'win32':
def setUp(self): super(IsolateServerStorageSmokeTest, self).setUp() self.tempdir = tempfile.mkdtemp(prefix=u'isolateserver') self.server = isolateserver_mock.MockIsolateServer()
def setUp(self): auto_stub.TestCase.setUp(self) Common.setUp(self) self._isolate = isolateserver_mock.MockIsolateServer() self._swarming = MockSwarmingServer()
def server(self): """Creates the Isolate Server mock on first reference.""" if not self._server: self._server = isolateserver_mock.MockIsolateServer() return self._server
def test_isolated_io_signal_no_grace_grand_children(self): """Handles grand-children process hanging and signal management. In this case, the I/O timeout is implemented by task_runner. An hard timeout would be implemented by run_isolated (depending on overhead). """ # Uses load_and_run() # https://msdn.microsoft.com/library/cc704588.aspx # STATUS_CONTROL_C_EXIT=0xC000013A. Python sees it as -1073741510. exit_code = -1073741510 if sys.platform == 'win32' else -signal.SIGTERM files = { 'parent.py': ( 'import subprocess, sys\n' 'print(\'parent\')\n' 'p = subprocess.Popen([sys.executable, \'-u\', \'children.py\'])\n' 'print(p.pid)\n' 'p.wait()\n' 'sys.exit(p.returncode)\n'), 'children.py': ( 'import subprocess, sys\n' 'print(\'children\')\n' 'p = subprocess.Popen([sys.executable,\'-u\',\'grand_children.py\'])\n' 'print(p.pid)\n' 'p.wait()\n' 'sys.exit(p.returncode)\n'), 'grand_children.py': self.SCRIPT_SIGNAL_HANG, } # We need to catch the pid of the grand children to be able to kill it, so # create our own check_final() instead of using self._gen_requests(). to_kill = [] def check_final(kwargs): self.assertLess(self.SHORT_TIME_OUT, kwargs['data'].pop('cost_usd')) self.assertLess(self.SHORT_TIME_OUT, kwargs['data'].pop('duration')) self.assertLess(0., kwargs['data'].pop('bot_overhead')) self.assertLess( 0., kwargs['data']['isolated_stats']['download'].pop('duration')) self.assertLess( 0., kwargs['data']['isolated_stats']['upload'].pop('duration')) # Makes the diffing easier. for k in ('download', 'upload'): for j in ('items_cold', 'items_hot'): kwargs['data']['isolated_stats'][k][j] = large.unpack( base64.b64decode(kwargs['data']['isolated_stats'][k][j])) # The command print the pid of this child and grand-child processes, each # on its line. output = base64.b64decode(kwargs['data'].pop('output', '')) for line in output.splitlines(): try: to_kill.append(int(line)) except ValueError: pass self.assertEqual( { 'data': { 'exit_code': exit_code, 'hard_timeout': False, 'id': u'localhost', 'io_timeout': True, 'isolated_stats': { u'download': { u'initial_number_items': 0, u'initial_size': 0, u'items_cold': [144, 150, 285, 307], u'items_hot': [], }, u'upload': { u'items_cold': [], u'items_hot': [], }, }, 'output_chunk_start': 0, 'task_id': 23, }, }, kwargs) requests = [ ( 'https://localhost:1/swarming/api/v1/bot/task_update/23', self.get_check_first(0.), {'ok': True}, ), ( 'https://localhost:1/swarming/api/v1/bot/task_update/23', check_final, {'ok': True}, ), ] self.expected_requests(requests) server = isolateserver_mock.MockIsolateServer() try: # TODO(maruel): -u is needed if you don't want python buffering to # interfere. isolated = json.dumps({ 'command': ['python', '-u', 'parent.py'], 'files': { name: { 'h': server.add_content_compressed('default-gzip', content), 's': len(content), } for name, content in files.iteritems() }, }) isolated_digest = server.add_content_compressed('default-gzip', isolated) try: manifest = get_manifest( inputs_ref={ 'isolated': isolated_digest, 'namespace': 'default-gzip', 'isolatedserver': server.url, }, # TODO(maruel): A bit cheezy, we'd want the I/O timeout to be just # enough to have the time for the PID to be printed but not more. io_timeout=1, grace_period=self.SHORT_TIME_OUT) expected = { u'exit_code': exit_code, u'hard_timeout': False, u'io_timeout': True, u'must_signal_internal_failure': None, u'version': task_runner.OUT_VERSION, } self.assertEqual(expected, self._load_and_run(manifest)) self.assertEqual(2, len(to_kill)) finally: for k in to_kill: try: if sys.platform == 'win32': os.kill(k, signal.SIGTERM) else: os.kill(k, signal.SIGKILL) except OSError: pass finally: server.close()
def test_isolated_grand_children(self): """Runs a normal test involving 3 level deep subprocesses.""" # Uses load_and_run() files = { 'parent.py': ( 'import subprocess, sys\n' 'sys.exit(subprocess.call([sys.executable,\'-u\',\'children.py\']))\n'), 'children.py': ( 'import subprocess, sys\n' 'sys.exit(subprocess.call(' '[sys.executable, \'-u\', \'grand_children.py\']))\n'), 'grand_children.py': 'print \'hi\'', } def check_final(kwargs): # Warning: this modifies input arguments. self.assertLess(0, kwargs['data'].pop('cost_usd')) self.assertLess(0, kwargs['data'].pop('bot_overhead')) self.assertLess(0, kwargs['data'].pop('duration')) self.assertLess( 0., kwargs['data']['isolated_stats']['download'].pop('duration')) # duration==0 can happen on Windows when the clock is in the default # resolution, 15.6ms. self.assertLessEqual( 0., kwargs['data']['isolated_stats']['upload'].pop('duration')) # Makes the diffing easier. kwargs['data']['output'] = base64.b64decode(kwargs['data']['output']) for k in ('download', 'upload'): for j in ('items_cold', 'items_hot'): kwargs['data']['isolated_stats'][k][j] = large.unpack( base64.b64decode(kwargs['data']['isolated_stats'][k][j])) self.assertEqual( { 'data': { 'exit_code': 0, 'hard_timeout': False, 'id': u'localhost', 'io_timeout': False, 'isolated_stats': { u'download': { u'initial_number_items': 0, u'initial_size': 0, u'items_cold': [10, 86, 94, 276], u'items_hot': [], }, u'upload': { u'items_cold': [], u'items_hot': [], }, }, 'output': 'hi\n', 'output_chunk_start': 0, 'task_id': 23, }, }, kwargs) requests = [ ( 'https://localhost:1/swarming/api/v1/bot/task_update/23', self.get_check_first(0.), {'ok': True}, ), ( 'https://localhost:1/swarming/api/v1/bot/task_update/23', check_final, {'ok': True}, ), ] self.expected_requests(requests) server = isolateserver_mock.MockIsolateServer() try: isolated = json.dumps({ 'command': ['python', 'parent.py'], 'files': { name: { 'h': server.add_content_compressed('default-gzip', content), 's': len(content), } for name, content in files.iteritems() }, }) isolated_digest = server.add_content_compressed('default-gzip', isolated) manifest = get_manifest( inputs_ref={ 'isolated': isolated_digest, 'namespace': 'default-gzip', 'isolatedserver': server.url, }) expected = { u'exit_code': 0, u'hard_timeout': False, u'io_timeout': False, u'must_signal_internal_failure': None, u'version': task_runner.OUT_VERSION, } self.assertEqual(expected, self._load_and_run(manifest)) finally: server.close()