def setUp(self): super(RunIsolatedTest, self).setUp() self.tempdir = run_isolated.make_temp_dir( u'run_isolated_smoke_test', ROOT_DIR) logging.debug(self.tempdir) # run_isolated.zip executable package. self.run_isolated_zip = os.path.join(self.tempdir, 'run_isolated.zip') run_isolated.get_as_zip_package().zip_into_file( self.run_isolated_zip, compress=False) # The run_isolated local cache. self.cache = os.path.join(self.tempdir, 'cache') self.server = isolateserver_mock.MockIsolateServer()
def setUp(self): super(RunIsolatedTest, self).setUp() self.tempdir = run_isolated.make_temp_dir(u'run_isolated_smoke_test', ROOT_DIR) logging.debug(self.tempdir) # run_isolated.zip executable package. self.run_isolated_zip = os.path.join(self.tempdir, 'run_isolated.zip') run_isolated.get_as_zip_package().zip_into_file(self.run_isolated_zip, compress=False) # The run_isolated local cache. self.cache = os.path.join(self.tempdir, 'cache') self.server = isolateserver_mock.MockIsolateServer()
def setUp(self): self.tempdir = run_isolated.make_temp_dir( 'run_isolated_smoke_test', ROOT_DIR) logging.debug(self.tempdir) # run_isolated.zip executable package. self.run_isolated_zip = os.path.join(self.tempdir, 'run_isolated.zip') run_isolated.get_as_zip_package().zip_into_file( self.run_isolated_zip, compress=False) # The "source" hash table. self.table = os.path.join(self.tempdir, 'table') os.mkdir(self.table) # The slave-side cache. self.cache = os.path.join(self.tempdir, 'cache') self.data_dir = os.path.join(ROOT_DIR, 'tests', 'run_isolated')
def setUp(self): self.tempdir = run_isolated.make_temp_dir('run_isolated_smoke_test', ROOT_DIR) logging.debug(self.tempdir) # run_isolated.zip executable package. self.run_isolated_zip = os.path.join(self.tempdir, 'run_isolated.zip') run_isolated.get_as_zip_package().zip_into_file(self.run_isolated_zip, compress=False) # The "source" hash table. self.table = os.path.join(self.tempdir, 'table') os.mkdir(self.table) # The slave-side cache. self.cache = os.path.join(self.tempdir, 'cache') self.data_dir = os.path.join(ROOT_DIR, 'tests', 'run_isolated')
def chromium_setup(manifest): """Sets up the commands to run. Highly chromium specific. """ # Add uncompressed zip here. It'll be compressed as part of the package sent # to Swarming server. run_test_name = 'run_isolated.zip' manifest.bundle.add_buffer(run_test_name, run_isolated.get_as_zip_package().zip_into_buffer(compress=False)) cleanup_script_name = 'swarm_cleanup.py' manifest.bundle.add_file(os.path.join(TOOLS_PATH, cleanup_script_name), cleanup_script_name) run_cmd = [ 'python', run_test_name, '--hash', manifest.isolated_hash, '--isolate-server', manifest.isolate_server, ] if manifest.verbose or manifest.profile: # Have it print the profiling section. run_cmd.append('--verbose') manifest.add_task('Run Test', run_cmd) # Clean up manifest.add_task('Clean Up', ['python', cleanup_script_name])
def chromium_setup(manifest): """Sets up the commands to run. Highly chromium specific. """ # Add uncompressed zip here. It'll be compressed as part of the package sent # to Swarming server. run_test_name = "run_isolated.zip" manifest.bundle.add_buffer(run_test_name, run_isolated.get_as_zip_package().zip_into_buffer(compress=False)) cleanup_script_name = "swarm_cleanup.py" manifest.bundle.add_file(os.path.join(TOOLS_PATH, cleanup_script_name), cleanup_script_name) run_cmd = ["python", run_test_name, "--hash", manifest.isolated_hash, "--namespace", manifest.namespace] if file_path.is_url(manifest.isolate_server): run_cmd.extend(("--isolate-server", manifest.isolate_server)) else: run_cmd.extend(("--indir", manifest.isolate_server)) if manifest.verbose or manifest.profile: # Have it print the profiling section. run_cmd.append("--verbose") # Pass all extra args for run_isolated.py, it will pass them to the command. if manifest.extra_args: run_cmd.append("--") run_cmd.extend(manifest.extra_args) manifest.add_task("Run Test", run_cmd) # Clean up manifest.add_task("Clean Up", ["python", cleanup_script_name])
def setup_run_isolated(manifest, bundle): """Sets up the manifest to run an isolated task via run_isolated.py. Modifies |bundle| (by adding files) and |manifest| (by adding commands) in place. Args: manifest: Manifest with swarm task definition. bundle: ZipPackage with files that would be transfered to swarm bot. If None, only |manifest| is modified (useful in tests). """ # Add uncompressed zip here. It'll be compressed as part of the package sent # to Swarming server. run_test_name = 'run_isolated.zip' if bundle and run_test_name not in bundle.files: bundle.add_buffer( run_test_name, run_isolated.get_as_zip_package().zip_into_buffer(compress=False)) cleanup_script_name = 'swarm_cleanup.py' if bundle and cleanup_script_name not in bundle.files: bundle.add_file(os.path.join(TOOLS_PATH, cleanup_script_name), cleanup_script_name) run_cmd = [ 'python', run_test_name, '--hash', manifest.isolated_hash, '--namespace', manifest.namespace, ] if file_path.is_url(manifest.isolate_server): run_cmd.extend(('--isolate-server', manifest.isolate_server)) else: run_cmd.extend(('--indir', manifest.isolate_server)) if manifest.verbose or manifest.profile: # Have it print the profiling section. run_cmd.append('--verbose') # Pass all extra args for run_isolated.py, it will pass them to the command. if manifest.extra_args: run_cmd.append('--') run_cmd.extend(manifest.extra_args) manifest.add_task('Run Test', run_cmd) # Clean up manifest.add_task('Clean Up', ['python', cleanup_script_name])
def isolated_get_data(isolate_server): """Returns the 'data' section with all files necessary to bootstrap a task execution running an isolated task. It's mainly zipping run_isolated.zip over and over again. TODO(maruel): Get rid of this with. https://code.google.com/p/swarming/issues/detail?id=173 """ bundle = zip_package.ZipPackage(ROOT_DIR) bundle.add_buffer( 'run_isolated.zip', run_isolated.get_as_zip_package().zip_into_buffer(compress=False)) bundle_url = isolated_upload_zip_bundle(isolate_server, bundle) return [(bundle_url, 'swarm_data.zip')]
def setup_run_isolated(manifest, bundle): """Sets up the manifest to run an isolated task via run_isolated.py. Modifies |bundle| (by adding files) and |manifest| (by adding commands) in place. Args: manifest: Manifest with swarm task definition. bundle: ZipPackage with files that would be transfered to swarm bot. If None, only |manifest| is modified (useful in tests). """ # Add uncompressed zip here. It'll be compressed as part of the package sent # to Swarming server. run_test_name = 'run_isolated.zip' if bundle and run_test_name not in bundle.files: bundle.add_buffer( run_test_name, run_isolated.get_as_zip_package().zip_into_buffer(compress=False)) cleanup_script_name = 'swarm_cleanup.py' if bundle and cleanup_script_name not in bundle.files: bundle.add_file( os.path.join(TOOLS_PATH, cleanup_script_name), cleanup_script_name) run_cmd = [ 'python', run_test_name, '--hash', manifest.isolated_hash, '--namespace', manifest.namespace, ] if file_path.is_url(manifest.isolate_server): run_cmd.extend(('--isolate-server', manifest.isolate_server)) else: run_cmd.extend(('--indir', manifest.isolate_server)) if manifest.verbose or manifest.profile: # Have it print the profiling section. run_cmd.append('--verbose') # Pass all extra args for run_isolated.py, it will pass them to the command. if manifest.extra_args: run_cmd.append('--') run_cmd.extend(manifest.extra_args) manifest.add_task('Run Test', run_cmd) # Clean up manifest.add_task('Clean Up', ['python', cleanup_script_name])
def main(): zip_package = run_isolated.get_as_zip_package() zip_package.zip_into_file('run_isolated.zip') return 0
def test_simple(self): # Create a directory with nothing in it and progressively add more stuff. isolate = os.path.join(self.srcdir, 'gtest_fake_pass.isolate') condition = 'OS=="linux" and chromeos==1' with open(isolate, 'w') as f: # Write a minimal .isolate file. f.write( str({ 'conditions': [ [ condition, { 'variables': { 'command': [ 'run_test_cases.py', 'gtest_fake_pass.py', ], }, } ], ], })) def _copy(filename): shutil.copy(os.path.join(BASE_DIR, 'gtest_fake', filename), os.path.join(self.srcdir, filename)) _copy('gtest_fake_base.py') _copy('gtest_fake_pass.py') shutil.copy(os.path.join(GOOGLETEST_DIR, 'run_test_cases.py'), os.path.join(self.srcdir, 'run_test_cases.py')) # Deploy run_isolated with dependencies as zip into srcdir. run_isolated.get_as_zip_package(executable=False).zip_into_file( os.path.join(self.srcdir, 'run_isolated.zip')) logging.debug('1. Create a .isolated file out of the .isolate file.') isolated = os.path.join(self.srcdir, 'gtest_fake_pass.isolated') out = self._run([ os.path.join(ROOT_DIR, 'isolate.py'), 'check', '-i', isolate, '-s', isolated, '--config-variable', 'OS', 'linux', '--config-variable', 'chromeos', '1', ]) if not VERBOSE: self.assertEqual('', out) logging.debug('2. Run fix_test_cases.py on it.') # Give up on looking at stdout. cmd = [ os.path.join(GOOGLETEST_DIR, 'fix_test_cases.py'), '-s', isolated, '--trace-blacklist', '.*\\.run_test_cases', ] _ = self._run(cmd) logging.debug('3. Asserting the content of the .isolated file.') with open(isolated) as f: actual_isolated = json.load(f) gtest_fake_base_py = os.path.join(self.srcdir, 'gtest_fake_base.py') gtest_fake_pass_py = os.path.join(self.srcdir, 'gtest_fake_pass.py') run_isolated_zip = os.path.join(self.srcdir, 'run_isolated.zip') run_test_cases_py = os.path.join(self.srcdir, 'run_test_cases.py') algo = hashlib.sha1 expected_isolated = { u'algo': u'sha-1', u'command': [u'run_test_cases.py', u'gtest_fake_pass.py'], u'files': { u'gtest_fake_base.py': { u'm': 416, u'h': unicode(isolateserver.hash_file(gtest_fake_base_py, algo)), u's': os.stat(gtest_fake_base_py).st_size, }, u'gtest_fake_pass.py': { u'm': 488, u'h': unicode(isolateserver.hash_file(gtest_fake_pass_py, algo)), u's': os.stat(gtest_fake_pass_py).st_size, }, u'run_isolated.zip': { u'm': 416, u'h': unicode(isolateserver.hash_file(run_isolated_zip, algo)), u's': os.stat(run_isolated_zip).st_size, }, u'run_test_cases.py': { u'm': 488, u'h': unicode(isolateserver.hash_file(run_test_cases_py, algo)), u's': os.stat(run_test_cases_py).st_size, }, }, u'relative_cwd': u'.', u'version': unicode(isolateserver.ISOLATED_FILE_VERSION), } if sys.platform == 'win32': for value in expected_isolated['files'].itervalues(): self.assertTrue(value.pop('m')) self.assertEqual(expected_isolated, actual_isolated) # Now verify the .isolate file was updated! (That's the magical part where # you say wow!) with open(isolate) as f: actual = eval(f.read(), {'__builtins__': None}, None) expected = { 'conditions': [ [ condition, { 'variables': { 'command': ['run_test_cases.py', 'gtest_fake_pass.py'], 'isolate_dependency_tracked': [ 'gtest_fake_base.py', 'gtest_fake_pass.py', 'run_isolated.zip', 'run_test_cases.py', ], }, } ], ], } self.assertEqual(expected, actual)
def test_simple(self): # Create a directory with nothing in it and progressively add more stuff. isolate = os.path.join(self.srcdir, 'gtest_fake_pass.isolate') condition = 'OS=="linux" and chromeos==1' with open(isolate, 'w') as f: # Write a minimal .isolate file. f.write(str({ 'conditions': [ [condition, { 'variables': { 'command': [ 'run_test_cases.py', 'gtest_fake_pass.py', ], }, }], ], })) def _copy(filename): shutil.copy( os.path.join(BASE_DIR, 'gtest_fake', filename), os.path.join(self.srcdir, filename)) _copy('gtest_fake_base.py') _copy('gtest_fake_pass.py') shutil.copy( os.path.join(GOOGLETEST_DIR, 'run_test_cases.py'), os.path.join(self.srcdir, 'run_test_cases.py')) # Deploy run_isolated with dependencies as zip into srcdir. run_isolated.get_as_zip_package(executable=False).zip_into_file( os.path.join(self.srcdir, 'run_isolated.zip')) logging.debug('1. Create a .isolated file out of the .isolate file.') isolated = os.path.join(self.srcdir, 'gtest_fake_pass.isolated') out = self._run( [ os.path.join(ROOT_DIR, 'isolate.py'), 'check', '-i', isolate, '-s', isolated, '--config-variable', 'OS', 'linux', '--config-variable', 'chromeos', '1', ]) if not VERBOSE: self.assertEqual('', out) logging.debug('2. Run fix_test_cases.py on it.') # Give up on looking at stdout. cmd = [ os.path.join(GOOGLETEST_DIR, 'fix_test_cases.py'), '-s', isolated, '--trace-blacklist', '.*\\.run_test_cases', ] _ = self._run(cmd) logging.debug('3. Asserting the content of the .isolated file.') with open(isolated) as f: actual_isolated = json.load(f) gtest_fake_base_py = os.path.join(self.srcdir, 'gtest_fake_base.py') gtest_fake_pass_py = os.path.join(self.srcdir, 'gtest_fake_pass.py') run_isolated_zip = os.path.join(self.srcdir, 'run_isolated.zip') run_test_cases_py = os.path.join(self.srcdir, 'run_test_cases.py') algo = hashlib.sha1 expected_isolated = { u'algo': u'sha-1', u'command': [u'run_test_cases.py', u'gtest_fake_pass.py'], u'files': { u'gtest_fake_base.py': { u'm': 416, u'h': unicode(isolateserver.hash_file(gtest_fake_base_py, algo)), u's': os.stat(gtest_fake_base_py).st_size, }, u'gtest_fake_pass.py': { u'm': 488, u'h': unicode(isolateserver.hash_file(gtest_fake_pass_py, algo)), u's': os.stat(gtest_fake_pass_py).st_size, }, u'run_isolated.zip': { u'm': 416, u'h': unicode(isolateserver.hash_file(run_isolated_zip, algo)), u's': os.stat(run_isolated_zip).st_size, }, u'run_test_cases.py': { u'm': 488, u'h': unicode(isolateserver.hash_file(run_test_cases_py, algo)), u's': os.stat(run_test_cases_py).st_size, }, }, u'relative_cwd': u'.', u'version': unicode(isolateserver.ISOLATED_FILE_VERSION), } if sys.platform == 'win32': for value in expected_isolated['files'].itervalues(): self.assertTrue(value.pop('m')) self.assertEqual(expected_isolated, actual_isolated) # Now verify the .isolate file was updated! (That's the magical part where # you say wow!) with open(isolate) as f: actual = eval(f.read(), {'__builtins__': None}, None) expected = { 'conditions': [ [condition, { 'variables': { 'command': [ 'run_test_cases.py', 'gtest_fake_pass.py' ], 'isolate_dependency_tracked': [ 'gtest_fake_base.py', 'gtest_fake_pass.py', 'run_isolated.zip', 'run_test_cases.py', ], }, }], ], } self.assertEqual(expected, actual)