def setUp(self):
super(RunIsolatedTest, self).setUp()
self.tempdir = run_isolated.make_temp_dir(
u'run_isolated_smoke_test', ROOT_DIR)
logging.debug(self.tempdir)
# run_isolated.zip executable package.
self.run_isolated_zip = os.path.join(self.tempdir, 'run_isolated.zip')
run_isolated.get_as_zip_package().zip_into_file(
self.run_isolated_zip, compress=False)
# The run_isolated local cache.
self.cache = os.path.join(self.tempdir, 'cache')
self.server = isolateserver_mock.MockIsolateServer()
def setUp(self):
super(RunIsolatedTest, self).setUp()
self.tempdir = run_isolated.make_temp_dir(u'run_isolated_smoke_test',
ROOT_DIR)
logging.debug(self.tempdir)
# run_isolated.zip executable package.
self.run_isolated_zip = os.path.join(self.tempdir, 'run_isolated.zip')
run_isolated.get_as_zip_package().zip_into_file(self.run_isolated_zip,
compress=False)
# The run_isolated local cache.
self.cache = os.path.join(self.tempdir, 'cache')
self.server = isolateserver_mock.MockIsolateServer()
def setUp(self):
self.tempdir = run_isolated.make_temp_dir(
'run_isolated_smoke_test', ROOT_DIR)
logging.debug(self.tempdir)
# run_isolated.zip executable package.
self.run_isolated_zip = os.path.join(self.tempdir, 'run_isolated.zip')
run_isolated.get_as_zip_package().zip_into_file(
self.run_isolated_zip, compress=False)
# The "source" hash table.
self.table = os.path.join(self.tempdir, 'table')
os.mkdir(self.table)
# The slave-side cache.
self.cache = os.path.join(self.tempdir, 'cache')
self.data_dir = os.path.join(ROOT_DIR, 'tests', 'run_isolated')
def setUp(self):
self.tempdir = run_isolated.make_temp_dir('run_isolated_smoke_test',
ROOT_DIR)
logging.debug(self.tempdir)
# run_isolated.zip executable package.
self.run_isolated_zip = os.path.join(self.tempdir, 'run_isolated.zip')
run_isolated.get_as_zip_package().zip_into_file(self.run_isolated_zip,
compress=False)
# The "source" hash table.
self.table = os.path.join(self.tempdir, 'table')
os.mkdir(self.table)
# The slave-side cache.
self.cache = os.path.join(self.tempdir, 'cache')
self.data_dir = os.path.join(ROOT_DIR, 'tests', 'run_isolated')
def chromium_setup(manifest):
"""Sets up the commands to run.
Highly chromium specific.
"""
# Add uncompressed zip here. It'll be compressed as part of the package sent
# to Swarming server.
run_test_name = 'run_isolated.zip'
manifest.bundle.add_buffer(run_test_name,
run_isolated.get_as_zip_package().zip_into_buffer(compress=False))
cleanup_script_name = 'swarm_cleanup.py'
manifest.bundle.add_file(os.path.join(TOOLS_PATH, cleanup_script_name),
cleanup_script_name)
run_cmd = [
'python', run_test_name,
'--hash', manifest.isolated_hash,
'--isolate-server', manifest.isolate_server,
]
if manifest.verbose or manifest.profile:
# Have it print the profiling section.
run_cmd.append('--verbose')
manifest.add_task('Run Test', run_cmd)
# Clean up
manifest.add_task('Clean Up', ['python', cleanup_script_name])
def chromium_setup(manifest):
"""Sets up the commands to run.
Highly chromium specific.
"""
# Add uncompressed zip here. It'll be compressed as part of the package sent
# to Swarming server.
run_test_name = "run_isolated.zip"
manifest.bundle.add_buffer(run_test_name, run_isolated.get_as_zip_package().zip_into_buffer(compress=False))
cleanup_script_name = "swarm_cleanup.py"
manifest.bundle.add_file(os.path.join(TOOLS_PATH, cleanup_script_name), cleanup_script_name)
run_cmd = ["python", run_test_name, "--hash", manifest.isolated_hash, "--namespace", manifest.namespace]
if file_path.is_url(manifest.isolate_server):
run_cmd.extend(("--isolate-server", manifest.isolate_server))
else:
run_cmd.extend(("--indir", manifest.isolate_server))
if manifest.verbose or manifest.profile:
# Have it print the profiling section.
run_cmd.append("--verbose")
# Pass all extra args for run_isolated.py, it will pass them to the command.
if manifest.extra_args:
run_cmd.append("--")
run_cmd.extend(manifest.extra_args)
manifest.add_task("Run Test", run_cmd)
# Clean up
manifest.add_task("Clean Up", ["python", cleanup_script_name])
def setup_run_isolated(manifest, bundle):
"""Sets up the manifest to run an isolated task via run_isolated.py.
Modifies |bundle| (by adding files) and |manifest| (by adding commands) in
place.
Args:
manifest: Manifest with swarm task definition.
bundle: ZipPackage with files that would be transfered to swarm bot.
If None, only |manifest| is modified (useful in tests).
"""
# Add uncompressed zip here. It'll be compressed as part of the package sent
# to Swarming server.
run_test_name = 'run_isolated.zip'
if bundle and run_test_name not in bundle.files:
bundle.add_buffer(
run_test_name,
run_isolated.get_as_zip_package().zip_into_buffer(compress=False))
cleanup_script_name = 'swarm_cleanup.py'
if bundle and cleanup_script_name not in bundle.files:
bundle.add_file(os.path.join(TOOLS_PATH, cleanup_script_name),
cleanup_script_name)
run_cmd = [
'python',
run_test_name,
'--hash',
manifest.isolated_hash,
'--namespace',
manifest.namespace,
]
if file_path.is_url(manifest.isolate_server):
run_cmd.extend(('--isolate-server', manifest.isolate_server))
else:
run_cmd.extend(('--indir', manifest.isolate_server))
if manifest.verbose or manifest.profile:
# Have it print the profiling section.
run_cmd.append('--verbose')
# Pass all extra args for run_isolated.py, it will pass them to the command.
if manifest.extra_args:
run_cmd.append('--')
run_cmd.extend(manifest.extra_args)
manifest.add_task('Run Test', run_cmd)
# Clean up
manifest.add_task('Clean Up', ['python', cleanup_script_name])
def isolated_get_data(isolate_server):
"""Returns the 'data' section with all files necessary to bootstrap a task
execution running an isolated task.
It's mainly zipping run_isolated.zip over and over again.
TODO(maruel): Get rid of this with.
https://code.google.com/p/swarming/issues/detail?id=173
"""
bundle = zip_package.ZipPackage(ROOT_DIR)
bundle.add_buffer(
'run_isolated.zip',
run_isolated.get_as_zip_package().zip_into_buffer(compress=False))
bundle_url = isolated_upload_zip_bundle(isolate_server, bundle)
return [(bundle_url, 'swarm_data.zip')]
def setup_run_isolated(manifest, bundle):
"""Sets up the manifest to run an isolated task via run_isolated.py.
Modifies |bundle| (by adding files) and |manifest| (by adding commands) in
place.
Args:
manifest: Manifest with swarm task definition.
bundle: ZipPackage with files that would be transfered to swarm bot.
If None, only |manifest| is modified (useful in tests).
"""
# Add uncompressed zip here. It'll be compressed as part of the package sent
# to Swarming server.
run_test_name = 'run_isolated.zip'
if bundle and run_test_name not in bundle.files:
bundle.add_buffer(
run_test_name,
run_isolated.get_as_zip_package().zip_into_buffer(compress=False))
cleanup_script_name = 'swarm_cleanup.py'
if bundle and cleanup_script_name not in bundle.files:
bundle.add_file(
os.path.join(TOOLS_PATH, cleanup_script_name), cleanup_script_name)
run_cmd = [
'python', run_test_name,
'--hash', manifest.isolated_hash,
'--namespace', manifest.namespace,
]
if file_path.is_url(manifest.isolate_server):
run_cmd.extend(('--isolate-server', manifest.isolate_server))
else:
run_cmd.extend(('--indir', manifest.isolate_server))
if manifest.verbose or manifest.profile:
# Have it print the profiling section.
run_cmd.append('--verbose')
# Pass all extra args for run_isolated.py, it will pass them to the command.
if manifest.extra_args:
run_cmd.append('--')
run_cmd.extend(manifest.extra_args)
manifest.add_task('Run Test', run_cmd)
# Clean up
manifest.add_task('Clean Up', ['python', cleanup_script_name])
def main():
zip_package = run_isolated.get_as_zip_package()
zip_package.zip_into_file('run_isolated.zip')
return 0
def main():
zip_package = run_isolated.get_as_zip_package()
zip_package.zip_into_file('run_isolated.zip')
return 0
def test_simple(self):
# Create a directory with nothing in it and progressively add more stuff.
isolate = os.path.join(self.srcdir, 'gtest_fake_pass.isolate')
condition = 'OS=="linux" and chromeos==1'
with open(isolate, 'w') as f:
# Write a minimal .isolate file.
f.write(
str({
'conditions': [
[
condition, {
'variables': {
'command': [
'run_test_cases.py',
'gtest_fake_pass.py',
],
},
}
],
],
}))
def _copy(filename):
shutil.copy(os.path.join(BASE_DIR, 'gtest_fake', filename),
os.path.join(self.srcdir, filename))
_copy('gtest_fake_base.py')
_copy('gtest_fake_pass.py')
shutil.copy(os.path.join(GOOGLETEST_DIR, 'run_test_cases.py'),
os.path.join(self.srcdir, 'run_test_cases.py'))
# Deploy run_isolated with dependencies as zip into srcdir.
run_isolated.get_as_zip_package(executable=False).zip_into_file(
os.path.join(self.srcdir, 'run_isolated.zip'))
logging.debug('1. Create a .isolated file out of the .isolate file.')
isolated = os.path.join(self.srcdir, 'gtest_fake_pass.isolated')
out = self._run([
os.path.join(ROOT_DIR, 'isolate.py'),
'check',
'-i',
isolate,
'-s',
isolated,
'--config-variable',
'OS',
'linux',
'--config-variable',
'chromeos',
'1',
])
if not VERBOSE:
self.assertEqual('', out)
logging.debug('2. Run fix_test_cases.py on it.')
# Give up on looking at stdout.
cmd = [
os.path.join(GOOGLETEST_DIR, 'fix_test_cases.py'),
'-s',
isolated,
'--trace-blacklist',
'.*\\.run_test_cases',
]
_ = self._run(cmd)
logging.debug('3. Asserting the content of the .isolated file.')
with open(isolated) as f:
actual_isolated = json.load(f)
gtest_fake_base_py = os.path.join(self.srcdir, 'gtest_fake_base.py')
gtest_fake_pass_py = os.path.join(self.srcdir, 'gtest_fake_pass.py')
run_isolated_zip = os.path.join(self.srcdir, 'run_isolated.zip')
run_test_cases_py = os.path.join(self.srcdir, 'run_test_cases.py')
algo = hashlib.sha1
expected_isolated = {
u'algo': u'sha-1',
u'command': [u'run_test_cases.py', u'gtest_fake_pass.py'],
u'files': {
u'gtest_fake_base.py': {
u'm':
416,
u'h':
unicode(isolateserver.hash_file(gtest_fake_base_py, algo)),
u's':
os.stat(gtest_fake_base_py).st_size,
},
u'gtest_fake_pass.py': {
u'm':
488,
u'h':
unicode(isolateserver.hash_file(gtest_fake_pass_py, algo)),
u's':
os.stat(gtest_fake_pass_py).st_size,
},
u'run_isolated.zip': {
u'm': 416,
u'h':
unicode(isolateserver.hash_file(run_isolated_zip, algo)),
u's': os.stat(run_isolated_zip).st_size,
},
u'run_test_cases.py': {
u'm': 488,
u'h':
unicode(isolateserver.hash_file(run_test_cases_py, algo)),
u's': os.stat(run_test_cases_py).st_size,
},
},
u'relative_cwd': u'.',
u'version': unicode(isolateserver.ISOLATED_FILE_VERSION),
}
if sys.platform == 'win32':
for value in expected_isolated['files'].itervalues():
self.assertTrue(value.pop('m'))
self.assertEqual(expected_isolated, actual_isolated)
# Now verify the .isolate file was updated! (That's the magical part where
# you say wow!)
with open(isolate) as f:
actual = eval(f.read(), {'__builtins__': None}, None)
expected = {
'conditions': [
[
condition, {
'variables': {
'command':
['run_test_cases.py', 'gtest_fake_pass.py'],
'isolate_dependency_tracked': [
'gtest_fake_base.py',
'gtest_fake_pass.py',
'run_isolated.zip',
'run_test_cases.py',
],
},
}
],
],
}
self.assertEqual(expected, actual)
def test_simple(self):
# Create a directory with nothing in it and progressively add more stuff.
isolate = os.path.join(self.srcdir, 'gtest_fake_pass.isolate')
condition = 'OS=="linux" and chromeos==1'
with open(isolate, 'w') as f:
# Write a minimal .isolate file.
f.write(str({
'conditions': [
[condition, {
'variables': {
'command': [
'run_test_cases.py', 'gtest_fake_pass.py',
],
},
}],
],
}))
def _copy(filename):
shutil.copy(
os.path.join(BASE_DIR, 'gtest_fake', filename),
os.path.join(self.srcdir, filename))
_copy('gtest_fake_base.py')
_copy('gtest_fake_pass.py')
shutil.copy(
os.path.join(GOOGLETEST_DIR, 'run_test_cases.py'),
os.path.join(self.srcdir, 'run_test_cases.py'))
# Deploy run_isolated with dependencies as zip into srcdir.
run_isolated.get_as_zip_package(executable=False).zip_into_file(
os.path.join(self.srcdir, 'run_isolated.zip'))
logging.debug('1. Create a .isolated file out of the .isolate file.')
isolated = os.path.join(self.srcdir, 'gtest_fake_pass.isolated')
out = self._run(
[
os.path.join(ROOT_DIR, 'isolate.py'),
'check', '-i', isolate, '-s', isolated,
'--config-variable', 'OS', 'linux',
'--config-variable', 'chromeos', '1',
])
if not VERBOSE:
self.assertEqual('', out)
logging.debug('2. Run fix_test_cases.py on it.')
# Give up on looking at stdout.
cmd = [
os.path.join(GOOGLETEST_DIR, 'fix_test_cases.py'),
'-s', isolated,
'--trace-blacklist', '.*\\.run_test_cases',
]
_ = self._run(cmd)
logging.debug('3. Asserting the content of the .isolated file.')
with open(isolated) as f:
actual_isolated = json.load(f)
gtest_fake_base_py = os.path.join(self.srcdir, 'gtest_fake_base.py')
gtest_fake_pass_py = os.path.join(self.srcdir, 'gtest_fake_pass.py')
run_isolated_zip = os.path.join(self.srcdir, 'run_isolated.zip')
run_test_cases_py = os.path.join(self.srcdir, 'run_test_cases.py')
algo = hashlib.sha1
expected_isolated = {
u'algo': u'sha-1',
u'command': [u'run_test_cases.py', u'gtest_fake_pass.py'],
u'files': {
u'gtest_fake_base.py': {
u'm': 416,
u'h': unicode(isolateserver.hash_file(gtest_fake_base_py, algo)),
u's': os.stat(gtest_fake_base_py).st_size,
},
u'gtest_fake_pass.py': {
u'm': 488,
u'h': unicode(isolateserver.hash_file(gtest_fake_pass_py, algo)),
u's': os.stat(gtest_fake_pass_py).st_size,
},
u'run_isolated.zip': {
u'm': 416,
u'h': unicode(isolateserver.hash_file(run_isolated_zip, algo)),
u's': os.stat(run_isolated_zip).st_size,
},
u'run_test_cases.py': {
u'm': 488,
u'h': unicode(isolateserver.hash_file(run_test_cases_py, algo)),
u's': os.stat(run_test_cases_py).st_size,
},
},
u'relative_cwd': u'.',
u'version': unicode(isolateserver.ISOLATED_FILE_VERSION),
}
if sys.platform == 'win32':
for value in expected_isolated['files'].itervalues():
self.assertTrue(value.pop('m'))
self.assertEqual(expected_isolated, actual_isolated)
# Now verify the .isolate file was updated! (That's the magical part where
# you say wow!)
with open(isolate) as f:
actual = eval(f.read(), {'__builtins__': None}, None)
expected = {
'conditions': [
[condition, {
'variables': {
'command': [
'run_test_cases.py', 'gtest_fake_pass.py'
],
'isolate_dependency_tracked': [
'gtest_fake_base.py',
'gtest_fake_pass.py',
'run_isolated.zip',
'run_test_cases.py',
],
},
}],
],
}
self.assertEqual(expected, actual)
