def test_load_and_run_isolated(self):
self.expected_requests([])
FakeAuthSystem.local_auth_context = None
self.mock(bot_auth, 'AuthSystem', FakeAuthSystem)
def run_command(
remote, task_details, work_dir,
cost_usd_hour, start, min_free_space, bot_file):
self.assertTrue(remote.uses_auth) # mainly to avoid unused arg warning
self.assertTrue(isinstance(task_details, task_runner.TaskDetails))
# Necessary for OSX.
self.assertEqual(
os.path.realpath(self.work_dir), os.path.realpath(work_dir))
self.assertEqual(3600., cost_usd_hour)
self.assertEqual(time.time(), start)
self.assertEqual(1, min_free_space)
self.assertEqual('/path/to/bot-file', bot_file)
self.assertIsNone(luci_context.read('local_auth'))
return {
u'exit_code': 0,
u'hard_timeout': False,
u'io_timeout': False,
u'must_signal_internal_failure': None,
u'version': task_runner.OUT_VERSION,
}
self.mock(task_runner, 'run_command', run_command)
manifest = os.path.join(self.root_dir, 'manifest')
with open(manifest, 'wb') as f:
data = {
'bot_id': 'localhost',
'command': None,
'env': {'d': 'e'},
'extra_args': ['foo', 'bar'],
'grace_period': 30.,
'hard_timeout': 10,
'io_timeout': 11,
'isolated': {
'input': '123',
'server': 'http://localhost:1',
'namespace': 'default-gzip',
},
'task_id': 23,
}
json.dump(data, f)
out_file = os.path.join(self.root_dir, 'w', 'task_runner_out.json')
task_runner.load_and_run(
manifest, 'localhost:1', 3600., time.time(), out_file, 1,
'/path/to/bot-file', '/path/to/auth-params-file')
expected = {
u'exit_code': 0,
u'hard_timeout': False,
u'io_timeout': False,
u'must_signal_internal_failure': None,
u'version': task_runner.OUT_VERSION,
}
with open(out_file, 'rb') as f:
self.assertEqual(expected, json.load(f))
def test_load_and_run_raw(self):
requests = [
(
'https://localhost:1/f',
{},
compress_to_zip({'file3': 'content3'}),
None,
),
]
self.expected_requests(requests)
server = xsrf_client.XsrfRemote('https://localhost:1/')
def run_command(swarming_server, task_details, work_dir, cost_usd_hour,
start):
self.assertEqual(server, swarming_server)
# Necessary for OSX.
self.assertEqual(os.path.realpath(self.work_dir), work_dir)
self.assertTrue(isinstance(task_details, task_runner.TaskDetails))
self.assertEqual(3600., cost_usd_hour)
self.assertEqual(time.time(), start)
return {
u'exit_code': 1,
u'hard_timeout': False,
u'io_timeout': False,
u'must_signal_internal_failure': None,
u'version': task_runner.OUT_VERSION,
}
self.mock(task_runner, 'run_command', run_command)
manifest = os.path.join(self.root_dir, 'manifest')
with open(manifest, 'wb') as f:
data = {
'bot_id': 'localhost',
'command': ['a'],
'data': [('https://localhost:1/f', 'foo.zip')],
'env': {
'd': 'e'
},
'extra_args': [],
'grace_period': 30.,
'hard_timeout': 10,
'inputs_ref': None,
'io_timeout': 11,
'task_id': 23,
}
json.dump(data, f)
out_file = os.path.join(self.root_dir, 'task_runner_out.json')
task_runner.load_and_run(manifest, server, 3600., time.time(),
out_file)
expected = {
u'exit_code': 1,
u'hard_timeout': False,
u'io_timeout': False,
u'must_signal_internal_failure': None,
u'version': task_runner.OUT_VERSION,
}
with open(out_file, 'rb') as f:
self.assertEqual(expected, json.load(f))
def test_load_and_run_raw(self):
requests = [
(
'https://localhost:1/f',
{},
compress_to_zip({'file3': 'content3'}),
None,
),
]
self.expected_requests(requests)
server = xsrf_client.XsrfRemote('https://localhost:1/')
def run_command(
swarming_server, task_details, work_dir, cost_usd_hour, start):
self.assertEqual(server, swarming_server)
# Necessary for OSX.
self.assertEqual(os.path.realpath(self.work_dir), work_dir)
self.assertTrue(isinstance(task_details, task_runner.TaskDetails))
self.assertEqual(3600., cost_usd_hour)
self.assertEqual(time.time(), start)
return {
u'exit_code': 1,
u'hard_timeout': False,
u'io_timeout': False,
u'must_signal_internal_failure': None,
u'version': task_runner.OUT_VERSION,
}
self.mock(task_runner, 'run_command', run_command)
manifest = os.path.join(self.root_dir, 'manifest')
with open(manifest, 'wb') as f:
data = {
'bot_id': 'localhost',
'command': ['a'],
'data': [('https://localhost:1/f', 'foo.zip')],
'env': {'d': 'e'},
'extra_args': [],
'grace_period': 30.,
'hard_timeout': 10,
'inputs_ref': None,
'io_timeout': 11,
'task_id': 23,
}
json.dump(data, f)
out_file = os.path.join(self.root_dir, 'task_runner_out.json')
task_runner.load_and_run(manifest, server, 3600., time.time(), out_file)
expected = {
u'exit_code': 1,
u'hard_timeout': False,
u'io_timeout': False,
u'must_signal_internal_failure': None,
u'version': task_runner.OUT_VERSION,
}
with open(out_file, 'rb') as f:
self.assertEqual(expected, json.load(f))
def _load_and_run(self, manifest):
# Dot not mock time since this test class is testing timeouts.
server = xsrf_client.XsrfRemote('https://localhost:1/')
in_file = os.path.join(self.work_dir, 'manifest.json')
with open(in_file, 'wb') as f:
json.dump(manifest, f)
out_file = os.path.join(self.work_dir, 'task_summary.json')
task_runner.load_and_run(in_file, server, 3600., time.time(), out_file)
with open(out_file, 'rb') as f:
return json.load(f)
def _load_and_run(self, manifest):
# Dot not mock time since this test class is testing timeouts.
server = xsrf_client.XsrfRemote('https://localhost:1/')
in_file = os.path.join(self.work_dir, 'manifest.json')
with open(in_file, 'wb') as f:
json.dump(manifest, f)
out_file = os.path.join(self.work_dir, 'task_summary.json')
task_runner.load_and_run(in_file, server, 3600., time.time(), out_file)
with open(out_file, 'rb') as f:
return json.load(f)
def _load_and_run(self, manifest):
# Dot not mock time since this test class is testing timeouts.
server = 'https://localhost:1'
in_file = os.path.join(self.work_dir, 'task_runner_in.json')
with open(in_file, 'wb') as f:
json.dump(manifest, f)
out_file = os.path.join(self.work_dir, 'task_runner_out.json')
task_runner.load_and_run(in_file, server, 3600., time.time(), out_file, 1)
with open(out_file, 'rb') as f:
return json.load(f)
def test_load_and_run_isolated(self):
self.expected_requests([])
server = 'https://localhost:1'
def run_command(
swarming_server, task_details, work_dir, cost_usd_hour, start,
min_free_space):
self.assertEqual(server, swarming_server)
# Necessary for OSX.
self.assertEqual(
os.path.realpath(self.work_dir), os.path.realpath(work_dir))
self.assertTrue(isinstance(task_details, task_runner.TaskDetails))
self.assertEqual(3600., cost_usd_hour)
self.assertEqual(time.time(), start)
self.assertEqual(1, min_free_space)
return {
u'exit_code': 0,
u'hard_timeout': False,
u'io_timeout': False,
u'must_signal_internal_failure': None,
u'version': task_runner.OUT_VERSION,
}
self.mock(task_runner, 'run_command', run_command)
manifest = os.path.join(self.root_dir, 'manifest')
with open(manifest, 'wb') as f:
data = {
'bot_id': 'localhost',
'command': None,
'env': {'d': 'e'},
'extra_args': ['foo', 'bar'],
'grace_period': 30.,
'hard_timeout': 10,
'io_timeout': 11,
'inputs_ref': {
'isolated': '123',
'isolatedserver': 'http://localhost:1',
'namespace': 'default-gzip',
},
'task_id': 23,
}
json.dump(data, f)
out_file = os.path.join(self.root_dir, 'work', 'task_runner_out.json')
task_runner.load_and_run(manifest, server, 3600., time.time(), out_file, 1)
expected = {
u'exit_code': 0,
u'hard_timeout': False,
u'io_timeout': False,
u'must_signal_internal_failure': None,
u'version': task_runner.OUT_VERSION,
}
with open(out_file, 'rb') as f:
self.assertEqual(expected, json.load(f))
def load_and_run(server_url, work_dir, manifest, auth_params_file):
"""Wraps task_runner.load_and_run() which runs a Swarming task."""
in_file = os.path.join(work_dir, 'task_runner_in.json')
with open(in_file, 'wb') as f:
json.dump(manifest, f)
out_file = os.path.join(work_dir, 'task_runner_out.json')
task_runner.load_and_run(
in_file, server_url, False, 3600., time.time(), out_file,
['--min-free-space', '1'], None, auth_params_file)
with open(out_file, 'rb') as f:
return json.load(f)
def test_load_and_run_fail(self):
requests = [
(
'https://localhost:1/f',
{},
compress_to_zip({'file3': 'content3'}),
None,
),
]
self.expected_requests(requests)
server = xsrf_client.XsrfRemote('https://localhost:1/')
runs = []
def run_command(swarming_server, task_details, work_dir, cost_usd_hour,
start, json_file):
self.assertEqual(server, swarming_server)
# Necessary for OSX.
self.assertEqual(os.path.realpath(self.work_dir), work_dir)
self.assertTrue(isinstance(task_details, task_runner.TaskDetails))
self.assertEqual(3600., cost_usd_hour)
self.assertEqual(time.time(), start)
self.assertEqual('task_summary.json', json_file)
runs.append(0)
# Fails the first, pass the second.
return 1 if len(runs) == 1 else 0
self.mock(task_runner, 'run_command', run_command)
manifest = os.path.join(self.root_dir, 'manifest')
with open(manifest, 'wb') as f:
data = {
'bot_id': 'localhost',
'command': ['a'],
'data': [('https://localhost:1/f', 'foo.zip')],
'env': {
'd': 'e'
},
'grace_period': 30.,
'hard_timeout': 10,
'io_timeout': 11,
'task_id': 23,
}
json.dump(data, f)
self.assertEqual(
False,
task_runner.load_and_run(manifest, server, 3600., time.time(),
'task_summary.json'))
self.assertEqual([0], runs)
def test_load_and_run_fail(self):
requests = [
(
'https://localhost:1/f',
{},
compress_to_zip({'file3': 'content3'}),
None,
),
]
self.expected_requests(requests)
server = xsrf_client.XsrfRemote('https://localhost:1/')
runs = []
def run_command(
swarming_server, task_details, work_dir, cost_usd_hour, start,
json_file):
self.assertEqual(server, swarming_server)
# Necessary for OSX.
self.assertEqual(os.path.realpath(self.work_dir), work_dir)
self.assertTrue(isinstance(task_details, task_runner.TaskDetails))
self.assertEqual(3600., cost_usd_hour)
self.assertEqual(time.time(), start)
self.assertEqual('task_summary.json', json_file)
runs.append(0)
# Fails the first, pass the second.
return 1 if len(runs) == 1 else 0
self.mock(task_runner, 'run_command', run_command)
manifest = os.path.join(self.root_dir, 'manifest')
with open(manifest, 'wb') as f:
data = {
'bot_id': 'localhost',
'command': ['a'],
'data': [('https://localhost:1/f', 'foo.zip')],
'env': {'d': 'e'},
'grace_period': 30.,
'hard_timeout': 10,
'io_timeout': 11,
'task_id': 23,
}
json.dump(data, f)
self.assertEqual(
False, task_runner.load_and_run(
manifest, server, 3600., time.time(), 'task_summary.json'))
self.assertEqual([0], runs)
def test_load_and_run_raw(self):
local_auth_ctx = {
'accounts': [{'id': 'a'}, {'id': 'b'}],
'default_account_id': 'a',
'rpc_port': 123,
'secret': 'abcdef',
}
FakeAuthSystem.local_auth_context = local_auth_ctx
self.mock(bot_auth, 'AuthSystem', FakeAuthSystem)
def run_command(
remote, task_details, work_dir,
cost_usd_hour, start, run_isolated_flags, bot_file, ctx_file):
self.assertTrue(remote.uses_auth) # mainly to avoid "unused arg" warning
self.assertTrue(isinstance(task_details, task_runner.TaskDetails))
# Necessary for OSX.
self.assertEqual(
os.path.realpath(self.work_dir), os.path.realpath(work_dir))
self.assertEqual(3600., cost_usd_hour)
self.assertEqual(time.time(), start)
self.assertEqual(['--min-free-space', '1'], run_isolated_flags)
self.assertEqual('/path/to/bot-file', bot_file)
with open(ctx_file, 'r') as f:
self.assertDictEqual(local_auth_ctx, json.load(f)['local_auth'])
return {
u'exit_code': 1,
u'hard_timeout': False,
u'io_timeout': False,
u'must_signal_internal_failure': None,
u'version': task_runner.OUT_VERSION,
}
self.mock(task_runner, 'run_command', run_command)
manifest = os.path.join(self.root_dir, 'manifest')
with open(manifest, 'wb') as f:
data = {
'bot_id': 'localhost',
'command': ['a'],
'env': {'d': 'e'},
'env_prefixes': {},
'extra_args': [],
'grace_period': 30.,
'hard_timeout': 10,
'io_timeout': 11,
'isolated': None,
'task_id': 23,
}
json.dump(data, f)
out_file = os.path.join(self.root_dir, 'w', 'task_runner_out.json')
task_runner.load_and_run(
manifest, 'localhost:1', False, 3600., time.time(), out_file,
['--min-free-space', '1'], '/path/to/bot-file',
'/path/to/auth-params-file')
expected = {
u'exit_code': 1,
u'hard_timeout': False,
u'io_timeout': False,
u'must_signal_internal_failure': None,
u'version': task_runner.OUT_VERSION,
}
with open(out_file, 'rb') as f:
self.assertEqual(expected, json.load(f))
def _load_and_run(self, manifest):
# Dot not mock time since this test class is testing timeouts.
server = xsrf_client.XsrfRemote('https://localhost:1/')
with open('manifest.json', 'w') as f:
json.dump(manifest, f)
return task_runner.load_and_run('manifest.json', server, 3600., time.time())