def test_ensures_same_server(self):
self.mock(logging, 'error', lambda *_: None)
# Two shard results, attempt to use different servers.
actual_calls = []
self.mock(
isolateserver, 'fetch_isolated',
lambda *args: actual_calls.append(args))
data = [
gen_result_response(
outputs_ref={
'isolatedserver': 'https://server1',
'namespace': 'namespace',
'isolated':'hash1',
}),
gen_result_response(
outputs_ref={
'isolatedserver': 'https://server2',
'namespace': 'namespace',
'isolated':'hash1',
}),
]
# Feed them to collector.
collector = swarming.TaskOutputCollector(self.tempdir, 2)
for index, result in enumerate(data):
collector.process_shard_result(index, result)
collector.finalize()
# Only first fetch is made, second one is ignored.
self.assertEqual(1, len(actual_calls))
isolated_hash, storage, _, outdir, _ = actual_calls[0]
self.assertEqual(
('hash1', os.path.join(self.tempdir, '0')),
(isolated_hash, outdir))
self.assertEqual('https://server1', storage.location)
def test_collect_multi(self):
actual_calls = []
def fetch_isolated(isolated_hash, storage, cache, outdir, use_symlinks):
self.assertIs(storage.__class__, isolateserver.Storage)
self.assertIs(cache.__class__, isolateserver.MemoryCache)
# Ensure storage is pointing to required location.
self.assertEqual('https://localhost:2', storage.location)
self.assertEqual('default', storage.namespace)
self.assertEqual(False, use_symlinks)
actual_calls.append((isolated_hash, outdir))
self.mock(isolateserver, 'fetch_isolated', fetch_isolated)
collector = swarming.TaskOutputCollector(self.tempdir, 2)
for index in xrange(2):
collector.process_shard_result(
index,
gen_result_response(
outputs_ref={
'isolated': str(index) * 40,
'isolatedserver': 'https://localhost:2',
'namespace': 'default',
}))
summary = collector.finalize()
expected_calls = [
('0'*40, os.path.join(self.tempdir, '0')),
('1'*40, os.path.join(self.tempdir, '1')),
]
self.assertEqual(expected_calls, actual_calls)
# Ensure collected summary is correct.
outputs_refs = [
{
'isolated': '0'*40,
'isolatedserver': 'https://localhost:2',
'namespace': 'default',
'view_url':
'https://localhost:2/browse?namespace=default&hash=' + '0'*40,
},
{
'isolated': '1'*40,
'isolatedserver': 'https://localhost:2',
'namespace': 'default',
'view_url':
'https://localhost:2/browse?namespace=default&hash=' + '1'*40,
},
]
expected = {
'shards': [gen_result_response(outputs_ref=o) for o in outputs_refs],
}
self.assertEqual(expected, summary)
# Ensure summary dumped to a file is correct as well.
with open(os.path.join(self.tempdir, 'summary.json'), 'r') as f:
summary_dump = json.load(f)
self.assertEqual(expected, summary_dump)
def test_works(self):
# Output logs of shards.
logs = [
gen_run_isolated_out_hack_log('https://server', 'namespace', 'hash1'),
gen_run_isolated_out_hack_log('https://server', 'namespace', 'hash2'),
SWARM_OUTPUT_SUCCESS,
]
# Feed three shard results to collector, last one without output files.
collector = swarming.TaskOutputCollector(
self.tempdir, 'task/name', len(logs))
for index, log in enumerate(logs):
collector.process_shard_result(index, gen_data(log, '0, 0'))
collector.finalize()
# Ensure it fetches the files from first two shards only.
expected_calls = [
('hash1', None, None, os.path.join(self.tempdir, '0'), False),
('hash2', None, None, os.path.join(self.tempdir, '1'), False),
]
self.assertEqual(len(expected_calls), len(self.fetch_isolated_calls))
storage_instances = set()
for expected, used in zip(expected_calls, self.fetch_isolated_calls):
isolated_hash, storage, cache, outdir, require_command = used
storage_instances.add(storage)
# Compare everything but |storage| and |cache| (use None in their place).
self.assertEqual(
expected, (isolated_hash, None, None, outdir, require_command))
# Ensure cache is set.
self.assertTrue(cache)
# Only one instance of Storage should be used.
self.assertEqual(1, len(storage_instances))
# Ensure storage is pointing to required location.
storage = storage_instances.pop()
self.assertEqual('https://server', storage.location)
self.assertEqual('namespace', storage.namespace)
# Ensure summary dump is correct.
with open(os.path.join(self.tempdir, 'summary.json'), 'r') as f:
summary = json.load(f)
expected_summary = {
'task_name': 'task/name',
'shards': [
gen_data(log, '0, 0') for index, log in enumerate(logs)
]
}
self.assertEqual(expected_summary, summary)
def test_ensures_same_server(self):
# Two shard results, attempt to use different servers.
data = [
gen_data(
gen_run_isolated_out_hack_log('https://server1', 'namespace', 'hash1'),
'0, 0'),
gen_data(
gen_run_isolated_out_hack_log('https://server2', 'namespace', 'hash2'),
'0, 0'),
]
# Feed them to collector.
collector = swarming.TaskOutputCollector(self.tempdir, 'task/name', 2)
for index, result in enumerate(data):
collector.process_shard_result(index, result)
collector.finalize()
# Only first fetch is made, second one is ignored.
self.assertEqual(1, len(self.fetch_isolated_calls))
isolated_hash, storage, _, outdir, _ = self.fetch_isolated_calls[0]
self.assertEqual(
('hash1', os.path.join(self.tempdir, '0')),
(isolated_hash, outdir))
self.assertEqual('https://server1', storage.location)