def test_can_merge_with_incomplete_ids(self):
# this shouldn't happen if the _interpret_*() methods are
# written correctly, but just in case
errors = [
dict(
attempt_id='attempt_201512232143_0008_r_000000_0',
hadoop_error=dict(message='BOOM'),
split=dict(path='trade_secrets.dat'),
),
dict(
attempt_id='attempt_201512232143_0008_r_000000_0',
hadoop_error=dict(message='BOOM'),
task_id='task_201512232143_0008_r_000000',
),
]
self.assertEqual(_merge_and_sort_errors(errors), [
dict(
attempt_id='attempt_201512232143_0008_r_000000_0',
hadoop_error=dict(message='BOOM'),
split=dict(path='trade_secrets.dat'),
task_id='task_201512232143_0008_r_000000',
),
])
def test_single_error(self):
error = dict(
container_id='container_1450486922681_0005_01_000003',
hadoop_error=dict(message='BOOM'),
)
self.assertEqual(_merge_and_sort_errors([error]), [error])
def test_single_error(self):
error = dict(
container_id='container_1450486922681_0005_01_000003',
hadoop_error=dict(message='BOOM'),
)
self.assertEqual(_merge_and_sort_errors([error]), [error])
def test_can_merge_with_incomplete_ids(self):
# this shouldn't happen if the _interpret_*() methods are
# written correctly, but just in case
errors = [
dict(
attempt_id='attempt_201512232143_0008_r_000000_0',
hadoop_error=dict(message='BOOM'),
split=dict(path='trade_secrets.dat'),
),
dict(
attempt_id='attempt_201512232143_0008_r_000000_0',
hadoop_error=dict(message='BOOM'),
task_id='task_201512232143_0008_r_000000',
),
]
self.assertEqual(
_merge_and_sort_errors(errors),
[
dict(
attempt_id='attempt_201512232143_0008_r_000000_0',
hadoop_error=dict(message='BOOM'),
split=dict(path='trade_secrets.dat'),
task_id='task_201512232143_0008_r_000000',
),
]
)
def test_merge_errors(self):
errors = [
dict(container_id="container_1450486922681_0005_01_000003", hadoop_error=dict(message="BOOM")),
dict( # from a different container, shouldn't be merged
container_id="container_1450486922681_0005_01_000004", hadoop_error=dict(message="bad stuff, maybe?")
),
dict(
container_id="container_1450486922681_0005_01_000003",
hadoop_error=dict(message="BOOM", path="history.jhist"),
split=dict(path="tricky_input"),
),
dict(
container_id="container_1450486922681_0005_01_000003",
hadoop_error=dict(message="BOOM\n", path="some_syslog"),
task_error=dict(message="it was probably snakes", path="some_stderr"),
),
]
self.assertEqual(
_merge_and_sort_errors(errors),
[
dict(
container_id="container_1450486922681_0005_01_000004",
hadoop_error=dict(message="bad stuff, maybe?"),
),
dict(
container_id="container_1450486922681_0005_01_000003",
hadoop_error=dict(message="BOOM\n", path="some_syslog"),
split=dict(path="tricky_input"),
task_error=dict(message="it was probably snakes", path="some_stderr"),
),
],
)
def test_merge_errors(self):
errors = [
dict(
container_id='container_1450486922681_0005_01_000003',
hadoop_error=dict(message='BOOM')
),
dict( # from a different container, shouldn't be merged
container_id='container_1450486922681_0005_01_000004',
hadoop_error=dict(message='bad stuff, maybe?')
),
dict(
container_id='container_1450486922681_0005_01_000003',
hadoop_error=dict(
message='BOOM',
path='history.jhist',
),
split=dict(path='tricky_input')
),
dict(
container_id='container_1450486922681_0005_01_000003',
hadoop_error=dict(
message='BOOM\n',
path='some_syslog',
),
task_error=dict(
message='it was probably snakes',
path='some_stderr',
),
),
]
self.assertEqual(
_merge_and_sort_errors(errors),
[
dict(
container_id='container_1450486922681_0005_01_000004',
hadoop_error=dict(message='bad stuff, maybe?')
),
dict(
container_id='container_1450486922681_0005_01_000003',
hadoop_error=dict(
message='BOOM\n',
path='some_syslog',
),
split=dict(path='tricky_input'),
task_error=dict(
message='it was probably snakes',
path='some_stderr'
),
)
])
def test_merge_errors(self):
errors = [
dict(
container_id='container_1450486922681_0005_01_000003',
hadoop_error=dict(message='BOOM')
),
dict( # from a different container, shouldn't be merged
container_id='container_1450486922681_0005_01_000004',
hadoop_error=dict(message='bad stuff, maybe?')
),
dict(
container_id='container_1450486922681_0005_01_000003',
hadoop_error=dict(
message='BOOM',
path='history.jhist',
),
split=dict(path='tricky_input')
),
dict(
container_id='container_1450486922681_0005_01_000003',
hadoop_error=dict(
message='BOOM\n',
path='some_syslog',
),
task_error=dict(
message='it was probably snakes',
path='some_stderr',
),
),
]
self.assertEqual(
_merge_and_sort_errors(errors),
[
dict(
container_id='container_1450486922681_0005_01_000004',
hadoop_error=dict(message='bad stuff, maybe?')
),
dict(
container_id='container_1450486922681_0005_01_000003',
hadoop_error=dict(
message='BOOM\n',
path='some_syslog',
),
split=dict(path='tricky_input'),
task_error=dict(
message='it was probably snakes',
path='some_stderr'
),
)
])
def test_attempt_to_container_id(self):
errors = [
dict(
attempt_id='attempt_201512232143_0008_r_000000_0',
hadoop_error=dict(message='BOOM'),
),
dict(
attempt_id='attempt_201512232143_0008_r_000000_1',
hadoop_error=dict(message='BOOM again'),
),
dict(
container_id='container_1450486922681_0005_01_000003',
task_error=dict(message='it was probably snakes'),
),
]
attempt_to_container_id = {
'attempt_201512232143_0008_r_000000_1':
'container_1450486922681_0005_01_000003',
}
self.assertEqual(
_merge_and_sort_errors(errors, attempt_to_container_id),
[
dict(
attempt_id='attempt_201512232143_0008_r_000000_1',
container_id='container_1450486922681_0005_01_000003',
hadoop_error=dict(message='BOOM again'),
task_error=dict(message='it was probably snakes'),
),
dict(
attempt_id='attempt_201512232143_0008_r_000000_0',
hadoop_error=dict(message='BOOM'),
),
],
)
def test_attempt_to_container_id(self):
errors = [
dict(
attempt_id='attempt_201512232143_0008_r_000000_0',
hadoop_error=dict(message='BOOM'),
),
dict(
attempt_id='attempt_201512232143_0008_r_000000_1',
hadoop_error=dict(message='BOOM again'),
),
dict(
container_id='container_1450486922681_0005_01_000003',
task_error=dict(message='it was probably snakes'),
),
]
attempt_to_container_id = {
'attempt_201512232143_0008_r_000000_1':
'container_1450486922681_0005_01_000003',
}
self.assertEqual(
_merge_and_sort_errors(errors, attempt_to_container_id),
[
dict(
attempt_id='attempt_201512232143_0008_r_000000_0',
hadoop_error=dict(message='BOOM'),
),
dict(
attempt_id='attempt_201512232143_0008_r_000000_1',
container_id='container_1450486922681_0005_01_000003',
hadoop_error=dict(message='BOOM again'),
task_error=dict(message='it was probably snakes'),
),
],
)
def test_empty(self):
self.assertEqual(_merge_and_sort_errors([]), [])
def test_empty(self):
self.assertEqual(_merge_and_sort_errors([]), [])
