def test_get_files_for_tasks_all():
mock_task = asynctest.MagicMock(spec=task.Task)
mock_file = Mock()
mock_file.exists = asynctest.CoroutineMock(return_value=True)
mock_task.file.return_value = mock_file
files = cluster.get_files_for_tasks([mock_task], ["myfile"], 1)
files = a_sync.block(aiter_to_list, files)
assert files == [mock_file]
def test_get_files_for_tasks_all():
mock_task = MagicMock()
mock_file = Mock()
mock_file.exists.return_value = True
mock_task.file.return_value = mock_file
files = cluster.get_files_for_tasks([mock_task], ['myfile'], 1)
files = list(files)
assert files == [mock_file]
def test_get_files_for_tasks_all():
attrs = {'id': 'foo'}
mock_task = MagicMock()
mock_task.__getitem__.side_effect = lambda x: attrs[x]
mock_file = Mock()
mock_file.exists.return_value = True
mock_task.file.return_value = mock_file
files = cluster.get_files_for_tasks([mock_task], ['myfile'], 1)
files = list(files)
assert files == [mock_file]
def test_get_files_for_tasks_all():
attrs = {'id': 'foo'}
mock_task = MagicMock()
mock_task.__getitem__.side_effect = lambda x: attrs[x]
mock_file = Mock()
mock_file.exists.return_value = True
mock_task.file.return_value = mock_file
files = cluster.get_files_for_tasks([mock_task], ['myfile'], 1)
files = list(files)
assert files == [mock_file]
def test_get_files_for_tasks_some():
mock_task = MagicMock()
mock_file = Mock()
mock_file_2 = Mock()
mock_file.exists.side_effect = [False, True]
mock_task.file.side_effect = [mock_file, mock_file_2]
files = cluster.get_files_for_tasks([mock_task], ['myfile', 'myotherfile'],
1)
files = list(files)
assert files == [mock_file_2]
def test_get_files_for_tasks_no_files():
attrs = {'id': 'foo'}
mock_task = MagicMock()
mock_task.__getitem__.side_effect = lambda x: attrs[x]
mock_file = Mock()
mock_file.exists.return_value = False
mock_task.file.return_value = mock_file
files = cluster.get_files_for_tasks([mock_task], ['myfile'], 1)
with raises(exceptions.FileNotFoundForTaskException) as excinfo:
files = list(files)
assert 'None of the tasks in foo contain the files in list myfile' in str(excinfo.value)
def test_get_files_for_tasks_some():
attrs = {'id': 'foo'}
mock_task = MagicMock()
mock_task.__getitem__.side_effect = lambda x: attrs[x]
mock_file = Mock()
mock_file_2 = Mock()
mock_file.exists.side_effect = [False, True]
mock_task.file.side_effect = [mock_file, mock_file_2]
files = cluster.get_files_for_tasks([mock_task], ['myfile', 'myotherfile'], 1)
files = list(files)
assert files == [mock_file_2]
def test_get_files_for_tasks_some():
mock_task = asynctest.MagicMock(spec=task.Task)
mock_file = Mock()
mock_file_2 = Mock()
mock_file.exists = asynctest.CoroutineMock(return_value=False)
mock_file_2.exists = asynctest.CoroutineMock(return_value=True)
mock_task.file.side_effect = [mock_file, mock_file_2]
files = cluster.get_files_for_tasks([mock_task], ["myfile", "myotherfile"],
1)
files = a_sync.block(aiter_to_list, files)
assert files == [mock_file_2]
async def format_stdstreams_tail_for_task(task, get_short_task_id, nlines=10):
"""Returns the formatted "tail" of stdout/stderr, for a given a task.
:param get_short_task_id: A function which given a
task_id returns a short task_id suitable for
printing.
"""
error_message = PaastaColors.red(
" couldn't read stdout/stderr for %s (%s)")
output = []
mesos_cli_config = get_mesos_config()
try:
fobjs = await aiter_to_list(
cluster.get_files_for_tasks(
task_list=[task],
file_list=['stdout', 'stderr'],
max_workers=mesos_cli_config["max_workers"],
))
fobjs.sort(key=lambda fobj: fobj.path, reverse=True)
if not fobjs:
output.append(
PaastaColors.blue(" no stdout/stderrr for %s" %
get_short_task_id(task['id'])))
return output
for fobj in fobjs:
output.append(
PaastaColors.blue(" {} tail for {}".format(
fobj.path, get_short_task_id(task['id']))))
# read nlines, starting from EOF
tail = []
lines_seen = 0
if nlines > 0:
async for line in fobj._readlines_reverse():
tail.append(line)
lines_seen += 1
if lines_seen >= nlines:
break
# reverse the tail, so that EOF is at the bottom again
if tail:
output.extend(tail[::-1])
output.append(PaastaColors.blue(" %s EOF" % fobj.path))
except (
mesos_exceptions.MasterNotAvailableException,
mesos_exceptions.SlaveDoesNotExist,
mesos_exceptions.TaskNotFoundException,
mesos_exceptions.FileNotFoundForTaskException,
) as e:
output.append(error_message % (get_short_task_id(task['id']), str(e)))
except TimeoutError:
output.append(error_message %
(get_short_task_id(task['id']), 'timeout'))
return output
def test_get_files_for_tasks_some():
attrs = {'id': 'foo'}
mock_task = MagicMock()
mock_task.__getitem__.side_effect = lambda x: attrs[x]
mock_file = Mock()
mock_file_2 = Mock()
mock_file.exists.side_effect = [False, True]
mock_task.file.side_effect = [mock_file, mock_file_2]
files = cluster.get_files_for_tasks([mock_task], ['myfile', 'myotherfile'],
1)
files = list(files)
assert files == [mock_file_2]
def test_get_files_for_tasks_no_files():
attrs = {'id': 'foo'}
mock_task = MagicMock()
mock_task.__getitem__.side_effect = lambda x: attrs[x]
mock_file = Mock()
mock_file.exists.return_value = False
mock_task.file.return_value = mock_file
files = cluster.get_files_for_tasks([mock_task], ['myfile'], 1)
with raises(exceptions.FileNotFoundForTaskException) as excinfo:
files = list(files)
assert 'None of the tasks in foo contain the files in list myfile' in str(
excinfo.value)
def test_get_files_for_tasks_no_files():
attrs = {"id": "foo"}
mock_task = asynctest.MagicMock(spec=task.Task)
mock_task.__getitem__.side_effect = lambda x: attrs[x]
mock_file = Mock()
mock_file.exists = asynctest.CoroutineMock(return_value=False)
mock_task.file.return_value = mock_file
files = cluster.get_files_for_tasks([mock_task], ["myfile"], 1)
with raises(exceptions.FileNotFoundForTaskException) as excinfo:
files = a_sync.block(aiter_to_list, files)
assert "None of the tasks in foo contain the files in list myfile" in str(
excinfo.value)
def format_stdstreams_tail_for_task(task, get_short_task_id, nlines=10):
"""Returns the formatted "tail" of stdout/stderr, for a given a task.
:param get_short_task_id: A function which given a
task_id returns a short task_id suitable for
printing.
"""
error_message = PaastaColors.red(
" couldn't read stdout/stderr for %s (%s)")
output = []
mesos_cli_config = get_mesos_config()
try:
fobjs = list(
cluster.get_files_for_tasks(
task_list=[task],
file_list=['stdout', 'stderr'],
max_workers=mesos_cli_config["max_workers"],
))
fobjs.sort(key=lambda fobj: fobj.path, reverse=True)
if not fobjs:
output.append(
PaastaColors.blue(" no stdout/stderrr for %s" %
get_short_task_id(task['id'])))
return output
for fobj in fobjs:
output.append(
PaastaColors.blue(" %s tail for %s" %
(fobj.path, get_short_task_id(task['id']))))
# read nlines, starting from EOF
# mesos.cli is smart and can efficiently read a file backwards
reversed_file = reversed(fobj)
tail = []
for _ in range(nlines):
line = next(reversed_file, None)
if line is None:
break
tail.append(line)
# reverse the tail, so that EOF is at the bottom again
if tail:
output.extend(tail[::-1])
output.append(PaastaColors.blue(" %s EOF" % fobj.path))
except (
mesos_exceptions.MasterNotAvailableException,
mesos_exceptions.SlaveDoesNotExist,
mesos_exceptions.TaskNotFoundException,
mesos_exceptions.FileNotFoundForTaskException,
) as e:
output.append(error_message % (get_short_task_id(task['id']), str(e)))
except TimeoutError:
output.append(error_message %
(get_short_task_id(task['id']), 'timeout'))
return output
async def get_tail_lines_for_mesos_task(
task: Task, get_short_task_id: Callable[[str], str],
num_tail_lines: int) -> MutableMapping[str, Sequence[str]]:
tail_lines_dict: MutableMapping[str, Sequence[str]] = {}
mesos_cli_config = get_mesos_config()
try:
fobjs = await aiter_to_list(
cluster.get_files_for_tasks(
task_list=[task],
file_list=["stdout", "stderr"],
max_workers=mesos_cli_config["max_workers"],
))
if not fobjs:
return {"stdout": [], "stderr": []}
fobjs.sort(key=lambda fobj: fobj.path, reverse=True)
for fobj in fobjs:
# read nlines, starting from EOF
tail = []
lines_seen = 0
async for line in fobj._readlines_reverse():
tail.append(line)
lines_seen += 1
if lines_seen >= num_tail_lines:
break
# reverse the tail, so that EOF is at the bottom again
tail_lines_dict[fobj.path] = tail[::-1]
except (
mesos_exceptions.MasterNotAvailableException,
mesos_exceptions.SlaveDoesNotExist,
mesos_exceptions.TaskNotFoundException,
mesos_exceptions.FileNotFoundForTaskException,
TimeoutError,
) as e:
short_task_id = get_short_task_id(task["id"])
error_name = e.__class__.__name__
return {
"error_message":
f"couldn't read stdout/stderr for {short_task_id} ({error_name})"
}
return tail_lines_dict
def format_stdstreams_tail_for_task(task, get_short_task_id, nlines=10):
"""Returns the formatted "tail" of stdout/stderr, for a given a task.
:param get_short_task_id: A function which given a
task_id returns a short task_id suitable for
printing.
"""
error_message = PaastaColors.red(" couldn't read stdout/stderr for %s (%s)")
output = []
mesos_cli_config = get_mesos_config()
try:
fobjs = list(cluster.get_files_for_tasks(
task_list=[task],
file_list=['stdout', 'stderr'],
max_workers=mesos_cli_config["max_workers"]
))
fobjs.sort(key=lambda fobj: fobj.path, reverse=True)
if not fobjs:
output.append(PaastaColors.blue(" no stdout/stderrr for %s" % get_short_task_id(task['id'])))
return output
for fobj in fobjs:
output.append(PaastaColors.blue(" %s tail for %s" % (fobj.path, get_short_task_id(task['id']))))
# read nlines, starting from EOF
# mesos.cli is smart and can efficiently read a file backwards
reversed_file = reversed(fobj)
tail = []
for _ in xrange(nlines):
line = next(reversed_file, None)
if line is None:
break
tail.append(line)
# reverse the tail, so that EOF is at the bottom again
if tail:
output.extend(tail[::-1])
output.append(PaastaColors.blue(" %s EOF" % fobj.path))
except (mesos_exceptions.MasterNotAvailableException,
mesos_exceptions.SlaveDoesNotExist,
mesos_exceptions.TaskNotFoundException,
mesos_exceptions.FileNotFoundForTaskException) as e:
output.append(error_message % (get_short_task_id(task['id']), e.message))
except TimeoutError:
output.append(error_message % (get_short_task_id(task['id']), 'timeout'))
return output
