def test_get_files_for_tasks_all(): mock_task = asynctest.MagicMock(spec=task.Task) mock_file = Mock() mock_file.exists = asynctest.CoroutineMock(return_value=True) mock_task.file.return_value = mock_file files = cluster.get_files_for_tasks([mock_task], ["myfile"], 1) files = a_sync.block(aiter_to_list, files) assert files == [mock_file]
def test_get_files_for_tasks_all(): mock_task = MagicMock() mock_file = Mock() mock_file.exists.return_value = True mock_task.file.return_value = mock_file files = cluster.get_files_for_tasks([mock_task], ['myfile'], 1) files = list(files) assert files == [mock_file]
def test_get_files_for_tasks_all(): attrs = {'id': 'foo'} mock_task = MagicMock() mock_task.__getitem__.side_effect = lambda x: attrs[x] mock_file = Mock() mock_file.exists.return_value = True mock_task.file.return_value = mock_file files = cluster.get_files_for_tasks([mock_task], ['myfile'], 1) files = list(files) assert files == [mock_file]
def test_get_files_for_tasks_some(): mock_task = MagicMock() mock_file = Mock() mock_file_2 = Mock() mock_file.exists.side_effect = [False, True] mock_task.file.side_effect = [mock_file, mock_file_2] files = cluster.get_files_for_tasks([mock_task], ['myfile', 'myotherfile'], 1) files = list(files) assert files == [mock_file_2]
def test_get_files_for_tasks_no_files(): attrs = {'id': 'foo'} mock_task = MagicMock() mock_task.__getitem__.side_effect = lambda x: attrs[x] mock_file = Mock() mock_file.exists.return_value = False mock_task.file.return_value = mock_file files = cluster.get_files_for_tasks([mock_task], ['myfile'], 1) with raises(exceptions.FileNotFoundForTaskException) as excinfo: files = list(files) assert 'None of the tasks in foo contain the files in list myfile' in str(excinfo.value)
def test_get_files_for_tasks_some(): attrs = {'id': 'foo'} mock_task = MagicMock() mock_task.__getitem__.side_effect = lambda x: attrs[x] mock_file = Mock() mock_file_2 = Mock() mock_file.exists.side_effect = [False, True] mock_task.file.side_effect = [mock_file, mock_file_2] files = cluster.get_files_for_tasks([mock_task], ['myfile', 'myotherfile'], 1) files = list(files) assert files == [mock_file_2]
def test_get_files_for_tasks_some(): mock_task = asynctest.MagicMock(spec=task.Task) mock_file = Mock() mock_file_2 = Mock() mock_file.exists = asynctest.CoroutineMock(return_value=False) mock_file_2.exists = asynctest.CoroutineMock(return_value=True) mock_task.file.side_effect = [mock_file, mock_file_2] files = cluster.get_files_for_tasks([mock_task], ["myfile", "myotherfile"], 1) files = a_sync.block(aiter_to_list, files) assert files == [mock_file_2]
async def format_stdstreams_tail_for_task(task, get_short_task_id, nlines=10): """Returns the formatted "tail" of stdout/stderr, for a given a task. :param get_short_task_id: A function which given a task_id returns a short task_id suitable for printing. """ error_message = PaastaColors.red( " couldn't read stdout/stderr for %s (%s)") output = [] mesos_cli_config = get_mesos_config() try: fobjs = await aiter_to_list( cluster.get_files_for_tasks( task_list=[task], file_list=['stdout', 'stderr'], max_workers=mesos_cli_config["max_workers"], )) fobjs.sort(key=lambda fobj: fobj.path, reverse=True) if not fobjs: output.append( PaastaColors.blue(" no stdout/stderrr for %s" % get_short_task_id(task['id']))) return output for fobj in fobjs: output.append( PaastaColors.blue(" {} tail for {}".format( fobj.path, get_short_task_id(task['id'])))) # read nlines, starting from EOF tail = [] lines_seen = 0 if nlines > 0: async for line in fobj._readlines_reverse(): tail.append(line) lines_seen += 1 if lines_seen >= nlines: break # reverse the tail, so that EOF is at the bottom again if tail: output.extend(tail[::-1]) output.append(PaastaColors.blue(" %s EOF" % fobj.path)) except ( mesos_exceptions.MasterNotAvailableException, mesos_exceptions.SlaveDoesNotExist, mesos_exceptions.TaskNotFoundException, mesos_exceptions.FileNotFoundForTaskException, ) as e: output.append(error_message % (get_short_task_id(task['id']), str(e))) except TimeoutError: output.append(error_message % (get_short_task_id(task['id']), 'timeout')) return output
def test_get_files_for_tasks_no_files(): attrs = {'id': 'foo'} mock_task = MagicMock() mock_task.__getitem__.side_effect = lambda x: attrs[x] mock_file = Mock() mock_file.exists.return_value = False mock_task.file.return_value = mock_file files = cluster.get_files_for_tasks([mock_task], ['myfile'], 1) with raises(exceptions.FileNotFoundForTaskException) as excinfo: files = list(files) assert 'None of the tasks in foo contain the files in list myfile' in str( excinfo.value)
def test_get_files_for_tasks_no_files(): attrs = {"id": "foo"} mock_task = asynctest.MagicMock(spec=task.Task) mock_task.__getitem__.side_effect = lambda x: attrs[x] mock_file = Mock() mock_file.exists = asynctest.CoroutineMock(return_value=False) mock_task.file.return_value = mock_file files = cluster.get_files_for_tasks([mock_task], ["myfile"], 1) with raises(exceptions.FileNotFoundForTaskException) as excinfo: files = a_sync.block(aiter_to_list, files) assert "None of the tasks in foo contain the files in list myfile" in str( excinfo.value)
def format_stdstreams_tail_for_task(task, get_short_task_id, nlines=10): """Returns the formatted "tail" of stdout/stderr, for a given a task. :param get_short_task_id: A function which given a task_id returns a short task_id suitable for printing. """ error_message = PaastaColors.red( " couldn't read stdout/stderr for %s (%s)") output = [] mesos_cli_config = get_mesos_config() try: fobjs = list( cluster.get_files_for_tasks( task_list=[task], file_list=['stdout', 'stderr'], max_workers=mesos_cli_config["max_workers"], )) fobjs.sort(key=lambda fobj: fobj.path, reverse=True) if not fobjs: output.append( PaastaColors.blue(" no stdout/stderrr for %s" % get_short_task_id(task['id']))) return output for fobj in fobjs: output.append( PaastaColors.blue(" %s tail for %s" % (fobj.path, get_short_task_id(task['id'])))) # read nlines, starting from EOF # mesos.cli is smart and can efficiently read a file backwards reversed_file = reversed(fobj) tail = [] for _ in range(nlines): line = next(reversed_file, None) if line is None: break tail.append(line) # reverse the tail, so that EOF is at the bottom again if tail: output.extend(tail[::-1]) output.append(PaastaColors.blue(" %s EOF" % fobj.path)) except ( mesos_exceptions.MasterNotAvailableException, mesos_exceptions.SlaveDoesNotExist, mesos_exceptions.TaskNotFoundException, mesos_exceptions.FileNotFoundForTaskException, ) as e: output.append(error_message % (get_short_task_id(task['id']), str(e))) except TimeoutError: output.append(error_message % (get_short_task_id(task['id']), 'timeout')) return output
async def get_tail_lines_for_mesos_task( task: Task, get_short_task_id: Callable[[str], str], num_tail_lines: int) -> MutableMapping[str, Sequence[str]]: tail_lines_dict: MutableMapping[str, Sequence[str]] = {} mesos_cli_config = get_mesos_config() try: fobjs = await aiter_to_list( cluster.get_files_for_tasks( task_list=[task], file_list=["stdout", "stderr"], max_workers=mesos_cli_config["max_workers"], )) if not fobjs: return {"stdout": [], "stderr": []} fobjs.sort(key=lambda fobj: fobj.path, reverse=True) for fobj in fobjs: # read nlines, starting from EOF tail = [] lines_seen = 0 async for line in fobj._readlines_reverse(): tail.append(line) lines_seen += 1 if lines_seen >= num_tail_lines: break # reverse the tail, so that EOF is at the bottom again tail_lines_dict[fobj.path] = tail[::-1] except ( mesos_exceptions.MasterNotAvailableException, mesos_exceptions.SlaveDoesNotExist, mesos_exceptions.TaskNotFoundException, mesos_exceptions.FileNotFoundForTaskException, TimeoutError, ) as e: short_task_id = get_short_task_id(task["id"]) error_name = e.__class__.__name__ return { "error_message": f"couldn't read stdout/stderr for {short_task_id} ({error_name})" } return tail_lines_dict
def format_stdstreams_tail_for_task(task, get_short_task_id, nlines=10): """Returns the formatted "tail" of stdout/stderr, for a given a task. :param get_short_task_id: A function which given a task_id returns a short task_id suitable for printing. """ error_message = PaastaColors.red(" couldn't read stdout/stderr for %s (%s)") output = [] mesos_cli_config = get_mesos_config() try: fobjs = list(cluster.get_files_for_tasks( task_list=[task], file_list=['stdout', 'stderr'], max_workers=mesos_cli_config["max_workers"] )) fobjs.sort(key=lambda fobj: fobj.path, reverse=True) if not fobjs: output.append(PaastaColors.blue(" no stdout/stderrr for %s" % get_short_task_id(task['id']))) return output for fobj in fobjs: output.append(PaastaColors.blue(" %s tail for %s" % (fobj.path, get_short_task_id(task['id'])))) # read nlines, starting from EOF # mesos.cli is smart and can efficiently read a file backwards reversed_file = reversed(fobj) tail = [] for _ in xrange(nlines): line = next(reversed_file, None) if line is None: break tail.append(line) # reverse the tail, so that EOF is at the bottom again if tail: output.extend(tail[::-1]) output.append(PaastaColors.blue(" %s EOF" % fobj.path)) except (mesos_exceptions.MasterNotAvailableException, mesos_exceptions.SlaveDoesNotExist, mesos_exceptions.TaskNotFoundException, mesos_exceptions.FileNotFoundForTaskException) as e: output.append(error_message % (get_short_task_id(task['id']), e.message)) except TimeoutError: output.append(error_message % (get_short_task_id(task['id']), 'timeout')) return output