def _stream_files(logger, fn, mesos_files, line_prefixes):
"""Apply `fn` in parallel to each file in `mesos_files`. `fn` must
return a list of strings, and these strings are then printed
serially as separate lines.
"""
reachable_files = list(mesos_files)
# TODO switch to map
for job, mesos_file in util.stream(fn, mesos_files):
try:
lines = job.result()
except DCOSException:
# The read function might throw an exception if read.json
# is unavailable, or if the file doesn't exist in the
# sandbox. In any case, we silently remove the file and
# continue.
reachable_files.remove(mesos_file)
continue
if lines:
_output(logger,
str(mesos_file),
lines,
line_prefixes)
return reachable_files
def _load_slaves_state(slaves):
"""Fetch each slave's state.json in parallel, and return the reachable
slaves.
"""
reachable_slaves = []
for job, slave in util.stream(lambda slave: slave.state(), slaves):
try:
job.result()
reachable_slaves.append(slave)
# job.result() will throw a DCOSException when it is unable to contact
# a given slave for any reason
except DCOSException:
pass
return reachable_slaves
def _stream_files(curr_header, fn, mesos_files):
"""Apply `fn` in parallel to each file in `mesos_files`. `fn` must
return a list of strings, and these strings are then printed
serially as separate lines.
`curr_header` is the most recently printed header. It's used to
group lines. Each line has an associated header (e.g. a string
representation of the MesosFile it was read from), and we only
print the header before printing a line with a different header
than the previous line. This effectively groups lines together
when the have the same header.
:param curr_header: Most recently printed header
:type curr_header: str
:param fn: function that reads a sequence of lines from a MesosFile
:type fn: MesosFile -> [str]
:param mesos_files: files to read
:type mesos_files: [MesosFile]
:returns: Returns the most recently printed header, and a list of
files that are still reachable. Once we detect a file is
unreachable, we stop trying to read from it.
:rtype: (str, [MesosFile])
"""
reachable_files = list(mesos_files)
# TODO switch to map
for job, mesos_file in util.stream(fn, mesos_files):
try:
lines = job.result()
except DCOSException as e:
# The read function might throw an exception if read.json
# is unavailable, or if the file doesn't exist in the
# sandbox. In any case, we silently remove the file and
# continue.
logger.exception("Error reading file: {}".format(e))
reachable_files.remove(mesos_file)
continue
if lines:
curr_header = _output(curr_header,
len(reachable_files) > 1,
six.text_type(mesos_file),
lines)
return curr_header, reachable_files
def _stream_files(curr_header, fn, mesos_files):
"""Apply `fn` in parallel to each file in `mesos_files`. `fn` must
return a list of strings, and these strings are then printed
serially as separate lines.
`curr_header` is the most recently printed header. It's used to
group lines. Each line has an associated header (e.g. a string
representation of the MesosFile it was read from), and we only
print the header before printing a line with a different header
than the previous line. This effectively groups lines together
when the have the same header.
:param curr_header: Most recently printed header
:type curr_header: str
:param fn: function that reads a sequence of lines from a MesosFile
:type fn: MesosFile -> [str]
:param mesos_files: files to read
:type mesos_files: [MesosFile]
:returns: Returns the most recently printed header, and a list of
files that are still reachable. Once we detect a file is
unreachable, we stop trying to read from it.
:rtype: (str, [MesosFile])
"""
reachable_files = list(mesos_files)
# TODO switch to map
for job, mesos_file in util.stream(fn, mesos_files):
try:
lines = job.result()
except DCOSException as e:
# The read function might throw an exception if read.json
# is unavailable, or if the file doesn't exist in the
# sandbox. In any case, we silently remove the file and
# continue.
logger.warning("Error reading file: {}".format(e))
reachable_files.remove(mesos_file)
continue
if lines:
curr_header = _output(curr_header,
len(reachable_files) > 1,
str(mesos_file),
lines)
return curr_header, reachable_files
def _load_slaves_state(slaves):
"""Fetch each slave's state.json in parallel, and return the reachable
slaves.
:param slaves: slaves to fetch
:type slaves: [MesosSlave]
:returns: MesosSlave objects that were successfully reached
:rtype: [MesosSlave]
"""
reachable_slaves = []
for job, slave in util.stream(lambda slave: slave.state(), slaves):
try:
job.result()
reachable_slaves.append(slave)
except DCOSException as e:
emitter.publish(
DefaultError('Error accessing slave: {0}'.format(e)))
return reachable_slaves
def _load_slaves_state(slaves):
"""Fetch each slave's state.json in parallel, and return the reachable
slaves.
:param slaves: slaves to fetch
:type slaves: [MesosSlave]
:returns: MesosSlave objects that were successfully reached
:rtype: [MesosSlave]
"""
reachable_slaves = []
for job, slave in util.stream(lambda slave: slave.state(), slaves):
try:
job.result()
reachable_slaves.append(slave)
except DCOSException as e:
emitter.publish(
DefaultError('Error accessing slave: {0}'.format(e)))
return reachable_slaves
