def read_process_queues_and_report(outs_q, errs_q):
outputs = get_nowait_from_queue(outs_q)
for chunk in outputs:
logger.debug('Command output: %s', chunk.strip('\n'))
if chunk.strip('\n') == 'unknown':
worker.close()
raise RuntimeError(
'Phone "%s" has an unknown state. Please check device authorization and state'
% self.source)
errors = get_nowait_from_queue(errs_q)
if errors:
worker.close()
# FIXME
# keep in mind this Sierra bug:
# https://discussions.apple.com/thread/7690618
# that causes:
# RuntimeError:
# There were errors trying to test connection to the phone 0x819CE30D080A6.
# Errors :
# ['cfgutil: error: Error Domain=NSCocoaErrorDomain Code=4097
# "connection to service named com.apple.configurator.xpc.InternetService"
# UserInfo={NSDebugDescription=connection to service named com.apple.configurator.xpc.InternetService}
# FIXME
raise RuntimeError(
'There were errors trying to test connection to the phone %s. Errors :%s'
% (self.source, errors))
def _collect_data(self, end=False):
"""
Collect data, cache it and send to listeners
"""
data = get_nowait_from_queue(self.results)
stats = get_nowait_from_queue(self.stats_results)
logger.debug("Data timestamps: %s" % [d.get('ts') for d in data])
logger.debug("Stats timestamps: %s" % [d.get('ts') for d in stats])
for item in data:
ts = item['ts']
if ts in self.stat_cache:
# send items
data_item = item
stat_item = self.stat_cache.pop(ts)
self.__notify_listeners(data_item, stat_item)
else:
self.data_cache[ts] = item
for item in stats:
ts = item['ts']
if ts in self.data_cache:
# send items
data_item = self.data_cache.pop(ts)
stat_item = item
self.__notify_listeners(data_item, stat_item)
else:
self.stat_cache[ts] = item
if end and len(self.data_cache) > 0:
logger.info('Timestamps without stats:')
for ts, data_item in sorted(self.data_cache.items(),
key=lambda i: i[0]):
logger.info(ts)
self.__notify_listeners(data_item,
StatsReader.stats_item(ts, 0, 0))
def read_process_queues_and_report(outs_q, errs_q):
outputs = get_nowait_from_queue(outs_q)
for chunk in outputs:
logger.debug('Command \'%s\' output: %s', cmd, chunk.strip())
errors = get_nowait_from_queue(errs_q)
for err_chunk in errors:
logger.warning('Errors in command \'%s\' output: %s', cmd, err_chunk.strip())
def _collect_data(self, end=False):
"""
Collect data, cache it and send to listeners
"""
data = get_nowait_from_queue(self.results)
stats = get_nowait_from_queue(self.stats_results)
logger.debug("Data timestamps: %s" % [d.get('ts') for d in data])
logger.debug("Stats timestamps: %s" % [d.get('ts') for d in stats])
for item in data:
ts = item['ts']
if ts in self.stat_cache:
# send items
data_item = item
stat_item = self.stat_cache.pop(ts)
self.__notify_listeners(data_item, stat_item)
else:
self.data_cache[ts] = item
for item in stats:
ts = item['ts']
if ts in self.data_cache:
# send items
data_item = self.data_cache.pop(ts)
stat_item = item
self.__notify_listeners(data_item, stat_item)
else:
self.stat_cache[ts] = item
if end and len(self.data_cache) > 0:
logger.info('Timestamps without stats:')
for ts, data_item in sorted(self.data_cache.items(), key=lambda i: i[0]):
logger.info(ts)
self.__notify_listeners(data_item, StatsReader.stats_item(ts, 0, 0))
def _read_chunk(self):
data = get_nowait_from_queue(self.source)
if not data:
time.sleep(1)
else:
ready_to_go_chunks = []
for chunk in data:
if isinstance(chunk, bytes):
chunk = chunk.decode(
'utf-8') # not sure if this is a good practice
match = self.log_fmt_regexp.match(chunk)
# we need this for multiline log entries concatenation
if match:
if not self.buffer:
self.buffer.append(match.groupdict())
else:
ready_to_go_chunk = self.buffer.pop(0)
self.buffer.append(match.groupdict())
ready_to_go_chunks.append(ready_to_go_chunk)
else:
if not self.buffer:
logger.warning(
'Trash data in logs, dropped data: \n%s', chunk)
else:
self.buffer[0][
'value'] = self.buffer[0]['value'] + str(chunk)
return ready_to_go_chunks
def run(self):
while not self._interrupted.is_set():
self.data_buffer = {}
self.metrics_buffer = {}
data = get_nowait_from_queue(self.source)
for df, metric in data:
if metric.type in self.data_buffer:
# append df to existing buffer
self.data_buffer[metric.type] = pd.concat(
[self.data_buffer[metric.type], df])
else:
# create buffer for new metric type
self.data_buffer[metric.type] = df
self.metrics_buffer[metric.type] = metric
if self._interrupted.is_set():
break
logger.debug('Buffer after routing: %s', self.data_buffer)
for client in self.clients:
for type_ in self.data_buffer:
client.put(self.data_buffer[type_],
self.metrics_buffer[type_])
if self._interrupted.is_set():
break
time.sleep(1)
self._finished.set()
def read_process_queues_and_report(outs_q, errs_q):
outputs = get_nowait_from_queue(outs_q)
for chunk in outputs:
logger.debug('Command output: %s', chunk.strip('\n'))
if chunk.strip('\n') == 'unknown':
worker.close()
raise RuntimeError(
'Phone "%s" has an unknown state. Please check device authorization and state'
% self.source)
errors = get_nowait_from_queue(errs_q)
if errors:
worker.close()
raise RuntimeError(
'There were errors trying to test connection to the phone %s. Errors :%s'
% (self.source, errors))
def __get_from_queue_prepare_and_send(self):
pending_batch = self.__prepare_batch_of_chunks(
get_nowait_from_queue(self.uploader.inner_queue))
for type_ in self.uploader.data_types_to_tables:
if pending_batch[type_]:
prepared_body = "".join(key for key in pending_batch[type_])
url = "{addr}/?query={query}".format(
addr=self.uploader.addr,
query="INSERT INTO {db}.{type} FORMAT TSV".format(
db=self.uploader.dbname,
type=self.uploader.data_types_to_tables[type_]))
try:
send_chunk(url, prepared_body)
except RetryError:
logger.warning(
'Failed to send chunk via uploader. Dropped')
logger.debug(
'Failed to send chunk via uploader. Dropped: %s %s',
url, prepared_body)
def _read_chunk(self):
data = get_nowait_from_queue(self.source)
if not data:
time.sleep(1)
else:
ready_to_go_chunks = []
for chunk in data:
match = self.log_fmt_regexp.match(chunk)
# we need this for multiline log entries concatenation
if match:
if not self.buffer:
self.buffer.append(match.groupdict())
else:
ready_to_go_chunk = self.buffer.pop(0)
self.buffer.append(match.groupdict())
ready_to_go_chunks.append(ready_to_go_chunk)
else:
if not self.buffer:
logger.warn('Trash data in logs, dropped data: \n%s',
chunk)
else:
self.buffer[0][
'value'] = self.buffer[0]['value'] + str(chunk)
return ready_to_go_chunks
def get(self):
while True:
yield get_nowait_from_queue(self.incoming_session_data)
if self.finished:
break
