def getStderr(self, block="AUTO"):
"""Return the entire stderr for the process as a list of strings.
Parameters
----------
block : bool
Whether to block until the process has finished running.
Returns
-------
error : [str]
The list of stderror strings.
"""
# Wait for the process to finish.
if block is True:
self.wait()
elif block == "AUTO" and self._is_blocked:
self.wait()
# Append any new lines to the stdout list.
for line in _pygtail.Pygtail(self._stderr_file):
self._stderr.append(line.rstrip())
return self._stderr.copy()
def watch(self):
for line in pygtail.Pygtail(self.log_file,
offset_file=self.offset_file,
read_from_end=True):
if line:
self.logger.debug(line)
match = self.regex.search(line)
if match:
self.logger.debug('Match found {}'.format(match.group(0)))
if match and match.group(1) == 'start':
self.current_event = Event(match.group(2))
elif match and match.group(1) == 'stop':
self.current_event = None
if self.current_event:
self.logger.debug('current event {}'.format(self.current_event))
if self.current_event:
time_diff = datetime.datetime.now() - self.current_event.start_time
self.logger.debug('checking event {} again {}'.format(
self.current_event, time_diff.total_seconds()))
if time_diff.total_seconds() > 10:
self.mqtt_client.publish('{}/{}'.format(
constants.ALERT_TOPIC, self.camera_name),
payload='on')
self.logger.info('Alert threshold hit for event {}'.format(
self.current_event.id))
self.current_event = None
def stderr(self, n=10):
"""Print the last n lines of the stderr buffer.
Parameters
----------
n : int
The number of lines to print.
"""
# Ensure that the number of lines is positive.
if n < 0:
raise ValueError("The number of lines must be positive!")
# Append any new lines to the stdout list.
for line in _pygtail.Pygtail(self._stderr_file):
self._stderr.append(line.rstrip())
# Get the current number of lines.
num_lines = len(self._stderr)
# Set the line from which to start printing.
if num_lines < n:
start = 0
else:
start = num_lines - n
# Print the lines.
for x in range(start, num_lines):
print(self._stderr[x])
def tail_file(filename):
''' Tail a file using pygtail. Note: this could probably be improved '''
with make_temp_file() as offset_file:
while True:
for line in pygtail.Pygtail(filename, offset_file=offset_file):
yield line
time.sleep(1.0)
def main():
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument("--transform-func",
default="jsonlog2elasticsearch.no_transform",
help="python function to transform each json line")
parser.add_argument("--debug",
action="store_true",
help="force debug mode")
parser.add_argument("--index-override-func",
default="no",
help="python function to override the ES_INDEX value")
parser.add_argument("ES_HOST", help="ES hostname/ip")
parser.add_argument("ES_PORT", help="ES port", type=int)
parser.add_argument("ES_INDEX", help="ES index name", type=str)
parser.add_argument("LOG_PATH", help="json log file fullpath")
args = parser.parse_args()
if args.debug:
set_config(minimal_level="DEBUG")
if not os.path.isfile(args.LOG_PATH):
touch(args.LOG_PATH)
transform_func = get_transform_func(args.transform_func)
if args.index_override_func == "no":
index_func = functools.partial(default_index_func, args.ES_INDEX)
else:
index_func = get_index_func(args.index_override_func)
pygtail.Pygtail._is_new_file = patched_is_new_file
pygtail.Pygtail._update_offset_file = patched_update_offset_file
pt = pygtail.Pygtail(filename=args.LOG_PATH, read_from_end=True)
es = elasticsearch.Elasticsearch(hosts=[{
"host": args.ES_HOST,
"port": args.ES_PORT,
"use_ssl": False
}])
signal.signal(signal.SIGTERM, signal_handler)
LOG.info("started")
while RUNNING:
while True:
try:
line = pt.next()
except StopIteration:
break
except FileNotFoundError:
touch(args.LOG_PATH)
break
else:
if process(line, transform_func, index_func):
commit(es)
commit(es, True)
LOG.debug("sleeping %i seconds..." % SLEEP_AFTER_EACH_ITERATION)
time.sleep(SLEEP_AFTER_EACH_ITERATION)
LOG.info("exited")
def kafkaProducer(filename, kafka_broker, topic):
pygTail = pygtail.Pygtail(filename)
lines = pygTail.readlines()
producer = KafkaProducer(bootstrap_servers=kafka_broker,
partitioner=partitioner)
# Asynchronous by default
if lines:
for line in lines:
future = producer.send(topic,
value=bytes(line.strip(), encoding="utf-8"))
future.get()
return 1, "All sent"
return 0, "Nothing to send to Kafka"
async def corelogwatcher(self):
'''
Watches for activity on the hid-io-core log file
'''
while self.retry_task:
# Make sure there is a log file to watch
if self.core_current_log_file:
try:
for line in pygtail.Pygtail(
self.core_current_log_file,
offset_file=self.core_current_log_file_offset):
self.on_core_log_entry(line)
except Exception as err:
logger.error(err)
await asyncio.sleep(0.5)
else:
await asyncio.sleep(1)
def main():
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-l',
'--log_level',
action='count',
default=0,
help='Set logging level, multiples for more detailed.')
parser.add_argument(
'-m',
'--metrics_log',
default="/var/log/dataone/daemon/cn-process-metric.log",
help="Metrics log file to process.")
parser.add_argument(
'-o',
'--offset_file',
default="/var/log/dataone/daemon/cn-process-metric.log.offset",
help="Offset file to record log reader position.")
parser.add_argument('-j',
'--json_file',
default="/var/www/processing_metrics.json",
help="Metrics state file in JSON.")
parser.add_argument('-t',
'--text_output',
default=None,
help="Text output file (stdout)")
args = parser.parse_args()
# Setup logging verbosity
levels = [logging.WARNING, logging.INFO, logging.DEBUG]
level = levels[min(len(levels) - 1, args.log_level)]
logging.basicConfig(level=level,
format="%(asctime)s %(levelname)s %(message)s")
collation = Collator(args.json_file)
pyg = pygtail.Pygtail(args.metrics_log, offset_file=args.offset_file)
for entry in pyg:
collation.addEntry(entry)
collation.save()
fdest = sys.stdout
if args.text_output is not None:
fdest = open(args.text_output, "wt")
fdest.write(collation.asText())
collation.emitToStatsd("measure-unm-1.dataone.org")
def _update_hills_dict(self):
"""Read the HILLS file and update any records."""
# Exit if the HILLS file hasn't been created.
if not _os.path.isfile(self._hills_file):
return
# Loop over all new lines in the file.
for line in _pygtail.Pygtail(self._hills_file):
# Is this a header line. If so, store the keys.
if line[3:9] == "FIELDS":
self._hills_keys = line[10:].split()
# This is an actual data record. Update the multi-dictionary.
elif line[0] != "#":
data = [float(x) for x in line.split()]
for key, value in zip(self._hills_keys, data):
self._hills_dict[key] = value
def getGradient(self, time_series=False, block="AUTO"):
"""Get the free energy gradient.
Parameters
----------
time_series : bool
Whether to return a list of time series records.
block : bool
Whether to block until the process has finished running.
Returns
-------
gradient : float
The free energy gradient.
"""
# Wait for the process to finish.
if block is True:
self.wait()
elif block == "AUTO" and self._is_blocked:
self.wait()
# No gradient file.
if not _os.path.isfile(self._gradient_file):
return None
# Append any new lines to the gradients list.
for line in _pygtail.Pygtail(self._gradient_file):
# Ignore comments.
if line[0] != "#":
self._gradients.append(float(line.rstrip().split()[-1]))
if len(self._gradients) == 0:
return None
if time_series:
return self._gradients
else:
return self._gradients[-1]
def watch_csv_log(path):
for line in pygtail.Pygtail(path):
try:
yield parse_alert_line(line)
except:
eprint(f"Error processing CSV line: '{line}' - skipping")
def ireadlines(self):
tailer = pygtail.Pygtail(self.logfile, offset_file=self.statefile)
for line in tailer:
yield line
