def main(args):
c = Client()
s = c.read_monitor_point('diagnostics/spectra', 'dr%s' % args.beam)
t = datetime.utcfromtimestamp(s.timestamp)
s = ImageMonitorPoint(s)
s.to_file(args.output)
print(f"Saved spectra from power beam {args.beam} at {t} to {args.output}")
def main(argv):
parser = argparse.ArgumentParser(
description="Data recorder manager for slow/fast visibility data")
parser.add_argument(
'-b',
'--begin-band',
type=int,
default=1,
help='beginning dr_visibility.py band number to manage')
parser.add_argument(
'-e',
'--end-band',
type=int,
default=16,
help='ending dr_visibility.py IP band number to manage')
parser.add_argument('-l',
'--logfile',
type=str,
help='file to write logging to')
parser.add_argument('-q',
'--quick',
action='store_true',
help='run in fast visibiltiy mode')
parser.add_argument('-f',
'--fork',
action='store_true',
help='fork and run in the background')
args = parser.parse_args()
# Fork, if requested
if args.fork:
stderr = '/tmp/%s_%i.stderr' % (os.path.splitext(
os.path.basename(__file__))[0], args.port)
daemonize(stdin='/dev/null', stdout='/dev/null', stderr=stderr)
# Setup logging
log = logging.getLogger(__name__)
logFormat = logging.Formatter('%(asctime)s [%(levelname)-8s] %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
logFormat.converter = time.gmtime
if args.logfile is None:
logHandler = logging.StreamHandler(sys.stdout)
else:
logHandler = LogFileHandler(args.logfile)
logHandler.setFormatter(logFormat)
log.addHandler(logHandler)
log.setLevel(logging.DEBUG)
log.info("Starting %s with PID %i", os.path.basename(__file__),
os.getpid())
log.info("Cmdline args:")
for arg in vars(args):
log.info(" %s: %s", arg, getattr(args, arg))
# Setup the IDs to use/manage
mcs_id = 'drv'
if args.quick:
mcs_id += 'f'
else:
mcs_id += 's'
MANAGE_ID = []
for band in range(args.begin_band, args.end_band + 1):
sub_id = mcs_id + str(band)
MANAGE_ID.append(sub_id)
# Setup signal handling
shutdown_event = setup_signal_handling([])
c = Client(mcs_id)
for cmd in ('ping', 'sync', 'start', 'stop'):
cb = CommandCallbackBase(c.client)
def wrapper(value):
status = True
response = {}
for id in MANAGE_ID:
s, r = c.send_command(id, cmd, **value)
status &= s
response[id] = r
return status, response
cb.action = wrapper
c.set_command_callback(cmd, cb)
tlast = 0.0
while not shutdown_event.is_set():
if time.time() - tlast > 15:
status = "normal"
info = ""
first = True
for id in MANAGE_ID:
t0 = time.time()
svalue = c.read_monitor_point('summary', id=id)
ivalue = c.read_monitor_point('info', id=id)
if svalue is None:
svalue = MonitorPoint("timeout", timestamp=0)
if ivalue is None:
ivalue = MonitorPoint("timeout", timestamp=0)
log.info("%s -> %s (%s) at %.0f", id, svalue.value,
ivalue.value, svalue.timestamp)
age = svalue.timestamp - t0
if age > 120:
status = "timeout"
if status == "error":
status = svalue.value
elif svalue.value == "warning":
if status == "normal":
status = svalue.value
if not first:
info += "; "
info += "%s: %s (%s)" % (id, svalue.value, ivalue.value)
first = False
ts = time.time()
c.write_monitor_point('summary', status, timestamp=ts)
c.write_monitor_point('info', info, timestamp=ts)
tlast = ts
time.sleep(2)
return 0
class StatusLogger(object):
"""
Monitoring class for logging the overall status of a pipeline. This aggregates
other monitoring points of the pipeline and uses that information to compute
an overall state of the pipeline.
"""
def __init__(self, log, id, queue, shutdown_event=None, update_interval=10):
self.log = log
self.id = id
self.queue = queue
if shutdown_event is None:
shutdown_event = threading.Event()
self.shutdown_event = shutdown_event
self.update_interval = update_interval
self.client = Client(id)
def _update(self):
pass
def main(self, once=False):
"""
Main logging loop. May be run only once with the "once" keyword set to
True.
"""
while not self.shutdown_event.is_set():
# Active operation
ts = time.time()
is_active = False if self.queue.active is None else True
active_filename = None
time_left = None
if is_active:
active_filename = self.queue.active.filename
time_left = self.queue.active.stop_time - self.queue.active.utcnow()
self.client.write_monitor_point('op-type', active_filename, timestamp=ts)
self.client.write_monitor_point('op-tag', active_filename, timestamp=ts)
# TODO: Overall system system/health
# What goes into this?
# * RX rate/missing packets?
# * Block processing time?
# * Free disk space?
# * Thread check?
# * ???
missing = self.client.read_monitor_point('bifrost/rx_missing')
if missing is None:
missing = MonitorPoint(0.0)
processing = self.client.read_monitor_point('bifrost/max_process')
total = self.client.read_monitor_point('storage/active_disk_size')
free = self.client.read_monitor_point('storage/active_disk_free')
dfree = 1.0*free.value / total.value
ts = min([v.timestamp for v in (missing, processing, total, free)])
if dfree < 0.99 and missing.value < 0.01:
self.client.write_monitor_point('summary', 'normal', timestamp=ts)
self.client.write_monitor_point('info', 'A-OK', timestamp=ts)
elif dfree > 0.99 and missing.value < 0.01:
self.client.write_monitor_point('summary', 'warning', timestamp=ts)
self.client.write_monitor_point('info', "no space (%.1f%% used)" % (dfree*100.0,), timestamp=ts)
elif dfree < 0.99 and missing.value > 0.01:
self.client.write_monitor_point('summary', 'warning', timestamp=ts)
self.client.write_monitor_point('info', "missing packets (%.1f%% missing)" % (missing.value*100.0,), timestamp=ts)
else:
self.client.write_monitor_point('summary', 'error', timestamp=ts)
self.client.write_monitor_point('info', "it's bad", timestamp=ts)
# Report
self.log.debug("=== Status Report ===")
self.log.debug(" queue size: %i", len(self.queue))
self.log.debug(" active operation: %s", is_active)
if is_active:
self.log.debug(" active filename: %s", os.path.basename(active_filename))
self.log.debug(" active time remaining: %s", time_left)
self.log.debug("=== ===")
# Sleep
if once:
break
time.sleep(self.update_interval)