def locate_station(args, show_geo=False):
"""Return the location of station_name or station_number as contained in args.locate.
If sub_array data exists, print subarray name."""
station, station_col = geo_location.station_name_or_number(args.locate)
v = None
db = mc.connect_to_mc_db(args)
with db.sessionmaker() as session:
sub_arrays = session.split_arrays()
for a in session.query(geo_location.GeoLocation).filter(station_col==station):
for key in sub_arrays.keys():
if a.station_name in sub_arrays[key]['Stations']:
this_sub_array = key
break
else:
this_sub_array = 'No sub-array information.'
v = [a.easting, a.northing, a.elevation, a.station_name, a.station_number, this_sub_array]
if show_geo:
if args.verbosity=='m' or args.verbosity=='h':
print('station_name: ',a.station_name)
print('\tstation_number: ',a.station_number)
print('\teasting: ',a.easting)
print('\tnorthing: ',a.northing)
print('\televation: ',a.elevation)
print('\tsub-array: ',this_sub_array,sub_arrays[this_sub_array]['Description'])
elif args.verbosity=='l':
print(a,this_sub_array)
if show_geo:
if not v and args.verbosity == 'm' or args.verbosity == 'h':
print(args.locate, ' not found.')
return v
def get_cminfo():
print 'Attempting to retreive hookup from CM database'
from hera_mc import mc, sys_handling, cm_utils
parser = mc.get_mc_argument_parser()
args = parser.parse_args(
args=[]) # args=[] to throw away command line arguments
db = mc.connect_to_mc_db(args)
session = db.sessionmaker()
h = sys_handling.Handling(session)
return h.get_cminfo_correlator()
def handle(self, *args, **options):
"""Perform necessary calculation and inser antennas in to DB."""
mc_args = Namespace()
mc_args.mc_db_name = options["mc_db_name"]
mc_args.mc_config_path = options["mc_config_path"]
db = mc.connect_to_mc_db(args=mc_args)
antpos = np.genfromtxt(
os.path.join(mc.data_path, "HERA_350.txt"),
usecols=(0, 1, 2, 3),
dtype={
"names": ("ANTNAME", "EAST", "NORTH", "UP"),
"formats": ("<U5", "<f8", "<f8", "<f8"),
},
encoding=None,
)
antnames = antpos["ANTNAME"]
inds = [int(j[2:]) for j in antnames]
inds = np.argsort(inds)
antnames = np.take(antnames, inds)
antpos = np.array([antpos["EAST"], antpos["NORTH"], antpos["UP"]])
array_center = np.mean(antpos, axis=1, keepdims=True)
antpos -= array_center
antpos = np.take(antpos, inds, axis=1)
with db.sessionmaker() as session:
hsession = cm_sysutils.Handling(session)
stations = []
for station_type in hsession.geo.parse_station_types_to_check(
"default"):
for stn in hsession.geo.station_types[station_type][
"Stations"]:
stations.append(stn)
# stations is a list of HH??? numbers we just want the ints
stations = list(map(int, [j[2:] for j in stations]))
bulk_add = []
for ind, name in enumerate(antnames):
ant_number = int(name[2:])
for pol in ["e", "n"]:
bulk_add.append(
Antenna(
ant_number=ant_number,
ant_name=name,
polarization=pol,
antpos_enu=antpos[:, ind].tolist(),
constructed=ant_number in stations,
))
Antenna.objects.bulk_create(bulk_add, ignore_conflicts=True)
def plot_arrays(args, overplot=None, label_station=False):
"""Plot the various sub-array types"""
coord = {'E': 'easting', 'N': 'northing', 'Z': 'elevation'}
plt.figure(args.xgraph + args.ygraph)
db = mc.connect_to_mc_db(args)
with db.sessionmaker() as session:
station_meta = session.get_station_meta()
for key in station_meta.keys():
for loc in station_meta[key]['Stations']:
for a in session.query(GeoLocation).filter(GeoLocation.station_name == loc):
show_it = True
if args.active:
show_it = is_in_connections_db(args,loc,True)
if show_it:
pt = {'easting': a.easting, 'northing': a.northing,
'elevation': a.elevation}
plt.plot(pt[coord[args.xgraph]], pt[coord[args.ygraph]],
station_meta[key]['Marker'], label=a.station_name)
if label_station:
if args.label_type=='station_name':
labeling = a.station_name
elif args.label_type=='station_number':
labeling = is_in_connections_db(args,loc,True)
if not labeling:
labeling = 'NA'
else:
labeling = 'S'
plt.annotate(labeling, xy=(pt[coord[args.xgraph]], pt[coord[args.ygraph]]),
xytext=(pt[coord[args.xgraph]] + 5, pt[coord[args.ygraph]]))
if overplot:
if overplot['connected'] and overplot['active']:
over_marker = 'g*'
mkr_lbl = 'ca'
elif overplot['connected'] and not overplot['active']:
over_marker = 'gx'
mkr_lbl = 'cx'
elif overplot['acive'] and not overplot['connected']:
over_marker = 'yx'
mkr_lbl = 'xa'
else:
over_marker = 'rx'
mkr_lbl = 'xx'
overplot_station = plt.plot(overplot[coord[args.xgraph]], overplot[coord[args.ygraph]],
over_marker, markersize=14)
legendEntries = [overplot_station]
legendText = [overplot['station_name'] + ':' + str(overplot['active'])]
plt.legend((overplot_station), (legendText), numpoints=1, loc='upper right')
if args.xgraph != 'Z' and args.ygraph != 'Z':
plt.axis('equal')
plt.plot(xaxis=args.xgraph, yaxis=args.ygraph)
plt.show()
def _get_new_mc_session():
"""
Helper function for getting a new connection to the M&C database.
Args:
====================
None
Return:
====================
None
"""
# use default settings for connecting to the db
mc_db = mc.connect_to_mc_db(None)
mcs = mc_db.sessionmaker()
return mcs
def get_part(self, args, hpn_query=None, show_part=False):
"""
Return information on a part. It will return all matching first characters.
"""
if hpn_query is None:
hpn_query = args.hpn
part_dict = {}
db = mc.connect_to_mc_db(args)
with db.sessionmaker() as session:
for part in session.query(part_connect.Parts).filter(
part_connect.Parts.hpn.like(hpn_query + '%')):
part_dict[part.hpn] = {
'hptype': part.hptype,
'manufacturer_number': part.manufacturer_number,
'manufacture_date': part.manufacture_date,
'a_ports': [],
'b_ports': []
}
part_dict[part.hpn]['repr'] = part.__repr__() # Keep for now
for part_info in session.query(part_connect.PartInfo).filter(
part_connect.PartInfo.hpn == part.hpn):
part_dict[part.hpn][
'short_description'] = part_info.short_description
for connection in session.query(
part_connect.Connections).filter(
part_connect.Connections.a.like(hpn_query + '%')):
if connection.port_on_a not in part_dict[
part.hpn]['a_ports']:
part_dict[part.hpn]['a_ports'].append(
connection.port_on_a)
for connection in session.query(
part_connect.Connections).filter(
part_connect.Connections.b.like(hpn_query + '%')):
if connection.port_on_b not in part_dict[
part.hpn]['b_ports']:
part_dict[part.hpn]['b_ports'].append(
connection.port_on_b)
if part.hptype == 'station':
sub_arrays = session.split_arrays(
geo.sub_array_designators.keys())
args.locate = part.hpn
part_dict[part.hpn]['geo'] = geo.locate_station(
args, show_geo=False)
if show_part:
self.show_part(args, part_dict)
return part_dict
def is_in_connections_db(args,station_name,check_if_active=False):
db = mc.connect_to_mc_db(args)
with db.sessionmaker() as session:
connected_station = session.query(part_connect.Connections).filter(part_connect.Connections.up == station_name)
if connected_station.count() > 0:
station_connected = True
else:
station_connected = False
if station_connected and check_if_active:
current = cm_utils._get_datetime(args.date,args.time)
for connection in connected_station.all():
stop_date = cm_utils._get_stopdate(connection.stop_date)
if current>connection.start_date and current<stop_date:
station_connected = int(connection.down.strip('A'))
else:
station_connected = False
return station_connected
def is_in_connections_db(self,args,hpn_query,rev_query='last'):
revq = rev_query.upper()
if revq == 'LAST':
revq = self.last_revisions[hpn_query]
if hpn_query in self.parts_dictionary.keys() and self.parts_dictionary[hpn_query]['rev']==revq:
return self.parts_dictionary[hpn_query]['is_connected']
db = mc.connect_to_mc_db(args)
with db.sessionmaker() as session:
connected_query = session.query(part_connect.Connections).filter( ((part_connect.Connections.up == hpn_query) &
(part_connect.Connections.up_rev == revq) ) |
((part_connect.Connections.down == hpn_query) &
(part_connect.Connections.down_rev== revq) ))
if connected_query.count() > 0:
found_connected = True
else:
found_connected = False
return found_connected
def locate_station(args, show_geo=False):
"""Return the location of station_name or station_number as contained in args.locate.
If sub_array data exists, print subarray name."""
station_name = False
try:
station = int(args.locate)
station_name = find_station_name(args,station)
except ValueError:
station_name = args.locate.upper()
v = None
if station_name:
db = mc.connect_to_mc_db(args)
with db.sessionmaker() as session:
station_meta = session.get_station_meta()
for a in session.query(GeoLocation).filter(GeoLocation.station_name == station_name):
for key in station_meta.keys():
if a.station_name in station_meta[key]['Stations']:
this_station = key
break
else:
this_station = 'No station metadata.'
ever_connected = is_in_connections_db(args,a.station_name)
active = is_in_connections_db(args,a.station_name,True)
v = {'easting': a.easting, 'northing': a.northing, 'elevation': a.elevation,
'station_name': a.station_name,
'station_type': this_station,
'connected':ever_connected,
'active':active}
if show_geo:
if args.verbosity == 'm' or args.verbosity == 'h':
print('station_name: ', a.station_name)
print('\teasting: ', a.easting)
print('\tnorthing: ', a.northing)
print('\televation: ', a.elevation)
print('\tstation description (%s): %s' %
(this_station,station_meta[this_station]['Description']))
print('\tever connected: ',ever_connected)
print('\tactive: ',active)
elif args.verbosity == 'l':
print(a, this_station)
if show_geo:
if not v and args.verbosity == 'm' or args.verbosity == 'h':
print(args.locate, ' not found.')
return v
def find_station_name(args,station_number):
station = 'A'+str(station_number)
db = mc.connect_to_mc_db(args)
with db.sessionmaker() as session:
connected_station = session.query(part_connect.Connections).filter(part_connect.Connections.down == station)
if connected_station.count() > 0:
station_connected = True
else:
station_connected = False
if station_connected:
current = cm_utils._get_datetime(args.date,args.time)
for connection in connected_station.all():
stop_date = cm_utils._get_stopdate(connection.stop_date)
if current>connection.start_date and current<stop_date:
station_connected = connection.up
else:
station_connected = False
return station_connected
def get_part_types(self, args, show_hptype=False):
"""
Goes through database and pulls out part types and some other info to display in a table.
Returns part_type_dict, a dictionary keyed on part type
Parameters
-----------
args: arguments as per mc and parts argument parser
show_hptype: boolean variable to print it out
"""
self.part_type_dict = {}
db = mc.connect_to_mc_db(args)
with db.sessionmaker() as session:
for part in session.query(part_connect.Parts).all():
if part.hptype not in self.part_type_dict.keys():
self.part_type_dict[part.hptype] = {'part_list':[part.hpn], 'a_ports':[], 'b_ports':[]}
else:
self.part_type_dict[part.hptype]['part_list'].append(part.hpn)
if show_hptype:
headers = ['Part type','# in dbase','A ports','B ports']
table_data = []
for k in self.part_type_dict.keys(): ###ASSUME FIRST PART IS FULLY CONNECTED
pa = self.part_type_dict[k]['part_list'][0]
pd = self.get_part(args,pa,show_part=False)
self.part_type_dict[k]['a_ports'] = pd[pa]['a_ports']
self.part_type_dict[k]['b_ports'] = pd[pa]['b_ports']
if show_hptype:
td = [k,len(self.part_type_dict[k]['part_list'])]
pts = ''
for a in self.part_type_dict[k]['a_ports']:
pts+=(a+', ')
td.append(pts.strip().strip(','))
pts = ''
for b in self.part_type_dict[k]['b_ports']:
pts+=(b+', ')
td.append(pts.strip().strip(','))
table_data.append(td)
if show_hptype:
print(tabulate(table_data,headers=headers,tablefmt='orgtbl'))
return self.part_type_dict
def update(args, data):
"""
update the database given a station_name and station_number with columns/values and provides some checking
use with caution -- should usually use in a script which will do datetime primary key etc
Parameters:
------------
data: [[station_name0,column0,value0],[...]]
station_nameN: station_name (starts with char)
values: corresponding list of values
"""
data = format_check_update_request(data)
if data is None:
print('Error: invalid update')
return False
station_name = data[0][0]
db = mc.connect_to_mc_db(args)
with db.sessionmaker() as session:
geo_rec = session.query(GeoLocation).filter(GeoLocation.station_name == station_name)
ngr = geo_rec.count()
if ngr == 0:
if args.add_new_geo:
gr = GeoLocation()
else:
print(d[0],"exists and add_new_geo not enabled.")
gr = None
elif ngr == 1:
gr = geo_rec.first()
else:
print("Shouldn't ever get here.")
gr = None
if gr:
for d in data:
try:
setattr(gr, d[1], d[2])
except AttributeError:
print(d[1], 'does not exist as a field')
continue
session.add(gr)
return True
def update(self, args, data):
"""
update the database given a station_name/_number with columns/values
Parameters:
------------
data: [[station0,column0,value0],[...]]
stationN: may be station_name (starts with char) or station_number (is an int)
values: corresponding list of values
"""
db = mc.connect_to_mc_db(args)
with db.sessionmaker() as session:
for d in data:
station, station_col = self.station_name_or_number(d[0])
for geo_rec in session.query(self).filter(
station_col == station):
try:
xxx = getattr(geo_rec, d[1])
setattr(geo_rec, d[1], d[2])
except AttributeError:
print(d[1], 'does not exist')
def get_connection(self, args, hpn_query=None, show_connection=False):
"""
Return information on parts connected to args.connection -- NEED TO INCLUDE USING START/STOP_TIME!!!
It should get connections immediately adjacent to one part (upstream and downstream).
"""
if hpn_query is None:
hpn_query = args.connection
connection_dict = {
'a': [],
'port_on_a': [],
'start_on_a': [],
'stop_on_a': [],
'repr_a': [],
'b': [],
'port_on_b': [],
'start_on_b': [],
'stop_on_b': [],
'repr_b': []
}
db = mc.connect_to_mc_db(args)
with db.sessionmaker() as session:
for connection in session.query(part_connect.Connections).filter(
part_connect.Connections.a.like(hpn_query + '%')):
# connected.append(self.get_part(args, connection.b, show_part=False))
connection_dict['b'].append(connection.b)
connection_dict['port_on_b'].append(connection.port_on_b)
connection_dict['start_on_b'].append(connection.start_time)
connection_dict['stop_on_b'].append(connection.stop_time)
connection_dict['repr_b'].append(connection.__repr__())
for connection in session.query(part_connect.Connections).filter(
part_connect.Connections.b.like(hpn_query + '%')):
# connected.append(self.get_part(args, connection.a, show_part=False))
connection_dict['a'].append(connection.a)
connection_dict['port_on_a'].append(connection.port_on_a)
connection_dict['start_on_a'].append(connection.start_time)
connection_dict['stop_on_a'].append(connection.stop_time)
connection_dict['repr_a'].append(connection.__repr__())
if show_connection:
self.show_connection(args, connection_dict)
return connection_dict
def plot_arrays(args, overplot=None):
"""Plot the various sub-array types"""
vpos = {'E':0,'N':1,'Z':2}
plt.figure(args.xgraph+args.ygraph)
db = mc.connect_to_mc_db(args)
with db.sessionmaker() as session:
sub_arrays = session.split_arrays()
for key in sub_arrays.keys():
for loc in sub_arrays[key]['Stations']:
for a in session.query(geo_location.GeoLocation).filter(geo_location.GeoLocation.station_name==loc):
v = [a.easting,a.northing,a.elevation]
plt.plot(v[vpos[args.xgraph]],v[vpos[args.ygraph]],sub_arrays[key]['Marker'],label=a.station_name)
if overplot:
overplot_station = plt.plot(overplot[vpos[args.xgraph]], overplot[vpos[args.ygraph]],
'ys', markersize=10)
legendEntries = [overplot_station]
legendText = [overplot[3]+':'+str(overplot[4])]
plt.legend((overplot_station),(legendText),numpoints=1,loc='upper right')
if args.xgraph != 'Z' and args.ygraph != 'Z':
plt.axis('equal')
plt.plot(xaxis=args.xgraph, yaxis=args.ygraph)
plt.show()
"""Onetime transfer of geo-locations of antennas into the M&C database.
"""
from __future__ import absolute_import, division, print_function
from hera_mc import geo_location, mc
data = {}
data['HH'] = ['herahex','HERA Hex locations', 'ro']
data['PH'] = ['paperhex','PAPER Hex locations', 'rs']
data['PI'] = ['paperimaging','PAPER Imaging locations', 'gs']
data['PP'] = ['paperpolarized','PAPER Polarized locations', 'bd']
data['S'] = ['stationgrid','Station grid locations', 'ks']
data['CR'] = ['container','Container location', 'k*']
data['ND'] = ['node','Node location', 'r*']
sorted_keys = sorted(data.keys())
parser = mc.get_mc_argument_parser()
args = parser.parse_args()
db = mc.connect_to_mc_db(args)
with db.sessionmaker() as session:
for k in sorted_keys:
d = geo_location.StationMeta()
d.prefix = k
d.meta_class_name = data[k][0]
d.description = data[k][1]
d.plot_marker = data[k][2]
session.add(d)
'--query',
help="Set flag if wished to be queried",
action='store_true')
cm_utils.add_date_time_args(parser)
parser.add_argument('--date2',
help="Stop date (if not None)",
default=None)
parser.add_argument('--time2',
help="Stop time (if not None)",
default=None)
parser.add_argument('--verbose',
help="Turn verbose mode on.",
action='store_true')
args = parser.parse_args()
if args.query:
args = query_args(args)
# Pre-process some args
start_date = cm_utils.get_astropytime(args.date, args.time)
stop_date = cm_utils.get_astropytime(args.date2, args.time2)
db = mc.connect_to_mc_db(args)
session = db.sessionmaker()
# Check for part
if args.verbose:
print("Adding part_rosetta {}: - {}".format(args.hpn, args.syspn))
cm_partconnect.update_part_rosetta(args.hpn, args.syspn, start_date,
stop_date, session)
def main():
# templates are stored relative to the script dir
# stored one level up, find the parent directory
# and split the parent directory away
script_dir = os.path.dirname(os.path.realpath(__file__))
split_dir = os.path.split(script_dir)
template_dir = os.path.join(split_dir[0], "templates")
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True)
if sys.version_info[0] < 3:
# py2
computer_hostname = os.uname()[1]
else:
# py3
computer_hostname = os.uname().nodename
parser = mc.get_mc_argument_parser()
args = parser.parse_args()
try:
db = mc.connect_to_mc_db(args)
except RuntimeError as e:
raise SystemExit(str(e))
plotnames = [[n1 + "-" + n2 for n1 in ["lib", "rtp"]]
for n2 in ["load", "disk", "mem", "bandwidth", "timediff"]]
colsize = 6
TIME_WINDOW = 14 # days
now = Time.now()
cutoff = now - TimeDelta(TIME_WINDOW, format="jd")
time_axis_range = [cutoff.isot, now.isot]
caption = {}
caption["text"] = ("An overview of many computer statisics."
"<br><br>Plotted statistics"
"""<div class="table-responsive">
<table class="table table-striped" style="border:1px solid black; border-top; 1px solid black;">
<thead>
<tr>
<td style="border-left:1px solid black;">Librarian Related Computers
<td style="border-left:1px solid black;">RTP Related Computers</td>
</tr>
</thead>
<tbody>
<tr>
<td style="border-left:1px solid black;">Load % per Computer
<td style="border-left:1px solid black;">Load % per Computer</td>
</tr>
<tr>
<td style="border-left:1px solid black;">Local Disk Usage %
<td style="border-left:1px solid black;">Local Disk Usage %</td></tr>
<tr>
<td style="border-left:1px solid black;">Local Memory Usage %
<td style="border-left:1px solid black;">Local Memory Usage %</td></tr>
<tr>
<td style="border-left:1px solid black;">Network I/O rate (MB/s)
<td style="border-left:1px solid black;">Network I/O rate (MB/s)</td>
</tr>
<tr>
<td style="border-left:1px solid black;">M&C time diff (s)
<td style="border-left:1px solid black;">M&C time diff (s)</td>
</tr>
</tbody>
</table>
</div>
""")
caption["title"] = "Compute Help"
html_template = env.get_template("plotly_base.html")
rendered_html = html_template.render(
plotname=plotnames,
plotstyle="height: 19.5%",
colsize=colsize,
gen_date=now.iso,
gen_time_unix_ms=now.unix * 1000,
js_name="compute",
hostname=computer_hostname,
scriptname=os.path.basename(__file__),
caption=caption,
)
with open("compute.html", "w") as h_file:
h_file.write(rendered_html)
with db.sessionmaker() as session:
lib_data = get_status(session, LibServerStatus, LIB_HOSTNAMES, cutoff)
rtp_data = get_status(session, RTPServerStatus, RTP_HOSTNAMES, cutoff)
layout = {
"xaxis": {
"range": time_axis_range
},
"yaxis": {
"title": "placeholder",
"rangemode": "tozero"
},
"title": {
"text": "placeholder",
"font": {
"size": 18
}
},
"height": 200,
"margin": {
"t": 25,
"r": 10,
"b": 10,
"l": 40
},
"legend": {
"orientation": "h",
"x": 0.15,
"y": -0.15
},
"showlegend": True,
"hovermode": "closest",
}
yaxis_titles = {
"load": "Load % per CPU",
"disk": "Local disk usage (%)",
"mem": "Memory usage (%)",
"bandwidth": "Network I/O (MB/s)",
"timediff": "M&C time diff. (s)",
}
titles = {
"load": "CPU Load",
"disk": "Disk Usage",
"mem": "Memory Usage",
"bandwidth": "Network I/O",
"timediff": "M&C time diff. ",
}
for server_type, data_dict in zip(["lib", "rtp"],
[lib_data, rtp_data]):
js_template = env.get_template("plotly_base.js")
for pname in ["load", "disk", "mem", "bandwidth", "timediff"]:
layout["yaxis"]["title"] = yaxis_titles[pname]
layout["title"]["text"] = server_type + " " + titles[pname]
_name = server_type + "-" + pname
rendered_js = js_template.render(plotname=_name,
data=data_dict[pname],
layout=layout)
if _name == "lib-load":
open_type = "w"
else:
open_type = "a"
with open("compute.js", open_type) as js_file:
js_file.write(rendered_js)
js_file.write("\n\n")
def main():
# templates are stored relative to the script dir
# stored one level up, find the parent directory
# and split the parent directory away
script_dir = os.path.dirname(os.path.realpath(__file__))
split_dir = os.path.split(script_dir)
template_dir = os.path.join(split_dir[0], "templates")
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True)
if sys.version_info[0] < 3:
# py2
computer_hostname = os.uname()[1]
else:
# py3
computer_hostname = os.uname().nodename
# The standard M&C argument parser
parser = mc.get_mc_argument_parser()
# we'll have to add some extra options too
parser.add_argument(
"--redishost",
dest="redishost",
type=str,
default="redishost",
help=(
'The host name for redis to connect to, defualts to "redishost"'),
)
parser.add_argument("--port",
dest="port",
type=int,
default=6379,
help="Redis port to connect.")
args = parser.parse_args()
try:
db = mc.connect_to_mc_db(args)
except RuntimeError as e:
raise SystemExit(str(e))
try:
redis_db = redis.Redis(args.redishost, port=args.port)
redis_db.keys()
except Exception as err:
raise SystemExit(str(err))
with db.sessionmaker() as session:
# without item this will be an array which will break database queries
latest = Time(
np.frombuffer(redis_db.get("auto:timestamp"),
dtype=np.float64).item(),
format="jd",
)
latest.out_subfmt = u"date_hm"
now = Time.now()
amps = {}
keys = [
k.decode() for k in redis_db.keys()
if k.startswith(b"auto") and not k.endswith(b"timestamp")
]
for key in keys:
match = re.search(r"auto:(?P<ant>\d+)(?P<pol>e|n)", key)
if match is not None:
ant, pol = int(match.group("ant")), match.group("pol")
d = redis_db.get(key)
if d is not None:
# need to copy because frombuffer creates a read-only array
auto = np.frombuffer(d, dtype=np.float32).copy()
eq_coeff = redis_db.hget(
bytes("eq:ant:{ant}:{pol}".format(ant=ant,
pol=pol).encode()),
"values",
)
if eq_coeff is not None:
eq_coeffs = np.fromstring(
eq_coeff.decode("utf-8").strip("[]"), sep=",")
if eq_coeffs.size == 0:
eq_coeffs = np.ones_like(auto)
else:
eq_coeffs = np.ones_like(auto)
# divide out the equalization coefficients
# eq_coeffs are stored as a length 1024 array but only a
# single number is used. Taking the median to not deal with
# a size mismatch
eq_coeffs = np.median(eq_coeffs)
auto /= eq_coeffs**2
auto[auto < 10**-10.0] = 10**-10.0
auto = np.median(auto)
amps[(ant, pol)] = 10.0 * np.log10(auto)
hsession = cm_sysutils.Handling(session)
ants = np.unique([ant for (ant, pol) in amps.keys()])
pols = np.unique([pol for (ant, pol) in amps.keys()])
antpos = np.genfromtxt(
os.path.join(mc.data_path, "HERA_350.txt"),
usecols=(0, 1, 2, 3),
dtype={
"names": ("ANTNAME", "EAST", "NORTH", "UP"),
"formats": ("<U5", "<f8", "<f8", "<f8"),
},
encoding=None,
)
antnames = antpos["ANTNAME"]
inds = [int(j[2:]) for j in antnames]
inds = np.argsort(inds)
antnames = np.take(antnames, inds)
antpos = np.array([antpos["EAST"], antpos["NORTH"], antpos["UP"]])
array_center = np.mean(antpos, axis=1, keepdims=True)
antpos -= array_center
antpos = np.take(antpos, inds, axis=1)
stations = hsession.get_connected_stations(at_date="now")
for station in stations:
if station.antenna_number not in ants:
ants = np.append(ants, station.antenna_number)
ants = np.unique(ants)
stations = []
for station_type in hsession.geo.parse_station_types_to_check(
"default"):
for stn in hsession.geo.station_types[station_type]["Stations"]:
stations.append(stn)
# stations is a list of HH??? numbers we just want the ints
stations = list(map(int, [j[2:] for j in stations]))
built_but_not_on = np.setdiff1d(stations, ants)
# Get node and PAM info
node_ind = np.zeros_like(ants, dtype=np.int)
pam_ind = np.zeros_like(ants, dtype=np.int)
# defaul the snap name to "No Data"
hostname = np.full_like(ants, "No\tData", dtype=object)
snap_serial = np.full_like(ants, "No\tData", dtype=object)
pam_power = {}
adc_power = {}
adc_rms = {}
time_array = {}
fem_imu_theta = {}
fem_imu_phi = {}
eq_coeffs = {}
for ant in ants:
for pol in pols:
amps.setdefault((ant, pol), np.Inf)
pam_power.setdefault((ant, pol), np.Inf)
adc_power.setdefault((ant, pol), np.Inf)
adc_rms.setdefault((ant, pol), np.Inf)
eq_coeffs.setdefault((ant, pol), np.Inf)
fem_imu_theta.setdefault((ant, pol), np.Inf)
fem_imu_phi.setdefault((ant, pol), np.Inf)
time_array.setdefault((ant, pol), now - Time(0, format="gps"))
for ant_cnt, ant in enumerate(ants):
station_status = session.get_antenna_status(
most_recent=True, antenna_number=int(ant))
for status in station_status:
antpol = (status.antenna_number, status.antenna_feed_pol)
if status.pam_power is not None:
pam_power[antpol] = status.pam_power
if status.adc_power is not None:
adc_power[antpol] = 10 * np.log10(status.adc_power)
if status.adc_rms is not None:
adc_rms[antpol] = status.adc_rms
if status.time is not None:
time_array[antpol] = now - Time(status.time, format="gps")
if status.fem_imu_phi is not None:
fem_imu_phi[antpol] = status.fem_imu_phi
if status.fem_imu_theta is not None:
fem_imu_theta[antpol] = status.fem_imu_theta
if status.eq_coeffs is not None:
_coeffs = np.fromstring(status.eq_coeffs.strip("[]"),
sep=",")
# just track the median coefficient for now
eq_coeffs[antpol] = np.median(_coeffs)
# Try to get the snap info. Output is a dictionary with 'e' and 'n' keys
mc_name = antnames[ant]
snap_info = hsession.get_part_at_station_from_type(
mc_name, "now", "snap")
# get the first key in the dict to index easier
_key = list(snap_info.keys())[0]
pol_key = [key for key in snap_info[_key].keys() if "E" in key]
if pol_key:
# 'E' should be in one of the keys, extract the 0th entry
pol_key = pol_key[0]
else:
# a hacky solution for a key that should work
pol_key = "E<ground"
if snap_info[_key][pol_key] is not None:
snap_serial[ant_cnt] = snap_info[_key][pol_key]
# Try to get the pam info. Output is a dictionary with 'e' and 'n' keys
pam_info = hsession.get_part_at_station_from_type(
mc_name, "now", "post-amp")
# get the first key in the dict to index easier
_key = list(pam_info.keys())[0]
if pam_info[_key][pol_key] is not None:
_pam_num = re.findall(r"PAM(\d+)", pam_info[_key][pol_key])[0]
pam_ind[ant_cnt] = np.int(_pam_num)
else:
pam_ind[ant_cnt] = -1
# Try to get the ADC info. Output is a dictionary with 'e' and 'n' keys
node_info = hsession.get_part_at_station_from_type(
mc_name, "now", "node")
# get the first key in the dict to index easier
_key = list(node_info.keys())[0]
if node_info[_key][pol_key] is not None:
_node_num = re.findall(r"N(\d+)", node_info[_key][pol_key])[0]
node_ind[ant_cnt] = np.int(_node_num)
_hostname = session.get_snap_hostname_from_serial(
snap_serial[ant_cnt])
if _hostname is not None:
hostname[ant_cnt] = _hostname
else:
snap_status = session.get_snap_status(
most_recent=True, nodeID=np.int(_node_num))
for _status in snap_status:
if _status.serial_number == snap_serial[ant_cnt]:
hostname[ant_cnt] = _status.hostname
else:
node_ind[ant_cnt] = -1
pams, _pam_ind = np.unique(pam_ind, return_inverse=True)
nodes, _node_ind = np.unique(node_ind, return_inverse=True)
xs_offline = np.ma.masked_array(
antpos[0, :],
mask=[
True if int(name[2:]) in ants else False for name in antnames
],
)
ys_offline = np.ma.masked_array(antpos[1, :], mask=xs_offline.mask)
name_offline = np.ma.masked_array(
[aname + "<br>OFFLINE" for aname in antnames],
mask=xs_offline.mask,
dtype=object,
)
xs_offline = xs_offline
names = [
"Auto [dB]",
"PAM [dB]",
"ADC [dB]",
"ADC RMS",
"FEM IMU THETA",
"FEM IMU PHI",
"EQ COEF",
]
powers = [
amps,
pam_power,
adc_power,
adc_rms,
fem_imu_theta,
fem_imu_phi,
eq_coeffs,
]
powers = [
np.ma.masked_invalid([[p[ant, pol] for ant in ants]
for pol in pols]) for p in powers
]
write_csv("ant_stats.csv", antnames, ants, pols, names, powers,
built_but_not_on)
time_array = np.array(
[[time_array[ant, pol].to("hour").value for ant in ants]
for pol in pols])
xs = np.ma.masked_array(antpos[0, ants], mask=powers[0][0].mask)
ys = np.ma.masked_array(
[
antpos[1, ants] + 3 * (pol_cnt - 0.5)
for pol_cnt, pol in enumerate(pols)
],
mask=powers[0].mask,
)
_text = np.array(
[[
antnames[ant] + pol + "<br>" + str(hostname[ant_cnt]) +
"<br>" + "PAM\t#:\t" + str(pam_ind[ant_cnt])
for ant_cnt, ant in enumerate(ants)
] for pol_cnt, pol in enumerate(pols)],
dtype="object",
)
# want to format No Data where data was not retrieved for each type of power
for pol_cnt, pol in enumerate(pols):
for ant_cnt, ant in enumerate(ants):
for _name, _power in zip(names, powers):
if not _power.mask[pol_cnt, ant_cnt]:
_text[pol_cnt, ant_cnt] += (
"<br>" + _name +
": {0:.2f}".format(_power[pol_cnt, ant_cnt]))
else:
_text[pol_cnt, ant_cnt] += "<br>" + _name + ": No Data"
if time_array[pol_cnt, ant_cnt] > 2 * 24 * 365:
# if the value is older than 2 years it is bad
# value are stored in hours.
# 2 was chosen arbitraritly.
_text[pol_cnt, ant_cnt] += "<br>" + "Ant Status: No Date"
else:
_text[pol_cnt,
ant_cnt] += ("<br>" +
"Ant Status: {0:.2f} hrs old".format(
time_array[pol_cnt, ant_cnt]))
# having spaces will cause odd wrapping issues, replace all
# spaces by \t
_text[pol_cnt, ant_cnt] = _text[pol_cnt,
ant_cnt].replace(" ", "\t")
masks = [[True] for p in powers]
# Offline antennas
data_hex = []
offline_ants = {
"x": xs_offline.compressed().tolist(),
"y": ys_offline.compressed().tolist(),
"text": name_offline,
"mode": "markers",
"visible": True,
"marker": {
"color":
np.ma.masked_array(["black"] * len(name_offline),
mask=xs_offline.mask),
"size":
14,
"opacity":
0.5,
"symbol":
"hexagon",
},
"hovertemplate": "%{text}<extra></extra>",
}
# now we want to Fill in the conneted ones
offline_ants["marker"]["color"][built_but_not_on] = "red"
offline_ants["text"].data[built_but_not_on] = [
offline_ants["text"].data[ant].split("<br>")[0] +
"<br>Constructed<br>Not\tOnline" for ant in built_but_not_on
]
offline_ants["marker"]["color"] = (
offline_ants["marker"]["color"].compressed().tolist())
offline_ants["text"] = offline_ants["text"].compressed().tolist()
data_hex.append(offline_ants)
# for each type of power, loop over pols and print out the data
# save up a mask array used for the buttons later
# also plot the bad ones!3
colorscale = "Viridis"
# define some custom scale values for the ADC RMS page
rms_scale_vals = [2, 20]
relavitve_values = [0.4, 0.7]
rms_color_scale = [
["0.0", "rgb(68,1,84)"],
["0.2", "rgb(62,74,137)"],
["0.3", "rgb(38,130,142)"],
["0.4", "rgb(53,183,121)"],
["0.5", "rgb(53,183,121)"],
["0.6", "rgb(53,183,121)"],
["0.7", "rgb(109,205,89)"],
["0.8", "rgb(180,222,44)"],
["1.0", "rgb(253,231,37)"],
]
for pow_ind, power in enumerate(powers):
if power.compressed().size > 0:
vmax = np.max(power.compressed())
vmin = np.min(power.compressed())
else:
vmax = 1
vmin = 0
colorscale = "Viridis"
if pow_ind == 3:
cbar_title = "RMS\tlinear"
vmin = rms_scale_vals[0] * relavitve_values[0]
vmax = rms_scale_vals[1] / relavitve_values[1]
colorscale = rms_color_scale
elif pow_ind == 4 or pow_ind == 5:
cbar_title = "Degrees"
elif pow_ind == len(powers) - 1:
cbar_title = "Median\tCoeff"
else:
cbar_title = "dB"
if pow_ind == 0:
visible = True
else:
visible = False
for pol_ind, pol in enumerate(pols):
for mask_cnt, mask in enumerate(masks):
if mask_cnt == pow_ind:
mask.extend([True] * 2)
else:
mask.extend([False] * 2)
_power = {
"x": xs.data[~power[pol_ind].mask].tolist(),
"y": ys[pol_ind].data[~power[pol_ind].mask].tolist(),
"text": _text[pol_ind][~power[pol_ind].mask].tolist(),
"mode": "markers",
"visible": visible,
"marker": {
"color":
power[pol_ind].data[~power[pol_ind].mask].tolist(),
"size": 14,
"cmin": vmin,
"cmax": vmax,
"colorscale": colorscale,
"colorbar": {
"thickness": 20,
"title": cbar_title
},
},
"hovertemplate": "%{text}<extra></extra>",
}
data_hex.append(_power)
_power_offline = {
"x": xs.data[power[pol_ind].mask].tolist(),
"y": ys[pol_ind].data[power[pol_ind].mask].tolist(),
"text": _text[pol_ind][power[pol_ind].mask].tolist(),
"mode": "markers",
"visible": visible,
"marker": {
"color": "orange",
"size": 14,
"cmin": vmin,
"cmax": vmax,
"colorscale": colorscale,
"colorbar": {
"thickness": 20,
"title": cbar_title
},
},
"hovertemplate": "%{text}<extra></extra>",
}
data_hex.append(_power_offline)
buttons = []
for _name, mask in zip(names, masks):
_button = {
"args": [{
"visible": mask
}, {
"title": "",
"annotations": {}
}],
"label": _name,
"method": "restyle",
}
buttons.append(_button)
updatemenus_hex = [{
"buttons": buttons,
"showactive": True,
"type": "buttons"
}]
layout_hex = {
"xaxis": {
"title": "East-West Position [m]"
},
"yaxis": {
"title": "North-South Position [m]",
"scaleanchor": "x"
},
"title": {
"text": "Per Antpol Stats vs Hex position",
"font": {
"size": 24
},
},
"hoverlabel": {
"align": "left"
},
"margin": {
"t": 40
},
"autosize": True,
"showlegend": False,
"hovermode": "closest",
}
caption = {}
caption["title"] = "Stats vs Hex pos Help"
caption["text"] = (
"This plot shows various statistics and measurements "
"per ant-pol versus its position in the array."
"<br>Antennas which are build but not fully hooked up "
"are shown in light red."
"<br>Grey antennas are not yet constructed."
"<br><br><h4>Available plotting options</h4>"
"<ul>"
"<li>Auto Corr - Median Auto Correlation (in db) "
"from the correlator with equalization coefficients "
"divided out</li>"
"<li>Pam Power - Latest Pam Power (in db) recorded in M&C</li>"
"<li>ADC Power - Latest ADC Power (in db) recorded in M&C</li>"
"<li>ADC RMS - Latest linear ADC RMS recorded in M&C</li>"
"<li>FEM IMU THETA - IMU-reported theta, in degrees</li>"
"<li>FEM IMU PHI - IMU-reported phi, in degrees</li>"
"<li>EQ Coeffs - Latest Median Equalization Coefficient recorded in M&C</li>"
"</ul>"
"Any antpol showing with an orange color means "
"no data is avaible for the currenty plot selection."
"<h4>Hover label Formatting</h4>"
"<ul>"
"<li>Antenna Name from M&C<br>(e.g. HH0n = Hera Hex Antenna 0 Polarization N)</li>"
"<li>Snap hostname from M&C<br>(e.g. heraNode0Snap0)</li>"
"<li>PAM Number</li>"
"<li>Median Auto Correlation power in dB</li>"
"<li>PAM power in dB</li>"
"<li>ADC power in dB</li>"
"<li>Linear ADC RMS</li>"
"<li>FEM IMU reported theta in degrees</li>"
"<li>FEM IMU reported phi in degrees</li>"
"<li>Median Equalization Coefficient</li>"
"<li>Time ago in hours the M&C Antenna Status was updated. "
"This time stamp applies to all data for this antenna "
"except the Auto Correlation.</li>"
"</ul>"
"In any hover label entry 'No Data' means "
"information not currrently available in M&C.")
# Render all the power vs position files
plotname = "plotly-hex"
html_template = env.get_template("plotly_base.html")
js_template = env.get_template("plotly_base.js")
if sys.version_info.minor >= 8 and sys.version_info.major > 2:
time_jd = latest.to_value('jd', subfmt='float')
time_unix = latest.to_value('unix')
else:
time_jd = latest.jd
time_unix = latest.unix
rendered_hex_html = html_template.render(
plotname=plotname,
data_type="Auto correlations",
plotstyle="height: 100%",
gen_date=now.iso,
data_date_iso=latest.iso,
data_date_jd="{:.3f}".format(time_jd),
data_date_unix_ms=time_unix * 1000,
js_name="hex_amp",
gen_time_unix_ms=now.unix * 1000,
scriptname=os.path.basename(__file__),
hostname=computer_hostname,
caption=caption,
)
rendered_hex_js = js_template.render(
data=data_hex,
layout=layout_hex,
updatemenus=updatemenus_hex,
plotname=plotname,
)
with open("hex_amp.html", "w") as h_file:
h_file.write(rendered_hex_html)
with open("hex_amp.js", "w") as js_file:
js_file.write(rendered_hex_js)
# now prepare the data to be plotted vs node number
data_node = []
masks = [[] for p in powers]
vmax = [
np.max(power.compressed()) if power.compressed().size > 1 else 1
for power in powers
]
vmin = [
np.min(power.compressed()) if power.compressed().size > 1 else 0
for power in powers
]
vmin[3] = rms_scale_vals[0] * relavitve_values[0]
vmax[3] = rms_scale_vals[1] / relavitve_values[1]
for node in nodes:
node_index = np.where(node_ind == node)[0]
hosts = hostname[node_index]
host_index = np.argsort(hosts)
ys = np.ma.masked_array(
[
np.arange(node_index.size) + 0.3 * pol_cnt
for pol_cnt, pol in enumerate(pols)
],
mask=powers[0][:, node_index].mask,
)
xs = np.zeros_like(ys)
xs[:] = node
powers_node = [pow[:, node_index] for pow in powers]
__text = _text[:, node_index]
for pow_ind, power in enumerate(powers_node):
cbar_title = "dB"
if pow_ind == 4 or pow_ind == 5:
cbar_title = "Degrees"
if pow_ind == 3:
colorscale = rms_color_scale
else:
colorscale = "Viridis"
colorscale = "Viridis"
if pow_ind == 3:
cbar_title = "RMS\tlinear"
colorscale = rms_color_scale
elif pow_ind == 4 or pow_ind == 5:
cbar_title = "Degrees"
elif pow_ind == len(powers) - 1:
cbar_title = "Median\tCoeff"
else:
cbar_title = "dB"
if pow_ind == 0:
visible = True
else:
visible = False
for pol_ind, pol in enumerate(pols):
for mask_cnt, mask in enumerate(masks):
if mask_cnt == pow_ind:
mask.extend([True] * 2)
else:
mask.extend([False] * 2)
__power = power[pol_ind][host_index]
___text = __text[pol_ind][host_index]
_power = {
"x": xs[pol_ind].data[~__power.mask].tolist(),
"y": ys[pol_ind].data[~__power.mask].tolist(),
"text": ___text[~__power.mask].tolist(),
"mode": "markers",
"visible": visible,
"marker": {
"color": __power.data[~__power.mask].tolist(),
"size": 14,
"cmin": vmin[pow_ind],
"cmax": vmax[pow_ind],
"colorscale": colorscale,
"colorbar": {
"thickness": 20,
"title": cbar_title
},
},
"hovertemplate": "%{text}<extra></extra>",
}
data_node.append(_power)
_power_offline = {
"x": xs[pol_ind].data[__power.mask].tolist(),
"y": ys[pol_ind].data[__power.mask].tolist(),
"text": ___text[__power.mask].tolist(),
"mode": "markers",
"visible": visible,
"marker": {
"color": "orange",
"size": 14,
"cmin": vmin[pow_ind],
"cmax": vmax[pow_ind],
"colorscale": colorscale,
"colorbar": {
"thickness": 20,
"title": cbar_title
},
},
"hovertemplate": "%{text}<extra></extra>",
}
data_node.append(_power_offline)
buttons = []
for _name, mask in zip(names, masks):
_button = {
"args": [{
"visible": mask
}, {
"title": "",
"annotations": {}
}],
"label": _name,
"method": "restyle",
}
buttons.append(_button)
updatemenus_node = [{
"buttons": buttons,
"showactive": True,
"type": "buttons"
}]
layout_node = {
"xaxis": {
"title": "Node Number",
"dtick": 1,
"tick0": 0,
"showgrid": False,
"zeroline": False,
},
"yaxis": {
"showticklabels": False,
"showgrid": False,
"zeroline": False
},
"title": {
"text": "Per Antpol Stats vs Node #",
"font": {
"size": 24
}
},
"hoverlabel": {
"align": "left"
},
"margin": {
"t": 40
},
"autosize": True,
"showlegend": False,
"hovermode": "closest",
}
caption_node = {}
caption_node["title"] = "Stats vs Node Help"
caption_node["text"] = (
"This plot shows various statistics and measurements "
"per ant-pol versus the node number to which it is connected."
"<br><br><h4>Available plotting options</h4>"
"<ul>"
"<li>Auto Corr - Median Auto Correlation (in db) "
"from the correlator with equalization coefficients "
"divided out</li>"
"<li>Pam Power - Latest Pam Power (in db) recorded in M&C</li>"
"<li>ADC Power - Latest ADC Power (in db) recorded in M&C</li>"
"<li>ADC RMS - Latest linear ADC RMS recorded in M&C</li>"
"<li>EQ Coeffs - Latest Median Equalization Coefficient recorded in M&C</li>"
"</ul>"
"Any antpol showing with an orange color means "
"no data is avaible for the currenty plot selection."
"<h4>Hover label Formatting</h4>"
"<ul>"
"<li>Antenna Name from M&C<br>(e.g. HH0n = Hera Hex Antenna 0 Polarization N)</li>"
"<li>Snap hostname from M&C<br>(e.g. heraNode0Snap0)</li>"
"<li>PAM Number</li>"
"<li>Median Auto Correlation power in dB</li>"
"<li>PAM power in dB</li>"
"<li>ADC power in dB</li>"
"<li>Linear ADC RMS</li>"
"<li>Median Equalization Coefficient</li>"
"<li>Time ago in hours the M&C Antenna Status was updated. "
"This time stamp applies to all data for this antenna "
"except the Auto Correlation.</li>"
"</ul>"
"In any hover label entry 'No Data' means "
"information not currrently available in M&C.")
# Render all the power vs ndde files
plotname = "plotly-node"
html_template = env.get_template("plotly_base.html")
js_template = env.get_template("plotly_base.js")
if sys.version_info.minor >= 8 and sys.version_info.major > 2:
time_jd = latest.to_value('jd', subfmt='float')
time_unix = latest.to_value('unix')
else:
time_jd = latest.jd
time_unix = latest.unix
rendered_node_html = html_template.render(
plotname=plotname,
data_type="Auto correlations",
plotstyle="height: 100%",
gen_date=now.iso,
gen_time_unix_ms=now.unix * 1000,
data_date_iso=latest.iso,
data_date_jd="{:.3f}".format(time_jd),
data_date_unix_ms=time_unix * 1000,
js_name="node_amp",
scriptname=os.path.basename(__file__),
hostname=computer_hostname,
caption=caption_node,
)
rendered_node_js = js_template.render(
data=data_node,
layout=layout_node,
updatemenus=updatemenus_node,
plotname=plotname,
)
with open("node_amp.html", "w") as h_file:
h_file.write(rendered_node_html)
with open("node_amp.js", "w") as js_file:
js_file.write(rendered_node_js)
def get_part(self, args, hpn_query=None, rev_query=None, exact_match=False, return_dictionary=True, show_part=False):
"""
Return information on a part. It will return all matching first characters unless exact_match==True.
Returns part_dict, a dictionary keyed on hera part number with part data as found in hera_mc tables
Parameters
-----------
args: arguments as per mc and parts argument parser
hpn_query: the input hera part number (whole or first part thereof)
exact_match: boolean to enforce full part number match
show_part: boolean to call show_part or not
"""
if hpn_query is None:
hpn_query = args.hpn
exact_match = args.exact_match
if rev_query is None:
rev_query = args.revision_number
if not exact_match and hpn_query[-1]!='%':
hpn_query = hpn_query+'%'
local_parts_keys = []
part_dict = {}
db = mc.connect_to_mc_db(args)
with db.sessionmaker() as session:
### Get parts to check and fill in last_revisions and connections_dictionary as able
for match_part in session.query(part_connect.Parts).filter(part_connect.Parts.hpn.like(hpn_query)):
if match_part.hpn not in local_parts_keys:
local_parts_keys.append(match_part.hpn)
if match_part.hpn not in self.last_revisions.keys():
self.last_revisions[match_part.hpn] = part_connect.get_last_revision_number(args,match_part.hpn,False)
if match_part.hpn not in self.connections_dictionary.keys():
self.get_connection(args, hpn_query=match_part.hpn, rev_query = rev_query, port_query='all',
exact_match=True, return_dictionary=False, show_connection=False)
### Now get unique part/rev and put into dictionary
for match_key in local_parts_keys:
revq = rev_query.upper()
if revq == 'LAST':
revq = self.last_revisions[match_key]
part_query = session.query(part_connect.Parts).filter( (part_connect.Parts.hpn==match_key) &
(part_connect.Parts.hpn_rev==revq) )
part_and_rev = part_query.all()
part_cnt = part_query.count()
if not part_cnt: ### None found.
continue
elif part_cnt == 1:
part = part_and_rev[0] ### Found only one.
is_connected = self.is_in_connections_db(args,part.hpn,part.hpn_rev)
psd_for_dict = part.stop_date
if not part.stop_date:
psd_for_dict = 'N/A'
part_dict[part.hpn] = {'rev':part.hpn_rev, 'is_connected':is_connected,
'hptype': part.hptype,
'manufacturer_number': part.manufacturer_number,
'start_date': part.start_date, 'stop_date': psd_for_dict,
'a_ports': [], 'b_ports': [], 'short_description':'', 'geo':None}
part_dict[part.hpn]['repr'] = part.__repr__() # Keep for now
for part_info in session.query(part_connect.PartInfo).filter( (part_connect.PartInfo.hpn == part.hpn) &
(part_connect.PartInfo.hpn_rev == part.hpn_rev) ):
part_dict[part.hpn]['short_description'] = part_info.short_description
part_dict[part.hpn]['a_ports'] = self.connections_dictionary[part.hpn]['a_ports']
part_dict[part.hpn]['b_ports'] = self.connections_dictionary[part.hpn]['b_ports']
if part.hptype == 'station':
args.locate = part.hpn
part_dict[part.hpn]['geo'] = geo_location.locate_station(args, show_geo=False)
if part.hpn not in self.parts_dictionary.keys():
self.parts_dictionary[part.hpn] = part_dict[part.hpn] ### This only handles the most recent revs.
else: ### Found more than one, which shouldn't happen.
print("Warning part_handling:175: Well, being here is a surprise -- should only be one part.", part.hpn)
if show_part:
if len(part_dict.keys()) == 0:
print(hpn_query,' not found.')
else:
self.show_part(args, part_dict)
if return_dictionary:
return part_dict
def get_connection(self, args, hpn_query=None, rev_query=None, port_query=None, exact_match=False,
return_dictionary=True, show_connection=False):
"""
Return information on parts connected to args.connection -- NEED TO INCLUDE USING START/STOP_TIME!!!
It should get connections immediately adjacent to one part (upstream and downstream).
Returns connection_dict, a dictionary keyed on part number of adjacent connections
Parameters
-----------
args: arguments as per mc and parts argument parser
hpn_query: the input hera part number (whole or first part thereof)
port_query: a specifiable port name, default is 'all'
exact_match: boolean to enforce full part number match
show_connection: boolean to call show_part or not
"""
if hpn_query is None:
hpn_query = args.connection
exact_match = args.exact_match
if rev_query is None:
rev_query = args.revision_number
if not exact_match and hpn_query[-1]!='%':
hpn_query = hpn_query+'%'
if port_query is None:
port_query = args.specify_port
connection_dict = {}
db = mc.connect_to_mc_db(args)
with db.sessionmaker() as session:
### Find where the part is in the upward connection
for match_connection in session.query(part_connect.Connections).filter(part_connect.Connections.up.like(hpn_query)):
if port_query=='all' or match_connection.b_on_up == port_query:
#-#THIS SHOULD HAVE WORKED BUT DOESN'T#-#
#-#if match_connection.up not in connection_dict.keys() and match_connection.up in self.connections_dictionary.keys():
#-# connection_dict[match_connection.up] = copy.deepcopy(self.connections_dictionary[match_connection.up])
#-# continue
if match_connection.up not in connection_dict.keys():
revq = rev_query.upper()
if revq == 'LAST':
if match_connection.up not in self.last_revisions.keys():
self.last_revisions[match_connection.up] = part_connect.get_last_revision_number(args,match_connection.up,False)
revq = self.last_revisions[match_connection.up]
connection_dict[match_connection.up] = {'rev':revq,
'a_ports':[], 'up_parts':[], 'up_rev':[], 'b_on_up':[],
'b_ports':[], 'down_parts':[], 'down_rev':[],'a_on_down':[],
'start_on_down':[], 'stop_on_down':[], 'repr_down':[],
'start_on_up':[], 'stop_on_up':[], 'repr_up':[]}
connection_dict[match_connection.up]['b_ports'].append(match_connection.b_on_up)
connection_dict[match_connection.up]['down_parts'].append(match_connection.down)
connection_dict[match_connection.up]['down_rev'].append(match_connection.down_rev)
connection_dict[match_connection.up]['a_on_down'].append(match_connection.a_on_down)
connection_dict[match_connection.up]['start_on_down'].append(match_connection.start_date)
connection_dict[match_connection.up]['stop_on_down'].append(match_connection.stop_date)
connection_dict[match_connection.up]['repr_down'].append(match_connection.__repr__())
### Find where the part is in the downward connection
for match_connection in session.query(part_connect.Connections).filter(part_connect.Connections.down.like(hpn_query)):
if port_query=='all' or match_connection.a_on_down == port_query:
#-#THIS SHOULD HAVE WORKED BUT DOESN'T#-#
#-#if match_connection.down not in connection_dict.keys() and match_connection.down in self.connections_dictionary.keys():
#-# connection_dict[match_connection.down] = copy.deepcopy(self.connections_dictionary[match_connection.down])
#-# continue
if match_connection.down not in connection_dict.keys():
revq = rev_query.upper()
if revq == 'LAST':
if match_connection.down not in self.last_revisions.keys():
self.last_revisions[match_connection.down] = part_connect.get_last_revision_number(args,match_connection.down,False)
revq = self.last_revisions[match_connection.down]
connection_dict[match_connection.down] = {'rev':revq,
'a_ports':[], 'up_parts':[], 'up_rev':[], 'b_on_up':[],
'b_ports':[], 'down_parts':[], 'down_rev':[],'a_on_down':[],
'start_on_down':[], 'stop_on_down':[], 'repr_down':[],
'start_on_up':[], 'stop_on_up':[], 'repr_up':[]}
connection_dict[match_connection.down]['a_ports'].append(match_connection.a_on_down)
connection_dict[match_connection.down]['up_parts'].append(match_connection.up)
connection_dict[match_connection.down]['up_rev'].append(match_connection.up_rev)
connection_dict[match_connection.down]['b_on_up'].append(match_connection.b_on_up)
connection_dict[match_connection.down]['start_on_up'].append(match_connection.start_date)
connection_dict[match_connection.down]['stop_on_up'].append(match_connection.stop_date)
connection_dict[match_connection.down]['repr_up'].append(match_connection.__repr__())
connection_dict = self.__check_ports(args,connection_dict,rev_query)
for pkey in connection_dict.keys():
if pkey not in self.connections_dictionary.keys():
self.connections_dictionary[pkey] = copy.copy(connection_dict[pkey])
if show_connection:
self.show_connection(args, connection_dict)
if return_dictionary:
return connection_dict
def main():
if platform.python_version().startswith("3"):
hostname = os.uname().nodename
else:
hostname = os.uname()[1]
# templates are stored relative to the script dir
# stored one level up, find the parent directory
# and split the parent directory away
script_dir = os.path.dirname(os.path.realpath(__file__))
split_dir = os.path.split(script_dir)
template_dir = os.path.join(split_dir[0], "templates")
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True)
parser = mc.get_mc_argument_parser()
args = parser.parse_args()
db = mc.connect_to_mc_db(args)
session = db.sessionmaker()
# get the most recent observation logged by the correlator
most_recent_obs = session.get_obs_by_time()[0]
# load all rows of a table into a list
# make rows from the row object
table = []
dt = Time.now().gps - Time(
most_recent_obs.starttime, format="gps", scale="utc").gps
dt_days = int(floor((dt / 3600.0) / 24))
dt_hours = (dt - dt_days * 3600 * 24) / 3600.0
last_obs_row = row(
label="Time Since Last Obs",
text="{dt_days} days {dt_hours} hours".format(dt_days=dt_days,
dt_hours=int(dt_hours)),
)
table.append(last_obs_row)
# get the number of raw files in the last 24 hours
numfiles = (session.query(LibFiles).filter(
LibFiles.time > (Time.now() -
TimeDelta(Quantity(1, "day"))).gps).filter(
LibFiles.filename.like("%uvh5")).count())
nfiles_row = row(label="Raw Files Recorded (last 24 hours)", text=numfiles)
table.append(nfiles_row)
# get the number of samples recorded by each node in the last 24 hours
result = (session.query(NodeSensor.node, func.count(
NodeSensor.time)).filter(
NodeSensor.time > (Time.now() -
TimeDelta(Quantity(1, "day"))).gps).group_by(
NodeSensor.node))
node_pings = ""
for l in result:
node_pings += "Node{node}:{pings} ".format(node=l[0], pings=l[1])
ping_row = row(label="Node Sensor Readings (last 24 hours)",
text=node_pings)
table.append(ping_row)
# get the current state of is_recording()
result = (session.query(
CorrelatorControlState.state, CorrelatorControlState.time).filter(
CorrelatorControlState.state_type.like("taking_data")).order_by(
CorrelatorControlState.time.desc()).limit(1).one())
is_recording = result[0]
last_update = Time(result[1], scale="utc", format="gps")
on_off_row = row(label="Correlator is")
# retro palette from https://venngage.com/blog/color-blind-friendly-palette/
if is_recording:
on_off_row.text = "ON"
on_off_row.color = "#63ACBE"
else:
on_off_row.text = "OFF"
on_off_row.color = "#EE442f"
on_off_row.text += " (last change: {})".format(last_update.iso)
table.append(on_off_row)
html_template = env.get_template("mc_stat_table.html")
rendered_html = html_template.render(
table=table,
gen_date=Time.now().iso,
gen_time_unix_ms=Time.now().unix * 1000,
scriptname=os.path.basename(__file__),
hostname=hostname,
)
with open("mc_html_summary.html", "w") as h_file:
h_file.write(rendered_html)
def main():
# templates are stored relative to the script dir
# stored one level up, find the parent directory
# and split the parent directory away
script_dir = os.path.dirname(os.path.realpath(__file__))
split_dir = os.path.split(script_dir)
template_dir = os.path.join(split_dir[0], 'templates')
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True)
# this filter is used to see if there is more than one table
env.filters['islist'] = is_list
if sys.version_info[0] < 3:
# py2
computer_hostname = os.uname()[1]
else:
# py3
computer_hostname = os.uname().nodename
# The standard M&C argument parser
parser = mc.get_mc_argument_parser()
# we'll have to add some extra options too
parser.add_argument(
'--redishost',
dest='redishost',
type=str,
default='redishost',
help=(
'The host name for redis to connect to, defualts to "redishost"'))
parser.add_argument('--port',
dest='port',
type=int,
default=6379,
help='Redis port to connect.')
args = parser.parse_args()
try:
db = mc.connect_to_mc_db(args)
except RuntimeError as e:
raise SystemExit(str(e))
try:
redis_db = redis.Redis(args.redishost, port=args.port)
redis_db.keys()
except Exception as err:
raise SystemExit(str(err))
with db.sessionmaker() as session:
corr_cm = hera_corr_cm.HeraCorrCM(redishost=args.redishost)
hsession = cm_sysutils.Handling(session)
stations = hsession.get_connected_stations(at_date='now')
antpos = np.genfromtxt(os.path.join(mc.data_path, "HERA_350.txt"),
usecols=(0, 1, 2, 3),
dtype={
'names': ('ANTNAME', 'EAST', 'NORTH', 'UP'),
'formats': ('<U5', '<f8', '<f8', '<f8')
},
encoding=None)
antnames = antpos['ANTNAME']
inds = [int(j[2:]) for j in antnames]
inds = np.argsort(inds)
antnames = np.take(antnames, inds)
ants = []
for station in stations:
if station.antenna_number not in ants:
ants = np.append(ants, station.antenna_number)
ants = np.unique(ants).astype(int)
hostname_lookup = {}
# all_snap_statuses = session.get_snap_status(most_recent=True)
all_snap_statuses = corr_cm.get_f_status()
snapautos = {}
ant_status_from_snaps = corr_cm.get_ant_status()
all_snaprf_stats = corr_cm.get_snaprf_status()
table_snap = {}
table_snap["title"] = "Snap hookups with No Data"
rows = []
bad_snaps = []
for snap_chan in all_snaprf_stats:
host, loc_num = snap_chan.split(":")
loc_num = int(loc_num)
# initialize this key if not already in the dict
auto_group = snapautos.setdefault(host, {})
try:
tmp_auto = np.array(
all_snaprf_stats[snap_chan]["autocorrelation"])
if np.all(tmp_auto == "None"):
print("No Data for {} port {}".format(host, loc_num))
bad_snaps.append(snap_chan)
tmp_auto = np.full(1024, np.nan)
eq_coeffs = np.array(all_snaprf_stats[snap_chan]["eq_coeffs"])
if np.all(eq_coeffs == "None"):
eq_coeffs = np.full_like(tmp_auto, 1.0)
tmp_auto /= eq_coeffs**2
tmp_auto = np.ma.masked_invalid(10 *
np.log10(np.real(tmp_auto)))
auto_group[loc_num] = tmp_auto.filled(-50)
except KeyError:
print("Snap connection with no autocorrelation", snap_chan)
raise
except TypeError:
print("Received TypeError when taking log.")
print("Type of item in dictionary: ",
type(all_snaprf_stats[snap_chan]["autocorrelation"]))
print("Value of item: ", tmp_auto)
raise
hostname_lookup.setdefault(host, {})
hostname_lookup[host].setdefault(loc_num, {})
hostname_lookup[host][loc_num]['MC'] = 'NC'
row = {}
row["text"] = '\t'.join(bad_snaps)
rows.append(row)
table_snap["rows"] = rows
table_ants = {}
table_ants["title"] = "Antennas with no mapping"
rows = []
bad_ants = []
bad_hosts = []
for ant_cnt, ant in enumerate(ants):
# get the status for both polarizations for this antenna
ant_status = {
key: ant_status_from_snaps[key]
for key in ant_status_from_snaps
if ant == int(key.split(':')[0])
}
# check if the antenna status from M&C has the host and
# channel number, if it does not we have to do some gymnastics
for antkey in ant_status:
pol_key = antkey.split(":")[1]
stat = ant_status[antkey]
name = "{ant:d}:{pol}".format(ant=ant, pol=pol_key)
# check that key is in the dictionary, is not None or the string "None"
if ('f_host' in stat and 'host_ant_id' in stat
and stat['f_host'] is not None
and stat['host_ant_id'] is not None
and stat['f_host'] != "None"
and stat['host_ant_id'] != "None"):
hostname = stat['f_host']
loc_num = stat['host_ant_id']
hostname_lookup[hostname][loc_num]['MC'] = name
else:
# Try to get the snap info from M&C. Output is a dictionary with 'e' and 'n' keys
# connect to M&C to find all the hooked up Snap hostnames and corresponding ant-pols
mc_name = antnames[ant]
# these two may not be used, but it is easier to grab them now
snap_info = hsession.get_part_at_station_from_type(
mc_name, 'now', 'snap', include_ports=True)
for _key in snap_info.keys():
# initialize a dict if they key does not exist already
pol_key = pol_key.upper() + "<ground"
if snap_info[_key][pol_key] is not None:
serial_with_ports = snap_info[_key][pol_key]
snap_serial = serial_with_ports.split(
'>')[1].split('<')[0]
ant_channel = int(
serial_with_ports.split('>')[0][1:]) // 2
hostname = session.get_snap_hostname_from_serial(
snap_serial)
if hostname is None:
node_info = hsession.get_part_at_station_from_type(
mc_name, 'now', 'node')
_node_num = re.findall(
r'N(\d+)', node_info[_key][pol_key])[0]
snap_stats = session.get_snap_status(
nodeID=int(_node_num), most_recent=True)
for _stat in snap_stats:
if _stat.serial_number == snap_serial:
hostname = _stat.hostname
# if hostname is still None then we don't have a known hookup
if hostname is None:
print("No MC snap information for antennna: " +
name)
bad_ants.append(name)
else:
if hostname not in hostname_lookup.keys():
err = "host from M&C not found in corr_cm 'status:snaprf' : {}".format(
hostname)
err += '\nThis host may not have data populated yet or is offline.'
err += '\nAll anteanns on this host will be full of 0.'
print(err)
bad_hosts.append(hostname)
grp1 = hostname_lookup.setdefault(hostname, {})
# if this loc num is not in lookup table initialize
# empty list
if ant_channel not in grp1.keys():
if hostname not in bad_hosts:
print(
"loc_num from M&C not found in hera_corr_cm `status:snaprf` (host, location number): {}"
.format([hostname, ant_channel]))
print(
"filling with bad array full of 0."
)
else:
_name = "{host}:{loc}".format(
host=hostname, loc=ant_channel)
if _name not in table_snap["rows"][0][
"text"]:
table_snap["rows"][0][
"text"] += _name + '\t'
snap_grp1 = snapautos.setdefault(
hostname, {})
snap_grp1[ant_channel] = np.full(1024, 0)
grp2 = grp1.setdefault(ant_channel, {})
grp2['MC'] = name
else:
print("No MC snap information for antennna: " +
name)
bad_ants.append(name)
row = {}
row["text"] = '\t'.join(bad_ants)
rows.append(row)
table_ants["rows"] = rows
host_masks = []
for h1 in sorted(hostname_lookup.keys()):
_mask = []
for h2 in sorted(hostname_lookup.keys()):
_mask.extend([
True if h2 == h1 else False
for loc_num in hostname_lookup[h2]
])
host_masks.append(_mask)
# Generate frequency axis
freqs = np.linspace(0, 250e6, 1024)
freqs /= 1e6
data = []
for host_cnt, host in enumerate(sorted(hostname_lookup.keys())):
if host_cnt == 0:
visible = True
else:
visible = False
for loc_num in sorted(hostname_lookup[host].keys()):
mc_name = hostname_lookup[host][loc_num]['MC']
name = '{loc}:{mcname}'.format(loc=loc_num,
mcname=mc_name.replace(":", ""))
try:
_data = {
"x": freqs.tolist(),
"y": snapautos[host][loc_num].tolist(),
"name": name,
"visible": visible,
"hovertemplate": "%{x:.1f}\tMHz<br>%{y:.3f}\t[dB]"
}
except KeyError:
print("Given host, location pair: ({0}, {1})".format(
host, loc_num))
print("All possible keys for host {0}: {1}".format(
host, list(snapautos[host].keys())))
raise
data.append(_data)
buttons = []
for host_cnt, host in enumerate(sorted(hostname_lookup.keys())):
prog_time = all_snap_statuses[host]['last_programmed']
timestamp = all_snap_statuses[host]['timestamp']
temp = all_snap_statuses[host]['temp']
uptime = all_snap_statuses[host]['uptime']
pps = all_snap_statuses[host]['pps_count']
if host in bad_hosts:
label = ('{host}<br>programmed:\t{start}'
'<br>spectra\trecorded:\tNO DATA OBSERVED<br>'
'temp:\t{temp:.0f}\tC\t'
'pps\tcount:\t{pps}\tCycles\t\t\t'
'uptime:\t{uptime}'
''.format(host=host,
start=prog_time.isoformat(' '),
temp=temp,
pps=pps,
uptime=uptime))
else:
label = ('{host}<br>programmed:\t{start}'
'<br>spectra\trecorded:\t{obs}<br>'
'temp:\t{temp:.0f}\tC\t'
'pps\tcount:\t{pps}\tCycles\t\t\t'
'uptime:\t{uptime}'
''.format(host=host,
start=prog_time.isoformat(' '),
obs=timestamp.isoformat(' '),
temp=temp,
pps=pps,
uptime=uptime))
_button = {
"args": [
{
"visible": host_masks[host_cnt]
},
{
"title": '',
# "annotations": {}
}
],
"label":
label,
"method":
"restyle"
}
buttons.append(_button)
updatemenus = [{
"buttons": buttons,
"showactive": True,
"type": "dropdown",
"x": .7,
"y": 1.1,
}]
layout = {
"xaxis": {
"title": "Frequency [MHz]",
"showticklabels": True,
"tick0": 0,
"dtick": 10,
},
"yaxis": {
"title": 'Power [dB]',
"showticklabels": True,
},
"hoverlabel": {
"align": "left"
},
"margin": {
"l": 40,
"b": 30,
"r": 40,
"t": 30
},
"autosize":
True,
"showlegend":
True,
"hovermode":
'closest',
"annotations": [{
"x": 1.03,
"y": 1.03,
"align": "right",
"valign": "top",
"text": 'ADC Port # : Antpol',
"showarrow": False,
"xref": "paper",
"yref": "paper",
"xanchor": "center",
"yanchor": "top"
}]
}
caption = {}
caption["title"] = "Snap Spectra Help"
caption["text"] = (
"Autocorrelations (in dB) calculated by each Snap "
"with equalization coefficients divided out. "
"Calculation is independent of the corrlator. "
"<br><br>"
"Data is organized by the ADC Port number from "
"each Snap. "
"A mapping dictionary is used to look up the "
"associated ant-pol for each ADC Port. "
"Some ADC Ports may not have a known ant-pol "
"and are labeled as N/C."
"<br>Some antennas known to M&C may not have a known ADC port"
" mapping and are labeled below the plot."
"<br><br>Some Snap ports my not have data and are "
"labeled in a table below the plot."
"<br><br>The node selection button contains a lot "
"of information described here."
"<br><h4>Button Label Information</h4>"
"<ul>"
"<li>Snap hostname</li>"
"<li>Last time the Snap was programmed</li>"
"<li>Last time data was recorded from this Snap</li>"
"<li>Last known temperature in C | PPS count | Uptime</li>"
"</ul>")
plotname = "plotly-snap"
html_template = env.get_template("refresh_with_table.html")
js_template = env.get_template("plotly_base.js")
rendered_html = html_template.render(
plotname=plotname,
plotstyle="height: 100%",
gen_date=Time.now().iso,
js_name='snapspectra',
gen_time_unix_ms=Time.now().unix * 1000,
scriptname=os.path.basename(__file__),
hostname=computer_hostname,
table=[table_snap, table_ants],
caption=caption)
rendered_js = js_template.render(data=data,
layout=layout,
updatemenus=updatemenus,
plotname=plotname)
with open('snapspectra.html', 'w') as h_file:
h_file.write(rendered_html)
with open('snapspectra.js', 'w') as js_file:
js_file.write(rendered_js)
def main():
# templates are stored relative to the script dir
# stored one level up, find the parent directory
# and split the parent directory away
script_dir = os.path.dirname(os.path.realpath(__file__))
split_dir = os.path.split(script_dir)
template_dir = os.path.join(split_dir[0], "templates")
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True)
if sys.version_info[0] < 3:
# py2
computer_hostname = os.uname()[1]
else:
# py3
computer_hostname = os.uname().nodename
parser = mc.get_mc_argument_parser()
args = parser.parse_args()
try:
db = mc.connect_to_mc_db(args)
except RuntimeError as e:
raise SystemExit(str(e))
colsize = 6
TIME_WINDOW = 14 # days
now = Time.now()
cutoff = now - TimeDelta(TIME_WINDOW, format="jd")
time_axis_range = [cutoff.isot, now.isot]
plotnames = [
["server-loads", "upload-ages"],
["disk-space", "bandwidths"],
["num-files", "ping-times"],
["file-compare"],
]
layout = {
"xaxis": {"range": time_axis_range},
"yaxis": {"title": "Load % per CPU"},
"title": {"text": "NONE"},
"height": 200,
"margin": {"t": 30, "r": 50, "b": 2, "l": 50},
"legend": {"orientation": "h", "x": 0.15, "y": -0.15},
"showlegend": True,
"hovermode": "closest",
}
html_template = env.get_template("librarian_table.html")
js_template = env.get_template("plotly_base.js")
with db.sessionmaker() as session:
data = do_server_loads(session, cutoff)
layout["title"]["text"] = "CPU Loads"
rendered_js = js_template.render(
plotname="server-loads", data=data, layout=layout
)
with open("librarian.js", "w") as js_file:
js_file.write(rendered_js)
js_file.write("\n\n")
data = do_upload_ages(session, cutoff)
layout["yaxis"]["title"] = "Minutes"
layout["yaxis"]["zeroline"] = False
layout["title"]["text"] = "Time Since last upload"
rendered_js = js_template.render(
plotname="upload-ages", data=data, layout=layout
)
with open("librarian.js", "a") as js_file:
js_file.write(rendered_js)
js_file.write("\n\n")
data = do_disk_space(session, cutoff)
layout["yaxis"]["title"] = "Data Volume [GiB]"
layout["yaxis"]["zeroline"] = True
layout["yaxis2"] = {
"title": "Free Space [GiB]",
"overlaying": "y",
"side": "right",
}
layout["title"]["text"] = "Disk Usage"
rendered_js = js_template.render(
plotname="disk-space", data=data, layout=layout
)
with open("librarian.js", "a") as js_file:
js_file.write(rendered_js)
js_file.write("\n\n")
layout.pop("yaxis2", None)
data = do_bandwidths(session, cutoff)
layout["yaxis"]["title"] = "MB/s"
layout["title"]["text"] = "Librarian Transfer Rates"
rendered_js = js_template.render(
plotname="bandwidths", data=data, layout=layout
)
with open("librarian.js", "a") as js_file:
js_file.write(rendered_js)
js_file.write("\n\n")
data = do_num_files(session, cutoff)
layout["yaxis"]["title"] = "Number"
layout["yaxis"]["zeroline"] = False
layout["title"]["text"] = "Total Number of Files in Librarian"
rendered_js = js_template.render(plotname="num-files", data=data, layout=layout)
with open("librarian.js", "a") as js_file:
js_file.write(rendered_js)
js_file.write("\n\n")
data = do_ping_times(session, cutoff)
layout["yaxis"]["title"] = "ms"
layout["yaxis"]["rangemode"] = "tozero"
layout["yaxis"]["zeroline"] = True
layout["title"]["text"] = "Server Ping Times"
rendered_js = js_template.render(
plotname="ping-times", data=data, layout=layout
)
with open("librarian.js", "a") as js_file:
js_file.write(rendered_js)
js_file.write("\n\n")
data = do_compare_file_types(TIME_WINDOW)
if data is not None:
layout["yaxis"]["title"] = "Files in <br><b>temporary staging</b>"
layout["yaxis"]["zeroline"] = True
layout["margin"]["l"] = 60
layout["title"]["text"] = "Files in <b>Temporary Staging</b>"
rendered_js = js_template.render(
plotname="file-compare", data=data, layout=layout
)
with open("librarian.js", "a") as js_file:
js_file.write(rendered_js)
js_file.write("\n\n")
tables = []
tables.append(do_raid_errors(session, cutoff))
tables.append(do_raid_status(session, cutoff))
caption = {}
caption["title"] = "Librarian Help"
caption["text"] = (
"An overview of many Librarian related statisics. "
"<br>The Plots are organized as follows: "
"""<div class="table-responsive">
<table class="table table-striped" style="border:1px solid black; border-top; 1px solid black;">
<tbody>
<tr>
<td style="border-left:1px solid black;">CPU Load
<td style="border-left:1px solid black;">Time Since last Upload</td>
</tr>
<tr>
<td style="border-left:1px solid black;">Disk Space Usage
<td style="border-left:1px solid black;">AOC Transfer Speeds</td></tr>
<tr>
<td style="border-left:1px solid black;">Total Number of files in Librarian
<td style="border-left:1px solid black;">AOC ping time</td></tr>
<tr>
<td style="border-left:1px solid black;">Files in <b>Temporary Storage</b>
<td style="border-left:1px solid black;"></td></tr>
</tbody>
</table>
</div>
"""
"The files in temporary storage counts the number of raw files at /mnt/sn1 "
"on qmaster and compares that to the number of processed files whose "
"JDs are >= oldest raw file.<br>It is a hacky proxy for 'DID RTP RUN?' "
"<br>Assuming RTP runs successfully on all files from an observation, "
"both lines should report the same number of files."
"<br><br>"
"The final two tables give some recent RAID errors and status reports."
)
rendered_html = html_template.render(
plotname=plotnames,
title="Librarian",
plotstyle="height: 24.5%",
colsize=colsize,
gen_date=now.iso,
gen_time_unix_ms=now.unix * 1000,
js_name="librarian",
hostname=computer_hostname,
scriptname=os.path.basename(__file__),
tables=tables,
caption=caption,
)
with open("librarian.html", "w") as h_file:
h_file.write(rendered_html)
def main():
# templates are stored relative to the script dir
# stored one level up, find the parent directory
# and split the parent directory away
script_dir = os.path.dirname(os.path.realpath(__file__))
split_dir = os.path.split(script_dir)
template_dir = os.path.join(split_dir[0], "templates")
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True)
if sys.version_info[0] < 3:
# py2
computer_hostname = os.uname()[1]
else:
# py3
computer_hostname = os.uname().nodename
parser = mc.get_mc_argument_parser()
args = parser.parse_args()
try:
db = mc.connect_to_mc_db(args)
except RuntimeError as e:
raise SystemExit(str(e))
plotnames = [
["am-xants", "am-meanVij"],
["am-redCorr", "am-meanVijXpol"],
["fc-agg_std", "fc-max_std"],
["oc-ant_phs_std_max", "oc-chisq_tot_avg"],
]
colsize = 6
TIME_WINDOW = 14 # days
now = Time.now()
cutoff = now - TimeDelta(TIME_WINDOW, format="jd")
time_axis_range = [cutoff.isot, now.isot]
caption = {}
caption["title"] = "Daily Quality Metrics"
caption["text"] = (
"An overview of the Daily Quality metrics run on data from RTP. "
"A plot with no data indicates that particular metric was not run, "
"or the data could not be found."
"<br><br>"
"Currenly only the Ant Metrics are run on data during RTP."
"""<div class="table-responsive">
<table class="table table-striped" style="border:1px solid black; border-top; 1px solid black;">
<tbody>
<tr>
<td style="border-left:1px solid black;">Ant Metrics Number of Excluded Ant-pols
<td style="border-left:1px solid black;">Ant Metrics Mean Vij Ampltiude</td>
</tr>
<tr>
<td style="border-left:1px solid black;">Ant Metrics red Corr Mean Ampltidue
<td style="border-left:1px solid black;">Ant Metrics Mean Vij Cross Pol Ampltidue</td></tr>
<tr>
<td style="border-left:1px solid black;">Frist Cal Agg Std
<td style="border-left:1px solid black;">First Cal Max Std</td></tr>
<tr>
<td style="border-left:1px solid black;">Ominical Phase Std Max
<td style="border-left:1px solid black;">Omnical Chi-square Total Average</td></tr>
</tbody>
</table>
</div>
""")
html_template = env.get_template("plotly_base.html")
rendered_html = html_template.render(
plotname=plotnames,
plotstyle="height: 24.5%",
colsize=colsize,
gen_date=now.iso,
gen_time_unix_ms=now.unix * 1000,
js_name="qm",
hostname=computer_hostname,
scriptname=os.path.basename(__file__),
caption=caption,
)
with open("qm.html", "w") as h_file:
h_file.write(rendered_html)
js_template = env.get_template("plotly_base.js")
with db.sessionmaker() as session:
layout = {
"xaxis": {
"range": time_axis_range
},
"yaxis": {
"title": "placeholder"
},
"title": {
"text": "placeholder"
},
"height": 200,
"margin": {
"t": 30,
"r": 10,
"b": 20,
"l": 40
},
"legend": {
"orientation": "h",
"x": 0.15,
"y": -0.15
},
"showlegend": False,
"hovermode": "closest",
}
# If an antpol is detected as bad (`val` not used).
data = do_ant_metric(
session,
"ant_metrics_xants",
sqlalchemy.func.count(),
ymode="markers",
yname="Data",
cutoff=cutoff,
)
layout["yaxis"]["title"] = "Count"
layout["title"]["text"] = "Ant Metrics # of Xants"
rendered_js = js_template.render(plotname="am-xants",
data=data,
layout=layout)
with open("qm.js", "w") as js_file:
js_file.write(rendered_js)
js_file.write("\n\n")
# "Mean of the absolute value of all visibilities associated with an
# antenna".
data = do_ant_metric(
session,
"ant_metrics_meanVij",
sqlalchemy.func.avg(AntMetrics.val),
yname="Data",
cutoff=cutoff,
)
layout["yaxis"]["title"] = "Average Amplitude"
layout["title"]["text"] = "Ant Metrics MeanVij"
rendered_js = js_template.render(plotname="am-meanVij",
data=data,
layout=layout)
with open("qm.js", "a") as js_file:
js_file.write(rendered_js)
js_file.write("\n\n")
# "Extent to which baselines involving an antenna do not correlate
# with others they are nominmally redundant with".
data = do_ant_metric(
session,
"ant_metrics_redCorr",
sqlalchemy.func.avg(AntMetrics.val),
yname="Data",
cutoff=cutoff,
)
layout["yaxis"]["title"] = "Average Amplitude"
layout["title"]["text"] = "Ant Metrics redCorr"
rendered_js = js_template.render(plotname="am-redCorr",
data=data,
layout=layout)
with open("qm.js", "a") as js_file:
js_file.write(rendered_js)
js_file.write("\n\n")
# "Ratio of mean cross-pol visibilities to mean same-pol visibilities:
# (Vxy+Vyx)/(Vxx+Vyy)".
data = do_ant_metric(
session,
"ant_metrics_meanVijXPol",
sqlalchemy.func.avg(AntMetrics.val),
yname="Data",
cutoff=cutoff,
)
layout["yaxis"]["title"] = "Average Amplitude"
layout["title"]["text"] = "Ant Metrics MeanVij CrossPol"
rendered_js = js_template.render(plotname="am-meanVijXpol",
data=data,
layout=layout)
with open("qm.js", "a") as js_file:
js_file.write(rendered_js)
js_file.write("\n\n")
# "Aggregate standard deviation of delay solutions".
data = do_xy_array_metric(session,
"firstcal_metrics_agg_std",
cutoff=cutoff)
layout["yaxis"]["title"] = "std"
layout["title"]["text"] = "FirstCal Metrics Agg Std"
rendered_js = js_template.render(plotname="fc-agg_std",
data=data,
layout=layout)
with open("qm.js", "a") as js_file:
js_file.write(rendered_js)
js_file.write("\n\n")
# "Maximum antenna standard deviation of delay solutions".
data = do_xy_array_metric(session,
"firstcal_metrics_max_std",
cutoff=cutoff)
layout["yaxis"]["title"] = "FC max_std"
layout["title"]["text"] = "FirstCal Metrics Max Std"
rendered_js = js_template.render(plotname="fc-max_std",
data=data,
layout=layout)
with open("qm.js", "a") as js_file:
js_file.write(rendered_js)
js_file.write("\n\n")
# Maximum of "gain phase standard deviation per-antenna across file".
data = do_xy_array_metric(
session,
"omnical_metrics_ant_phs_std_max",
doubled_suffix=True,
cutoff=cutoff,
)
layout["yaxis"]["title"] = "OC ant_phs_std_max"
layout["title"]["text"] = "OmniCal Metrics Ant Phase Std max"
rendered_js = js_template.render(plotname="oc-ant_phs_std_max",
data=data,
layout=layout)
with open("qm.js", "a") as js_file:
js_file.write(rendered_js)
js_file.write("\n\n")
# "Median of chi-square across entire file".
data = do_xy_array_metric(session,
"omnical_metrics_chisq_tot_avg",
doubled_suffix=True,
cutoff=cutoff)
layout["yaxis"]["title"] = "OC chisq_tot_avg"
layout["title"]["text"] = "OmniCal Metrics Chi-square total avg"
rendered_js = js_template.render(plotname="oc-chisq_tot_avg",
data=data,
layout=layout)
with open("qm.js", "a") as js_file:
js_file.write(rendered_js)
js_file.write("\n\n")
def main():
# templates are stored relative to the script dir
# stored one level up, find the parent directory
# and split the parent directory away
script_dir = os.path.dirname(os.path.realpath(__file__))
split_dir = os.path.split(script_dir)
template_dir = os.path.join(split_dir[0], "templates")
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True)
# this filter is used to see if there is more than one table
env.filters["islist"] = is_list
if sys.version_info[0] < 3:
# py2
computer_hostname = os.uname()[1]
else:
# py3
computer_hostname = os.uname().nodename
parser = mc.get_mc_argument_parser()
parser.add_argument(
"--redishost",
dest="redishost",
type=str,
default="redishost",
help=(
'The host name for redis to connect to, defaults to "redishost"'),
)
parser.add_argument("--port",
dest="port",
type=int,
default=6379,
help="Redis port to connect.")
args = parser.parse_args()
try:
db = mc.connect_to_mc_db(args)
except RuntimeError as e:
raise SystemExit(str(e))
try:
redis_db = redis.Redis(args.redishost, port=args.port)
redis_db.keys()
except Exception as err:
raise SystemExit(str(err))
with db.sessionmaker() as session:
now = Time.now()
hsession = cm_sysutils.Handling(session)
stations = hsession.get_connected_stations(at_date="now")
ants = []
for station in stations:
if station.antenna_number not in ants:
ants = np.append(ants, station.antenna_number)
ants = np.unique(ants).astype(int)
antpos = np.genfromtxt(
os.path.join(mc.data_path, "HERA_350.txt"),
usecols=(0, 1, 2, 3),
dtype={
"names": ("ANTNAME", "EAST", "NORTH", "UP"),
"formats": ("<U5", "<f8", "<f8", "<f8"),
},
encoding=None,
)
antnames = antpos["ANTNAME"]
inds = [int(j[2:]) for j in antnames]
inds = np.argsort(inds)
antnames = np.take(antnames, inds)
nodes = []
hists = []
bad_ants = []
bad_node = []
for ant_cnt, ant in enumerate(ants):
ant_status = session.get_antenna_status(most_recent=True,
antenna_number=int(ant))
mc_name = antnames[int(ant)]
node_info = hsession.get_part_at_station_from_type(
mc_name, "now", "node")
if len(ant_status) == 0:
for pol in ["e", "n"]:
name = "{ant}:{pol}".format(ant=ant, pol=pol)
print("No histogram data for ", name)
bad_ants.append(name)
for stat in ant_status:
name = "ant{ant}{pol}".format(ant=stat.antenna_number,
pol=stat.antenna_feed_pol)
# try to find the associated node
# get the first key in the dict to index easier
_key = list(node_info.keys())[0]
pol_key = [
key for key in node_info[_key].keys()
if stat.antenna_feed_pol.upper() in key
]
if pol_key:
# 'E' should be in one of the keys, extract the 0th entry
pol_key = pol_key[0]
else:
# a hacky solution for a key that should work
pol_key = "E<ground"
if node_info[_key][pol_key] is not None:
_node_num = re.findall(r"N(\d+)",
node_info[_key][pol_key])[0]
else:
print("No Node mapping for antennna: " + name)
_node_num = -1
bad_node.append("{ant}:{pol}".format(
ant=stat.antenna_number, pol=stat.antenna_feed_pol))
nodes.append(_node_num)
timestamp = Time(stat.time, format="gps")
if (stat.histogram_bin_centers is not None
and stat.histogram is not None):
bins = np.fromstring(
stat.histogram_bin_centers.strip("[]"), sep=",")
hist = np.fromstring(stat.histogram.strip("[]"), sep=",")
if stat.eq_coeffs is not None:
eq_coeffs = np.fromstring(stat.eq_coeffs.strip("[]"),
sep=",")
else:
eq_coeffs = np.ones_like(hist)
hist /= np.median(eq_coeffs)**2
text = "observed at {iso}<br>(JD {jd})".format(
iso=timestamp.iso, jd=timestamp.jd)
# spaces cause weird wrapping issues, replace them all with \t
text = text.replace(" ", "\t")
_data = {
"x": bins.tolist(),
"y": hist.tolist(),
"name": name,
"node": _node_num,
"text": [text] * bins.size,
"hovertemplate": "(%{x:.1},\t%{y})<br>%{text}",
}
hists.append(_data)
else:
name = "{ant}:{pol}".format(ant=stat.antenna_number,
pol=stat.antenna_feed_pol)
print("No histogram data for ", name)
bad_ants.append(name)
table = {}
table["title"] = "Ants with no Histogram"
table["rows"] = []
row = {}
row["text"] = ",\t".join(bad_ants)
table["rows"].append(row)
table_node = {}
table_node["title"] = "Antennas with no Node mapping"
row_node = {}
row_node["text"] = "\t".join(bad_node)
rows_node = [row_node]
table_node["rows"] = rows_node
layout = {
"xaxis": {
"title": "ADC value"
},
"yaxis": {
"title": "Occurance",
"type": "linear"
},
"title": {
"text": "ADC Histograms",
"xref": "paper",
"x": 0.5,
"yref": "paper",
"y": 1.5,
"font": {
"size": 24,
},
},
"margin": {
"l": 40,
"b": 30,
"r": 40,
"t": 70
},
"hovermode": "closest",
"autosize": True,
"showlegend": True,
}
# Make all the buttons for this plot
nodes = np.unique(nodes)
# if an antenna was not mapped, roll the -1 to the end
# this makes making buttons easier so the unmapped show last
if -1 in nodes:
nodes = np.roll(nodes, -1)
# create a mask to find all the matching nodes
node_mask = [[True if s["node"] == node else False for s in hists]
for node in nodes]
buttons_node = []
_button_node = {
"args": [
{
"visible": [True for s in hists]
},
{
"title": "",
"annotations": {}
},
],
"label":
"All\tAnts",
"method":
"restyle",
}
buttons_node.append(_button_node)
for node_cnt, node in enumerate(nodes):
if node != -1:
label = "Node\t{}".format(node)
else:
label = "Unmapped\tAnts"
_button_node = {
"args": [
{
"visible": node_mask[node_cnt]
},
{
"title": "",
"annotations": {}
},
],
"label":
label,
"method":
"restyle",
}
buttons_node.append(_button_node)
buttons = []
log_buttons = {
"args": [{}, {
"yaxis": {
"type": "log"
}
}],
"label": "Log",
"method": "update",
}
lin_buttons = {
"args": [{}, {
"yaxis": {
"type": "linear"
}
}],
"label": "Linear",
"method": "update",
}
buttons.append(lin_buttons)
buttons.append(log_buttons)
updatemenus = [
{
"buttons": buttons_node,
"showactive": True,
"active": 0,
"type": "dropdown",
"x": 0.535,
"y": 1.03,
},
{
"buttons": buttons,
"showactive": True,
"type": "buttons",
"active": 0,
},
]
plotname = "plotly-adc-hist"
html_template = env.get_template("ploty_with_multi_table.html")
js_template = env.get_template("plotly_base.js")
caption = {}
caption["text"] = (
"The ADC Histograms with equalization coefficients "
"divided out."
"<br><br>Some antennas known to M&C may not have a histogram "
" and are listed below the image."
"<br><br>Some antennas may not have "
"a known node mapping and are listed below the image.\n "
"<br><br>Plot can be downselected to display "
"individual nodes or show the entire array.\n "
"<br><br>Double click on an entry in the legend "
"to select only that entry, "
"double click again to restore all plots.\n "
"<br><br><h4>Formatting options</h4>"
"<ul>"
"<li>Linear - Display with Linear y-axis</li>"
"<li>Log - Display with Log y-axis</li>"
"</ul>"
"<br><br>Single click an entry in the legend to un-plot it, "
"single click again to restore it to the plot.")
caption["title"] = "Histogram Help"
rendered_html = html_template.render(
plotname=plotname,
plotstyle="height: 100%",
gen_date=now.iso,
js_name="adchist",
caption=caption,
gen_time_unix_ms=now.unix * 1000,
scriptname=os.path.basename(__file__),
hostname=computer_hostname,
table=[table, table_node],
)
rendered_js = js_template.render(data=hists,
layout=layout,
plotname=plotname,
updatemenus=updatemenus)
with open("adchist.html", "w") as h_file:
h_file.write(rendered_html)
with open("adchist.js", "w") as js_file:
js_file.write(rendered_js)
