def handle(self, *args, **options):
"""Perform necessary calculation and inser antennas in to DB."""
mc_args = Namespace()
mc_args.mc_db_name = options["mc_db_name"]
mc_args.mc_config_path = options["mc_config_path"]
db = mc.connect_to_mc_db(args=mc_args)
antpos = np.genfromtxt(
os.path.join(mc.data_path, "HERA_350.txt"),
usecols=(0, 1, 2, 3),
dtype={
"names": ("ANTNAME", "EAST", "NORTH", "UP"),
"formats": ("<U5", "<f8", "<f8", "<f8"),
},
encoding=None,
)
antnames = antpos["ANTNAME"]
inds = [int(j[2:]) for j in antnames]
inds = np.argsort(inds)
antnames = np.take(antnames, inds)
antpos = np.array([antpos["EAST"], antpos["NORTH"], antpos["UP"]])
array_center = np.mean(antpos, axis=1, keepdims=True)
antpos -= array_center
antpos = np.take(antpos, inds, axis=1)
with db.sessionmaker() as session:
hsession = cm_sysutils.Handling(session)
stations = []
for station_type in hsession.geo.parse_station_types_to_check(
"default"):
for stn in hsession.geo.station_types[station_type][
"Stations"]:
stations.append(stn)
# stations is a list of HH??? numbers we just want the ints
stations = list(map(int, [j[2:] for j in stations]))
bulk_add = []
for ind, name in enumerate(antnames):
ant_number = int(name[2:])
for pol in ["e", "n"]:
bulk_add.append(
Antenna(
ant_number=ant_number,
ant_name=name,
polarization=pol,
antpos_enu=antpos[:, ind].tolist(),
constructed=ant_number in stations,
))
Antenna.objects.bulk_create(bulk_add, ignore_conflicts=True)
def main():
# templates are stored relative to the script dir
# stored one level up, find the parent directory
# and split the parent directory away
script_dir = os.path.dirname(os.path.realpath(__file__))
split_dir = os.path.split(script_dir)
template_dir = os.path.join(split_dir[0], "templates")
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True)
if sys.version_info[0] < 3:
# py2
computer_hostname = os.uname()[1]
else:
# py3
computer_hostname = os.uname().nodename
# The standard M&C argument parser
parser = mc.get_mc_argument_parser()
# we'll have to add some extra options too
parser.add_argument(
"--redishost",
dest="redishost",
type=str,
default="redishost",
help=(
'The host name for redis to connect to, defualts to "redishost"'),
)
parser.add_argument("--port",
dest="port",
type=int,
default=6379,
help="Redis port to connect.")
args = parser.parse_args()
try:
db = mc.connect_to_mc_db(args)
except RuntimeError as e:
raise SystemExit(str(e))
try:
redis_db = redis.Redis(args.redishost, port=args.port)
redis_db.keys()
except Exception as err:
raise SystemExit(str(err))
with db.sessionmaker() as session:
# without item this will be an array which will break database queries
latest = Time(
np.frombuffer(redis_db.get("auto:timestamp"),
dtype=np.float64).item(),
format="jd",
)
latest.out_subfmt = u"date_hm"
now = Time.now()
amps = {}
keys = [
k.decode() for k in redis_db.keys()
if k.startswith(b"auto") and not k.endswith(b"timestamp")
]
for key in keys:
match = re.search(r"auto:(?P<ant>\d+)(?P<pol>e|n)", key)
if match is not None:
ant, pol = int(match.group("ant")), match.group("pol")
d = redis_db.get(key)
if d is not None:
# need to copy because frombuffer creates a read-only array
auto = np.frombuffer(d, dtype=np.float32).copy()
eq_coeff = redis_db.hget(
bytes("eq:ant:{ant}:{pol}".format(ant=ant,
pol=pol).encode()),
"values",
)
if eq_coeff is not None:
eq_coeffs = np.fromstring(
eq_coeff.decode("utf-8").strip("[]"), sep=",")
if eq_coeffs.size == 0:
eq_coeffs = np.ones_like(auto)
else:
eq_coeffs = np.ones_like(auto)
# divide out the equalization coefficients
# eq_coeffs are stored as a length 1024 array but only a
# single number is used. Taking the median to not deal with
# a size mismatch
eq_coeffs = np.median(eq_coeffs)
auto /= eq_coeffs**2
auto[auto < 10**-10.0] = 10**-10.0
auto = np.median(auto)
amps[(ant, pol)] = 10.0 * np.log10(auto)
hsession = cm_sysutils.Handling(session)
ants = np.unique([ant for (ant, pol) in amps.keys()])
pols = np.unique([pol for (ant, pol) in amps.keys()])
antpos = np.genfromtxt(
os.path.join(mc.data_path, "HERA_350.txt"),
usecols=(0, 1, 2, 3),
dtype={
"names": ("ANTNAME", "EAST", "NORTH", "UP"),
"formats": ("<U5", "<f8", "<f8", "<f8"),
},
encoding=None,
)
antnames = antpos["ANTNAME"]
inds = [int(j[2:]) for j in antnames]
inds = np.argsort(inds)
antnames = np.take(antnames, inds)
antpos = np.array([antpos["EAST"], antpos["NORTH"], antpos["UP"]])
array_center = np.mean(antpos, axis=1, keepdims=True)
antpos -= array_center
antpos = np.take(antpos, inds, axis=1)
stations = hsession.get_connected_stations(at_date="now")
for station in stations:
if station.antenna_number not in ants:
ants = np.append(ants, station.antenna_number)
ants = np.unique(ants)
stations = []
for station_type in hsession.geo.parse_station_types_to_check(
"default"):
for stn in hsession.geo.station_types[station_type]["Stations"]:
stations.append(stn)
# stations is a list of HH??? numbers we just want the ints
stations = list(map(int, [j[2:] for j in stations]))
built_but_not_on = np.setdiff1d(stations, ants)
# Get node and PAM info
node_ind = np.zeros_like(ants, dtype=np.int)
pam_ind = np.zeros_like(ants, dtype=np.int)
# defaul the snap name to "No Data"
hostname = np.full_like(ants, "No\tData", dtype=object)
snap_serial = np.full_like(ants, "No\tData", dtype=object)
pam_power = {}
adc_power = {}
adc_rms = {}
time_array = {}
fem_imu_theta = {}
fem_imu_phi = {}
eq_coeffs = {}
for ant in ants:
for pol in pols:
amps.setdefault((ant, pol), np.Inf)
pam_power.setdefault((ant, pol), np.Inf)
adc_power.setdefault((ant, pol), np.Inf)
adc_rms.setdefault((ant, pol), np.Inf)
eq_coeffs.setdefault((ant, pol), np.Inf)
fem_imu_theta.setdefault((ant, pol), np.Inf)
fem_imu_phi.setdefault((ant, pol), np.Inf)
time_array.setdefault((ant, pol), now - Time(0, format="gps"))
for ant_cnt, ant in enumerate(ants):
station_status = session.get_antenna_status(
most_recent=True, antenna_number=int(ant))
for status in station_status:
antpol = (status.antenna_number, status.antenna_feed_pol)
if status.pam_power is not None:
pam_power[antpol] = status.pam_power
if status.adc_power is not None:
adc_power[antpol] = 10 * np.log10(status.adc_power)
if status.adc_rms is not None:
adc_rms[antpol] = status.adc_rms
if status.time is not None:
time_array[antpol] = now - Time(status.time, format="gps")
if status.fem_imu_phi is not None:
fem_imu_phi[antpol] = status.fem_imu_phi
if status.fem_imu_theta is not None:
fem_imu_theta[antpol] = status.fem_imu_theta
if status.eq_coeffs is not None:
_coeffs = np.fromstring(status.eq_coeffs.strip("[]"),
sep=",")
# just track the median coefficient for now
eq_coeffs[antpol] = np.median(_coeffs)
# Try to get the snap info. Output is a dictionary with 'e' and 'n' keys
mc_name = antnames[ant]
snap_info = hsession.get_part_at_station_from_type(
mc_name, "now", "snap")
# get the first key in the dict to index easier
_key = list(snap_info.keys())[0]
pol_key = [key for key in snap_info[_key].keys() if "E" in key]
if pol_key:
# 'E' should be in one of the keys, extract the 0th entry
pol_key = pol_key[0]
else:
# a hacky solution for a key that should work
pol_key = "E<ground"
if snap_info[_key][pol_key] is not None:
snap_serial[ant_cnt] = snap_info[_key][pol_key]
# Try to get the pam info. Output is a dictionary with 'e' and 'n' keys
pam_info = hsession.get_part_at_station_from_type(
mc_name, "now", "post-amp")
# get the first key in the dict to index easier
_key = list(pam_info.keys())[0]
if pam_info[_key][pol_key] is not None:
_pam_num = re.findall(r"PAM(\d+)", pam_info[_key][pol_key])[0]
pam_ind[ant_cnt] = np.int(_pam_num)
else:
pam_ind[ant_cnt] = -1
# Try to get the ADC info. Output is a dictionary with 'e' and 'n' keys
node_info = hsession.get_part_at_station_from_type(
mc_name, "now", "node")
# get the first key in the dict to index easier
_key = list(node_info.keys())[0]
if node_info[_key][pol_key] is not None:
_node_num = re.findall(r"N(\d+)", node_info[_key][pol_key])[0]
node_ind[ant_cnt] = np.int(_node_num)
_hostname = session.get_snap_hostname_from_serial(
snap_serial[ant_cnt])
if _hostname is not None:
hostname[ant_cnt] = _hostname
else:
snap_status = session.get_snap_status(
most_recent=True, nodeID=np.int(_node_num))
for _status in snap_status:
if _status.serial_number == snap_serial[ant_cnt]:
hostname[ant_cnt] = _status.hostname
else:
node_ind[ant_cnt] = -1
pams, _pam_ind = np.unique(pam_ind, return_inverse=True)
nodes, _node_ind = np.unique(node_ind, return_inverse=True)
xs_offline = np.ma.masked_array(
antpos[0, :],
mask=[
True if int(name[2:]) in ants else False for name in antnames
],
)
ys_offline = np.ma.masked_array(antpos[1, :], mask=xs_offline.mask)
name_offline = np.ma.masked_array(
[aname + "<br>OFFLINE" for aname in antnames],
mask=xs_offline.mask,
dtype=object,
)
xs_offline = xs_offline
names = [
"Auto [dB]",
"PAM [dB]",
"ADC [dB]",
"ADC RMS",
"FEM IMU THETA",
"FEM IMU PHI",
"EQ COEF",
]
powers = [
amps,
pam_power,
adc_power,
adc_rms,
fem_imu_theta,
fem_imu_phi,
eq_coeffs,
]
powers = [
np.ma.masked_invalid([[p[ant, pol] for ant in ants]
for pol in pols]) for p in powers
]
write_csv("ant_stats.csv", antnames, ants, pols, names, powers,
built_but_not_on)
time_array = np.array(
[[time_array[ant, pol].to("hour").value for ant in ants]
for pol in pols])
xs = np.ma.masked_array(antpos[0, ants], mask=powers[0][0].mask)
ys = np.ma.masked_array(
[
antpos[1, ants] + 3 * (pol_cnt - 0.5)
for pol_cnt, pol in enumerate(pols)
],
mask=powers[0].mask,
)
_text = np.array(
[[
antnames[ant] + pol + "<br>" + str(hostname[ant_cnt]) +
"<br>" + "PAM\t#:\t" + str(pam_ind[ant_cnt])
for ant_cnt, ant in enumerate(ants)
] for pol_cnt, pol in enumerate(pols)],
dtype="object",
)
# want to format No Data where data was not retrieved for each type of power
for pol_cnt, pol in enumerate(pols):
for ant_cnt, ant in enumerate(ants):
for _name, _power in zip(names, powers):
if not _power.mask[pol_cnt, ant_cnt]:
_text[pol_cnt, ant_cnt] += (
"<br>" + _name +
": {0:.2f}".format(_power[pol_cnt, ant_cnt]))
else:
_text[pol_cnt, ant_cnt] += "<br>" + _name + ": No Data"
if time_array[pol_cnt, ant_cnt] > 2 * 24 * 365:
# if the value is older than 2 years it is bad
# value are stored in hours.
# 2 was chosen arbitraritly.
_text[pol_cnt, ant_cnt] += "<br>" + "Ant Status: No Date"
else:
_text[pol_cnt,
ant_cnt] += ("<br>" +
"Ant Status: {0:.2f} hrs old".format(
time_array[pol_cnt, ant_cnt]))
# having spaces will cause odd wrapping issues, replace all
# spaces by \t
_text[pol_cnt, ant_cnt] = _text[pol_cnt,
ant_cnt].replace(" ", "\t")
masks = [[True] for p in powers]
# Offline antennas
data_hex = []
offline_ants = {
"x": xs_offline.compressed().tolist(),
"y": ys_offline.compressed().tolist(),
"text": name_offline,
"mode": "markers",
"visible": True,
"marker": {
"color":
np.ma.masked_array(["black"] * len(name_offline),
mask=xs_offline.mask),
"size":
14,
"opacity":
0.5,
"symbol":
"hexagon",
},
"hovertemplate": "%{text}<extra></extra>",
}
# now we want to Fill in the conneted ones
offline_ants["marker"]["color"][built_but_not_on] = "red"
offline_ants["text"].data[built_but_not_on] = [
offline_ants["text"].data[ant].split("<br>")[0] +
"<br>Constructed<br>Not\tOnline" for ant in built_but_not_on
]
offline_ants["marker"]["color"] = (
offline_ants["marker"]["color"].compressed().tolist())
offline_ants["text"] = offline_ants["text"].compressed().tolist()
data_hex.append(offline_ants)
# for each type of power, loop over pols and print out the data
# save up a mask array used for the buttons later
# also plot the bad ones!3
colorscale = "Viridis"
# define some custom scale values for the ADC RMS page
rms_scale_vals = [2, 20]
relavitve_values = [0.4, 0.7]
rms_color_scale = [
["0.0", "rgb(68,1,84)"],
["0.2", "rgb(62,74,137)"],
["0.3", "rgb(38,130,142)"],
["0.4", "rgb(53,183,121)"],
["0.5", "rgb(53,183,121)"],
["0.6", "rgb(53,183,121)"],
["0.7", "rgb(109,205,89)"],
["0.8", "rgb(180,222,44)"],
["1.0", "rgb(253,231,37)"],
]
for pow_ind, power in enumerate(powers):
if power.compressed().size > 0:
vmax = np.max(power.compressed())
vmin = np.min(power.compressed())
else:
vmax = 1
vmin = 0
colorscale = "Viridis"
if pow_ind == 3:
cbar_title = "RMS\tlinear"
vmin = rms_scale_vals[0] * relavitve_values[0]
vmax = rms_scale_vals[1] / relavitve_values[1]
colorscale = rms_color_scale
elif pow_ind == 4 or pow_ind == 5:
cbar_title = "Degrees"
elif pow_ind == len(powers) - 1:
cbar_title = "Median\tCoeff"
else:
cbar_title = "dB"
if pow_ind == 0:
visible = True
else:
visible = False
for pol_ind, pol in enumerate(pols):
for mask_cnt, mask in enumerate(masks):
if mask_cnt == pow_ind:
mask.extend([True] * 2)
else:
mask.extend([False] * 2)
_power = {
"x": xs.data[~power[pol_ind].mask].tolist(),
"y": ys[pol_ind].data[~power[pol_ind].mask].tolist(),
"text": _text[pol_ind][~power[pol_ind].mask].tolist(),
"mode": "markers",
"visible": visible,
"marker": {
"color":
power[pol_ind].data[~power[pol_ind].mask].tolist(),
"size": 14,
"cmin": vmin,
"cmax": vmax,
"colorscale": colorscale,
"colorbar": {
"thickness": 20,
"title": cbar_title
},
},
"hovertemplate": "%{text}<extra></extra>",
}
data_hex.append(_power)
_power_offline = {
"x": xs.data[power[pol_ind].mask].tolist(),
"y": ys[pol_ind].data[power[pol_ind].mask].tolist(),
"text": _text[pol_ind][power[pol_ind].mask].tolist(),
"mode": "markers",
"visible": visible,
"marker": {
"color": "orange",
"size": 14,
"cmin": vmin,
"cmax": vmax,
"colorscale": colorscale,
"colorbar": {
"thickness": 20,
"title": cbar_title
},
},
"hovertemplate": "%{text}<extra></extra>",
}
data_hex.append(_power_offline)
buttons = []
for _name, mask in zip(names, masks):
_button = {
"args": [{
"visible": mask
}, {
"title": "",
"annotations": {}
}],
"label": _name,
"method": "restyle",
}
buttons.append(_button)
updatemenus_hex = [{
"buttons": buttons,
"showactive": True,
"type": "buttons"
}]
layout_hex = {
"xaxis": {
"title": "East-West Position [m]"
},
"yaxis": {
"title": "North-South Position [m]",
"scaleanchor": "x"
},
"title": {
"text": "Per Antpol Stats vs Hex position",
"font": {
"size": 24
},
},
"hoverlabel": {
"align": "left"
},
"margin": {
"t": 40
},
"autosize": True,
"showlegend": False,
"hovermode": "closest",
}
caption = {}
caption["title"] = "Stats vs Hex pos Help"
caption["text"] = (
"This plot shows various statistics and measurements "
"per ant-pol versus its position in the array."
"<br>Antennas which are build but not fully hooked up "
"are shown in light red."
"<br>Grey antennas are not yet constructed."
"<br><br><h4>Available plotting options</h4>"
"<ul>"
"<li>Auto Corr - Median Auto Correlation (in db) "
"from the correlator with equalization coefficients "
"divided out</li>"
"<li>Pam Power - Latest Pam Power (in db) recorded in M&C</li>"
"<li>ADC Power - Latest ADC Power (in db) recorded in M&C</li>"
"<li>ADC RMS - Latest linear ADC RMS recorded in M&C</li>"
"<li>FEM IMU THETA - IMU-reported theta, in degrees</li>"
"<li>FEM IMU PHI - IMU-reported phi, in degrees</li>"
"<li>EQ Coeffs - Latest Median Equalization Coefficient recorded in M&C</li>"
"</ul>"
"Any antpol showing with an orange color means "
"no data is avaible for the currenty plot selection."
"<h4>Hover label Formatting</h4>"
"<ul>"
"<li>Antenna Name from M&C<br>(e.g. HH0n = Hera Hex Antenna 0 Polarization N)</li>"
"<li>Snap hostname from M&C<br>(e.g. heraNode0Snap0)</li>"
"<li>PAM Number</li>"
"<li>Median Auto Correlation power in dB</li>"
"<li>PAM power in dB</li>"
"<li>ADC power in dB</li>"
"<li>Linear ADC RMS</li>"
"<li>FEM IMU reported theta in degrees</li>"
"<li>FEM IMU reported phi in degrees</li>"
"<li>Median Equalization Coefficient</li>"
"<li>Time ago in hours the M&C Antenna Status was updated. "
"This time stamp applies to all data for this antenna "
"except the Auto Correlation.</li>"
"</ul>"
"In any hover label entry 'No Data' means "
"information not currrently available in M&C.")
# Render all the power vs position files
plotname = "plotly-hex"
html_template = env.get_template("plotly_base.html")
js_template = env.get_template("plotly_base.js")
if sys.version_info.minor >= 8 and sys.version_info.major > 2:
time_jd = latest.to_value('jd', subfmt='float')
time_unix = latest.to_value('unix')
else:
time_jd = latest.jd
time_unix = latest.unix
rendered_hex_html = html_template.render(
plotname=plotname,
data_type="Auto correlations",
plotstyle="height: 100%",
gen_date=now.iso,
data_date_iso=latest.iso,
data_date_jd="{:.3f}".format(time_jd),
data_date_unix_ms=time_unix * 1000,
js_name="hex_amp",
gen_time_unix_ms=now.unix * 1000,
scriptname=os.path.basename(__file__),
hostname=computer_hostname,
caption=caption,
)
rendered_hex_js = js_template.render(
data=data_hex,
layout=layout_hex,
updatemenus=updatemenus_hex,
plotname=plotname,
)
with open("hex_amp.html", "w") as h_file:
h_file.write(rendered_hex_html)
with open("hex_amp.js", "w") as js_file:
js_file.write(rendered_hex_js)
# now prepare the data to be plotted vs node number
data_node = []
masks = [[] for p in powers]
vmax = [
np.max(power.compressed()) if power.compressed().size > 1 else 1
for power in powers
]
vmin = [
np.min(power.compressed()) if power.compressed().size > 1 else 0
for power in powers
]
vmin[3] = rms_scale_vals[0] * relavitve_values[0]
vmax[3] = rms_scale_vals[1] / relavitve_values[1]
for node in nodes:
node_index = np.where(node_ind == node)[0]
hosts = hostname[node_index]
host_index = np.argsort(hosts)
ys = np.ma.masked_array(
[
np.arange(node_index.size) + 0.3 * pol_cnt
for pol_cnt, pol in enumerate(pols)
],
mask=powers[0][:, node_index].mask,
)
xs = np.zeros_like(ys)
xs[:] = node
powers_node = [pow[:, node_index] for pow in powers]
__text = _text[:, node_index]
for pow_ind, power in enumerate(powers_node):
cbar_title = "dB"
if pow_ind == 4 or pow_ind == 5:
cbar_title = "Degrees"
if pow_ind == 3:
colorscale = rms_color_scale
else:
colorscale = "Viridis"
colorscale = "Viridis"
if pow_ind == 3:
cbar_title = "RMS\tlinear"
colorscale = rms_color_scale
elif pow_ind == 4 or pow_ind == 5:
cbar_title = "Degrees"
elif pow_ind == len(powers) - 1:
cbar_title = "Median\tCoeff"
else:
cbar_title = "dB"
if pow_ind == 0:
visible = True
else:
visible = False
for pol_ind, pol in enumerate(pols):
for mask_cnt, mask in enumerate(masks):
if mask_cnt == pow_ind:
mask.extend([True] * 2)
else:
mask.extend([False] * 2)
__power = power[pol_ind][host_index]
___text = __text[pol_ind][host_index]
_power = {
"x": xs[pol_ind].data[~__power.mask].tolist(),
"y": ys[pol_ind].data[~__power.mask].tolist(),
"text": ___text[~__power.mask].tolist(),
"mode": "markers",
"visible": visible,
"marker": {
"color": __power.data[~__power.mask].tolist(),
"size": 14,
"cmin": vmin[pow_ind],
"cmax": vmax[pow_ind],
"colorscale": colorscale,
"colorbar": {
"thickness": 20,
"title": cbar_title
},
},
"hovertemplate": "%{text}<extra></extra>",
}
data_node.append(_power)
_power_offline = {
"x": xs[pol_ind].data[__power.mask].tolist(),
"y": ys[pol_ind].data[__power.mask].tolist(),
"text": ___text[__power.mask].tolist(),
"mode": "markers",
"visible": visible,
"marker": {
"color": "orange",
"size": 14,
"cmin": vmin[pow_ind],
"cmax": vmax[pow_ind],
"colorscale": colorscale,
"colorbar": {
"thickness": 20,
"title": cbar_title
},
},
"hovertemplate": "%{text}<extra></extra>",
}
data_node.append(_power_offline)
buttons = []
for _name, mask in zip(names, masks):
_button = {
"args": [{
"visible": mask
}, {
"title": "",
"annotations": {}
}],
"label": _name,
"method": "restyle",
}
buttons.append(_button)
updatemenus_node = [{
"buttons": buttons,
"showactive": True,
"type": "buttons"
}]
layout_node = {
"xaxis": {
"title": "Node Number",
"dtick": 1,
"tick0": 0,
"showgrid": False,
"zeroline": False,
},
"yaxis": {
"showticklabels": False,
"showgrid": False,
"zeroline": False
},
"title": {
"text": "Per Antpol Stats vs Node #",
"font": {
"size": 24
}
},
"hoverlabel": {
"align": "left"
},
"margin": {
"t": 40
},
"autosize": True,
"showlegend": False,
"hovermode": "closest",
}
caption_node = {}
caption_node["title"] = "Stats vs Node Help"
caption_node["text"] = (
"This plot shows various statistics and measurements "
"per ant-pol versus the node number to which it is connected."
"<br><br><h4>Available plotting options</h4>"
"<ul>"
"<li>Auto Corr - Median Auto Correlation (in db) "
"from the correlator with equalization coefficients "
"divided out</li>"
"<li>Pam Power - Latest Pam Power (in db) recorded in M&C</li>"
"<li>ADC Power - Latest ADC Power (in db) recorded in M&C</li>"
"<li>ADC RMS - Latest linear ADC RMS recorded in M&C</li>"
"<li>EQ Coeffs - Latest Median Equalization Coefficient recorded in M&C</li>"
"</ul>"
"Any antpol showing with an orange color means "
"no data is avaible for the currenty plot selection."
"<h4>Hover label Formatting</h4>"
"<ul>"
"<li>Antenna Name from M&C<br>(e.g. HH0n = Hera Hex Antenna 0 Polarization N)</li>"
"<li>Snap hostname from M&C<br>(e.g. heraNode0Snap0)</li>"
"<li>PAM Number</li>"
"<li>Median Auto Correlation power in dB</li>"
"<li>PAM power in dB</li>"
"<li>ADC power in dB</li>"
"<li>Linear ADC RMS</li>"
"<li>Median Equalization Coefficient</li>"
"<li>Time ago in hours the M&C Antenna Status was updated. "
"This time stamp applies to all data for this antenna "
"except the Auto Correlation.</li>"
"</ul>"
"In any hover label entry 'No Data' means "
"information not currrently available in M&C.")
# Render all the power vs ndde files
plotname = "plotly-node"
html_template = env.get_template("plotly_base.html")
js_template = env.get_template("plotly_base.js")
if sys.version_info.minor >= 8 and sys.version_info.major > 2:
time_jd = latest.to_value('jd', subfmt='float')
time_unix = latest.to_value('unix')
else:
time_jd = latest.jd
time_unix = latest.unix
rendered_node_html = html_template.render(
plotname=plotname,
data_type="Auto correlations",
plotstyle="height: 100%",
gen_date=now.iso,
gen_time_unix_ms=now.unix * 1000,
data_date_iso=latest.iso,
data_date_jd="{:.3f}".format(time_jd),
data_date_unix_ms=time_unix * 1000,
js_name="node_amp",
scriptname=os.path.basename(__file__),
hostname=computer_hostname,
caption=caption_node,
)
rendered_node_js = js_template.render(
data=data_node,
layout=layout_node,
updatemenus=updatemenus_node,
plotname=plotname,
)
with open("node_amp.html", "w") as h_file:
h_file.write(rendered_node_html)
with open("node_amp.js", "w") as js_file:
js_file.write(rendered_node_js)
if __name__ == '__main__':
default_hookup_cols = [
'station', 'feed', 'front-end', 'node-bulkhead', 'post-amp',
'snap', 'node'
]
parser = mc.get_mc_argument_parser()
# set values for 'action' to use
parser.add_argument('-p', '--hpn', help="Part number, csv-list or [default]",
default='default')
parser.add_argument('-e', '--exact-match',
help="Force exact matches on part numbers, not beginning N char. [False]",
dest='exact_match', action='store_true')
parser.add_argument('--hookup-cols',
help="Specify a subset of parts to show comma-delimited no-space list.",
dest='hookup_cols', default=default_hookup_cols)
args = parser.parse_args()
# Pre-process the args
args.hpn = cm_utils.listify(args.hpn)
args.hookup_cols = cm_utils.listify(args.hookup_cols)
# Start session
db = mc.connect_to_mc_db(args)
session = db.sessionmaker()
system = cm_sysutils.Handling(session)
system.publish_summary(hlist=args.hpn, exact_match=args.exact_match,
hookup_cols=args.hookup_cols,
)
def get_cm_info():
from hera_mc import cm_sysutils
h = cm_sysutils.Handling()
return h.get_cminfo_correlator()
def main():
# templates are stored relative to the script dir
# stored one level up, find the parent directory
# and split the parent directory away
script_dir = os.path.dirname(os.path.realpath(__file__))
split_dir = os.path.split(script_dir)
template_dir = os.path.join(split_dir[0], "templates")
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True)
# this filter is used to see if there is more than one table
env.filters["islist"] = is_list
if sys.version_info[0] < 3:
# py2
computer_hostname = os.uname()[1]
else:
# py3
computer_hostname = os.uname().nodename
parser = mc.get_mc_argument_parser()
parser.add_argument(
"--redishost",
dest="redishost",
type=str,
default="redishost",
help=(
'The host name for redis to connect to, defaults to "redishost"'),
)
parser.add_argument("--port",
dest="port",
type=int,
default=6379,
help="Redis port to connect.")
args = parser.parse_args()
try:
db = mc.connect_to_mc_db(args)
except RuntimeError as e:
raise SystemExit(str(e))
try:
redis_db = redis.Redis(args.redishost, port=args.port)
redis_db.keys()
except Exception as err:
raise SystemExit(str(err))
with db.sessionmaker() as session:
now = Time.now()
hsession = cm_sysutils.Handling(session)
stations = hsession.get_connected_stations(at_date="now")
ants = []
for station in stations:
if station.antenna_number not in ants:
ants = np.append(ants, station.antenna_number)
ants = np.unique(ants).astype(int)
antpos = np.genfromtxt(
os.path.join(mc.data_path, "HERA_350.txt"),
usecols=(0, 1, 2, 3),
dtype={
"names": ("ANTNAME", "EAST", "NORTH", "UP"),
"formats": ("<U5", "<f8", "<f8", "<f8"),
},
encoding=None,
)
antnames = antpos["ANTNAME"]
inds = [int(j[2:]) for j in antnames]
inds = np.argsort(inds)
antnames = np.take(antnames, inds)
nodes = []
hists = []
bad_ants = []
bad_node = []
for ant_cnt, ant in enumerate(ants):
ant_status = session.get_antenna_status(most_recent=True,
antenna_number=int(ant))
mc_name = antnames[int(ant)]
node_info = hsession.get_part_at_station_from_type(
mc_name, "now", "node")
if len(ant_status) == 0:
for pol in ["e", "n"]:
name = "{ant}:{pol}".format(ant=ant, pol=pol)
print("No histogram data for ", name)
bad_ants.append(name)
for stat in ant_status:
name = "ant{ant}{pol}".format(ant=stat.antenna_number,
pol=stat.antenna_feed_pol)
# try to find the associated node
# get the first key in the dict to index easier
_key = list(node_info.keys())[0]
pol_key = [
key for key in node_info[_key].keys()
if stat.antenna_feed_pol.upper() in key
]
if pol_key:
# 'E' should be in one of the keys, extract the 0th entry
pol_key = pol_key[0]
else:
# a hacky solution for a key that should work
pol_key = "E<ground"
if node_info[_key][pol_key] is not None:
_node_num = re.findall(r"N(\d+)",
node_info[_key][pol_key])[0]
else:
print("No Node mapping for antennna: " + name)
_node_num = -1
bad_node.append("{ant}:{pol}".format(
ant=stat.antenna_number, pol=stat.antenna_feed_pol))
nodes.append(_node_num)
timestamp = Time(stat.time, format="gps")
if (stat.histogram_bin_centers is not None
and stat.histogram is not None):
bins = np.fromstring(
stat.histogram_bin_centers.strip("[]"), sep=",")
hist = np.fromstring(stat.histogram.strip("[]"), sep=",")
if stat.eq_coeffs is not None:
eq_coeffs = np.fromstring(stat.eq_coeffs.strip("[]"),
sep=",")
else:
eq_coeffs = np.ones_like(hist)
hist /= np.median(eq_coeffs)**2
text = "observed at {iso}<br>(JD {jd})".format(
iso=timestamp.iso, jd=timestamp.jd)
# spaces cause weird wrapping issues, replace them all with \t
text = text.replace(" ", "\t")
_data = {
"x": bins.tolist(),
"y": hist.tolist(),
"name": name,
"node": _node_num,
"text": [text] * bins.size,
"hovertemplate": "(%{x:.1},\t%{y})<br>%{text}",
}
hists.append(_data)
else:
name = "{ant}:{pol}".format(ant=stat.antenna_number,
pol=stat.antenna_feed_pol)
print("No histogram data for ", name)
bad_ants.append(name)
table = {}
table["title"] = "Ants with no Histogram"
table["rows"] = []
row = {}
row["text"] = ",\t".join(bad_ants)
table["rows"].append(row)
table_node = {}
table_node["title"] = "Antennas with no Node mapping"
row_node = {}
row_node["text"] = "\t".join(bad_node)
rows_node = [row_node]
table_node["rows"] = rows_node
layout = {
"xaxis": {
"title": "ADC value"
},
"yaxis": {
"title": "Occurance",
"type": "linear"
},
"title": {
"text": "ADC Histograms",
"xref": "paper",
"x": 0.5,
"yref": "paper",
"y": 1.5,
"font": {
"size": 24,
},
},
"margin": {
"l": 40,
"b": 30,
"r": 40,
"t": 70
},
"hovermode": "closest",
"autosize": True,
"showlegend": True,
}
# Make all the buttons for this plot
nodes = np.unique(nodes)
# if an antenna was not mapped, roll the -1 to the end
# this makes making buttons easier so the unmapped show last
if -1 in nodes:
nodes = np.roll(nodes, -1)
# create a mask to find all the matching nodes
node_mask = [[True if s["node"] == node else False for s in hists]
for node in nodes]
buttons_node = []
_button_node = {
"args": [
{
"visible": [True for s in hists]
},
{
"title": "",
"annotations": {}
},
],
"label":
"All\tAnts",
"method":
"restyle",
}
buttons_node.append(_button_node)
for node_cnt, node in enumerate(nodes):
if node != -1:
label = "Node\t{}".format(node)
else:
label = "Unmapped\tAnts"
_button_node = {
"args": [
{
"visible": node_mask[node_cnt]
},
{
"title": "",
"annotations": {}
},
],
"label":
label,
"method":
"restyle",
}
buttons_node.append(_button_node)
buttons = []
log_buttons = {
"args": [{}, {
"yaxis": {
"type": "log"
}
}],
"label": "Log",
"method": "update",
}
lin_buttons = {
"args": [{}, {
"yaxis": {
"type": "linear"
}
}],
"label": "Linear",
"method": "update",
}
buttons.append(lin_buttons)
buttons.append(log_buttons)
updatemenus = [
{
"buttons": buttons_node,
"showactive": True,
"active": 0,
"type": "dropdown",
"x": 0.535,
"y": 1.03,
},
{
"buttons": buttons,
"showactive": True,
"type": "buttons",
"active": 0,
},
]
plotname = "plotly-adc-hist"
html_template = env.get_template("ploty_with_multi_table.html")
js_template = env.get_template("plotly_base.js")
caption = {}
caption["text"] = (
"The ADC Histograms with equalization coefficients "
"divided out."
"<br><br>Some antennas known to M&C may not have a histogram "
" and are listed below the image."
"<br><br>Some antennas may not have "
"a known node mapping and are listed below the image.\n "
"<br><br>Plot can be downselected to display "
"individual nodes or show the entire array.\n "
"<br><br>Double click on an entry in the legend "
"to select only that entry, "
"double click again to restore all plots.\n "
"<br><br><h4>Formatting options</h4>"
"<ul>"
"<li>Linear - Display with Linear y-axis</li>"
"<li>Log - Display with Log y-axis</li>"
"</ul>"
"<br><br>Single click an entry in the legend to un-plot it, "
"single click again to restore it to the plot.")
caption["title"] = "Histogram Help"
rendered_html = html_template.render(
plotname=plotname,
plotstyle="height: 100%",
gen_date=now.iso,
js_name="adchist",
caption=caption,
gen_time_unix_ms=now.unix * 1000,
scriptname=os.path.basename(__file__),
hostname=computer_hostname,
table=[table, table_node],
)
rendered_js = js_template.render(data=hists,
layout=layout,
plotname=plotname,
updatemenus=updatemenus)
with open("adchist.html", "w") as h_file:
h_file.write(rendered_html)
with open("adchist.js", "w") as js_file:
js_file.write(rendered_js)
def main():
# templates are stored relative to the script dir
# stored one level up, find the parent directory
# and split the parent directory away
script_dir = os.path.dirname(os.path.realpath(__file__))
split_dir = os.path.split(script_dir)
template_dir = os.path.join(split_dir[0], 'templates')
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True)
# this filter is used to see if there is more than one table
env.filters['islist'] = is_list
if sys.version_info[0] < 3:
# py2
computer_hostname = os.uname()[1]
else:
# py3
computer_hostname = os.uname().nodename
# The standard M&C argument parser
parser = mc.get_mc_argument_parser()
# we'll have to add some extra options too
parser.add_argument(
'--redishost',
dest='redishost',
type=str,
default='redishost',
help=(
'The host name for redis to connect to, defualts to "redishost"'))
parser.add_argument('--port',
dest='port',
type=int,
default=6379,
help='Redis port to connect.')
args = parser.parse_args()
try:
db = mc.connect_to_mc_db(args)
except RuntimeError as e:
raise SystemExit(str(e))
try:
redis_db = redis.Redis(args.redishost, port=args.port)
redis_db.keys()
except Exception as err:
raise SystemExit(str(err))
with db.sessionmaker() as session:
corr_cm = hera_corr_cm.HeraCorrCM(redishost=args.redishost)
hsession = cm_sysutils.Handling(session)
stations = hsession.get_connected_stations(at_date='now')
antpos = np.genfromtxt(os.path.join(mc.data_path, "HERA_350.txt"),
usecols=(0, 1, 2, 3),
dtype={
'names': ('ANTNAME', 'EAST', 'NORTH', 'UP'),
'formats': ('<U5', '<f8', '<f8', '<f8')
},
encoding=None)
antnames = antpos['ANTNAME']
inds = [int(j[2:]) for j in antnames]
inds = np.argsort(inds)
antnames = np.take(antnames, inds)
ants = []
for station in stations:
if station.antenna_number not in ants:
ants = np.append(ants, station.antenna_number)
ants = np.unique(ants).astype(int)
hostname_lookup = {}
# all_snap_statuses = session.get_snap_status(most_recent=True)
all_snap_statuses = corr_cm.get_f_status()
snapautos = {}
ant_status_from_snaps = corr_cm.get_ant_status()
all_snaprf_stats = corr_cm.get_snaprf_status()
table_snap = {}
table_snap["title"] = "Snap hookups with No Data"
rows = []
bad_snaps = []
for snap_chan in all_snaprf_stats:
host, loc_num = snap_chan.split(":")
loc_num = int(loc_num)
# initialize this key if not already in the dict
auto_group = snapautos.setdefault(host, {})
try:
tmp_auto = np.array(
all_snaprf_stats[snap_chan]["autocorrelation"])
if np.all(tmp_auto == "None"):
print("No Data for {} port {}".format(host, loc_num))
bad_snaps.append(snap_chan)
tmp_auto = np.full(1024, np.nan)
eq_coeffs = np.array(all_snaprf_stats[snap_chan]["eq_coeffs"])
if np.all(eq_coeffs == "None"):
eq_coeffs = np.full_like(tmp_auto, 1.0)
tmp_auto /= eq_coeffs**2
tmp_auto = np.ma.masked_invalid(10 *
np.log10(np.real(tmp_auto)))
auto_group[loc_num] = tmp_auto.filled(-50)
except KeyError:
print("Snap connection with no autocorrelation", snap_chan)
raise
except TypeError:
print("Received TypeError when taking log.")
print("Type of item in dictionary: ",
type(all_snaprf_stats[snap_chan]["autocorrelation"]))
print("Value of item: ", tmp_auto)
raise
hostname_lookup.setdefault(host, {})
hostname_lookup[host].setdefault(loc_num, {})
hostname_lookup[host][loc_num]['MC'] = 'NC'
row = {}
row["text"] = '\t'.join(bad_snaps)
rows.append(row)
table_snap["rows"] = rows
table_ants = {}
table_ants["title"] = "Antennas with no mapping"
rows = []
bad_ants = []
bad_hosts = []
for ant_cnt, ant in enumerate(ants):
# get the status for both polarizations for this antenna
ant_status = {
key: ant_status_from_snaps[key]
for key in ant_status_from_snaps
if ant == int(key.split(':')[0])
}
# check if the antenna status from M&C has the host and
# channel number, if it does not we have to do some gymnastics
for antkey in ant_status:
pol_key = antkey.split(":")[1]
stat = ant_status[antkey]
name = "{ant:d}:{pol}".format(ant=ant, pol=pol_key)
# check that key is in the dictionary, is not None or the string "None"
if ('f_host' in stat and 'host_ant_id' in stat
and stat['f_host'] is not None
and stat['host_ant_id'] is not None
and stat['f_host'] != "None"
and stat['host_ant_id'] != "None"):
hostname = stat['f_host']
loc_num = stat['host_ant_id']
hostname_lookup[hostname][loc_num]['MC'] = name
else:
# Try to get the snap info from M&C. Output is a dictionary with 'e' and 'n' keys
# connect to M&C to find all the hooked up Snap hostnames and corresponding ant-pols
mc_name = antnames[ant]
# these two may not be used, but it is easier to grab them now
snap_info = hsession.get_part_at_station_from_type(
mc_name, 'now', 'snap', include_ports=True)
for _key in snap_info.keys():
# initialize a dict if they key does not exist already
pol_key = pol_key.upper() + "<ground"
if snap_info[_key][pol_key] is not None:
serial_with_ports = snap_info[_key][pol_key]
snap_serial = serial_with_ports.split(
'>')[1].split('<')[0]
ant_channel = int(
serial_with_ports.split('>')[0][1:]) // 2
hostname = session.get_snap_hostname_from_serial(
snap_serial)
if hostname is None:
node_info = hsession.get_part_at_station_from_type(
mc_name, 'now', 'node')
_node_num = re.findall(
r'N(\d+)', node_info[_key][pol_key])[0]
snap_stats = session.get_snap_status(
nodeID=int(_node_num), most_recent=True)
for _stat in snap_stats:
if _stat.serial_number == snap_serial:
hostname = _stat.hostname
# if hostname is still None then we don't have a known hookup
if hostname is None:
print("No MC snap information for antennna: " +
name)
bad_ants.append(name)
else:
if hostname not in hostname_lookup.keys():
err = "host from M&C not found in corr_cm 'status:snaprf' : {}".format(
hostname)
err += '\nThis host may not have data populated yet or is offline.'
err += '\nAll anteanns on this host will be full of 0.'
print(err)
bad_hosts.append(hostname)
grp1 = hostname_lookup.setdefault(hostname, {})
# if this loc num is not in lookup table initialize
# empty list
if ant_channel not in grp1.keys():
if hostname not in bad_hosts:
print(
"loc_num from M&C not found in hera_corr_cm `status:snaprf` (host, location number): {}"
.format([hostname, ant_channel]))
print(
"filling with bad array full of 0."
)
else:
_name = "{host}:{loc}".format(
host=hostname, loc=ant_channel)
if _name not in table_snap["rows"][0][
"text"]:
table_snap["rows"][0][
"text"] += _name + '\t'
snap_grp1 = snapautos.setdefault(
hostname, {})
snap_grp1[ant_channel] = np.full(1024, 0)
grp2 = grp1.setdefault(ant_channel, {})
grp2['MC'] = name
else:
print("No MC snap information for antennna: " +
name)
bad_ants.append(name)
row = {}
row["text"] = '\t'.join(bad_ants)
rows.append(row)
table_ants["rows"] = rows
host_masks = []
for h1 in sorted(hostname_lookup.keys()):
_mask = []
for h2 in sorted(hostname_lookup.keys()):
_mask.extend([
True if h2 == h1 else False
for loc_num in hostname_lookup[h2]
])
host_masks.append(_mask)
# Generate frequency axis
freqs = np.linspace(0, 250e6, 1024)
freqs /= 1e6
data = []
for host_cnt, host in enumerate(sorted(hostname_lookup.keys())):
if host_cnt == 0:
visible = True
else:
visible = False
for loc_num in sorted(hostname_lookup[host].keys()):
mc_name = hostname_lookup[host][loc_num]['MC']
name = '{loc}:{mcname}'.format(loc=loc_num,
mcname=mc_name.replace(":", ""))
try:
_data = {
"x": freqs.tolist(),
"y": snapautos[host][loc_num].tolist(),
"name": name,
"visible": visible,
"hovertemplate": "%{x:.1f}\tMHz<br>%{y:.3f}\t[dB]"
}
except KeyError:
print("Given host, location pair: ({0}, {1})".format(
host, loc_num))
print("All possible keys for host {0}: {1}".format(
host, list(snapautos[host].keys())))
raise
data.append(_data)
buttons = []
for host_cnt, host in enumerate(sorted(hostname_lookup.keys())):
prog_time = all_snap_statuses[host]['last_programmed']
timestamp = all_snap_statuses[host]['timestamp']
temp = all_snap_statuses[host]['temp']
uptime = all_snap_statuses[host]['uptime']
pps = all_snap_statuses[host]['pps_count']
if host in bad_hosts:
label = ('{host}<br>programmed:\t{start}'
'<br>spectra\trecorded:\tNO DATA OBSERVED<br>'
'temp:\t{temp:.0f}\tC\t'
'pps\tcount:\t{pps}\tCycles\t\t\t'
'uptime:\t{uptime}'
''.format(host=host,
start=prog_time.isoformat(' '),
temp=temp,
pps=pps,
uptime=uptime))
else:
label = ('{host}<br>programmed:\t{start}'
'<br>spectra\trecorded:\t{obs}<br>'
'temp:\t{temp:.0f}\tC\t'
'pps\tcount:\t{pps}\tCycles\t\t\t'
'uptime:\t{uptime}'
''.format(host=host,
start=prog_time.isoformat(' '),
obs=timestamp.isoformat(' '),
temp=temp,
pps=pps,
uptime=uptime))
_button = {
"args": [
{
"visible": host_masks[host_cnt]
},
{
"title": '',
# "annotations": {}
}
],
"label":
label,
"method":
"restyle"
}
buttons.append(_button)
updatemenus = [{
"buttons": buttons,
"showactive": True,
"type": "dropdown",
"x": .7,
"y": 1.1,
}]
layout = {
"xaxis": {
"title": "Frequency [MHz]",
"showticklabels": True,
"tick0": 0,
"dtick": 10,
},
"yaxis": {
"title": 'Power [dB]',
"showticklabels": True,
},
"hoverlabel": {
"align": "left"
},
"margin": {
"l": 40,
"b": 30,
"r": 40,
"t": 30
},
"autosize":
True,
"showlegend":
True,
"hovermode":
'closest',
"annotations": [{
"x": 1.03,
"y": 1.03,
"align": "right",
"valign": "top",
"text": 'ADC Port # : Antpol',
"showarrow": False,
"xref": "paper",
"yref": "paper",
"xanchor": "center",
"yanchor": "top"
}]
}
caption = {}
caption["title"] = "Snap Spectra Help"
caption["text"] = (
"Autocorrelations (in dB) calculated by each Snap "
"with equalization coefficients divided out. "
"Calculation is independent of the corrlator. "
"<br><br>"
"Data is organized by the ADC Port number from "
"each Snap. "
"A mapping dictionary is used to look up the "
"associated ant-pol for each ADC Port. "
"Some ADC Ports may not have a known ant-pol "
"and are labeled as N/C."
"<br>Some antennas known to M&C may not have a known ADC port"
" mapping and are labeled below the plot."
"<br><br>Some Snap ports my not have data and are "
"labeled in a table below the plot."
"<br><br>The node selection button contains a lot "
"of information described here."
"<br><h4>Button Label Information</h4>"
"<ul>"
"<li>Snap hostname</li>"
"<li>Last time the Snap was programmed</li>"
"<li>Last time data was recorded from this Snap</li>"
"<li>Last known temperature in C | PPS count | Uptime</li>"
"</ul>")
plotname = "plotly-snap"
html_template = env.get_template("refresh_with_table.html")
js_template = env.get_template("plotly_base.js")
rendered_html = html_template.render(
plotname=plotname,
plotstyle="height: 100%",
gen_date=Time.now().iso,
js_name='snapspectra',
gen_time_unix_ms=Time.now().unix * 1000,
scriptname=os.path.basename(__file__),
hostname=computer_hostname,
table=[table_snap, table_ants],
caption=caption)
rendered_js = js_template.render(data=data,
layout=layout,
updatemenus=updatemenus,
plotname=plotname)
with open('snapspectra.html', 'w') as h_file:
h_file.write(rendered_html)
with open('snapspectra.js', 'w') as js_file:
js_file.write(rendered_js)