def handle(self, *args, **options): """Perform necessary calculation and inser antennas in to DB.""" mc_args = Namespace() mc_args.mc_db_name = options["mc_db_name"] mc_args.mc_config_path = options["mc_config_path"] db = mc.connect_to_mc_db(args=mc_args) antpos = np.genfromtxt( os.path.join(mc.data_path, "HERA_350.txt"), usecols=(0, 1, 2, 3), dtype={ "names": ("ANTNAME", "EAST", "NORTH", "UP"), "formats": ("<U5", "<f8", "<f8", "<f8"), }, encoding=None, ) antnames = antpos["ANTNAME"] inds = [int(j[2:]) for j in antnames] inds = np.argsort(inds) antnames = np.take(antnames, inds) antpos = np.array([antpos["EAST"], antpos["NORTH"], antpos["UP"]]) array_center = np.mean(antpos, axis=1, keepdims=True) antpos -= array_center antpos = np.take(antpos, inds, axis=1) with db.sessionmaker() as session: hsession = cm_sysutils.Handling(session) stations = [] for station_type in hsession.geo.parse_station_types_to_check( "default"): for stn in hsession.geo.station_types[station_type][ "Stations"]: stations.append(stn) # stations is a list of HH??? numbers we just want the ints stations = list(map(int, [j[2:] for j in stations])) bulk_add = [] for ind, name in enumerate(antnames): ant_number = int(name[2:]) for pol in ["e", "n"]: bulk_add.append( Antenna( ant_number=ant_number, ant_name=name, polarization=pol, antpos_enu=antpos[:, ind].tolist(), constructed=ant_number in stations, )) Antenna.objects.bulk_create(bulk_add, ignore_conflicts=True)
def main(): # templates are stored relative to the script dir # stored one level up, find the parent directory # and split the parent directory away script_dir = os.path.dirname(os.path.realpath(__file__)) split_dir = os.path.split(script_dir) template_dir = os.path.join(split_dir[0], "templates") env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True) if sys.version_info[0] < 3: # py2 computer_hostname = os.uname()[1] else: # py3 computer_hostname = os.uname().nodename # The standard M&C argument parser parser = mc.get_mc_argument_parser() # we'll have to add some extra options too parser.add_argument( "--redishost", dest="redishost", type=str, default="redishost", help=( 'The host name for redis to connect to, defualts to "redishost"'), ) parser.add_argument("--port", dest="port", type=int, default=6379, help="Redis port to connect.") args = parser.parse_args() try: db = mc.connect_to_mc_db(args) except RuntimeError as e: raise SystemExit(str(e)) try: redis_db = redis.Redis(args.redishost, port=args.port) redis_db.keys() except Exception as err: raise SystemExit(str(err)) with db.sessionmaker() as session: # without item this will be an array which will break database queries latest = Time( np.frombuffer(redis_db.get("auto:timestamp"), dtype=np.float64).item(), format="jd", ) latest.out_subfmt = u"date_hm" now = Time.now() amps = {} keys = [ k.decode() for k in redis_db.keys() if k.startswith(b"auto") and not k.endswith(b"timestamp") ] for key in keys: match = re.search(r"auto:(?P<ant>\d+)(?P<pol>e|n)", key) if match is not None: ant, pol = int(match.group("ant")), match.group("pol") d = redis_db.get(key) if d is not None: # need to copy because frombuffer creates a read-only array auto = np.frombuffer(d, dtype=np.float32).copy() eq_coeff = redis_db.hget( bytes("eq:ant:{ant}:{pol}".format(ant=ant, pol=pol).encode()), "values", ) if eq_coeff is not None: eq_coeffs = np.fromstring( eq_coeff.decode("utf-8").strip("[]"), sep=",") if eq_coeffs.size == 0: eq_coeffs = np.ones_like(auto) else: eq_coeffs = np.ones_like(auto) # divide out the equalization coefficients # eq_coeffs are stored as a length 1024 array but only a # single number is used. Taking the median to not deal with # a size mismatch eq_coeffs = np.median(eq_coeffs) auto /= eq_coeffs**2 auto[auto < 10**-10.0] = 10**-10.0 auto = np.median(auto) amps[(ant, pol)] = 10.0 * np.log10(auto) hsession = cm_sysutils.Handling(session) ants = np.unique([ant for (ant, pol) in amps.keys()]) pols = np.unique([pol for (ant, pol) in amps.keys()]) antpos = np.genfromtxt( os.path.join(mc.data_path, "HERA_350.txt"), usecols=(0, 1, 2, 3), dtype={ "names": ("ANTNAME", "EAST", "NORTH", "UP"), "formats": ("<U5", "<f8", "<f8", "<f8"), }, encoding=None, ) antnames = antpos["ANTNAME"] inds = [int(j[2:]) for j in antnames] inds = np.argsort(inds) antnames = np.take(antnames, inds) antpos = np.array([antpos["EAST"], antpos["NORTH"], antpos["UP"]]) array_center = np.mean(antpos, axis=1, keepdims=True) antpos -= array_center antpos = np.take(antpos, inds, axis=1) stations = hsession.get_connected_stations(at_date="now") for station in stations: if station.antenna_number not in ants: ants = np.append(ants, station.antenna_number) ants = np.unique(ants) stations = [] for station_type in hsession.geo.parse_station_types_to_check( "default"): for stn in hsession.geo.station_types[station_type]["Stations"]: stations.append(stn) # stations is a list of HH??? numbers we just want the ints stations = list(map(int, [j[2:] for j in stations])) built_but_not_on = np.setdiff1d(stations, ants) # Get node and PAM info node_ind = np.zeros_like(ants, dtype=np.int) pam_ind = np.zeros_like(ants, dtype=np.int) # defaul the snap name to "No Data" hostname = np.full_like(ants, "No\tData", dtype=object) snap_serial = np.full_like(ants, "No\tData", dtype=object) pam_power = {} adc_power = {} adc_rms = {} time_array = {} fem_imu_theta = {} fem_imu_phi = {} eq_coeffs = {} for ant in ants: for pol in pols: amps.setdefault((ant, pol), np.Inf) pam_power.setdefault((ant, pol), np.Inf) adc_power.setdefault((ant, pol), np.Inf) adc_rms.setdefault((ant, pol), np.Inf) eq_coeffs.setdefault((ant, pol), np.Inf) fem_imu_theta.setdefault((ant, pol), np.Inf) fem_imu_phi.setdefault((ant, pol), np.Inf) time_array.setdefault((ant, pol), now - Time(0, format="gps")) for ant_cnt, ant in enumerate(ants): station_status = session.get_antenna_status( most_recent=True, antenna_number=int(ant)) for status in station_status: antpol = (status.antenna_number, status.antenna_feed_pol) if status.pam_power is not None: pam_power[antpol] = status.pam_power if status.adc_power is not None: adc_power[antpol] = 10 * np.log10(status.adc_power) if status.adc_rms is not None: adc_rms[antpol] = status.adc_rms if status.time is not None: time_array[antpol] = now - Time(status.time, format="gps") if status.fem_imu_phi is not None: fem_imu_phi[antpol] = status.fem_imu_phi if status.fem_imu_theta is not None: fem_imu_theta[antpol] = status.fem_imu_theta if status.eq_coeffs is not None: _coeffs = np.fromstring(status.eq_coeffs.strip("[]"), sep=",") # just track the median coefficient for now eq_coeffs[antpol] = np.median(_coeffs) # Try to get the snap info. Output is a dictionary with 'e' and 'n' keys mc_name = antnames[ant] snap_info = hsession.get_part_at_station_from_type( mc_name, "now", "snap") # get the first key in the dict to index easier _key = list(snap_info.keys())[0] pol_key = [key for key in snap_info[_key].keys() if "E" in key] if pol_key: # 'E' should be in one of the keys, extract the 0th entry pol_key = pol_key[0] else: # a hacky solution for a key that should work pol_key = "E<ground" if snap_info[_key][pol_key] is not None: snap_serial[ant_cnt] = snap_info[_key][pol_key] # Try to get the pam info. Output is a dictionary with 'e' and 'n' keys pam_info = hsession.get_part_at_station_from_type( mc_name, "now", "post-amp") # get the first key in the dict to index easier _key = list(pam_info.keys())[0] if pam_info[_key][pol_key] is not None: _pam_num = re.findall(r"PAM(\d+)", pam_info[_key][pol_key])[0] pam_ind[ant_cnt] = np.int(_pam_num) else: pam_ind[ant_cnt] = -1 # Try to get the ADC info. Output is a dictionary with 'e' and 'n' keys node_info = hsession.get_part_at_station_from_type( mc_name, "now", "node") # get the first key in the dict to index easier _key = list(node_info.keys())[0] if node_info[_key][pol_key] is not None: _node_num = re.findall(r"N(\d+)", node_info[_key][pol_key])[0] node_ind[ant_cnt] = np.int(_node_num) _hostname = session.get_snap_hostname_from_serial( snap_serial[ant_cnt]) if _hostname is not None: hostname[ant_cnt] = _hostname else: snap_status = session.get_snap_status( most_recent=True, nodeID=np.int(_node_num)) for _status in snap_status: if _status.serial_number == snap_serial[ant_cnt]: hostname[ant_cnt] = _status.hostname else: node_ind[ant_cnt] = -1 pams, _pam_ind = np.unique(pam_ind, return_inverse=True) nodes, _node_ind = np.unique(node_ind, return_inverse=True) xs_offline = np.ma.masked_array( antpos[0, :], mask=[ True if int(name[2:]) in ants else False for name in antnames ], ) ys_offline = np.ma.masked_array(antpos[1, :], mask=xs_offline.mask) name_offline = np.ma.masked_array( [aname + "<br>OFFLINE" for aname in antnames], mask=xs_offline.mask, dtype=object, ) xs_offline = xs_offline names = [ "Auto [dB]", "PAM [dB]", "ADC [dB]", "ADC RMS", "FEM IMU THETA", "FEM IMU PHI", "EQ COEF", ] powers = [ amps, pam_power, adc_power, adc_rms, fem_imu_theta, fem_imu_phi, eq_coeffs, ] powers = [ np.ma.masked_invalid([[p[ant, pol] for ant in ants] for pol in pols]) for p in powers ] write_csv("ant_stats.csv", antnames, ants, pols, names, powers, built_but_not_on) time_array = np.array( [[time_array[ant, pol].to("hour").value for ant in ants] for pol in pols]) xs = np.ma.masked_array(antpos[0, ants], mask=powers[0][0].mask) ys = np.ma.masked_array( [ antpos[1, ants] + 3 * (pol_cnt - 0.5) for pol_cnt, pol in enumerate(pols) ], mask=powers[0].mask, ) _text = np.array( [[ antnames[ant] + pol + "<br>" + str(hostname[ant_cnt]) + "<br>" + "PAM\t#:\t" + str(pam_ind[ant_cnt]) for ant_cnt, ant in enumerate(ants) ] for pol_cnt, pol in enumerate(pols)], dtype="object", ) # want to format No Data where data was not retrieved for each type of power for pol_cnt, pol in enumerate(pols): for ant_cnt, ant in enumerate(ants): for _name, _power in zip(names, powers): if not _power.mask[pol_cnt, ant_cnt]: _text[pol_cnt, ant_cnt] += ( "<br>" + _name + ": {0:.2f}".format(_power[pol_cnt, ant_cnt])) else: _text[pol_cnt, ant_cnt] += "<br>" + _name + ": No Data" if time_array[pol_cnt, ant_cnt] > 2 * 24 * 365: # if the value is older than 2 years it is bad # value are stored in hours. # 2 was chosen arbitraritly. _text[pol_cnt, ant_cnt] += "<br>" + "Ant Status: No Date" else: _text[pol_cnt, ant_cnt] += ("<br>" + "Ant Status: {0:.2f} hrs old".format( time_array[pol_cnt, ant_cnt])) # having spaces will cause odd wrapping issues, replace all # spaces by \t _text[pol_cnt, ant_cnt] = _text[pol_cnt, ant_cnt].replace(" ", "\t") masks = [[True] for p in powers] # Offline antennas data_hex = [] offline_ants = { "x": xs_offline.compressed().tolist(), "y": ys_offline.compressed().tolist(), "text": name_offline, "mode": "markers", "visible": True, "marker": { "color": np.ma.masked_array(["black"] * len(name_offline), mask=xs_offline.mask), "size": 14, "opacity": 0.5, "symbol": "hexagon", }, "hovertemplate": "%{text}<extra></extra>", } # now we want to Fill in the conneted ones offline_ants["marker"]["color"][built_but_not_on] = "red" offline_ants["text"].data[built_but_not_on] = [ offline_ants["text"].data[ant].split("<br>")[0] + "<br>Constructed<br>Not\tOnline" for ant in built_but_not_on ] offline_ants["marker"]["color"] = ( offline_ants["marker"]["color"].compressed().tolist()) offline_ants["text"] = offline_ants["text"].compressed().tolist() data_hex.append(offline_ants) # for each type of power, loop over pols and print out the data # save up a mask array used for the buttons later # also plot the bad ones!3 colorscale = "Viridis" # define some custom scale values for the ADC RMS page rms_scale_vals = [2, 20] relavitve_values = [0.4, 0.7] rms_color_scale = [ ["0.0", "rgb(68,1,84)"], ["0.2", "rgb(62,74,137)"], ["0.3", "rgb(38,130,142)"], ["0.4", "rgb(53,183,121)"], ["0.5", "rgb(53,183,121)"], ["0.6", "rgb(53,183,121)"], ["0.7", "rgb(109,205,89)"], ["0.8", "rgb(180,222,44)"], ["1.0", "rgb(253,231,37)"], ] for pow_ind, power in enumerate(powers): if power.compressed().size > 0: vmax = np.max(power.compressed()) vmin = np.min(power.compressed()) else: vmax = 1 vmin = 0 colorscale = "Viridis" if pow_ind == 3: cbar_title = "RMS\tlinear" vmin = rms_scale_vals[0] * relavitve_values[0] vmax = rms_scale_vals[1] / relavitve_values[1] colorscale = rms_color_scale elif pow_ind == 4 or pow_ind == 5: cbar_title = "Degrees" elif pow_ind == len(powers) - 1: cbar_title = "Median\tCoeff" else: cbar_title = "dB" if pow_ind == 0: visible = True else: visible = False for pol_ind, pol in enumerate(pols): for mask_cnt, mask in enumerate(masks): if mask_cnt == pow_ind: mask.extend([True] * 2) else: mask.extend([False] * 2) _power = { "x": xs.data[~power[pol_ind].mask].tolist(), "y": ys[pol_ind].data[~power[pol_ind].mask].tolist(), "text": _text[pol_ind][~power[pol_ind].mask].tolist(), "mode": "markers", "visible": visible, "marker": { "color": power[pol_ind].data[~power[pol_ind].mask].tolist(), "size": 14, "cmin": vmin, "cmax": vmax, "colorscale": colorscale, "colorbar": { "thickness": 20, "title": cbar_title }, }, "hovertemplate": "%{text}<extra></extra>", } data_hex.append(_power) _power_offline = { "x": xs.data[power[pol_ind].mask].tolist(), "y": ys[pol_ind].data[power[pol_ind].mask].tolist(), "text": _text[pol_ind][power[pol_ind].mask].tolist(), "mode": "markers", "visible": visible, "marker": { "color": "orange", "size": 14, "cmin": vmin, "cmax": vmax, "colorscale": colorscale, "colorbar": { "thickness": 20, "title": cbar_title }, }, "hovertemplate": "%{text}<extra></extra>", } data_hex.append(_power_offline) buttons = [] for _name, mask in zip(names, masks): _button = { "args": [{ "visible": mask }, { "title": "", "annotations": {} }], "label": _name, "method": "restyle", } buttons.append(_button) updatemenus_hex = [{ "buttons": buttons, "showactive": True, "type": "buttons" }] layout_hex = { "xaxis": { "title": "East-West Position [m]" }, "yaxis": { "title": "North-South Position [m]", "scaleanchor": "x" }, "title": { "text": "Per Antpol Stats vs Hex position", "font": { "size": 24 }, }, "hoverlabel": { "align": "left" }, "margin": { "t": 40 }, "autosize": True, "showlegend": False, "hovermode": "closest", } caption = {} caption["title"] = "Stats vs Hex pos Help" caption["text"] = ( "This plot shows various statistics and measurements " "per ant-pol versus its position in the array." "<br>Antennas which are build but not fully hooked up " "are shown in light red." "<br>Grey antennas are not yet constructed." "<br><br><h4>Available plotting options</h4>" "<ul>" "<li>Auto Corr - Median Auto Correlation (in db) " "from the correlator with equalization coefficients " "divided out</li>" "<li>Pam Power - Latest Pam Power (in db) recorded in M&C</li>" "<li>ADC Power - Latest ADC Power (in db) recorded in M&C</li>" "<li>ADC RMS - Latest linear ADC RMS recorded in M&C</li>" "<li>FEM IMU THETA - IMU-reported theta, in degrees</li>" "<li>FEM IMU PHI - IMU-reported phi, in degrees</li>" "<li>EQ Coeffs - Latest Median Equalization Coefficient recorded in M&C</li>" "</ul>" "Any antpol showing with an orange color means " "no data is avaible for the currenty plot selection." "<h4>Hover label Formatting</h4>" "<ul>" "<li>Antenna Name from M&C<br>(e.g. HH0n = Hera Hex Antenna 0 Polarization N)</li>" "<li>Snap hostname from M&C<br>(e.g. heraNode0Snap0)</li>" "<li>PAM Number</li>" "<li>Median Auto Correlation power in dB</li>" "<li>PAM power in dB</li>" "<li>ADC power in dB</li>" "<li>Linear ADC RMS</li>" "<li>FEM IMU reported theta in degrees</li>" "<li>FEM IMU reported phi in degrees</li>" "<li>Median Equalization Coefficient</li>" "<li>Time ago in hours the M&C Antenna Status was updated. " "This time stamp applies to all data for this antenna " "except the Auto Correlation.</li>" "</ul>" "In any hover label entry 'No Data' means " "information not currrently available in M&C.") # Render all the power vs position files plotname = "plotly-hex" html_template = env.get_template("plotly_base.html") js_template = env.get_template("plotly_base.js") if sys.version_info.minor >= 8 and sys.version_info.major > 2: time_jd = latest.to_value('jd', subfmt='float') time_unix = latest.to_value('unix') else: time_jd = latest.jd time_unix = latest.unix rendered_hex_html = html_template.render( plotname=plotname, data_type="Auto correlations", plotstyle="height: 100%", gen_date=now.iso, data_date_iso=latest.iso, data_date_jd="{:.3f}".format(time_jd), data_date_unix_ms=time_unix * 1000, js_name="hex_amp", gen_time_unix_ms=now.unix * 1000, scriptname=os.path.basename(__file__), hostname=computer_hostname, caption=caption, ) rendered_hex_js = js_template.render( data=data_hex, layout=layout_hex, updatemenus=updatemenus_hex, plotname=plotname, ) with open("hex_amp.html", "w") as h_file: h_file.write(rendered_hex_html) with open("hex_amp.js", "w") as js_file: js_file.write(rendered_hex_js) # now prepare the data to be plotted vs node number data_node = [] masks = [[] for p in powers] vmax = [ np.max(power.compressed()) if power.compressed().size > 1 else 1 for power in powers ] vmin = [ np.min(power.compressed()) if power.compressed().size > 1 else 0 for power in powers ] vmin[3] = rms_scale_vals[0] * relavitve_values[0] vmax[3] = rms_scale_vals[1] / relavitve_values[1] for node in nodes: node_index = np.where(node_ind == node)[0] hosts = hostname[node_index] host_index = np.argsort(hosts) ys = np.ma.masked_array( [ np.arange(node_index.size) + 0.3 * pol_cnt for pol_cnt, pol in enumerate(pols) ], mask=powers[0][:, node_index].mask, ) xs = np.zeros_like(ys) xs[:] = node powers_node = [pow[:, node_index] for pow in powers] __text = _text[:, node_index] for pow_ind, power in enumerate(powers_node): cbar_title = "dB" if pow_ind == 4 or pow_ind == 5: cbar_title = "Degrees" if pow_ind == 3: colorscale = rms_color_scale else: colorscale = "Viridis" colorscale = "Viridis" if pow_ind == 3: cbar_title = "RMS\tlinear" colorscale = rms_color_scale elif pow_ind == 4 or pow_ind == 5: cbar_title = "Degrees" elif pow_ind == len(powers) - 1: cbar_title = "Median\tCoeff" else: cbar_title = "dB" if pow_ind == 0: visible = True else: visible = False for pol_ind, pol in enumerate(pols): for mask_cnt, mask in enumerate(masks): if mask_cnt == pow_ind: mask.extend([True] * 2) else: mask.extend([False] * 2) __power = power[pol_ind][host_index] ___text = __text[pol_ind][host_index] _power = { "x": xs[pol_ind].data[~__power.mask].tolist(), "y": ys[pol_ind].data[~__power.mask].tolist(), "text": ___text[~__power.mask].tolist(), "mode": "markers", "visible": visible, "marker": { "color": __power.data[~__power.mask].tolist(), "size": 14, "cmin": vmin[pow_ind], "cmax": vmax[pow_ind], "colorscale": colorscale, "colorbar": { "thickness": 20, "title": cbar_title }, }, "hovertemplate": "%{text}<extra></extra>", } data_node.append(_power) _power_offline = { "x": xs[pol_ind].data[__power.mask].tolist(), "y": ys[pol_ind].data[__power.mask].tolist(), "text": ___text[__power.mask].tolist(), "mode": "markers", "visible": visible, "marker": { "color": "orange", "size": 14, "cmin": vmin[pow_ind], "cmax": vmax[pow_ind], "colorscale": colorscale, "colorbar": { "thickness": 20, "title": cbar_title }, }, "hovertemplate": "%{text}<extra></extra>", } data_node.append(_power_offline) buttons = [] for _name, mask in zip(names, masks): _button = { "args": [{ "visible": mask }, { "title": "", "annotations": {} }], "label": _name, "method": "restyle", } buttons.append(_button) updatemenus_node = [{ "buttons": buttons, "showactive": True, "type": "buttons" }] layout_node = { "xaxis": { "title": "Node Number", "dtick": 1, "tick0": 0, "showgrid": False, "zeroline": False, }, "yaxis": { "showticklabels": False, "showgrid": False, "zeroline": False }, "title": { "text": "Per Antpol Stats vs Node #", "font": { "size": 24 } }, "hoverlabel": { "align": "left" }, "margin": { "t": 40 }, "autosize": True, "showlegend": False, "hovermode": "closest", } caption_node = {} caption_node["title"] = "Stats vs Node Help" caption_node["text"] = ( "This plot shows various statistics and measurements " "per ant-pol versus the node number to which it is connected." "<br><br><h4>Available plotting options</h4>" "<ul>" "<li>Auto Corr - Median Auto Correlation (in db) " "from the correlator with equalization coefficients " "divided out</li>" "<li>Pam Power - Latest Pam Power (in db) recorded in M&C</li>" "<li>ADC Power - Latest ADC Power (in db) recorded in M&C</li>" "<li>ADC RMS - Latest linear ADC RMS recorded in M&C</li>" "<li>EQ Coeffs - Latest Median Equalization Coefficient recorded in M&C</li>" "</ul>" "Any antpol showing with an orange color means " "no data is avaible for the currenty plot selection." "<h4>Hover label Formatting</h4>" "<ul>" "<li>Antenna Name from M&C<br>(e.g. HH0n = Hera Hex Antenna 0 Polarization N)</li>" "<li>Snap hostname from M&C<br>(e.g. heraNode0Snap0)</li>" "<li>PAM Number</li>" "<li>Median Auto Correlation power in dB</li>" "<li>PAM power in dB</li>" "<li>ADC power in dB</li>" "<li>Linear ADC RMS</li>" "<li>Median Equalization Coefficient</li>" "<li>Time ago in hours the M&C Antenna Status was updated. " "This time stamp applies to all data for this antenna " "except the Auto Correlation.</li>" "</ul>" "In any hover label entry 'No Data' means " "information not currrently available in M&C.") # Render all the power vs ndde files plotname = "plotly-node" html_template = env.get_template("plotly_base.html") js_template = env.get_template("plotly_base.js") if sys.version_info.minor >= 8 and sys.version_info.major > 2: time_jd = latest.to_value('jd', subfmt='float') time_unix = latest.to_value('unix') else: time_jd = latest.jd time_unix = latest.unix rendered_node_html = html_template.render( plotname=plotname, data_type="Auto correlations", plotstyle="height: 100%", gen_date=now.iso, gen_time_unix_ms=now.unix * 1000, data_date_iso=latest.iso, data_date_jd="{:.3f}".format(time_jd), data_date_unix_ms=time_unix * 1000, js_name="node_amp", scriptname=os.path.basename(__file__), hostname=computer_hostname, caption=caption_node, ) rendered_node_js = js_template.render( data=data_node, layout=layout_node, updatemenus=updatemenus_node, plotname=plotname, ) with open("node_amp.html", "w") as h_file: h_file.write(rendered_node_html) with open("node_amp.js", "w") as js_file: js_file.write(rendered_node_js)
if __name__ == '__main__': default_hookup_cols = [ 'station', 'feed', 'front-end', 'node-bulkhead', 'post-amp', 'snap', 'node' ] parser = mc.get_mc_argument_parser() # set values for 'action' to use parser.add_argument('-p', '--hpn', help="Part number, csv-list or [default]", default='default') parser.add_argument('-e', '--exact-match', help="Force exact matches on part numbers, not beginning N char. [False]", dest='exact_match', action='store_true') parser.add_argument('--hookup-cols', help="Specify a subset of parts to show comma-delimited no-space list.", dest='hookup_cols', default=default_hookup_cols) args = parser.parse_args() # Pre-process the args args.hpn = cm_utils.listify(args.hpn) args.hookup_cols = cm_utils.listify(args.hookup_cols) # Start session db = mc.connect_to_mc_db(args) session = db.sessionmaker() system = cm_sysutils.Handling(session) system.publish_summary(hlist=args.hpn, exact_match=args.exact_match, hookup_cols=args.hookup_cols, )
def get_cm_info(): from hera_mc import cm_sysutils h = cm_sysutils.Handling() return h.get_cminfo_correlator()
def main(): # templates are stored relative to the script dir # stored one level up, find the parent directory # and split the parent directory away script_dir = os.path.dirname(os.path.realpath(__file__)) split_dir = os.path.split(script_dir) template_dir = os.path.join(split_dir[0], "templates") env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True) # this filter is used to see if there is more than one table env.filters["islist"] = is_list if sys.version_info[0] < 3: # py2 computer_hostname = os.uname()[1] else: # py3 computer_hostname = os.uname().nodename parser = mc.get_mc_argument_parser() parser.add_argument( "--redishost", dest="redishost", type=str, default="redishost", help=( 'The host name for redis to connect to, defaults to "redishost"'), ) parser.add_argument("--port", dest="port", type=int, default=6379, help="Redis port to connect.") args = parser.parse_args() try: db = mc.connect_to_mc_db(args) except RuntimeError as e: raise SystemExit(str(e)) try: redis_db = redis.Redis(args.redishost, port=args.port) redis_db.keys() except Exception as err: raise SystemExit(str(err)) with db.sessionmaker() as session: now = Time.now() hsession = cm_sysutils.Handling(session) stations = hsession.get_connected_stations(at_date="now") ants = [] for station in stations: if station.antenna_number not in ants: ants = np.append(ants, station.antenna_number) ants = np.unique(ants).astype(int) antpos = np.genfromtxt( os.path.join(mc.data_path, "HERA_350.txt"), usecols=(0, 1, 2, 3), dtype={ "names": ("ANTNAME", "EAST", "NORTH", "UP"), "formats": ("<U5", "<f8", "<f8", "<f8"), }, encoding=None, ) antnames = antpos["ANTNAME"] inds = [int(j[2:]) for j in antnames] inds = np.argsort(inds) antnames = np.take(antnames, inds) nodes = [] hists = [] bad_ants = [] bad_node = [] for ant_cnt, ant in enumerate(ants): ant_status = session.get_antenna_status(most_recent=True, antenna_number=int(ant)) mc_name = antnames[int(ant)] node_info = hsession.get_part_at_station_from_type( mc_name, "now", "node") if len(ant_status) == 0: for pol in ["e", "n"]: name = "{ant}:{pol}".format(ant=ant, pol=pol) print("No histogram data for ", name) bad_ants.append(name) for stat in ant_status: name = "ant{ant}{pol}".format(ant=stat.antenna_number, pol=stat.antenna_feed_pol) # try to find the associated node # get the first key in the dict to index easier _key = list(node_info.keys())[0] pol_key = [ key for key in node_info[_key].keys() if stat.antenna_feed_pol.upper() in key ] if pol_key: # 'E' should be in one of the keys, extract the 0th entry pol_key = pol_key[0] else: # a hacky solution for a key that should work pol_key = "E<ground" if node_info[_key][pol_key] is not None: _node_num = re.findall(r"N(\d+)", node_info[_key][pol_key])[0] else: print("No Node mapping for antennna: " + name) _node_num = -1 bad_node.append("{ant}:{pol}".format( ant=stat.antenna_number, pol=stat.antenna_feed_pol)) nodes.append(_node_num) timestamp = Time(stat.time, format="gps") if (stat.histogram_bin_centers is not None and stat.histogram is not None): bins = np.fromstring( stat.histogram_bin_centers.strip("[]"), sep=",") hist = np.fromstring(stat.histogram.strip("[]"), sep=",") if stat.eq_coeffs is not None: eq_coeffs = np.fromstring(stat.eq_coeffs.strip("[]"), sep=",") else: eq_coeffs = np.ones_like(hist) hist /= np.median(eq_coeffs)**2 text = "observed at {iso}<br>(JD {jd})".format( iso=timestamp.iso, jd=timestamp.jd) # spaces cause weird wrapping issues, replace them all with \t text = text.replace(" ", "\t") _data = { "x": bins.tolist(), "y": hist.tolist(), "name": name, "node": _node_num, "text": [text] * bins.size, "hovertemplate": "(%{x:.1},\t%{y})<br>%{text}", } hists.append(_data) else: name = "{ant}:{pol}".format(ant=stat.antenna_number, pol=stat.antenna_feed_pol) print("No histogram data for ", name) bad_ants.append(name) table = {} table["title"] = "Ants with no Histogram" table["rows"] = [] row = {} row["text"] = ",\t".join(bad_ants) table["rows"].append(row) table_node = {} table_node["title"] = "Antennas with no Node mapping" row_node = {} row_node["text"] = "\t".join(bad_node) rows_node = [row_node] table_node["rows"] = rows_node layout = { "xaxis": { "title": "ADC value" }, "yaxis": { "title": "Occurance", "type": "linear" }, "title": { "text": "ADC Histograms", "xref": "paper", "x": 0.5, "yref": "paper", "y": 1.5, "font": { "size": 24, }, }, "margin": { "l": 40, "b": 30, "r": 40, "t": 70 }, "hovermode": "closest", "autosize": True, "showlegend": True, } # Make all the buttons for this plot nodes = np.unique(nodes) # if an antenna was not mapped, roll the -1 to the end # this makes making buttons easier so the unmapped show last if -1 in nodes: nodes = np.roll(nodes, -1) # create a mask to find all the matching nodes node_mask = [[True if s["node"] == node else False for s in hists] for node in nodes] buttons_node = [] _button_node = { "args": [ { "visible": [True for s in hists] }, { "title": "", "annotations": {} }, ], "label": "All\tAnts", "method": "restyle", } buttons_node.append(_button_node) for node_cnt, node in enumerate(nodes): if node != -1: label = "Node\t{}".format(node) else: label = "Unmapped\tAnts" _button_node = { "args": [ { "visible": node_mask[node_cnt] }, { "title": "", "annotations": {} }, ], "label": label, "method": "restyle", } buttons_node.append(_button_node) buttons = [] log_buttons = { "args": [{}, { "yaxis": { "type": "log" } }], "label": "Log", "method": "update", } lin_buttons = { "args": [{}, { "yaxis": { "type": "linear" } }], "label": "Linear", "method": "update", } buttons.append(lin_buttons) buttons.append(log_buttons) updatemenus = [ { "buttons": buttons_node, "showactive": True, "active": 0, "type": "dropdown", "x": 0.535, "y": 1.03, }, { "buttons": buttons, "showactive": True, "type": "buttons", "active": 0, }, ] plotname = "plotly-adc-hist" html_template = env.get_template("ploty_with_multi_table.html") js_template = env.get_template("plotly_base.js") caption = {} caption["text"] = ( "The ADC Histograms with equalization coefficients " "divided out." "<br><br>Some antennas known to M&C may not have a histogram " " and are listed below the image." "<br><br>Some antennas may not have " "a known node mapping and are listed below the image.\n " "<br><br>Plot can be downselected to display " "individual nodes or show the entire array.\n " "<br><br>Double click on an entry in the legend " "to select only that entry, " "double click again to restore all plots.\n " "<br><br><h4>Formatting options</h4>" "<ul>" "<li>Linear - Display with Linear y-axis</li>" "<li>Log - Display with Log y-axis</li>" "</ul>" "<br><br>Single click an entry in the legend to un-plot it, " "single click again to restore it to the plot.") caption["title"] = "Histogram Help" rendered_html = html_template.render( plotname=plotname, plotstyle="height: 100%", gen_date=now.iso, js_name="adchist", caption=caption, gen_time_unix_ms=now.unix * 1000, scriptname=os.path.basename(__file__), hostname=computer_hostname, table=[table, table_node], ) rendered_js = js_template.render(data=hists, layout=layout, plotname=plotname, updatemenus=updatemenus) with open("adchist.html", "w") as h_file: h_file.write(rendered_html) with open("adchist.js", "w") as js_file: js_file.write(rendered_js)
def main(): # templates are stored relative to the script dir # stored one level up, find the parent directory # and split the parent directory away script_dir = os.path.dirname(os.path.realpath(__file__)) split_dir = os.path.split(script_dir) template_dir = os.path.join(split_dir[0], 'templates') env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True) # this filter is used to see if there is more than one table env.filters['islist'] = is_list if sys.version_info[0] < 3: # py2 computer_hostname = os.uname()[1] else: # py3 computer_hostname = os.uname().nodename # The standard M&C argument parser parser = mc.get_mc_argument_parser() # we'll have to add some extra options too parser.add_argument( '--redishost', dest='redishost', type=str, default='redishost', help=( 'The host name for redis to connect to, defualts to "redishost"')) parser.add_argument('--port', dest='port', type=int, default=6379, help='Redis port to connect.') args = parser.parse_args() try: db = mc.connect_to_mc_db(args) except RuntimeError as e: raise SystemExit(str(e)) try: redis_db = redis.Redis(args.redishost, port=args.port) redis_db.keys() except Exception as err: raise SystemExit(str(err)) with db.sessionmaker() as session: corr_cm = hera_corr_cm.HeraCorrCM(redishost=args.redishost) hsession = cm_sysutils.Handling(session) stations = hsession.get_connected_stations(at_date='now') antpos = np.genfromtxt(os.path.join(mc.data_path, "HERA_350.txt"), usecols=(0, 1, 2, 3), dtype={ 'names': ('ANTNAME', 'EAST', 'NORTH', 'UP'), 'formats': ('<U5', '<f8', '<f8', '<f8') }, encoding=None) antnames = antpos['ANTNAME'] inds = [int(j[2:]) for j in antnames] inds = np.argsort(inds) antnames = np.take(antnames, inds) ants = [] for station in stations: if station.antenna_number not in ants: ants = np.append(ants, station.antenna_number) ants = np.unique(ants).astype(int) hostname_lookup = {} # all_snap_statuses = session.get_snap_status(most_recent=True) all_snap_statuses = corr_cm.get_f_status() snapautos = {} ant_status_from_snaps = corr_cm.get_ant_status() all_snaprf_stats = corr_cm.get_snaprf_status() table_snap = {} table_snap["title"] = "Snap hookups with No Data" rows = [] bad_snaps = [] for snap_chan in all_snaprf_stats: host, loc_num = snap_chan.split(":") loc_num = int(loc_num) # initialize this key if not already in the dict auto_group = snapautos.setdefault(host, {}) try: tmp_auto = np.array( all_snaprf_stats[snap_chan]["autocorrelation"]) if np.all(tmp_auto == "None"): print("No Data for {} port {}".format(host, loc_num)) bad_snaps.append(snap_chan) tmp_auto = np.full(1024, np.nan) eq_coeffs = np.array(all_snaprf_stats[snap_chan]["eq_coeffs"]) if np.all(eq_coeffs == "None"): eq_coeffs = np.full_like(tmp_auto, 1.0) tmp_auto /= eq_coeffs**2 tmp_auto = np.ma.masked_invalid(10 * np.log10(np.real(tmp_auto))) auto_group[loc_num] = tmp_auto.filled(-50) except KeyError: print("Snap connection with no autocorrelation", snap_chan) raise except TypeError: print("Received TypeError when taking log.") print("Type of item in dictionary: ", type(all_snaprf_stats[snap_chan]["autocorrelation"])) print("Value of item: ", tmp_auto) raise hostname_lookup.setdefault(host, {}) hostname_lookup[host].setdefault(loc_num, {}) hostname_lookup[host][loc_num]['MC'] = 'NC' row = {} row["text"] = '\t'.join(bad_snaps) rows.append(row) table_snap["rows"] = rows table_ants = {} table_ants["title"] = "Antennas with no mapping" rows = [] bad_ants = [] bad_hosts = [] for ant_cnt, ant in enumerate(ants): # get the status for both polarizations for this antenna ant_status = { key: ant_status_from_snaps[key] for key in ant_status_from_snaps if ant == int(key.split(':')[0]) } # check if the antenna status from M&C has the host and # channel number, if it does not we have to do some gymnastics for antkey in ant_status: pol_key = antkey.split(":")[1] stat = ant_status[antkey] name = "{ant:d}:{pol}".format(ant=ant, pol=pol_key) # check that key is in the dictionary, is not None or the string "None" if ('f_host' in stat and 'host_ant_id' in stat and stat['f_host'] is not None and stat['host_ant_id'] is not None and stat['f_host'] != "None" and stat['host_ant_id'] != "None"): hostname = stat['f_host'] loc_num = stat['host_ant_id'] hostname_lookup[hostname][loc_num]['MC'] = name else: # Try to get the snap info from M&C. Output is a dictionary with 'e' and 'n' keys # connect to M&C to find all the hooked up Snap hostnames and corresponding ant-pols mc_name = antnames[ant] # these two may not be used, but it is easier to grab them now snap_info = hsession.get_part_at_station_from_type( mc_name, 'now', 'snap', include_ports=True) for _key in snap_info.keys(): # initialize a dict if they key does not exist already pol_key = pol_key.upper() + "<ground" if snap_info[_key][pol_key] is not None: serial_with_ports = snap_info[_key][pol_key] snap_serial = serial_with_ports.split( '>')[1].split('<')[0] ant_channel = int( serial_with_ports.split('>')[0][1:]) // 2 hostname = session.get_snap_hostname_from_serial( snap_serial) if hostname is None: node_info = hsession.get_part_at_station_from_type( mc_name, 'now', 'node') _node_num = re.findall( r'N(\d+)', node_info[_key][pol_key])[0] snap_stats = session.get_snap_status( nodeID=int(_node_num), most_recent=True) for _stat in snap_stats: if _stat.serial_number == snap_serial: hostname = _stat.hostname # if hostname is still None then we don't have a known hookup if hostname is None: print("No MC snap information for antennna: " + name) bad_ants.append(name) else: if hostname not in hostname_lookup.keys(): err = "host from M&C not found in corr_cm 'status:snaprf' : {}".format( hostname) err += '\nThis host may not have data populated yet or is offline.' err += '\nAll anteanns on this host will be full of 0.' print(err) bad_hosts.append(hostname) grp1 = hostname_lookup.setdefault(hostname, {}) # if this loc num is not in lookup table initialize # empty list if ant_channel not in grp1.keys(): if hostname not in bad_hosts: print( "loc_num from M&C not found in hera_corr_cm `status:snaprf` (host, location number): {}" .format([hostname, ant_channel])) print( "filling with bad array full of 0." ) else: _name = "{host}:{loc}".format( host=hostname, loc=ant_channel) if _name not in table_snap["rows"][0][ "text"]: table_snap["rows"][0][ "text"] += _name + '\t' snap_grp1 = snapautos.setdefault( hostname, {}) snap_grp1[ant_channel] = np.full(1024, 0) grp2 = grp1.setdefault(ant_channel, {}) grp2['MC'] = name else: print("No MC snap information for antennna: " + name) bad_ants.append(name) row = {} row["text"] = '\t'.join(bad_ants) rows.append(row) table_ants["rows"] = rows host_masks = [] for h1 in sorted(hostname_lookup.keys()): _mask = [] for h2 in sorted(hostname_lookup.keys()): _mask.extend([ True if h2 == h1 else False for loc_num in hostname_lookup[h2] ]) host_masks.append(_mask) # Generate frequency axis freqs = np.linspace(0, 250e6, 1024) freqs /= 1e6 data = [] for host_cnt, host in enumerate(sorted(hostname_lookup.keys())): if host_cnt == 0: visible = True else: visible = False for loc_num in sorted(hostname_lookup[host].keys()): mc_name = hostname_lookup[host][loc_num]['MC'] name = '{loc}:{mcname}'.format(loc=loc_num, mcname=mc_name.replace(":", "")) try: _data = { "x": freqs.tolist(), "y": snapautos[host][loc_num].tolist(), "name": name, "visible": visible, "hovertemplate": "%{x:.1f}\tMHz<br>%{y:.3f}\t[dB]" } except KeyError: print("Given host, location pair: ({0}, {1})".format( host, loc_num)) print("All possible keys for host {0}: {1}".format( host, list(snapautos[host].keys()))) raise data.append(_data) buttons = [] for host_cnt, host in enumerate(sorted(hostname_lookup.keys())): prog_time = all_snap_statuses[host]['last_programmed'] timestamp = all_snap_statuses[host]['timestamp'] temp = all_snap_statuses[host]['temp'] uptime = all_snap_statuses[host]['uptime'] pps = all_snap_statuses[host]['pps_count'] if host in bad_hosts: label = ('{host}<br>programmed:\t{start}' '<br>spectra\trecorded:\tNO DATA OBSERVED<br>' 'temp:\t{temp:.0f}\tC\t' 'pps\tcount:\t{pps}\tCycles\t\t\t' 'uptime:\t{uptime}' ''.format(host=host, start=prog_time.isoformat(' '), temp=temp, pps=pps, uptime=uptime)) else: label = ('{host}<br>programmed:\t{start}' '<br>spectra\trecorded:\t{obs}<br>' 'temp:\t{temp:.0f}\tC\t' 'pps\tcount:\t{pps}\tCycles\t\t\t' 'uptime:\t{uptime}' ''.format(host=host, start=prog_time.isoformat(' '), obs=timestamp.isoformat(' '), temp=temp, pps=pps, uptime=uptime)) _button = { "args": [ { "visible": host_masks[host_cnt] }, { "title": '', # "annotations": {} } ], "label": label, "method": "restyle" } buttons.append(_button) updatemenus = [{ "buttons": buttons, "showactive": True, "type": "dropdown", "x": .7, "y": 1.1, }] layout = { "xaxis": { "title": "Frequency [MHz]", "showticklabels": True, "tick0": 0, "dtick": 10, }, "yaxis": { "title": 'Power [dB]', "showticklabels": True, }, "hoverlabel": { "align": "left" }, "margin": { "l": 40, "b": 30, "r": 40, "t": 30 }, "autosize": True, "showlegend": True, "hovermode": 'closest', "annotations": [{ "x": 1.03, "y": 1.03, "align": "right", "valign": "top", "text": 'ADC Port # : Antpol', "showarrow": False, "xref": "paper", "yref": "paper", "xanchor": "center", "yanchor": "top" }] } caption = {} caption["title"] = "Snap Spectra Help" caption["text"] = ( "Autocorrelations (in dB) calculated by each Snap " "with equalization coefficients divided out. " "Calculation is independent of the corrlator. " "<br><br>" "Data is organized by the ADC Port number from " "each Snap. " "A mapping dictionary is used to look up the " "associated ant-pol for each ADC Port. " "Some ADC Ports may not have a known ant-pol " "and are labeled as N/C." "<br>Some antennas known to M&C may not have a known ADC port" " mapping and are labeled below the plot." "<br><br>Some Snap ports my not have data and are " "labeled in a table below the plot." "<br><br>The node selection button contains a lot " "of information described here." "<br><h4>Button Label Information</h4>" "<ul>" "<li>Snap hostname</li>" "<li>Last time the Snap was programmed</li>" "<li>Last time data was recorded from this Snap</li>" "<li>Last known temperature in C | PPS count | Uptime</li>" "</ul>") plotname = "plotly-snap" html_template = env.get_template("refresh_with_table.html") js_template = env.get_template("plotly_base.js") rendered_html = html_template.render( plotname=plotname, plotstyle="height: 100%", gen_date=Time.now().iso, js_name='snapspectra', gen_time_unix_ms=Time.now().unix * 1000, scriptname=os.path.basename(__file__), hostname=computer_hostname, table=[table_snap, table_ants], caption=caption) rendered_js = js_template.render(data=data, layout=layout, updatemenus=updatemenus, plotname=plotname) with open('snapspectra.html', 'w') as h_file: h_file.write(rendered_html) with open('snapspectra.js', 'w') as js_file: js_file.write(rendered_js)