コード例 #1
0
    def handle(self, *args, **options):
        """Perform necessary calculation and inser antennas in to DB."""
        mc_args = Namespace()
        mc_args.mc_db_name = options["mc_db_name"]
        mc_args.mc_config_path = options["mc_config_path"]
        db = mc.connect_to_mc_db(args=mc_args)
        antpos = np.genfromtxt(
            os.path.join(mc.data_path, "HERA_350.txt"),
            usecols=(0, 1, 2, 3),
            dtype={
                "names": ("ANTNAME", "EAST", "NORTH", "UP"),
                "formats": ("<U5", "<f8", "<f8", "<f8"),
            },
            encoding=None,
        )
        antnames = antpos["ANTNAME"]
        inds = [int(j[2:]) for j in antnames]
        inds = np.argsort(inds)

        antnames = np.take(antnames, inds)

        antpos = np.array([antpos["EAST"], antpos["NORTH"], antpos["UP"]])
        array_center = np.mean(antpos, axis=1, keepdims=True)
        antpos -= array_center
        antpos = np.take(antpos, inds, axis=1)

        with db.sessionmaker() as session:
            hsession = cm_sysutils.Handling(session)

            stations = []
            for station_type in hsession.geo.parse_station_types_to_check(
                    "default"):
                for stn in hsession.geo.station_types[station_type][
                        "Stations"]:
                    stations.append(stn)

            # stations is a list of HH??? numbers we just want the ints
            stations = list(map(int, [j[2:] for j in stations]))

            bulk_add = []
            for ind, name in enumerate(antnames):
                ant_number = int(name[2:])
                for pol in ["e", "n"]:
                    bulk_add.append(
                        Antenna(
                            ant_number=ant_number,
                            ant_name=name,
                            polarization=pol,
                            antpos_enu=antpos[:, ind].tolist(),
                            constructed=ant_number in stations,
                        ))

        Antenna.objects.bulk_create(bulk_add, ignore_conflicts=True)
コード例 #2
0
def main():
    # templates are stored relative to the script dir
    # stored one level up, find the parent directory
    # and split the parent directory away
    script_dir = os.path.dirname(os.path.realpath(__file__))
    split_dir = os.path.split(script_dir)
    template_dir = os.path.join(split_dir[0], "templates")

    env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True)
    if sys.version_info[0] < 3:
        # py2
        computer_hostname = os.uname()[1]
    else:
        # py3
        computer_hostname = os.uname().nodename

    # The standard M&C argument parser
    parser = mc.get_mc_argument_parser()
    # we'll have to add some extra options too
    parser.add_argument(
        "--redishost",
        dest="redishost",
        type=str,
        default="redishost",
        help=(
            'The host name for redis to connect to, defualts to "redishost"'),
    )
    parser.add_argument("--port",
                        dest="port",
                        type=int,
                        default=6379,
                        help="Redis port to connect.")
    args = parser.parse_args()

    try:
        db = mc.connect_to_mc_db(args)
    except RuntimeError as e:
        raise SystemExit(str(e))

    try:
        redis_db = redis.Redis(args.redishost, port=args.port)
        redis_db.keys()
    except Exception as err:
        raise SystemExit(str(err))

    with db.sessionmaker() as session:
        # without item this will be an array which will break database queries
        latest = Time(
            np.frombuffer(redis_db.get("auto:timestamp"),
                          dtype=np.float64).item(),
            format="jd",
        )
        latest.out_subfmt = u"date_hm"

        now = Time.now()
        amps = {}
        keys = [
            k.decode() for k in redis_db.keys()
            if k.startswith(b"auto") and not k.endswith(b"timestamp")
        ]

        for key in keys:
            match = re.search(r"auto:(?P<ant>\d+)(?P<pol>e|n)", key)
            if match is not None:
                ant, pol = int(match.group("ant")), match.group("pol")
                d = redis_db.get(key)
                if d is not None:
                    # need to copy because frombuffer creates a read-only array
                    auto = np.frombuffer(d, dtype=np.float32).copy()

                    eq_coeff = redis_db.hget(
                        bytes("eq:ant:{ant}:{pol}".format(ant=ant,
                                                          pol=pol).encode()),
                        "values",
                    )
                    if eq_coeff is not None:
                        eq_coeffs = np.fromstring(
                            eq_coeff.decode("utf-8").strip("[]"), sep=",")
                        if eq_coeffs.size == 0:
                            eq_coeffs = np.ones_like(auto)
                    else:
                        eq_coeffs = np.ones_like(auto)

                    # divide out the equalization coefficients
                    # eq_coeffs are stored as a length 1024 array but only a
                    # single number is used. Taking the median to not deal with
                    # a size mismatch
                    eq_coeffs = np.median(eq_coeffs)
                    auto /= eq_coeffs**2
                    auto[auto < 10**-10.0] = 10**-10.0
                    auto = np.median(auto)
                    amps[(ant, pol)] = 10.0 * np.log10(auto)

        hsession = cm_sysutils.Handling(session)
        ants = np.unique([ant for (ant, pol) in amps.keys()])
        pols = np.unique([pol for (ant, pol) in amps.keys()])

        antpos = np.genfromtxt(
            os.path.join(mc.data_path, "HERA_350.txt"),
            usecols=(0, 1, 2, 3),
            dtype={
                "names": ("ANTNAME", "EAST", "NORTH", "UP"),
                "formats": ("<U5", "<f8", "<f8", "<f8"),
            },
            encoding=None,
        )
        antnames = antpos["ANTNAME"]
        inds = [int(j[2:]) for j in antnames]
        inds = np.argsort(inds)

        antnames = np.take(antnames, inds)

        antpos = np.array([antpos["EAST"], antpos["NORTH"], antpos["UP"]])
        array_center = np.mean(antpos, axis=1, keepdims=True)
        antpos -= array_center
        antpos = np.take(antpos, inds, axis=1)

        stations = hsession.get_connected_stations(at_date="now")

        for station in stations:
            if station.antenna_number not in ants:
                ants = np.append(ants, station.antenna_number)
        ants = np.unique(ants)

        stations = []
        for station_type in hsession.geo.parse_station_types_to_check(
                "default"):
            for stn in hsession.geo.station_types[station_type]["Stations"]:
                stations.append(stn)

        # stations is a list of HH??? numbers we just want the ints
        stations = list(map(int, [j[2:] for j in stations]))
        built_but_not_on = np.setdiff1d(stations, ants)
        # Get node and PAM info
        node_ind = np.zeros_like(ants, dtype=np.int)
        pam_ind = np.zeros_like(ants, dtype=np.int)
        # defaul the snap name to "No Data"
        hostname = np.full_like(ants, "No\tData", dtype=object)
        snap_serial = np.full_like(ants, "No\tData", dtype=object)

        pam_power = {}
        adc_power = {}
        adc_rms = {}
        time_array = {}
        fem_imu_theta = {}
        fem_imu_phi = {}
        eq_coeffs = {}
        for ant in ants:
            for pol in pols:
                amps.setdefault((ant, pol), np.Inf)
                pam_power.setdefault((ant, pol), np.Inf)
                adc_power.setdefault((ant, pol), np.Inf)
                adc_rms.setdefault((ant, pol), np.Inf)
                eq_coeffs.setdefault((ant, pol), np.Inf)
                fem_imu_theta.setdefault((ant, pol), np.Inf)
                fem_imu_phi.setdefault((ant, pol), np.Inf)
                time_array.setdefault((ant, pol), now - Time(0, format="gps"))

        for ant_cnt, ant in enumerate(ants):
            station_status = session.get_antenna_status(
                most_recent=True, antenna_number=int(ant))

            for status in station_status:
                antpol = (status.antenna_number, status.antenna_feed_pol)
                if status.pam_power is not None:
                    pam_power[antpol] = status.pam_power
                if status.adc_power is not None:
                    adc_power[antpol] = 10 * np.log10(status.adc_power)
                if status.adc_rms is not None:
                    adc_rms[antpol] = status.adc_rms
                if status.time is not None:
                    time_array[antpol] = now - Time(status.time, format="gps")
                if status.fem_imu_phi is not None:
                    fem_imu_phi[antpol] = status.fem_imu_phi
                if status.fem_imu_theta is not None:
                    fem_imu_theta[antpol] = status.fem_imu_theta
                if status.eq_coeffs is not None:
                    _coeffs = np.fromstring(status.eq_coeffs.strip("[]"),
                                            sep=",")
                    # just track the median coefficient for now
                    eq_coeffs[antpol] = np.median(_coeffs)

            # Try to get the snap info. Output is a dictionary with 'e' and 'n' keys
            mc_name = antnames[ant]
            snap_info = hsession.get_part_at_station_from_type(
                mc_name, "now", "snap")
            # get the first key in the dict to index easier
            _key = list(snap_info.keys())[0]
            pol_key = [key for key in snap_info[_key].keys() if "E" in key]
            if pol_key:
                # 'E' should be in one of the keys, extract the 0th entry
                pol_key = pol_key[0]
            else:
                # a hacky solution for a key that should work
                pol_key = "E<ground"
            if snap_info[_key][pol_key] is not None:
                snap_serial[ant_cnt] = snap_info[_key][pol_key]

            # Try to get the pam info. Output is a dictionary with 'e' and 'n' keys
            pam_info = hsession.get_part_at_station_from_type(
                mc_name, "now", "post-amp")
            # get the first key in the dict to index easier
            _key = list(pam_info.keys())[0]
            if pam_info[_key][pol_key] is not None:
                _pam_num = re.findall(r"PAM(\d+)", pam_info[_key][pol_key])[0]
                pam_ind[ant_cnt] = np.int(_pam_num)
            else:
                pam_ind[ant_cnt] = -1

            # Try to get the ADC info. Output is a dictionary with 'e' and 'n' keys
            node_info = hsession.get_part_at_station_from_type(
                mc_name, "now", "node")
            # get the first key in the dict to index easier
            _key = list(node_info.keys())[0]
            if node_info[_key][pol_key] is not None:
                _node_num = re.findall(r"N(\d+)", node_info[_key][pol_key])[0]
                node_ind[ant_cnt] = np.int(_node_num)

                _hostname = session.get_snap_hostname_from_serial(
                    snap_serial[ant_cnt])

                if _hostname is not None:
                    hostname[ant_cnt] = _hostname
                else:
                    snap_status = session.get_snap_status(
                        most_recent=True, nodeID=np.int(_node_num))
                    for _status in snap_status:
                        if _status.serial_number == snap_serial[ant_cnt]:
                            hostname[ant_cnt] = _status.hostname
            else:
                node_ind[ant_cnt] = -1

        pams, _pam_ind = np.unique(pam_ind, return_inverse=True)
        nodes, _node_ind = np.unique(node_ind, return_inverse=True)

        xs_offline = np.ma.masked_array(
            antpos[0, :],
            mask=[
                True if int(name[2:]) in ants else False for name in antnames
            ],
        )
        ys_offline = np.ma.masked_array(antpos[1, :], mask=xs_offline.mask)
        name_offline = np.ma.masked_array(
            [aname + "<br>OFFLINE" for aname in antnames],
            mask=xs_offline.mask,
            dtype=object,
        )
        xs_offline = xs_offline

        names = [
            "Auto  [dB]",
            "PAM [dB]",
            "ADC [dB]",
            "ADC RMS",
            "FEM IMU THETA",
            "FEM IMU PHI",
            "EQ COEF",
        ]
        powers = [
            amps,
            pam_power,
            adc_power,
            adc_rms,
            fem_imu_theta,
            fem_imu_phi,
            eq_coeffs,
        ]
        powers = [
            np.ma.masked_invalid([[p[ant, pol] for ant in ants]
                                  for pol in pols]) for p in powers
        ]
        write_csv("ant_stats.csv", antnames, ants, pols, names, powers,
                  built_but_not_on)

        time_array = np.array(
            [[time_array[ant, pol].to("hour").value for ant in ants]
             for pol in pols])
        xs = np.ma.masked_array(antpos[0, ants], mask=powers[0][0].mask)
        ys = np.ma.masked_array(
            [
                antpos[1, ants] + 3 * (pol_cnt - 0.5)
                for pol_cnt, pol in enumerate(pols)
            ],
            mask=powers[0].mask,
        )
        _text = np.array(
            [[
                antnames[ant] + pol + "<br>" + str(hostname[ant_cnt]) +
                "<br>" + "PAM\t#:\t" + str(pam_ind[ant_cnt])
                for ant_cnt, ant in enumerate(ants)
            ] for pol_cnt, pol in enumerate(pols)],
            dtype="object",
        )

        #  want to format No Data where data was not retrieved for each type of power
        for pol_cnt, pol in enumerate(pols):
            for ant_cnt, ant in enumerate(ants):
                for _name, _power in zip(names, powers):
                    if not _power.mask[pol_cnt, ant_cnt]:
                        _text[pol_cnt, ant_cnt] += (
                            "<br>" + _name +
                            ": {0:.2f}".format(_power[pol_cnt, ant_cnt]))
                    else:
                        _text[pol_cnt, ant_cnt] += "<br>" + _name + ": No Data"
                if time_array[pol_cnt, ant_cnt] > 2 * 24 * 365:
                    # if the value is older than 2 years it is bad
                    # value are stored in hours.
                    # 2 was chosen arbitraritly.
                    _text[pol_cnt, ant_cnt] += "<br>" + "Ant Status:  No Date"
                else:
                    _text[pol_cnt,
                          ant_cnt] += ("<br>" +
                                       "Ant Status: {0:.2f} hrs old".format(
                                           time_array[pol_cnt, ant_cnt]))
                # having spaces will cause odd wrapping issues, replace all
                # spaces by \t
                _text[pol_cnt, ant_cnt] = _text[pol_cnt,
                                                ant_cnt].replace(" ", "\t")

        masks = [[True] for p in powers]

        # Offline antennas
        data_hex = []
        offline_ants = {
            "x": xs_offline.compressed().tolist(),
            "y": ys_offline.compressed().tolist(),
            "text": name_offline,
            "mode": "markers",
            "visible": True,
            "marker": {
                "color":
                np.ma.masked_array(["black"] * len(name_offline),
                                   mask=xs_offline.mask),
                "size":
                14,
                "opacity":
                0.5,
                "symbol":
                "hexagon",
            },
            "hovertemplate": "%{text}<extra></extra>",
        }
        # now we want to Fill in the conneted ones
        offline_ants["marker"]["color"][built_but_not_on] = "red"
        offline_ants["text"].data[built_but_not_on] = [
            offline_ants["text"].data[ant].split("<br>")[0] +
            "<br>Constructed<br>Not\tOnline" for ant in built_but_not_on
        ]

        offline_ants["marker"]["color"] = (
            offline_ants["marker"]["color"].compressed().tolist())
        offline_ants["text"] = offline_ants["text"].compressed().tolist()
        data_hex.append(offline_ants)

        #  for each type of power, loop over pols and print out the data
        #  save up a mask array used for the buttons later
        #  also plot the bad ones!3
        colorscale = "Viridis"

        # define some custom scale values for the ADC RMS page
        rms_scale_vals = [2, 20]
        relavitve_values = [0.4, 0.7]
        rms_color_scale = [
            ["0.0", "rgb(68,1,84)"],
            ["0.2", "rgb(62,74,137)"],
            ["0.3", "rgb(38,130,142)"],
            ["0.4", "rgb(53,183,121)"],
            ["0.5", "rgb(53,183,121)"],
            ["0.6", "rgb(53,183,121)"],
            ["0.7", "rgb(109,205,89)"],
            ["0.8", "rgb(180,222,44)"],
            ["1.0", "rgb(253,231,37)"],
        ]

        for pow_ind, power in enumerate(powers):
            if power.compressed().size > 0:
                vmax = np.max(power.compressed())
                vmin = np.min(power.compressed())
            else:
                vmax = 1
                vmin = 0

            colorscale = "Viridis"

            if pow_ind == 3:
                cbar_title = "RMS\tlinear"
                vmin = rms_scale_vals[0] * relavitve_values[0]
                vmax = rms_scale_vals[1] / relavitve_values[1]
                colorscale = rms_color_scale
            elif pow_ind == 4 or pow_ind == 5:
                cbar_title = "Degrees"
            elif pow_ind == len(powers) - 1:
                cbar_title = "Median\tCoeff"
            else:
                cbar_title = "dB"

            if pow_ind == 0:
                visible = True
            else:
                visible = False

            for pol_ind, pol in enumerate(pols):
                for mask_cnt, mask in enumerate(masks):
                    if mask_cnt == pow_ind:
                        mask.extend([True] * 2)
                    else:
                        mask.extend([False] * 2)

                _power = {
                    "x": xs.data[~power[pol_ind].mask].tolist(),
                    "y": ys[pol_ind].data[~power[pol_ind].mask].tolist(),
                    "text": _text[pol_ind][~power[pol_ind].mask].tolist(),
                    "mode": "markers",
                    "visible": visible,
                    "marker": {
                        "color":
                        power[pol_ind].data[~power[pol_ind].mask].tolist(),
                        "size": 14,
                        "cmin": vmin,
                        "cmax": vmax,
                        "colorscale": colorscale,
                        "colorbar": {
                            "thickness": 20,
                            "title": cbar_title
                        },
                    },
                    "hovertemplate": "%{text}<extra></extra>",
                }
                data_hex.append(_power)

                _power_offline = {
                    "x": xs.data[power[pol_ind].mask].tolist(),
                    "y": ys[pol_ind].data[power[pol_ind].mask].tolist(),
                    "text": _text[pol_ind][power[pol_ind].mask].tolist(),
                    "mode": "markers",
                    "visible": visible,
                    "marker": {
                        "color": "orange",
                        "size": 14,
                        "cmin": vmin,
                        "cmax": vmax,
                        "colorscale": colorscale,
                        "colorbar": {
                            "thickness": 20,
                            "title": cbar_title
                        },
                    },
                    "hovertemplate": "%{text}<extra></extra>",
                }
                data_hex.append(_power_offline)

        buttons = []
        for _name, mask in zip(names, masks):
            _button = {
                "args": [{
                    "visible": mask
                }, {
                    "title": "",
                    "annotations": {}
                }],
                "label": _name,
                "method": "restyle",
            }
            buttons.append(_button)

        updatemenus_hex = [{
            "buttons": buttons,
            "showactive": True,
            "type": "buttons"
        }]

        layout_hex = {
            "xaxis": {
                "title": "East-West Position [m]"
            },
            "yaxis": {
                "title": "North-South Position [m]",
                "scaleanchor": "x"
            },
            "title": {
                "text": "Per Antpol Stats vs Hex position",
                "font": {
                    "size": 24
                },
            },
            "hoverlabel": {
                "align": "left"
            },
            "margin": {
                "t": 40
            },
            "autosize": True,
            "showlegend": False,
            "hovermode": "closest",
        }
        caption = {}
        caption["title"] = "Stats vs Hex pos Help"
        caption["text"] = (
            "This plot shows various statistics and measurements "
            "per ant-pol versus its position in the array."
            "<br>Antennas which are build but not fully hooked up "
            "are shown in light red."
            "<br>Grey antennas are not yet constructed."
            "<br><br><h4>Available plotting options</h4>"
            "<ul>"
            "<li>Auto Corr - Median Auto Correlation (in db) "
            "from the correlator with equalization coefficients "
            "divided out</li>"
            "<li>Pam Power - Latest Pam Power (in db) recorded in M&C</li>"
            "<li>ADC Power - Latest ADC Power (in db) recorded in M&C</li>"
            "<li>ADC RMS - Latest linear ADC RMS recorded in M&C</li>"
            "<li>FEM IMU THETA - IMU-reported theta, in degrees</li>"
            "<li>FEM IMU PHI - IMU-reported phi, in degrees</li>"
            "<li>EQ Coeffs - Latest Median Equalization Coefficient recorded in M&C</li>"
            "</ul>"
            "Any antpol showing with an orange color means "
            "no data is avaible for the currenty plot selection."
            "<h4>Hover label Formatting</h4>"
            "<ul>"
            "<li>Antenna Name from M&C<br>(e.g. HH0n = Hera Hex Antenna 0 Polarization N)</li>"
            "<li>Snap hostname from M&C<br>(e.g. heraNode0Snap0)</li>"
            "<li>PAM Number</li>"
            "<li>Median Auto Correlation power in dB</li>"
            "<li>PAM power in dB</li>"
            "<li>ADC power in dB</li>"
            "<li>Linear ADC RMS</li>"
            "<li>FEM IMU reported theta in degrees</li>"
            "<li>FEM IMU reported phi in degrees</li>"
            "<li>Median Equalization Coefficient</li>"
            "<li>Time ago in hours the M&C Antenna Status was updated. "
            "This time stamp applies to all data for this antenna "
            "except the Auto Correlation.</li>"
            "</ul>"
            "In any hover label entry 'No Data' means "
            "information not currrently available in M&C.")

        # Render all the power vs position files
        plotname = "plotly-hex"
        html_template = env.get_template("plotly_base.html")
        js_template = env.get_template("plotly_base.js")

        if sys.version_info.minor >= 8 and sys.version_info.major > 2:
            time_jd = latest.to_value('jd', subfmt='float')
            time_unix = latest.to_value('unix')
        else:
            time_jd = latest.jd
            time_unix = latest.unix

        rendered_hex_html = html_template.render(
            plotname=plotname,
            data_type="Auto correlations",
            plotstyle="height: 100%",
            gen_date=now.iso,
            data_date_iso=latest.iso,
            data_date_jd="{:.3f}".format(time_jd),
            data_date_unix_ms=time_unix * 1000,
            js_name="hex_amp",
            gen_time_unix_ms=now.unix * 1000,
            scriptname=os.path.basename(__file__),
            hostname=computer_hostname,
            caption=caption,
        )

        rendered_hex_js = js_template.render(
            data=data_hex,
            layout=layout_hex,
            updatemenus=updatemenus_hex,
            plotname=plotname,
        )

        with open("hex_amp.html", "w") as h_file:
            h_file.write(rendered_hex_html)

        with open("hex_amp.js", "w") as js_file:
            js_file.write(rendered_hex_js)

        # now prepare the data to be plotted vs node number
        data_node = []

        masks = [[] for p in powers]

        vmax = [
            np.max(power.compressed()) if power.compressed().size > 1 else 1
            for power in powers
        ]
        vmin = [
            np.min(power.compressed()) if power.compressed().size > 1 else 0
            for power in powers
        ]
        vmin[3] = rms_scale_vals[0] * relavitve_values[0]
        vmax[3] = rms_scale_vals[1] / relavitve_values[1]

        for node in nodes:
            node_index = np.where(node_ind == node)[0]
            hosts = hostname[node_index]

            host_index = np.argsort(hosts)

            ys = np.ma.masked_array(
                [
                    np.arange(node_index.size) + 0.3 * pol_cnt
                    for pol_cnt, pol in enumerate(pols)
                ],
                mask=powers[0][:, node_index].mask,
            )
            xs = np.zeros_like(ys)
            xs[:] = node
            powers_node = [pow[:, node_index] for pow in powers]
            __text = _text[:, node_index]

            for pow_ind, power in enumerate(powers_node):
                cbar_title = "dB"
                if pow_ind == 4 or pow_ind == 5:
                    cbar_title = "Degrees"

                if pow_ind == 3:
                    colorscale = rms_color_scale
                else:
                    colorscale = "Viridis"
                colorscale = "Viridis"

                if pow_ind == 3:
                    cbar_title = "RMS\tlinear"
                    colorscale = rms_color_scale
                elif pow_ind == 4 or pow_ind == 5:
                    cbar_title = "Degrees"
                elif pow_ind == len(powers) - 1:
                    cbar_title = "Median\tCoeff"
                else:
                    cbar_title = "dB"

                if pow_ind == 0:
                    visible = True
                else:
                    visible = False

                for pol_ind, pol in enumerate(pols):
                    for mask_cnt, mask in enumerate(masks):
                        if mask_cnt == pow_ind:
                            mask.extend([True] * 2)
                        else:
                            mask.extend([False] * 2)

                    __power = power[pol_ind][host_index]
                    ___text = __text[pol_ind][host_index]

                    _power = {
                        "x": xs[pol_ind].data[~__power.mask].tolist(),
                        "y": ys[pol_ind].data[~__power.mask].tolist(),
                        "text": ___text[~__power.mask].tolist(),
                        "mode": "markers",
                        "visible": visible,
                        "marker": {
                            "color": __power.data[~__power.mask].tolist(),
                            "size": 14,
                            "cmin": vmin[pow_ind],
                            "cmax": vmax[pow_ind],
                            "colorscale": colorscale,
                            "colorbar": {
                                "thickness": 20,
                                "title": cbar_title
                            },
                        },
                        "hovertemplate": "%{text}<extra></extra>",
                    }

                    data_node.append(_power)

                    _power_offline = {
                        "x": xs[pol_ind].data[__power.mask].tolist(),
                        "y": ys[pol_ind].data[__power.mask].tolist(),
                        "text": ___text[__power.mask].tolist(),
                        "mode": "markers",
                        "visible": visible,
                        "marker": {
                            "color": "orange",
                            "size": 14,
                            "cmin": vmin[pow_ind],
                            "cmax": vmax[pow_ind],
                            "colorscale": colorscale,
                            "colorbar": {
                                "thickness": 20,
                                "title": cbar_title
                            },
                        },
                        "hovertemplate": "%{text}<extra></extra>",
                    }

                    data_node.append(_power_offline)
        buttons = []
        for _name, mask in zip(names, masks):
            _button = {
                "args": [{
                    "visible": mask
                }, {
                    "title": "",
                    "annotations": {}
                }],
                "label": _name,
                "method": "restyle",
            }
            buttons.append(_button)

        updatemenus_node = [{
            "buttons": buttons,
            "showactive": True,
            "type": "buttons"
        }]

        layout_node = {
            "xaxis": {
                "title": "Node Number",
                "dtick": 1,
                "tick0": 0,
                "showgrid": False,
                "zeroline": False,
            },
            "yaxis": {
                "showticklabels": False,
                "showgrid": False,
                "zeroline": False
            },
            "title": {
                "text": "Per Antpol Stats vs Node #",
                "font": {
                    "size": 24
                }
            },
            "hoverlabel": {
                "align": "left"
            },
            "margin": {
                "t": 40
            },
            "autosize": True,
            "showlegend": False,
            "hovermode": "closest",
        }

        caption_node = {}
        caption_node["title"] = "Stats vs Node Help"
        caption_node["text"] = (
            "This plot shows various statistics and measurements "
            "per ant-pol versus the node number to which it is connected."
            "<br><br><h4>Available plotting options</h4>"
            "<ul>"
            "<li>Auto Corr - Median Auto Correlation (in db) "
            "from the correlator with equalization coefficients "
            "divided out</li>"
            "<li>Pam Power - Latest Pam Power (in db) recorded in M&C</li>"
            "<li>ADC Power - Latest ADC Power (in db) recorded in M&C</li>"
            "<li>ADC RMS - Latest linear ADC RMS recorded in M&C</li>"
            "<li>EQ Coeffs - Latest Median Equalization Coefficient recorded in M&C</li>"
            "</ul>"
            "Any antpol showing with an orange color means "
            "no data is avaible for the currenty plot selection."
            "<h4>Hover label Formatting</h4>"
            "<ul>"
            "<li>Antenna Name from M&C<br>(e.g. HH0n = Hera Hex Antenna 0 Polarization N)</li>"
            "<li>Snap hostname from M&C<br>(e.g. heraNode0Snap0)</li>"
            "<li>PAM Number</li>"
            "<li>Median Auto Correlation power in dB</li>"
            "<li>PAM power in dB</li>"
            "<li>ADC power in dB</li>"
            "<li>Linear ADC RMS</li>"
            "<li>Median Equalization Coefficient</li>"
            "<li>Time ago in hours the M&C Antenna Status was updated. "
            "This time stamp applies to all data for this antenna "
            "except the Auto Correlation.</li>"
            "</ul>"
            "In any hover label entry 'No Data' means "
            "information not currrently available in M&C.")

        # Render all the power vs ndde files
        plotname = "plotly-node"
        html_template = env.get_template("plotly_base.html")
        js_template = env.get_template("plotly_base.js")

        if sys.version_info.minor >= 8 and sys.version_info.major > 2:
            time_jd = latest.to_value('jd', subfmt='float')
            time_unix = latest.to_value('unix')
        else:
            time_jd = latest.jd
            time_unix = latest.unix

        rendered_node_html = html_template.render(
            plotname=plotname,
            data_type="Auto correlations",
            plotstyle="height: 100%",
            gen_date=now.iso,
            gen_time_unix_ms=now.unix * 1000,
            data_date_iso=latest.iso,
            data_date_jd="{:.3f}".format(time_jd),
            data_date_unix_ms=time_unix * 1000,
            js_name="node_amp",
            scriptname=os.path.basename(__file__),
            hostname=computer_hostname,
            caption=caption_node,
        )

        rendered_node_js = js_template.render(
            data=data_node,
            layout=layout_node,
            updatemenus=updatemenus_node,
            plotname=plotname,
        )

        with open("node_amp.html", "w") as h_file:
            h_file.write(rendered_node_html)

        with open("node_amp.js", "w") as js_file:
            js_file.write(rendered_node_js)
コード例 #3
0
ファイル: mc_publish_summary.py プロジェクト: mfkiwl/hera_mc
if __name__ == '__main__':
    default_hookup_cols = [
        'station', 'feed', 'front-end', 'node-bulkhead', 'post-amp',
        'snap', 'node'
    ]
    parser = mc.get_mc_argument_parser()
    # set values for 'action' to use
    parser.add_argument('-p', '--hpn', help="Part number, csv-list or [default]",
                        default='default')
    parser.add_argument('-e', '--exact-match',
                        help="Force exact matches on part numbers, not beginning N char. [False]",
                        dest='exact_match', action='store_true')
    parser.add_argument('--hookup-cols',
                        help="Specify a subset of parts to show comma-delimited no-space list.",
                        dest='hookup_cols', default=default_hookup_cols)

    args = parser.parse_args()

    # Pre-process the args
    args.hpn = cm_utils.listify(args.hpn)
    args.hookup_cols = cm_utils.listify(args.hookup_cols)

    # Start session
    db = mc.connect_to_mc_db(args)
    session = db.sessionmaker()

    system = cm_sysutils.Handling(session)
    system.publish_summary(hlist=args.hpn, exact_match=args.exact_match,
                           hookup_cols=args.hookup_cols,
                           )
コード例 #4
0
def get_cm_info():
    from hera_mc import cm_sysutils
    h = cm_sysutils.Handling()
    return h.get_cminfo_correlator()
コード例 #5
0
def main():
    # templates are stored relative to the script dir
    # stored one level up, find the parent directory
    # and split the parent directory away
    script_dir = os.path.dirname(os.path.realpath(__file__))
    split_dir = os.path.split(script_dir)
    template_dir = os.path.join(split_dir[0], "templates")

    env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True)
    # this filter is used to see if there is more than one table
    env.filters["islist"] = is_list

    if sys.version_info[0] < 3:
        # py2
        computer_hostname = os.uname()[1]
    else:
        # py3
        computer_hostname = os.uname().nodename

    parser = mc.get_mc_argument_parser()
    parser.add_argument(
        "--redishost",
        dest="redishost",
        type=str,
        default="redishost",
        help=(
            'The host name for redis to connect to, defaults to "redishost"'),
    )
    parser.add_argument("--port",
                        dest="port",
                        type=int,
                        default=6379,
                        help="Redis port to connect.")
    args = parser.parse_args()

    try:
        db = mc.connect_to_mc_db(args)
    except RuntimeError as e:
        raise SystemExit(str(e))

    try:
        redis_db = redis.Redis(args.redishost, port=args.port)
        redis_db.keys()
    except Exception as err:
        raise SystemExit(str(err))

    with db.sessionmaker() as session:
        now = Time.now()

        hsession = cm_sysutils.Handling(session)
        stations = hsession.get_connected_stations(at_date="now")

        ants = []
        for station in stations:
            if station.antenna_number not in ants:
                ants = np.append(ants, station.antenna_number)
        ants = np.unique(ants).astype(int)

        antpos = np.genfromtxt(
            os.path.join(mc.data_path, "HERA_350.txt"),
            usecols=(0, 1, 2, 3),
            dtype={
                "names": ("ANTNAME", "EAST", "NORTH", "UP"),
                "formats": ("<U5", "<f8", "<f8", "<f8"),
            },
            encoding=None,
        )
        antnames = antpos["ANTNAME"]
        inds = [int(j[2:]) for j in antnames]
        inds = np.argsort(inds)

        antnames = np.take(antnames, inds)

        nodes = []
        hists = []
        bad_ants = []
        bad_node = []
        for ant_cnt, ant in enumerate(ants):
            ant_status = session.get_antenna_status(most_recent=True,
                                                    antenna_number=int(ant))
            mc_name = antnames[int(ant)]
            node_info = hsession.get_part_at_station_from_type(
                mc_name, "now", "node")
            if len(ant_status) == 0:
                for pol in ["e", "n"]:
                    name = "{ant}:{pol}".format(ant=ant, pol=pol)
                    print("No histogram data for ", name)
                    bad_ants.append(name)
            for stat in ant_status:
                name = "ant{ant}{pol}".format(ant=stat.antenna_number,
                                              pol=stat.antenna_feed_pol)

                # try to find the associated node
                # get the first key in the dict to index easier
                _key = list(node_info.keys())[0]
                pol_key = [
                    key for key in node_info[_key].keys()
                    if stat.antenna_feed_pol.upper() in key
                ]
                if pol_key:
                    # 'E' should be in one of the keys, extract the 0th entry
                    pol_key = pol_key[0]
                else:
                    # a hacky solution for a key that should work
                    pol_key = "E<ground"

                if node_info[_key][pol_key] is not None:
                    _node_num = re.findall(r"N(\d+)",
                                           node_info[_key][pol_key])[0]
                else:
                    print("No Node mapping for antennna: " + name)
                    _node_num = -1
                    bad_node.append("{ant}:{pol}".format(
                        ant=stat.antenna_number, pol=stat.antenna_feed_pol))
                nodes.append(_node_num)

                timestamp = Time(stat.time, format="gps")
                if (stat.histogram_bin_centers is not None
                        and stat.histogram is not None):
                    bins = np.fromstring(
                        stat.histogram_bin_centers.strip("[]"), sep=",")
                    hist = np.fromstring(stat.histogram.strip("[]"), sep=",")
                    if stat.eq_coeffs is not None:
                        eq_coeffs = np.fromstring(stat.eq_coeffs.strip("[]"),
                                                  sep=",")
                    else:
                        eq_coeffs = np.ones_like(hist)
                    hist /= np.median(eq_coeffs)**2

                    text = "observed at {iso}<br>(JD {jd})".format(
                        iso=timestamp.iso, jd=timestamp.jd)
                    # spaces cause weird wrapping issues, replace them all with \t
                    text = text.replace(" ", "\t")
                    _data = {
                        "x": bins.tolist(),
                        "y": hist.tolist(),
                        "name": name,
                        "node": _node_num,
                        "text": [text] * bins.size,
                        "hovertemplate": "(%{x:.1},\t%{y})<br>%{text}",
                    }
                    hists.append(_data)
                else:
                    name = "{ant}:{pol}".format(ant=stat.antenna_number,
                                                pol=stat.antenna_feed_pol)
                    print("No histogram data for ", name)
                    bad_ants.append(name)
        table = {}
        table["title"] = "Ants with no Histogram"
        table["rows"] = []
        row = {}
        row["text"] = ",\t".join(bad_ants)
        table["rows"].append(row)

        table_node = {}
        table_node["title"] = "Antennas with no Node mapping"
        row_node = {}
        row_node["text"] = "\t".join(bad_node)
        rows_node = [row_node]
        table_node["rows"] = rows_node

        layout = {
            "xaxis": {
                "title": "ADC value"
            },
            "yaxis": {
                "title": "Occurance",
                "type": "linear"
            },
            "title": {
                "text": "ADC Histograms",
                "xref": "paper",
                "x": 0.5,
                "yref": "paper",
                "y": 1.5,
                "font": {
                    "size": 24,
                },
            },
            "margin": {
                "l": 40,
                "b": 30,
                "r": 40,
                "t": 70
            },
            "hovermode": "closest",
            "autosize": True,
            "showlegend": True,
        }

        # Make all the buttons for this plot
        nodes = np.unique(nodes)
        # if an antenna was not mapped, roll the -1 to the end
        # this makes making buttons easier so the unmapped show last
        if -1 in nodes:
            nodes = np.roll(nodes, -1)
        # create a mask to find all the matching nodes
        node_mask = [[True if s["node"] == node else False for s in hists]
                     for node in nodes]

        buttons_node = []
        _button_node = {
            "args": [
                {
                    "visible": [True for s in hists]
                },
                {
                    "title": "",
                    "annotations": {}
                },
            ],
            "label":
            "All\tAnts",
            "method":
            "restyle",
        }
        buttons_node.append(_button_node)

        for node_cnt, node in enumerate(nodes):
            if node != -1:
                label = "Node\t{}".format(node)
            else:
                label = "Unmapped\tAnts"

            _button_node = {
                "args": [
                    {
                        "visible": node_mask[node_cnt]
                    },
                    {
                        "title": "",
                        "annotations": {}
                    },
                ],
                "label":
                label,
                "method":
                "restyle",
            }
            buttons_node.append(_button_node)

        buttons = []

        log_buttons = {
            "args": [{}, {
                "yaxis": {
                    "type": "log"
                }
            }],
            "label": "Log",
            "method": "update",
        }
        lin_buttons = {
            "args": [{}, {
                "yaxis": {
                    "type": "linear"
                }
            }],
            "label": "Linear",
            "method": "update",
        }

        buttons.append(lin_buttons)
        buttons.append(log_buttons)

        updatemenus = [
            {
                "buttons": buttons_node,
                "showactive": True,
                "active": 0,
                "type": "dropdown",
                "x": 0.535,
                "y": 1.03,
            },
            {
                "buttons": buttons,
                "showactive": True,
                "type": "buttons",
                "active": 0,
            },
        ]

        plotname = "plotly-adc-hist"

        html_template = env.get_template("ploty_with_multi_table.html")
        js_template = env.get_template("plotly_base.js")

        caption = {}

        caption["text"] = (
            "The ADC Histograms with equalization coefficients "
            "divided out."
            "<br><br>Some antennas known to M&C may not have a histogram "
            " and are listed below the image."
            "<br><br>Some antennas may not have "
            "a known node mapping and are listed below the image.\n  "
            "<br><br>Plot can be downselected to display "
            "individual nodes  or show the entire array.\n "
            "<br><br>Double click on an entry in the legend "
            "to select only that entry, "
            "double click again to restore all plots.\n  "
            "<br><br><h4>Formatting options</h4>"
            "<ul>"
            "<li>Linear - Display with Linear y-axis</li>"
            "<li>Log - Display with Log y-axis</li>"
            "</ul>"
            "<br><br>Single click an entry in the legend to un-plot it, "
            "single click again to restore it to the plot.")

        caption["title"] = "Histogram Help"

        rendered_html = html_template.render(
            plotname=plotname,
            plotstyle="height: 100%",
            gen_date=now.iso,
            js_name="adchist",
            caption=caption,
            gen_time_unix_ms=now.unix * 1000,
            scriptname=os.path.basename(__file__),
            hostname=computer_hostname,
            table=[table, table_node],
        )

        rendered_js = js_template.render(data=hists,
                                         layout=layout,
                                         plotname=plotname,
                                         updatemenus=updatemenus)
        with open("adchist.html", "w") as h_file:
            h_file.write(rendered_html)
        with open("adchist.js", "w") as js_file:
            js_file.write(rendered_js)
コード例 #6
0
def main():
    # templates are stored relative to the script dir
    # stored one level up, find the parent directory
    # and split the parent directory away
    script_dir = os.path.dirname(os.path.realpath(__file__))
    split_dir = os.path.split(script_dir)
    template_dir = os.path.join(split_dir[0], 'templates')

    env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True)
    # this filter is used to see if there is more than one table
    env.filters['islist'] = is_list

    if sys.version_info[0] < 3:
        # py2
        computer_hostname = os.uname()[1]
    else:
        # py3
        computer_hostname = os.uname().nodename

    # The standard M&C argument parser
    parser = mc.get_mc_argument_parser()
    # we'll have to add some extra options too
    parser.add_argument(
        '--redishost',
        dest='redishost',
        type=str,
        default='redishost',
        help=(
            'The host name for redis to connect to, defualts to "redishost"'))
    parser.add_argument('--port',
                        dest='port',
                        type=int,
                        default=6379,
                        help='Redis port to connect.')
    args = parser.parse_args()

    try:
        db = mc.connect_to_mc_db(args)
    except RuntimeError as e:
        raise SystemExit(str(e))

    try:
        redis_db = redis.Redis(args.redishost, port=args.port)
        redis_db.keys()
    except Exception as err:
        raise SystemExit(str(err))

    with db.sessionmaker() as session:
        corr_cm = hera_corr_cm.HeraCorrCM(redishost=args.redishost)
        hsession = cm_sysutils.Handling(session)
        stations = hsession.get_connected_stations(at_date='now')

        antpos = np.genfromtxt(os.path.join(mc.data_path, "HERA_350.txt"),
                               usecols=(0, 1, 2, 3),
                               dtype={
                                   'names': ('ANTNAME', 'EAST', 'NORTH', 'UP'),
                                   'formats': ('<U5', '<f8', '<f8', '<f8')
                               },
                               encoding=None)
        antnames = antpos['ANTNAME']
        inds = [int(j[2:]) for j in antnames]
        inds = np.argsort(inds)

        antnames = np.take(antnames, inds)

        ants = []
        for station in stations:
            if station.antenna_number not in ants:
                ants = np.append(ants, station.antenna_number)
        ants = np.unique(ants).astype(int)

        hostname_lookup = {}

        # all_snap_statuses = session.get_snap_status(most_recent=True)
        all_snap_statuses = corr_cm.get_f_status()
        snapautos = {}

        ant_status_from_snaps = corr_cm.get_ant_status()
        all_snaprf_stats = corr_cm.get_snaprf_status()

        table_snap = {}
        table_snap["title"] = "Snap hookups with No Data"
        rows = []
        bad_snaps = []

        for snap_chan in all_snaprf_stats:
            host, loc_num = snap_chan.split(":")
            loc_num = int(loc_num)

            # initialize this key if not already in the dict
            auto_group = snapautos.setdefault(host, {})

            try:
                tmp_auto = np.array(
                    all_snaprf_stats[snap_chan]["autocorrelation"])
                if np.all(tmp_auto == "None"):
                    print("No Data for {} port {}".format(host, loc_num))
                    bad_snaps.append(snap_chan)
                    tmp_auto = np.full(1024, np.nan)

                eq_coeffs = np.array(all_snaprf_stats[snap_chan]["eq_coeffs"])
                if np.all(eq_coeffs == "None"):
                    eq_coeffs = np.full_like(tmp_auto, 1.0)

                tmp_auto /= eq_coeffs**2
                tmp_auto = np.ma.masked_invalid(10 *
                                                np.log10(np.real(tmp_auto)))
                auto_group[loc_num] = tmp_auto.filled(-50)

            except KeyError:
                print("Snap connection with no autocorrelation", snap_chan)
                raise
            except TypeError:
                print("Received TypeError when taking log.")
                print("Type of item in dictionary: ",
                      type(all_snaprf_stats[snap_chan]["autocorrelation"]))
                print("Value of item: ", tmp_auto)
                raise

            hostname_lookup.setdefault(host, {})
            hostname_lookup[host].setdefault(loc_num, {})
            hostname_lookup[host][loc_num]['MC'] = 'NC'
        row = {}
        row["text"] = '\t'.join(bad_snaps)
        rows.append(row)
        table_snap["rows"] = rows

        table_ants = {}
        table_ants["title"] = "Antennas with no mapping"
        rows = []

        bad_ants = []
        bad_hosts = []

        for ant_cnt, ant in enumerate(ants):

            # get the status for both polarizations for this antenna
            ant_status = {
                key: ant_status_from_snaps[key]
                for key in ant_status_from_snaps
                if ant == int(key.split(':')[0])
            }

            # check if the antenna status from M&C has the host and
            # channel number, if it does not we have to do some gymnastics
            for antkey in ant_status:
                pol_key = antkey.split(":")[1]
                stat = ant_status[antkey]
                name = "{ant:d}:{pol}".format(ant=ant, pol=pol_key)
                # check that key is in the dictionary, is not None or the string "None"
                if ('f_host' in stat and 'host_ant_id' in stat
                        and stat['f_host'] is not None
                        and stat['host_ant_id'] is not None
                        and stat['f_host'] != "None"
                        and stat['host_ant_id'] != "None"):
                    hostname = stat['f_host']
                    loc_num = stat['host_ant_id']
                    hostname_lookup[hostname][loc_num]['MC'] = name
                else:
                    # Try to get the snap info from M&C. Output is a dictionary with 'e' and 'n' keys
                    # connect to M&C to find all the hooked up Snap hostnames and corresponding ant-pols
                    mc_name = antnames[ant]
                    # these two may not be used, but it is easier to grab them now
                    snap_info = hsession.get_part_at_station_from_type(
                        mc_name, 'now', 'snap', include_ports=True)
                    for _key in snap_info.keys():
                        # initialize a dict if they key does not exist already
                        pol_key = pol_key.upper() + "<ground"
                        if snap_info[_key][pol_key] is not None:
                            serial_with_ports = snap_info[_key][pol_key]
                            snap_serial = serial_with_ports.split(
                                '>')[1].split('<')[0]
                            ant_channel = int(
                                serial_with_ports.split('>')[0][1:]) // 2

                            hostname = session.get_snap_hostname_from_serial(
                                snap_serial)

                            if hostname is None:
                                node_info = hsession.get_part_at_station_from_type(
                                    mc_name, 'now', 'node')
                                _node_num = re.findall(
                                    r'N(\d+)', node_info[_key][pol_key])[0]

                                snap_stats = session.get_snap_status(
                                    nodeID=int(_node_num), most_recent=True)

                                for _stat in snap_stats:
                                    if _stat.serial_number == snap_serial:
                                        hostname = _stat.hostname

                            # if hostname is still None then we don't have a known hookup
                            if hostname is None:
                                print("No MC snap information for antennna: " +
                                      name)
                                bad_ants.append(name)
                            else:
                                if hostname not in hostname_lookup.keys():
                                    err = "host from M&C not found in corr_cm 'status:snaprf' : {}".format(
                                        hostname)
                                    err += '\nThis host may not have data populated yet or is offline.'
                                    err += '\nAll anteanns on this host will be full of 0.'
                                    print(err)
                                    bad_hosts.append(hostname)
                                grp1 = hostname_lookup.setdefault(hostname, {})
                                # if this loc num is not in lookup table initialize
                                # empty list
                                if ant_channel not in grp1.keys():
                                    if hostname not in bad_hosts:
                                        print(
                                            "loc_num from M&C not found in hera_corr_cm `status:snaprf` (host, location number): {}"
                                            .format([hostname, ant_channel]))
                                        print(
                                            "filling with bad array full of 0."
                                        )
                                    else:
                                        _name = "{host}:{loc}".format(
                                            host=hostname, loc=ant_channel)
                                        if _name not in table_snap["rows"][0][
                                                "text"]:
                                            table_snap["rows"][0][
                                                "text"] += _name + '\t'
                                    snap_grp1 = snapautos.setdefault(
                                        hostname, {})
                                    snap_grp1[ant_channel] = np.full(1024, 0)
                                grp2 = grp1.setdefault(ant_channel, {})
                                grp2['MC'] = name

                        else:
                            print("No MC snap information for antennna: " +
                                  name)
                            bad_ants.append(name)
        row = {}
        row["text"] = '\t'.join(bad_ants)
        rows.append(row)
        table_ants["rows"] = rows

        host_masks = []
        for h1 in sorted(hostname_lookup.keys()):
            _mask = []
            for h2 in sorted(hostname_lookup.keys()):
                _mask.extend([
                    True if h2 == h1 else False
                    for loc_num in hostname_lookup[h2]
                ])
            host_masks.append(_mask)

        # Generate frequency axis
        freqs = np.linspace(0, 250e6, 1024)
        freqs /= 1e6

        data = []
        for host_cnt, host in enumerate(sorted(hostname_lookup.keys())):
            if host_cnt == 0:
                visible = True
            else:
                visible = False

            for loc_num in sorted(hostname_lookup[host].keys()):
                mc_name = hostname_lookup[host][loc_num]['MC']

                name = '{loc}:{mcname}'.format(loc=loc_num,
                                               mcname=mc_name.replace(":", ""))
                try:
                    _data = {
                        "x": freqs.tolist(),
                        "y": snapautos[host][loc_num].tolist(),
                        "name": name,
                        "visible": visible,
                        "hovertemplate": "%{x:.1f}\tMHz<br>%{y:.3f}\t[dB]"
                    }
                except KeyError:
                    print("Given host, location pair: ({0}, {1})".format(
                        host, loc_num))
                    print("All possible keys for host {0}: {1}".format(
                        host, list(snapautos[host].keys())))
                    raise
                data.append(_data)
        buttons = []
        for host_cnt, host in enumerate(sorted(hostname_lookup.keys())):
            prog_time = all_snap_statuses[host]['last_programmed']
            timestamp = all_snap_statuses[host]['timestamp']
            temp = all_snap_statuses[host]['temp']
            uptime = all_snap_statuses[host]['uptime']
            pps = all_snap_statuses[host]['pps_count']
            if host in bad_hosts:
                label = ('{host}<br>programmed:\t{start}'
                         '<br>spectra\trecorded:\tNO DATA OBSERVED<br>'
                         'temp:\t{temp:.0f}\tC\t'
                         'pps\tcount:\t{pps}\tCycles\t\t\t'
                         'uptime:\t{uptime}'
                         ''.format(host=host,
                                   start=prog_time.isoformat(' '),
                                   temp=temp,
                                   pps=pps,
                                   uptime=uptime))
            else:
                label = ('{host}<br>programmed:\t{start}'
                         '<br>spectra\trecorded:\t{obs}<br>'
                         'temp:\t{temp:.0f}\tC\t'
                         'pps\tcount:\t{pps}\tCycles\t\t\t'
                         'uptime:\t{uptime}'
                         ''.format(host=host,
                                   start=prog_time.isoformat(' '),
                                   obs=timestamp.isoformat(' '),
                                   temp=temp,
                                   pps=pps,
                                   uptime=uptime))
            _button = {
                "args": [
                    {
                        "visible": host_masks[host_cnt]
                    },
                    {
                        "title": '',
                        # "annotations": {}
                    }
                ],
                "label":
                label,
                "method":
                "restyle"
            }
            buttons.append(_button)
        updatemenus = [{
            "buttons": buttons,
            "showactive": True,
            "type": "dropdown",
            "x": .7,
            "y": 1.1,
        }]
        layout = {
            "xaxis": {
                "title": "Frequency [MHz]",
                "showticklabels": True,
                "tick0": 0,
                "dtick": 10,
            },
            "yaxis": {
                "title": 'Power [dB]',
                "showticklabels": True,
            },
            "hoverlabel": {
                "align": "left"
            },
            "margin": {
                "l": 40,
                "b": 30,
                "r": 40,
                "t": 30
            },
            "autosize":
            True,
            "showlegend":
            True,
            "hovermode":
            'closest',
            "annotations": [{
                "x": 1.03,
                "y": 1.03,
                "align": "right",
                "valign": "top",
                "text": 'ADC Port # : Antpol',
                "showarrow": False,
                "xref": "paper",
                "yref": "paper",
                "xanchor": "center",
                "yanchor": "top"
            }]
        }

        caption = {}
        caption["title"] = "Snap Spectra Help"
        caption["text"] = (
            "Autocorrelations (in dB) calculated by each Snap "
            "with equalization coefficients divided out. "
            "Calculation is independent of the corrlator. "
            "<br><br>"
            "Data is organized by the ADC Port number from "
            "each Snap. "
            "A mapping dictionary is used to look up the "
            "associated ant-pol for each ADC Port. "
            "Some ADC Ports may not have a known ant-pol "
            "and are labeled as N/C."
            "<br>Some antennas known to M&C may not have a known ADC port"
            " mapping and are labeled below the plot."
            "<br><br>Some Snap ports my not have data and are "
            "labeled in a table below the plot."
            "<br><br>The node selection button contains a lot "
            "of information described here."
            "<br><h4>Button Label Information</h4>"
            "<ul>"
            "<li>Snap hostname</li>"
            "<li>Last time the Snap was programmed</li>"
            "<li>Last time data was recorded from this Snap</li>"
            "<li>Last known temperature in C | PPS count | Uptime</li>"
            "</ul>")

        plotname = "plotly-snap"
        html_template = env.get_template("refresh_with_table.html")
        js_template = env.get_template("plotly_base.js")

        rendered_html = html_template.render(
            plotname=plotname,
            plotstyle="height: 100%",
            gen_date=Time.now().iso,
            js_name='snapspectra',
            gen_time_unix_ms=Time.now().unix * 1000,
            scriptname=os.path.basename(__file__),
            hostname=computer_hostname,
            table=[table_snap, table_ants],
            caption=caption)
        rendered_js = js_template.render(data=data,
                                         layout=layout,
                                         updatemenus=updatemenus,
                                         plotname=plotname)

        with open('snapspectra.html', 'w') as h_file:
            h_file.write(rendered_html)

        with open('snapspectra.js', 'w') as js_file:
            js_file.write(rendered_js)