def test_yday_format(self): """Year:Day_of_year format""" # Heterogeneous input formats with in_subfmt='*' (default) times = ["2000-12-01", "2001-12-01 01:01:01.123"] t = Time(times, format="iso", scale="tai") t.out_subfmt = "date_hm" assert np.all(t.yday == np.array(["2000:336:00:00", "2001:335:01:01"])) t.out_subfmt = "*" assert np.all(t.yday == np.array(["2000:336:00:00:00.000", "2001:335:01:01:01.123"]))
def compile_datasets(target, old=0, returnColors=True): lcvs = glob.glob(target + 'lcvs/optical/*.lcv') # all_datasets = np.zeros(1, dtype='S30') # all_jds = np.zeros(1, dtype=float) num_images = 0 for lcv in lcvs: U, B, V, R, I = lightcurves.read_optical_lcv(lcv, old=old) if lcv == lcvs[0]: all_datasets = np.array(U[3], dtype='S35') all_jds = np.array(U[2], dtype=float) num_images += len(U[0]) + len(B[0]) + len(V[0]) + len(R[0]) + len( I[0]) else: all_datasets = np.append(all_datasets, U[3]) all_jds = np.append(all_jds, U[2]) num_images += len(U[0]) + len(B[0]) + len(V[0]) + len(R[0]) + len( I[0]) all_datasets = np.append(all_datasets, B[3]) all_datasets = np.append(all_datasets, V[3]) all_datasets = np.append(all_datasets, R[3]) all_datasets = np.append(all_datasets, I[3]) all_jds = np.append(all_jds, B[2]) all_jds = np.append(all_jds, V[2]) all_jds = np.append(all_jds, R[2]) all_jds = np.append(all_jds, I[2]) all_jds = all_jds + 2400000.5 datasets_prefix = np.zeros(len(all_datasets), dtype='S30') print 'Total number of images: {} '.format(num_images) for ind, string in enumerate(all_datasets): datasets_prefix[ind] = string.split(':')[0] unique, counts = np.unique(datasets_prefix, return_counts=True) dataset_names = unique[np.argsort(counts)[::-1]] dataset_counts = counts[np.argsort(counts)[::-1]] colors = np.zeros(len(dataset_names), dtype='S25') print '\n\nDatasets:\n' for ind, dataset in enumerate(dataset_names): jds = all_jds[datasets_prefix == dataset] t_min = Time(jds.min(), format='jd') t_max = Time(jds.max(), format='jd') t_min.out_subfmt = 'date' t_max.out_subfmt = 'date' colors[ind] = plotting_utilities.get_color(ind) print '%3i %10s %6i %s %s %s' % (ind + 1, dataset, dataset_counts[ind], t_min.iso, t_max.iso, plotting_utilities.get_color(ind)) if returnColors == True: return dataset_names, colors if returnColors == False: return dataset_names
def write_fits(img, metadata, fitsobj): imgtime = Time(metadata[0][0] + config.inttime*0.5, scale='utc', format='unix', location=(LOFAR_CS002_LONG, LOFAR_CS002_LAT)) imgtime.format='isot' imgtime.out_subfmt = 'date_hms' filename = '%s_S%0.1f_I%ix%i_W%i_A%0.1f.fits' % (imgtime.datetime.strftime("%Y%m%d%H%M%SUTC"), np.mean(subbands), len(subbands), config.inttime, config.window, config.alpha) filename = os.path.join(config.output, filename) if os.path.exists(filename): logger.info("'%s' exists - skipping", filename) return # CRVAL1 should hold RA in degrees. sidereal_time returns hour angle in # hours. fitsobj.header['CRVAL1'] = imgtime.sidereal_time(kind='apparent').value * 15 fitsobj.header['DATE-OBS'] = str(imgtime) imgtime_end = imgtime + TimeDelta(config.inttime, format='sec') fitsobj.header['END_UTC'] = str(imgtime_end) t = Time.now(); t.format = 'isot' fitsobj.header['DATE'] = str(t) fitsobj.data[0, 0, :, :] = img data = img[np.logical_not(np.isnan(img))] quality = rms.rms(rms.clip(data)) high = data.max() low = data.min() fitsobj.header['DATAMAX'] = high fitsobj.header['DATAMIN'] = low fitsobj.header['HISTORY'][0] = 'AARTFAAC 6 stations superterp' fitsobj.header['HISTORY'][1] = 'RMS {}'.format(quality) fitsobj.header['HISTORY'][2] = 'DYNAMIC RANGE {}:{}'.format(int(round(high)), int(round(quality))) fitsobj.writeto(filename) logger.info("%s %0.3f %i:%i", filename, quality, int(round(high)), int(round(quality)))
def get_datetime(filename): # Reading binary data with open(filename, "rb") as f: bin_data = f.read() # Searching datetime start byte date_key = r'\xe0.\x00\x00\x0b' matches = [] for match in finditer(date_key, bin_data): matches.append(match.span()) date_offset = matches[-1][0] - 8 # Unpacking datetime time_values = {} hours, days = unpack_from("<II", bin_data, date_offset) secs = days * 45.0 * 60.0 + hours / (4294967295.0 / 45.0) * 60.0 time_values["UNIX"] = secs - 2938117104000.0 # Convert to ISO and MJD time formats t = Time(time_values["UNIX"], format="unix") t.format = "iso" t.out_subfmt = "date_hm" time_values["ISO"] = t.value t.format = "mjd" time_values["MJD"] = str(int(np.floor(t.value))) return time_values
def extract_1708_08971(self): """Extract SLSN entries for arxiv paper 1708.08971 (page 11)""" path = "source_lists/tabula-1708.08971 (dragged).csv" arxiv = "1708.08971 (p.11)" with open(path, 'rb') as csvfile: reader = csv.reader(csvfile, delimiter=',', quotechar='|') for i, row in enumerate(reader): if i < 1: pass else: new = SLSN() new.name = row[0] new.redshift = float(row[5]) new.ref = row[8] new.arxiv = arxiv date_type = row[6][-1] mjd = Time(float(row[7]), format="mjd") mjd.out_subfmt = "date" ra = row[1] dec = row[2] new.add_coordinates(ra, dec) if date_type == "p": new.peak_date = mjd elif date_type == "d": new.disc_date = mjd else: raise Exception("Unknown date type!") self.entries.append(new)
def _get_time(): t = Time([[1], [2]], format='cxcsec', location=EarthLocation(1000, 2000, 3000, unit=u.km)) t.format = 'iso' t.precision = 5 t.delta_ut1_utc = np.array([[3.0], [4.0]]) t.delta_tdb_tt = np.array([[5.0], [6.0]]) t.out_subfmt = 'date_hm' return t
def save_obs_uvfits(obs, fname, force_singlepol=None, polrep_out='circ'): """Save observation data to uvfits. To save Stokes I as a single polarization (e.g., only RR) set force_singlepol='R' or 'L' """ # output times must be in utc obs = obs.switch_timetype(timetype_out='UTC') if polrep_out == 'circ': obs = obs.switch_polrep('circ') elif polrep_out == 'stokes': obs = obs.switch_polrep('stokes') else: raise Exception( "'polrep_out' in 'save_obs_uvfits' must be 'circ' or 'stokes'!") hdulist_new = fits.HDUList() hdulist_new.append(fits.GroupsHDU()) ##################### # AIPS Data TABLE ##################### # Data header (based on the BU format) MJD_0 = 2400000.5 header = hdulist_new['PRIMARY'].header header['OBSRA'] = obs.ra * 180. / 12. header['OBSDEC'] = obs.dec header['OBJECT'] = obs.source header['MJD'] = float(obs.mjd) header['DATE-OBS'] = Time(obs.mjd + MJD_0, format='jd', scale='utc', out_subfmt='date').iso header['BSCALE'] = 1.0 header['BZERO'] = 0.0 header['BUNIT'] = 'JY' header['VELREF'] = 3 # TODO ?? header['EQUINOX'] = 'J2000' header['ALTRPIX'] = 1.e0 header['ALTRVAL'] = 0.e0 header['TELESCOP'] = 'VLBA' # TODO Can we change this field? header['INSTRUME'] = 'VLBA' header['OBSERVER'] = 'EHT' header['CTYPE2'] = 'COMPLEX' header['CRVAL2'] = 1.e0 header['CDELT2'] = 1.e0 header['CRPIX2'] = 1.e0 header['CROTA2'] = 0.e0 header['CTYPE3'] = 'STOKES' if polrep_out == 'circ': header['CRVAL3'] = -1.e0 header['CDELT3'] = -1.e0 elif polrep_out == 'stokes': header['CRVAL3'] = 1.e0 header['CDELT3'] = 1.e0 header['CRPIX3'] = 1.e0 header['CROTA3'] = 0.e0 header['CTYPE4'] = 'FREQ' header['CRVAL4'] = obs.rf header['CDELT4'] = obs.bw header['CRPIX4'] = 1.e0 header['CROTA4'] = 0.e0 header['CTYPE6'] = 'RA' header['CRVAL6'] = header['OBSRA'] header['CDELT6'] = 1.e0 header['CRPIX6'] = 1.e0 header['CROTA6'] = 0.e0 header['CTYPE7'] = 'DEC' header['CRVAL7'] = header['OBSDEC'] header['CDELT7'] = 1.e0 header['CRPIX7'] = 1.e0 header['CROTA7'] = 0.e0 header['PTYPE1'] = 'UU---SIN' header['PSCAL1'] = 1.0 / obs.rf header['PZERO1'] = 0.e0 header['PTYPE2'] = 'VV---SIN' header['PSCAL2'] = 1.0 / obs.rf header['PZERO2'] = 0.e0 header['PTYPE3'] = 'WW---SIN' header['PSCAL3'] = 1.0 / obs.rf header['PZERO3'] = 0.e0 header['PTYPE4'] = 'BASELINE' header['PSCAL4'] = 1.e0 header['PZERO4'] = 0.e0 header['PTYPE5'] = 'DATE' header['PSCAL5'] = 1.e0 header['PZERO5'] = 0.e0 header['PTYPE6'] = 'DATE' header['PSCAL6'] = 1.e0 header['PZERO6'] = 0.0 header['PTYPE7'] = 'INTTIM' header['PSCAL7'] = 1.e0 header['PZERO7'] = 0.e0 header['PTYPE8'] = 'TAU1' header['PSCAL8'] = 1.e0 header['PZERO8'] = 0.e0 header['PTYPE9'] = 'TAU2' header['PSCAL9'] = 1.e0 header['PZERO9'] = 0.e0 header['history'] = "AIPS SORT ORDER='TB'" # Get data if polrep_out == 'circ': obsdata = obs.unpack([ 'time', 'tint', 'u', 'v', 'rrvis', 'llvis', 'rlvis', 'lrvis', 'rrsigma', 'llsigma', 'rlsigma', 'lrsigma', 't1', 't2', 'tau1', 'tau2' ]) elif polrep_out == 'stokes': obsdata = obs.unpack([ 'time', 'tint', 'u', 'v', 'vis', 'qvis', 'uvis', 'vvis', 'sigma', 'qsigma', 'usigma', 'vsigma', 't1', 't2', 'tau1', 'tau2' ]) ndat = len(obsdata['time']) # times and tints jds = (2400000.5 + obs.mjd) * np.ones(len(obsdata)) fractimes = (obsdata['time'] / 24.0) tints = obsdata['tint'] # Baselines t1 = [obs.tkey[scope] + 1 for scope in obsdata['t1']] t2 = [obs.tkey[scope] + 1 for scope in obsdata['t2']] bl = 256 * np.array(t1) + np.array(t2) # opacities tau1 = obsdata['tau1'] tau2 = obsdata['tau2'] # uv are in lightseconds u = obsdata['u'] v = obsdata['v'] # rr, ll, lr, rl, weights if polrep_out == 'circ': rr = obsdata['rrvis'] ll = obsdata['llvis'] rl = obsdata['rlvis'] lr = obsdata['lrvis'] weightrr = 1.0 / (obsdata['rrsigma']**2) weightll = 1.0 / (obsdata['llsigma']**2) weightrl = 1.0 / (obsdata['rlsigma']**2) weightlr = 1.0 / (obsdata['lrsigma']**2) # If necessary, enforce single polarization if force_singlepol == 'L': if obs.polrep == 'stokes': raise Exception( "force_singlepol only works with obs.polrep=='stokes'!") print( "force_singlepol='L': treating Stokes 'I' as LL and ignoring Q,U,V!!" ) ll = obsdata['vis'] rr = rr * 0.0 rl = rl * 0.0 lr = lr * 0.0 weightrr = weightrr * 0.0 weightrl = weightrl * 0.0 weightlr = weightlr * 0.0 elif force_singlepol == 'R': if obs.polrep == 'stokes': raise Exception( "force_singlepol only works with obs.polrep=='stokes'!") print( "force_singlepol='R': treating Stokes 'I' as RR and ignoring Q,U,V!!" ) rr = obsdata['vis'] ll = rr * 0.0 rl = rl * 0.0 lr = lr * 0.0 weightll = weightll * 0.0 weightrl = weightrl * 0.0 weightlr = weightlr * 0.0 dat1 = rr dat2 = ll dat3 = rl dat4 = lr weight1 = weightrr weight2 = weightll weight3 = weightrl weight4 = weightlr elif polrep_out == 'stokes': dat1 = obsdata['vis'] dat2 = obsdata['qvis'] dat3 = obsdata['uvis'] dat4 = obsdata['vvis'] weight1 = 1.0 / (obsdata['sigma']**2) weight2 = 1.0 / (obsdata['qsigma']**2) weight3 = 1.0 / (obsdata['usigma']**2) weight4 = 1.0 / (obsdata['vsigma']**2) # Replace nans by zeros (including zero weights) dat1 = np.nan_to_num(dat1) dat2 = np.nan_to_num(dat2) dat3 = np.nan_to_num(dat3) dat4 = np.nan_to_num(dat4) weight1 = np.nan_to_num(weight1) weight2 = np.nan_to_num(weight2) weight3 = np.nan_to_num(weight3) weight4 = np.nan_to_num(weight4) # Data array outdat = np.zeros((ndat, 1, 1, 1, 1, 4, 3)) outdat[:, 0, 0, 0, 0, 0, 0] = np.real(dat1) outdat[:, 0, 0, 0, 0, 0, 1] = np.imag(dat1) outdat[:, 0, 0, 0, 0, 0, 2] = weight1 outdat[:, 0, 0, 0, 0, 1, 0] = np.real(dat2) outdat[:, 0, 0, 0, 0, 1, 1] = np.imag(dat2) outdat[:, 0, 0, 0, 0, 1, 2] = weight2 outdat[:, 0, 0, 0, 0, 2, 0] = np.real(dat3) outdat[:, 0, 0, 0, 0, 2, 1] = np.imag(dat3) outdat[:, 0, 0, 0, 0, 2, 2] = weight3 outdat[:, 0, 0, 0, 0, 3, 0] = np.real(dat4) outdat[:, 0, 0, 0, 0, 3, 1] = np.imag(dat4) outdat[:, 0, 0, 0, 0, 3, 2] = weight4 # Save data pars = [ 'UU---SIN', 'VV---SIN', 'WW---SIN', 'BASELINE', 'DATE', 'DATE', 'INTTIM', 'TAU1', 'TAU2' ] x = fits.GroupData( outdat, parnames=pars, pardata=[u, v, np.zeros(ndat), bl, jds, fractimes, tints, tau1, tau2], bitpix=-32) hdulist_new['PRIMARY'].data = x hdulist_new['PRIMARY'].header = header # TODO necessary ?? ##################### # AIPS AN TABLE ##################### # Load the array data tarr = obs.tarr tnames = tarr['site'] tnums = np.arange(1, len(tarr) + 1) xyz = np.array([[tarr[i]['x'], tarr[i]['y'], tarr[i]['z']] for i in np.arange(len(tarr))]) sefd = tarr['sefdr'] nsta = len(tnames) col1 = fits.Column(name='ANNAME', format='8A', array=tnames) col2 = fits.Column(name='STABXYZ', format='3D', unit='METERS', array=xyz) col3 = fits.Column(name='NOSTA', format='1J', array=tnums) colfin = fits.Column(name='SEFD', format='1D', array=sefd) # TODO these antenna fields+header are questionable - look into them col4 = fits.Column(name='MNTSTA', format='1J', array=np.zeros(nsta)) col5 = fits.Column(name='STAXOF', format='1E', unit='METERS', array=np.zeros(nsta)) col6 = fits.Column(name='POLTYA', format='1A', array=np.array(['R' for i in range(nsta)], dtype='|S1')) col7 = fits.Column(name='POLAA', format='1E', unit='DEGREES', array=np.zeros(nsta)) col8 = fits.Column(name='POLCALA', format='3E', array=np.zeros((nsta, 3))) col9 = fits.Column(name='POLTYB', format='1A', array=np.array(['L' for i in range(nsta)], dtype='|S1')) col10 = fits.Column(name='POLAB', format='1E', unit='DEGREES', array=(90. * np.ones(nsta))) col11 = fits.Column(name='POLCALB', format='3E', array=np.zeros((nsta, 3))) col25 = fits.Column(name='ORBPARM', format='1E', array=np.zeros(0)) # Antenna Header params # TODO do we need to change more of these?? collist = [ col1, col2, col25, col3, col4, col5, col6, col7, col8, col9, col10, col11, colfin ] tbhdu = fits.BinTableHDU.from_columns(fits.ColDefs(collist), name='AIPS AN') hdulist_new.append(tbhdu) head = hdulist_new['AIPS AN'].header head['EXTVER'] = 1 head['ARRAYX'] = 0.e0 head['ARRAYY'] = 0.e0 head['ARRAYZ'] = 0.e0 # TODO change the reference date rdate_tt_new = Time(obs.mjd + MJD_0, format='jd', scale='utc', out_subfmt='date') rdate_out = rdate_tt_new.iso rdate_tt_new.out_subfmt = 'float' # TODO -- needed to fix subformat issue in astropy 4.0 rdate_jd_out = rdate_tt_new.jd rdate_gstiao_out = rdate_tt_new.sidereal_time('apparent', 'greenwich').degree rdate_offset_out = (rdate_tt_new.ut1.datetime.second - rdate_tt_new.utc.datetime.second) rdate_offset_out += 1.e-6 * (rdate_tt_new.ut1.datetime.microsecond - rdate_tt_new.utc.datetime.microsecond) head['RDATE'] = rdate_out head['GSTIA0'] = rdate_gstiao_out head['DEGPDY'] = 360.9856 head['UT1UTC'] = rdate_offset_out # difference between UT1 and UTC ? head['DATUTC'] = 0.e0 head['TIMESYS'] = 'UTC' head['FREQ'] = obs.rf head['POLARX'] = 0.e0 head['POLARY'] = 0.e0 head['ARRNAM'] = 'VLBA' # TODO must be recognized by aips/casa head['XYZHAND'] = 'RIGHT' head['FRAME'] = '????' head['NUMORB'] = 0 head['NO_IF'] = 1 # TODO nchan head['NOPCAL'] = 0 # TODO add pol cal information head['POLTYPE'] = 'VLBI' head['FREQID'] = 1 hdulist_new['AIPS AN'].header = head # TODO necessary, or is it a pointer? ##################### # AIPS FQ TABLE ##################### # Convert types & columns nif = 1 col1 = np.array(1, dtype=np.int32).reshape([nif]) # frqsel col2 = np.array(0.0, dtype=np.float64).reshape([nif]) # iffreq col3 = np.array([obs.bw], dtype=np.float32).reshape([nif]) # chwidth col4 = np.array([obs.bw], dtype=np.float32).reshape([nif]) # bw col5 = np.array([1], dtype=np.int32).reshape([nif]) # sideband col1 = fits.Column(name="FRQSEL", format="1J", array=col1) col2 = fits.Column(name="IF FREQ", format="%dD" % (nif), array=col2) col3 = fits.Column(name="CH WIDTH", format="%dE" % (nif), array=col3) col4 = fits.Column(name="TOTAL BANDWIDTH", format="%dE" % (nif), array=col4) col5 = fits.Column(name="SIDEBAND", format="%dJ" % (nif), array=col5) cols = fits.ColDefs([col1, col2, col3, col4, col5]) # create table tbhdu = fits.BinTableHDU.from_columns(cols) # add header information tbhdu.header.append(("NO_IF", nif, "Number IFs")) tbhdu.header.append(("EXTNAME", "AIPS FQ")) tbhdu.header.append(("EXTVER", 1)) hdulist_new.append(tbhdu) ##################### # AIPS NX TABLE ##################### scan_times = [] scan_time_ints = [] start_vis = [] stop_vis = [] # TODO make sure jds AND scan_info MUST be time sorted!! jj = 0 ROUND_SCAN_INT = 5 comp_fac = 3600 * 24 * 100 # compare to 100th of a second scan_arr = obs.scans print('Building NX table') if (scan_arr is None or len(scan_arr) == 0): print("No NX table in saved uvfits") else: try: scan_arr = scan_arr / 24. for scan in scan_arr: scan_start = round(scan[0], ROUND_SCAN_INT) scan_stop = round(scan[1], ROUND_SCAN_INT) scan_dur = (scan_stop - scan_start) if jj >= len(fractimes): # print start_vis, stop_vis break # print ("%.12f %.12f %.12f" % (fractimes[jj], scan_start, scan_stop)) jd = round(fractimes[jj], ROUND_SCAN_INT) * comp_fac # TODO precision?? if ((np.floor(jd) >= np.floor(scan_start * comp_fac)) and (np.ceil(jd) <= np.ceil(comp_fac * scan_stop))): start_vis.append(jj) # TODO AIPS MEMO 117 says scan_times should be midpoint! # but AIPS data looks likes it's at the start? scan_times.append(scan_start + 0.5 * scan_dur) # - rdate_jd_out) scan_time_ints.append(scan_dur) ceilcut = np.ceil(comp_fac * scan_stop) while ((jj < len(fractimes) and np.floor( round(fractimes[jj], ROUND_SCAN_INT) * comp_fac) <= ceilcut)): jj += 1 stop_vis.append(jj - 1) else: continue if jj < len(fractimes): print(scan_arr[-1]) print(round(scan_arr[-1][0], ROUND_SCAN_INT), round(scan_arr[-1][1], ROUND_SCAN_INT)) print(jj, len(jds), round(jds[jj], ROUND_SCAN_INT)) print( "WARNING!!!: in save_uvfits NX table, " + "didn't get to all entries when computing scan start/stop!" ) print(scan_times) time_nx = fits.Column(name="TIME", format="1D", unit='DAYS', array=np.array(scan_times)) timeint_nx = fits.Column(name="TIME INTERVAL", format="1E", unit='DAYS', array=np.array(scan_time_ints)) sourceid_nx = fits.Column(name="SOURCE ID", format="1J", unit='', array=np.ones(len(scan_times))) subarr_nx = fits.Column(name="SUBARRAY", format="1J", unit='', array=np.ones(len(scan_times))) freqid_nx = fits.Column(name="FREQ ID", format="1J", unit='', array=np.ones(len(scan_times))) startvis_nx = fits.Column(name="START VIS", format="1J", unit='', array=np.array(start_vis) + 1) endvis_nx = fits.Column(name="END VIS", format="1J", unit='', array=np.array(stop_vis) + 1) cols = fits.ColDefs([ time_nx, timeint_nx, sourceid_nx, subarr_nx, freqid_nx, startvis_nx, endvis_nx ]) tbhdu = fits.BinTableHDU.from_columns(cols) # header information tbhdu.header.append(("EXTNAME", "AIPS NX")) tbhdu.header.append(("EXTVER", 1)) hdulist_new.append(tbhdu) except TypeError: print("No NX table in saved uvfits") # Write final HDUList to file hdulist_new.writeto(fname, overwrite=True) return
def hjd2datetime(hjd): t = Time(hjd, format='jd') t.format = 'iso' t.out_subfmt = 'date_hms' return t.iso
def main(): # templates are stored relative to the script dir # stored one level up, find the parent directory # and split the parent directory away script_dir = os.path.dirname(os.path.realpath(__file__)) split_dir = os.path.split(script_dir) template_dir = os.path.join(split_dir[0], "templates") env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True) env.filters["islist"] = is_list if sys.version_info[0] < 3: # py2 computer_hostname = os.uname()[1] else: # py3 computer_hostname = os.uname().nodename parser = argparse.ArgumentParser( description=("Create auto-correlation spectra plot for heranow dashboard") ) parser.add_argument( "--redishost", dest="redishost", type=str, default="redishost", help=('The host name for redis to connect to, defaults to "redishost"'), ) parser.add_argument( "--port", dest="port", type=int, default=6379, help="Redis port to connect." ) args = parser.parse_args() r = redis.Redis(args.redishost, port=args.port) keys = [ k.decode() for k in r.keys() if k.startswith(b"auto") and not k.endswith(b"timestamp") ] ants = [] for key in keys: match = re.search(r"auto:(?P<ant>\d+)(?P<pol>e|n)", key) if match is not None: ant, pol = int(match.group("ant")), match.group("pol") ants.append(ant) ants = np.unique(ants) corr_map = r.hgetall(b"corr:map") ant_to_snap = json.loads(corr_map[b"ant_to_snap"]) node_map = {} nodes = [] # want to be smart against the length of the autos, they sometimes change # depending on the mode of the array for i in ants: for pol in ["e", "n"]: d = r.get("auto:{ant:d}{pol:s}".format(ant=i, pol=pol)) if d is not None: auto = np.frombuffer(d, dtype=np.float32).copy() break auto_size = auto.size # Generate frequency axis # Some times we have 6144 length inputs, others 1536, this should # set the length to match whatever the auto we got was NCHANS = int(8192 // 4 * 3) NCHANS_F = 8192 NCHAN_SUM = NCHANS // auto_size NCHANS = auto_size frange = np.linspace(0, 250e6, NCHANS_F + 1)[1536 : 1536 + (8192 // 4 * 3)] # average over channels frange = frange.reshape(NCHANS, NCHAN_SUM).sum(axis=1) / NCHAN_SUM frange_mhz = frange / 1e6 got_time = False n_signals = 0 try: t_plot_jd = np.frombuffer(r["auto:timestamp"], dtype=np.float64)[0] t_plot = Time(t_plot_jd, format="jd") t_plot.out_subfmt = u"date_hm" got_time = True except: pass # grab data from redis and format it according to plotly's javascript api autospectra = [] table_ants = {} table_ants["title"] = "Antennas with no Node mapping" rows = [] bad_ants = [] for i in ants: for pol in ["e", "n"]: # get the timestamp from redis for the first ant-pol if not got_time: t_plot_jd = float( r.hget( "visdata://{i:d}/{j:d}/{i_pol:s}{j_pol:s}".format( i=i, j=i, i_pol=pol, j_pol=pol ), "time", ) ) if t_plot_jd is not None: got_time = True linename = "ant{ant:d}{pol:s}".format(ant=i, pol=pol) try: hostname = ant_to_snap[str(i)][pol]["host"] match = re.search(r"heraNode(?P<node>\d+)Snap", hostname) if match is not None: _node = int(match.group("node")) nodes.append(_node) node_map[linename] = _node else: print("No Node mapping for antennna: " + linename) bad_ants.append(linename) node_map[linename] = -1 nodes.append(-1) except (KeyError): print("No Node mapping for antennna: " + linename) bad_ants.append(linename) node_map[linename] = -1 nodes.append(-1) d = r.get("auto:{ant:d}{pol:s}".format(ant=i, pol=pol)) if d is not None: n_signals += 1 auto = np.frombuffer(d, dtype=np.float32)[0:NCHANS].copy() eq_coeffs = r.hget( bytes("eq:ant:{ant}:{pol}".format(ant=i, pol=pol).encode()), "values", ) if eq_coeffs is not None: eq_coeffs = np.fromstring( eq_coeffs.decode("utf-8").strip("[]"), sep="," ) if eq_coeffs.size == 0: eq_coeffs = np.ones_like(auto) else: eq_coeffs = np.ones_like(auto) # divide out the equalization coefficients # eq_coeffs are stored as a length 1024 array but only a # single number is used. Taking the median to not deal with # a size mismatch eq_coeffs = np.median(eq_coeffs) auto /= eq_coeffs ** 2 auto[auto < 10 ** -10.0] = 10 ** -10.0 auto = 10 * np.log10(auto) _auto = { "x": frange_mhz.tolist(), "y": auto.tolist(), "name": linename, "node": node_map[linename], "type": "scatter", "hovertemplate": "%{x:.1f}\tMHz<br>%{y:.3f}\t[dB]", } autospectra.append(_auto) row = {} row["text"] = "\t".join(bad_ants) rows.append(row) table_ants["rows"] = rows nodes = np.unique(nodes) # if an antenna was not mapped, roll the -1 to the end # this makes making buttons easier so the unmapped show last if -1 in nodes: nodes = np.roll(nodes, -1) # create a mask to find all the matching nodes node_mask = [ [True if s["node"] == node else False for s in autospectra] for node in nodes ] buttons = [] _button = { "args": [ {"visible": [True for s in autospectra]}, { "title": "", # "annotations": {} }, ], "label": "All\tAnts", "method": "restyle", } buttons.append(_button) for node_cnt, node in enumerate(nodes): if node != -1: label = "Node\t{}".format(node) else: label = "Unmapped\tAnts" _button = { "args": [ {"visible": node_mask[node_cnt]}, { "title": "", # "annotations": {} }, ], "label": label, "method": "update", } buttons.append(_button) updatemenus = [ { "buttons": buttons, "showactive": True, "active": 0, "type": "dropdown", "x": 0.535, "y": 1.03, } ] layout = { "xaxis": {"title": "Frequency [MHz]"}, "yaxis": {"title": "Power [dB]"}, "title": { "text": "Autocorrelations", "xref": "paper", "x": 0.5, "yref": "paper", "y": 1.5, "font": {"size": 24}, }, "autosize": True, "showlegend": True, "legend": {"x": 1, "y": 1}, "margin": {"l": 40, "b": 30, "r": 40, "t": 46}, "hovermode": "closest", } plotname = "plotly-autos" caption = {} caption["text"] = ( "The Autocorrelations from the correlator (in dB) versus frequency " "with equalization coefficients divided out.\n " "<br><br>Some antennas may not have " "a known node mapping and are listed below the image.\n " "<br><br>Plot can be downselected to display individual nodes " "or show the entire array.\n " "<br><br>Double click on an entry in the legend to select only that " "entry, double click again to restore all plots.\n " "<br><br>Single click an entry in the legend to un-plot it, " "single click again to restore it to the plot." ) caption["title"] = "Autocorrelations Help" html_template = env.get_template("refresh_with_table.html") js_template = env.get_template("plotly_base.js") if sys.version_info.minor >= 8 and sys.version_info.major > 2: time_jd = t_plot.to_value('jd', subfmt='float') time_unix = t_plot.to_value('unix') else: time_jd = t_plot.jd time_unix = t_plot.unix rendered_html = html_template.render( plotname=plotname, data_type="Auto correlations", plotstyle="height: 100%", div_height="height: 73%", gen_date=Time.now().iso, data_date_iso=t_plot.iso, data_date_jd="{:.3f}".format(time_jd), data_date_unix_ms=time_unix * 1000, js_name="spectra", gen_time_unix_ms=Time.now().unix * 1000, scriptname=os.path.basename(__file__), hostname=computer_hostname, table=table_ants, caption=caption, ) rendered_js = js_template.render( data=autospectra, layout=layout, updatemenus=updatemenus, plotname=plotname ) print("Got {n_sig:d} signals".format(n_sig=n_signals)) with open("spectra.html", "w") as h_file: h_file.write(rendered_html) with open("spectra.js", "w") as js_file: js_file.write(rendered_js)
def main(): # templates are stored relative to the script dir # stored one level up, find the parent directory # and split the parent directory away script_dir = os.path.dirname(os.path.realpath(__file__)) split_dir = os.path.split(script_dir) template_dir = os.path.join(split_dir[0], "templates") env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True) if sys.version_info[0] < 3: # py2 computer_hostname = os.uname()[1] else: # py3 computer_hostname = os.uname().nodename # The standard M&C argument parser parser = mc.get_mc_argument_parser() # we'll have to add some extra options too parser.add_argument( "--redishost", dest="redishost", type=str, default="redishost", help=( 'The host name for redis to connect to, defualts to "redishost"'), ) parser.add_argument("--port", dest="port", type=int, default=6379, help="Redis port to connect.") args = parser.parse_args() try: db = mc.connect_to_mc_db(args) except RuntimeError as e: raise SystemExit(str(e)) try: redis_db = redis.Redis(args.redishost, port=args.port) redis_db.keys() except Exception as err: raise SystemExit(str(err)) with db.sessionmaker() as session: # without item this will be an array which will break database queries latest = Time( np.frombuffer(redis_db.get("auto:timestamp"), dtype=np.float64).item(), format="jd", ) latest.out_subfmt = u"date_hm" now = Time.now() amps = {} keys = [ k.decode() for k in redis_db.keys() if k.startswith(b"auto") and not k.endswith(b"timestamp") ] for key in keys: match = re.search(r"auto:(?P<ant>\d+)(?P<pol>e|n)", key) if match is not None: ant, pol = int(match.group("ant")), match.group("pol") d = redis_db.get(key) if d is not None: # need to copy because frombuffer creates a read-only array auto = np.frombuffer(d, dtype=np.float32).copy() eq_coeff = redis_db.hget( bytes("eq:ant:{ant}:{pol}".format(ant=ant, pol=pol).encode()), "values", ) if eq_coeff is not None: eq_coeffs = np.fromstring( eq_coeff.decode("utf-8").strip("[]"), sep=",") if eq_coeffs.size == 0: eq_coeffs = np.ones_like(auto) else: eq_coeffs = np.ones_like(auto) # divide out the equalization coefficients # eq_coeffs are stored as a length 1024 array but only a # single number is used. Taking the median to not deal with # a size mismatch eq_coeffs = np.median(eq_coeffs) auto /= eq_coeffs**2 auto[auto < 10**-10.0] = 10**-10.0 auto = np.median(auto) amps[(ant, pol)] = 10.0 * np.log10(auto) hsession = cm_sysutils.Handling(session) ants = np.unique([ant for (ant, pol) in amps.keys()]) pols = np.unique([pol for (ant, pol) in amps.keys()]) antpos = np.genfromtxt( os.path.join(mc.data_path, "HERA_350.txt"), usecols=(0, 1, 2, 3), dtype={ "names": ("ANTNAME", "EAST", "NORTH", "UP"), "formats": ("<U5", "<f8", "<f8", "<f8"), }, encoding=None, ) antnames = antpos["ANTNAME"] inds = [int(j[2:]) for j in antnames] inds = np.argsort(inds) antnames = np.take(antnames, inds) antpos = np.array([antpos["EAST"], antpos["NORTH"], antpos["UP"]]) array_center = np.mean(antpos, axis=1, keepdims=True) antpos -= array_center antpos = np.take(antpos, inds, axis=1) stations = hsession.get_connected_stations(at_date="now") for station in stations: if station.antenna_number not in ants: ants = np.append(ants, station.antenna_number) ants = np.unique(ants) stations = [] for station_type in hsession.geo.parse_station_types_to_check( "default"): for stn in hsession.geo.station_types[station_type]["Stations"]: stations.append(stn) # stations is a list of HH??? numbers we just want the ints stations = list(map(int, [j[2:] for j in stations])) built_but_not_on = np.setdiff1d(stations, ants) # Get node and PAM info node_ind = np.zeros_like(ants, dtype=np.int) pam_ind = np.zeros_like(ants, dtype=np.int) # defaul the snap name to "No Data" hostname = np.full_like(ants, "No\tData", dtype=object) snap_serial = np.full_like(ants, "No\tData", dtype=object) pam_power = {} adc_power = {} adc_rms = {} time_array = {} fem_imu_theta = {} fem_imu_phi = {} eq_coeffs = {} for ant in ants: for pol in pols: amps.setdefault((ant, pol), np.Inf) pam_power.setdefault((ant, pol), np.Inf) adc_power.setdefault((ant, pol), np.Inf) adc_rms.setdefault((ant, pol), np.Inf) eq_coeffs.setdefault((ant, pol), np.Inf) fem_imu_theta.setdefault((ant, pol), np.Inf) fem_imu_phi.setdefault((ant, pol), np.Inf) time_array.setdefault((ant, pol), now - Time(0, format="gps")) for ant_cnt, ant in enumerate(ants): station_status = session.get_antenna_status( most_recent=True, antenna_number=int(ant)) for status in station_status: antpol = (status.antenna_number, status.antenna_feed_pol) if status.pam_power is not None: pam_power[antpol] = status.pam_power if status.adc_power is not None: adc_power[antpol] = 10 * np.log10(status.adc_power) if status.adc_rms is not None: adc_rms[antpol] = status.adc_rms if status.time is not None: time_array[antpol] = now - Time(status.time, format="gps") if status.fem_imu_phi is not None: fem_imu_phi[antpol] = status.fem_imu_phi if status.fem_imu_theta is not None: fem_imu_theta[antpol] = status.fem_imu_theta if status.eq_coeffs is not None: _coeffs = np.fromstring(status.eq_coeffs.strip("[]"), sep=",") # just track the median coefficient for now eq_coeffs[antpol] = np.median(_coeffs) # Try to get the snap info. Output is a dictionary with 'e' and 'n' keys mc_name = antnames[ant] snap_info = hsession.get_part_at_station_from_type( mc_name, "now", "snap") # get the first key in the dict to index easier _key = list(snap_info.keys())[0] pol_key = [key for key in snap_info[_key].keys() if "E" in key] if pol_key: # 'E' should be in one of the keys, extract the 0th entry pol_key = pol_key[0] else: # a hacky solution for a key that should work pol_key = "E<ground" if snap_info[_key][pol_key] is not None: snap_serial[ant_cnt] = snap_info[_key][pol_key] # Try to get the pam info. Output is a dictionary with 'e' and 'n' keys pam_info = hsession.get_part_at_station_from_type( mc_name, "now", "post-amp") # get the first key in the dict to index easier _key = list(pam_info.keys())[0] if pam_info[_key][pol_key] is not None: _pam_num = re.findall(r"PAM(\d+)", pam_info[_key][pol_key])[0] pam_ind[ant_cnt] = np.int(_pam_num) else: pam_ind[ant_cnt] = -1 # Try to get the ADC info. Output is a dictionary with 'e' and 'n' keys node_info = hsession.get_part_at_station_from_type( mc_name, "now", "node") # get the first key in the dict to index easier _key = list(node_info.keys())[0] if node_info[_key][pol_key] is not None: _node_num = re.findall(r"N(\d+)", node_info[_key][pol_key])[0] node_ind[ant_cnt] = np.int(_node_num) _hostname = session.get_snap_hostname_from_serial( snap_serial[ant_cnt]) if _hostname is not None: hostname[ant_cnt] = _hostname else: snap_status = session.get_snap_status( most_recent=True, nodeID=np.int(_node_num)) for _status in snap_status: if _status.serial_number == snap_serial[ant_cnt]: hostname[ant_cnt] = _status.hostname else: node_ind[ant_cnt] = -1 pams, _pam_ind = np.unique(pam_ind, return_inverse=True) nodes, _node_ind = np.unique(node_ind, return_inverse=True) xs_offline = np.ma.masked_array( antpos[0, :], mask=[ True if int(name[2:]) in ants else False for name in antnames ], ) ys_offline = np.ma.masked_array(antpos[1, :], mask=xs_offline.mask) name_offline = np.ma.masked_array( [aname + "<br>OFFLINE" for aname in antnames], mask=xs_offline.mask, dtype=object, ) xs_offline = xs_offline names = [ "Auto [dB]", "PAM [dB]", "ADC [dB]", "ADC RMS", "FEM IMU THETA", "FEM IMU PHI", "EQ COEF", ] powers = [ amps, pam_power, adc_power, adc_rms, fem_imu_theta, fem_imu_phi, eq_coeffs, ] powers = [ np.ma.masked_invalid([[p[ant, pol] for ant in ants] for pol in pols]) for p in powers ] write_csv("ant_stats.csv", antnames, ants, pols, names, powers, built_but_not_on) time_array = np.array( [[time_array[ant, pol].to("hour").value for ant in ants] for pol in pols]) xs = np.ma.masked_array(antpos[0, ants], mask=powers[0][0].mask) ys = np.ma.masked_array( [ antpos[1, ants] + 3 * (pol_cnt - 0.5) for pol_cnt, pol in enumerate(pols) ], mask=powers[0].mask, ) _text = np.array( [[ antnames[ant] + pol + "<br>" + str(hostname[ant_cnt]) + "<br>" + "PAM\t#:\t" + str(pam_ind[ant_cnt]) for ant_cnt, ant in enumerate(ants) ] for pol_cnt, pol in enumerate(pols)], dtype="object", ) # want to format No Data where data was not retrieved for each type of power for pol_cnt, pol in enumerate(pols): for ant_cnt, ant in enumerate(ants): for _name, _power in zip(names, powers): if not _power.mask[pol_cnt, ant_cnt]: _text[pol_cnt, ant_cnt] += ( "<br>" + _name + ": {0:.2f}".format(_power[pol_cnt, ant_cnt])) else: _text[pol_cnt, ant_cnt] += "<br>" + _name + ": No Data" if time_array[pol_cnt, ant_cnt] > 2 * 24 * 365: # if the value is older than 2 years it is bad # value are stored in hours. # 2 was chosen arbitraritly. _text[pol_cnt, ant_cnt] += "<br>" + "Ant Status: No Date" else: _text[pol_cnt, ant_cnt] += ("<br>" + "Ant Status: {0:.2f} hrs old".format( time_array[pol_cnt, ant_cnt])) # having spaces will cause odd wrapping issues, replace all # spaces by \t _text[pol_cnt, ant_cnt] = _text[pol_cnt, ant_cnt].replace(" ", "\t") masks = [[True] for p in powers] # Offline antennas data_hex = [] offline_ants = { "x": xs_offline.compressed().tolist(), "y": ys_offline.compressed().tolist(), "text": name_offline, "mode": "markers", "visible": True, "marker": { "color": np.ma.masked_array(["black"] * len(name_offline), mask=xs_offline.mask), "size": 14, "opacity": 0.5, "symbol": "hexagon", }, "hovertemplate": "%{text}<extra></extra>", } # now we want to Fill in the conneted ones offline_ants["marker"]["color"][built_but_not_on] = "red" offline_ants["text"].data[built_but_not_on] = [ offline_ants["text"].data[ant].split("<br>")[0] + "<br>Constructed<br>Not\tOnline" for ant in built_but_not_on ] offline_ants["marker"]["color"] = ( offline_ants["marker"]["color"].compressed().tolist()) offline_ants["text"] = offline_ants["text"].compressed().tolist() data_hex.append(offline_ants) # for each type of power, loop over pols and print out the data # save up a mask array used for the buttons later # also plot the bad ones!3 colorscale = "Viridis" # define some custom scale values for the ADC RMS page rms_scale_vals = [2, 20] relavitve_values = [0.4, 0.7] rms_color_scale = [ ["0.0", "rgb(68,1,84)"], ["0.2", "rgb(62,74,137)"], ["0.3", "rgb(38,130,142)"], ["0.4", "rgb(53,183,121)"], ["0.5", "rgb(53,183,121)"], ["0.6", "rgb(53,183,121)"], ["0.7", "rgb(109,205,89)"], ["0.8", "rgb(180,222,44)"], ["1.0", "rgb(253,231,37)"], ] for pow_ind, power in enumerate(powers): if power.compressed().size > 0: vmax = np.max(power.compressed()) vmin = np.min(power.compressed()) else: vmax = 1 vmin = 0 colorscale = "Viridis" if pow_ind == 3: cbar_title = "RMS\tlinear" vmin = rms_scale_vals[0] * relavitve_values[0] vmax = rms_scale_vals[1] / relavitve_values[1] colorscale = rms_color_scale elif pow_ind == 4 or pow_ind == 5: cbar_title = "Degrees" elif pow_ind == len(powers) - 1: cbar_title = "Median\tCoeff" else: cbar_title = "dB" if pow_ind == 0: visible = True else: visible = False for pol_ind, pol in enumerate(pols): for mask_cnt, mask in enumerate(masks): if mask_cnt == pow_ind: mask.extend([True] * 2) else: mask.extend([False] * 2) _power = { "x": xs.data[~power[pol_ind].mask].tolist(), "y": ys[pol_ind].data[~power[pol_ind].mask].tolist(), "text": _text[pol_ind][~power[pol_ind].mask].tolist(), "mode": "markers", "visible": visible, "marker": { "color": power[pol_ind].data[~power[pol_ind].mask].tolist(), "size": 14, "cmin": vmin, "cmax": vmax, "colorscale": colorscale, "colorbar": { "thickness": 20, "title": cbar_title }, }, "hovertemplate": "%{text}<extra></extra>", } data_hex.append(_power) _power_offline = { "x": xs.data[power[pol_ind].mask].tolist(), "y": ys[pol_ind].data[power[pol_ind].mask].tolist(), "text": _text[pol_ind][power[pol_ind].mask].tolist(), "mode": "markers", "visible": visible, "marker": { "color": "orange", "size": 14, "cmin": vmin, "cmax": vmax, "colorscale": colorscale, "colorbar": { "thickness": 20, "title": cbar_title }, }, "hovertemplate": "%{text}<extra></extra>", } data_hex.append(_power_offline) buttons = [] for _name, mask in zip(names, masks): _button = { "args": [{ "visible": mask }, { "title": "", "annotations": {} }], "label": _name, "method": "restyle", } buttons.append(_button) updatemenus_hex = [{ "buttons": buttons, "showactive": True, "type": "buttons" }] layout_hex = { "xaxis": { "title": "East-West Position [m]" }, "yaxis": { "title": "North-South Position [m]", "scaleanchor": "x" }, "title": { "text": "Per Antpol Stats vs Hex position", "font": { "size": 24 }, }, "hoverlabel": { "align": "left" }, "margin": { "t": 40 }, "autosize": True, "showlegend": False, "hovermode": "closest", } caption = {} caption["title"] = "Stats vs Hex pos Help" caption["text"] = ( "This plot shows various statistics and measurements " "per ant-pol versus its position in the array." "<br>Antennas which are build but not fully hooked up " "are shown in light red." "<br>Grey antennas are not yet constructed." "<br><br><h4>Available plotting options</h4>" "<ul>" "<li>Auto Corr - Median Auto Correlation (in db) " "from the correlator with equalization coefficients " "divided out</li>" "<li>Pam Power - Latest Pam Power (in db) recorded in M&C</li>" "<li>ADC Power - Latest ADC Power (in db) recorded in M&C</li>" "<li>ADC RMS - Latest linear ADC RMS recorded in M&C</li>" "<li>FEM IMU THETA - IMU-reported theta, in degrees</li>" "<li>FEM IMU PHI - IMU-reported phi, in degrees</li>" "<li>EQ Coeffs - Latest Median Equalization Coefficient recorded in M&C</li>" "</ul>" "Any antpol showing with an orange color means " "no data is avaible for the currenty plot selection." "<h4>Hover label Formatting</h4>" "<ul>" "<li>Antenna Name from M&C<br>(e.g. HH0n = Hera Hex Antenna 0 Polarization N)</li>" "<li>Snap hostname from M&C<br>(e.g. heraNode0Snap0)</li>" "<li>PAM Number</li>" "<li>Median Auto Correlation power in dB</li>" "<li>PAM power in dB</li>" "<li>ADC power in dB</li>" "<li>Linear ADC RMS</li>" "<li>FEM IMU reported theta in degrees</li>" "<li>FEM IMU reported phi in degrees</li>" "<li>Median Equalization Coefficient</li>" "<li>Time ago in hours the M&C Antenna Status was updated. " "This time stamp applies to all data for this antenna " "except the Auto Correlation.</li>" "</ul>" "In any hover label entry 'No Data' means " "information not currrently available in M&C.") # Render all the power vs position files plotname = "plotly-hex" html_template = env.get_template("plotly_base.html") js_template = env.get_template("plotly_base.js") if sys.version_info.minor >= 8 and sys.version_info.major > 2: time_jd = latest.to_value('jd', subfmt='float') time_unix = latest.to_value('unix') else: time_jd = latest.jd time_unix = latest.unix rendered_hex_html = html_template.render( plotname=plotname, data_type="Auto correlations", plotstyle="height: 100%", gen_date=now.iso, data_date_iso=latest.iso, data_date_jd="{:.3f}".format(time_jd), data_date_unix_ms=time_unix * 1000, js_name="hex_amp", gen_time_unix_ms=now.unix * 1000, scriptname=os.path.basename(__file__), hostname=computer_hostname, caption=caption, ) rendered_hex_js = js_template.render( data=data_hex, layout=layout_hex, updatemenus=updatemenus_hex, plotname=plotname, ) with open("hex_amp.html", "w") as h_file: h_file.write(rendered_hex_html) with open("hex_amp.js", "w") as js_file: js_file.write(rendered_hex_js) # now prepare the data to be plotted vs node number data_node = [] masks = [[] for p in powers] vmax = [ np.max(power.compressed()) if power.compressed().size > 1 else 1 for power in powers ] vmin = [ np.min(power.compressed()) if power.compressed().size > 1 else 0 for power in powers ] vmin[3] = rms_scale_vals[0] * relavitve_values[0] vmax[3] = rms_scale_vals[1] / relavitve_values[1] for node in nodes: node_index = np.where(node_ind == node)[0] hosts = hostname[node_index] host_index = np.argsort(hosts) ys = np.ma.masked_array( [ np.arange(node_index.size) + 0.3 * pol_cnt for pol_cnt, pol in enumerate(pols) ], mask=powers[0][:, node_index].mask, ) xs = np.zeros_like(ys) xs[:] = node powers_node = [pow[:, node_index] for pow in powers] __text = _text[:, node_index] for pow_ind, power in enumerate(powers_node): cbar_title = "dB" if pow_ind == 4 or pow_ind == 5: cbar_title = "Degrees" if pow_ind == 3: colorscale = rms_color_scale else: colorscale = "Viridis" colorscale = "Viridis" if pow_ind == 3: cbar_title = "RMS\tlinear" colorscale = rms_color_scale elif pow_ind == 4 or pow_ind == 5: cbar_title = "Degrees" elif pow_ind == len(powers) - 1: cbar_title = "Median\tCoeff" else: cbar_title = "dB" if pow_ind == 0: visible = True else: visible = False for pol_ind, pol in enumerate(pols): for mask_cnt, mask in enumerate(masks): if mask_cnt == pow_ind: mask.extend([True] * 2) else: mask.extend([False] * 2) __power = power[pol_ind][host_index] ___text = __text[pol_ind][host_index] _power = { "x": xs[pol_ind].data[~__power.mask].tolist(), "y": ys[pol_ind].data[~__power.mask].tolist(), "text": ___text[~__power.mask].tolist(), "mode": "markers", "visible": visible, "marker": { "color": __power.data[~__power.mask].tolist(), "size": 14, "cmin": vmin[pow_ind], "cmax": vmax[pow_ind], "colorscale": colorscale, "colorbar": { "thickness": 20, "title": cbar_title }, }, "hovertemplate": "%{text}<extra></extra>", } data_node.append(_power) _power_offline = { "x": xs[pol_ind].data[__power.mask].tolist(), "y": ys[pol_ind].data[__power.mask].tolist(), "text": ___text[__power.mask].tolist(), "mode": "markers", "visible": visible, "marker": { "color": "orange", "size": 14, "cmin": vmin[pow_ind], "cmax": vmax[pow_ind], "colorscale": colorscale, "colorbar": { "thickness": 20, "title": cbar_title }, }, "hovertemplate": "%{text}<extra></extra>", } data_node.append(_power_offline) buttons = [] for _name, mask in zip(names, masks): _button = { "args": [{ "visible": mask }, { "title": "", "annotations": {} }], "label": _name, "method": "restyle", } buttons.append(_button) updatemenus_node = [{ "buttons": buttons, "showactive": True, "type": "buttons" }] layout_node = { "xaxis": { "title": "Node Number", "dtick": 1, "tick0": 0, "showgrid": False, "zeroline": False, }, "yaxis": { "showticklabels": False, "showgrid": False, "zeroline": False }, "title": { "text": "Per Antpol Stats vs Node #", "font": { "size": 24 } }, "hoverlabel": { "align": "left" }, "margin": { "t": 40 }, "autosize": True, "showlegend": False, "hovermode": "closest", } caption_node = {} caption_node["title"] = "Stats vs Node Help" caption_node["text"] = ( "This plot shows various statistics and measurements " "per ant-pol versus the node number to which it is connected." "<br><br><h4>Available plotting options</h4>" "<ul>" "<li>Auto Corr - Median Auto Correlation (in db) " "from the correlator with equalization coefficients " "divided out</li>" "<li>Pam Power - Latest Pam Power (in db) recorded in M&C</li>" "<li>ADC Power - Latest ADC Power (in db) recorded in M&C</li>" "<li>ADC RMS - Latest linear ADC RMS recorded in M&C</li>" "<li>EQ Coeffs - Latest Median Equalization Coefficient recorded in M&C</li>" "</ul>" "Any antpol showing with an orange color means " "no data is avaible for the currenty plot selection." "<h4>Hover label Formatting</h4>" "<ul>" "<li>Antenna Name from M&C<br>(e.g. HH0n = Hera Hex Antenna 0 Polarization N)</li>" "<li>Snap hostname from M&C<br>(e.g. heraNode0Snap0)</li>" "<li>PAM Number</li>" "<li>Median Auto Correlation power in dB</li>" "<li>PAM power in dB</li>" "<li>ADC power in dB</li>" "<li>Linear ADC RMS</li>" "<li>Median Equalization Coefficient</li>" "<li>Time ago in hours the M&C Antenna Status was updated. " "This time stamp applies to all data for this antenna " "except the Auto Correlation.</li>" "</ul>" "In any hover label entry 'No Data' means " "information not currrently available in M&C.") # Render all the power vs ndde files plotname = "plotly-node" html_template = env.get_template("plotly_base.html") js_template = env.get_template("plotly_base.js") if sys.version_info.minor >= 8 and sys.version_info.major > 2: time_jd = latest.to_value('jd', subfmt='float') time_unix = latest.to_value('unix') else: time_jd = latest.jd time_unix = latest.unix rendered_node_html = html_template.render( plotname=plotname, data_type="Auto correlations", plotstyle="height: 100%", gen_date=now.iso, gen_time_unix_ms=now.unix * 1000, data_date_iso=latest.iso, data_date_jd="{:.3f}".format(time_jd), data_date_unix_ms=time_unix * 1000, js_name="node_amp", scriptname=os.path.basename(__file__), hostname=computer_hostname, caption=caption_node, ) rendered_node_js = js_template.render( data=data_node, layout=layout_node, updatemenus=updatemenus_node, plotname=plotname, ) with open("node_amp.html", "w") as h_file: h_file.write(rendered_node_html) with open("node_amp.js", "w") as js_file: js_file.write(rendered_node_js)
def UTC(mjd): " convert MJD value to ISO date string" t = Time(mjd, format='mjd') t.format = 'iso' t.out_subfmt = 'date_hm' return t.value
def extract_the_open_supernova_catalog(self): """Extracts SLSN from the Open supernovae catalog with the type SLSN. The catalog can be found at https://sne.space/ . """ arxiv = "1605.01054 (https://sne.space/)" new_entries = [] path = "source_lists/The Open Supernova Catalog.csv" with open(path, 'rb') as csvfile: reader = csv.reader(csvfile, delimiter=',', quotechar='"') for i, row in enumerate(reader): if i < 1: pass # Check for weird artifact entry in SLSN catalog elif row[0] != "PISN_Jerkstrand": new = SLSN() new.name = row[0] new.alias = list( set([ x.strip() for x in row[1].split(",") if x != new.name ])) for j, attr in enumerate(["disc_date", "peak_date"]): try: date = "-".join(row[j + 2].split("/")) date = Time(date, format="iso") date.out_subfmt = "date" date.format = "mjd" setattr(new, attr, date) except ValueError: if row[j + 2] != "": new.notes += "Problem in " + attr + ", only have " new.notes += row[j + 2] + " (not ISO format), " if row[4] != "": new.abs_peak = float(row[4]) coords = [np.nan, np.nan] for j, angle in enumerate(coords): if row[5 + j] != "": values = row[5 + j].split(",") coords[j] = values[0] if len(values) > 1: new.notes += "Several values reported for " + \ ["ra", "dec"][j] + " " + \ str(values) + ", " new.add_coordinates(coords[0], coords[1]) for j, attr in enumerate(["redshift", "ebv"]): val = row[7 + (2 * j)] if val != "": all_vals = list(set(val.split(","))) setattr(new, attr, float(all_vals[0])) if len(all_vals) > 1: new.notes += "Several values reported for " + \ attr + " " + str(all_vals) + ", " type_val = row[8] if type_val != "": all_vals = list(set(type_val.split(","))) setattr(new, "type", all_vals[0]) if len(all_vals) > 1: new.notes += "Several values reported for " + \ attr + " " + str(all_vals) + ", " ref_val = row[10] if ref_val != "": setattr(new, "ref", ref_val) new.arxiv = arxiv new_entries.append(new) self.entries.extend(new_entries)
def extract_1604_08207(self): """Extracts SLSN entries for arxiv paper 1604.08207 (page 3)""" path = "source_lists/tabula-1604.08207.csv" arxiv = "1604.08207 (p.3)" with open(path, 'rb') as csvfile: reader = csv.reader(csvfile, delimiter=',', quotechar='|') for i, row in enumerate(reader): if i < 1: pass else: new = SLSN() new.name = "PTF" + row[0] ra = row[1] dec = [x for x in row[2]] if len(dec) > 12: dec.pop(0) dec.pop(0) dec[0] = "-" dec = "".join(dec) new.add_coordinates(ra, dec) if len(row[3]) > 2: new.type = "SLSN-R" else: new.type = "SLSN-" + row[3] new.redshift = float(row[4]) new.arxiv = arxiv new.ref = arxiv peak = [x for x in row[5]][3:] while not peak[0].isdigit(): peak.pop(0) peak = -1. * float("".join(peak)) new.abs_peak = peak date = [x for x in row[10]] if len(date) > 10: date.pop(0) date = "".join(date) date = Time(date, format="iso") date.out_subfmt = "date" date.format = "mjd" new.peak_date = date new.ebv = float(row[13]) notes = row[16] if notes != "": if notes[0] == "=": split = notes.split(".") new.alias.append(split[0][2:]) if len(split) > 1: notes = split[1] else: notes = "" else: notes = [x for x in row[16]] while not notes[-1].isalpha(): notes.pop(-1) notes = "".join(notes) new.notes = notes self.entries.append(new)
def main(): # templates are stored relative to the script dir # stored one level up, find the parent directory # and split the parent directory away script_dir = os.path.dirname(os.path.realpath(__file__)) split_dir = os.path.split(script_dir) template_dir = os.path.join(split_dir[0], "templates") env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True) if sys.version_info[0] < 3: # py2 computer_hostname = os.uname()[1] else: # py3 computer_hostname = os.uname().nodename parser = argparse.ArgumentParser( description=("Create snap hookup tables for heranow dashboard")) parser.add_argument( "--redishost", dest="redishost", type=str, default="redishost", help=( 'The host name for redis to connect to, defaults to "redishost"'), ) parser.add_argument("--port", dest="port", type=int, default=6379, help="Redis port to connect.") args = parser.parse_args() redis_db = redis.Redis(args.redishost, port=args.port) corr_map = redis_db.hgetall("corr:map") update_time = Time(float(corr_map[b"update_time"]), format="unix") update_time.out_subfmt = u"date_hm" all_tables = [] # make a table of the antenna to snap mapping table_a_to_s = {} table_a_to_s["title"] = "Antenna -> SNAP mappings" table_a_to_s["tab_style"] = "float: left" rows_a = [] ant_to_snap = json.loads(corr_map[b"ant_to_snap"]) for ant in sorted(map(int, ant_to_snap)): ant = str(ant) pol = ant_to_snap[ant] for p in pol: vals = pol[p] row = {} host = vals["host"] chan = vals["channel"] if isinstance(host, bytes): host = host.decode("utf-8") if isinstance(chan, bytes): chan = chan.decode("utf-8") row["text"] = "{ant}:{pol} -> {host}:{chan}".format(ant=ant, pol=p, host=host, chan=chan) rows_a.append(row) table_a_to_s["rows"] = rows_a table_a_to_s["div_style"] = 'style="max-height: 2500px;"' table_a_to_s["colsize"] = 6 all_tables.append(table_a_to_s) # make a table of the snap to antenna mapping table_s_to_a = {} table_s_to_a["title"] = "SNAP -> Antenna mappings" table_s_to_a["style"] = "float: right" rows_s = [] snap_to_ant = json.loads(corr_map[b"snap_to_ant"]) for snap in sorted(snap_to_ant): ant = snap_to_ant[snap] for i in range(6): if ant[i] is None: ant[i] = "n/c" if isinstance(ant[i], bytes): ant[i] = ant[i].decode("utf-8") if isinstance(snap, bytes): snap = snap.decode("utf-8") row = {} row["text"] = "{snap} -> {ants}".format(snap=snap, ants=", ".join(ant)) rows_s.append(row) table_s_to_a["rows"] = rows_s table_s_to_a["colsize"] = 6 all_tables.append(table_s_to_a) # Make a table of the snap to antenna indices mapping table_ant_ind = {} table_ant_ind["title"] = "SNAP -> Antenna indices" snap_to_ant_i = redis_db.hgetall("corr:snap_ants") rows_ant_ind = [] for snap in sorted(snap_to_ant_i): ant = snap_to_ant_i[snap] row = {} if isinstance(snap, bytes): snap = snap.decode("utf-8") if isinstance(ant, bytes): ant = ant.decode("utf-8") row["text"] = "{snap} -> {ant}".format(snap=snap, ant=str(ant)) rows_ant_ind.append(row) table_ant_ind["rows"] = rows_ant_ind table_ant_ind["style"] = "float: right" table_ant_ind["colsize"] = 6 all_tables.append(table_ant_ind) # Make a table of the XENG channel indices table_xeng = {} table_xeng["title"] = "XENG -> Channel indices" rows_xeng = [] xeng_to_chan_i = redis_db.hgetall("corr:xeng_chans") for xeng in sorted(map(int, xeng_to_chan_i)): xeng = bytes(str(xeng).encode()) chans = xeng_to_chan_i[xeng] row = {} if isinstance(xeng, bytes): xeng = xeng.decode("utf-8") if isinstance(chans, bytes): chans = chans.decode("utf-8") row["text"] = "{xeng} -> {chans}...".format(xeng=xeng, chans=chans[0:5]) rows_xeng.append(row) table_xeng["rows"] = rows_xeng table_xeng["style"] = "float: right" table_xeng["colsize"] = 6 all_tables.append(table_xeng) html_template = env.get_template("tables_with_footer.html") if sys.version_info.minor >= 8 and sys.version_info.major > 2: time_jd = update_time.to_value('jd', subfmt='float') time_unix = update_time.to_value('unix') else: time_jd = update_time.jd time_unix = update_time.unix rendered_html = html_template.render( tables=all_tables, data_type="Hookup information", data_date_iso=update_time.iso, data_date_jd="{:.3f}".format(time_jd), data_date_unix_ms=time_unix * 1000, gen_date=Time.now().iso, gen_time_unix_ms=Time.now().unix * 1000, scriptname=os.path.basename(__file__), hostname=computer_hostname, ) with open("snaphookup.html", "w") as h_file: h_file.write(rendered_html)