try:
t_stop = dt_to_timestamp(datetime.datetime.strptime(opts.stop, "%Y-%m-%d_%H:%M:%S"))
print("Stop Time: " + ts_to_datestring(t_stop))
except:
print("Bad t_stop time format detected (must be YYYY-MM-DD_HH:MM:SS)")
#print t_date, t_start, t_stop
# Load configuration file
station.load_configuration_file(opts.config)
station_name = station.configuration['station']['name']
print("\nStation Name: ", station_name)
print("Checking directory: ", opts.directory + "\n")
if opts.type == "channel":
file_manager = ChannelFormatFileManager(root_path=opts.directory, daq_mode=modo)
elif opts.type == "raw":
file_manager = RawFormatFileManager(root_path=opts.directory, daq_mode=modo)
else:
print("\n Please specify a data format (channel, raw)")
exit()
print("\tFILE\t\t TIMESTAMP\t\tSTART\t\t\tSTOP\t\tSIZE (MB)\tBLOCKS")
print("---------------------+-----------------+------------------+--------------------------+--------------+-----------")
if opts.mode == "null":
lista = sorted(glob.glob(opts.directory + "/" + opts.type + "_%d_*hdf5" % (int(opts.tile)-1)))
else:
lista = sorted(glob.glob(opts.directory + "/" + opts.type + "_" + opts.mode + "_%d_*hdf5" % (int(opts.tile)-1)))
nof_tiles = 16
print("\nFound %d hdf5 files for Tile-%02d\n" % (len(lista), int(opts.tile)))
for nn, l in enumerate(lista):
try:
t_stop = dt_to_timestamp(datetime.datetime.strptime(opts.stop, "%Y-%m-%d_%H:%M:%S"))
print "Stop Time: " + ts_to_datestring(t_stop)
except:
print "Bad t_stop time format detected (must be YYYY-MM-DD_HH:MM:SS)"
#print t_date, t_start, t_stop
# Load configuration file
station.load_configuration_file(opts.config)
station_name = station.configuration['station']['name']
print "\nStation Name: ", station_name
print "Checking directory: ", opts.directory + "\n"
if opts.type == "channel":
file_manager = ChannelFormatFileManager(root_path=opts.directory, daq_mode=modo)
elif opts.type == "raw":
file_manager = RawFormatFileManager(root_path=opts.directory, daq_mode=modo)
else:
print "\n Please specify a data format (channel, raw)"
exit()
print "\tFILE\t\t TIMESTAMP\t\tSTART\t\t\tSTOP\t\tSIZE (MB)\tBLOCKS"
print "---------------------+-----------------+------------------+--------------------------+--------------+-----------"
if opts.mode == "null":
lista = sorted(glob.glob(opts.directory + "/" + opts.type + "_%d_*hdf5" % (int(opts.tile)-1)))
else:
lista = sorted(glob.glob(opts.directory + "/" + opts.type + "_" + opts.mode + "_%d_*hdf5" % (int(opts.tile)-1)))
nof_tiles = 16
for l in lista:
#dic = file_manager.get_metadata(timestamp=fname_to_tstamp(l[-21:-7]), tile_id=(int(opts.tile)-1))
file_manager.read_data(n_samples=1)
try:
t_stop = dt_to_timestamp(datetime.datetime.strptime(opts.stop, "%Y-%m-%d_%H:%M:%S"))
print "Stop Time: " + ts_to_datestring(t_stop)
except:
print "Bad t_stop time format detected (must be YYYY-MM-DD_HH:MM:SS)"
#print t_date, t_start, t_stop
# Load configuration file
station.load_configuration_file(opts.config)
station_name = station.configuration['station']['name']
#print "\nStation Name: ", station_name
print "Processing directory: ", opts.directory,
if opts.type == "channel":
file_manager = ChannelFormatFileManager(root_path=opts.directory, daq_mode=modo)
elif opts.type == "raw":
file_manager = RawFormatFileManager(root_path=opts.directory, daq_mode=modo)
else:
print "\n Please specify a data format (channel, raw)"
exit()
#print "\tFILE\t\t TIMESTAMP\t\tSTART\t\t\tSTOP\t\tSIZE (MB)\tBLOCKS"
#print "---------------------+-----------------+------------------+--------------------------+--------------+-----------"
if opts.mode == "null":
lista = sorted(glob.glob(opts.directory + "/" + opts.type + "_%d_*hdf5" % (int(opts.tile)-1)))
else:
lista = sorted(glob.glob(opts.directory + "/" + opts.type + "_" + opts.mode + "_%d_*hdf5" % (int(opts.tile)-1)))
nof_tiles = 16
for l in lista:
#dic = file_manager.get_metadata(timestamp=fname_to_tstamp(l[-21:-7]), tile_id=(int(opts.tile)-1))
#file_manager.read_data(n_samples=1)
#ind = np.arange(nplot)
# Load configuration file
station.load_configuration_file(opts.config)
station_name = station.configuration['station']['name']
# Store number of tiles
nof_tiles = len(station.configuration['tiles'])
print "\nStation Name: ", station_name
print "\nNumber of Tiles: ", nof_tiles
print "Checking directory: ", opts.directory + station_name.lower() + "\n"
print "Looking for tiles/antenna: ", opts.tile, "/", opts.input, "\n"
file_manager = ChannelFormatFileManager(root_path=opts.directory +
station_name.lower(),
daq_mode=FileDAQModes.Integrated)
base, x, y = get_antenna_positions(station_name)
ants = []
for j in base:
ants += ["ANT-%03d" % int(j)]
fig = plt.figure(figsize=(11, 7), facecolor='w')
ax = fig.add_subplot(1, 1, 1)
all_data = np.zeros((512, nof_tiles * 16, 2, 1))
xl, = ax.plot(range(512), range(512), color='b')
yl, = ax.plot(range(512), range(512), color='g')
ax.tick_params(axis='both', which='both', labelsize=12)
ax.set_ylim(0, 50)
import time
import glob
import datetime
from aavs_utils import ts_to_datestring, tstamp_to_fname, dt_to_timestamp, fname_to_tstamp
import os
import numpy as np
from tqdm import tqdm
from slacker import Slacker
#
slack = Slacker("")
conf = "/opt/aavs/config/aavs1_full_station.yml"
station.load_configuration_file(conf)
station_name = station.configuration['station']['name']
modo = FileDAQModes.Continuous
file_manager = ChannelFormatFileManager(
root_path="/data/data_1/2019_05_15_204_station_beam/", daq_mode=modo)
tiles = range(16)
for t in tiles:
try:
message = "Processing AAVS1 Tile-%02d" % (t + 1)
slack.chat.post_message("lab-test", message, as_user=True)
except:
pass
print "Processing Tile-%02d" % (t + 1)
lista = sorted(
glob.glob(
"/data/data_1/2019_05_15_204_station_beam/channel_cont_%d_*hdf5" %
t))
for l in tqdm(range(len(lista))):
dic = file_manager.get_metadata(timestamp=fname_to_tstamp(
lista[l][-21:-7]),
# Load configuration file
station.load_configuration_file(opts.config)
station_name = station.configuration['station']['name']
if station_name == "AAVS1.5":
if "all" in opts.tile.lower():
tiles = [1, 2, 3]
print "\nStation Name: ", station_name
print "Checking directory: ", opts.directory + station_name.lower() + "\n"
print "Looking for tiles: ", tiles, "\n"
# Instantiate a file manager
file_manager = ChannelFormatFileManager(root_path=opts.directory +
station_name.lower(),
daq_mode=FileDAQModes.Integrated)
base, x, y = get_antenna_positions(station_name)
#print len(base), len(x), len(y)
ants = []
for j in base:
ants += ["ANT-%03d" % int(j)]
for tile in tiles:
t_cnt = 0
lista = sorted(
glob.glob(opts.directory + station_name.lower() +
"/channel_integ_%d_*hdf5" % (tile - 1)))
def plotting_thread(directory, cadence):
""" PLotting thread
:param cadence: Sleeps between plot generations """
global stop_plotting
station_name = station.configuration['station']['name']
logging.info("Starting plotting threads for station " + station_name)
remap = [0, 1, 2, 3, 8, 9, 10, 11, 15, 14, 13, 12, 7, 6, 5, 4]
if not os.path.isdir(img_dir+station_name):
os.mkdir(img_dir+station_name)
# Store number of tiles
nof_tiles = len(station.configuration['tiles'])
# Create station instance
aavs_station = station.Station(station.configuration)
aavs_station.connect()
_connect_station(aavs_station)
sleep(16)
station_dir = ""
station_file = ""
if station_name == "AAVS2":
station_dir = "skala-4/"
station_file = "STATION_SKALA-4.png"
elif station_name == "EDA2":
station_dir = "eda-2/"
station_file = "STATION_EDA-2.png"
# Grab antenna base numbers and positions
base, x, y = get_antenna_positions(station_name)
# AAVS2 Tile-14 Patch
if (datetime.datetime.utcnow() > datetime.datetime(2020, 3, 1)) and (station_name == "AAVS2") and \
(datetime.datetime.utcnow() < datetime.datetime(2020, 3, 16, 18, 00)):
print "Patching antenna name and positions"
base = base[:16*13] + base[16*14:]
x = x[:16*13] + x[16*14:]
y = y[:16*13] + y[16*14:]
ants = []
for j in range(16*nof_tiles):
ants += ["ANT-%03d" % int(base[j])]
tile_names = []
tiles = get_antenna_tile_names(station_name)
# AAVS2 Tile-14 Patch
if (datetime.datetime.utcnow() > datetime.datetime(2020, 3, 1)) and (station_name == "AAVS2") and \
(datetime.datetime.utcnow() < datetime.datetime(2020, 3, 16, 18, 00)):
print "Patching tile names"
tiles = tiles[:16*13] + tiles[16*14:]
#print tiles
for i in tiles:
if not i.replace("TPM", "Tile") in tile_names:
tile_names += [i.replace("TPM", "Tile")]
# Instantiate a file manager
file_manager = ChannelFormatFileManager(root_path=opts.directory, daq_mode=FileDAQModes.Integrated)
plt.ioff()
#asse_x = np.linspace(0, 400, 512)
asse_x = np.arange(512) * 400/512.
# gridspec inside gridspec
outer_grid = gridspec.GridSpec(nof_tiles, 1, hspace=0.8, left=0.02, right=0.98, bottom=0.04, top=0.98)
fig = plt.figure(figsize=(FIG_W, TILE_H * nof_tiles), facecolor='w')
t_axes = []
axes = []
tstamp_label = []
x_lines = []
y_lines = []
ind = np.arange(16)
x_bar = []
y_bar = []
for i in range(nof_tiles):
# print tile_active[i]
gs = gridspec.GridSpecFromSubplotSpec(2, 17, wspace=0.05, hspace=0.5, subplot_spec=outer_grid[i])
t_axes += [
[plt.subplot(gs[0:2, 0:3]), plt.subplot(gs[0:2, 3:5]), plt.subplot(gs[0, 6:8]), plt.subplot(gs[1, 6:8])]]
t_axes[i][0].cla()
t_axes[i][0].set_axis_off()
t_axes[i][0].plot([0.001, 0.002], color='w')
t_axes[i][0].set_xlim(-20, 20)
t_axes[i][0].set_ylim(-20, 20)
t_axes[i][0].annotate(tile_names[i], (-11, 5), fontsize=26, color='black')
t_axes[i][0].annotate("Acquisition Time (UTC)", (-17.7, -6), fontsize=12, color='black')
tl = t_axes[i][0].annotate("--- UTC", (-17.8, -12), fontsize=12, color='black')
tstamp_label += [tl]
t_axes[i][1].cla()
t_axes[i][1].set_axis_off()
t_axes[i][1].plot([0.001, 0.002], color='wheat')
t_axes[i][1].set_xlim(-25, 25)
t_axes[i][1].set_ylim(-25, 25)
circle1 = plt.Circle((0, 0), 20, color='wheat', linewidth=2.5) # , fill=False)
t_axes[i][1].add_artist(circle1)
t_axes[i][1].annotate("E", (21, -1), fontsize=10, color='black')
t_axes[i][1].annotate("W", (-25, -1), fontsize=10, color='black')
t_axes[i][1].annotate("N", (-1, 21), fontsize=10, color='black')
t_axes[i][1].annotate("S", (-1, -24), fontsize=10, color='black')
t_axes[i][2].cla()
t_axes[i][2].plot([0.001, 0.002], color='w')
t_axes[i][2].tick_params(axis='both', which='both', labelsize=6)
t_axes[i][2].set_xticks(xrange(1, 17))
t_axes[i][2].set_xticklabels(np.array(range(1, 17)).astype("str").tolist(), fontsize=4)
if not station_name == "EDA2":
t_axes[i][2].set_ylim([0, 40])
t_axes[i][2].set_yticks([15, 20])
t_axes[i][2].set_yticklabels(["15", "20"], fontsize=7)
else:
t_axes[i][2].set_ylim([0, 20])
t_axes[i][2].set_yticks([0, 5, 10, 15, 20])
t_axes[i][2].set_yticklabels(["0", "5", "10", "15", "20"], fontsize=7)
t_axes[i][2].set_xlim([0, 16])
t_axes[i][2].set_ylabel("RMS", fontsize=10)
t_axes[i][2].grid()
t_axes[i][2].set_title("Power Pol X", fontsize=10)
xb = t_axes[i][2].bar(ind + 0.5, ind, 0.8, color='b')
x_bar += [xb]
t_axes[i][3].cla()
t_axes[i][3].plot([0.001, 0.002], color='w')
t_axes[i][3].tick_params(axis='both', which='both', labelsize=6)
t_axes[i][3].set_xticks(xrange(1, 17))
t_axes[i][3].set_xticklabels(np.array(range(1, 17)).astype("str").tolist(), fontsize=4)
t_axes[i][3].set_yticks([15, 20])
t_axes[i][3].set_yticklabels(["15", "20"], fontsize=7)
if not station_name == "EDA2":
t_axes[i][3].set_ylim([0, 40])
t_axes[i][3].set_yticks([15, 20])
t_axes[i][3].set_yticklabels(["15", "20"], fontsize=7)
else:
t_axes[i][3].set_ylim([0, 5])
t_axes[i][3].set_yticks([0, 5, 10, 15, 20])
t_axes[i][3].set_yticklabels(["0", "5", "10", "15", "20"], fontsize=7)
t_axes[i][3].set_xlim([0, 16])
t_axes[i][3].set_ylabel("RMS", fontsize=10)
t_axes[i][3].set_xlabel("Power Pol Y", fontsize=10)
t_axes[i][3].grid()
yb = t_axes[i][3].bar(ind + 0.5, ind, 0.8, color='g')
y_bar += [yb]
for r in range(2):
for c in range(8):
ant_index = (16*i) + (8*r) + c
axes += [plt.subplot(gs[(r, c + 9)])]
axes[ant_index].cla()
axes[ant_index].set_xlim(0, 400)
axes[ant_index].set_ylim(0, 50)
if c:
axes[ant_index].get_yaxis().set_visible(False)
else:
axes[ant_index].set_yticks([0, 10, 20, 30, 40, 50])
axes[ant_index].set_yticklabels([0, 10, 20, 30, 40, 50], fontsize=8)
axes[ant_index].set_ylabel("dB", fontsize=10)
if r:
axes[ant_index].set_xticks([100, 200, 300, 400])
axes[ant_index].set_xticklabels([100, 200, 300, 400], fontsize=8, rotation=45)
axes[ant_index].set_xlabel("MHz", fontsize=10)
else:
axes[ant_index].set_xticks([100, 200, 300, 400])
axes[ant_index].set_xticklabels(["", "", "", ""], fontsize=1)
axes[ant_index].set_title(ants[(16*i) + remap[(8*r) + c]], fontsize=10)
xl, = axes[ant_index].plot(asse_x, range(512), color='b')
x_lines += [xl]
yl, = axes[ant_index].plot(asse_x, range(512), color='g')
y_lines += [yl]
# Draw antenna positions
t_axes[i][1].plot(float(x[ant_index]), float(y[ant_index]), marker='+', markersize=4,
linestyle='None', color='k')
all_data = np.zeros((512, nof_tiles * 16, 2, 1))
tile_acq_timestamp = []
current_day = "2019-05-01"
while not stop_plotting:
# if not os.path.exists(LOCK_FILE):
# Wait for a while
sleep(cadence)
try:
# Connect to the station
_connect_station(aavs_station)
# Read latest spectra
tile_rms = []
for i in range(nof_tiles):
# Grab tile data
data, timestamps = file_manager.read_data(tile_id=i, n_samples=1, sample_offset=-1)
all_data[:, i * 16 : (i + 1) * 16, :, :] = data
# Grab antenna RMS
tile_rms.extend(aavs_station.tiles[i].get_adc_rms())
#asse_x_secs = []
timestamp_day = datetime.datetime.strftime(datetime.datetime.utcfromtimestamp(timestamps[0][0]), "%Y-%m-%d")
if not current_day == timestamp_day:
current_day = timestamp_day
tile_acq_timestamp = [int(timestamps[0][0])]
# asse_x_secs = [(datetime.datetime.utcfromtimestamp(tile_acq_timestamp[-1]) -
# datetime.datetime.utcfromtimestamp(tile_acq_timestamp[-1]).replace(hour=0,
# minute=0,
# second=0,
# microsecond=0)).seconds]
if not os.path.isdir(img_dir + station_name + "/" + current_day):
os.mkdir(img_dir + station_name + "/" + current_day)
else:
tile_acq_timestamp += [int(timestamps[0][0])]
# asse_x_secs += [(datetime.datetime.utcfromtimestamp(tile_acq_timestamp[-1]) -
# datetime.datetime.utcfromtimestamp(tile_acq_timestamp[-1]).replace(hour=0,
# minute=0,
# second=0,
# microsecond=0)).seconds]
t_timestamp = datetime.datetime.strftime(datetime.datetime.utcfromtimestamp(tile_acq_timestamp[-1]), "%Y-%m-%d %H:%M:%S UTC")
for i in range(nof_tiles):
tstamp_label[i].set_text(t_timestamp)
for r in range(2):
for c in range(8):
ant_index = (16*i) + (8*r) + c
remapped_ant = (16*i) + remap[(8*r) + c]
singolo = all_data[:, remapped_ant: remapped_ant + 1, 0, 0]
with np.errstate(divide='ignore'):
x_lines[ant_index].set_ydata(10*np.log10(singolo))
singolo = all_data[:, remapped_ant: remapped_ant + 1, 1, 0]
with np.errstate(divide='ignore'):
y_lines[ant_index].set_ydata(10*np.log10(singolo))
x_bar[i][(8*r) + c].set_height(tile_rms[remapped_ant * 2])
y_bar[i][(8*r) + c].set_height(tile_rms[remapped_ant * 2 + 1])
fig.canvas.draw()
fname = img_dir + station_dir + station_file
fig.savefig(fname)
logging.info("Generated plots for timestamp " + t_timestamp + " on " + fname)
except:
logging.warning("Something went wrong plotting timestamp " + t_timestamp + " ...skipping...")
logging.warning("Tile RMS len: "+str(len(tile_rms)))
logging.warning("Wait for a minute to automatic restart...")
sleep(60)
pass
tiles = [1, 2, 3]
tile_names = ["7", "11", "16"]
else:
if opts.tile == "":
tiles = range(16)
else:
tiles = [int(t) for t in opts.tile.split(",")]
directory = opts.directory
if not opts.lab:
directory += station_name.lower()
print "\nStation Name: ", station_name
print "Checking directory: ", directory + "\n"
print "Looking for tiles: ", tiles, "\n"
file_manager = ChannelFormatFileManager(root_path=directory,
daq_mode=FileDAQModes.Integrated)
da = tstamp_to_fname(t_start)[:-6]
date_path = da[:4] + "-" + da[4:6] + "-" + da[6:]
if opts.channel == "":
xmin = closest(asse_x, float(opts.startfreq))
xmax = closest(asse_x, float(opts.stopfreq))
else:
xmin = int(opts.channel)
xmax = int(opts.channel)
if xmin == xmax:
print "Using channel #" + str(xmin) + " (Freq: " + str(
asse_x[xmin]) + ")"
else:
print "Using channels from #" + str(xmin) + " (Freq: " + str(asse_x[xmin]) + ") to #" + str(xmax) + \