''' Get all front-end power-detector voltages for the given day

from the stateframe SQL database, and obtain the median of them,

to use as a flare monitor.

Returns ut times in plot_date format and median voltages.

'''

import dbutil

# timerange is 13 UT to 02 UT on next day, relative to the day in Time() object t

trange = Time([int(t.mjd) + 12./24,int(t.mjd) + 26./24],format='mjd')

tstart, tend = trange.lv.astype('str')

cursor = dbutil.get_cursor()

mjd = t.mjd

try:

verstr = dbutil.find_table_version(cursor,tstart)

if verstr is None:

print 'No stateframe table found for given time.'

return tstart, [], {}

cursor.execute('select * from fV'+verstr+'_vD15 where I15 < 4 and timestamp between '+tstart+' and '+tend+' order by timestamp')

except:

print 'Error with query of SQL database.'

return tstart, [], {}

data = np.transpose(np.array(cursor.fetchall(),'object'))

if len(data) == 0:

# No data found, so return timestamp and empty lists

print 'SQL Query was valid, but no data for',t.iso[:10],'were found (yet).'

return tstart, [], {}

ncol,ntot = data.shape

data.shape = (ncol,ntot/4,4)

names = np.array(cursor.description)[:,0]

mydict = dict(zip(names,data))

hv = []

ut = Time(mydict['Timestamp'][:,0].astype('float'),format='lv').plot_date

hfac = np.median(mydict['Ante_Fron_FEM_HPol_Voltage'].astype('float'),0)

vfac = np.median(mydict['Ante_Fron_FEM_VPol_Voltage'].astype('float'),0)

for i in range(4):

if hfac[i] > 0:

hv.append(mydict['Ante_Fron_FEM_HPol_Voltage'][:,i]/hfac[i])

if vfac[i] > 0:

hv.append(mydict['Ante_Fron_FEM_VPol_Voltage'][:,i]/vfac[i])

flm = np.median(np.array(hv),0)

good = np.where(abs(flm[1:]-flm[:-1])<0.01)[0]

# Get the project IDs for scans during the period

verstrh = dbutil.find_table_version(cursor,trange[0].lv,True)

if verstrh is None:

print 'No scan_header table found for given time.'

return ut[good], flm[good], {}

cursor.execute('select Timestamp,Project from hV'+verstrh+'_vD1 where Timestamp between '+tstart+' and '+tend+' order by Timestamp')

data = np.transpose(np.array(cursor.fetchall()))

names = np.array(cursor.description)[:,0]

if len(data) == 0:

# No Project ID found, so return data and empty projdict dictionary

print 'SQL Query was valid, but no Project data were found.'

return ut[good], flm[good], {}

projdict = dict(zip(names,data))

projdict['Timestamp'] = projdict['Timestamp'].astype('float') # Convert timestamps from string to float

# Get the times when scanstate is -1

cursor.execute('select Timestamp,Sche_Data_ScanState from fV'+verstr+'_vD1 where Timestamp between '+tstart+' and '+tend+' and Sche_Data_ScanState = -1 order by Timestamp')

scan_off_times = np.transpose(np.array(cursor.fetchall()))[0] #Just list of timestamps

if len(scan_off_times) > 2:

gaps = scan_off_times[1:] - scan_off_times[:-1] - 1

eos = np.where(gaps > 10)[0]

if len(eos) > 1:

if scan_off_times[eos[1]] < projdict['Timestamp'][0]:

# Gaps are not lined up, so drop the first:

eos = eos[1:]

EOS = scan_off_times[eos]

if scan_off_times[eos[0]] <= projdict['Timestamp'][0]:

# First EOS is earlier than first Project ID, so make first Project ID None.

projdict['Timestamp'] = np.append([scan_off_times[0]],projdict['Timestamp'])

projdict['Project'] = np.append(['None'],projdict['Project'])

if scan_off_times[eos[-1]+1] >= projdict['Timestamp'][-1]:

# Last EOS is later than last Project ID, so make last Project ID None.

projdict['Timestamp'] = np.append(projdict['Timestamp'],[scan_off_times[eos[-1]+1]])

projdict['Project'] = np.append(projdict['Project'],['None'])

EOS = np.append(EOS,[scan_off_times[eos[-1]+1],scan_off_times[-1]])

projdict.update({'EOS': EOS})

else:

# Not enough scan changes to determine EOS (end-of-scan) times

projdict.update({'EOS': []})

cursor.close()

''' Given the time as a Time object, search the FDB file for files

associated with the scan for that time and create a dynamic spectrogram

on the axis specified by ax, or on a new plot if no ax. If the requested

time is more than 10 minutes after the last file of that scan, returns

None to indicate no plot.

Skip SK flagging [2017-Mar-20 DG]

'''

import time, os

import dump_tsys

#import get_X_data2 as gd

import read_idb as ri

import spectrogram_fit as sp

fdb = dump_tsys.rd_fdb(t)

# Get files from next day, in case scan extends past current day

t1 = Time(t.mjd + 1,format='mjd')

fdb1 = dump_tsys.rd_fdb(t1)

# Concatenate the two days (if the second day exists)

if fdb1 != {}:

for key in fdb.keys():

fdb[key] = np.concatenate((fdb[key],fdb1[key]))

# Find unique scan IDs

scans, idx = np.unique(fdb['SCANID'],return_index=True)

# Limit to scans in 'NormalObserving' mode

good, = np.where(fdb['PROJECTID'][idx] == 'NormalObserving')

if len(good) > 0:

scans = scans[good]

else:

print 'No NormalObserving scans found.'

return None, None, None

# Find scanID that starts earlier than, but closest to, the current time

for i,scan in enumerate(scans):

dt = t - Time(time.strftime('%Y-%m-%d %H:%M:%S',time.strptime(scan,'%y%m%d%H%M%S')))

if dt.sec > 0.:

iout = i

scan = scans[iout]

# Find files for this scan

fidx, = np.where(fdb['SCANID'] == scan)

tlevel = None

bflag = None

if len(fidx) > 0:

files = fdb['FILE'][fidx]

# Find out how old last file of this scan is, and proceed only if less than 20 minutes

# earlier than the time given in t.

try:

dt = t - Time(time.strftime('%Y-%m-%d %H:%M:%S',time.strptime(files[-1],'IDB%Y%m%d%H%M%S')))

except:

dt = 10000. # Forces skip of plot creation

print 'Unexpected FDB file format.'

scan = None

if dt.sec < 1200.:

# This is a currently active scan, so create the figure

path = '/data1/IDB/'

if not os.path.isdir(path+files[0]):

# Look in /dppdata1

datstr = t.iso[:10].replace('-','')

path = '/data1/eovsa/fits/IDB/'+datstr+'/'

if not os.path.isdir(path+files[0]):

print 'No files found for this scan ID',scan

scan = None

return scan, tlevel, bflag, times

filelist = files

files = []

for i,file in enumerate(filelist):

files.append(path+file)

# data, uvw, fghz, times = gd.get_X_data(files)

out = ri.read_idb(files)

#out = ri.flag_sk(out) # Skip flagging for sk

fghz = out['fghz']

times = Time(out['time'],format='jd')

data = out['x']

if ax is not None:

datstr = times[0].iso[:10]

ax.set_xlabel('Time [UT on '+datstr+']')

ax.set_ylabel('Frequency [GHz]')

ax.set_title('EOVSA Summed Cross-Correlation Amplitude for '+datstr)

pdata = np.sum(np.sum(np.abs(data[0:11,:]),1),0) # Spectrogram to plot

X = np.sort(pdata.flatten()) # Sorted, flattened array

dmax = X[int(len(X)*0.95)] # Clip at 5% of points

sp.plot_spectrogram(fghz, times, pdata,

ax=ax, logsample=None, xdata=True, cbar=True, dmax=dmax)

#tlevel, bflag = flaremeter(data)

else:

print 'Time',dt.sec,'is > 1200 s after last file of last NormalObserving scan. No plot created.'

scan = None

else:

print 'No files found for this scan ID',scan

scan = None

''' Obtain median of data across baselines, polarizations, and frequencies to create a

time series indicated whether a flare has occurred. Values returned will be close

to unity if no flare. Returns:

tlevel: Array of levels at each time, nominally near unity

bflag: Array of flags indicating nominal background (where True) or

elevated background (where False) indicating possible flare

'''

nbl,npol,nf,nt = data.shape

tlevel = np.zeros(nt,'float')

background = np.sqrt(np.abs(data[:,0,:,:])**2 + np.abs(data[:,1,:,:])**2)

init_bg = np.nanmedian(background,2) # Initially take background as median over entire time range

bflag = np.ones(nt,'bool') # flags indicating "good" background times (not in flare)

for i in range(nt):

good, = np.where(bflag[:i] == True) # List of indexes of good background times up to current time

ngood = len(good) # Truncate list of indexes to last 100 elements (or fewer)

if ngood > 100:

good = good[ngood-100:]

# Calculate median over good background times

bg = np.nanmedian(background[:,:,good],2)

else:

# If there haven't been 100 times with good backgrounds yet, just use the initial one.

# This is supposed to avoid startup transients.

bg = init_bg

# Generate levels for each baseline and frequency for this time

level = np.sqrt(abs(data[:,0,:,i])**2 + abs(data[:,1,:,i])**2)/bg

# Take median over baseline and frequency to give a single number for this time

tlevel[i] = np.nanmedian(level)

if tlevel[i] > 1.05:

# If the level of the current time is higher than 1.05, do not include this time in future backgrounds

bflag[i] = False

''' Cleans up the background flag array to remove rapid fluctuations

and provide better in-flare designations.

'''

''' Given newly determined tlevel and bflag, see if a file already

exists for this date and append or replace with new information,

if so, otherwise create a new file.

File created is a binary file of records, 9 bytes per record:

float time, float tlevel, bool bflag

Returns data for entire day (contents of any existing file plus

the new data)

'''

import glob

import dump_tsys

import struct

datstr = times[0].iso[:10].replace('-','')

filename = '/common/webplots/flaremon/flaremeter/FLM'+datstr+'.dat'

if len(glob.glob(filename)) == 1:

# Filename exists, so read entire file at once

f = open(filename,'rb')

buf = f.read()

f.close()

nrec = len(buf)/13 # 13 bytes per record: double time, float level, bool flag

t = np.zeros(nrec,'double')

l = np.zeros(nrec,'float')

b = np.zeros(nrec,'bool')

for i in range(nrec):

t[i],l[i],b[i] = struct.unpack('dfB',buf[i*13:(i+1)*13])

# Since unique also sorts, and takes the first instance, it should be enough to

# concatenate times with t and get unique indexes

times_lv = np.concatenate(((times.lv+0.5).astype('int'), (t+0.5).astype('int')))

tlevel = np.concatenate((tlevel, l))

bflag = np.concatenate((bflag, b))

blah, idx = np.unique(times_lv,return_index=True)

times = Time(times_lv[idx],format='lv')

tlevel = tlevel[idx]

bflag = bflag[idx]

# Open same filename for writing (overwrites contents if file exists)

f = open(filename,'wb')

for i in range(len(times)):

f.write(struct.pack('dfB',*(times[i].lv,tlevel[i],bflag[i])))

f.close()