''' Read "real-time" data file once and obtain the median over antenna

and frequency, appending the result to /data1/RT/RT_<date>.txt.

Returns ut times in plot_date format and median values.

'''

tp = np.zeros((100,16))

amp = np.zeros((100,16))

f = open('/data1/RT/RT_latest.txt','r')

lines = f.readlines()

f.close()

mjd = Time.now().mjd - 0.5

datstr = Time(mjd,format='mjd').iso[:10]

tstr = lines[1].split(':')[1][1:7]

tstr = tstr[:2]+':'+tstr[2:4]+':'+tstr[4:]

print datstr+' '+tstr+' ('+Time.now().iso[:19]+')'

t = Time(datstr+' '+tstr)

try:

# Read last line of output and compare with input. Skip if the same.

f = open('/data1/RT/RT_'+datstr+'.txt','r')

lastline = f.readlines()[-1]

f.close()

lastt = Time(lastline[:19])

if lastt == t:

# Times are the same, so do not write to output file

return

except:

pass

# Read entire file and form median values of TP and AMP

for i,line in enumerate(lines[10::10]):

tp[i] = np.array(line.split()).astype(float)

for i,line in enumerate(lines[11::10]):

amp[i] = np.array(line.split()).astype(float)

tpmed = np.median(tp[:,:13])

ampmed = np.median(amp[:,:13])

# Open output file for appending, write median values, and close.

f = open('/data1/RT/RT_'+datstr+'.txt','a')

f.write(t.iso+' {:8.4f} {:8.4f}'.format(tpmed,ampmed)+'\n')

''' Read all projects from SQL for the current date and return a summary

as a dictionary with keys Timestamp, Project, and EOS (another timestamp)

'''

if nosql == True:

return get_projects_nosql(t)

import dbutil

# timerange is 12 UT to 12 UT on next day, relative to the day in Time() object t

trange = Time([int(t.mjd) + 12./24,int(t.mjd) + 36./24],format='mjd')

tstart, tend = trange.lv.astype('str')

cursor = dbutil.get_cursor()

mjd = t.mjd

# Get the project IDs for scans during the period

verstrh = dbutil.find_table_version(cursor,trange[0].lv,True)

if verstrh is None:

print 'No scan_header table found for given time.'

return {}

query = 'select Timestamp,Project from hV'+verstrh+'_vD1 where Timestamp between '+tstart+' and '+tend+' order by Timestamp'

projdict, msg = dbutil.do_query(cursor, query)

if msg != 'Success':

print msg

return {}

elif len(projdict) == 0:

# No Project ID found, so return data and empty projdict dictionary

print 'SQL Query was valid, but no Project data were found.'

return {}

projdict['Timestamp'] = projdict['Timestamp'].astype('float') # Convert timestamps from string to float

for i in range(len(projdict['Project'])): projdict['Project'][i] = projdict['Project'][i].replace('\x00','')

projdict.update({'EOS':projdict['Timestamp'][1:]})

projdict.update({'Timestamp':projdict['Timestamp'][:-1]})

projdict.update({'Project':projdict['Project'][:-1]})

cursor.close()

''' Read all projects from FDB file for the current date and return a summary

as a dictionary with keys Timestamp, Project, and EOS (another timestamp)

'''

import dump_tsys as dt

# timerange is 12 UT to 12 UT on next day, relative to the day in Time() object t

trange = Time([int(t.mjd) + 12./24,int(t.mjd) + 36./24],format='mjd')

tstart = t.iso[2:10].replace('-','')+'120000'

t2 = Time(int(t.mjd)+1, format='mjd')

tend = t2.iso[2:10].replace('-','')+'120000'

fdb = dt.rd_fdb(t)

fdb2 = dt.rd_fdb(t2)

if fdb == {}:

# No FDB file found, so return empty project dictionary

print 'No Project data [FDB file] found for the given date.'

return {}

if fdb2 == {}:

pass

else:

# Concatenate the two dicts into one

if fdb['FILE'][-1] == '':

fdb = dict([(k, np.concatenate((fdb.get(k,[])[:-1],fdb2.get(k,[])))) for k in set(fdb)|set(fdb2)])

else:

fdb = dict([(k, np.concatenate((fdb.get(k,[]),fdb2.get(k,[])))) for k in set(fdb)|set(fdb2)])

# Get "good" indexes for times between 12 UT on date and 12 UT on next date

gidx, = np.where(np.logical_and(fdb['SCANID']>tstart,fdb['SCANID']<tend))

scanid,idx = np.unique(fdb['SCANID'][gidx],return_index=True)

sidx = gidx[idx] # Indexes into fdb for the start of each scan

eidx = np.concatenate((sidx[1:]-1,np.array([gidx[-1]]))) # Indexes into fdb for the end of each scan

# Get the project IDs for scans during the period

projdict = {'Timestamp':fdb['ST_TS'][sidx].astype(float),

'Project':fdb['PROJECTID'][sidx],

'EOS':fdb['EN_TS'][eidx].astype(float)}

''' Get all front-end power-detector voltages for the given day

from the stateframe SQL database, and obtain the median of them,

to use as a flare monitor.

Returns ut times in plot_date format and median voltages.

'''

import dbutil

# timerange is 12 UT to 12 UT on next day, relative to the day in Time() object t

trange = Time([int(t.mjd) + 12./24,int(t.mjd) + 36./24],format='mjd')

tstart, tend = trange.lv.astype('str')

cursor = dbutil.get_cursor()

mjd = t.mjd

verstr = dbutil.find_table_version(cursor,tstart)

if verstr is None:

print 'No stateframe table found for given time.'

return tstart, [], {}

query = 'select Timestamp,Ante_Fron_FEM_HPol_Voltage,Ante_Fron_FEM_VPol_Voltage from fV'+verstr+'_vD15 where timestamp between '+tstart+' and '+tend+' order by timestamp'

data, msg = dbutil.do_query(cursor, query)

if msg != 'Success':

print msg

return tstart, [], {}

for k,v in data.items():

data[k].shape = (len(data[k])/15,15)

hv = []

try:

ut = Time(data['Timestamp'][:,0].astype('float'),format='lv').plot_date

except:

print 'Error for time',t.iso

print 'Query:',query,' returned msg:',msg

print 'Keys:', data.keys()

print data['Timestamp'][0,0]

hfac = np.median(data['Ante_Fron_FEM_HPol_Voltage'].astype('float'),0)

vfac = np.median(data['Ante_Fron_FEM_VPol_Voltage'].astype('float'),0)

for i in range(4):

if hfac[i] > 0:

hv.append(data['Ante_Fron_FEM_HPol_Voltage'][:,i]/hfac[i])

if vfac[i] > 0:

hv.append(data['Ante_Fron_FEM_VPol_Voltage'][:,i]/vfac[i])

#import pdb; pdb.set_trace()

flm = np.median(np.array(hv),0)

good = np.where(abs(flm[1:]-flm[:-1])<0.1)[0]

projdict = get_projects(t)

''' Given the time as a Time object, search the FDB file for files

associated with the scan for that time and create a dynamic spectrogram

on the axis specified by ax, or on a new plot if no ax. If the requested

time is more than 10 minutes after the last file of that scan, returns

None to indicate no plot.

Skip SK flagging [2017-Mar-20 DG]

'''

import time, os

import dump_tsys

#import get_X_data2 as gd

import read_idb as ri

import spectrogram_fit as sp

fdb = dump_tsys.rd_fdb(t)

# Get files from next day, in case scan extends past current day

t1 = Time(t.mjd + 1,format='mjd')

fdb1 = dump_tsys.rd_fdb(t1)

# Concatenate the two days (if the second day exists)

if fdb1 != {}:

for key in fdb.keys():

fdb[key] = np.concatenate((fdb[key],fdb1[key]))

# Find unique scan IDs

scans, idx = np.unique(fdb['SCANID'],return_index=True)

# Limit to scans in 'NormalObserving' mode

good, = np.where(fdb['PROJECTID'][idx] == 'NormalObserving')

if len(good) > 0:

scans = scans[good]

else:

print 'No NormalObserving scans found.'

return None, None, None, None

# Find scanID that starts earlier than, but closest to, the current time

for i,scan in enumerate(scans):

print scan

dt = t - Time(time.strftime('%Y-%m-%d %H:%M:%S',time.strptime(scan,'%y%m%d%H%M%S')))

if dt.sec > 0.:

iout = i

scan = scans[iout]

# Find files for this scan

fidx, = np.where(fdb['SCANID'] == scan)

tlevel = None

bflag = None

if len(fidx) > 0:

files = fdb['FILE'][fidx]

# Find out how old last file of this scan is, and proceed only if less than 20 minutes

# earlier than the time given in t.

try:

dt = t - Time(time.strftime('%Y-%m-%d %H:%M:%S',time.strptime(files[-1],'IDB%Y%m%d%H%M%S')))

except:

dt = 10000. # Forces skip of plot creation

print 'Unexpected FDB file format.'

scan = None

if dt.sec < 1200.:

# This is a currently active scan, so create the figure

path = '/data1/IDB/'

if not os.path.isdir(path+files[0]):

# Look in /dppdata1

datstr = t.iso[:10].replace('-','')

# path = '/data1/eovsa/fits/IDB/'+datstr+'/'

path = get_idbdir(t)+datstr+'/'

if not os.path.isdir(path+files[0]):

print 'No files found for this scan ID',scan

scan = None

times = None

return scan, tlevel, bflag, times

filelist = files

files = []

for i,file in enumerate(filelist):

files.append(path+file)

# data, uvw, fghz, times = gd.get_X_data(files)

out = ri.read_idb(files)

print 'Files read at',Time.now()

print files

#out = ri.flag_sk(out) # Skip flagging for sk

fghz = out['fghz']

times = Time(out['time'],format='jd')

data = out['x']

if ax is not None:

datstr = times[0].iso[:10]

ax.set_xlabel('Time [UT on '+datstr+']')

ax.set_ylabel('Frequency [GHz]')

ax.set_title('EOVSA Summed Cross-Correlation Amplitude for '+datstr)

pdata = np.nansum(np.nansum(np.abs(data[0:11,:]),1),0) # Spectrogram to plot

X = np.sort(pdata.flatten()) # Sorted, flattened array

dmax = X[int(len(X)*0.95)] # Clip at 5% of points

sp.plot_spectrogram(fghz, times, pdata,

ax=ax, logsample=None, xdata=True, cbar=True, dmax=dmax)

#tlevel, bflag = flaremeter(data)

else:

print 'Time',dt.sec,'is > 1200 s after last file of last NormalObserving scan. No plot created.'

scan = None

times = None

else:

print 'No files found for this scan ID',scan

scan = None

''' Obtain median of data across baselines, polarizations, and frequencies to create a

time series indicated whether a flare has occurred. Values returned will be close

to unity if no flare. Returns:

tlevel: Array of levels at each time, nominally near unity

bflag: Array of flags indicating nominal background (where True) or

elevated background (where False) indicating possible flare

'''

nbl,npol,nf,nt = data.shape

tlevel = np.zeros(nt,'float')

background = np.sqrt(np.abs(data[:,0,:,:])**2 + np.abs(data[:,1,:,:])**2)

init_bg = np.nanmedian(background,2) # Initially take background as median over entire time range

bflag = np.ones(nt,'bool') # flags indicating "good" background times (not in flare)

for i in range(nt):

good, = np.where(bflag[:i] == True) # List of indexes of good background times up to current time

ngood = len(good) # Truncate list of indexes to last 100 elements (or fewer)

if ngood > 100:

good = good[ngood-100:]

# Calculate median over good background times

bg = np.nanmedian(background[:,:,good],2)

else:

# If there haven't been 100 times with good backgrounds yet, just use the initial one.

# This is supposed to avoid startup transients.

bg = init_bg

# Generate levels for each baseline and frequency for this time

level = np.sqrt(abs(data[:,0,:,i])**2 + abs(data[:,1,:,i])**2)/bg

# Take median over baseline and frequency to give a single number for this time

tlevel[i] = np.nanmedian(level)

if tlevel[i] > 1.05:

# If the level of the current time is higher than 1.05, do not include this time in future backgrounds

bflag[i] = False

''' Cleans up the background flag array to remove rapid fluctuations

and provide better in-flare designations.

'''

''' Given newly determined tlevel and bflag, see if a file already

exists for this date and append or replace with new information,

if so, otherwise create a new file.

File created is a binary file of records, 9 bytes per record:

float time, float tlevel, bool bflag

Returns data for entire day (contents of any existing file plus

the new data)

'''

import glob

import dump_tsys

import struct

datstr = times[0].iso[:10].replace('-','')

filename = '/common/webplots/flaremon/flaremeter/FLM'+datstr+'.dat'

if len(glob.glob(filename)) == 1:

# Filename exists, so read entire file at once

f = open(filename,'rb')

buf = f.read()

f.close()

nrec = len(buf)/13 # 13 bytes per record: double time, float level, bool flag

t = np.zeros(nrec,'double')

l = np.zeros(nrec,'float')

b = np.zeros(nrec,'bool')

for i in range(nrec):

t[i],l[i],b[i] = struct.unpack('dfB',buf[i*13:(i+1)*13])

# Since unique also sorts, and takes the first instance, it should be enough to

# concatenate times with t and get unique indexes

times_lv = np.concatenate(((times.lv+0.5).astype('int'), (t+0.5).astype('int')))

tlevel = np.concatenate((tlevel, l))

bflag = np.concatenate((bflag, b))

blah, idx = np.unique(times_lv,return_index=True)

times = Time(times_lv[idx],format='lv')

tlevel = tlevel[idx]

bflag = bflag[idx]

# Open same filename for writing (overwrites contents if file exists)

f = open(filename,'wb')

for i in range(len(times)):

f.write(struct.pack('dfB',*(times[i].lv,tlevel[i],bflag[i])))

f.close()

import os

from util import Time

if filename is None:

datstr = Time.now().iso[:10]

filename = '/data1/RT/RT_'+datstr+'.txt'

if not os.path.exists(filename):

print 'File',filename,'not found.'

return {}

f = open(filename,'r')

lines = f.readlines()

f.close()

t = []

amp = []

tp = []

from util import Time

for line in lines:

t.append(Time(line[:19]).plot_date)

tp.append(float(line[24:31]))

amp.append(float(line[32:-3]))

t = np.array(t)

dt = t[1:] - t[:-1]

jmp, = np.where(dt < -0.75)

if len(jmp) == 1:

# This is a day jump, so add one to later times

t[jmp[0]+1:] += 1