# read in LOFAR calibrated pulse block, if desired with debug-lofar-pulse option
lofar_pulse = None
if options.debug_lofar_pulse:
options.debug_test_pulse = True # if not set, set to True
# Read database info, for crp_plotfiles key which contains the most actual results directory (i.e. the one used in the database)
import psycopg2 # for changing status in database
from pycrtools import crdatabase as crdb
dbManager = crdb.CRDatabase("crdb",
host="coma00.science.ru.nl",
user="******",
password="******",
dbname="crdb")
db = dbManager.db
print '### Replacing simulated data by LOFAR data processed by pipeline (calibrated-xyz) ###'
print 'Reading event data, id = %d ...' % eventid
event_data = crdb.Event(db=db, id=eventid)
#event_data.stations[0].polarization['0']["crp_integrated_pulse_power"]
station = [st for st in event_data.stations
if st.stationname == 'CS002'][0]
crp_integrated_pulse_power_01 = station.polarization['0'][
'crp_integrated_pulse_power']
crp_integrated_pulse_power_xyz = station.polarization['xyz'][
'crp_integrated_pulse_power']
crp_plotfiles = event_data["crp_plotfiles"]
results_dir = os.path.split(crp_plotfiles[0])[0]
print 'Reading in timeseries data from directory %s' % results_dir
print '(for now, reading in one antenna from CS002 and copying...)'
xyz_timeseries = np.load(
os.path.join(results_dir,
'xyz_calibrated_pulse_block-%d-CS002.npy' % eventid))
ldf_fit_energy_particle = np.zeros([nEvents])
lora_energy = np.zeros([nEvents])
lora_elevation = np.zeros([nEvents])
dbManager = crdb.CRDatabase(host='coma00.science.ru.nl',
user='******',
password='******',
dbname='crdb')
db = dbManager.db
for i in np.arange(15):
event_id = int(list1[i])
print event_id
try:
event = crdb.Event(db=db, id=event_id)
ldf_fit_energy[i] = event["ldf_fit_energy"]
ldf_fit_energy_particle[i] = event["ldf_fit_energy_particle"]
lora_elevation[i] = (90 - event["lora_elevation"]) * np.pi / 180.0
lora_energy[i] = event["lora_energy"]
except:
print '...... no data'
outfile_name = 'db_energy_info.dat'
analysis_info = {
'ldf_fit_energy': ldf_fit_energy,
'ldf_fit_energy_particle': ldf_fit_energy_particle,
'lora_energy': lora_energy,
def get_noise(event_id):
nstations=0
try:
#for u in np.arange(1):
event = crdb.Event(db=db, id=event_id)
event_time=np.asarray(event["lora_time"])
event_time=event_time[event_time>1.0]
event_time=np.min(event_time)
#if event_time<100.0:
# break
print(event_time)
time = Time(event_time, format='unix', scale='utc',location=loc)
time.delta_ut1_utc = 0.
LST=time.sidereal_time('apparent').hour
print('event time UTC: {0}'.format(event_time))
print('event time LST: {0}'.format(LST))
stations = []
#collect stations with "GOOD" status for event
for f in event.datafiles:
stn=[]
stn.extend(f.stations)
#print stn[0].stationname
#print f.stations.stationname
if stn[0].stationname == "CS001" or stn[0].stationname == "CS002" or stn[0].stationname == "CS003" or stn[0].stationname == "CS004" or stn[0].stationname == "CS005" or stn[0].stationname == "CS006" or stn[0].stationname == "CS007" or stn[0].stationname == "CS011" or stn[0].stationname == "CS013" or stn[0].stationname == "CS017" or stn[0].stationname == "CS021" or stn[0].stationname == "CS026" or stn[0].stationname == "CS028" or stn[0].stationname == "CS030" or stn[0].stationname == "CS031" or stn[0].stationname == "CS032" or stn[0].stationname == "CS101" or stn[0].stationname == "CS103" or stn[0].stationname == "CS301" or stn[0].stationname == "CS302" or stn[0].stationname == "CS401" or stn[0].stationname == "CS501":
if stn[0].status=="GOOD":
stations.extend(f.stations)
nstations=len(stations)
except:
print('no event at this point')
for s in np.arange(nstations):
station_flag=0
station=stations[s]
# The following steps are copied from cr_physics pipeline
# there are a million try/excepts because I ran into lots of specific errors I didn't want to handle
try:
# Open file
f = cr.open(station.datafile.settings.datapath + '/' + station.datafile.filename)
antenna_set= f["ANTENNA_SET"]
# Check if we are dealing with LBA or HBA observations
if "LBA" in f["ANTENNA_SET"]:
print ('LBA event')
else:
print ('HBA event')
continue
except:
print ('no event at antennas')
continue
# Read LORA information
try:
tbb_time = f["TIME"][0]
max_sample_number = max(f["SAMPLE_NUMBER"])
min_sample_number = min(f["SAMPLE_NUMBER"])
(tbb_time_sec, tbb_time_nsec) = lora.nsecFromSec(tbb_time, logfile=os.path.join(lora_directory,lora_logfile))
(block_number_lora, sample_number_lora) = lora.loraTimestampToBlocknumber(tbb_time_sec, tbb_time_nsec, tbb_time, max_sample_number, blocksize=blocksize)
except:
continue
# Check if starting time in sample units (SAMPLE_NUMBER) does not deviate among antennas
try:
sample_number_per_antenna = np.array(f["SAMPLE_NUMBER"])
median_sample_number = np.median(sample_number_per_antenna)
data_length = np.median(np.array(f["DATA_LENGTH"]))
deviating_antennas = np.where( np.abs(sample_number_per_antenna - median_sample_number) > data_length/4)[0]
nof_deviating_antennas = len(deviating_antennas)
print ('Number of deviating antennas: %d' % nof_deviating_antennas)
except:
continue
try:
frequencies = f["FREQUENCY_DATA"]
print ('blocksize: {0}'.format(f["BLOCKSIZE"]))
# Get bandpass filter
nf = f["BLOCKSIZE"] / 2 + 1
ne = int(10. * nf / f["CLOCK_FREQUENCY"])
bandpass_filter.fill(0.)
bandpass_filter[int(nf * 30.0 / 100.)-(ne/2):int(nf * 80.0 / 100.)+(ne/2)] = 1.0
gaussian_weights = cr.hArray(cr.hGaussianWeights(ne, 4.0))
cr.hRunningAverage(bandpass_filter, gaussian_weights)
except:
continue
try:
raw_data = f["TIMESERIES_DATA"].toNumpy()
# Find outliers
tmp = np.max(np.abs(raw_data), axis=1)
outlier_antennas = np.argwhere(np.abs(tmp-np.median(tmp[tmp>0.1])) > 2*np.std(tmp[tmp>0.1])).ravel()
print("Outlier antennas", outlier_antennas)
except:
print 'no raw data'
continue
try:
# Get calibration delays to flag antennas with wrong calibration values
try:
cabledelays = cr.hArray(f["DIPOLE_CALIBRATION_DELAY"])
cabledelays = np.abs(cabledelays.toNumpy())
except:
print 'problem with cable delays'
continue
# Find RFI and bad antennas
findrfi = cr.trun("FindRFI", f=f, nofblocks=10, plotlist=[], apply_hanning_window=True, hanning_fraction=0.2, bandpass_filter=bandpass_filter)
print "Bad antennas", findrfi.bad_antennas
antenna_ids_findrfi = f["SELECTED_DIPOLES"]
nAnt=len(f["SELECTED_DIPOLES"])
bad_antennas_spikes = []
bad_antennas = findrfi.bad_antennas[:]
dipole_names = f["SELECTED_DIPOLES"]
good_antennas = [n for n in dipole_names if n not in bad_antennas]
station["crp_bad_antennas_power"] = findrfi.bad_antennas
station["crp_bad_antennas_spikes"] = bad_antennas_spikes
selected_dipoles = []
for i in range(len(dipole_names) / 2):
if dipole_names[2 * i] in good_antennas and dipole_names[2 * i + 1] in good_antennas and f.nof_consecutive_zeros[2 * i] < 512 and f.nof_consecutive_zeros[2 * i + 1] < 512 and cabledelays[2 * i] < 150.e-9 and cabledelays[2 * i + 1] < 150.e-9:
selected_dipoles.extend([dipole_names[2 * i], dipole_names[2 * i + 1]])
f["SELECTED_DIPOLES"] = selected_dipoles
station["crp_selected_dipoles"] = selected_dipoles
nDipoles=len(selected_dipoles)
except:
print 'issue with RFI'
continue
try:
print block_number_lora
nF= len(frequencies.toNumpy())
all_ffts=np.zeros([nAvg,nDipoles,nF])
all_ffts_cleaned=np.zeros([nAvg,nDipoles,nF])
except:
continue
#______________________________________________________________________
block_number=0
for i in np.arange(nAvg):
try:
# make sure not to include the signal window in the average
if abs(block_number-block_number_lora)<5:
block_number=block_number+10
fft_data = f.empty("FFT_DATA")
#f.getFFTData(fft_data, block_number_lora, True, hanning_fraction=0.2, datacheck=True) # this is what is in the pipeline
f.getFFTData(fft_data, block_number, True, hanning_fraction=0.2, datacheck=True)
# Apply bandpass
fft_data[...].mul(bandpass_filter)
# Normalize spectrum
fft_data /= f["BLOCKSIZE"]
fft_hold=fft_data
# Reject DC component
fft_data[..., 0] = 0.0
# Also reject 1st harmonic (gives a lot of spurious power with Hanning window)
fft_data[..., 1] = 0.0
# Flag dirty channels (from RFI excission)
fft_data[..., cr.hArray(findrfi.dirty_channels)] = 0
# factor of two because reall FFT
all_ffts[i]=2*np.abs(fft_hold.toNumpy())**2
all_ffts_cleaned[i]=2*np.abs(fft_data.toNumpy())**2
badFreq=frequencies.toNumpy()[findrfi.dirty_channels]/1e6
nBadChannelsFilt=len(badFreq[(badFreq>=30.0)*(badFreq<=80.0)])
if nBadChannelsFilt>1:
print 'n bad channels: {0}'.format(nBadChannelsFilt)
continue
block_number=block_number+1
except:
print 'error'
continue
try:
#for y in np.arange(1):
fft_avg=np.average(all_ffts_cleaned,axis=0)
freq=frequencies.toNumpy()
df=(freq[1]-freq[0])/1e6
freq_new=np.arange(30,81,1)
fft_resample=np.zeros([nAnt,nResample])
except:
print 'error in average'
continue
for n in np.arange(nDipoles):
try:
fft_use=fft_avg[n][fft_avg[n]>1e-100]
freq_use=freq[fft_avg[n]>1e-100]
if len(fft_avg[n])>len(fft_use):
station_flag=1
f=interp1d(freq_use/1e6,fft_use)
f_new=f(freq_new)
start_f=np.argmin(np.abs((freq/1e6)-30))
stop_f=np.argmin(np.abs((freq/1e6)-80))
fft_resample[n]=f_new*(1/df)
except:
station_flag=1
print 'issue with interp'
analysisinfo={'event_number': event_id,'station': station.stationname,'UTC_time':event_time,'LST':LST,'frequencies':freq,'FFT_data':fft_avg,'frequencies_50':freq_new,'FFT_data_resampled':fft_resample,'flag': station_flag,'antenna_set':antenna_set,'selected_dipoles': dipole_names,'bad_dipoles':bad_antennas,'nBadChannelsFilt':nBadChannelsFilt}
outputfile=open(station.stationname+'/'+str(int(event_id))+'_noise_OUTER.p','w')
pickle.dump(analysisinfo,outputfile)
outputfile.close()
print '{0} done'.format(station.stationname)
print 'done with event'
dbManager = crdb.CRDatabase(db_filename, host=options.host, user=options.user, password=options.password, dbname=options.dbname)
db = dbManager.db
valid_status = ["DESIRED", "CONEX_STARTED", "CONEX_DONE", "COREAS_STARTED", "COREAS_DONE"]
print "skipping conex:", options.skip_conex
print "skipping coreas:", options.skip_coreas
print "skipping analysis:", options.skip_analysis
# Ignore if simulations are already scheduled
if options.id in simhelp.running_jobs(ignore_suspended=options.ignore_suspended_jobs):
print "Event {0} already scheduled, skipping...".format(options.id)
sys.exit(0)
# Get event from database and run pipeline on it
with process_event(crdb.Event(db=db, id=options.id)) as event:
if event.simulation_status not in valid_status:
sys.exit(1)
# Use options
az = options.azimuth
if az is None:
az = event["simulation_direction"][0]
az = pytmf.deg2rad(90. - az) # LOFAR convention has east = 0, north = 90 deg. Convert to 0=North, 90=East for further use
zen = options.elevation
if zen is None:
zen = event["simulation_direction"][1]
zen = pytmf.deg2rad(90. - zen)
def GetLDF(eventno):
# Open CR Database connection to read out data pipeline parameters for this event
# nofAttempts = 3
#thisAttempt = 0
#database_connection = False
#while not database_connection and (thisAttempt < nofAttempts):
# try:
# database_connection = True
# except:
# thisAttempt += 1
# print 'Database connection failed at attempt %d' % thisAttempt
# import time
# time.sleep(20 + 100*np.random.rand() ) # sleep for 20 to 120 seconds before retrying
# if not database_connection:
# raise ValueError("No database connection after {0} attempts!".format(nofAttempts))
dbManager = crdb.CRDatabase(host='coma00.science.ru.nl',
user='******',
password='******',
dbname='crdb')
db = dbManager.db
event = crdb.Event(db=db, id=eventno)
"""gotEvent = False
nofAttempts = 3
thisAttempt = 0
while not gotEvent and (thisAttempt < nofAttempts):
try:
gotEvent = True
except:
thisAttempt += 1
print 'Get event failed at attempt %d' % thisAttempt
import time
time.sleep(20 + 100*np.random.rand() )
if not gotEvent:
raise ValueError("Could not read event data after {0} attempts!".format(nofAttempts))
"""
lora_elevation = 0
try:
energy = event["lora_energy"]
core_x = event["lora_core_x"]
core_y = event["lora_core_y"]
azimuth = event["lora_azimuth"]
lora_elevation = event["lora_elevation"]
moliere = event["lora_moliere"]
lora_x = event["lora_posx"]
lora_y = event["lora_posy"]
lora_dens = event["lora_particle_density__m2"]
if LoraQualityPassed(moliere, core_x, core_y, lora_elevation):
print "GOOD quality event (LORA)", energy, azimuth, lora_elevation, core_x, core_y, moliere
else:
print "LORA Quality not passed", energy, azimuth, lora_elevation, core_x, core_y, moliere
except:
print "Skipping event, no LORA data"
return -1 # This should be tested for when calling the function... Nothing to return for a file without LORA info.
# Loop over all stations in event
stations = []
for f in event.datafiles:
stations.extend(f.stations)
positions = []
selected_dipoles = []
delays = []
amplitude = []
rms = []
power11 = []
power21 = []
power41 = []
noisepower = []
stationname = []
pulse_direction = []
pulse_delay_fit_residual = []
time = []
for station in stations:
if station.status == "GOOD":
try:
p = station.polarization["xyz"]
p0 = station.polarization["0"]
positions.append(station["local_antenna_positions"])
selected_dipoles.append(station["crp_selected_dipoles"])
amplitude.append(p["crp_pulse_peak_amplitude"])
power11.append(p0["crp_integrated_pulse_power"])
power21.append(p0["crp_integrated_pulse_power_wide"])
power41.append(p0["crp_integrated_pulse_power_double_wide"])
noisepower.append(p0["crp_integrated_noise_power"])
rms.append(p["crp_rms"])
stationname.append([station.stationname] * len(p["crp_rms"]))
pulse_direction.append(station["crp_pulse_direction"])
pulse_delay_fit_residual.append(
station["crp_pulse_delay_fit_residual"])
time.append(station["crp_pulse_time"])
except:
print "Do not have all pulse parameters for station", station.stationname
#print positions
positions = np.vstack(positions)
selected_dipoles = np.hstack(selected_dipoles)
amplitude = np.vstack(amplitude)
rms = np.vstack(rms)
power11 = np.vstack(power11)
power21 = np.vstack(power21)
power41 = np.vstack(power41)
noisepower = np.vstack(noisepower)
pulse_delay_fit_residual = np.hstack(pulse_delay_fit_residual)
#stationname = np.array(stationname)
stationname = np.array(sum(stationname, []))
time = np.vstack(time)
#print time[:,0]
lof_azimuth = np.mean(pulse_direction, axis=0)[0]
lof_elevation = np.mean(pulse_direction, axis=0)[1]
shape = positions.shape
positions = positions.reshape((shape[0] / 2, 2, shape[1]))[:, 0]
positions = positions.copy()
#uncer = np.sqrt(rms[:,0]**2+rms[:,1]**2 + rms[:,2]**2)
#total = np.sqrt(amplitude[:,0]*signals[:,0]+signals[:,1]*signals[:,1]+signals[:,2]*signals[:,2])
#print "Returning event:", eventno
dist = GetDistance([core_x, core_y, 0], [azimuth, lora_elevation],
positions)
x_err = GetTotalDistanceUncertainty([core_x, core_y, 0], [5., 5., 0],
positions, [azimuth, lora_elevation],
[2, 2, 0], dist)
dist = np.array(dist)
x_err = np.array(x_err)
#print positions.shape, amplitude.shape, power.shape, rms.shape
return core_x, core_y, stationname, selected_dipoles, positions, dist, x_err, amplitude, power11, power21, power41, rms, noisepower, pulse_delay_fit_residual, time[:, 0], lora_x, lora_y, lora_dens, lof_azimuth, lof_elevation, float(
lora_elevation)
def find_trigger(event_id):
#print 'running event: {0}'.format(event_id)
detector=np.zeros([nDet])
ymd=np.zeros([nDet])
gps=np.zeros([nDet])
ctd=np.zeros([nDet])
nsec=np.zeros([nDet])
trigg_condition=-1*np.ones([nDet])
trigg_pattern=np.zeros([nDet])
total_counts=np.zeros([nDet])
pulse_height=np.zeros([nDet])
pulse_width=np.zeros([nDet])
counts=np.zeros([nDet,nTrace])
on_off=np.zeros([nDet])
local_trigger=-1*np.ones([nLasa])
trigger_setting=-1
dbManager = crdb.CRDatabase("crdb", host="coma00.science.ru.nl",user="******", password="******", dbname="crdb")
db = dbManager.db
event = crdb.Event(db = db, id = event_id)
lora_nsec=event["lora_nsecs"]
lora_utc=event["lora_utc_time_secs"]
#print 'lora utc: {0}'.format(lora_utc)
data=np.genfromtxt(open(on_off_filename,'r'))
logUTC=data.T[0]
on_off_log=data.T[1:21].T
## find closest day to lora utc
idx = (np.abs(logUTC - lora_utc)).argmin()
if logUTC[idx]>lora_utc:
idx=idx-1
on_off= on_off_log[idx]
## find detectors that are on/off daily with
year=time.gmtime(lora_utc).tm_year
month=time.gmtime(lora_utc).tm_mon
day=time.gmtime(lora_utc).tm_mday
# find correct log file for event (daily, sometimes not with a standard name)
log_list= glob.glob(data_dir+'{0:04d}{1:02d}{2:02d}*.log'.format(year,month,day))
log_list.extend(glob.glob(data_dir+'{0:04d}{1:02d}{2:02d}*.log'.format(year,month,day-1)))
if day==1:
log_list.extend(glob.glob(data_dir+'{0:04d}{1:02d}{2:02d}*.log'.format(year,month-1,30)))
log_list.extend(glob.glob(data_dir+'{0:04d}{1:02d}{2:02d}*.log'.format(year,month-1,31)))
if day==1 and month==1:
log_list.extend(glob.glob(data_dir+'{0:04d}{1:02d}{2:02d}*.log'.format(year-1,12,31)))
log_list.extend(glob.glob(data_dir+'{0:04d}{1:02d}{2:02d}*.log'.format(year-1,12,30)))
found_utc=0
for l in np.arange(len(log_list)):
if str(int(lora_utc)) in open(log_list[l]).read():
log_file_name=log_list[l]
root_file_name=log_file_name.split('.')[0]+'.root'
found_utc=1
if found_utc==1:
log_file=open(log_file_name,'r')
root_file=ROOT.TFile.Open(root_file_name)
tree_sec = root_file.Get("Tree_sec")
tree_event = root_file.Get("Tree_event")
tree_log = root_file.Get("Tree_log")
tree_noise = root_file.Get("Tree_noise")
active_lasas,trigger_setting=find_active_stations(log_file)
event_index=find_entry_number(lora_utc,lora_nsec,tree_event)
for d in np.arange(nDet):
detname='Det'+str(d+1)
det=tree_event.GetBranch(detname)
detector[d],ymd[d],gps[d],ctd[d],nsec[d],trigg_condition[d],trigg_pattern[d],total_counts[d],pulse_height[d],pulse_width[d],counts[d]=getData(det,event_index)
if np.max(counts[d])>0.0:
on_off[d]=1
# to get the lasa local trigger condition
#katie: question- how can only 2 detectors have condition 2/4??
lasa1=np.min(trigg_condition[0:4])
lasa2=np.min(trigg_condition[4:8])
lasa3=np.min(trigg_condition[8:12])
lasa4=np.min(trigg_condition[12:16])
lasa5=np.min(trigg_condition[16:20])
local_trigger=np.asarray([lasa1,lasa2,lasa3,lasa4,lasa5])
## change to stricter condition in case of broken files
for t in np.arange(len(local_trigger)):
if local_trigger[t]<0.5:
local_trigger[t]=3.0
print 'event okay'
else:
print 'didn\'t find matching file'
## if there is a problem reading local trigger setting, change everything to strictest condition (3/4)
if (-1 in local_trigger)==True:
local_trigger=3*np.ones([nLasa])
## check if station or detector trigger
## catch weird settings
trigger_type='d'
if (int(trigger_setting)<6.0 and int(trigger_setting)>0.0):
print 'station trigger'
trigger_type='s'
elif (int(trigger_setting)>5.0 and int(trigger_setting)<21.0):
print 'detector trigger'
trigger_type='d'
else:
if lora_utc<(jan2012+6*30*utc_day):
trigger_setting=5.0
trigger_type='s'
elif lora_utc>(jan2012+6*30*utc_day) and lora_utc<jan2013:
trigger_setting=4.0
trigger_type='s'
else:
trigger_setting=13
trigger_type='d'
return on_off,int(trigger_setting),local_trigger,trigger_type