datadir = options.datadir doTestProcessFunction = options.test force_reprocess = options.force_reprocess lorafile_suffix = options.lorafile_suffix writedir = options.writedir lorafile_suffix = '' # read in LOFAR calibrated pulse block, if desired with debug-lofar-pulse option lofar_pulse = None if options.debug_lofar_pulse: options.debug_test_pulse = True # if not set, set to True # Read database info, for crp_plotfiles key which contains the most actual results directory (i.e. the one used in the database) import psycopg2 # for changing status in database from pycrtools import crdatabase as crdb dbManager = crdb.CRDatabase("crdb", host="coma00.science.ru.nl", user="******", password="******", dbname="crdb") db = dbManager.db print '### Replacing simulated data by LOFAR data processed by pipeline (calibrated-xyz) ###' print 'Reading event data, id = %d ...' % eventid event_data = crdb.Event(db=db, id=eventid) #event_data.stations[0].polarization['0']["crp_integrated_pulse_power"] station = [st for st in event_data.stations if st.stationname == 'CS002'][0] crp_integrated_pulse_power_01 = station.polarization['0'][ 'crp_integrated_pulse_power'] crp_integrated_pulse_power_xyz = station.polarization['xyz'][ 'crp_integrated_pulse_power'] crp_plotfiles = event_data["crp_plotfiles"] results_dir = os.path.split(crp_plotfiles[0])[0]
(options, args) = parser.parse_args() list_id = int(options.list_id) sim_list_file = open('xmax_dataset_nop_bias_1e17_1e18.txt', 'r') list1 = np.genfromtxt(sim_list_file, usecols=(0)) nEvents = len(list1) ldf_fit_energy = np.zeros([nEvents]) ldf_fit_energy_particle = np.zeros([nEvents]) lora_energy = np.zeros([nEvents]) lora_elevation = np.zeros([nEvents]) dbManager = crdb.CRDatabase(host='coma00.science.ru.nl', user='******', password='******', dbname='crdb') db = dbManager.db for i in np.arange(15): event_id = int(list1[i]) print event_id try: event = crdb.Event(db=db, id=event_id) ldf_fit_energy[i] = event["ldf_fit_energy"] ldf_fit_energy_particle[i] = event["ldf_fit_energy_particle"] lora_elevation[i] = (90 - event["lora_elevation"]) * np.pi / 180.0 lora_energy[i] = event["lora_energy"]
def GetLDF(eventno): # Open CR Database connection to read out data pipeline parameters for this event # nofAttempts = 3 #thisAttempt = 0 #database_connection = False #while not database_connection and (thisAttempt < nofAttempts): # try: # database_connection = True # except: # thisAttempt += 1 # print 'Database connection failed at attempt %d' % thisAttempt # import time # time.sleep(20 + 100*np.random.rand() ) # sleep for 20 to 120 seconds before retrying # if not database_connection: # raise ValueError("No database connection after {0} attempts!".format(nofAttempts)) dbManager = crdb.CRDatabase(host='coma00.science.ru.nl', user='******', password='******', dbname='crdb') db = dbManager.db event = crdb.Event(db=db, id=eventno) """gotEvent = False nofAttempts = 3 thisAttempt = 0 while not gotEvent and (thisAttempt < nofAttempts): try: gotEvent = True except: thisAttempt += 1 print 'Get event failed at attempt %d' % thisAttempt import time time.sleep(20 + 100*np.random.rand() ) if not gotEvent: raise ValueError("Could not read event data after {0} attempts!".format(nofAttempts)) """ lora_elevation = 0 try: energy = event["lora_energy"] core_x = event["lora_core_x"] core_y = event["lora_core_y"] azimuth = event["lora_azimuth"] lora_elevation = event["lora_elevation"] moliere = event["lora_moliere"] lora_x = event["lora_posx"] lora_y = event["lora_posy"] lora_dens = event["lora_particle_density__m2"] if LoraQualityPassed(moliere, core_x, core_y, lora_elevation): print "GOOD quality event (LORA)", energy, azimuth, lora_elevation, core_x, core_y, moliere else: print "LORA Quality not passed", energy, azimuth, lora_elevation, core_x, core_y, moliere except: print "Skipping event, no LORA data" return -1 # This should be tested for when calling the function... Nothing to return for a file without LORA info. # Loop over all stations in event stations = [] for f in event.datafiles: stations.extend(f.stations) positions = [] selected_dipoles = [] delays = [] amplitude = [] rms = [] power11 = [] power21 = [] power41 = [] noisepower = [] stationname = [] pulse_direction = [] pulse_delay_fit_residual = [] time = [] for station in stations: if station.status == "GOOD": try: p = station.polarization["xyz"] p0 = station.polarization["0"] positions.append(station["local_antenna_positions"]) selected_dipoles.append(station["crp_selected_dipoles"]) amplitude.append(p["crp_pulse_peak_amplitude"]) power11.append(p0["crp_integrated_pulse_power"]) power21.append(p0["crp_integrated_pulse_power_wide"]) power41.append(p0["crp_integrated_pulse_power_double_wide"]) noisepower.append(p0["crp_integrated_noise_power"]) rms.append(p["crp_rms"]) stationname.append([station.stationname] * len(p["crp_rms"])) pulse_direction.append(station["crp_pulse_direction"]) pulse_delay_fit_residual.append( station["crp_pulse_delay_fit_residual"]) time.append(station["crp_pulse_time"]) except: print "Do not have all pulse parameters for station", station.stationname #print positions positions = np.vstack(positions) selected_dipoles = np.hstack(selected_dipoles) amplitude = np.vstack(amplitude) rms = np.vstack(rms) power11 = np.vstack(power11) power21 = np.vstack(power21) power41 = np.vstack(power41) noisepower = np.vstack(noisepower) pulse_delay_fit_residual = np.hstack(pulse_delay_fit_residual) #stationname = np.array(stationname) stationname = np.array(sum(stationname, [])) time = np.vstack(time) #print time[:,0] lof_azimuth = np.mean(pulse_direction, axis=0)[0] lof_elevation = np.mean(pulse_direction, axis=0)[1] shape = positions.shape positions = positions.reshape((shape[0] / 2, 2, shape[1]))[:, 0] positions = positions.copy() #uncer = np.sqrt(rms[:,0]**2+rms[:,1]**2 + rms[:,2]**2) #total = np.sqrt(amplitude[:,0]*signals[:,0]+signals[:,1]*signals[:,1]+signals[:,2]*signals[:,2]) #print "Returning event:", eventno dist = GetDistance([core_x, core_y, 0], [azimuth, lora_elevation], positions) x_err = GetTotalDistanceUncertainty([core_x, core_y, 0], [5., 5., 0], positions, [azimuth, lora_elevation], [2, 2, 0], dist) dist = np.array(dist) x_err = np.array(x_err) #print positions.shape, amplitude.shape, power.shape, rms.shape return core_x, core_y, stationname, selected_dipoles, positions, dist, x_err, amplitude, power11, power21, power41, rms, noisepower, pulse_delay_fit_residual, time[:, 0], lora_x, lora_y, lora_dens, lof_azimuth, lof_elevation, float( lora_elevation)
parser.add_option("--xmax", type=float, default=None, help="If given use this xmax, otherwise fall back to database value") parser.add_option("--nof-conex-proton", type=int, default=150) parser.add_option("--nof-conex-iron", type=int, default=50) parser.add_option("--showers-around-xmax-estimate", type=int, default=11, help="Number of showers to simulate in a narrow region around the estimated Xmax. Default 11.") parser.add_option("--width-around-xmax-estimate", type=float, default=20.0, help="Width of densely simulated region around Xmax estimate. Default 20 g/cm2") parser.add_option("--no-atmosphere", default=False, action="store_true") parser.add_option("--skip-conex", default=False, action="store_true") parser.add_option("--skip-coreas", default=False, action="store_true") parser.add_option("--skip-analysis", default=False, action="store_true") parser.add_option("--ignore-suspended-jobs", default=False, action="store_true", help="Ignore suspended jobs in queue, i.e. submit duplicate jobs for these (if no other jobs RUNNING or PENDING for the same event)") parser.add_option("--hadronic-interaction-model", type=str, default="QGSII") (options, args) = parser.parse_args() db_filename = options.database dbManager = crdb.CRDatabase(db_filename, host=options.host, user=options.user, password=options.password, dbname=options.dbname) db = dbManager.db valid_status = ["DESIRED", "CONEX_STARTED", "CONEX_DONE", "COREAS_STARTED", "COREAS_DONE"] print "skipping conex:", options.skip_conex print "skipping coreas:", options.skip_coreas print "skipping analysis:", options.skip_analysis # Ignore if simulations are already scheduled if options.id in simhelp.running_jobs(ignore_suspended=options.ignore_suspended_jobs): print "Event {0} already scheduled, skipping...".format(options.id) sys.exit(0) # Get event from database and run pipeline on it with process_event(crdb.Event(db=db, id=options.id)) as event:
def find_trigger(event_id): #print 'running event: {0}'.format(event_id) detector=np.zeros([nDet]) ymd=np.zeros([nDet]) gps=np.zeros([nDet]) ctd=np.zeros([nDet]) nsec=np.zeros([nDet]) trigg_condition=-1*np.ones([nDet]) trigg_pattern=np.zeros([nDet]) total_counts=np.zeros([nDet]) pulse_height=np.zeros([nDet]) pulse_width=np.zeros([nDet]) counts=np.zeros([nDet,nTrace]) on_off=np.zeros([nDet]) local_trigger=-1*np.ones([nLasa]) trigger_setting=-1 dbManager = crdb.CRDatabase("crdb", host="coma00.science.ru.nl",user="******", password="******", dbname="crdb") db = dbManager.db event = crdb.Event(db = db, id = event_id) lora_nsec=event["lora_nsecs"] lora_utc=event["lora_utc_time_secs"] #print 'lora utc: {0}'.format(lora_utc) data=np.genfromtxt(open(on_off_filename,'r')) logUTC=data.T[0] on_off_log=data.T[1:21].T ## find closest day to lora utc idx = (np.abs(logUTC - lora_utc)).argmin() if logUTC[idx]>lora_utc: idx=idx-1 on_off= on_off_log[idx] ## find detectors that are on/off daily with year=time.gmtime(lora_utc).tm_year month=time.gmtime(lora_utc).tm_mon day=time.gmtime(lora_utc).tm_mday # find correct log file for event (daily, sometimes not with a standard name) log_list= glob.glob(data_dir+'{0:04d}{1:02d}{2:02d}*.log'.format(year,month,day)) log_list.extend(glob.glob(data_dir+'{0:04d}{1:02d}{2:02d}*.log'.format(year,month,day-1))) if day==1: log_list.extend(glob.glob(data_dir+'{0:04d}{1:02d}{2:02d}*.log'.format(year,month-1,30))) log_list.extend(glob.glob(data_dir+'{0:04d}{1:02d}{2:02d}*.log'.format(year,month-1,31))) if day==1 and month==1: log_list.extend(glob.glob(data_dir+'{0:04d}{1:02d}{2:02d}*.log'.format(year-1,12,31))) log_list.extend(glob.glob(data_dir+'{0:04d}{1:02d}{2:02d}*.log'.format(year-1,12,30))) found_utc=0 for l in np.arange(len(log_list)): if str(int(lora_utc)) in open(log_list[l]).read(): log_file_name=log_list[l] root_file_name=log_file_name.split('.')[0]+'.root' found_utc=1 if found_utc==1: log_file=open(log_file_name,'r') root_file=ROOT.TFile.Open(root_file_name) tree_sec = root_file.Get("Tree_sec") tree_event = root_file.Get("Tree_event") tree_log = root_file.Get("Tree_log") tree_noise = root_file.Get("Tree_noise") active_lasas,trigger_setting=find_active_stations(log_file) event_index=find_entry_number(lora_utc,lora_nsec,tree_event) for d in np.arange(nDet): detname='Det'+str(d+1) det=tree_event.GetBranch(detname) detector[d],ymd[d],gps[d],ctd[d],nsec[d],trigg_condition[d],trigg_pattern[d],total_counts[d],pulse_height[d],pulse_width[d],counts[d]=getData(det,event_index) if np.max(counts[d])>0.0: on_off[d]=1 # to get the lasa local trigger condition #katie: question- how can only 2 detectors have condition 2/4?? lasa1=np.min(trigg_condition[0:4]) lasa2=np.min(trigg_condition[4:8]) lasa3=np.min(trigg_condition[8:12]) lasa4=np.min(trigg_condition[12:16]) lasa5=np.min(trigg_condition[16:20]) local_trigger=np.asarray([lasa1,lasa2,lasa3,lasa4,lasa5]) ## change to stricter condition in case of broken files for t in np.arange(len(local_trigger)): if local_trigger[t]<0.5: local_trigger[t]=3.0 print 'event okay' else: print 'didn\'t find matching file' ## if there is a problem reading local trigger setting, change everything to strictest condition (3/4) if (-1 in local_trigger)==True: local_trigger=3*np.ones([nLasa]) ## check if station or detector trigger ## catch weird settings trigger_type='d' if (int(trigger_setting)<6.0 and int(trigger_setting)>0.0): print 'station trigger' trigger_type='s' elif (int(trigger_setting)>5.0 and int(trigger_setting)<21.0): print 'detector trigger' trigger_type='d' else: if lora_utc<(jan2012+6*30*utc_day): trigger_setting=5.0 trigger_type='s' elif lora_utc>(jan2012+6*30*utc_day) and lora_utc<jan2013: trigger_setting=4.0 trigger_type='s' else: trigger_setting=13 trigger_type='d' return on_off,int(trigger_setting),local_trigger,trigger_type