-
Notifications
You must be signed in to change notification settings - Fork 0
/
autovet.py
329 lines (252 loc) · 15.1 KB
/
autovet.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
#############################
#############################
## ##
## autovet.py ##
## ##
## updated:06/10/16 ##
## ##
#############################
#############################
#still needs to be cleaned up according to python style guide
#Need to add comments
import argparse
import glob
import numpy
from gwpy.segments import DataQualityFlag
#import ConfigParser
from gwpy.time import tconvert
import datetime
import os, sys
from gwpy.segments import SegmentList
from gwpy.segments import Segment
#command line parsing
parser = argparse.ArgumentParser(
description='autovet.py is a program that grabs triggers and segments for hveto, UPVh, and OVL for any time period, and concatenates them into one segment file, and one trigger file.It then creates a DQ Flag for the given type and time period, spits out a .xml file, and generates the .ini file needed to run VET. For questions or concerns, contact Erika Cowan at erika.cowan@ligo.org')
parser.add_argument('gps_start_time',type=int,help='Please enter GPS start time')
parser.add_argument('gps_end_time',type=int,help='Please enter GPS end time')
parser.add_argument('directory_path',type=str,help='Please enter directory path for triggers and segments')
#parser.add_argument('-s','--start_date', type=str, help='Please enter start date in YYYYMMDD format, required for the hveto option')
#parser.add_argument('-e','--end_date', type=str, help='Please enter end date in YYYYMMDD format, required for the hveto option')
parser.add_argument('type_dq_flag', type=str, help='Please enter either hveto, UPVh, OVL')
parser.add_argument('-a', '--hveto_analysis_seg', type=str, help='Please enter offline hveto O1 offline analysis segment, 4,5,6,8,9')
parser.add_argument('-o','--online_offline', type=str, help='Please enter either offline or online. This is for hveto.')
args = parser.parse_args()
#A check to make sure we're within the time window of aLIGO, and that end_time is after start_time
if args.gps_start_time < 971574400: #roughly the end of S6
parser.error("gps_start_time before S6")
if args.gps_end_time < args.gps_start_time:
parser.error("end_time is before gps_start_time")
#finds beginning of day for given gps time
start_of_day = tconvert(args.gps_start_time)
start_of_day = start_of_day.replace(hour=0,minute=0,second=0)
start_of_day = tconvert(start_of_day)
###################################################
#######CREATING TOTAL TRIGGER/SEGMENT FILES########
###################################################
###choosing to read in hveto!###
if args.type_dq_flag == 'hveto':
print 'Data Quality Flag chosen is hveto, stored in the path ' + args.directory_path
if args.online_offline == 'offline':
analysis_segs_45689 = ['4', '5', '6', '7', '9']
analysis_segs_237 = ['2', '3']
if args.hveto_analysis_seg in analysis_segs_45689:
pattern_trigs_hveto= os.path.join(args.directory_path, 'analysis' + args.hveto_analysis_seg , 'H1-omicron_BOTH-*-DARM','*VETO_SEGS_ROUND*.txt')
pattern_segs_hveto = os.path.join(args.directory_path, 'analysis' + args.hveto_analysis_seg , 'H1-omicron_BOTH-*-DARM','segs.txt')
elif args.hveto_analysis_seg in analysis_segs_237:
pattern_trigs_hveto = os.path.join(args.directory_path,'H1-omicron_BOTH-*-DARM','*VETO_SEGS_ROUND*.txt')
pattern_segs_hveto = os.path.join(args.directory_path,'H1-omicron_BOTH-*-DARM','segs.txt')
elif args.hveto_analysis_seg == '8':
pattern_trigs_hveto = os.path.join(args.directory_path,'*VETO_SEGS_ROUND*.txt')
pattern_segs_hveto = os.path.join(args.directory_path,'segs.txt')
else:
print 'Did not choose O1 analysis segment 1,2,3,4,5,6,7,8,9. Please choose.'
exit()
print pattern_trigs_hveto
print 'Data Quality Flag chosen is hveto, stored in the path ' + args.directory_path
#TRIGGER HANDLING: begin for loop that loops over the range of all days/months/years
f = open("total_hveto_trigs.txt", "w") #file that will hold collection of all triggers
#create pattern paths for the trigger segment files to loop over
#NOTE TO SELF: create option to specify which trigger files to loop over. default it to '*VETO_SEGS_ROUND*.txt', and then in the --help, specify how to put in your own list of trigger files.
#grabbing the trigger files
for filename in glob.glob(pattern_trigs_hveto):
#loading the triggers in
data = numpy.atleast_2d(numpy.loadtxt(filename))
print data
#creating and filling arrays to store the data
start_time = [data[i,0] for i in range(len(data))]
end_time = [data[i,1] for i in range(len(data))]
#writing the two arrays to total_hveto_trigs.txt
for index in range(len(start_time)):
f.write(str(start_time[index]) + " " + str(end_time[index]) + "\n")
f.close()
#SEGMENT HANDLING: begin for loop that loops over the range of all days/months/years
f = open("total_hveto_segs.txt","w") #file that will hold collection of all segments
#grabbing segment files
for filename in glob.glob(pattern_segs_hveto):
if os.path.isfile(filename):
print filename + " exists. Adding to total_hveto_segs.txt."
#loading segments in
knownsegments =numpy.atleast_2d(numpy.loadtxt(filename, delimiter =','))
#storing the segments in these two arrays
known_start = [knownsegments[i,0] for i in range(len(knownsegments))]
known_end = [knownsegments[i,1] for i in range(len(knownsegments))]
#writing the two arrays to total_hveto_segs.txt
for index in range(len(known_start)):
f.write(str(known_start[index]) + " " + str(known_end[index]) + "\n")
else:
print filename + " does not exist. Looking for the segment file in next time increment."
break
f.close()
elif args.online_offline == 'online':
#TRIGGER HANDLING: begin for loop that loops over the range of all days/months/years
f = open("total_hveto_trigs.txt", "w") #file that will hold collection of all triggers
#create pattern paths for the trigger segment files to loop over
#NOTE TO SELF: create option to specify which trigger files to loop over. default it to '*VETO_SEGS_ROUND*.txt', and then in the --help, specify how to put in your own list of trigger files.
pattern_trigs_hveto = os.path.join(args.directory_path, '{}{:02}','{}{:02}{:02}', '*86400-DARM','*VETO_SEGS_ROUND*.txt')
start_time_utc = tconvert(args.gps_start_time)
end_time_utc = tconvert(args.gps_end_time)
triggers = SegmentList([])
while start_time_utc < end_time_utc:
day = start_time_utc.day
month = start_time_utc.month
year = start_time_utc.year
wildcard_trigs_hveto = pattern_trigs_hveto.format(year, month, year, month, day)
triggers = SegmentList([])
#grabbing the trigger files
for filename in glob.glob(wildcard_trigs_hveto):
#loading the triggers in
data = SegmentList.read(filename)
print data
triggers += data
start_time_utc += datetime.timedelta(days=1)
#triggers.coalesce()
start_end_seg = Segment(args.gps_start_time, args.gps_end_time)
triggers = triggers & SegmentList([start_end_seg])
#print triggers
triggers.write("total_hveto_trigs.txt")
start_time_utc += datetime.timedelta(days=1)
#SEGMENT HANDLING: begin for loop that loops over the range of all days/months/years
f = open("total_hveto_segs.txt","w") #file that will hold collection of all segments
pattern_segs_hveto = os.path.join(args.directory_path, '{}{:02}','{}{:02}{:02}', '*86400-DARM', 'segs.txt')
for day in range(start_day, end_day + 1):
for month in range(start_month, end_month +1):
for year in range(start_year, end_year +1):
wildcard_segs_hveto = pattern_segs_hveto.format(year, month,year, month, day)
#grabbing segment files
for filename in glob.glob(wildcard_segs_hveto):
if os.path.isfile(filename):
print filename + " exists. Adding to total_hveto_segs.txt."
#loading segments in
knownsegments =numpy.atleast_2d(numpy.loadtxt(filename, delimiter =','))
#storing the segments in these two arrays
known_start = [knownsegments[i,0] for i in range(len(knownsegments))]
known_end = [knownsegments[i,1] for i in range(len(knownsegments))]
#writing the two arrays to total_hveto_segs.txt
for index in range(len(known_start)):
f.write(str(known_start[index]) + " " + str(known_end[index]) + "\n")
else:
print filename + " does not exist. Looking for the segment file in next time increment."
break
f.close()
else:
print 'Did not choose online or offline. Please choose.'
# exit()
###choosing to read in UPVh!###
elif args.type_dq_flag == 'UPVh':
print 'Data Quality Flag chosen is ' + args.type_dq_flag +', stored in the path ' + args.directory_path
#TRIGGER HANDLING: begin for loop that loops over the range of dates
f = open("total_UPVh_trigs.txt", "w")
pattern_trigs_UPVh = os.path.join(args.directory_path, 'DARM_LOCK_{}_{}-H', 'H1:*veto.txt')
for day in range(args.gps_start_time, args.gps_end_time + 1, 86400):
wildcard_UPVh_trigs = pattern_trigs_UPVh.format(day, day+86400)
print day
#grabbing segment files
for filename in glob.glob(wildcard_UPVh_trigs):
#loading segments in
data = numpy.loadtxt(filename)
#storing the segments in these two arrays
start_time = [data[i,0] for i in range(len(data))]
end_time = [data[i,1] for i in range(len(data))]
for index in range(len(start_time)):
if start_time[index] > args.gps_start_time and start_time[index] < args.gps_end_time and end_time[index] > args.gps_start_time:
if end_time[index] < args.gps_end_time:
f.write(str(start_time[index]) + " " + str(end_time[index]) + "\n")
elif end_time[index] >= args.gps_end_time:
f.write(str(start_time[index]) + " " + str(stop) + "\n")
else:
print "segments out of start/end time range."
#SEGMENT HANDLING: begin for loop that loops over the range of dates
f = open("total_UPVh_segs.txt","w")
pattern_segs_UPVh = os.path.join(args.directory_path, 'DARM_LOCK_{}_{}-H', 'segments.txt')
for day in range(args.gps_start_time, args.gps_end_time +1, 86400):
wildcard_UPVh_segs = pattern_segs_UPVh.format(day, day + 86400)
#grabbing segment files
for filename in glob.glob(wildcard_UPVh_segs):
if os.path.isfile(filename):
print filename + " exists. Adding to total_UPVh_segs.txt."
#loading segments in
knownsegments = numpy.loadtxt(filename)
#storing the segments in these two arrays
known_start = [knownsegments[i,0] for i in range(len(knownsegments))]
known_end = [knownsegments[i,1] for i in range(len(knownsegments))]
for index in range(len(known_start)):
f.write(str(known_start[index]) + " " + str(known_end[index]) + "\n")
else:
print filename + " does not exist. Looking for the segment file in next time increment."
break
f.close()
###whoops! you forgot to choose hveto, UPVh, or OVL!###
else:
print 'Did not give correct dq flag. Please choose from hveto, UPVh, OVL in command line.'
exit()
###################################################
###########CREATING DQ FLAG .XML FILE#############
###################################################
#construct flag and filename
flag_name = 'H1:' + args.type_dq_flag + '-RND:1'
name = 'segments_' + args.type_dq_flag + '_RND.xml'
#reading in segment files
try: knownsegments = numpy.loadtxt('total_'+ args.type_dq_flag + '_segs.txt')
except:
print 'No total_'+ args.type_dq_flag + '_segs.txt file in current working directory. It should have been produced from last loop. \n If this file is empty, that may mean you have no active segments during this time period.'
#NOTE TO SELF: If there were no active segments, so we still want to produce the DQ Flag? Or should we tell it to quit out?
known_start = [knownsegments[i,0] for i in range(len(knownsegments))]
known_end = [knownsegments[i,1] for i in range(len(knownsegments))]
# reading in trigger files
data = numpy.loadtxt('total_'+ args.type_dq_flag + '_trigs.txt')
# get an array for the start_time and end_time of each segment
start_time = [data[i,0] for i in range(len(data))]
end_time = [data[i,1] for i in range(len(data))]
# create a data quality flag object
#zip will truncate the start and end time. is this OK?
flag = DataQualityFlag(flag_name, active=zip(start_time, end_time), known=zip(known_start, known_end))
# write flag
flag.write(name)
print "Created DQ Flag: " + flag_name + " in .xml form as: " + name
###################################################
##############CREATING VET .INI FILE###############
###################################################
config = ConfigParser.RawConfigParser()
config.add_section('plugins')
config.set('plugins','gwvet.tabs', ' ')
config.add_section('states')
config.set('states', 'Science', '%(ifo)s:DMT-ANALYSIS_READY:1')
config.add_section('segment-database')
config.set('segment-database','url','https://segments.ligo.org')
config.add_section('')
config.set('','type','veto-flag')
config.set('','event-channel','%(ifo)s:GDS-CALIB_STRAIN')
config.set('','event-generator','Omicron')
config.set('','metrics',"'Deadtime',\n'Efficiency', \n'Efficiency/Deadtime', \n'Efficiency | SNR>=8', \n'Efficiency/Deadtime | SNR>=8', \n'Efficiency | SNR>=20', \n'Efficiency/Deadtime | SNR>=20', \n'Efficiency | SNR>=100', \n'Efficiency/Deadtime | SNR>=100',\n'Use percentage', \n'Loudest event by SNR'")
config.add_section('tab-SNR-6')
config.set('tab-SNR-6', 'name','SNR 6')
config.set('tab-SNR-6', 'type', 'veto-flag')
config.set('tab-SNR-6', 'shortname', 'SNR 6')
config.set('tab-SNR-6', 'flags', flag_name )
config.set('tab-SNR-6', 'states', "Science")
config.set('tab-SNR-6', 'segmentfile', name )
with open(args.type_dq_flag + '_segs.ini','wb') as configfile:
config.write(configfile)
print "\n Created " + args.type_dq_flag + '_segs.ini. You have everything you need to run VET now! \n(Advance to GO and collect $200.)'
print "To run VET,first go into " + args.type_dq_flag + "_segs.ini, and delete the line that only contains []. Save and exit the .ini file.\n"
print "Now, use the command: gw_summary gps " + str(args.gps_start_time) + " " + str(args.gps_end_time) + " -f /home/detchar/etc/summary/configurations/defaults.ini -f "+ args.type_dq_flag + "_segs.ini"