def open_cache(lcf):
"""Read a LAL-format cache file into memory as a
:class:`glue.lal.Cache`.
"""
if isinstance(lcf, file):
return Cache.fromfile(lcf)
else:
with open(lcf, 'r') as f:
return Cache.fromfile(f)
def open_cache(lcf):
"""Read a LAL-format cache file into memory as a
:class:`glue.lal.Cache`.
"""
if isinstance(lcf, file):
return Cache.fromfile(lcf)
else:
with open(lcf, 'r') as f:
return Cache.fromfile(f)
def process(self, *args, **kwargs):
error = None
# read the cache files
if isinstance(self.cache, string_types) and os.path.isfile(self.cache):
with open(self.cache, 'r') as fobj:
try:
self.cache = Cache.fromfile(fobj).sieve(
segment=self.span)
except ValueError as e:
if "could not convert \'\\n\' to CacheEntry" in str(e):
error = 'could not parse event cache file'
else:
raise
elif isinstance(self.cache, string_types):
error = 'could not locate event cache file'
warn("Cache file %s not found." % self.cache)
elif self.cache is not None and not isinstance(self.cache, Cache):
raise ValueError("Cannot parse EventTriggerTab.cache of type %r"
% type(self.cache))
# push error to all states for HTML writing
if error:
for state in self.states:
self.error[state] = (
'danger',
'This analysis seems to have failed: %s.' % error)
# only process if the cachfile was found
if kwargs.get('trigcache', None) is None:
kwargs['trigcache'] = self.cache
try:
super(EventTriggerTab, self).process(*args, **kwargs)
except IOError as e:
msg = "GWSumm failed to process these data.<pre>%s</pre>" % str(e)
for state in self.states:
self.error[state] = ('danger', msg)
def process(self, *args, **kwargs):
error = None
# read the cache files
if isinstance(self.cache, str) and os.path.isfile(self.cache):
with open(self.cache, 'r') as fobj:
try:
self.cache = Cache.fromfile(fobj).sieve(
segment=self.span)
except ValueError as e:
if "could not convert \'\\n\' to CacheEntry" in str(e):
error = 'could not parse event cache file'
else:
raise
elif isinstance(self.cache, str):
error = 'could not locate event cache file'
warn("Cache file %s not found." % self.cache)
elif self.cache is not None and not isinstance(self.cache, Cache):
raise ValueError("Cannot parse EventTriggerTab.cache of type %r"
% type(self.cache))
# push error to all states for HTML writing
if error:
for state in self.states:
self.error[state] = (
'danger', 'This analysis seems to have failed: %s.' % error)
# only process if the cachfile was found
if kwargs.get('trigcache', None) is None:
kwargs['trigcache'] = self.cache
try:
super(EventTriggerTab, self).process(*args, **kwargs)
except IOError as e:
warn('Caught %s: %s' % (type(e).__name__, str(e)))
msg = "GWSumm failed to process these data.<pre>%s</pre>" % str(e)
for state in self.states:
self.error[state] = ( 'danger', msg)
def process(self, *args, **kwargs):
# read the segment files
if os.path.isfile(self.segmentfile):
segs = DataQualityFlag.read(self.segmentfile, coalesce=False)
self.states[0].known = segs.known
self.states[0].active = segs.active
self.states[0].ready = True
else:
warn('Segment file %s not found.' % self.segmentfile)
return
if len(self.states[0].active) == 0:
warn('No segments analysed by daily ahope.')
return
# read the cache files
if os.path.isfile(self.inspiralcachefile):
with open(self.inspiralcachefile, 'r') as fobj:
try:
self.inspiralcache = Cache.fromfile(fobj).sieve(
segment=self.span)
except ValueError as e:
if "could not convert \'\\n\' to CacheEntry" in str(e):
self.inspiralcache = Cache()
else:
raise
else:
warn("Cache file %s not found." % self.inspiralcachefile)
return
if os.path.isfile(self.tmpltbankcachefile):
with open(self.tmpltbankcachefile, 'r') as fobj:
try:
self.tmpltbankcache = Cache.fromfile(fobj).sieve(
segment=self.span)
except ValueError:
if "could not convert \'\\n\' to CacheEntry" in str(e):
self.tmpltbankcache = Cache()
else:
raise
else:
warn("Cache file %s not found." % self.tmpltbankcachefile)
self.tmpltbankcache = Cache()
# only process if the cachfile was found
super(DailyAhopeTab, self).process(*args, **kwargs)
def process(self, *args, **kwargs):
# read the segment files
if os.path.isfile(self.segmentfile):
segs = DataQualityFlag.read(self.segmentfile, coalesce=False)
self.states[0].known = segs.known
self.states[0].active = segs.active
self.states[0].ready = True
else:
warn('Segment file %s not found.' % self.segmentfile)
return
if len(self.states[0].active) == 0:
warn('No segments analysed by daily ahope.')
return
# read the cache files
if os.path.isfile(self.inspiralcachefile):
with open(self.inspiralcachefile, 'r') as fobj:
try:
self.inspiralcache = Cache.fromfile(fobj).sieve(
segment=self.span)
except ValueError as e:
if "could not convert \'\\n\' to CacheEntry" in str(e):
self.inspiralcache = Cache()
else:
raise
else:
warn("Cache file %s not found." % self.inspiralcachefile)
return
if os.path.isfile(self.tmpltbankcachefile):
with open(self.tmpltbankcachefile, 'r') as fobj:
try:
self.tmpltbankcache = Cache.fromfile(fobj).sieve(
segment=self.span)
except ValueError:
if "could not convert \'\\n\' to CacheEntry" in str(e):
self.tmpltbankcache = Cache()
else:
raise
else:
warn("Cache file %s not found." % self.tmpltbankcachefile)
self.tmpltbankcache = Cache()
# only process if the cachfile was found
super(DailyAhopeTab, self).process(*args, **kwargs)
def get_omicron_triggers(channel, ifo, segments, cachefile):
print "Reading channel: %s\n" %channel
with open(cachefile, 'r') as f:
mycache = Cache.fromfile(f)
# Let's try and catch failed reads
try:
triggers = get_triggers(ifo + ':' + channel, 'sngl_burst', segments,\
cache=mycache)
except:
print "No Omicron triggers read for channel %s" %channel
return None
return triggers
def load_omic_trigs(omicroncachefile, segs):
# Read in the Omicron triggers
with open(omicroncachefile, 'r') as cachefile:
cache = Cache.fromfile(cachefile)
omic_trigs = SnglBurstTable.read(cache, verbose=True, filt=lambda x: x.snr <\
100 and x.peak_frequency < 100)
# Check if Omicron triggers have been read in successfully
if not omic_trigs:
sys.exit("ERROR: No triggers for Omicron channel: %s" % cachefile.split('.')[0])
else:
print "%d Omicron triggers read" % len(omic_trigs)
#Get the Omicron triggers that lie within the valid segment list
omic_trigs = omic_trigs.vetoed(segs)
return omic_trigs
def run_command(self, options={}, args=[]):
if len(args) not in [1]:
self.parser.error("cachfile is required.")
config = getLarsConfig()
if not config:
print "This analysis does not appear to have a reservation. (no %s)" % INI_NAME
print "If a reservation has been lost, try 'lars info [--repair]'"
print "to try to recover your '%s'" % INI_NAME
return
id = config.get('lars', 'id')
cachefilename = args[0]
cachefile = open(cachefilename, "r")
cache = Cache.fromfile(cachefile)
segdir = cache.to_segmentlistdict()
extent = segdir.extent_all()
gpsStart = int(extent[0])
gpsEnd = int(extent[1])
ifos = mkIfos(segdir.keys())
duration = gpsEnd - gpsStart
url = makeNiceUrl(os.getcwd())
if options.dry_run:
print "Dry run. Results not saved"
print "gpsStart: ", gpsStart
print "gpsEnd: ", gpsEnd
print "duration: ", duration
print "IFOs: ", ifos
print "Cachefile: ", cachefilename
print "Location: ", url
return
server = serviceProxy(config.get('lars', 'serviceUrl'))
rv = server.publish(id, ifos, gpsStart, duration, url,
makeNiceUrl(cachefilename))
rv = objectify(rv)
print "Published:", rv.uid
def run_command(self, options={}, args=[]):
if len(args) not in [1]:
self.parser.error("cachfile is required.")
config = getLarsConfig()
if not config:
print "This analysis does not appear to have a reservation. (no %s)" % INI_NAME
print "If a reservation has been lost, try 'lars info [--repair]'"
print "to try to recover your '%s'" % INI_NAME
return
id = config.get('lars','id')
cachefilename = args[0]
cachefile = open(cachefilename, "r")
cache = Cache.fromfile(cachefile)
segdir = cache.to_segmentlistdict()
extent = segdir.extent_all()
gpsStart = int(extent[0])
gpsEnd = int(extent[1])
ifos = mkIfos(segdir.keys())
duration = gpsEnd - gpsStart
url = makeNiceUrl(os.getcwd())
if options.dry_run:
print "Dry run. Results not saved"
print "gpsStart: ", gpsStart
print "gpsEnd: ", gpsEnd
print "duration: ", duration
print "IFOs: ", ifos
print "Cachefile: ", cachefilename
print "Location: ", url
return
server = serviceProxy(config.get('lars', 'serviceUrl'))
rv = server.publish(id, ifos, gpsStart, duration, url, makeNiceUrl(cachefilename))
rv = objectify(rv)
print "Published:", rv.uid
def hoge(gps):
if np.isnan(gps):
return None
from gwpy.time import tconvert
#print tconvert(gps)
from gwpy.timeseries import TimeSeries
from glue.lal import Cache
channel = 'K1:PEM-EXV_SEIS_Z_SENSINF_OUT_DQ'
start = gps #- 5*60
end = gps + 30*60
gwf_cache = 'full_Sep01-Nov01.cache'
with open(gwf_cache, 'r') as fobj:
cache = Cache.fromfile(fobj)
#print cache
data = TimeSeries.read(cache,channel,start=start,end=end,verbose=True,nproc=2,pad=np.nan,format='gwf.lalframe')
plot = data.plot(
title = 'hoge'
#ylabel='Strain amplitude',
)
plot.savefig('{0}.png'.format(gps))
plot.close()
def find_daily_cache(start,
end,
ifo,
clustering=None,
check_files=False,
**kwargs):
"""Find Daily ihope files from the daily runs for the given span
@param start
GPS start time for search
@param end
GPS end time for search
@param ifo
observatory for search
@param clustering
tag for clustering stage to search, default: unclustered
@param check_files
check that the returned files can be read on disk, default False
@param kwargs UNDOCUMENTED
"""
out = Cache()
# set clustering tag
if clustering == None or clustering.upper() == 'UNCLUSTERED':
file_tag = 'INSPIRAL_UNCLUSTERED'
elif clustering.upper() in ["100MS", "100MILLISEC"]:
file_tag = 'INSPIRAL_100MILLISEC_CLUSTERED'
elif clustering.upper() in ["30MS", "30MILLISEC"]:
file_tag = 'INSPIRAL_30MILLISEC_CLUSTERED'
elif clustering.upper() in ["16S", "16SECOND"]:
file_tag = 'INSPIRAL_16SEC_CLUSTERED'
# set base directory
directory = kwargs.pop("directory", os.path.expanduser("~cbc/ihope_daily"))
# work out days
span = Segment(start, end)
start = int(start)
start_d = lal.UTCToGPS(
datetime(*lal.GPSToUTC(start)[:6]).replace(hour=0, minute=0,
second=0).timetuple())
days = []
day = start_d
while day <= end:
days.append(day)
day += 86400
# optimise
append = out.append
splitext = os.path.splitext
isfile = os.path.isfile
pjoin = os.path.join
intersects = span.intersects
from_T050017 = CacheEntry.from_T050017
# loop over days gathering files
for day in days:
utc = datetime(*lal.GPSToUTC(day)[:6])
day_path = pjoin(directory, utc.strftime("%Y%m"),
utc.strftime("%Y%m%d"))
day_cache = os.path.join(day_path, "%s-%s.cache" % (ifo, file_tag))
if isfile(day_cache):
with open(day_cache, "r") as f:
filenames = Cache.fromfile(f).pfnlist()
else:
filenames = glob(
os.path.join(day_path, ("%s-%s-*.xml.gz" % (ifo, file_tag))))
for filename in filenames:
e = from_T050017(filename)
if intersects(e.segment):
append(e)
out.sort(key=lambda e: e.path)
return out
def find_daily_cache(start, end, ifo, clustering=None, check_files=False,
**kwargs):
"""Find Daily ihope files from the daily runs for the given span
@param start
GPS start time for search
@param end
GPS end time for search
@param ifo
observatory for search
@param clustering
tag for clustering stage to search, default: unclustered
@param check_files
check that the returned files can be read on disk, default False
@param kwargs UNDOCUMENTED
"""
out = Cache()
# set clustering tag
if clustering==None or clustering.upper()=='UNCLUSTERED':
file_tag='INSPIRAL_UNCLUSTERED'
elif clustering.upper() in ["100MS", "100MILLISEC"]:
file_tag='INSPIRAL_100MILLISEC_CLUSTERED'
elif clustering.upper() in ["30MS", "30MILLISEC"]:
file_tag='INSPIRAL_30MILLISEC_CLUSTERED'
elif clustering.upper() in ["16S", "16SECOND"]:
file_tag='INSPIRAL_16SEC_CLUSTERED'
# set base directory
directory = kwargs.pop("directory", os.path.expanduser("~cbc/ihope_daily"))
# work out days
span = Segment(start, end)
start = int(start)
start_d = lal.UTCToGPS(datetime(*lal.GPSToUTC(start)[:6]).replace(
hour=0, minute=0, second=0).timetuple())
days = []
day = start_d
while day <= end:
days.append(day)
day+=86400
# optimise
append = out.append
splitext = os.path.splitext
isfile = os.path.isfile
pjoin = os.path.join
intersects = span.intersects
from_T050017 = CacheEntry.from_T050017
# loop over days gathering files
for day in days:
utc = datetime(*lal.GPSToUTC(day)[:6])
day_path = pjoin(directory, utc.strftime("%Y%m"),
utc.strftime("%Y%m%d"))
day_cache = os.path.join(day_path, "%s-%s.cache" % (ifo, file_tag))
if isfile(day_cache):
with open(day_cache, "r") as f:
filenames = Cache.fromfile(f).pfnlist()
else:
filenames = glob(os.path.join(day_path,
("%s-%s-*.xml.gz"
% (ifo, file_tag))))
for filename in filenames:
e = from_T050017(filename)
if intersects(e.segment):
append(e)
out.sort(key=lambda e: e.path)
return out
########## apply segment list (of good times)
segments = SegmentList.read('L1_ER7_segments.txt')
for cachefile in cachelist:
### open trigger cache
# Make a tag for the saving the plot as well as the title
# The tag is the name of the channel extracted from the path
tag = cachefile.split('/')[-1]
tag = tag.split('.')[0]
tag = tag.replace('_Omicron','')
print ('\n\nReading file: %s now ...\n' % tag)
with open(cachefile, 'r') as fobj:
cache = Cache.fromfile(fobj)
### read triggers
# filter to select for triggers with frequency < 100
#trigs = SnglBurstTable.read(cache, verbose=True, filt=lambda t: t.peak_frequency < 100)
#filter to select for triggers with frequency <100 and snr <100
trigs = get_triggers('L1:'+tag, 'sngl_burst', segments, cache=cache)
### check triggers read successfully
if not trigs:
print(" WARNING: No triggers for channel '%s'." % channel,
file=sys.stderr)
else:
# conditions = map(parse_specification, opts.specifier) # # Put the conditions together # channel_cond = defaultdict(list) for inst, channel_name, op, threshold in conditions: channel = "%s:%s" % (inst, channel_name) channel_cond[channel].append((op, threshold)) # # Read the datas and such # ifos = list(set([c[:2] for c in channel_cond.keys()])) cache = Cache.fromfile(open(opts.frame_cache)) seg = cache.to_segmentlistdict()[ifos[0][0]][0] if opts.verbose: print "Loaded %s, total coverage time: %f" % (opts.frame_cache, abs(seg)) # # Set up the XML document # xmldoc = ligolw.Document() xmldoc.appendChild(ligolw.LIGO_LW()) # Append the process information procrow = utils.process.append_process(xmldoc, program=sys.argv[0]) utils.process.append_process_params(xmldoc, procrow, process.process_params_from_dict(opts.__dict__)) # # Segment storage
def mkSegment(gst, get, utc_date, txt=True) :
chGRDLSC = 'K1:GRD-LSC_LOCK_STATE_N'
chGRDIFO = 'K1:GRD-IFO_STATE_N'
chGRDEQ = 'K1:GRD-PEM_EARTHQUAKE_STATE_N'
chOMCADC = 'K1:FEC-32_ADC_OVERFLOW_0_0'
channels = [chGRDLSC,chGRDIFO,chGRDEQ,chOMCADC]
if getpass.getuser() == "controls":
gwf_cache = '/users/DET/Cache/latest.cache'
with open(gwf_cache, 'r') as fobj:
cache = Cache.fromfile(fobj)
else:
# add 1sec margin for locked segments contract.
cache = GetFilelist(gst-1, get+1)
#------------------------------------------------------------
#print('Reading {0} timeseries data...'.format(date))
# add 1sec margin for locked segments contract.
channeldata = TimeSeriesDict.read(cache, channels, start=gst-1, end=get+1, format='gwf.lalframe', gap='pad')
channeldataGRDIFO = channeldata[chGRDIFO]
channeldataGRDLSC = channeldata[chGRDLSC]
channeldataGRDEQ = channeldata[chGRDEQ]
channeldataOMCADC = channeldata[chOMCADC]
sv={}
sv['K1-GRD_SCIENCE_MODE'] = channeldataGRDIFO == 1000
# Locked will be defined by inverse of unlocked segments for technical reason.
#sv['K1-GRD_LOCKED'] = channeldataGRDLSC == 1000
sv['K1-GRD_UNLOCKED'] = channeldataGRDLSC != 1000
sv['K1-GRD_PEM_EARTHQUAKE'] = channeldataGRDEQ == 1000
sv['K1-OMC_OVERFLOW_VETO'] = channeldataOMCADC != 0
# OMC_OVERFLOW_OK will be defined by inverse of veto segments for technical reason.
#sv['K1-OMC_OVERFLOW_OK'] = channeldataOMCADC == 0
dqflag = {}
for key in keys:
if key == 'K1-GRD_LOCKED' or key == 'K1-OMC_OVERFLOW_OK':
continue
dqflag[key] = sv[key].to_dqflag(round=True)
# To omit fraction. round=True option is inclusive in default.
dqflag['K1-GRD_SCIENCE_MODE'].active = dqflag['K1-GRD_SCIENCE_MODE'].active.contract(1.0)
dqflag['K1-GRD_LOCKED'] = ~dqflag['K1-GRD_UNLOCKED']
dqflag['K1-GRD_LOCKED'].name = "K1:GRD-LSC_LOCK_STATE_N == 1000"
dqflag['K1-OMC_OVERFLOW_OK'] = ~dqflag['K1-OMC_OVERFLOW_VETO']
dqflag['K1-OMC_OVERFLOW_OK'].name = "K1:FEC-32_ADC_OVERFLOW_0_0 == 0"
dqflag['K1-GRD_SCIENCE_MODE'].description = "Observation mode. K1:GRD-IFO_STATE_N == 1000"
dqflag['K1-GRD_UNLOCKED'].description = "Interferometer is not locked. K1:GRD-LSC_LOCK_STATE_N != 1000"
dqflag['K1-GRD_LOCKED'].description = "Interferometer is locked. K1:GRD-LSC_LOCK_STATE_N == 1000"
dqflag['K1-OMC_OVERFLOW_VETO'].description = "OMC overflow happened. K1:FEC-32_ADC_OVERFLOW_0_0 != 0"
dqflag['K1-OMC_OVERFLOW_OK'].description = "OMC overflow does not happened. K1:FEC-32_ADC_OVERFLOW_0_0 == 0"
for key in keys:
# added 1sec margin for locked segments contract is removed.
margin = DataQualityFlag(known=[(gst,get)],active=[(gst-1,gst),(get,get+1)])
dqflag[key] -= margin
# write down 15 min segments.
if txt:
with open(filepath_txt[key], mode='w') as f:
for seg in dqflag[key].active :
f.write('{0} {1}\n'.format(int(seg[0]), int(seg[1])))
# if accumulated file exists, it is added.
if os.path.exists(filepath_xml[key]):
tmp = DataQualityFlag.read(filepath_xml[key])
dqflag[key] = dqflag[key] + tmp
dqflag[key].write(filepath_xml[key],overwrite=True)
def CouplingArrange(cache_drc,
ach,
bch,
s,
d,
a=None,
df=None,
f0=None,
f1=None):
## [1] cache_drc :: Enter Cache file direction
## [2] ach :: Enter first channel name to compare
## [3] bch :: Enter second channel name to compare
## [4] s :: Enter start GPS time
## [5] d :: Enter time from star GPS time
## [6] a :: Enter Averaging Number, CAN NOT be used with -df at the same time
## [7] df :: Enter Frequency interval, df, CAN NOT be used with -a at the same time.
## [8] f0 :: Enter start frequency to obtain amplitude value list
## [9] f1 :: Enter end frequency to obtain amplitude value list
import sys
import os
import numpy as np
from scipy.interpolate import interp1d
from gwpy.timeseries import TimeSeries
from glue.lal import Cache
from gwpy.segments import Segment
########### List of times to want to calculate ###########
gst = s # GPS start time
get = gst + d # GPS end time
dur = d # Time duration
ach = str(ach) # Slected A Channel
bch = str(bch) # Slected B Chennel
avg = a # Slected Averaging Number
df = df # Slected Frequency Interval
if avg == None and df == None:
avg = 100
bvg = float(dur / avg)
elif avg == 0 and df == None:
avg = 1
bvg = float(dur / avg)
elif avg != None and df == None:
avg = int(avg)
bvg = float(dur / avg)
elif avg == None and df != None:
df = float(df)
avg = int(df * dur)
bvg = float(1 / df)
else:
avg = 1
bvg = float(dur / avg)
########### Reading Cache ###########
gwf_cache = cache_drc
with open(gwf_cache, 'r') as fobj:
cache = Cache.fromfile(fobj)
########### Reaiding TimeSeries Data ###########
data1 = TimeSeries.read(cache, ach, gst, get, format='lalframe')
data2 = TimeSeries.read(cache, bch, gst, get, format='lalframe')
########### TimeSeries Data Averaging Process ###########
data1_psd_seg = TimeSeries.read(cache,
ach,
gst,
gst + (bvg),
format='lalframe').psd()**(1 / 2.)
for n in range(avg):
if gst + (bvg * (n + 2)) > gst + (bvg *
(n + 1)) and gst + (bvg *
(n + 2)) < get:
data1_seg = TimeSeries.read(cache,
ach,
gst + (bvg * (n + 1)),
gst + (bvg * (n + 2)),
format='lalframe')
data1_psd_seg += data1_seg.psd()**(1 / 2.)
else:
pass
data1_psd = data1_psd_seg / int(avg)
data2_psd_seg = TimeSeries.read(cache,
bch,
gst,
gst + (bvg),
format='lalframe').psd()**(1 / 2.)
for n in range(avg):
if gst + (bvg * (n + 2)) > gst + (bvg *
(n + 1)) and gst + (bvg *
(n + 2)) < get:
data2_seg = TimeSeries.read(cache,
bch,
gst + (bvg * (n + 1)),
gst + (bvg * (n + 2)),
format='lalframe')
data2_psd_seg += data2_seg.psd()**(1 / 2.)
else:
pass
data2_psd = data2_psd_seg / int(avg)
########### Calculate PSD Ratio of two channels using Interpolation process ###########
data1_df = float(str(data1_psd.df)[:-2])
data2_df = float(str(data2_psd.df)[:-2])
data1_psd_yarr = np.array(data1_psd)
data1_psd_xarr = np.linspace(data1_psd.xspan[0], data1_psd.xspan[-1],
len(data1_psd_yarr))
data2_psd_yarr = np.array(data2_psd)
data2_psd_xarr = np.linspace(data2_psd.xspan[0], data2_psd.xspan[-1],
len(data2_psd_yarr))
g1 = interp1d(data1_psd_xarr, data1_psd_yarr)
g2 = interp1d(data2_psd_xarr, data2_psd_yarr)
h_xlist = []
h_ylist = []
if len(data1_psd_xarr) >= len(data2_psd_xarr):
for a in range((len(data2_psd_xarr))):
h_xlist.append(a * data2_df)
h_ylist.append(float(g1(a * data2_df)) / float(g2(a * data2_df)))
hx = np.array(h_xlist)
hy = np.array(h_ylist)
else:
for b in range((len(data1_psd_xarr))):
h_xlist.append(b * data1_df)
h_ylist.append(float(g1(b * data1_df)) / float(g2(b * data1_df)))
hx = np.array(h_xlist)
hy = np.array(h_ylist)
h = interp1d(hx, hy)
############ Find amplitude for selected frequency ############
freq0 = f0
freq1 = f1
if freq0 != None and freq1 != None:
amp_list = []
amp_list_x = []
amp_list_y = []
for f in range(len(h_xlist)):
if h_xlist[f] <= freq1 and freq0 <= h_xlist[f]:
amp_list_x.append(h_xlist[f])
amp_list_y.append(h(h_xlist[f]))
amp_list.append(amp_list_x)
amp_list.append(amp_list_y)
return amp_list
else:
amp_list = []
amp_list.append(h_xlist)
amp_list.append(h_ylist)
return amp_list
def main(args=None):
"""Run the GWSumm command-line interface
"""
parser = create_parser()
args = parser.parse_args(args=args)
if args.debug:
warnings.simplefilter('error', DeprecationWarning)
# set verbose output options
globalv.VERBOSE = args.verbose
# find all config files
args.config_file = [
os.path.expanduser(fp) for csv in args.config_file
for fp in csv.split(',')
]
# check segdb option
if args.on_segdb_error not in ['raise', 'warn', 'ignore']:
parser.error("Invalid option --on-segdb-error='%s'" %
args.on_segdb_error)
# read configuration file
config = GWSummConfigParser()
config.optionxform = str
if args.ifo:
config.set_ifo_options(args.ifo, section=DEFAULTSECT)
config.set(DEFAULTSECT, 'user', getpass.getuser())
config.read(args.config_file)
try:
ifo = config.get(DEFAULTSECT, 'IFO')
except NoOptionError:
ifo = None
finally:
globalv.IFO = ifo
# interpolate section names
interp = {}
if ifo:
interp['ifo'] = ifo.lower()
interp['IFO'] = ifo.title()
config.interpolate_section_names(**interp)
# double-check week mode matches calendar setting
if args.mode == 'week':
if config.has_option("calendar", "start-of-week"):
weekday = getattr(calendar,
config.get("calendar", "start-of-week").upper())
if weekday != args.week.timetuple().tm_wday:
msg = ("Cannot process week starting on %s. The "
"'start-of-week' option in the [calendar] section "
"of the INI file specifies weeks start on %ss." %
(args.week.strftime('%Y%m%d'),
config.get("calendar", "start-of-week")))
raise parser.error(msg)
# record times in ConfigParser
config.set_date_options(args.gpsstart, args.gpsend, section=DEFAULTSECT)
# convert times for convenience
span = Segment(args.gpsstart, args.gpsend)
utc = tconvert(args.gpsstart)
starttime = Time(float(args.gpsstart), format='gps')
endtime = Time(float(args.gpsend), format='gps')
# set mode and output directory
mode.set_mode(args.mode)
try:
path = mode.get_base(utc)
except ValueError:
path = os.path.join('%d-%d' % (args.gpsstart, args.gpsend))
# set LAL FFT plan wisdom level
duration = min(globalv.NOW, args.gpsend) - args.gpsstart
if duration > 200000:
fft_lal.LAL_FFTPLAN_LEVEL = 3
elif duration > 40000:
fft_lal.LAL_FFTPLAN_LEVEL = 2
else:
fft_lal.LAL_FFTPLAN_LEVEL = 1
# set global html only flag
if args.html_only:
globalv.HTMLONLY = True
# build directories
mkdir(args.output_dir)
os.chdir(args.output_dir)
plotdir = os.path.join(path, 'plots')
mkdir(plotdir)
# -- setup --------------------------------------
LOGGER.info(" -- GW interferometer summary information system -- ")
LOGGER.debug("This is process {}".format(os.getpid()))
LOGGER.debug("You have selected {} mode".format(mode.get_mode().name))
LOGGER.debug("Start time: {0} ({1})".format(starttime.utc.iso,
starttime.gps))
LOGGER.debug("End time: {0} ({1})".format(endtime.utc.iso, endtime.gps))
LOGGER.debug("Output directory: {}".format(
os.path.abspath(os.path.join(args.output_dir, path))))
# -- Finalise configuration
LOGGER.info("Loading configuration")
plugins = config.load_plugins()
if plugins:
LOGGER.debug(" -- Loaded {} plugins:".format(len(plugins)))
for mod in plugins:
LOGGER.debug(" %s" % mod)
units = config.load_units()
LOGGER.debug(" Loaded %d units" % len(units))
channels = config.load_channels()
LOGGER.debug(" Loaded %d channels" % len(channels))
states = config.load_states()
LOGGER.debug(" Loaded %d states" % len(states))
rcp = config.load_rcParams()
LOGGER.debug(" Loaded %d rcParams" % len(rcp))
# read list of tabs
tablist = TabList.from_ini(config,
match=args.process_tab,
path=path,
plotdir=plotdir)
tablist.sort(reverse=True)
tabs = sorted(tablist.get_hierarchy(), key=tablist._sortkey)
LOGGER.info(" Loaded %d tabs [%d parents overall]" %
(len(tablist), len(tabs)))
# read caches
cache = {}
for (key,
var) in zip(['datacache', 'trigcache', 'segmentcache'],
[args.data_cache, args.event_cache, args.segment_cache]):
if var:
LOGGER.info("Reading %s from %d files... " % (key, len(var)))
cache[key] = Cache()
for fp in var:
with open(fp, 'r') as f:
cache[key].extend(Cache.fromfile(f))
cache[key] = cache[key].sieve(segment=span)
LOGGER.debug("Done [%d entries]" % len(cache[key]))
# -- read archive -------------------------------
if not hasattr(args, 'archive'):
args.archive = False
if args.html_only:
args.archive = False
args.daily_archive = False
elif args.archive is True:
args.archive = 'GW_SUMMARY_ARCHIVE'
archives = []
if args.archive:
archivedir = os.path.join(path, 'archive')
mkdir(archivedir)
args.archive = os.path.join(
archivedir, '%s-%s-%d-%d.h5' %
(ifo, args.archive, args.gpsstart, args.gpsend - args.gpsstart))
if os.path.isfile(args.archive):
archives.append(args.archive)
else:
LOGGER.debug(
"No archive found in %s, one will be created at the end" %
args.archive)
# read daily archive for week/month/... mode
if hasattr(args, 'daily_archive') and args.daily_archive:
# find daily archive files
archives.extend(
archive.find_daily_archives(args.gpsstart, args.gpsend, ifo,
args.daily_archive, archivedir))
# then don't read any actual data
cache['datacache'] = Cache()
for arch in archives:
LOGGER.info("Reading archived data from %s" % arch)
archive.read_data_archive(arch)
LOGGER.debug("Archive data loaded")
# -- read HTML configuration --------------------
css = config.get_css(section='html')
javascript = config.get_javascript(section='html')
# enable comments
try:
globalv.HTML_COMMENTS_NAME = config.get('html', 'disqus-shortname')
except (NoOptionError, NoSectionError):
pass
# find new ifo bases
ifobases = {}
try:
bases_ = config.nditems('html')
except NoSectionError:
pass
else:
base_reg = re.compile(r'-base\Z')
for key, val in bases_:
if base_reg.search(key):
ifobases[key.rsplit('-', 1)[0]] = val
ifobases = OrderedDict(sorted(ifobases.items(), key=lambda x: x[0]))
# -- write auxiliary pages ----------------------
# get URL from output directory
if 'public_html' in os.getcwd():
urlbase = os.path.sep + os.path.join(
'~%s' % config.get(DEFAULTSECT, 'user'),
os.getcwd().split('public_html', 1)[1][1:])
base = urlbase
# otherwise get URL from html config
elif ifo in ifobases:
urlbase = urlparse(ifobases[ifo]).path
base = urlbase
# otherwise let the write_html processor work it out on-the-fly
else:
urlbase = None
base = None
# get link to issues report page
try:
issues = config.get('html', 'issues')
except KeyError:
issues = True
# write 404 error page
if not args.no_htaccess and not args.no_html and urlbase:
top = os.path.join(urlbase, path)
four0four = get_tab('404')(span=span,
parent=None,
path=path,
index=os.path.join(path, '404.html'))
four0four.write_html(css=css,
js=javascript,
tabs=tabs,
ifo=ifo,
ifomap=ifobases,
top=top,
base=base,
writedata=not args.html_only,
writehtml=not args.no_html,
issues=issues)
url404 = os.path.join(urlbase, four0four.index)
with open(os.path.join(path, '.htaccess'), 'w') as htaccess:
print('Options -Indexes', file=htaccess)
print('ErrorDocument 404 %s' % url404, file=htaccess)
print('ErrorDocument 403 %s' % url404, file=htaccess)
# write config page
about = get_tab('about')(span=span, parent=None, path=path)
if not args.no_html:
mkdir(about.path)
about.write_html(css=css,
js=javascript,
tabs=tabs,
config=config.files,
prog=PROG,
ifo=ifo,
ifomap=ifobases,
about=about.index,
base=base,
issues=issues,
writedata=not args.html_only,
writehtml=not args.no_html)
# -- read bulk data -----------------------------
# XXX: bulk data reading could optimise things
# XXX: but has never been used, so should remove (DMM 18/01/16)
if args.bulk_read and not args.html_only:
LOGGER.info("Reading all data in BULK")
allts = set()
allsv = set()
allflags = set()
for tab in tablist:
snames = []
for state in tab.states:
snames.append(state.name)
if state.definition:
allflags.update(re_flagdiv.split(state.definition))
# get all data defined for the 'all' state
if ALLSTATE in snames:
allts.update(
tab.get_channels('timeseries', 'spectrogram', 'spectrum',
'histogram'))
allsv.update(tab.get_channels('statevector'))
allflags.update(tab.get_flags('segments'))
# or get data for plots defined over all states
else:
for plot in tab.plots:
if plot.state is not None:
continue
if plot.type in [
'timeseries', 'spectrogram', 'spectrum',
'histogram'
]:
allts.update(plot.channels)
elif plot.type in ['statevector']:
allsv.update(plot.channels)
elif plot.type in ['segments']:
allflags.update([
f for cflag in plot.flags
for f in re_flagdiv.split(cflag)[::2] if f
])
allseg = SegmentList([span])
if len(allflags):
LOGGER.info(
"%d data-quality flags identified for segment query from all "
"tabs" % len(allflags))
get_segments(allflags, allseg, config=config, return_=False)
if len(allts):
LOGGER.info("%d channels identified for TimeSeries from all tabs" %
len(allts))
get_timeseries_dict(allts,
allseg,
config=config,
nds=args.nds,
nproc=args.multiprocess,
return_=False)
if len(allsv):
LOGGER.info(
"%d channels identified for StateVector from all tabs" %
len(allsv))
get_timeseries_dict(allsv,
allseg,
config=config,
nds=args.nds,
statevector=True,
nproc=args.multiprocess,
return_=False)
# -- process all tabs ---------------------------
# TODO: consider re-working this loop as TabList.process_all
for tab in tablist:
if tab.parent:
name = '%s/%s' % (tab.parent.name, tab.name)
else:
name = tab.name
if not args.html_only and isinstance(tab, get_tab('_processed')):
LOGGER.debug("Processing %s" % name)
tab.process(config=config,
nds=args.nds,
nproc=args.multiprocess,
segdb_error=args.on_segdb_error,
datafind_error=args.on_datafind_error,
**cache)
if not tab.hidden and not isinstance(tab, get_tab('link')):
mkdir(tab.href)
tab.write_html(css=css,
js=javascript,
tabs=tabs,
ifo=ifo,
ifomap=ifobases,
about=about.index,
base=base,
issues=issues,
writedata=not args.html_only,
writehtml=not args.no_html)
# archive this tab
if args.archive:
LOGGER.info("Writing data to archive")
archive.write_data_archive(args.archive)
LOGGER.debug("Archive written to {}".format(
os.path.abspath(args.archive)))
LOGGER.debug("%s complete" % (name))
LOGGER.info("-- Data products written, all done --")
def process_options(): """ Process options and check for required values. """ opt = OptionParser() opt.add_option( "-c", "--input-cache", help="Read triggers from the files in this cache." ) opt.add_option( "-v", "--verbose", action="store_true", help="Be verbose." ) veto_settings = OptionGroup( opt, "HVeto settings" ) veto_settings.add_option( "-i", "--instrument", help="Instrument against which to veto. Required." ) veto_settings.add_option( "-r", "--reference-channel", help="Channel against which to veto. Required." ) veto_settings.add_option( "-t", "--reference-triggers", help="File path to load reference triggers. Required." ) veto_settings.add_option( "-s", "--significance-threshold", type=float, default=15, help="Significance below which to terminate the rounds. Default is 15." ) veto_settings.add_option( "--snr-thresh", action="append", help="Add an SNR threshold to use in veto round. Can be given multiple times for different values. WARNING: This will override the default settings, *not* append to them." ) veto_settings.add_option( "--time-window", action="append", help="Add a time window to use in veto round. Can be given multiple times for different values. WARNING: This will override the default settings, *not* append to them." ) veto_settings.add_option( "-S", "--min-ref-snr", type=float, default=8, help="Minimum SNR threshold to load a trigger in the reference channel." ) # FIXME: Strictly speaking the ignore list is required because I'm not # sure what the function will do with out one? veto_settings.add_option( "-I", "--ignore-list", help="Text file, one channel per line with a list of channels to ignore when loading triggers." ) # FIXME: #veto_settings.add_option( "-C", "--ignore-channel", action="append", help="Ignore these channels. Given several times, will ignore several channels. Do not prepend instrument. E.g. -C LSC-DARM_CTRL." ) veto_settings.add_option( "--write-coinc", action="store_true", default=False, help="If set, output table will include coinc tables indicating which triggers were coincided in the process of execution." ) opt.add_option_group( veto_settings ) livetime_settings = OptionGroup( opt, "livetime settings" ) # FIXME: #livetime_settings.add_option( "-L", "--livetime-definer", action="append", help="Name of segment definer entry from which to draw live segments. See '-l' option. If none is indicated, use all segments. Provide several times for deveral different segment definers." ) livetime_settings.add_option( "-l", "--livetime-segments", help="File from which to parse livetime segments. Will assume, at first, a LIGOLW XML file with valid segment definer and segment tables. If this fails, will try segwizard format. Required." ) livetime_settings.add_option( "--segment-definer", help="In tandem with --livetime-segments will retrieve segments with this definer. If none is provided, all segments will be used. Note: this option is REQUIRED if querying a databse. Example: H1:DMT-SCIENCE:1 (version is required)" ) livetime_settings.add_option( "--segment-database", help="Query this URL for segments. Takes precedence over providing a file." ) livetime_settings.add_option( "--gps-start", type=int, help="GPS start of analysis." ) livetime_settings.add_option( "--gps-end", type=int, help="GPS end of analysis." ) opt.add_option_group( livetime_settings ) opts, args = opt.parse_args() if opts.instrument is None: print >>sys.stderr, "Instrument must be indicated." exit() if opts.reference_channel is None: print >>sys.stderr, "Reference channel must be indicated." exit() if opts.reference_triggers is None: print >>sys.stderr, "Reference triggers must be present." exit() if (opts.livetime_segments or opts.segment_database) is None: print >>sys.stderr, "Must provide livetime segments file or segment database location." exit() if opts.segment_database and (opts.segment_definer is None): print >>sys.stderr, "Must provide definer for segment database querying." exit() if len(args) == 0 and opts.input_cache is None: print >>sys.stderr, "Must provide input arguments or set --input-cache." exit() if opts.input_cache is not None: with open(opts.input_cache) as cache: c = Cache.fromfile(cache) args.extend( c.pfnlist() ) if opts.ignore_list is None: print >>sys.stderr, "Must provide a channel ignore list." exit() return opts, args
def main(args=None):
"""Run the online trigger visualization tool
"""
parser = create_parser()
args = parser.parse_args(args=args)
args.epoch = args.epoch or args.gpsstart
args.columns = args.columns or DEFAULT_COLUMNS
if len(args.columns) < 2:
parser.error("--columns must receive at least two columns, "
"got {0}".format(len(args.columns)))
args.x_column = args.x_column or args.columns[0]
args.y_column = args.y_column or args.columns[1]
if not args.color and len(args.columns) >= 3:
args.color = args.columns[2]
# add columns for tile plot
for c in ('duration', 'bandwidth'):
if args.tiles and c not in args.columns:
args.columns.append(c)
span = Segment(args.gpsstart, args.gpsend)
# format default params
params = {
'xscale': 'auto-gps',
'epoch': args.epoch or args.gpsstart,
'xlim': (args.gpsstart, args.gpsend),
'yscale': 'log',
'ylabel': 'Frequency [Hz]',
'cmap': get_plot('triggers').defaults.get('cmap', 'YlGnBu'),
'clim': (3, 50),
'colorlabel': 'Signal-to-noise ratio (SNR)',
}
# update with user params
for input_ in args.plot_params or []:
key, val = input_.split('=', 1)
params[key.strip('-')] = safe_eval(val)
# -- load triggers ------------------------------
# get segments
if args.state:
segs = get_segments(args.state, [span])
# read cache
if args.cache_file:
with open(args.cache_file, 'r') as f:
cache = Cache.fromfile(f).sieve(segment=span)
LOGGER.info("Read cache of {0} files".format(len(cache)))
else:
cache = None
# get triggers
trigs = get_triggers(
args.channel,
args.etg, [span],
cache=cache,
columns=args.columns,
verbose='Reading {0.channel} ({0.etg}) events'.format(args))
LOGGER.info("Read {0} events".format(len(trigs)))
if args.state:
trigs = keep_in_segments(trigs, segs.active, etg=args.etg)
LOGGER.info("{0} events in state {1!r}".format(len(trigs), args.state))
if args.snr:
trigs = trigs[trigs['snr'] >= args.snr]
LOGGER.info("{0} events remaining with snr >= {1.snr}".format(
len(trigs), args))
# -- make plot ----------------------------------
# format keywords for plot creation
plot_kw = OrderedDict(
(key, params.pop(key))
for key in ('xscale', 'xlim', 'epoch', 'yscale', 'ylabel'))
# create plot
if args.tiles:
plot = trigs.tile(args.x_column,
args.y_column,
'duration',
'bandwidth',
color=args.color,
edgecolor=params.pop('edgecolor', 'face'),
linewidth=params.pop('linewidth', .8),
**plot_kw)
else:
plot = trigs.scatter(args.x_column,
args.y_column,
color=args.color,
edgecolor=params.pop('edgecolor', 'none'),
s=params.pop('s', 12),
**plot_kw)
ax = plot.gca()
mappable = ax.collections[0]
# set mappable properties
vmin, vmax = params.pop('clim', (3, 50))
if params.pop('logcolor', True):
mappable.set_norm(LogNorm(vmin=vmin, vmax=vmax))
if mappable._A is None:
mappable._A = ndarray((0, ))
# draw colorbar
clabel = params.pop(
'colorlabel',
'Signal-to-noise ratio (SNR)',
)
cmap = params.pop(
'cmap',
get_plot('triggers').defaults.get('cmap', 'YlGnBu'),
)
ax.colorbar(mappable=mappable, label=clabel, cmap=cmap, clim=(vmin, vmax))
for key, val in params.items():
try:
getattr(ax, 'set_%s' % key)(val)
except AttributeError:
setattr(ax, key, val)
# add segments
if args.state:
sax = plot.add_segments_bar(segs,
ax=ax,
label=args.state_label or args.state)
setp(sax.get_yticklabels(), fontsize=10)
sax.set_epoch(args.epoch)
# save and exit
plot.save(args.output_file)
LOGGER.info("Plot saved to {0.output_file}".format(args))
