def find_dmt_omega(channel, start, end, base=None): """Find DMT-Omega trigger XML files """ span = Segment(to_gps(start), to_gps(end)) channel = get_channel(channel) ifo = channel.ifo if base is None and channel.name.split(':', 1)[-1] == 'GDS-CALIB_STRAIN': base = '/gds-%s/dmt/triggers/%s-HOFT_Omega' % ( ifo.lower(), ifo[0].upper()) elif base is None: raise NotImplementedError("This method doesn't know how to locate DMT " "Omega trigger files for %r" % str(channel)) gps5 = int('%.5s' % start) end5 = int('%.5s' % end) out = Cache() append = out.append while gps5 <= end5: trigglob = os.path.join( base, str(gps5), '%s-%s_%s_%s_OmegaC-*-*.xml' % ( ifo, channel.system, channel.subsystem, channel.signal)) found = glob.glob(trigglob) for f in found: ce = CacheEntry.from_T050017(f) if ce.segment.intersects(span): append(ce) gps5 += 1 out.sort(key=lambda e: e.path) vprint(" Found %d files for %s (DMT-Omega)\n" % (len(out), channel.ndsname)) return out
def find_kw(channel, start, end, base=None): """Find KW trigger XML files """ span = Segment(to_gps(start), to_gps(end)) channel = get_channel(channel) ifo = channel.ifo if base is None and channel.name.split(':', 1)[-1] == 'GDS-CALIB_STRAIN': tag = '%s-KW_HOFT' % ifo[0].upper() base = '/gds-%s/dmt/triggers/%s' % (ifo.lower(), tag) elif base is None: tag = '%s-KW_TRIGGERS' % ifo[0].upper() base = '/gds-%s/dmt/triggers/%s' % (ifo.lower(), tag) gps5 = int('%.5s' % start) end5 = int('%.5s' % end) out = Cache() append = out.append while gps5 <= end5: trigglob = os.path.join( base, '%s-%d' % (tag, gps5), '%s-*-*.xml' % tag) found = glob.glob(trigglob) for f in found: ce = CacheEntry.from_T050017(f) if ce.segment.intersects(span): append(ce) gps5 += 1 out.sort(key=lambda e: e.path) vprint(" Found %d files for %s (KW)\n" % (len(out), channel.ndsname)) return out
def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False): """Find the paths of trigger files that represent the given observatory, channel, and ETG (event trigger generator) for a given GPS [start, end) segment. """ if etg.lower() == 'omicron': etg = '?micron' # construct search span = Segment(gpsstart, gpsend) ifo, channel = channel.split(':', 1) trigtype = "%s_%s" % (channel, etg.lower()) epoch = '*' searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype) gpsdirs = range(int(str(gpsstart)[:5]), int(str(gpsend)[:5]) + 1) trigform = ('%s-%s_%s-%s-*.xml*' % (ifo, re_dash.sub('_', channel), etg.lower(), '[0-9]' * 10)) # perform and cache results out = Cache() for gpsdir in gpsdirs: gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform) if verbose: gprint("Searching %s..." % os.path.split(gpssearchpath)[0], end=' ') gpscache = Cache( map(CacheEntry.from_T050017, glob.glob(os.path.join(searchbase, str(gpsdir), trigform)))) out.extend(gpscache.sieve(segment=span)) if verbose: gprint("%d found" % len(gpscache.sieve(segment=span))) out.sort(key=lambda e: e.path) return out
def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False): """Find the paths of trigger files that represent the given observatory, channel, and ETG (event trigger generator) for a given GPS [start, end) segment. """ if etg.lower() == 'omicron': etg = '?micron' # construct search span = Segment(gpsstart, gpsend) ifo, channel = channel.split(':', 1) trigtype = "%s_%s" % (channel, etg.lower()) epoch = '*' searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype) gpsdirs = range(int(str(gpsstart)[:5]), int(str(gpsend)[:5])+1) trigform = ('%s-%s_%s-%s-*.xml*' % (ifo, re_dash.sub('_', channel), etg.lower(), '[0-9]'*10)) # perform and cache results out = Cache() for gpsdir in gpsdirs: gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform) if verbose: gprint("Searching %s..." % os.path.split(gpssearchpath)[0], end =' ') gpscache = Cache(map(CacheEntry.from_T050017, glob.glob(os.path.join(searchbase, str(gpsdir), trigform)))) out.extend(gpscache.sieve(segment=span)) if verbose: gprint("%d found" % len(gpscache.sieve(segment=span))) out.sort(key=lambda e: e.path) return out
def find_online_cache(start, end, channel, **kwargs): """Find ExcessPower files from the online GSTLAL analysis for the given span @param start GPS start time for search @param end GPS end time for search @param channel UNDOCUMENTED @param kwargs UNDOCUMENTED 'ifo' observatory for search 'clustering' tag for clustering stage to search, default: unclustered 'check_files' check that the returned files can be read on disk, default False """ out = Cache() # set base directory directory = kwargs.pop("directory", ER3_RUN_DIRECTORY) ifo,channel = channel.split(":", 1) channel_dir = os.path.join(directory, ifo, "%s_excesspower" % channel) glob_query = "%s-%s_excesspower-*.xml" % (ifo, channel.replace("-", "_")) span = Segment(start, end) # optimise append = out.append splitext = os.path.splitext isfile = os.path.isfile pjoin = os.path.join intersects = span.intersects from_T050017 = CacheEntry.from_T050017 # loop over days gathering files t = start // 1e4 * 1e4 while t < end: gps_dir = os.path.join(channel_dir, "%.6s" % t) if os.path.isdir(gps_dir): file_list = glob(os.path.join(gps_dir, glob_query)) for f in file_list: e = from_T050017(f) if intersects(e.segment): append(e) t += 1e4 out.sort(key=lambda e: e.segment[0]) return out
def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False): """Find the paths of trigger files that represent the given observatory, channel, and ETG (event trigger generator) for a given GPS [start, end) segment. """ if etg.lower().startswith('omicron'): etg = '?' + etg[1:] # construct search gpsstart = to_gps(gpsstart).seconds gpsend = to_gps(gpsend).seconds span = Segment(gpsstart, gpsend) ifo, channel = channel.split(':', 1) trigtype = "%s_%s" % (channel, etg.lower()) epoch = '*' searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype) gpsdirs = range(int(str(gpsstart)[:5]), int(str(gpsend)[:5]) + 1) trigform = ('%s-%s_%s-%s-*.xml*' % (ifo, re_dash.sub('_', channel), etg.lower(), '[0-9]' * 10)) # test for channel-level directory if not glob.glob(searchbase): raise ValueError("No channel-level directory found at %s. Either the " "channel name or ETG names are wrong, or this " "channel is not configured for this ETG." % searchbase) # perform and cache results out = Cache() append = out.append for gpsdir in gpsdirs: gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform) if verbose: gprint("Searching %s..." % os.path.split(gpssearchpath)[0], end=' ') found = set( map(os.path.realpath, glob.glob(os.path.join(searchbase, str(gpsdir), trigform)))) n = 0 for f in found: ce = CacheEntry.from_T050017(f) if ce.segment.intersects(span): append(ce) n += 1 if verbose: gprint("%d found" % n) out.sort(key=lambda e: e.path) return out
def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False): """Find the paths of trigger files that represent the given observatory, channel, and ETG (event trigger generator) for a given GPS [start, end) segment. """ if etg.lower().startswith('omicron'): etg = '?' + etg[1:] # construct search gpsstart = to_gps(gpsstart).seconds gpsend = to_gps(gpsend).seconds span = Segment(gpsstart, gpsend) ifo, channel = channel.split(':', 1) trigtype = "%s_%s" % (channel, etg.lower()) epoch = '*' searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype) gpsdirs = range(int(str(gpsstart)[:5]), int(str(gpsend)[:5])+1) trigform = ('%s-%s_%s-%s-*.xml*' % (ifo, re_dash.sub('_', channel), etg.lower(), '[0-9]'*10)) # test for channel-level directory if not glob.glob(searchbase): raise ValueError("No channel-level directory found at %s. Either the " "channel name or ETG names are wrong, or this " "channel is not configured for this ETG." % searchbase) # perform and cache results out = Cache() append = out.append for gpsdir in gpsdirs: gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform) if verbose: gprint("Searching %s..." % os.path.split(gpssearchpath)[0], end=' ') found = set(map( os.path.realpath, glob.glob(os.path.join(searchbase, str(gpsdir), trigform)))) n = 0 for f in found: ce = CacheEntry.from_T050017(f) if ce.segment.intersects(span): append(ce) n += 1 if verbose: gprint("%d found" % n) out.sort(key=lambda e: e.path) return out
def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False, **kwargs): """Find the paths of trigger files that represent the given observatory, channel, and ETG (event trigger generator) for a given GPS [start, end) segment. """ # special case for KW if etg.lower() in ['kw', 'kleinewelle']: from .kw import find_dmt_cache ifo = channel.split(':')[0] kwargs.setdefault('extension', 'xml') kwargs.setdefault('check_files', True) return find_dmt_cache(gpsstart, gpsend, ifo, **kwargs) elif etg.lower() == 'omega': from .omega import find_dmt_cache ifo = channel.split(':')[0] kwargs.setdefault('check_files', True) return find_dmt_cache(gpsstart, gpsend, ifo, **kwargs) elif etg.lower() == 'omicron': etg = '?micron' # construct search span = segments.segment(gpsstart, gpsend) ifo, channel = channel.split(':', 1) trigtype = "%s_%s" % (channel, etg.lower()) epoch = '*' searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype) gpsdirs = numpy.arange(int(str(gpsstart)[:5]), int(str(gpsend)[:5])+1) trigform = ('%s-%s_%s-%s-*.xml*' % (ifo, re.sub('-', '_', channel), etg.lower(), '[0-9]'*10)) # perform and cache results out = Cache() for gpsdir in gpsdirs: gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform) if verbose: sys.stdout.write("Searching %s..." % os.path.split(gpssearchpath)[0]) sys.stdout.flush() gpscache = Cache(map(CacheEntry.from_T050017, glob.glob(os.path.join(searchbase, str(gpsdir), trigform)))) out.extend(gpscache.sieve(segment=span)) if verbose: sys.stdout.write(" %d found\n" % len(gpscache.sieve(segment=span))) out.sort(key=lambda e: e.path) return out
def find_dmt_cache(start, end, ifo, extension="xml", check_files=False, **kwargs): """Find DMT KW files for the given GPS period. @param start GPS start time for search @param end GPS end time for search @param ifo observatory for search @param extension UNDOCUMENTED @param check_files check that the returned files can be read on disk, default False @param kwargs UNDOCUMENTED """ out = Cache() # verify host host = {'G1': 'atlas', 'H1': 'ligo-wa', 'H2': 'ligo-wa', 'L1': 'ligo-la'} if (not kwargs.has_key('directory') and not re.search(host[ifo], getfqdn())): sys.stderr.write("WARNING: KW online files are not available for " "IFO=%s on this host." % ifo) sys.stderr.flush() return out span = segments.segment(start, end) # set known epochs known_epochs = [1026263104] # get parameters dt = int(kwargs.pop("duration", 64)) epoch = kwargs.pop("epoch", known_epochs) filetag = kwargs.pop("filetag", "KW_TRIGGERS") dirtag = filetag.endswith("_TRENDS") and filetag[:-7] or filetag try: iter(epoch) except TypeError: epoch = [int(epoch)] overlap = int(kwargs.pop("overlap", 0)) directory = kwargs.pop( "duration", "/gds-%s/dmt/triggers/%s-%s" % (ifo.lower(), ifo[0].upper(), dirtag)) # optimise append = out.append splitext = os.path.splitext isfile = os.path.isfile intersects = span.intersects segment = segments.segment from_T050017 = CacheEntry.from_T050017 # get times epoch_idx = bisect.bisect_right(epoch, start) - 1 next_epoch = len(epoch) >= epoch_idx + 2 and epoch[epoch_idx + 1] or 0 start_time = int(start - numpy.mod(start - epoch[epoch_idx], dt - overlap)) t = start_time def _kw_file(gps, ifo): return ("%s/%s-%s-%.5s/" "%s-%s-%.10d-%d.%s" % (directory, ifo.upper()[0], dirtag, gps, ifo.upper()[0], filetag, gps, dt, extension)) # loop over time segments constructing file paths while t < end: fp = _kw_file(t, ifo) if (intersects(segment(t, t + dt)) and (not check_files or isfile(fp))): append(from_T050017(fp)) t += dt - overlap if next_epoch and t > next_epoch: t = next_epoch epoch_idx += 1 next_epoch = len(epoch) >= epoch_idx + 2 and epoch[epoch_idx + 1] or 0 out.sort(key=lambda e: e.path) return out
def find_online_cache(start, end, ifo, mask='DOWNSELECT', check_files=False, **kwargs): """Find KW Online files for the given GPS period. @param start GPS start time for search @param end GPS end time for search @param ifo observatory for search @param mask description tag of KW ASCII to search @param check_files check that the returned files can be read on disk, default False @param kwargs UNDOCUMENTED """ out = Cache() # verify host host = {'G1': 'atlas', 'H1': 'ligo-wa', 'H2': 'ligo-wa', 'L1': 'ligo-la'} if (not kwargs.has_key('directory') and not re.search(host[ifo], getfqdn())): sys.stderr.write("WARNING: KW online files are not available for " "IFO=%s on this host." % ifo) sys.stderr.flush() return out span = segments.segment(start, end) # get parameters dt = kwargs.pop("duration", 64) overlap = kwargs.pop("overlap", 8) if ifo == "G1": directory = kwargs.pop("directory", "/home/omega/online/G1/segments") epoch = kwargs.pop("epoch", 983669456) else: directory = kwargs.pop("directory",\ "/home/omega/online/%s/archive/S6/segments" % ifo) # optimise append = out.append splitext = os.path.splitext isfile = os.path.isfile intersects = span.intersects segment = segments.segment from_T050017 = CacheEntry.from_T050017 # get times start_time = int(start - numpy.mod(start - epoch, dt - overlap)) t = start_time if ifo == "G1": def _omega_file(gps, ifo): return ("%s/%.5d/%.10d-%.10d/%s-KW_TRIGGERS_%s-%.10d-%d.txt" % (basedir, gps / 100000, gps, gps + dt, ifo, mask, gps, dt)) else: def _omega_file(gps, ifo): return ("%s/%s-%s/%s-KW_TRIGGERS_%s-%s-%s.txt" % (basedir, gps, gps + dt, ifo, mask, gps, dt)) # loop over time segments constructing file paths while t < end: fp = _omega_file(t, ifo) if (intersects(segment(t, t + dt)) and (not check_files or isfile(fp))): append(from_T050017(fp)) t += dt - overlap out.sort(key=lambda e: e.path) return out
def find_dmt_cache(start, end, ifo, check_files=False, **kwargs): """Find DMTOmega files for the given GPS period. @param start GPS start time for search @param end GPS end time for search @param ifo observatory for search @param check_files check that the returned files can be read on disk, default False @param kwargs UNDOCUMENTED """ out = Cache() # verify host host = {'G1': 'atlas', 'H1': 'ligo-wa', 'H2': 'ligo-wa', 'L1': 'ligo-la'} if (not kwargs.has_key('directory') and not re.search(host[ifo], getfqdn())): sys.stderr.write("WARNING: Omega online files are not available for " "IFO=%s on this host." % ifo) sys.stderr.flush() return out span = segments.segment(start, end) # set known epochs known_epochs = { 1031340854: 55, 1041657635: 55, 1041669472: 55, 1041682187: 55, 1044093810: 38, 1044111232: 38, 1044111282: 38, 1044112180: 38, 1057700030: 38, 1057722672: 38 } # get parameters epoch = kwargs.pop("epoch", sorted(known_epochs.keys())) dt = kwargs.pop("duration", 55) try: iter(epoch) except TypeError: epoch = [epoch] overlap = kwargs.pop("overlap", 0) directory = kwargs.pop( "duration", "/gds-%s/dmt/triggers/%s-Omega_Triggers" % (ifo.lower(), ifo[0].upper())) # optimise append = out.append splitext = os.path.splitext isfile = os.path.isfile intersects = span.intersects segment = segments.segment from_T050017 = CacheEntry.from_T050017 # get times epoch_idx = bisect.bisect_right(epoch, start) - 1 print epoch_idx try: dt = known_epochs[epoch[epoch_idx]] except KeyError: dt = 38 next_epoch = len(epoch) >= epoch_idx + 2 and epoch[epoch_idx + 1] or 0 start_time = int(start - numpy.mod(start - epoch[epoch_idx], dt - overlap)) t = start_time def _omega_file(gps, ifo, deltaT): return ("%s/%s-OMEGA_TRIGGERS_CLUSTER-%.5s/" "%s-OMEGA_TRIGGERS_CLUSTER-%.10d-%d.xml" % (directory, ifo.upper(), gps, ifo.upper(), gps, deltaT)) # loop over time segments constructing file paths while t < end: fp = _omega_file(t, ifo, dt) if (intersects(segment(t, t + dt)) and (not check_files or isfile(fp))): append(from_T050017(fp)) t += dt - overlap if next_epoch and t > next_epoch: try: dt = known_epochs[next_epoch] except KeyError: dt = 55 t = next_epoch epoch_idx += 1 next_epoch = len(epoch) >= epoch_idx + 2 and epoch[epoch_idx + 1] or 0 out.sort(key=lambda e: e.path) return out
def find_dmt_cache(start, end, ifo, check_files=False, **kwargs): """Find DMTOmega files for the given GPS period. @param start GPS start time for search @param end GPS end time for search @param ifo observatory for search @param check_files check that the returned files can be read on disk, default False @param kwargs UNDOCUMENTED """ out = Cache() # verify host host = { 'G1':'atlas', 'H1':'ligo-wa', 'H2':'ligo-wa', 'L1':'ligo-la'} if (not kwargs.has_key('directory') and not re.search(host[ifo],getfqdn())): sys.stderr.write("WARNING: Omega online files are not available for " "IFO=%s on this host." % ifo) sys.stderr.flush() return out span = segments.segment(start,end) # set known epochs known_epochs = {1031340854:55, 1041657635:55, 1041669472:55, 1041682187:55, 1044093810:38, 1044111232:38, 1044111282:38, 1044112180:38, 1057700030:38, 1057722672:38} # get parameters epoch = kwargs.pop("epoch", sorted(known_epochs.keys())) dt = kwargs.pop("duration", 55) try: iter(epoch) except TypeError: epoch = [epoch] overlap = kwargs.pop("overlap", 0) directory = kwargs.pop("duration", "/gds-%s/dmt/triggers/%s-Omega_Triggers" % (ifo.lower(), ifo[0].upper())) # optimise append = out.append splitext = os.path.splitext isfile = os.path.isfile intersects = span.intersects segment = segments.segment from_T050017 = CacheEntry.from_T050017 # get times epoch_idx = bisect.bisect_right(epoch, start)-1 print epoch_idx try: dt = known_epochs[epoch[epoch_idx]] except KeyError: dt = 38 next_epoch = len(epoch) >= epoch_idx+2 and epoch[epoch_idx+1] or 0 start_time = int(start-numpy.mod(start-epoch[epoch_idx], dt-overlap)) t = start_time def _omega_file(gps, ifo, deltaT): return ("%s/%s-OMEGA_TRIGGERS_CLUSTER-%.5s/" "%s-OMEGA_TRIGGERS_CLUSTER-%.10d-%d.xml" % (directory, ifo.upper(), gps, ifo.upper(), gps, deltaT)) # loop over time segments constructing file paths while t<end: fp = _omega_file(t, ifo, dt) if (intersects(segment(t, t+dt)) and (not check_files or isfile(fp))): append(from_T050017(fp)) t += dt - overlap if next_epoch and t > next_epoch: try: dt = known_epochs[next_epoch] except KeyError: dt = 55 t = next_epoch epoch_idx += 1 next_epoch = len(epoch) >= epoch_idx+2 and epoch[epoch_idx+1] or 0 out.sort(key=lambda e: e.path) return out
def find_online_cache(start, end, ifo, mask='DOWNSELECT', check_files=False, **kwargs): """Find Omega Online files for the given GPS period. @param start GPS start time for search @param end GPS end time for search @param ifo observatory for search @param mask description tag of Omega ASCII to search @param check_files check that the returned files can be read on disk, default False @param kwargs UNDOCUMENTED """ out = Cache() # verify host host = { 'G1':'atlas', 'H1':'ligo-wa', 'H2':'ligo-wa', 'L1':'ligo-la'} if (not kwargs.has_key('directory') and not re.search(host[ifo],getfqdn())): sys.stderr.write("WARNING: Omega online files are not available for " "IFO=%s on this host." % ifo) sys.stderr.flush() return out span = segments.segment(start,end) # get parameters dt = kwargs.pop("duration", 64) overlap = kwargs.pop("overlap", 8) if ifo == "G1": directory = kwargs.pop("directory", "/home/omega/online/G1/segments") epoch = kwargs.pop("epoch", 983669456) else: directory = kwargs.pop("directory",\ "/home/omega/online/%s/archive/S6/segments" % ifo) # optimise append = out.append splitext = os.path.splitext isfile = os.path.isfile intersects = span.intersects segment = segments.segment from_T050017 = CacheEntry.from_T050017 # get times start_time = int(start-numpy.mod(start-epoch, dt-overlap)) t = start_time if ifo == "G1": def _omega_file(gps, ifo): return ("%s/%.5d/%.10d-%.10d/%s-OMEGA_TRIGGERS_%s-%.10d-%d.txt" % (basedir, gps/100000, gps, gps+dt, ifo, mask, gps, dt)) else: def _omega_file(gps, ifo): return ("%s/%s-%s/%s-OMEGA_TRIGGERS_%s-%s-%s.txt" % (basedir, gps, gps+dt, ifo, mask, gps, dt)) # loop over time segments constructing file paths while t<end: fp = _omega_file(t, ifo) if (intersects(segment(t, t+dt)) and (not check_files or isfile(fp))): append(from_T050017(fp)) t += dt - overlap out.sort(key=lambda e: e.path) return out
def get_cache(start, end, ifo, channel, mask='DOWNSELECT', checkfilesexist=False,\ **kwargs): """ Returns a glue.lal.Cache contatining CacheEntires for all omega online trigger files between the given start and end time for the given ifo. """ cache = Cache() # verify host host = { 'G1':'atlas', 'H1':'ligo-wa', 'H2':'ligo-wa', 'L1':'ligo-la'} if (not kwargs.has_key('directory') and not re.search(host[ifo],getfqdn())): sys.stderr.write("warning: Omega online files are not available for "+\ "IFO=%s on this host." % ifo) sys.stderr.flush() return cache span = segments.segment(start,end) if ifo == 'G1': if channel: kwargs.setdefault('directory', '/home/omega/online/%s/segments' % channel.replace(':','_')) else: kwargs.setdefault('directory', '/home/omega/online/G1/segments') kwargs.setdefault('epoch', 0) else: kwargs.setdefault('directory',\ '/home/omega/online/%s/archive/S6/segments' % ifo) kwargs.setdefault('epoch', 931211808) kwargs.setdefault('duration', 64) kwargs.setdefault('overlap', 8) # optimise append = cache.append splitext = os.path.splitext isfile = os.path.isfile intersects = span.intersects segment = segments.segment from_T050017 = CacheEntry.from_T050017 basedir = kwargs['directory'] basetime = kwargs['epoch'] triglength = kwargs['duration'] overlap = kwargs['overlap'] # get times start_time = int(start-numpy.mod(start-basetime,triglength-overlap)) t = start_time # loop over time segments constructing file paths and appending to the cache while t<end: if ifo == 'G1': trigfile = '%s/%.5d/%.10d-%10.d/%s-OMEGA_TRIGGERS_%s-%.10d-%d.txt'\ % (basedir, t/100000, t, t+triglength, ifo, mask, t, triglength) else: trigfile = '%s/%.10d-%10.d/%s-OMEGA_TRIGGERS_%s-%.10d-%d.txt'\ % (basedir, t, t+triglength, ifo, mask, t, triglength) if intersects(segment(t, t+triglength))\ and (not checkfilesexist or isfile(trigfile)): append(from_T050017(trigfile)) t+=triglength-overlap cache.sort(key=lambda e: e.path) return cache
def get_cache(start, end, ifo, channel, mask='DOWNSELECT', checkfilesexist=False,\ **kwargs): """ Returns a glue.lal.Cache contatining CacheEntires for all omega online trigger files between the given start and end time for the given ifo. """ cache = Cache() # verify host host = {'G1': 'atlas', 'H1': 'ligo-wa', 'H2': 'ligo-wa', 'L1': 'ligo-la'} if (not kwargs.has_key('directory') and not re.search(host[ifo], getfqdn())): sys.stderr.write("warning: Omega online files are not available for "+\ "IFO=%s on this host." % ifo) sys.stderr.flush() return cache span = segments.segment(start, end) if ifo == 'G1': if channel: kwargs.setdefault( 'directory', '/home/omega/online/%s/segments' % channel.replace(':', '_')) else: kwargs.setdefault('directory', '/home/omega/online/G1/segments') kwargs.setdefault('epoch', 0) else: kwargs.setdefault('directory',\ '/home/omega/online/%s/archive/S6/segments' % ifo) kwargs.setdefault('epoch', 931211808) kwargs.setdefault('duration', 64) kwargs.setdefault('overlap', 8) # optimise append = cache.append splitext = os.path.splitext isfile = os.path.isfile intersects = span.intersects segment = segments.segment from_T050017 = CacheEntry.from_T050017 basedir = kwargs['directory'] basetime = kwargs['epoch'] triglength = kwargs['duration'] overlap = kwargs['overlap'] # get times start_time = int(start - numpy.mod(start - basetime, triglength - overlap)) t = start_time # loop over time segments constructing file paths and appending to the cache while t < end: if ifo == 'G1': trigfile = '%s/%.5d/%.10d-%10.d/%s-OMEGA_TRIGGERS_%s-%.10d-%d.txt'\ % (basedir, t/100000, t, t+triglength, ifo, mask, t, triglength) else: trigfile = '%s/%.10d-%10.d/%s-OMEGA_TRIGGERS_%s-%.10d-%d.txt'\ % (basedir, t, t+triglength, ifo, mask, t, triglength) if intersects(segment(t, t+triglength))\ and (not checkfilesexist or isfile(trigfile)): append(from_T050017(trigfile)) t += triglength - overlap cache.sort(key=lambda e: e.path) return cache
def find_dmt_cache(start, end, ifo, extension="xml", check_files=False, **kwargs): """Find DMT KW files for the given GPS period. @param start GPS start time for search @param end GPS end time for search @param ifo observatory for search @param extension UNDOCUMENTED @param check_files check that the returned files can be read on disk, default False @param kwargs UNDOCUMENTED """ out = Cache() # verify host host = { 'G1':'atlas', 'H1':'ligo-wa', 'H2':'ligo-wa', 'L1':'ligo-la'} if (not kwargs.has_key('directory') and not re.search(host[ifo],getfqdn())): sys.stderr.write("WARNING: KW online files are not available for " "IFO=%s on this host." % ifo) sys.stderr.flush() return out span = segments.segment(start,end) # set known epochs known_epochs = [1026263104] # get parameters dt = int(kwargs.pop("duration", 64)) epoch = kwargs.pop("epoch", known_epochs) filetag = kwargs.pop("filetag", "KW_TRIGGERS") dirtag = filetag.endswith("_TRENDS") and filetag[:-7] or filetag try: iter(epoch) except TypeError: epoch = [int(epoch)] overlap = int(kwargs.pop("overlap", 0)) directory = kwargs.pop("duration", "/gds-%s/dmt/triggers/%s-%s" % (ifo.lower(), ifo[0].upper(), dirtag)) # optimise append = out.append splitext = os.path.splitext isfile = os.path.isfile intersects = span.intersects segment = segments.segment from_T050017 = CacheEntry.from_T050017 # get times epoch_idx = bisect.bisect_right(epoch, start)-1 next_epoch = len(epoch) >= epoch_idx+2 and epoch[epoch_idx+1] or 0 start_time = int(start-numpy.mod(start-epoch[epoch_idx], dt-overlap)) t = start_time def _kw_file(gps, ifo): return ("%s/%s-%s-%.5s/" "%s-%s-%.10d-%d.%s" % (directory, ifo.upper()[0], dirtag, gps, ifo.upper()[0], filetag, gps, dt, extension)) # loop over time segments constructing file paths while t<end: fp = _kw_file(t, ifo) if (intersects(segment(t, t+dt)) and (not check_files or isfile(fp))): append(from_T050017(fp)) t += dt - overlap if next_epoch and t > next_epoch: t = next_epoch epoch_idx += 1 next_epoch = len(epoch) >= epoch_idx+2 and epoch[epoch_idx+1] or 0 out.sort(key=lambda e: e.path) return out
def find_daily_cache(start, end, ifo, clustering=None, check_files=False, **kwargs): """Find Daily ihope files from the daily runs for the given span @param start GPS start time for search @param end GPS end time for search @param ifo observatory for search @param clustering tag for clustering stage to search, default: unclustered @param check_files check that the returned files can be read on disk, default False @param kwargs UNDOCUMENTED """ out = Cache() # set clustering tag if clustering==None or clustering.upper()=='UNCLUSTERED': file_tag='INSPIRAL_UNCLUSTERED' elif clustering.upper() in ["100MS", "100MILLISEC"]: file_tag='INSPIRAL_100MILLISEC_CLUSTERED' elif clustering.upper() in ["30MS", "30MILLISEC"]: file_tag='INSPIRAL_30MILLISEC_CLUSTERED' elif clustering.upper() in ["16S", "16SECOND"]: file_tag='INSPIRAL_16SEC_CLUSTERED' # set base directory directory = kwargs.pop("directory", os.path.expanduser("~cbc/ihope_daily")) # work out days span = Segment(start, end) start = int(start) start_d = lal.UTCToGPS(datetime(*lal.GPSToUTC(start)[:6]).replace( hour=0, minute=0, second=0).timetuple()) days = [] day = start_d while day <= end: days.append(day) day+=86400 # optimise append = out.append splitext = os.path.splitext isfile = os.path.isfile pjoin = os.path.join intersects = span.intersects from_T050017 = CacheEntry.from_T050017 # loop over days gathering files for day in days: utc = datetime(*lal.GPSToUTC(day)[:6]) day_path = pjoin(directory, utc.strftime("%Y%m"), utc.strftime("%Y%m%d")) day_cache = os.path.join(day_path, "%s-%s.cache" % (ifo, file_tag)) if isfile(day_cache): with open(day_cache, "r") as f: filenames = Cache.fromfile(f).pfnlist() else: filenames = glob(os.path.join(day_path, ("%s-%s-*.xml.gz" % (ifo, file_tag)))) for filename in filenames: e = from_T050017(filename) if intersects(e.segment): append(e) out.sort(key=lambda e: e.path) return out
def find_daily_cache(start, end, ifo, clustering=None, check_files=False, **kwargs): """Find Daily ihope files from the daily runs for the given span @param start GPS start time for search @param end GPS end time for search @param ifo observatory for search @param clustering tag for clustering stage to search, default: unclustered @param check_files check that the returned files can be read on disk, default False @param kwargs UNDOCUMENTED """ out = Cache() # set clustering tag if clustering == None or clustering.upper() == 'UNCLUSTERED': file_tag = 'INSPIRAL_UNCLUSTERED' elif clustering.upper() in ["100MS", "100MILLISEC"]: file_tag = 'INSPIRAL_100MILLISEC_CLUSTERED' elif clustering.upper() in ["30MS", "30MILLISEC"]: file_tag = 'INSPIRAL_30MILLISEC_CLUSTERED' elif clustering.upper() in ["16S", "16SECOND"]: file_tag = 'INSPIRAL_16SEC_CLUSTERED' # set base directory directory = kwargs.pop("directory", os.path.expanduser("~cbc/ihope_daily")) # work out days span = Segment(start, end) start = int(start) start_d = lal.UTCToGPS( datetime(*lal.GPSToUTC(start)[:6]).replace(hour=0, minute=0, second=0).timetuple()) days = [] day = start_d while day <= end: days.append(day) day += 86400 # optimise append = out.append splitext = os.path.splitext isfile = os.path.isfile pjoin = os.path.join intersects = span.intersects from_T050017 = CacheEntry.from_T050017 # loop over days gathering files for day in days: utc = datetime(*lal.GPSToUTC(day)[:6]) day_path = pjoin(directory, utc.strftime("%Y%m"), utc.strftime("%Y%m%d")) day_cache = os.path.join(day_path, "%s-%s.cache" % (ifo, file_tag)) if isfile(day_cache): with open(day_cache, "r") as f: filenames = Cache.fromfile(f).pfnlist() else: filenames = glob( os.path.join(day_path, ("%s-%s-*.xml.gz" % (ifo, file_tag)))) for filename in filenames: e = from_T050017(filename) if intersects(e.segment): append(e) out.sort(key=lambda e: e.path) return out