Esempio n. 1
0
def read_cache(lcf, coltype=LIGOTimeGPS):
    """Read a LAL-format cache file into memory as a `Cache`

    Parameters
    ----------
    lcf : `str`, `file`
        input file or file path to read

    coltype : `LIGOTimeGPS`, `int`, optional
        `type` for GPS times

    Returns
    -------
    cache : :class:`glue.lal.Cache`
        a cache object, representing each line in the file as a
        :class:`~lal.utils.CacheEntry`
    """
    from glue.lal import Cache
    # open file
    if not isinstance(lcf, FILE_LIKE):
        with open(lcf, 'r') as f:
            return open_cache(f, coltype=coltype)

    # read file
    out = Cache()
    for line in lcf:
        if isinstance(line, bytes):
            line = line.decode('utf-8')
        out.append(out.entry_class(line, coltype=coltype))
    return out
def FrameCachetoLALCache(fcache):

  lcache = LALCache()

  files = fcache.get_files()

  for f in files:
    lcache.append(LALCacheEntry.from_T050017(f))
  
  return lcache
Esempio n. 3
0
    def make_cache():
        try:
            from lal.utils import CacheEntry
        except ImportError as e:
            pytest.skip(str(e))

        segs = SegmentList()
        cache = Cache()
        for seg in [(0, 1), (1, 2), (4, 5)]:
            d = seg[1] - seg[0]
            f = 'A-B-%d-%d.tmp' % (seg[0], d)
            cache.append(CacheEntry.from_T050017(f, coltype=int))
            segs.append(Segment(*seg))
        return cache, segs
Esempio n. 4
0
def find_frames(ifo,
                frametype,
                gpsstart,
                gpsend,
                config=GWSummConfigParser(),
                urltype='file',
                gaps='warn',
                onerror='raise'):
    """Query the datafind server for GWF files for the given type

    Parameters
    ----------
    ifo : `str`
        prefix for the IFO of interest (either one or two characters)

    frametype : `str`
        name of the frametype to find

    gpsstart : `int`
        GPS start time of the query

    gpsend : `int`
        GPS end time of the query

    config : `~ConfigParser.ConfigParser`, optional
        configuration with `[datafind]` section containing `server`
        specification, otherwise taken from the environment

    urltype : `str`, optional
        what type of file paths to return, default: `file`

    gaps : `str`, optional
        what to do when gaps are detected, one of

        - `ignore` : do nothing
        - `warn` : display the existence of gaps but carry on
        - `raise` : raise an exception

    onerror : `str`, optional
        what to do when the `~glue.datafind` query itself fails, same
        options as for ``gaps``

    Returns
    -------
    cache : `~glue.lal.Cache`
        a list of structured frame file descriptions matching the ifo and
        frametype requested
    """
    vprint('    Finding %s-%s frames for [%d, %d)...' %
           (ifo[0], frametype, int(gpsstart), int(gpsend)))
    # find datafind host:port
    try:
        host = config.get('datafind', 'server')
    except (NoOptionError, NoSectionError):
        try:
            host = os.environ['LIGO_DATAFIND_SERVER']
        except KeyError:
            host = None
            port = None
        else:
            try:
                host, port = host.rsplit(':', 1)
            except ValueError:
                port = None
            else:
                port = int(port)
    else:
        port = config.getint('datafind', 'port')
    # get credentials
    if port == 80:
        cert = None
        key = None
    else:
        cert, key = datafind.find_credential()

    # XXX HACK: LLO changed frame types on Dec 6 2013:
    LLOCHANGE = 1070291904
    if re.match('L1_{CRMT}', frametype) and gpsstart < LLOCHANGE:
        frametype = frametype[-1]

    # query frames
    ifo = ifo[0].upper()
    gpsstart = int(floor(gpsstart))
    gpsend = int(ceil(min(globalv.NOW, gpsend)))
    if gpsend <= gpsstart:
        return Cache()

    # parse match
    try:
        frametype, match = frametype.split('|', 1)
    except ValueError:
        match = None

    def _query():
        if cert is not None:
            dfconn = datafind.GWDataFindHTTPSConnection(host=host,
                                                        port=port,
                                                        cert_file=cert,
                                                        key_file=key)
        else:
            dfconn = datafind.GWDataFindHTTPConnection(host=host, port=port)
        return dfconn.find_frame_urls(ifo[0].upper(),
                                      frametype,
                                      gpsstart,
                                      gpsend,
                                      urltype=urltype,
                                      on_gaps=gaps,
                                      match=match)

    try:
        cache = _query()
    except RuntimeError as e:
        sleep(1)
        try:
            cache = _query()
        except RuntimeError:
            if 'Invalid GPS times' in str(e):
                e.args = ('%s: %d ... %s' % (str(e), gpsstart, gpsend), )
            if onerror in ['ignore', None]:
                pass
            elif onerror in ['warn']:
                warnings.warn('Caught %s: %s' % (type(e).__name__, str(e)))
            else:
                raise
            cache = Cache()

    # XXX: if querying for day of LLO frame type change, do both
    if (ifo[0].upper() == 'L' and frametype in ['C', 'R', 'M', 'T']
            and gpsstart < LLOCHANGE < gpsend):
        start = len(cache) and cache[-1].segment[1] or gpsstart
        if start < gpsend:
            cache.extend(
                dfconn.find_frame_urls(ifo[0].upper(),
                                       'L1_%s' % frametype,
                                       start,
                                       gpsend,
                                       urltype=urltype,
                                       on_gaps=gaps)[1:])

    # extend cache beyond datafind's knowledge to reduce latency
    try:
        latest = cache[-1]
        ngps = len(
            re_gwf_gps_epoch.search(os.path.dirname(
                latest.path)).groupdict()['gpsepoch'])
    except (IndexError, AttributeError):
        pass
    else:
        while True:
            s, e = latest.segment
            if s >= gpsend:
                break
            # replace GPS time of file basename
            new = latest.path.replace('-%d-' % s, '-%d-' % e)
            # replace GPS epoch in dirname
            new = new.replace('%s/' % str(s)[:ngps], '%s/' % str(e)[:ngps])
            if os.path.isfile(new):
                latest = CacheEntry.from_T050017(new)
                cache.append(latest)
            else:
                break

    # validate files existing and return
    cache, _ = cache.checkfilesexist()
    vprint(' %d found.\n' % len(cache))
    return cache
def get_cache(start, end, ifo, ftype, framecache=False, server=None,\
              verbose=False):

  """
    Queries the LSC datafind server and returns a glue.lal.Cache object
    containing the frame file paths in the given GPS (start, end) interval
    for the given ifo and type (can be lists).

    framecache=True returns a pylal.dq.dqFrameUTils.FrameCache object in stead.

    Arguments:

      start : float
        GPS start time (s).
      end : float
        GPS end time (s).
      ifo : [ string | list ]
        ifo (or list of) to find, e.g. 'G1'.
      ftype : [ string | list ]
        frame data type (or list of) to find, e.g. 'G1_RDS_C01_L3'.

  """

  # set lists
  if isinstance(ftype, str):
    types = [ftype]
  else:
    types = ftype
  if isinstance(ifo, str):
    ifos = [ifo]
  else:
    ifos = ifo

  # construct span
  span = segments.segment(start,end)

  # set options
  cache = LALCache()
  entry_class = LALCacheEntry

  # try querying the ligo_data_find server
  if not server:
    server = _find_datafind_server()

  if verbose: sys.stdout.write("Opening connection to %s...\n" % server)

  if re.search(':', server):
    port = int(server.split(':')[-1])
  else:
    port = None

  cert, key = _get_grid_proxy()

  # if we have a credential then use it when setting up the connection
  if cert and key and port!=80:
    h = httplib.HTTPSConnection(server, key_file=key, cert_file=cert)
  else:
    h = httplib.HTTPConnection(server)

  if verbose: sys.stdout.write("Querying server for frames...\n")

  # loop over ifos and types
  for ifo in ifos:
    for t in types:
      # construct the URL for a simple data find query
      url = "/LDR/services/data/v1/gwf/%s/%s/%s,%s/file.json" % (ifo[0], t,\
                                                                 str(start),\
                                                                 str(end))
      # query the server
      h.request("GET", url)
      response = h.getresponse()
      _verify_response(response)
      # unravel the response
      urlList = cjson.decode(response.read())
      for url in urlList:
        cache.append(entry_class.from_T050017(url))

  # close the server connection
  h.close()
  if verbose: sys.stdout.write("Connection to %s closed.\n" % server)

  # convert to FrameCache if needed
  if framecache:
    cache = LALCachetoFrameCache(cache)

  return cache