def connect(host, port=None): """Open an `nds2.connection` to a given host and port Parameters ---------- host : `str` name of server with which to connect port : `int`, optional connection port Returns ------- connection : `nds2.connection` a new open connection to the given NDS host """ import nds2 # pylint: disable=no-member # set default port for NDS1 connections (required, I think) if port is None and NDS1_HOSTNAME.match(host): port = 8088 if port is None: return nds2.connection(host) return nds2.connection(host, port)
def read(start, end, chlst, nds_hostname='k1nds0', **kwargs): ''' read data using nds. Parameters ---------- start : int Start GPS time. end : int End GPS time. chlst : list List of the CDS channel. Returns ------- data : list Example ------- >>> channel = 'K1:PEM-EX1_SEIS_WE_SENSINF_OUT16' >>> read(1205201472,1205205568,channel) <list of the numpy array> ''' if not isinstance(chlst, list): raise ChannelNameException('Please give chlst as list type.\n'\ 'Given chlst is {}, which type is {}'\ .format(chlst,type(chlst))) elif isinstance(chlst, str): chlst = [chlst] conn = nds2.connection('10.68.10.121', 8088) # nds0 buffers = conn.fetch(start, end, chlst) return buffers
def connection_tester(self, ifo, s): self.logger.info('Testing connection to server at {0} ({1})...'.format(ifo, s)) try: conn = nds2.connection(s) self.logger.success('Connection established, printing details... \n {0}'.format(conn)) return conn except Exception as e: self.logger.error('Unable to open connection with server... \n {0}, {1}'.format(s, e)) return e
def stream_data(channels, duration=600, start_time=0, fs_up=256, portNumber=31200): nds_osx = ('/opt/local/Library/Frameworks/Python.framework/' + 'Versions/2.7/lib/python2.7/site-packages/') nds_sandbox = '/usr/lib/python2.7/dist-packages/' if os.path.exists(nds_osx): sys.path.append(nds_osx) elif os.path.exists(nds_sandbox): sys.path.append(nds_sandbox) # Connect to the right server ifo = channels[0][:2] if ifo == 'L1': ndsServer = 'nds.ligo-la.caltech.edu' elif ifo == 'H1': ndsServer = 'nds.ligo-wa.caltech.edu' else: sys.exit("unknown IFO specified") # Setup connection to the NDS try: conn = nds2.connection(ndsServer, portNumber) except RuntimeError: alert( 'ERROR: Need to run `kinit albert.einstein` before nds2 ' 'can establish a connection', color='FAIL') sys.exit(1) # get data data = conn.fetch(start_time, start_time + duration, channels) data = np.array(data) # stack data and downsample vdata = [] for k in range(len(channels)): fsdown = data[k].channel.sample_rate down_factor = int(fsdown // fs_up) fir_aa = sig.firwin(20 * down_factor + 1, 0.8 / down_factor, window='blackmanharris') # Using fir_aa[1:-1] cuts off a leading and trailing zero downdata = sig.decimate(data[k].data, down_factor, ftype=sig.dlti(fir_aa[1:-1], 1.0), zero_phase=True) vdata.append(downdata) return np.array(vdata).T
def connect(host, port=None): """Open an `nds2.connection` to a given host and port Parameters ---------- host : `str` name of server with which to connect port : `int`, optional connection port Returns ------- connection : `nds2.connection` a new open connection to the given NDS host """ if port is None: return nds2.connection(host) return nds2.connection(host, port)
def stream_data(start, channels, dur = 600, fsup = 512, ifo = 'H1'): """ Collect LIGO data using nds2 Parameters ---------- start : `int` GPS start time (UTC) channels : `list` channel names to scrape dur : `int` duration (in seconds) of data to scrape fsup : `int` sample rate to return data with ifo : `str` Interferometer ('H1' or 'L1') Returns ------- vdata : `numpy.ndarray` array of collected data """ import nds2 if ifo == 'H1': server = 'nds.ligo-wa.caltech.edu' else: server = 'nds.ligo-la.caltech.edu' # Setup connection to the NDS conn = nds2.connection(server, 31200) data = [] for i in range(len(channels)): temp = conn.fetch(start, start + dur, [channels[i]]) data.append(temp) # Get the data and stack it (the data are the columns) vdata = [] for k in range(len(channels)): fsdown = data[k][0].channel.sample_rate down_factor = int(fsdown // fsup) fir_aa = sig.firwin(20 * down_factor + 1, 0.8 / down_factor, window='blackmanharris') # Using fir_aa[1:-1] cuts off a leading and trailing zero downdata = sig.decimate(data[k][0].data, down_factor, ftype = sig.dlti(fir_aa[1:-1], 1.0), zero_phase = True) vdata.append(downdata) return np.array(vdata)
def _ligo_model_overflow_channels_nds(dcuid, ifo, gpstime, host): import nds2 if host is True: try: host = os.getenv('NDSSERVER', '').split(',', 1)[0] except KeyError: raise ValueError("Cannot determine default NDSSERVER, please pass " "nds=<host:port> or set NDSSERVER environment " "variable") try: host, port = host.rsplit(':', 1) except ValueError: connection = nds2.connection(host) else: connection = nds2.connection(host, int(port)) if connection.get_protocol() > 1: connection.set_epoch(gpstime, gpstime + 1) # NOTE: the `3` here is the channel type mask for 'ONLINE | RAW' return map(attrgetter('name'), connection.find_channels( '{ifo}:FEC-{dcuid}_*_OVERFLOW_*'.format(ifo=ifo, dcuid=dcuid), 3))
def test_nds2_conversion(self): try: import nds2 except ImportError as e: self.skipTest(str(e)) else: try: conn = nds2.connection(NDSHOST) except Exception as f: self.skipTest(str(f)) else: nds2channel = conn.find_channels(self.channel)[0] new = Channel.from_nds2(nds2channel) self.assertTrue(str(new) == self.channel) self.assertTrue(new.ifo == self.channel.split(':', 1)[0]) self.assertTrue(new.sample_rate == units.Quantity(32768, 'Hz'))
def connect(start_time): global conn global connected global server fast_server = ('l1nds0', 8088) slow_server = ('nds.ligo-la.caltech.edu', 31200) next_server = server # if we're on the slow server, decide if the fast server could be used if server == slow_server: current_time = int(subprocess.check_output('tconvert now', shell=True).strip()) fast_time = current_time - 12*24*60*60 # earliest time the fast server will have data if start_time >= fast_time: next_server = fast_server if not connected or next_server != server: conn = nds2.connection(next_server[0], next_server[1]) # connect to the next server connected = True server = next_server # set the current server
def quickplot(chanList, gpsLength=3600, gpsStop=correct_time().gpsSeconds): ''' quickplot takes in a list of channels, and plots an hour worth of data for all channels specified. Inputs: chanList: list of channels valid for the CTN Lab fb4. gpsLength: length of time in seconds to plot from gpsStop. Default is one hour. gpsStop: gpstime to plot until. Default is now. ''' conn = nds2.connection('10.0.1.156', 8088) gpsStart = gpsStop - gpsLength data = conn.fetch(gpsStart, gpsStop, chanList) if gpsLength <= 60: units = 'seconds' timeDivisor = 1 elif gpsLength <= 3600: units = 'minutes' timeDivisor = 60 elif gpsLength <= 86400: units = 'hours' timeDivisor = 3600 elif gpsLength <= 604800: units = 'days' timeDivisor = 86400 displayTime = gpsLength / timeDivisor t = linspace(0, displayTime, gpsLength * 16) for dat in data: plot(t, dat.data) xlabel('Time [{}] from {} ({})'.format(units, tconvert(gpsStart), gpsStart)) print(chanList) print('gpsLength = {}'.format(gpsLength)) print('gpsStop = {}'.format(gpsStop))
def connh(): return nds2.connection('nds.ligo-wa.caltech.edu', 31200)
with open(fname, 'r') as f: chanlines = f.read().split() channels = [chan_head + line for line in chanlines] # Get data start time if ifo == 'L1': ndsServer = 'nds.ligo-la.caltech.edu' elif ifo == 'H1': ndsServer = 'nds.ligo-wa.caltech.edu' else: sys.exit("unknown IFO specified") # Setup connection to the NDS try: conn = nds2.connection(ndsServer, portNumber) except RuntimeError: print('ERROR: Need to run `kinit albert.einstein` before nds2 ' 'can establish a connection') sys.exit(1) # Setup start and stop times t = Time(times, format='iso', scale='utc') t_start = int(t.gps) if __debug__: print("Output sample rate: {} Hz".format(fsup)) print("Channel List:\n-------------") print("\n".join(channels)) print("Getting data from " + ndsServer + "...")
def _connect(): return nds2.connection(host, port)
import nds2 conn = nds2.connection('nds.ligo-la.caltech.edu') print conn channel = ['L1:ASC-AS_A_RF45_Q_PIT_OUT_DQ'] print "channel = {0}".format(channel) print "Now testing the command 'conn.iterate(channel)' " try: iter = conn.iterate(channel) except RuntimeError, err: print "Failed" print RuntimeError, err print "Now testing the command 'conn.iterate(1, channel)' " try: iter = conn.iterate(1, channel) except RuntimeError, err: print "Failed" print RuntimeError, err print "Now testing the command 'conn.iterate(1125417950, 1125418050, channel)' " try: iter = conn.iterate(1125417950, 1125418050, channel) except RuntimeError, err: print "Failed" print RuntimeError, err
# ---------------------------------------- # Parse Arguments # ---------------------------------------- import argparse parser = argparse.ArgumentParser(description='Find Channels in NDS') parser.add_argument('channel', help='channel name') parser.add_argument('-trend', default='minutes', help='choose in ["minutes","seconds"]',) parser.add_argument('-types', default='mean', help='choose in ["mean","max","min","rms"]') args = parser.parse_args() chname = args.channel trend = args.trend types = args.types # ---------------------------------------- # Fetch data from NDS # ---------------------------------------- import nds2 conn = nds2.connection('10.68.10.121', 8088) fmt = '{c}.{type},{trend}-trend'.format(c=chname,type=types,trend=trend[0]) found = conn.find_channels(fmt) for ch in found: print ch.name
def _get_timeseries_dict(channels, segments, config=None, cache=None, query=True, nds=None, frametype=None, multiprocess=True, return_=True, statevector=False, archive=True, datafind_error='raise', **ioargs): """Internal method to retrieve the data for a set of like-typed channels using the :meth:`TimeSeriesDict.read` accessor. """ channels = map(get_channel, channels) # set classes if statevector: ListClass = StateVectorList DictClass = StateVectorDict else: ListClass = TimeSeriesList DictClass = TimeSeriesDict # check we have a configparser if config is None: config = GWSummConfigParser() # read segments from global memory keys = dict((c.ndsname, make_globalv_key(c)) for c in channels) havesegs = reduce( operator.and_, (globalv.DATA.get(keys[channel.ndsname], ListClass()).segments for channel in channels)) new = segments - havesegs # get processes if multiprocess is True: nproc = count_free_cores() elif multiprocess is False: nproc = 1 else: nproc = multiprocess # read channel information filter_ = dict() resample = dict() dtype_ = dict() for channel in channels: try: filter_[channel.ndsname] = channel.filter except AttributeError: pass try: resample[channel] = float(channel.resample) except AttributeError: pass if channel.dtype is None: dtype_[channel] = ioargs.get('dtype') else: dtype_[channel] = channel.dtype # work out whether to use NDS or not if nds is None and cache is not None: nds = False elif nds is None: nds = 'LIGO_DATAFIND_SERVER' not in os.environ # read new data query &= (abs(new) > 0) if cache is not None: query &= len(cache) > 0 if query: for channel in channels: globalv.DATA.setdefault(keys[channel.ndsname], ListClass()) # open NDS connection if nds and config.has_option('nds', 'host'): host = config.get('nds', 'host') port = config.getint('nds', 'port') try: ndsconnection = nds2.connection(host, port) except RuntimeError as e: if 'SASL authentication' in str(e): from gwpy.io.nds import kinit kinit() ndsconnection = nds2.connection(host, port) else: raise frametype = source = 'nds' ndstype = channels[0].type elif nds: ndsconnection = None frametype = source = 'nds' ndstype = channels[0].type # or find frame type and check cache else: ifo = channels[0].ifo frametype = frametype or channels[0].frametype if frametype is not None and frametype.endswith('%s_M' % ifo): new = type(new)([s for s in new if abs(s) >= 60.]) elif frametype is not None and frametype.endswith('%s_T' % ifo): new = type(new)([s for s in new if abs(s) >= 1.]) if cache is not None: fcache = cache.sieve(ifos=ifo[0], description=frametype, exact_match=True) else: fcache = Cache() if (cache is None or len(fcache) == 0) and len(new): span = new.extent().protract(8) fcache = find_frames(ifo, frametype, span[0], span[1], config=config, gaps='ignore', onerror=datafind_error) cachesegments = find_cache_segments(fcache) gaps = SegmentList([span]) - cachesegments if abs(gaps) and frametype == '%s_HOFT_C00' % ifo: f2 = '%s_DMT_C00' % ifo vprint(" Gaps discovered in aggregated h(t) type " "%s, checking %s\n" % (frametype, f2)) c2 = find_frames(ifo, f2, span[0], span[1], config=config, gaps='ignore', onerror=datafind_error) g2 = SegmentList([span]) - find_cache_segments(c2) if abs(g2) < abs(gaps): vprint(" Greater coverage with frametype %s\n" % f2) fcache = c2 frametype = f2 else: vprint(" No extra coverage with frametype %s\n" % f2) # parse discontiguous cache blocks and rebuild segment list cachesegments = find_cache_segments(fcache) new &= cachesegments source = 'frames' # set ctype if reading with framecpp if cache is None and frametype in ADC_TYPES and HAS_FRAMECPP: ioargs['type'] = 'adc' for channel in channels: channel.frametype = frametype # check whether each channel exists for all new times already qchannels = [] qresample = {} qdtype = {} for channel in channels: oldsegs = globalv.DATA.get(channel.ndsname, ListClass()).segments if abs(new - oldsegs) != 0: qchannels.append(channel) if channel in resample: qresample[channel] = resample[channel] qdtype[channel] = dtype_.get(channel, ioargs.get('dtype')) ioargs['dtype'] = qdtype # loop through segments, recording data for each if len(new) and nproc > 1: vprint(" Fetching data (from %s) for %d channels [%s]" % (source, len(qchannels), nds and ndstype or frametype)) for segment in new: # force reading integer-precision segments segment = type(segment)(int(segment[0]), int(segment[1])) if abs(segment) < 1: continue if nds: tsd = DictClass.fetch(qchannels, segment[0], segment[1], connection=ndsconnection, type=ndstype, **ioargs) else: # pad resampling if segment[1] == cachesegments[-1][1] and qresample: resamplepad = 8 if abs(segment) <= resamplepad: continue segment = type(segment)(segment[0], segment[1] - resamplepad) segcache = fcache.sieve( segment=segment.protract(resamplepad)) else: segcache = fcache.sieve(segment=segment) # set minute trend times modulo 60 from GPS 0 if (re.match('(?:(.*)_)?[A-Z]\d_M', str(frametype)) or (ifo == 'C1' and frametype == 'M')): segstart = int(segment[0]) // 60 * 60 segend = int(segment[1]) // 60 * 60 if segend >= segment[1]: segend -= 60 # and ignore segments shorter than 1 full average if (segend - segstart) < 60: continue segcache = segcache.sieve( segment=type(segment)(segstart, segend)) else: segstart, segend = map(float, segment) # pull filters out because they can break multiprocessing if nproc > 1: for c in qchannels: if c.ndsname in filter_: del c.filter # read data tsd = DictClass.read(segcache, qchannels, start=segstart, end=segend, nproc=nproc, resample=qresample, **ioargs) # put filters back for c in qchannels: if c.ndsname in filter_: c.filter = filter_[c.ndsname] for (channel, data) in tsd.iteritems(): key = keys[channel.ndsname] if (key in globalv.DATA and data.span in globalv.DATA[key].segments): continue if data.unit is None: data.unit = 'undef' for seg in globalv.DATA[key].segments: if seg.intersects(data.span): data = data.crop(*(data.span - seg)) break try: filt = filter_[channel.ndsname] except KeyError: pass else: # filter with function if callable(filt): try: data = filt(data) except TypeError as e: if 'Can only apply' in str(e): data.value[:] = filt(data.value) else: raise # filter with gain elif (isinstance(filt, tuple) and len(filt) == 3 and len(filt[0] + filt[1]) == 0): try: data *= filt[2] except TypeError: data = data * filt[2] # filter zpk elif isinstance(filt, tuple): data = data.filter(*filt) # filter fail else: raise ValueError("Cannot parse filter for %s: %r" % (channel.ndsname, filt)) if isinstance(data, StateVector) or ':GRD-' in str(channel): try: data.unit = units.dimensionless_unscaled except AttributeError: data._unit = units.dimensionless_unscaled if hasattr(channel, 'bits'): data.bits = channel.bits elif data.unit is None: data._unit = channel.unit # XXX: HACK for failing unit check if len(globalv.DATA[key]): data._unit = globalv.DATA[key][-1].unit # update channel type for trends if data.channel.type is None and (data.channel.trend is not None): if data.dt.to('s').value == 1: data.channel.type = 's-trend' elif data.dt.to('s').value == 60: data.channel.type = 'm-trend' # append and coalesce add_timeseries(data, key=key, coalesce=True) if multiprocess: vprint('.') if len(new): vprint("\n") if not return_: return # return correct data out = OrderedDict() for channel in channels: data = ListClass() if keys[channel.ndsname] not in globalv.DATA: out[channel.ndsname] = ListClass() else: for ts in globalv.DATA[keys[channel.ndsname]]: for seg in segments: if abs(seg) == 0 or abs(seg) < ts.dt.value: continue if ts.span.intersects(seg): common = map(float, ts.span & seg) cropped = ts.crop(*common, copy=False) if cropped.size: data.append(cropped) out[channel.ndsname] = data.coalesce() return out
f.close() print "Saved GPS times in logfile: %s" % filename + ".log" else: ### load logfile f = open(sys.argv[1]) lines = f.readlines() logfile = [] for line in lines: tok = line.split() logfile.append([int(tok[0]), int(tok[1]), tok[2]]) f.close() ######## Main loop for data processing ######################################################################################################## # open NDS connection conn = nds2.connection("nds.ligo-wa.caltech.edu", 31200) # preallocate the sensing matrix and the coherence matrix matrix = zeros([len(readout), len(excitations)], dtype="complex128") cohe = zeros([len(readout), len(excitations)]) # loop over all excitation channels for exc, mon, log, i, frequency in zip(excitations, monitors, logfile, range(len(excitations)), freqs): print "Analyzing excitation %s, monitored with %s, data from %d to %d" % (exc, mon, log[0], log[1]) ### read data (trying more times if the data is not yet on disk) waitfordata = True while waitfordata: try: channels = readout[:] channels.append(mon) bufs = conn.fetch(log[0], log[1], channels) waitfordata = False
def nonna_get_data(target_channel, aux_channels, gps_start, duration, band_freqs, outfs, fs = 4096, nds_server = 'nds.ligo-wa.caltech.edu', nds_port = 31200): """ This function reads data and prepares it for the analysis. It computes the band-limited RMS and downsamples the auxiliary channels. Input arguments: target_channel = compute the BLRMS of this channel aux_channels = all the slow channels that will be used to predict the BLRMS time variation gps_start = start reading data at this time duration = number of seconds of data to read band_freqs = [fmin, fmax] edge frequencies of the band used to compute the BLRMS outfs = output sampling frequency for BLRMS and the other channels fs = first downsample the target channel to this sampling rate, to avoid numerical instabilities of the band-pass filter, which would return only a bunch of NaNs nds_server = address of the NDS2 server nds_port = port used to contact the server Output data: t = vector of time values for each sample blrms = vector containing the BLRMS (at the rate outfs) aux = matrix of all auxiliary channels, downsampled to the output rate Note that the first two and last two seconds of data are discarded, to cope with the band-pass and low-pass filter transients. """ ##### READ DATA # open connection conn = nds2.connection(nds_server, nds_port) # read all data buffers = conn.fetch(gps_start, gps_start+duration, [target_channel]+aux_channels) ##### COMPUTE BLRMS # decimate target signal tg = scipy.signal.decimate(buffers[0].data, int(buffers[0].length/duration/fs)) # band pass b,a = scipy.signal.butter(6, scipy.array(band_freqs)/(fs/2.), btype='bandpass') tg = scipy.signal.filtfilt(b, a, tg) # square and low pass tg = tg**2 b,a = scipy.signal.butter(4, outfs/(fs/2.), btype='lowpass') blrms = scipy.signal.filtfilt(b, a, tg) # decimate blrms = blrms[::fs/outfs] ##### DECIMATE THE OTHER CHANNELS aux = scipy.zeros((duration*outfs, len(buffers)-1)) # loop over all channels for i in range(1,len(buffers)): # low pass and decimate fs_aux = int(buffers[i].length/duration) aux[:,i-1] = decimate(buffers[i].data, fs_aux/out_fs) #b,a = scipy.signal.butter(4, outfs/(fs_aux/2.), btype='lowpass') #x = scipy.signal.filtfilt(b, a, buffers[i].data) #aux[:,i-1] = x[::fs_aux/outfs] # get rid of initial and final transients aux = aux[2*outfs:-2*outfs] blrms = blrms[2*outfs:-2*outfs] t = scipy.arange(0,len(blrms))/float(outfs) # RETURN RESULTS return t, blrms, aux
def fetch(cls, channel, start, end, host=None, port=None, verbose=False, connection=None, ndschanneltype=None): """Fetch data from NDS into a TimeSeries. Parameters ---------- channel : :class:`~gwpy.detector.channel.Channel`, or `str` required data channel start : `~gwpy.time.Time`, or float GPS start time of data span end : `~gwpy.time.Time`, or float GPS end time of data span host : `str`, optional URL of NDS server to use, defaults to observatory site host port : `int`, optional port number for NDS server query, must be given with `host` verbose : `bool`, optional print verbose output about NDS progress connection : :class:`~gwpy.io.nds.NDS2Connection` open NDS connection to use ndschanneltype : `int` NDS2 channel type integer Returns ------- TimeSeries a new `TimeSeries` containing the data read from NDS """ # import module and type-cast arguments from ..io import nds as ndsio import nds2 start = int(floor(isinstance(start, Time) and start.gps or start)) end = int(ceil(isinstance(end, Time) and end.gps or end)) # set context if verbose: outputcontext = ndsio.NDSOutputContext() else: outputcontext = ndsio.NDSOutputContext(open(os.devnull, 'w'), open(os.devnull, 'w')) # get type if not ndschanneltype and isinstance(channel, Channel) and channel.type: ndschanneltype = channel.type if not ndschanneltype: ndschanneltype = (nds2.channel.CHANNEL_TYPE_RAW | nds2.channel.CHANNEL_TYPE_RDS | nds2.channel.CHANNEL_TYPE_STREND | nds2.channel.CHANNEL_TYPE_MTREND) channel = str(channel) # user-defined host or open connection if connection or host: hostlist = [(host, port)] # logical host resolution order else: hostlist = ndsio.host_resolution_order(Channel(channel).ifo) # loop hosts, stopping on first success for host,port in hostlist: if connection: _conn = connection # open connection if needed - check kerberos ticket if connection is None: if verbose: print("Connecting to %s:%s" % (host, port)) try: with outputcontext: _conn = nds2.connection(host, port) except RuntimeError as e: if str(e).startswith('Request SASL authentication'): print('\nError authenticating against %s' % host, file=sys.stderr) ndsio.kinit() with outputcontext: _conn = nds2.connection(host, port) else: raise # double check channels against server with outputcontext: channelok = _conn.find_channels(str(channel), ndschanneltype) if not channelok: channels = _conn.find_channels( '*%s*' % str(channel), ndschanneltype) # if no channels and user didn't supply their own server # warn and move one if len(channels) == 0 and not connection: if verbose: warnings.warn("No matching channels found", ndsio.NDSWarning) continue elif len(channels) == 0: pass # if one channel, find elif len(channels) == 1: channel = channels[0].name # if more than one channel and user did supply their own # server, barf elif connection: raise ValueError("No channel '%s' found on server." " However, %d others were found, " "please restrict your search and " "try again:\n %s" % (str(channel), len(channels), "\n ".join(map(str, channels)))) if channel.endswith('m-trend') and (start % 60 or end % 60): warnings.warn("Requested channel is minute trend, but " "start and stop GPS times are not modulo " "60-seconds (from GPS epoch). Times will be " "expanded outwards to compensate") if start % 60: start = start // 60 * 60 if end % 60: end = end // 60 * 60 + 60 # fetch data try: if verbose: print("Downloading data...") buffer_ = _conn.fetch(start, end, [str(channel)])[0] except RuntimeError as e: # if error and user supplied their own server, raise if connection: raise # otherwise warn and move on elif verbose: warnings.warn(str(e), ndsio.NDSWarning) else: # cast as TimeSeries and return epoch = Time(buffer_.gps_seconds, buffer_.gps_nanoseconds, format='gps') channel = Channel.from_nds2(buffer_.channel) return cls(buffer_.data, epoch=epoch, channel=channel) raise RuntimeError("Cannot find relevant data on any known server")
import nds2 conn = nds2.connection('nds.ligo-wa.caltech.edu', 31200) print conn buf = conn.fetch(1024417918, 1024417919, 'H1:PSL-ISS_PDA_OUT_DQ') #,'H1:PSL_ISS_PDB_OUT_DQ']) print buf #'H1:PSL_ISS_PDB_OUT_DQ'
def frame_struct(params): """@create seismon frame structure @param params seismon params structure """ gpsStart = np.min(params["gpsStart"])-1000 gpsEnd = np.max(params["gpsEnd"]) if params["noFrames"]: datacache = [] elif params["ifo"] == "XG": frameDir = "/archive/frames/MBH/" frameDir = "/home/prestegard/Homestake_frames/" frameList = [os.path.join(root, name) for root, dirs, files in os.walk(frameDir) for name in files] datacache = [] for frame in frameList: thisFrame = frame.replace("file://localhost","") if thisFrame == "": continue thisFrameSplit = thisFrame.split(".") if thisFrameSplit[-1] == "log": continue thisFrameSplit = thisFrame.split("-") gps = float(thisFrameSplit[-2]) dur = float(thisFrameSplit[-1].replace(".gwf","")) if gps+dur < gpsStart: continue if gps > gpsEnd: continue #cacheFile = glue.lal.CacheEntry("%s %s %d %d %s"%("XG","Homestake",gps,dur,frame)) datacache.append(frame) datacache = glue.lal.Cache(map(glue.lal.CacheEntry.from_T050017, datacache)) elif params["ifo"] == "LUNAR": frameDir = "/home/mcoughlin/Lunar/data/" frameList = [os.path.join(root, name) for root, dirs, files in os.walk(frameDir) for name in files] datacache = [] for frame in frameList: datacache.append(frame) elif params["ifo"] == "CZKHC": frameDir = "/home/mcoughlin/Stochastic/Lunar/Seismometer/data/" frameList = [os.path.join(root, name) for root, dirs, files in os.walk(frameDir) for name in files] datacache = [] for frame in frameList: datacache.append(frame) elif params["ifo"] == "SR": frameDir = "/home/mcoughlin/Stochastic/Lunar/seismic_data_corrected/data/" frameDir = "/home/mcoughlin/Stochastic/Lunar/corrected/data/" frameList = [os.path.join(root, name) for root, dirs, files in os.walk(frameDir) for name in files] datacache = [] for frame in frameList: datacache.append(frame) elif params["ifo"] == "Gravimeter": frameDir = "/home/mcoughlin/Gravimeter/minute/" frameList = [os.path.join(root, name) for root, dirs, files in os.walk(frameDir) for name in files] datacache = [] for frame in frameList: datacache.append(frame) elif params["ifo"] == "Tiltmeter": frameDir = "/home/mcoughlin/Tiltmeter/frames/" frameList = [os.path.join(root, name) for root, dirs, files in os.walk(frameDir) for name in files] datacache = [] for frame in frameList: thisFrame = frame.replace("file://localhost","") if thisFrame == "": continue thisFrameSplit = thisFrame.split(".") if thisFrameSplit[-1] == "log": continue thisFrameSplit = thisFrame.split("-") gps = float(thisFrameSplit[-2]) dur = float(thisFrameSplit[-1].replace(".gwf","")) if gps+dur < gpsStart: continue if gps > gpsEnd: continue #cacheFile = glue.lal.CacheEntry("%s %s %d %d %s"%("XG","Homestake",gps,dur,frame)) datacache.append(frame) datacache = glue.lal.Cache(map(glue.lal.CacheEntry.from_T050017, datacache)) elif params["ifo"] == "IRIS": datacache = "IRIS" else: if params["frameType"] == "nds2": import nds2 conn = nds2.connection(params["ndsServer"]) y = conn.find_channels('*',nds2.channel.CHANNEL_TYPE_RAW,\ nds2.channel.DATA_TYPE_FLOAT32, 128, 16384) params["ndsConnection"] = conn pass else: connection = glue.datafind.GWDataFindHTTPConnection() datacache = connection.find_frame_urls(params["ifo"][0], params["frameType"], gpsStart, gpsEnd, urltype="file", on_gaps="warn") connection.close() params["frame"] = datacache return params
def stream_data(ini_file): # Read config file settings = ConfigParser() settings.optionxform = str settings.read(ini_file) # Unpack configs dur = settings.getint('Data', 'duration') fname = settings.get('Data', 'chanlist') fsup = settings.getint('Data', 'fs') ifo = settings.get('Data', 'ifo') output = settings.get('Data', 'output') portNumber = settings.getint('Data', 'portNumber') save = settings.getboolean('Data', 'save_mat') times = settings.get('Data', 'data_start') nds_osx = ('/opt/local/Library/Frameworks/Python.framework/' + 'Versions/2.7/lib/python2.7/site-packages/') nds_sandbox = '/usr/lib/python2.7/dist-packages/' import sys if os.path.exists(nds_osx): sys.path.append(nds_osx) elif os.path.exists(nds_sandbox): sys.path.append(nds_sandbox) # Collect channels and times chan_head = ifo + ':' chanlines, custom_times = read_chans_and_times(fname) channels = [chan_head + line for line in chanlines] # Get data start time if ifo == 'L1': ndsServer = 'nds.ligo-la.caltech.edu' elif ifo == 'H1': ndsServer = 'nds.ligo-wa.caltech.edu' else: sys.exit("unknown IFO specified") # Setup connection to the NDS try: conn = nds2.connection(ndsServer, portNumber) except RuntimeError: print('ERROR: Need to run `kinit albert.einstein` before nds2 ' 'can establish a connection') sys.exit(1) if __debug__: print("Output sample rate: {} Hz".format(fsup)) # print("Channel List:\n-------------") # print("\n".join(channels)) # Setup start and stop times t = Time(times, format='iso', scale='utc') t_start = int(t.gps) print("Getting data from " + ndsServer + "...") data = [] for i in range(len(custom_times)): if custom_times[i] == None: custom_times[i] = t_start try: temp = conn.fetch(custom_times[i], custom_times[i] + dur, [channels[i]]) sys.stdout.write("\033[0;32m") sys.stdout.write(u'\r [{}] '.format(u'\u2713')) sys.stdout.write("\033[0;0m") sys.stdout.write('{} '.format(channels[i])) sys.stdout.write('\n') sys.stdout.flush() except: sys.stdout.write("\033[1;31m") sys.stdout.write(u'\r [{}] '.format(u'\u2717')) sys.stdout.write("\033[0;0m") sys.stdout.write('{} '.format(channels[i])) sys.stdout.write('\n') sys.stdout.flush() data.append(temp) # Get the data and stack it (the data are the columns) vdata = [] for k in range(len(channels)): fsdown = data[k][0].channel.sample_rate down_factor = int(fsdown // fsup) fir_aa = sig.firwin(20 * down_factor + 1, 0.8 / down_factor, window='blackmanharris') # Prevent ringing from DC offset DC = np.mean(data[k][0].data) # Using fir_aa[1:-1] cuts off a leading and trailing zero downdata = sig.decimate(data[k][0].data, down_factor, ftype=sig.dlti(fir_aa[1:-1], 1.0), zero_phase=True) vdata.append(downdata) if save: if not os.path.isdir('Data'): os.system('mkdir Data') # save to a hdf5 format if output == "None": funame = 'Data/' + ifo + '_data_array.mat' else: funame = 'Data/' + output sio.savemat(funame, mdict={ 'data': vdata, 'fsample': fsup, 'chans': channels }, do_compression=True) print("Data saved as " + funame) else: return np.array(vdata).T, fsup
def stream_data(ini_file): """ Stream the requested data from nds2. Parameters ---------- ini_file : `str` path to configuration file which contains the pipeline parameters Returns ------- vdata : `ndarray` numpy array containing the requested channel data. if `save_mat` is True, the data is saved to a mat file and not returned fsup : `int` sample rate of the collected data """ # Read config file try: settings = ConfigParser.ConfigParser() except: settings = configparser.ConfigParser() settings.optionxform = str settings.read(ini_file) # Unpack configs dur = settings.getint('Data', 'duration') fname = settings.get('Data', 'chanlist') fs = settings.getint('Data', 'fs') ifo = settings.get('Data', 'ifo') output = settings.get('Data', 'output') portNumber = settings.getint('Data', 'portNumber') save = settings.getboolean('Data', 'save_mat') data_dir = settings.get('Data', 'data_dir') times = settings.get('Data', 'data_start') nds_osx = ('/opt/local/Library/Frameworks/Python.framework/' + 'Versions/2.7/lib/python2.7/site-packages/') nds_sandbox = '/usr/lib/python2.7/dist-packages/' import sys if os.path.exists(nds_osx): sys.path.append(nds_osx) elif os.path.exists(nds_sandbox): sys.path.append(nds_sandbox) # Collect channels and times chan_head = ifo + ':' chanlines, custom_times = read_chans_and_times(fname) channels = [chan_head + line for line in chanlines] # Get data start time if ifo == 'L1': ndsServer = 'nds.ligo-la.caltech.edu' elif ifo == 'H1': ndsServer = 'nds.ligo-wa.caltech.edu' else: sys.exit("unknown IFO specified") # Setup connection to the NDS try: conn = nds2.connection(ndsServer, portNumber) except RuntimeError: print('ERROR: Need to run `kinit albert.einstein` before nds2 ' 'can establish a connection') sys.exit(1) #if __debug__: # print(("Output sample rate: {} Hz".format(fsup))) # # print("Channel List:\n-------------") # # print("\n".join(channels)) # Setup start and stop times t = Time(times, format='iso', scale='utc') t_start = int(t.gps) print(("Getting data from " + ndsServer + "...")) data = [] for i in range(len(custom_times)): if custom_times[i] == None: custom_times[i] = t_start try: temp = conn.fetch(custom_times[i], custom_times[i] + dur, [channels[i]]) sys.stdout.write("\033[0;32m") sys.stdout.write('\r [{}] '.format('\u2713')) sys.stdout.write("\033[0;0m") sys.stdout.write('{} '.format(channels[i])) sys.stdout.write('\n') sys.stdout.flush() except: sys.stdout.write("\033[1;31m") sys.stdout.write('\r [{}] '.format('\u2717')) sys.stdout.write("\033[0;0m") sys.stdout.write('{} '.format(channels[i])) sys.stdout.write('\n') sys.stdout.flush() data.append(temp) # Get the data and stack it (the data are the columns) vdata = [] for k in range(len(channels)): fs_data = data[k][0].channel.sample_rate resample_data = sig.resample( data[k][0].data, int(1. * len(data[k][0].data) * fs / fs_data)) vdata.append(resample_data) if save: if data_dir == "None": data_dir = 'Data' if not os.path.isdir(data_dir): os.system('mkdir %s' % data_dir) # save to a hdf5 format if output == "None": funame = os.path.join(data_dir, '%s_%d_%d.mat' % (ifo, t_start, dur)) else: funame = os.path.join(data_dir, output) sio.savemat(funame, mdict={ 'data': vdata, 'fsample': fs, 'chans': channels }, do_compression=True) print(("Data saved as " + funame)) else: return np.array(vdata).T, fs
log_dir = "/opt/rtcds/llo/l1/log/l1susdrift/" log_filename = "%sSUSDRIFT_%s_%s-%s.log" % (log_dir,optic_name,int(opts.starttime),int(opts.duration)) logging.basicConfig(filename=log_filename, level=logging.DEBUG) def nds_fetch_old_data(connection, channel, start, end): """Fetch data from NDS using the standard fetch method """ return connection.fetch(int(math.floor(start)), int(math.ceil(end)), [channel])[0] logging.info("Start time: %s" % opts.starttime) logging.info("Duration: %s" % opts.duration) # open connection to NDS2 connection = nds2.connection(opts.host, opts.port) logging.info("Connected to NDS host %s:%d" % (opts.host, opts.port)) for optic in optics: stage = STAGE[optic] DEGREES_OF_FREEDOM = DOF[optic] logging.info("Updating %s..." % optic) for dof in DEGREES_OF_FREEDOM: bound_optic = BOUND_OPTIC[optic] bound = bound_optic[dof] logging.info(" %s: " % dof) channel = CHANNEL % (ifo, optic, stage, dof) buffer_ = nds_fetch_old_data(connection, channel, gpsstart, gpsstart+opts.duration) data = buffer_.data
f.close() print "Saved GPS times in logfile: %s" % filename+".log" else: ### load logfile f = open(sys.argv[1]) lines = f.readlines() logfile = [] for line in lines: tok = line.split() logfile.append([int(tok[0]), int(tok[1]), tok[2]]) f.close() ######## Main loop for data processing ######################################################################################################## # open NDS connection conn = nds2.connection('nds.ligo-wa.caltech.edu', 31200) # preallocate the sensing matrix and the coherence matrix matrix = zeros([len(readout), len(excitations)], dtype='complex128') cohe = zeros([len(readout), len(excitations)]) # loop over all excitation channels for exc,mon,log,i,frequency in zip(excitations, monitors, logfile, range(len(excitations)), freqs): print "Analyzing excitation %s, monitored with %s, data from %d to %d" % (exc, mon, log[0], log[1]) ### read data (trying more times if the data is not yet on disk) waitfordata = True while waitfordata: try: channels = readout[:] channels.append(mon) bufs = conn.fetch(log[0], log[1], channels) waitfordata = False
def _get_timeseries_dict(channels, segments, config=None, cache=None, query=True, nds=None, frametype=None, multiprocess=True, return_=True, statevector=False, archive=True, datafind_error='raise', **ioargs): """Internal method to retrieve the data for a set of like-typed channels using the :meth:`TimeSeriesDict.read` accessor. """ channels = map(get_channel, channels) # set classes if statevector: ListClass = StateVectorList DictClass = StateVectorDict else: ListClass = TimeSeriesList DictClass = TimeSeriesDict # check we have a configparser if config is None: config = GWSummConfigParser() # read segments from global memory keys = dict((c.ndsname, make_globalv_key(c)) for c in channels) havesegs = reduce(operator.and_, (globalv.DATA.get(keys[channel.ndsname], ListClass()).segments for channel in channels)) new = segments - havesegs # get processes if multiprocess is True: nproc = count_free_cores() elif multiprocess is False: nproc = 1 else: nproc = multiprocess if globalv.VERBOSE and not multiprocess: verbose = ' ' else: verbose = False # read channel information filter_ = dict() resample = dict() dtype_ = dict() for channel in channels: try: filter_[channel.ndsname] = channel.filter except AttributeError: pass try: resample[channel] = float(channel.resample) except AttributeError: pass if channel.dtype is None: dtype_[channel] = ioargs.get('dtype') else: dtype_[channel] = channel.dtype # work out whether to use NDS or not if nds is None and cache is not None: nds = False elif nds is None: nds = 'LIGO_DATAFIND_SERVER' not in os.environ # read new data query &= (abs(new) > 0) if cache is not None: query &= len(cache) > 0 if query: for channel in channels: globalv.DATA.setdefault(keys[channel.ndsname], ListClass()) # open NDS connection if nds and config.has_option('nds', 'host'): host = config.get('nds', 'host') port = config.getint('nds', 'port') try: ndsconnection = nds2.connection(host, port) except RuntimeError as e: if 'SASL authentication' in str(e): from gwpy.io.nds import kinit kinit() ndsconnection = nds2.connection(host, port) else: raise frametype = source = 'nds' ndstype = channels[0].type elif nds: ndsconnection = None frametype = source = 'nds' ndstype = channels[0].type # or find frame type and check cache else: ifo = channels[0].ifo frametype = frametype or channels[0].frametype if frametype is not None and frametype.endswith('%s_M' % ifo): new = type(new)([s for s in new if abs(s) >= 60.]) elif frametype is not None and frametype.endswith('%s_T' % ifo): new = type(new)([s for s in new if abs(s) >= 1.]) #elif ((globalv.NOW - new[0][0]) < 86400 * 10 and # frametype == '%s_R' % ifo and # find_types(site=ifo[0], match='_C\Z')): # frametype = '%s_C' % ifo if cache is not None: fcache = cache.sieve(ifos=ifo[0], description=frametype, exact_match=True) else: fcache = Cache() if (cache is None or len(fcache) == 0) and len(new): span = new.extent().protract(8) fcache = find_frames(ifo, frametype, span[0], span[1], config=config, gaps='ignore', onerror=datafind_error) if len(fcache) == 0 and frametype == '%s_R' % ifo: frametype = '%s_C' % ifo vprint(" Moving to backup frametype %s\n" % frametype) fcache = find_frames(ifo, frametype, span[0], span[1], config=config, gaps='ignore', onerror=datafind_error) # parse discontiguous cache blocks and rebuild segment list cachesegments = find_cache_segments(fcache) new &= cachesegments source = 'frames' for channel in channels: channel.frametype = frametype # check whether each channel exists for all new times already qchannels = [] qresample = {} qdtype = {} for channel in channels: oldsegs = globalv.DATA.get(channel.ndsname, ListClass()).segments if abs(new - oldsegs) != 0: qchannels.append(channel) if channel in resample: qresample[channel] = resample[channel] qdtype[channel] = dtype_.get(channel, ioargs.get('dtype')) ioargs['dtype'] = qdtype # find channel type if not nds: ctype = set() for channel in qchannels: try: ctype.add(channel.ctype) except AttributeError: ctype.add(get_channel_type(channel)) if len(ctype) == 1: ctype = list(ctype)[0] else: ctype = None # loop through segments, recording data for each if len(new) and nproc > 1: vprint(" Fetching data (from %s) for %d channels [%s]" % (source, len(qchannels), nds and ndstype or frametype)) for segment in new: # force reading integer-precision segments segment = type(segment)(int(segment[0]), int(segment[1])) if abs(segment) < 1: continue if nds: tsd = DictClass.fetch(qchannels, segment[0], segment[1], connection=ndsconnection, type=ndstype, **ioargs) else: # pad resampling if segment[1] == cachesegments[-1][1] and qresample: resamplepad = 8 if abs(segment) <= resamplepad: continue segment = type(segment)(segment[0], segment[1] - resamplepad) segcache = fcache.sieve( segment=segment.protract(resamplepad)) else: segcache = fcache.sieve(segment=segment) # set minute trend times modulo 60 from GPS 0 if (re.match('(?:(.*)_)?[A-Z]\d_M', str(frametype)) or (ifo == 'C1' and frametype == 'M')): segstart = int(segment[0]) // 60 * 60 segend = int(segment[1]) // 60 * 60 if segend >= segment[1]: segend -= 60 # and ignore segments shorter than 1 full average if (segend - segstart) < 60: continue else: segstart, segend = map(float, segment) # pull filters out because they can break multiprocessing if nproc > 1: for c in qchannels: if c.ndsname in filter_: del c.filter # read data tsd = DictClass.read(segcache, qchannels, start=segstart, end=segend, type=ctype, nproc=nproc, resample=qresample, verbose=verbose, **ioargs) # put filters back for c in qchannels: if c.ndsname in filter_: c.filter = filter_[c.ndsname] for (channel, data) in tsd.iteritems(): key = keys[channel.ndsname] if (key in globalv.DATA and data.span in globalv.DATA[key].segments): continue if data.unit is None: data.unit = 'undef' for seg in globalv.DATA[key].segments: if seg.intersects(data.span): data = data.crop(*(data.span - seg)) break try: filt = filter_[channel.ndsname] except KeyError: pass else: # filter with function if callable(filt): try: data = filt(data) except TypeError as e: if 'Can only apply' in str(e): data.value[:] = filt(data.value) else: raise # filter with gain elif (isinstance(filt, tuple) and len(filt) == 3 and len(filt[0] + filt[1]) == 0): try: data *= filt[2] except TypeError: data = data * filt[2] # filter zpk elif isinstance(filt, tuple): data = data.filter(*filt) # filter fail else: raise ValueError("Cannot parse filter for %s: %r" % (channel.ndsname, filt)) if isinstance(data, StateVector) or ':GRD-' in str(channel): try: data.unit = units.dimensionless_unscaled except AttributeError: data._unit = units.dimensionless_unscaled if hasattr(channel, 'bits'): data.bits = channel.bits elif data.unit is None: data._unit = channel.unit # XXX: HACK for failing unit check if len(globalv.DATA[key]): data._unit = globalv.DATA[key][-1].unit # update channel type for trends if (data.channel.type is None and data.channel.trend is not None): if data.dt.to('s').value == 1: data.channel.type = 's-trend' elif data.dt.to('s').value == 60: data.channel.type = 'm-trend' # append and coalesce add_timeseries(data, key=key, coalesce=True) if multiprocess: vprint('.') if len(new): vprint("\n") if not return_: return # return correct data out = OrderedDict() for channel in channels: data = ListClass() if keys[channel.ndsname] not in globalv.DATA: out[channel.ndsname] = ListClass() else: for ts in globalv.DATA[keys[channel.ndsname]]: for seg in segments: if abs(seg) == 0 or abs(seg) < ts.dt.value: continue if ts.span.intersects(seg): common = map(float, ts.span & seg) cropped = ts.crop(*common, copy=False) if cropped.size: data.append(cropped) out[channel.ndsname] = data.coalesce() return out