def getCatData(date, opt): """ Download data from IRIS or Earthworm waveserver with padding and filter it. This is a specialized version getData() for catalog events, pulling a smaller amount of time around a known event. date: UTCDateTime of known catalog event opt: Options object describing station/run parameters Returns ObsPy stream objects, one for cutting and the other for triggering """ nets = opt.network.split(',') stas = opt.station.split(',') locs = opt.location.split(',') chas = opt.channel.split(',') if opt.server == "IRIS": client = Client("IRIS") else: client = EWClient(opt.server, opt.port) st = Stream() for n in range(len(stas)): try: stmp = client.get_waveforms(nets[n], stas[n], locs[n], chas[n], date - opt.atrig, date + 3*opt.atrig) stmp = stmp.filter("bandpass", freqmin=opt.fmin, freqmax=opt.fmax, corners=2, zerophase=True) stmp = stmp.merge(method=1, fill_value='interpolate') except (obspy.fdsn.header.FDSNException): try: # try again stmp = client.get_waveforms(nets[n], stas[n], locs[n], chas[n], date - opt.atrig, date + 3*opt.atrig) stmp = stmp.filter("bandpass", freqmin=opt.fmin, freqmax=opt.fmax, corners=2, zerophase=True) stmp = stmp.merge(method=1, fill_value='interpolate') except (obspy.fdsn.header.FDSNException): print('No data found for {0}.{1}'.format(stas[n],nets[n])) trtmp = Trace() trtmp.stats.sampling_rate = opt.samprate trtmp.stats.station = stas[n] stmp = Stream().extend([trtmp.copy()]) # Resample to ensure all traces are same length if stmp[0].stats.sampling_rate != opt.samprate: stmp = stmp.resample(opt.samprate) st.extend(stmp.copy()) st = st.trim(starttime=date-opt.atrig, endtime=date+3*opt.atrig, pad=True, fill_value=0) stC = st.copy() return st, stC
def getData(tstart, tend, opt): """ Download data from files in a folder, from IRIS, or a Earthworm waveserver A note on SAC/miniSEED files: as this makes no assumptions about the naming scheme of your data files, please ensure that your headers contain the correct SCNL information! tstart: UTCDateTime of beginning of period of interest tend: UTCDateTime of end of period of interest opt: Options object describing station/run parameters Returns ObsPy stream objects, one for cutting and the other for triggering """ nets = opt.network.split(',') stas = opt.station.split(',') locs = opt.location.split(',') chas = opt.channel.split(',') st = Stream() if opt.server == 'file': # Generate list of files if opt.server == 'file': flist = list(itertools.chain.from_iterable(glob.iglob(os.path.join( root,opt.filepattern)) for root, dirs, files in os.walk(opt.searchdir))) # Determine which subset of files to load based on start and end times and # station name; we'll fully deal with stations below flist_sub = [] for f in flist: # Load header only stmp = obspy.read(f, headonly=True) # Check if station is contained in the stas list if stmp[0].stats.station in stas: # Check if contains either start or end time ststart = stmp[0].stats.starttime stend = stmp[-1].stats.endtime if (ststart<=tstart and tstart<=stend) or (ststart<=tend and tend<=stend) or (tstart<=stend and ststart<=tend): flist_sub.append(f) # Fully load data from file stmp = Stream() for f in flist_sub: tmp = obspy.read(f, starttime=tstart, endtime=tend+opt.maxdt) if len(tmp) > 0: stmp = stmp.extend(tmp) # Filter and merge stmp = stmp.filter('bandpass', freqmin=opt.fmin, freqmax=opt.fmax, corners=2, zerophase=True) stmp = stmp.taper(0.05,type='hann',max_length=opt.mintrig) for m in range(len(stmp)): if stmp[m].stats.sampling_rate != opt.samprate: stmp[m] = stmp[m].resample(opt.samprate) stmp = stmp.merge(method=1, fill_value=0) # Only grab stations/channels that we want and in order netlist = [] stalist = [] chalist = [] loclist = [] for s in stmp: stalist.append(s.stats.station) chalist.append(s.stats.channel) netlist.append(s.stats.network) loclist.append(s.stats.location) # Find match of SCNL in header or fill empty for n in range(len(stas)): for m in range(len(stalist)): if (stas[n] in stalist[m] and chas[n] in chalist[m] and nets[n] in netlist[m] and locs[n] in loclist[m]): st = st.append(stmp[m]) if len(st) == n: print("Couldn't find "+stas[n]+'.'+chas[n]+'.'+nets[n]+'.'+locs[n]) trtmp = Trace() trtmp.stats.sampling_rate = opt.samprate trtmp.stats.station = stas[n] st = st.append(trtmp.copy()) else: if '.' not in opt.server: client = Client(opt.server) else: client = EWClient(opt.server, opt.port) for n in range(len(stas)): try: stmp = client.get_waveforms(nets[n], stas[n], locs[n], chas[n], tstart, tend+opt.maxdt) for m in range(len(stmp)): stmp[m].data = np.where(stmp[m].data == -2**31, 0, stmp[m].data) # replace -2**31 (Winston NaN token) w 0 stmp = stmp.filter('bandpass', freqmin=opt.fmin, freqmax=opt.fmax, corners=2, zerophase=True) stmp = stmp.taper(0.05,type='hann',max_length=opt.mintrig) for m in range(len(stmp)): if stmp[m].stats.sampling_rate != opt.samprate: stmp[m] = stmp[m].resample(opt.samprate) stmp = stmp.merge(method=1, fill_value=0) except (obspy.clients.fdsn.header.FDSNException): try: # try again stmp = client.get_waveforms(nets[n], stas[n], locs[n], chas[n], tstart, tend+opt.maxdt) for m in range(len(stmp)): stmp[m].data = np.where(stmp[m].data == -2**31, 0, stmp[m].data) # replace -2**31 (Winston NaN token) w 0 stmp = stmp.filter('bandpass', freqmin=opt.fmin, freqmax=opt.fmax, corners=2, zerophase=True) stmp = stmp.taper(0.05,type='hann',max_length=opt.mintrig) for m in range(len(stmp)): if stmp[m].stats.sampling_rate != opt.samprate: stmp[m] = stmp[m].resample(opt.samprate) stmp = stmp.merge(method=1, fill_value=0) except (obspy.clients.fdsn.header.FDSNException): print('No data found for {0}.{1}'.format(stas[n],nets[n])) trtmp = Trace() trtmp.stats.sampling_rate = opt.samprate trtmp.stats.station = stas[n] stmp = Stream().extend([trtmp.copy()]) # Last check for length; catches problem with empty waveserver if len(stmp) != 1: print('No data found for {0}.{1}'.format(stas[n],nets[n])) trtmp = Trace() trtmp.stats.sampling_rate = opt.samprate trtmp.stats.station = stas[n] stmp = Stream().extend([trtmp.copy()]) st.extend(stmp.copy()) # Edit 'start' time if using offset option if opt.maxdt: dts = np.fromstring(opt.offset, sep=',') for n, tr in enumerate(st): tr.stats.starttime = tr.stats.starttime-dts[n] st = st.trim(starttime=tstart, endtime=tend, pad=True, fill_value=0) stC = st.copy() return st, stC
def get_stream(datasource, scnl, tstart, tend, fill_value=0, filepattern='*', filter=None, samprate=100, verbose=False): """ Generalized (and more robust) way to retrieve waveform data through ObsPy Download data from files in a folder, from IRIS, or a Earthworm waveserver A note on SAC/miniSEED files: as this makes no assumptions about the naming scheme of your data files, please ensure that your headers contain the correct SCNL information! tstart: UTCDateTime of beginning of period of interest tend: UTCDateTime of end of period of interest filepattern='*' You can specify a pattern for your files to reduce the files within the directory searched. For example, filepattern=2019.06.*.mseed if your files are miniSEED files named by date and you only want those from June 2019. Simple wildcarding is supported (i.e., * and ?, [] for ranges of values or lists) but not full regular expressions. samprate=100 Resamples all waveforms to the same sample rate. Returns ObsPy stream objects Based on code by Alicia Hotovec-Ellis and Aaron Wech. Example: >>> get_stream(['vdap.org', 16024], ['HSR.EHZ.CC.--'], '2004-09-28T00:00:00', '2004-09-28T01:00:00') >>> get_stream(['file', '/Users/vdapseismo/data/'], ['HSR.EHZ.CC.--'], '2004-09-28T00:00:00', '2004-09-28T01:00:00') >>> get_stream(['IRIS'], ['HSR.EHZ.CC.--'], '2004-09-28T00:00:00', '2004-09-28T01:00:00') """ from obspy import UTCDateTime import obspy from obspy.clients.fdsn import Client from obspy.clients.earthworm import Client as EWClient from obspy.core.trace import Trace from obspy.core.stream import Stream from obspy.signal.trigger import coincidence_trigger import numpy as np from scipy import stats from scipy.fftpack import fft import glob, os, itertools #print(datasource) #print(scnl) #print(tstart) #print(tend) tstart = UTCDateTime(tstart) tend = UTCDateTime(tend) nets = [] stas = [] locs = [] chas = [] for s in scnl: #print(s) nets.append(s.split('.')[2]) stas.append(s.split('.')[0]) locs.append(s.split('.')[3]) chas.append(s.split('.')[1]) st = Stream() if '/' in datasource: # Retrieve data from file structure flist = list( itertools.chain.from_iterable( glob.iglob(os.path.join(root, filepattern)) for root, dirs, files in os.walk(datasource))) # Determine which subset of files to load based on start and end times and # station name; we'll fully deal with stations below flist_sub = [] for f in flist: # Load header only stmp = obspy.read(f, headonly=True) # Check if station is contained in the stas list if stmp[0].stats.station in stas: # Check if contains either start or end time ststart = stmp[0].stats.starttime stend = stmp[0].stats.endtime if (ststart <= tstart and tstart <= stend) or ( ststart <= tend and tend <= stend) or (tstart <= stend and ststart <= tend): flist_sub.append(f) # Fully load data from file stmp = Stream() for f in flist_sub: tmp = obspy.read(f, starttime=tstart, endtime=tend) if len(tmp) > 0: stmp = stmp.extend(tmp) # merge stmp = stmp.taper(max_percentage=0.01) for m in range(len(stmp)): if stmp[m].stats.sampling_rate != samprate: stmp[m] = stmp[m].resample(samprate) stmp = stmp.merge(method=1, fill_value=fill_value) # Only grab stations/channels that we want and in order netlist = [] stalist = [] chalist = [] loclist = [] for s in stmp: stalist.append(s.stats.station) chalist.append(s.stats.channel) netlist.append(s.stats.network) loclist.append(s.stats.location) # Find match of SCNL in header or fill empty for n in range(len(stas)): for m in range(len(stalist)): if (stas[n] in stalist[m] and chas[n] in chalist[m] and nets[n] in netlist[m] and locs[n] in loclist[m]): st = st.append(stmp[m]) if len(st) == n: print('No data found for {}.{}.{}.{}'.format( stas[n], chas[n], nets[n], locs[n])) trtmp = Trace() trtmp.stats.sampling_rate = samprate trtmp.stats.station = stas[n] st = st.append(trtmp.copy()) else: # retrieve data from server if '.' not in datasource: client = Client(datasource) else: datasource = datasource.split(':') client = EWClient(datasource[0], int(datasource[1])) for n in range(len(stas)): try: stmp = client.get_waveforms(nets[n], stas[n], locs[n], chas[n], tstart, tend) for m in range(len(stmp)): #stmp[m].data = np.ma.masked_where(stmp[m].data == -2**31, stmp[m].data) # masks out all values of -2**31 (Winston NaN Token) #stmp[m] = stmp[m].split().merge(method=0, fill_value='interpolate')[0] # splits trace at masked values; then re-merges using linear interpolation stmp[m].data = np.where(stmp[m].data == -2**31, 0, stmp[m].data) if stmp[m].stats.sampling_rate != samprate: stmp[m] = stmp[m].resample(samprate) stmp = stmp.taper(max_percentage=0.01) stmp = stmp.merge(method=1, fill_value=fill_value) except (obspy.clients.fdsn.header.FDSNException): try: # try again stmp = client.get_waveforms(nets[n], stas[n], locs[n], chas[n], tstart, tend) for m in range(len(stmp)): #stmp[m].data = np.ma.masked_where(stmp[m].data == -2**31, stmp[m].data) # masks out all values of -2**31 (Winston NaN Token) #stmp[m] = stmp[m].split().merge(method=0, fill_value='interpolate')[0] # splits trace at masked values; then re-merges using linear interpolation stmp[m].data = np.where(stmp[m].data == -2**31, 0, stmp[m].data) if stmp[m].stats.sampling_rate != samprate: stmp[m] = stmp[m].resample(samprate) stmp = stmp.taper(max_percentage=0.01) stmp = stmp.merge(method=1, fill_value=fill_value) except (obspy.clients.fdsn.header.FDSNException): print('No data found for {0}.{1}'.format(stas[n], nets[n])) trtmp = Trace() trtmp.stats.sampling_rate = samprate trtmp.stats.station = stas[n] stmp = Stream().extend([trtmp.copy()]) # Last check for length; catches problem with empty waveserver if len(stmp) != 1: print('No data found for {}.{}.{}.{}'.format( stas[n], chas[n], nets[n], locs[n])) trtmp = Trace() trtmp.stats.sampling_rate = samprate trtmp.stats.station = stas[n] stmp = Stream().extend([trtmp.copy()]) st.extend(stmp.copy()) st = st.trim(starttime=tstart, endtime=tend, pad=True, fill_value=fill_value) return st
def getData(tstart, tend, opt): """ Download data from files in a folder, from IRIS, or a Earthworm waveserver A note on SAC/miniSEED files: as this makes no assumptions about the naming scheme of your data files, please ensure that your headers contain the correct SCNL information! tstart: UTCDateTime of beginning of period of interest tend: UTCDateTime of end of period of interest opt: Options object describing station/run parameters Returns ObsPy stream objects, one for cutting and the other for triggering """ nets = opt.network.split(',') stas = opt.station.split(',') locs = opt.location.split(',') chas = opt.channel.split(',') st = Stream() if opt.server == 'SAC' or opt.server == 'miniSEED': # Generate list of files if opt.server == 'SAC': flist = glob.glob(opt.sacdir + '*.sac') + glob.glob(opt.sacdir + '*.SAC') elif opt.server == 'miniSEED': flist = glob.glob(opt.mseeddir + '*.mseed') + glob.glob(opt.mseeddir + '*.MSEED') # Load data from file stmp = Stream() for f in flist: tmp = obspy.read(f, starttime=tstart, endtime=tend) if len(tmp) > 0: stmp = stmp.extend(tmp) # Filter and merge stmp = stmp.filter('bandpass', freqmin=opt.fmin, freqmax=opt.fmax, corners=2, zerophase=True) stmp = stmp.taper(0.05, type='hann', max_length=opt.mintrig) for m in range(len(stmp)): if stmp[m].stats.sampling_rate != opt.samprate: stmp[m] = stmp[m].resample(opt.samprate) stmp = stmp.merge(method=1, fill_value=0) # Only grab stations/channels that we want and in order netlist = [] stalist = [] chalist = [] loclist = [] for s in stmp: stalist.append(s.stats.station) chalist.append(s.stats.channel) netlist.append(s.stats.network) loclist.append(s.stats.location) # Find match of SCNL in header or fill empty for n in range(len(stas)): for m in range(len(stalist)): if (stas[n] in stalist[m] and chas[n] in chalist[m] and nets[n] in netlist[m] and locs[n] in loclist[m]): st = st.append(stmp[m]) if len(st) == n: print("Couldn't find " + stas[n] + '.' + chas[n] + '.' + nets[n] + '.' + locs[n]) trtmp = Trace() trtmp.stats.sampling_rate = opt.samprate trtmp.stats.station = stas[n] st = st.append(trtmp.copy()) else: if '.' not in opt.server: client = Client(opt.server) else: client = EWClient(opt.server, opt.port) for n in range(len(stas)): try: stmp = client.get_waveforms(nets[n], stas[n], locs[n], chas[n], tstart, tend) stmp = stmp.filter('bandpass', freqmin=opt.fmin, freqmax=opt.fmax, corners=2, zerophase=True) stmp = stmp.taper(0.05, type='hann', max_length=opt.mintrig) for m in range(len(stmp)): if stmp[m].stats.sampling_rate != opt.samprate: stmp[m] = stmp[m].resample(opt.samprate) stmp = stmp.merge(method=1, fill_value=0) except (obspy.fdsn.header.FDSNException): try: # try again stmp = client.get_waveforms(nets[n], stas[n], locs[n], chas[n], tstart, tend) stmp = stmp.filter('bandpass', freqmin=opt.fmin, freqmax=opt.fmax, corners=2, zerophase=True) stmp = stmp.taper(0.05, type='hann', max_length=opt.mintrig) for m in range(len(stmp)): if stmp[m].stats.sampling_rate != opt.samprate: stmp[m] = stmp[m].resample(opt.samprate) stmp = stmp.merge(method=1, fill_value=0) except (obspy.fdsn.header.FDSNException): print('No data found for {0}.{1}'.format(stas[n], nets[n])) trtmp = Trace() trtmp.stats.sampling_rate = opt.samprate trtmp.stats.station = stas[n] stmp = Stream().extend([trtmp.copy()]) st.extend(stmp.copy()) st = st.trim(starttime=tstart, endtime=tend, pad=True, fill_value=0) stC = st.copy() return st, stC