def get_pile(self):
if self._pile is None:
fns = util.select_files([self.tempdir], regex=r'\.SAC$')
self._pile = pile.Pile()
self._pile.load_files(fns, fileformat='sac')
return self._pile
def testPileTraversal(self):
import shutil
config.show_progress = False
nfiles = 200
nsamples = 1000
abc = 'abcdefghijklmnopqrstuvwxyz'
def rn(n):
return ''.join([random.choice(abc) for i in xrange(n)])
stations = [rn(4) for i in xrange(10)]
channels = [rn(3) for i in xrange(3)]
networks = ['xx']
tmin = 1234567890
datadir = makeManyFiles(
nfiles, nsamples, networks, stations, channels, tmin)
filenames = util.select_files([datadir], show_progress=False)
cachedir = pjoin(datadir, '_cache_')
p = pile.Pile()
p.load_files(filenames=filenames, cache=pile.get_cache(cachedir),
show_progress=False)
assert set(p.networks) == set(networks)
assert set(p.stations) == set(stations)
assert set(p.channels) == set(channels)
toff = 0
while toff < nfiles*nsamples:
trs, loaded1 = p.chop(tmin+10, tmin+200)
for tr in trs:
assert num.all(tr.get_ydata() == num.ones(190))
trs, loaded2 = p.chop(tmin-100, tmin+100)
for tr in trs:
assert len(tr.get_ydata()) == 100
loaded = loaded1 | loaded2
while loaded:
file = loaded.pop()
file.drop_data()
toff += nsamples
s = 0
for traces in p.chopper(tmin=None, tmax=p.tmax+1., tinc=122.):
for tr in traces:
s += num.sum(tr.ydata)
assert s == nfiles*nsamples
for fn in filenames:
os.utime(fn, None)
p.reload_modified()
pile.get_cache(cachedir).clean()
shutil.rmtree(datadir)
def __init__(self, event_name=None):
self.events = []
self.pile = pile.Pile()
self.stations = {}
self.responses = defaultdict(list)
self.responses_stationxml = []
self.clippings = {}
self.blacklist = set()
self.whitelist_nslc = None
self.whitelist_nsl_xx = None
self.whitelist = None
self.station_corrections = {}
self.station_factors = {}
self.pick_markers = []
self.apply_correction_delays = True
self.apply_correction_factors = True
self.apply_displaced_sampling_workaround = False
self.extend_incomplete = False
self.clip_handling = 'by_nsl'
self.kite_scenes = []
self.gnss_campaigns = []
self.synthetic_test = None
self._picks = None
self._cache = {}
self._event_name = event_name
def testMemTracesFile(self):
tr = trace.Trace(ydata=num.arange(100, dtype=num.float))
f = pile.MemTracesFile(None, [tr])
p = pile.Pile()
p.add_file(f)
for tr in p.iter_all():
print tr.ydata
def testMemTracesFile(self):
tr = trace.Trace(ydata=num.arange(100, dtype=num.float))
f = pile.MemTracesFile(None, [tr])
p = pile.Pile()
p.add_file(f)
for tr in p.iter_all(include_last=True):
assert numeq(tr.ydata, num.arange(100, dtype=num.float), 0.001)
def get_pile(self):
if self._pile is None:
#fns = io.save( io.load(pjoin(self.tempdir, 'mini.seed')), pjoin(self.tempdir,
# 'raw-%(network)s-%(station)s-%(location)s-%(channel)s.mseed'))
fns = util.select_files([self.tempdir], regex=r'\.SAC$')
self._pile = pile.Pile()
self._pile.load_files(fns, fileformat='sac')
return self._pile
def test_to_obspy_stream(self):
pl = pile.Pile()
pl.load_files([common.test_data_file('test1.mseed')],
show_progress=False)
st = pl.to_obspy_stream()
assert isinstance(st, obspy.Stream)
assert len(st) == len([tr for tr in pl.iter_all()])
for tr in st:
assert isinstance(tr, obspy.Trace)
def __init__(self, basepile=None, tinc=360., tpad=0., storepath=None):
pile.Pile.__init__(self)
self._tinc = tinc
self._tpad = tpad
self._storepath = storepath
self._blocks = {}
if basepile is None:
basepile = pile.Pile()
self.set_basepile(basepile)
def get_pile(self):
if not self._pile:
fns = []
for irec, rec in enumerate(self._get_receivers()):
irec_fortran = irec+1
for comp in rec.components:
fn = '%s-%i-%s.%s' % (self._reference_fn_base, irec_fortran, comp, 'mseed')
fns.append(fn)
self._pile = pile.Pile()
self._pile.load_files(fns)
return self._pile
def get_waveform_pile(self):
self.ensure_data()
if self._pile is None:
path_waveforms = self.get_path('waveforms')
util.ensuredir(path_waveforms)
fns = util.select_files([path_waveforms], show_progress=False)
self._pile = pile.Pile()
if fns:
self._pile.load_files(fns,
fileformat='mseed',
show_progress=False)
return self._pile
def _get_pile(self, path):
apath = op.abspath(path)
assert op.isdir(apath)
if apath not in self._piles:
fns = util.select_files(
[apath], show_progress=False)
p = pile.Pile()
if fns:
p.load_files(fns, fileformat='mseed', show_progress=False)
self._piles[apath] = p
return self._piles[apath]
def get_pile(self):
if not self._pile:
fns = []
for sta in self.get_stations().values():
for cha in sta.get_channels():
fn = self._traces_fn_tmpl % dict(
network=sta.network,
station=sta.station,
location=sta.location,
channel=cha.name)
if os.path.exists(fn):
fns.append(fn)
self._pile = pile.Pile()
self._pile.load_files(fns)
return self._pile
def __init__(self, event_name=None):
self.events = []
self._pile = pile.Pile()
self._pile_update_args = []
self.stations = {}
self.responses = defaultdict(list)
self.responses_stationxml = []
self.clippings = {}
self.blacklist = set()
self.whitelist_nslc = None
self.whitelist_nsl = None
self.station_corrections = {}
self.station_factors = {}
self.pick_markers = []
self.apply_correction_delays = True
self.apply_correction_factors = True
self.extend_incomplete = False
self.clip_handling = 'by_nsl'
self._picks = None
self._cache = {}
self._event_name = event_name
def kiwi_setup(options, config, event_names):
from tunguska import prepare, gfdb
conf = config['iris_pull_config']
kiwi_conf = config['kiwi_config']
if kiwi_conf.has('gfdb_path'):
db = gfdb.Gfdb(kiwi_conf.path('gfdb_path'))
deltat = db.dt
else:
if kiwi_conf.has('deltat'):
deltat = kiwi_conf.deltat
else:
deltat = None
db = None
if not event_names:
sys.exit('need event name')
for event_name in event_names:
conf.event_name = event_name
kiwi_conf.event_name = event_name
event = _get_event_infos(conf)
stations = _get_stations(conf)
for station in stations:
station.set_event_relative_data(event)
traces = _get_prepared_traces(conf, stations)
raw_traces = _get_raw_traces(conf, stations)
p = pile.Pile()
buf = pile.MemTracesFile(None, traces)
p.add_file(buf)
processed = []
for station in stations:
tt1 = kiwi_conf.cut_span[0](station.dist_m, event.depth)
tt2 = kiwi_conf.cut_span[1](station.dist_m, event.depth)
if None in (tt1, tt2):
continue
tmin = tt1 + event.time
tmax = tt2 + event.time
traces = p.all(
tmin=tmin,
tmax=tmax,
want_incomplete=False,
trace_selector=lambda tr: get_nsl(tr) == get_nsl(station))
for proj, in_channels, out_channels in station.guess_projections_to_rtu(
out_channels=('R', 'T', 'Z')):
proc = trace.project(traces, proj, in_channels, out_channels)
processed.extend(proc)
for tr in proc:
for ch in out_channels:
if ch.name == tr.channel:
station.add_channel(ch)
for tr in processed:
if deltat is not None:
try:
tr.downsample_to(deltat, snap=True, allow_upsample_max=5)
except util.UnavailableDecimation, e:
logger.warn('Cannot downsample %s.%s.%s.%s: %s' %
(tr.nslc_id + (e, )))
continue
stations_by_nsl = dict((get_nsl(s), s) for s in stations)
acc = DummyAcc(raw_traces)
prepare.save_kiwi_dataset(acc, stations_by_nsl, processed, event,
kiwi_conf)
def snuffler_from_commandline(args=None):
if args is None:
args = sys.argv[1:]
usage = '''usage: %prog [options] waveforms ...'''
parser = OptionParser(usage=usage)
parser.add_option(
'--format',
dest='format',
default='detect',
choices=io.allowed_formats('load'),
help='assume input files are of given FORMAT. Choices: %s'
% io.allowed_formats('load', 'cli_help', 'detect'))
parser.add_option(
'--pattern',
dest='regex',
metavar='REGEX',
help='only include files whose paths match REGEX')
parser.add_option(
'--stations',
dest='station_fns',
action='append',
default=[],
metavar='STATIONS',
help='read station information from file STATIONS')
parser.add_option(
'--stationxml',
dest='stationxml_fns',
action='append',
default=[],
metavar='STATIONSXML',
help='read station information from XML file STATIONSXML')
parser.add_option(
'--event', '--events',
dest='event_fns',
action='append',
default=[],
metavar='EVENT',
help='read event information from file EVENT')
parser.add_option(
'--markers',
dest='marker_fns',
action='append',
default=[],
metavar='MARKERS',
help='read marker information file MARKERS')
parser.add_option(
'--follow',
type='float',
dest='follow',
metavar='N',
help='follow real time with a window of N seconds')
parser.add_option(
'--cache',
dest='cache_dir',
default=config.config().cache_dir,
metavar='DIR',
help='use directory DIR to cache trace metadata '
'(default=\'%default\')')
parser.add_option(
'--force-cache',
dest='force_cache',
action='store_true',
default=False,
help='use the cache even when trace attribute spoofing is active '
'(may have silly consequences)')
parser.add_option(
'--store-path',
dest='store_path',
metavar='PATH_TEMPLATE',
help='store data received through streams to PATH_TEMPLATE')
parser.add_option(
'--store-interval',
type='float',
dest='store_interval',
default=600,
metavar='N',
help='dump stream data to file every N seconds [default: %default]')
parser.add_option(
'--ntracks',
type='int',
dest='ntracks',
default=24,
metavar='N',
help='initially use N waveform tracks in viewer [default: %default]')
parser.add_option(
'--opengl',
dest='opengl',
action='store_true',
default=False,
help='use OpenGL for drawing')
parser.add_option(
'--qt5',
dest='gui_toolkit_qt5',
action='store_true',
default=False,
help='use Qt5 for the GUI')
parser.add_option(
'--qt4',
dest='gui_toolkit_qt4',
action='store_true',
default=False,
help='use Qt4 for the GUI')
parser.add_option(
'--debug',
dest='debug',
action='store_true',
default=False,
help='print debugging information to stderr')
options, args = parser.parse_args(list(args))
if options.debug:
util.setup_logging('snuffler', 'debug')
else:
util.setup_logging('snuffler', 'warning')
if options.gui_toolkit_qt4:
config.override_gui_toolkit = 'qt4'
if options.gui_toolkit_qt5:
config.override_gui_toolkit = 'qt5'
this_pile = pile.Pile()
stations = []
for stations_fn in extend_paths(options.station_fns):
stations.extend(model.station.load_stations(stations_fn))
for stationxml_fn in extend_paths(options.stationxml_fns):
stations.extend(
stationxml.load_xml(
filename=stationxml_fn).get_pyrocko_stations())
events = []
for event_fn in extend_paths(options.event_fns):
events.extend(model.event.Event.load_catalog(event_fn))
markers = []
for marker_fn in extend_paths(options.marker_fns):
markers.extend(marker.load_markers(marker_fn))
return snuffle(
this_pile,
stations=stations,
events=events,
markers=markers,
ntracks=options.ntracks,
follow=options.follow,
controls=True,
opengl=options.opengl,
paths=args,
cache_dir=options.cache_dir,
regex=options.regex,
format=options.format,
force_cache=options.force_cache,
store_path=options.store_path,
store_interval=options.store_interval)
def __init__(self, dirpath):
eventdata.EventDataAccess.__init__(self)
self._dirpath = dirpath
self._pile = pile.Pile()
self._pile.load_files(util.select_files([self._dirpath], regex=r'raw-[^/]+\.mseed$'))
def get_pile(self, tmin=None, tmax=None):
p = pile.Pile()
trf = pile.MemTracesFile(None, self.get_waveforms(tmin, tmax))
p.add_file(trf)
return p
