def make_pile( paths=None, selector=None, regex=None,
fileformat = 'mseed',
cachedirname=None, show_progress=True ):
'''Create pile from given file and directory names.
:param paths: filenames and/or directories to look for traces. If paths is
``None`` ``sys.argv[1:]`` is used.
:param selector: lambda expression taking group dict of regex match object as
a single argument and which returns true or false to keep or reject
a file
:param regex: regular expression which filenames have to match
:param fileformat: format of the files ('mseed', 'sac', 'kan',
'from_extension', 'try')
:param cachedirname: loader cache is stored under this directory. It is
created as neccessary.
:param show_progress: show progress bar and other progress information
'''
if isinstance(paths, str):
paths = [ paths ]
if paths is None:
paths = sys.argv[1:]
if cachedirname is None:
cachedirname = config.config().cache_dir
fns = util.select_files(paths, selector, regex, show_progress=show_progress)
cache = get_cache(cachedirname)
p = Pile()
p.load_files( sorted(fns), cache=cache, fileformat=fileformat, show_progress=show_progress)
return p
def get_pile(self):
if self._pile is None:
fns = util.select_files([self.tempdir], regex=r"\.SAC$")
self._pile = pile.Pile()
self._pile.load_files(fns, fileformat="sac")
return self._pile
def get_pile(self):
if self._pile is None:
fns = util.select_files([self.tempdir], regex=r'\.SAC$')
self._pile = pile.Pile()
self._pile.load_files(fns, fileformat='sac')
return self._pile
def add_gnss_campaigns(self, paths):
paths = util.select_files(paths,
regex=r'\.yml|\.yaml',
show_progress=False)
for path in paths:
self.add_gnss_campaign(filename=path)
def testPileTraversal(self):
import shutil
config.show_progress = False
nfiles = 200
nsamples = 1000
abc = 'abcdefghijklmnopqrstuvwxyz'
def rn(n):
return ''.join([random.choice(abc) for i in xrange(n)])
stations = [rn(4) for i in xrange(10)]
channels = [rn(3) for i in xrange(3)]
networks = ['xx']
tmin = 1234567890
datadir = makeManyFiles(
nfiles, nsamples, networks, stations, channels, tmin)
filenames = util.select_files([datadir], show_progress=False)
cachedir = pjoin(datadir, '_cache_')
p = pile.Pile()
p.load_files(filenames=filenames, cache=pile.get_cache(cachedir),
show_progress=False)
assert set(p.networks) == set(networks)
assert set(p.stations) == set(stations)
assert set(p.channels) == set(channels)
toff = 0
while toff < nfiles*nsamples:
trs, loaded1 = p.chop(tmin+10, tmin+200)
for tr in trs:
assert num.all(tr.get_ydata() == num.ones(190))
trs, loaded2 = p.chop(tmin-100, tmin+100)
for tr in trs:
assert len(tr.get_ydata()) == 100
loaded = loaded1 | loaded2
while loaded:
file = loaded.pop()
file.drop_data()
toff += nsamples
s = 0
for traces in p.chopper(tmin=None, tmax=p.tmax+1., tinc=122.):
for tr in traces:
s += num.sum(tr.ydata)
assert s == nfiles*nsamples
for fn in filenames:
os.utime(fn, None)
p.reload_modified()
pile.get_cache(cachedir).clean()
shutil.rmtree(datadir)
def testPileTraversal(self):
import tempfile, shutil
config.show_progress = False
nfiles = 200
nsamples = 1000
abc = 'abcdefghijklmnopqrstuvwxyz'
def rn(n):
return ''.join( [ random.choice(abc) for i in xrange(n) ] )
stations = [ rn(4) for i in xrange(10) ]
channels = [ rn(3) for i in xrange(3) ]
networks = [ 'xx' ]
tmin = 1234567890
datadir = makeManyFiles(nfiles, nsamples, networks, stations, channels, tmin)
filenames = util.select_files([datadir], show_progress=False)
cachedir = pjoin(datadir,'_cache_')
p = pile.Pile()
p.load_files(filenames=filenames, cache=pile.get_cache(cachedir),
show_progress=False)
assert set(p.networks) == set(networks)
assert set(p.stations) == set(stations)
assert set(p.channels) == set(channels)
toff = 0
while toff < nfiles*nsamples:
trs, loaded1 = p.chop(tmin+10, tmin+200)
for tr in trs:
assert num.all(tr.get_ydata() == num.ones(190))
trs, loaded2 = p.chop(tmin-100, tmin+100)
for tr in trs:
assert len(tr.get_ydata()) == 100
loaded = loaded1 | loaded2
while loaded:
file = loaded.pop()
file.drop_data()
toff += nsamples
s = 0
for traces in p.chopper(tmin=None, tmax=p.tmax+1., tinc=122. ): #tpad=10.):
for trace in traces:
s += num.sum(trace.ydata)
assert s == nfiles*nsamples
for fn in filenames:
os.utime(fn, None)
p.reload_modified()
pile.get_cache(cachedir).clean()
shutil.rmtree(datadir)
def get_pile(self):
if self._pile is None:
#fns = io.save( io.load(pjoin(self.tempdir, 'mini.seed')), pjoin(self.tempdir,
# 'raw-%(network)s-%(station)s-%(location)s-%(channel)s.mseed'))
fns = util.select_files([self.tempdir], regex=r'\.SAC$')
self._pile = pile.Pile()
self._pile.load_files(fns, fileformat='sac')
return self._pile
def get_pile(self):
if self._pile is None:
#fns = io.save( io.load(pjoin(self.tempdir, 'mini.seed')), pjoin(self.tempdir,
# 'raw-%(network)s-%(station)s-%(location)s-%(channel)s.mseed'))
fns = util.select_files( [ self.tempdir ], regex=r'\.SAC$')
self._pile = pile.Pile()
self._pile.load_files(fns, fileformat='sac')
return self._pile
def add_kite_scenes(self, paths):
logger.info('Loading kite InSAR scenes...')
paths = util.select_files(paths, regex=r'\.npz', show_progress=False)
for path in paths:
self.add_kite_scene(filename=path)
if not self.kite_scenes:
logger.warning('Could not find any kite scenes at %s.' %
quote_paths(self.kite_scene_paths))
def add_waveforms(self, paths, regex=None, fileformat='detect',
show_progress=False):
cachedirname = config.config().cache_dir
logger.debug('Selecting waveform files %s...' % quote_paths(paths))
fns = util.select_files(paths, regex=regex,
show_progress=show_progress)
cache = pile.get_cache(cachedirname)
logger.debug('Scanning waveform files %s...' % quote_paths(paths))
self.pile.load_files(sorted(fns), cache=cache,
fileformat=fileformat,
show_progress=show_progress)
def get_insar_scenes(self):
from kite import Scene
if self._scenes is None:
self._scenes = []
path_insar = self.get_path('insar')
util.ensuredir(path_insar)
fns = util.select_files([path_insar], regex='\\.(npz)$',
show_progress=False)
for f in fns:
self._scenes.append(Scene.load(f))
return self._scenes
def get_insar_scenes(self):
from kite import Scene
if self._scenes is None:
self._scenes = []
path_insar = self.get_path('insar')
util.ensuredir(path_insar)
fns = util.select_files([path_insar], regex='\.(npz)$',
show_progress=False)
for f in fns:
self._scenes.append(Scene.load(f))
return self._scenes
def _update_pile(self):
while self._pile_update_args:
paths, regex, fileformat, show_progress = \
self._pile_update_args.pop(0)
logger.debug('Loading waveform data from %s' % paths)
cachedirname = config.config().cache_dir
fns = util.select_files(paths, regex=regex,
show_progress=show_progress)
cache = pile.get_cache(cachedirname)
self._pile.load_files(sorted(fns), cache=cache,
fileformat=fileformat,
show_progress=show_progress)
def get_waveform_pile(self):
self.ensure_data()
if self._pile is None:
path_waveforms = self.get_path('waveforms')
util.ensuredir(path_waveforms)
fns = util.select_files([path_waveforms], show_progress=False)
self._pile = pile.Pile()
if fns:
self._pile.load_files(fns,
fileformat='mseed',
show_progress=False)
return self._pile
def _get_pile(self, path):
apath = op.abspath(path)
assert op.isdir(apath)
if apath not in self._piles:
fns = util.select_files(
[apath], show_progress=False)
p = pile.Pile()
if fns:
p.load_files(fns, fileformat='mseed', show_progress=False)
self._piles[apath] = p
return self._piles[apath]
def get_waveform_pile(self):
self.ensure_data()
if self._pile is None:
path_waveforms = self.get_path('waveforms')
util.ensuredir(path_waveforms)
fns = util.select_files(
[path_waveforms], show_progress=False)
self._pile = pile.Pile()
if fns:
self._pile.load_files(
fns, fileformat='mseed', show_progress=False)
return self._pile
def get_archive(self):
self.ensure_data()
path_tar = self.get_path('archive.tar')
if not op.exists(path_tar):
path_base = self.get_path()
path_waveforms = self.get_path('waveforms')
self.ensure_data()
fns = util.select_files([path_waveforms], show_progress=False)
f = tarfile.TarFile(path_tar, 'w')
for fn in fns:
fna = fn[len(path_base) + 1:]
f.add(fn, fna)
f.close()
return path_tar
def get_archive(self):
self.ensure_data()
path_tar = self.get_path('archive.tar')
if not op.exists(path_tar):
path_base = self.get_path()
path_waveforms = self.get_path('waveforms')
self.ensure_data()
fns = util.select_files(
[path_waveforms], show_progress=False)
f = tarfile.TarFile(path_tar, 'w')
for fn in fns:
fna = fn[len(path_base)+1:]
f.add(fn, fna)
f.close()
return path_tar
def __init__(self, dirpath):
eventdata.EventDataAccess.__init__(self)
self._dirpath = dirpath
self._pile = pile.Pile()
self._pile.load_files(util.select_files([self._dirpath], regex=r'raw-[^/]+\.mseed$'))
