def testPileTraversal(self):
import shutil
config.show_progress = False
nfiles = 200
nsamples = 1000
abc = 'abcdefghijklmnopqrstuvwxyz'
def rn(n):
return ''.join([random.choice(abc) for i in xrange(n)])
stations = [rn(4) for i in xrange(10)]
channels = [rn(3) for i in xrange(3)]
networks = ['xx']
tmin = 1234567890
datadir = makeManyFiles(
nfiles, nsamples, networks, stations, channels, tmin)
filenames = util.select_files([datadir], show_progress=False)
cachedir = pjoin(datadir, '_cache_')
p = pile.Pile()
p.load_files(filenames=filenames, cache=pile.get_cache(cachedir),
show_progress=False)
assert set(p.networks) == set(networks)
assert set(p.stations) == set(stations)
assert set(p.channels) == set(channels)
toff = 0
while toff < nfiles*nsamples:
trs, loaded1 = p.chop(tmin+10, tmin+200)
for tr in trs:
assert num.all(tr.get_ydata() == num.ones(190))
trs, loaded2 = p.chop(tmin-100, tmin+100)
for tr in trs:
assert len(tr.get_ydata()) == 100
loaded = loaded1 | loaded2
while loaded:
file = loaded.pop()
file.drop_data()
toff += nsamples
s = 0
for traces in p.chopper(tmin=None, tmax=p.tmax+1., tinc=122.):
for tr in traces:
s += num.sum(tr.ydata)
assert s == nfiles*nsamples
for fn in filenames:
os.utime(fn, None)
p.reload_modified()
pile.get_cache(cachedir).clean()
shutil.rmtree(datadir)
def testPileTraversal(self):
import tempfile, shutil
config.show_progress = False
nfiles = 200
nsamples = 1000
abc = 'abcdefghijklmnopqrstuvwxyz'
def rn(n):
return ''.join( [ random.choice(abc) for i in xrange(n) ] )
stations = [ rn(4) for i in xrange(10) ]
channels = [ rn(3) for i in xrange(3) ]
networks = [ 'xx' ]
tmin = 1234567890
datadir = makeManyFiles(nfiles, nsamples, networks, stations, channels, tmin)
filenames = util.select_files([datadir], show_progress=False)
cachedir = pjoin(datadir,'_cache_')
p = pile.Pile()
p.load_files(filenames=filenames, cache=pile.get_cache(cachedir),
show_progress=False)
assert set(p.networks) == set(networks)
assert set(p.stations) == set(stations)
assert set(p.channels) == set(channels)
toff = 0
while toff < nfiles*nsamples:
trs, loaded1 = p.chop(tmin+10, tmin+200)
for tr in trs:
assert num.all(tr.get_ydata() == num.ones(190))
trs, loaded2 = p.chop(tmin-100, tmin+100)
for tr in trs:
assert len(tr.get_ydata()) == 100
loaded = loaded1 | loaded2
while loaded:
file = loaded.pop()
file.drop_data()
toff += nsamples
s = 0
for traces in p.chopper(tmin=None, tmax=p.tmax+1., tinc=122. ): #tpad=10.):
for trace in traces:
s += num.sum(trace.ydata)
assert s == nfiles*nsamples
for fn in filenames:
os.utime(fn, None)
p.reload_modified()
pile.get_cache(cachedir).clean()
shutil.rmtree(datadir)
def add_waveforms(self, paths, regex=None, fileformat='detect',
show_progress=False):
cachedirname = config.config().cache_dir
logger.debug('Selecting waveform files %s...' % quote_paths(paths))
fns = util.select_files(paths, regex=regex,
show_progress=show_progress)
cache = pile.get_cache(cachedirname)
logger.debug('Scanning waveform files %s...' % quote_paths(paths))
self.pile.load_files(sorted(fns), cache=cache,
fileformat=fileformat,
show_progress=show_progress)
def _update_pile(self):
while self._pile_update_args:
paths, regex, fileformat, show_progress = \
self._pile_update_args.pop(0)
logger.debug('Loading waveform data from %s' % paths)
cachedirname = config.config().cache_dir
fns = util.select_files(paths, regex=regex,
show_progress=show_progress)
cache = pile.get_cache(cachedirname)
self._pile.load_files(sorted(fns), cache=cache,
fileformat=fileformat,
show_progress=show_progress)