def testPileTraversal(self): import shutil config.show_progress = False nfiles = 200 nsamples = 1000 abc = 'abcdefghijklmnopqrstuvwxyz' def rn(n): return ''.join([random.choice(abc) for i in xrange(n)]) stations = [rn(4) for i in xrange(10)] channels = [rn(3) for i in xrange(3)] networks = ['xx'] tmin = 1234567890 datadir = makeManyFiles( nfiles, nsamples, networks, stations, channels, tmin) filenames = util.select_files([datadir], show_progress=False) cachedir = pjoin(datadir, '_cache_') p = pile.Pile() p.load_files(filenames=filenames, cache=pile.get_cache(cachedir), show_progress=False) assert set(p.networks) == set(networks) assert set(p.stations) == set(stations) assert set(p.channels) == set(channels) toff = 0 while toff < nfiles*nsamples: trs, loaded1 = p.chop(tmin+10, tmin+200) for tr in trs: assert num.all(tr.get_ydata() == num.ones(190)) trs, loaded2 = p.chop(tmin-100, tmin+100) for tr in trs: assert len(tr.get_ydata()) == 100 loaded = loaded1 | loaded2 while loaded: file = loaded.pop() file.drop_data() toff += nsamples s = 0 for traces in p.chopper(tmin=None, tmax=p.tmax+1., tinc=122.): for tr in traces: s += num.sum(tr.ydata) assert s == nfiles*nsamples for fn in filenames: os.utime(fn, None) p.reload_modified() pile.get_cache(cachedir).clean() shutil.rmtree(datadir)
def testPileTraversal(self): import tempfile, shutil config.show_progress = False nfiles = 200 nsamples = 1000 abc = 'abcdefghijklmnopqrstuvwxyz' def rn(n): return ''.join( [ random.choice(abc) for i in xrange(n) ] ) stations = [ rn(4) for i in xrange(10) ] channels = [ rn(3) for i in xrange(3) ] networks = [ 'xx' ] tmin = 1234567890 datadir = makeManyFiles(nfiles, nsamples, networks, stations, channels, tmin) filenames = util.select_files([datadir], show_progress=False) cachedir = pjoin(datadir,'_cache_') p = pile.Pile() p.load_files(filenames=filenames, cache=pile.get_cache(cachedir), show_progress=False) assert set(p.networks) == set(networks) assert set(p.stations) == set(stations) assert set(p.channels) == set(channels) toff = 0 while toff < nfiles*nsamples: trs, loaded1 = p.chop(tmin+10, tmin+200) for tr in trs: assert num.all(tr.get_ydata() == num.ones(190)) trs, loaded2 = p.chop(tmin-100, tmin+100) for tr in trs: assert len(tr.get_ydata()) == 100 loaded = loaded1 | loaded2 while loaded: file = loaded.pop() file.drop_data() toff += nsamples s = 0 for traces in p.chopper(tmin=None, tmax=p.tmax+1., tinc=122. ): #tpad=10.): for trace in traces: s += num.sum(trace.ydata) assert s == nfiles*nsamples for fn in filenames: os.utime(fn, None) p.reload_modified() pile.get_cache(cachedir).clean() shutil.rmtree(datadir)
def add_waveforms(self, paths, regex=None, fileformat='detect', show_progress=False): cachedirname = config.config().cache_dir logger.debug('Selecting waveform files %s...' % quote_paths(paths)) fns = util.select_files(paths, regex=regex, show_progress=show_progress) cache = pile.get_cache(cachedirname) logger.debug('Scanning waveform files %s...' % quote_paths(paths)) self.pile.load_files(sorted(fns), cache=cache, fileformat=fileformat, show_progress=show_progress)
def _update_pile(self): while self._pile_update_args: paths, regex, fileformat, show_progress = \ self._pile_update_args.pop(0) logger.debug('Loading waveform data from %s' % paths) cachedirname = config.config().cache_dir fns = util.select_files(paths, regex=regex, show_progress=show_progress) cache = pile.get_cache(cachedirname) self._pile.load_files(sorted(fns), cache=cache, fileformat=fileformat, show_progress=show_progress)