def make_pile( paths=None, selector=None, regex=None, fileformat = 'mseed', cachedirname=None, show_progress=True ): '''Create pile from given file and directory names. :param paths: filenames and/or directories to look for traces. If paths is ``None`` ``sys.argv[1:]`` is used. :param selector: lambda expression taking group dict of regex match object as a single argument and which returns true or false to keep or reject a file :param regex: regular expression which filenames have to match :param fileformat: format of the files ('mseed', 'sac', 'kan', 'from_extension', 'try') :param cachedirname: loader cache is stored under this directory. It is created as neccessary. :param show_progress: show progress bar and other progress information ''' if isinstance(paths, str): paths = [ paths ] if paths is None: paths = sys.argv[1:] if cachedirname is None: cachedirname = config.config().cache_dir fns = util.select_files(paths, selector, regex, show_progress=show_progress) cache = get_cache(cachedirname) p = Pile() p.load_files( sorted(fns), cache=cache, fileformat=fileformat, show_progress=show_progress) return p
def get_pile(self): if self._pile is None: fns = util.select_files([self.tempdir], regex=r"\.SAC$") self._pile = pile.Pile() self._pile.load_files(fns, fileformat="sac") return self._pile
def get_pile(self): if self._pile is None: fns = util.select_files([self.tempdir], regex=r'\.SAC$') self._pile = pile.Pile() self._pile.load_files(fns, fileformat='sac') return self._pile
def add_gnss_campaigns(self, paths): paths = util.select_files(paths, regex=r'\.yml|\.yaml', show_progress=False) for path in paths: self.add_gnss_campaign(filename=path)
def testPileTraversal(self): import shutil config.show_progress = False nfiles = 200 nsamples = 1000 abc = 'abcdefghijklmnopqrstuvwxyz' def rn(n): return ''.join([random.choice(abc) for i in xrange(n)]) stations = [rn(4) for i in xrange(10)] channels = [rn(3) for i in xrange(3)] networks = ['xx'] tmin = 1234567890 datadir = makeManyFiles( nfiles, nsamples, networks, stations, channels, tmin) filenames = util.select_files([datadir], show_progress=False) cachedir = pjoin(datadir, '_cache_') p = pile.Pile() p.load_files(filenames=filenames, cache=pile.get_cache(cachedir), show_progress=False) assert set(p.networks) == set(networks) assert set(p.stations) == set(stations) assert set(p.channels) == set(channels) toff = 0 while toff < nfiles*nsamples: trs, loaded1 = p.chop(tmin+10, tmin+200) for tr in trs: assert num.all(tr.get_ydata() == num.ones(190)) trs, loaded2 = p.chop(tmin-100, tmin+100) for tr in trs: assert len(tr.get_ydata()) == 100 loaded = loaded1 | loaded2 while loaded: file = loaded.pop() file.drop_data() toff += nsamples s = 0 for traces in p.chopper(tmin=None, tmax=p.tmax+1., tinc=122.): for tr in traces: s += num.sum(tr.ydata) assert s == nfiles*nsamples for fn in filenames: os.utime(fn, None) p.reload_modified() pile.get_cache(cachedir).clean() shutil.rmtree(datadir)
def testPileTraversal(self): import tempfile, shutil config.show_progress = False nfiles = 200 nsamples = 1000 abc = 'abcdefghijklmnopqrstuvwxyz' def rn(n): return ''.join( [ random.choice(abc) for i in xrange(n) ] ) stations = [ rn(4) for i in xrange(10) ] channels = [ rn(3) for i in xrange(3) ] networks = [ 'xx' ] tmin = 1234567890 datadir = makeManyFiles(nfiles, nsamples, networks, stations, channels, tmin) filenames = util.select_files([datadir], show_progress=False) cachedir = pjoin(datadir,'_cache_') p = pile.Pile() p.load_files(filenames=filenames, cache=pile.get_cache(cachedir), show_progress=False) assert set(p.networks) == set(networks) assert set(p.stations) == set(stations) assert set(p.channels) == set(channels) toff = 0 while toff < nfiles*nsamples: trs, loaded1 = p.chop(tmin+10, tmin+200) for tr in trs: assert num.all(tr.get_ydata() == num.ones(190)) trs, loaded2 = p.chop(tmin-100, tmin+100) for tr in trs: assert len(tr.get_ydata()) == 100 loaded = loaded1 | loaded2 while loaded: file = loaded.pop() file.drop_data() toff += nsamples s = 0 for traces in p.chopper(tmin=None, tmax=p.tmax+1., tinc=122. ): #tpad=10.): for trace in traces: s += num.sum(trace.ydata) assert s == nfiles*nsamples for fn in filenames: os.utime(fn, None) p.reload_modified() pile.get_cache(cachedir).clean() shutil.rmtree(datadir)
def get_pile(self): if self._pile is None: #fns = io.save( io.load(pjoin(self.tempdir, 'mini.seed')), pjoin(self.tempdir, # 'raw-%(network)s-%(station)s-%(location)s-%(channel)s.mseed')) fns = util.select_files([self.tempdir], regex=r'\.SAC$') self._pile = pile.Pile() self._pile.load_files(fns, fileformat='sac') return self._pile
def get_pile(self): if self._pile is None: #fns = io.save( io.load(pjoin(self.tempdir, 'mini.seed')), pjoin(self.tempdir, # 'raw-%(network)s-%(station)s-%(location)s-%(channel)s.mseed')) fns = util.select_files( [ self.tempdir ], regex=r'\.SAC$') self._pile = pile.Pile() self._pile.load_files(fns, fileformat='sac') return self._pile
def add_kite_scenes(self, paths): logger.info('Loading kite InSAR scenes...') paths = util.select_files(paths, regex=r'\.npz', show_progress=False) for path in paths: self.add_kite_scene(filename=path) if not self.kite_scenes: logger.warning('Could not find any kite scenes at %s.' % quote_paths(self.kite_scene_paths))
def add_waveforms(self, paths, regex=None, fileformat='detect', show_progress=False): cachedirname = config.config().cache_dir logger.debug('Selecting waveform files %s...' % quote_paths(paths)) fns = util.select_files(paths, regex=regex, show_progress=show_progress) cache = pile.get_cache(cachedirname) logger.debug('Scanning waveform files %s...' % quote_paths(paths)) self.pile.load_files(sorted(fns), cache=cache, fileformat=fileformat, show_progress=show_progress)
def get_insar_scenes(self): from kite import Scene if self._scenes is None: self._scenes = [] path_insar = self.get_path('insar') util.ensuredir(path_insar) fns = util.select_files([path_insar], regex='\\.(npz)$', show_progress=False) for f in fns: self._scenes.append(Scene.load(f)) return self._scenes
def get_insar_scenes(self): from kite import Scene if self._scenes is None: self._scenes = [] path_insar = self.get_path('insar') util.ensuredir(path_insar) fns = util.select_files([path_insar], regex='\.(npz)$', show_progress=False) for f in fns: self._scenes.append(Scene.load(f)) return self._scenes
def _update_pile(self): while self._pile_update_args: paths, regex, fileformat, show_progress = \ self._pile_update_args.pop(0) logger.debug('Loading waveform data from %s' % paths) cachedirname = config.config().cache_dir fns = util.select_files(paths, regex=regex, show_progress=show_progress) cache = pile.get_cache(cachedirname) self._pile.load_files(sorted(fns), cache=cache, fileformat=fileformat, show_progress=show_progress)
def get_waveform_pile(self): self.ensure_data() if self._pile is None: path_waveforms = self.get_path('waveforms') util.ensuredir(path_waveforms) fns = util.select_files([path_waveforms], show_progress=False) self._pile = pile.Pile() if fns: self._pile.load_files(fns, fileformat='mseed', show_progress=False) return self._pile
def _get_pile(self, path): apath = op.abspath(path) assert op.isdir(apath) if apath not in self._piles: fns = util.select_files( [apath], show_progress=False) p = pile.Pile() if fns: p.load_files(fns, fileformat='mseed', show_progress=False) self._piles[apath] = p return self._piles[apath]
def get_waveform_pile(self): self.ensure_data() if self._pile is None: path_waveforms = self.get_path('waveforms') util.ensuredir(path_waveforms) fns = util.select_files( [path_waveforms], show_progress=False) self._pile = pile.Pile() if fns: self._pile.load_files( fns, fileformat='mseed', show_progress=False) return self._pile
def get_archive(self): self.ensure_data() path_tar = self.get_path('archive.tar') if not op.exists(path_tar): path_base = self.get_path() path_waveforms = self.get_path('waveforms') self.ensure_data() fns = util.select_files([path_waveforms], show_progress=False) f = tarfile.TarFile(path_tar, 'w') for fn in fns: fna = fn[len(path_base) + 1:] f.add(fn, fna) f.close() return path_tar
def get_archive(self): self.ensure_data() path_tar = self.get_path('archive.tar') if not op.exists(path_tar): path_base = self.get_path() path_waveforms = self.get_path('waveforms') self.ensure_data() fns = util.select_files( [path_waveforms], show_progress=False) f = tarfile.TarFile(path_tar, 'w') for fn in fns: fna = fn[len(path_base)+1:] f.add(fn, fna) f.close() return path_tar
def __init__(self, dirpath): eventdata.EventDataAccess.__init__(self) self._dirpath = dirpath self._pile = pile.Pile() self._pile.load_files(util.select_files([self._dirpath], regex=r'raw-[^/]+\.mseed$'))