def save_all(events, eventsclusters, clusters, conf, resdir): ''' Save all results of the clustering analysis ''' # save clustered events fn = os.path.join(resdir, 'clustered_events.pf') if os.path.isfile(fn): os.remove(fn) for iev, ev in enumerate(events): tag = 'cluster:' + str(eventsclusters[iev]) if ev.tags is not None: ev.tags.append(tag) else: ev.tags = [tag] model.dump_events(events, fn) # save events of each cluster for cluster_id in clusters: wished_events = [] fn = os.path.join(resdir, 'cluster.' + str(cluster_id) + '.events.pf') for iev, evcl in enumerate(eventsclusters): if evcl == cluster_id: wished_events.append(events[iev]) model.dump_events(wished_events, fn) # save clusters fn = os.path.join(resdir, 'processed.clusters') save_obj(clusters, fn) # save eventsclusters fn = os.path.join(resdir, 'processed.eventsclusters') save_obj(eventsclusters, fn)
def dump(x, gm, indices): if type == 'vector': print(' ', ' '.join('%16.7g' % problem.extract(x, i) for i in indices), '%16.7g' % gm, file=out) elif type == 'source': source = problem.get_source(x) if effective_lat_lon: source.set_origin(*source.effective_latlon) guts.dump(source, stream=out) elif type == 'event': ev = problem.get_source(x).pyrocko_event() if effective_lat_lon: ev.set_origin(*ev.effective_latlon) model.dump_events([ev], stream=out) elif type == 'event-yaml': ev = problem.get_source(x).pyrocko_event() if effective_lat_lon: ev.set_origin(*ev.effective_latlon) guts.dump_all([ev], stream=out) else: raise GrondError('Invalid argument: type=%s' % repr(type))
def testEventExtras(self): tempdir = self.make_tempdir() eextra = model.Event(lat=12., lon=12.) data = [ (dict(i=1, f=1.0, n=None, b=True, s='abc', e=eextra), None), ({1: 'abc'}, guts.ValidationError), ({'e': model.Event(lat=1, lon=1)}, guts.ValidationError)] for d, exc in data: ev1 = model.Event( lat=10., lon=11., depth=4000., magnitude=5., extras=d) fn = pjoin(tempdir, 'test.events') with self.assertRaises(model.EventExtrasDumpError): model.dump_events([ev1], fn) if exc is None: ev1.validate() ev2 = guts.load(string=ev1.dump()) for k in d: assert isinstance(ev2.extras[k], type(d[k])) else: with self.assertRaises(exc): ev1.validate()
def call(self): markers = self.get_selected_event_markers() events = [m.get_event() for m in markers] if len(events) == 0: self.fail('no events found') out_filename = self.output_filename('Template for output files') model.dump_events(events, filename=out_filename)
def save(synthetic_traces, event, stations, savedir, noise_events=False): model.dump_stations(stations, savedir + 'model.txt') io.save(synthetic_traces, savedir + 'traces.mseed') model.dump_events(event, savedir + 'event.txt') model.dump_stations(stations, savedir + 'stations.pf') st_xml = stationxml.FDSNStationXML.from_pyrocko_stations( stations, add_flat_responses_from='M') st_xml.dump_xml(filename=savedir + 'stations.xml') if noise_events is not False: model.dump_events(noise_events, savedir + 'events_noise.txt')
def command_init(args): ''' Execution of command init ''' def setup(parser): parser.add_option('--force', dest='force', action='store_true', help='overwrite existing project directory') parser, options, args = cl_parse('init', args, setup) if len(args) != 1: help_and_die(parser, 'missing argument') else: fn_config = args[0] if not os.path.isfile(fn_config): die('config file missing: %s' % fn_config) conf = load(filename=fn_config) config.check(conf) if ((not options.force) and (os.path.isdir(conf.project_dir))): die('project dir exists: %s; use force option' % conf.project_dir) else: if os.path.isdir(conf.project_dir): shutil.rmtree(conf.project_dir) os.mkdir(conf.project_dir) conf.dump(filename=os.path.join(conf.project_dir, 'seiscloud.config')) dst = os.path.join(conf.project_dir, 'catalog.pf') if conf.catalog_origin == 'file': src = conf.catalog_fn shutil.copyfile(src, dst) else: if conf.catalog_origin == 'globalcmt': orig_catalog = catalog.GlobalCMT() else: # geofon orig_catalog = catalog.Geofon() events = orig_catalog.get_events(time_range=(util.str_to_time( conf.tmin), util.str_to_time(conf.tmax)), magmin=conf.magmin, latmin=conf.latmin, latmax=conf.latmax, lonmin=conf.lonmin, lonmax=conf.lonmax) selevents = [ev for ev in events if ev.magnitude <= conf.magmax] model.dump_events(selevents, dst) print('Project directory prepared "%s"' % conf.project_dir)
def save_all(events,eventsclusters,clusters,conf,resdir): ''' Save all results of the clustering analysis ''' # save events of each cluster for cluster_id in clusters: wished_events=[] fn=os.path.join(resdir,'cluster.'+str(cluster_id)+'.events.pf') for iev,evcl in enumerate(eventsclusters): if evcl==cluster_id: wished_events.append(events[iev]) model.dump_events(wished_events,fn) # save clusters fn=os.path.join(resdir,'processed.clusters') save_obj(clusters, fn) # save eventsclusters fn=os.path.join(resdir,'processed.eventsclusters') save_obj(eventsclusters, fn)
from pyrocko.client import fdsn from pyrocko import util, io, trace, model from pyrocko.io import quakeml tmin = util.stt('2014-01-01 16:10:00.000') tmax = util.stt('2014-01-01 16:39:59.000') # request events at IRIS for the given time span request_event = fdsn.event(site='iris', starttime=tmin, endtime=tmax) # parse QuakeML and extract pyrocko events events = quakeml.QuakeML.load_xml(request_event).get_pyrocko_events() model.dump_events(events, 'iris-events.pf') # select stations by their NSLC id and wildcards (asterisk) for waveform # download selection = [ ('*', 'HMDT', '*', '*', tmin, tmax), # all available components ('GE', 'EIL', '*', '*Z', tmin, tmax), # all vertical components ] # Restricted access token # token = open('token.asc', 'rb').read() # request_waveform = fdsn.dataselect(site='geofon', selection=selection, # token=token) # setup a waveform data request request_waveform = fdsn.dataselect(site='geofon', selection=selection) # write the incoming data stream to 'traces.mseed' with open('traces.mseed', 'wb') as file:
from pyrocko import util, model from pyrocko.client import catalog tmin = util.str_to_time ('2017-01-02 13:00:00') tmax = util.str_to_time ('2017-01-02 14:00:00') # create an instance of the global CMT catalog global_cmt_catalog = catalog.GlobalCMT() # query the catalog events = global_cmt_catalog.get_events( time_range=(tmin, tmax), magmin=4.0, latmin=-5.0, latmax=-4.0, lonmin=-77, lonmax=-76) # dump events to catalog model.dump_events(events, 'GCMT.EQ.txt')
def search(config, override_tmin=None, override_tmax=None, show_detections=False, show_movie=False, show_window_traces=False, force=False, stop_after_first=False, nparallel=6, save_imax=False, bark=False): fp = config.expand_path run_path = fp(config.run_path) # if op.exists(run_path): # if force: # shutil.rmtree(run_path) # else: # raise common.LassieError( # 'run directory already exists: %s' % # run_path) util.ensuredir(run_path) write_config(config, op.join(run_path, 'config.yaml')) ifm_path_template = config.get_ifm_path_template() detections_path = config.get_detections_path() events_path = config.get_events_path() figures_path_template = config.get_figures_path_template() config.setup_image_function_contributions() ifcs = config.image_function_contributions grid = config.get_grid() receivers = config.get_receivers() norm_map = gridmod.geometrical_normalization(grid, receivers) data_paths = fp(config.data_paths) for data_path in fp(data_paths): if not op.exists(data_path): pass p = pile.make_pile(data_paths, fileformat='detect') if p.is_empty(): raise common.LassieError('no usable waveforms found') for ifc in ifcs: ifc.prescan(p) shift_tables = [] tshift_minmaxs = [] for ifc in ifcs: shift_tables.append(ifc.get_table(grid, receivers)) tshift_minmaxs.append(num.nanmin(shift_tables[-1])) tshift_minmaxs.append(num.nanmax(shift_tables[-1])) fsmooth_min = min(ifc.get_fsmooth() for ifc in ifcs) tshift_min = min(tshift_minmaxs) tshift_max = max(tshift_minmaxs) if config.detector_tpeaksearch is not None: tpeaksearch = config.detector_tpeaksearch else: tpeaksearch = (tshift_max - tshift_min) + 1.0 / fsmooth_min tpad = max(ifc.get_tpad() for ifc in ifcs) + \ (tshift_max - tshift_min) + tpeaksearch tinc = (tshift_max - tshift_min) * 10. + 3.0 * tpad tavail = p.tmax - p.tmin tinc = min(tinc, tavail - 2.0 * tpad) if tinc <= 0: raise common.LassieError('available waveforms too short \n' 'required: %g s\n' 'available: %g s\n' % (2. * tpad, tavail)) blacklist = set(tuple(s.split('.')) for s in config.blacklist) whitelist = set(tuple(s.split('.')) for s in config.whitelist) distances = grid.distances(receivers) distances_to_grid = num.min(distances, axis=0) distance_min = num.min(distances) distance_max = num.max(distances) station_index = dict( (rec.codes, i) for (i, rec) in enumerate(receivers) if rec.codes not in blacklist and ( not whitelist or rec.codes in whitelist) and ( config.distance_max is None or distances_to_grid[i] <= config.distance_max)) check_data_consistency(p, config) deltat_cf = max(p.deltats.keys()) assert deltat_cf > 0.0 while True: if not all(ifc.deltat_cf_is_available(deltat_cf * 2) for ifc in ifcs): break deltat_cf *= 2 logger.info('CF lassie sampling interval (rate): %g s (%g Hz)' % (deltat_cf, 1.0 / deltat_cf)) ngridpoints = grid.size() logger.info('number of grid points: %i' % ngridpoints) logger.info('minimum source-receiver distance: %g m' % distance_min) logger.info('maximum source-receiver distance: %g m' % distance_max) logger.info('minimum travel-time: %g s' % tshift_min) logger.info('maximum travel-time: %g s' % tshift_max) idetection = 0 tmin = override_tmin or config.tmin or p.tmin + tpad tmax = override_tmax or config.tmax or p.tmax - tpad events = config.get_events() twindows = [] if events is not None: for ev in events: if tmin <= ev.time <= tmax: twindows.append( (ev.time + tshift_min - (tshift_max - tshift_min) * config.event_time_window_factor, ev.time + tshift_min + (tshift_max - tshift_min) * config.event_time_window_factor)) else: twindows.append((tmin, tmax)) for iwindow_group, (tmin_win, tmax_win) in enumerate(twindows): nwin = int(math.ceil((tmax_win - tmin_win) / tinc)) logger.info('start processing time window group %i/%i: %s - %s' % (iwindow_group + 1, len(twindows), util.time_to_str(tmin_win), util.time_to_str(tmax_win))) logger.info('number of time windows: %i' % nwin) logger.info('time window length: %g s' % (tinc + 2.0 * tpad)) logger.info('time window payload: %g s' % tinc) logger.info('time window padding: 2 x %g s' % tpad) logger.info('time window overlap: %g%%' % (100.0 * 2.0 * tpad / (tinc + 2.0 * tpad))) iwin = -1 for trs in p.chopper( tmin=tmin_win, tmax=tmax_win, tinc=tinc, tpad=tpad, want_incomplete=config.fill_incomplete_with_zeros, trace_selector=lambda tr: tr.nslc_id[:3] in station_index): iwin += 1 trs_ok = [] for tr in trs: if tr.ydata.size == 0: logger.warn('skipping empty trace: %s.%s.%s.%s' % tr.nslc_id) continue if not num.all(num.isfinite(tr.ydata)): logger.warn('skipping trace because of invalid values: ' '%s.%s.%s.%s' % tr.nslc_id) continue trs_ok.append(tr) trs = trs_ok if not trs: continue logger.info('processing time window %i/%i: %s - %s' % (iwin + 1, nwin, util.time_to_str( trs[0].wmin), util.time_to_str(trs[0].wmax))) wmin = trs[0].wmin wmax = trs[0].wmax if config.fill_incomplete_with_zeros: trs = zero_fill(trs, wmin - tpad, wmax + tpad) t0 = math.floor(wmin / deltat_cf) * deltat_cf iwmin = int(round((wmin - tpeaksearch - t0) / deltat_cf)) iwmax = int(round((wmax + tpeaksearch - t0) / deltat_cf)) lengthout = iwmax - iwmin + 1 pdata = [] trs_debug = [] parstack_params = [] for iifc, ifc in enumerate(ifcs): dataset = ifc.preprocess(trs, wmin - tpeaksearch, wmax + tpeaksearch, tshift_max - tshift_min, deltat_cf) if not dataset: continue nstations_selected = len(dataset) nsls_selected, trs_selected = zip(*dataset) for tr in trs_selected: tr.meta = {'tabu': True} trs_debug.extend(trs + list(trs_selected)) istations_selected = num.array( [station_index[nsl] for nsl in nsls_selected], dtype=num.int) arrays = [tr.ydata.astype(num.float) for tr in trs_selected] offsets = num.array([ int(round((tr.tmin - t0) / deltat_cf)) for tr in trs_selected ], dtype=num.int32) w = ifc.get_weights(nsls_selected) weights = num.ones((ngridpoints, nstations_selected)) weights *= w[num.newaxis, :] weights *= ifc.weight shift_table = shift_tables[iifc] ok = num.isfinite(shift_table[:, istations_selected]) bad = num.logical_not(ok) shifts = -num.round(shift_table[:, istations_selected] / deltat_cf).astype(num.int32) weights[bad] = 0.0 shifts[bad] = num.max(shifts[ok]) pdata.append((list(trs_selected), shift_table, ifc)) parstack_params.append((arrays, offsets, shifts, weights)) if config.stacking_blocksize is not None: ipstep = config.stacking_blocksize frames = None else: ipstep = lengthout frames = num.zeros((ngridpoints, lengthout)) twall_start = time.time() frame_maxs = num.zeros(lengthout) frame_argmaxs = num.zeros(lengthout, dtype=num.int) ipmin = iwmin while ipmin < iwmin + lengthout: ipsize = min(ipstep, iwmin + lengthout - ipmin) if ipstep == lengthout: frames_p = frames else: frames_p = num.zeros((ngridpoints, ipsize)) for (arrays, offsets, shifts, weights) in parstack_params: frames_p, _ = parstack(arrays, offsets, shifts, weights, 0, offsetout=ipmin, lengthout=ipsize, result=frames_p, nparallel=nparallel, impl='openmp') if config.sharpness_normalization: frame_p_maxs = frames_p.max(axis=0) frame_p_means = num.abs(frames_p).mean(axis=0) frames_p *= (frame_p_maxs / frame_p_means)[num.newaxis, :] frames_p *= norm_map[:, num.newaxis] if config.ifc_count_normalization: frames_p *= 1.0 / len(ifcs) frame_maxs[ipmin-iwmin:ipmin-iwmin+ipsize] = \ frames_p.max(axis=0) frame_argmaxs[ipmin-iwmin:ipmin-iwmin+ipsize] = \ pargmax(frames_p) ipmin += ipstep del frames_p twall_end = time.time() logger.info('wallclock time for stacking: %g s' % (twall_end - twall_start)) tmin_frames = t0 + iwmin * deltat_cf tr_stackmax = trace.Trace('', 'SMAX', '', '', tmin=tmin_frames, deltat=deltat_cf, ydata=frame_maxs) tr_stackmax.meta = {'tabu': True} trs_debug.append(tr_stackmax) if show_window_traces: trace.snuffle(trs_debug) ydata_window = tr_stackmax.chop(wmin, wmax, inplace=False).get_ydata() logger.info('CF stats: min %g, max %g, median %g' % (num.min(ydata_window), num.max(ydata_window), num.median(ydata_window))) if nstations_selected != 17: logger.info( 'Warning, station outage detected! Nr of station operable: %s' % nstations_selected) detector_threshold_seiger = config.detector_threshold - ( (17 - nstations_selected) * 4 ) # 17 is maximum number of seiger stations, 4 is a mean baseline for noise if nstations_selected != 17: logger.info( 'Warning, station outage detected! Nr of station operable: %s, threshold now: %s' % (nstations_selected, detector_threshold_seiger)) tpeaks, apeaks = list( zip(*[(tpeak, apeak) for (tpeak, apeak) in zip( *tr_stackmax.peaks(detector_threshold_seiger, tpeaksearch)) if wmin <= tpeak and tpeak < wmax])) or ([], []) tr_stackmax_indx = tr_stackmax.copy(data=False) tr_stackmax_indx.set_ydata(frame_argmaxs.astype(num.int32)) tr_stackmax_indx.set_location('i') for (tpeak, apeak) in zip(tpeaks, apeaks): iframe = int(round((tpeak - tmin_frames) / deltat_cf)) imax = frame_argmaxs[iframe] latpeak, lonpeak, xpeak, ypeak, zpeak = \ grid.index_to_location(imax) idetection += 1 detection = Detection(id='%06i' % idetection, time=tpeak, location=geo.Point(lat=float(latpeak), lon=float(lonpeak), x=float(xpeak), y=float(ypeak), z=float(zpeak)), ifm=float(apeak)) if bark: common.bark() logger.info('detection found: %s' % str(detection)) f = open(detections_path, 'a') f.write( '%06i %s %g %g %g %g %g %g\n' % (idetection, util.time_to_str(tpeak, format='%Y-%m-%d %H:%M:%S.6FRAC'), apeak, latpeak, lonpeak, xpeak, ypeak, zpeak)) f.close() ev = detection.get_event() f = open(events_path, 'a') model.dump_events([ev], stream=f) f.close() if show_detections or config.save_figures: fmin = min(ifc.fmin for ifc in ifcs) fmax = min(ifc.fmax for ifc in ifcs) fn = figures_path_template % { 'id': util.tts(t0).replace(" ", "T"), 'format': 'png' } util.ensuredirs(fn) if frames is not None: frames_p = frames tmin_frames_p = tmin_frames iframe_p = iframe else: iframe_min = max( 0, int(round(iframe - tpeaksearch / deltat_cf))) iframe_max = min( lengthout - 1, int(round(iframe + tpeaksearch / deltat_cf))) ipsize = iframe_max - iframe_min + 1 frames_p = num.zeros((ngridpoints, ipsize)) tmin_frames_p = tmin_frames + iframe_min * deltat_cf iframe_p = iframe - iframe_min for (arrays, offsets, shifts, weights) \ in parstack_params: frames_p, _ = parstack(arrays, offsets, shifts, weights, 0, offsetout=iwmin + iframe_min, lengthout=ipsize, result=frames_p, nparallel=nparallel, impl='openmp') if config.sharpness_normalization: frame_p_maxs = frames_p.max(axis=0) frame_p_means = num.abs(frames_p).mean(axis=0) frames_p *= (frame_p_maxs / frame_p_means)[num.newaxis, :] frames_p *= norm_map[:, num.newaxis] if config.ifc_count_normalization: frames_p *= 1.0 / len(ifcs) try: plot.plot_detection(grid, receivers, frames_p, tmin_frames_p, deltat_cf, imax, iframe_p, xpeak, ypeak, zpeak, tr_stackmax, tpeaks, apeaks, detector_threshold_seiger, wmin, wmax, pdata, trs, fmin, fmax, idetection, tpeaksearch, movie=show_movie, show=show_detections, save_filename=fn, event=ev) except: pass del frames_p if stop_after_first: return tr_stackmax.chop(wmin, wmax) tr_stackmax_indx.chop(wmin, wmax) if save_imax is True: io.save([tr_stackmax, tr_stackmax_indx], ifm_path_template) del frames logger.info('end processing time window group: %s - %s' % (util.time_to_str(tmin_win), util.time_to_str(tmax_win))) cat = Catalog() files = glob("%s/../figures/*qml*" % run_path) files.sort(key=os.path.getmtime) for file in files: cat_read = read_events(file) for event in cat_read: cat.append(event) cat.write("%s/../all_events_stacking.qml" % run_path, format="QUAKEML")
phasename=phasename, event=event) picks.append(m) progbar.update(i) progbar.finish() with open(fn, 'w') as f: f.write(datastr) gui_util.save_markers(picks, 'picks.pf') def reduce_stf(self, event, t): t -= self.interp_stf(event.magnitude)/2. return t if __name__=='__main__': stations = model.load_stations('/media/usb/webnet/meta/stations.pf') events = [] for efile in glob.glob('/home/marius/src/swarming/unperturbed/*/event.pf'): events.append(model.Event(load=efile)) #events = model.load_events('/home/marius/src/swarming/events_swarm.pf') want_phases = {'p':('p', 'P'), 's':('s', 'S')} e = LocalEngine(use_config=True) s = e.get_store('vogtland_fischer_horalek_2000_vpvs169_minus4p') m = s.config.earthmodel_1d b = Bakery(stations=stations, events=events, wanted_phases=want_phases, model=m) model.dump_events(events, 'dumped_catalog.pf') b.run_n_write('picks.dat')
with open(fn, 'r') as f: # windows empty line separation events = f.read().split('\r\n\r\n') py_events = [] pattern = re.compile(r'((?P<mid>\D{3}\d{2}) (?P<t>\d{6}.\d) (?P<place>\D*\(\d{3}\)) (?P<lat>\d{2}.\d{2})N (?P<lon>\d{2}.\d{2})E h=(?P<depth>\d*)km mb=(?P<mb>\d.\d)', flags=re.IGNORECASE) for e in events: m = pattern.findall(e) if m: groupdict = m.groupdict() t = groupdict['t'] #mb = groupdict['mb'] #mw = groupdict['mw'] #print '...................' #print mb #print mw date_str = '%s-%s-%s %s:%s:%s' % (year, month, groupdict['mid'][3:], t[0:2], t[2:4], t[4:]) t = str_to_time(date_str) py_events.append(Event(name=groupdict['place'], time=t, lat=float(groupdict['lat']), lon=float(groupdict['lon']), depth=float(groupdict['depth']))) else: #print 'skipping ', e continue dump_events(py_events, 'events_crawled.pf')
from pyrocko import util, model from pyrocko.client import catalog tmin = util.str_to_time('2011-01-01 00:00:00') # beginning time of query tmax = util.str_to_time('2011-12-31 23:59:59') # create an instance of the global CMT catalog global_cmt_catalog = catalog.GlobalCMT() # query the catalog events = global_cmt_catalog.get_events( time_range=(tmin, tmax), magmin=2., latmin=-35., latmax=-20., lonmin=-76., lonmax=-65.) print('Downloaded %s events' % len(events)) print('The last one is:') print(events[-1]) # dump events to catalog model.dump_events(events, 'northern_chile_events.txt')
mechanisms=mechanisms, magnitudes=magnitudes, stf=stf) # setup stations/targets: #stats = load_stations(webnet+'/meta/stations.pf') stats = load_stations('stations.pf') #stats = [] # Scrutinize the swarm using matplotlib noise = Noise(files='/media/usb/webnet/mseed/2008') # convert loaded stations to targets (see function at the top). #targets = guess_targets_from_stations(stats) targets = get_targets(stations, noise.data_pile, store_id=store_id) Visualizer(swarm, stats) # Processing that data will return a pyrocko.gf.seismosizer.Reponse object. response = engine.process(sources=swarm.get_sources(), targets=targets) # Save the events dump_events(swarm.get_events(), 'events_swarm.pf') io.save(response.pyrocko_traces(), 'swarm.mseed') convolved_traces = stf.post_process(response) # Save traces: io.save(convolved_traces.traces_list(), 'swarm_stf.mseed')
def report(env, report_config=None, update_without_plotting=True, make_index=True, make_archive=True, nthreads=0): if report_config is None: report_config = ReportConfig() report_config.set_basepath('.') event_name = env.get_current_event_name() logger.info('Creating report entry for run "%s"...' % event_name) fp = report_config.expand_path entry_path = expand_template( op.join(fp(report_config.report_base_path), report_config.entries_sub_path), dict(event_name=event_name, problem_name=event_name)) if op.exists(entry_path) and not update_without_plotting: shutil.rmtree(entry_path) util.ensuredir(entry_path) plots_dir_out = op.join(entry_path, 'plots') util.ensuredir(plots_dir_out) configs_dir = op.join(op.split(__file__)[0], 'app/configs/') rundir_path = env.get_rundir_path() try: os.system("cp -r %s/grun/plots/* %s" % (rundir_path, plots_dir_out)) except: pass os.system("cp %s/plot_collection.yaml %s" % (configs_dir, plots_dir_out)) util.ensuredir("%s/shakemap/default/" % (plots_dir_out)) os.system( "cp %s/*shakemap.png %s/shakemap/default/shakemap.default.gf_shakemap.d100.png" % (rundir_path, plots_dir_out)) os.system("cp %s/shakemap.default.plot_group.yaml %s/shakemap/default/" % (configs_dir, plots_dir_out)) util.ensuredir("%s/location/default/" % (plots_dir_out)) os.system( "cp %s/*location.png %s/location/default/location.default.location.d100.png" % (rundir_path, plots_dir_out)) os.system("cp %s/location.default.plot_group.yaml %s/location/default/" % (configs_dir, plots_dir_out)) util.ensuredir("%s/production_data/default/" % (plots_dir_out)) os.system( "cp %s/*production_data.png %s/production_data/default/production_data.default.production_data.d100.png" % (rundir_path, plots_dir_out)) os.system( "cp %s/production_data.default.plot_group.yaml %s/production_data/default/" % (configs_dir, plots_dir_out)) util.ensuredir("%s/waveforms/default/" % (plots_dir_out)) os.system( "cp %s/waveforms_1.png %s/waveforms/default/waveforms.default.waveforms_1.d100.png" % (rundir_path, plots_dir_out)) os.system( "cp %s/waveforms_2.png %s/waveforms/default/waveforms.default.waveforms_2.d100.png" % (rundir_path, plots_dir_out)) os.system( "cp %s/waveforms_3.png %s/waveforms/default/waveforms.default.waveforms_3.d100.png" % (rundir_path, plots_dir_out)) os.system("cp %s/waveforms.default.plot_group.yaml %s/waveforms/default/" % (configs_dir, plots_dir_out)) os.system("cp %s/grun/config.yaml %s/config.yaml" % (rundir_path, entry_path)) try: event = model.load_events(rundir_path + "event.txt") model.dump_events(event, filename=op.join(entry_path, 'event_reference.yaml'), format="yaml") event = event[0] from silvertine import plot pcc = report_config.plot_config_collection.get_weeded(env) plot.make_plots(env, plots_path=op.join(entry_path, 'plots'), plot_config_collection=pcc) try: run_info = env.get_run_info() except environment.NoRundirAvailable: run_info = None rie = ReportIndexEntry(path='.', problem_name=event_name, silvertine_version="0.01", run_info=run_info) fn = op.join(entry_path, 'event_reference.yaml') if op.exists(fn): rie.event_best = guts.load(filename=fn) fn = op.join(entry_path, 'event_reference.yaml') if op.exists(fn): rie.event_reference = guts.load(filename=fn) fn = op.join(entry_path, 'index.yaml') guts.dump(rie, filename=fn) logger.info('Done creating report entry for run "%s".' % "test") # report_index(report_config) # if make_archive: # report_archive(report_config) except FileNotFoundError: pass
except util.UnavailableDecimation as e: logger.warn('Cannot downsample %s.%s.%s.%s: %s' % (tr.nslc_id + (e, ))) continue if options.out_components == 'rtu': pios = s.guess_projections_to_rtu(out_channels=('R', 'T', 'Z')) elif options.out_components == 'enu': pios = s.guess_projections_to_enu(out_channels=('E', 'N', 'Z')) else: assert False for (proj, in_channels, out_channels) in pios: proc = trace.project(traces, proj, in_channels, out_channels) for tr in proc: for ch in out_channels: if ch.name == tr.channel: s.add_channel(ch) if proc: io.save(proc, fn_waveforms) used_stations.append(s) stations = list(used_stations) util.ensuredirs(fn_stations) model.dump_stations(stations, fn_stations) model.dump_events([event], fn_event) logger.info('prepared waveforms from %i stations' % len(stations))
Origin = C.parseConfig('origin') Conf = globalConf() Config = C.parseConfig('config') filter = FilterCfg(Config) cfg = ConfigObj(dict=Config) minDist, maxDist = cfg.FloatRange('mindist', 'maxdist') ev = Event(Origin['lat'],Origin['lon'],Origin['depth'],Origin['time'] ) event = model.Event(lat=float(ev.lat), lon=float(ev.lon), depth=float(ev.depth)*1000., time=util.str_to_time(ev.time)) newFreq = float(filter.newFrequency()) options.time = Origin ['time'] options.duration = int(Conf['duration']) sdspath = os.path.join(options.eventpath,'data') model.dump_events([event], sdspath+'event.pf') tmin = util.str_to_time(ev.time)-600. tmax = util.str_to_time(ev.time)+1800. def get_stations(site, lat, lon, rmin, rmax, tmin, tmax, channel_pattern='BH*'): extra = {} if site == 'iris': extra.update(matchtimeseries=True) sx = fdsn.station( site=site, latitude=lat, longitude=lon, minradius=rmin, maxradius=rmax, startbefore=tmin, endafter=tmax, channel=channel_pattern, format='text', level='channel', includerestricted=False)
str(eml.lon), str(eml.depth), str(eml.magnitude), str(emw.magnitude) ]) event_params.append([ str(util.tts(eml.time)), str(eml.time), str(eml.lat), str(eml.lon), str(eml.depth), str(eml.magnitude), str(emw.magnitude) ]) model.dump_events(land_events, "landau_events.pf") model.dump_events(ins_events, "ins_events.pf") with open('seiger_catalog_alpha.csv', 'w', newline='') as csvfile: spamwriter = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) spamwriter.writerow( ["Time", "Time(s)unix time", "Lat", "Lon", "Tiefe", "Ml", "Mw"]) for p in event_params: spamwriter.writerow(p) with open('seiger_catalog_alpha_landau.csv', 'w', newline='') as csvfile: spamwriter = csv.writer(csvfile, delimiter=',',
"mt3", "mt4", "mt5", "dc?", "_", "_", ] order_mapping = dict(zip(order, range(len(order)))) wanted = ["name", "lat", "lon", "depth"] events = [] with open(fn, "r") as f: for line in f.readlines(): data = line.split() kwargs = {} for kw in wanted: val = data[order_mapping[kw]] val = float(val) if kw is not "name" else val val = float(val) * 1000.0 if kw is "depth" else val kwargs.update({kw: val}) time = str_to_time("%s %s" % (data[order_mapping["date"]].replace("/", "-"), data[order_mapping["time"]])) kwargs.update({"time": time}) m = num.array([num.float(data[order_mapping["mt%i" % i]]) for i in range(6)]) mt = MomentTensor(m=symmat6(*m) * float(data[order_mapping["moment"]])) kwargs.update({"moment_tensor": mt}) e = Event(**kwargs) events.append(e) dump_events(events, "events2008_mt.pf")
def command_matrix(args): ''' Execution of command matrix ''' def setup(parser): parser.add_option('--force', dest='force', action='store_true', help='overwrite existing project directory') parser.add_option('--view', dest='view', action='store_true', help='view similarity matrix') parser.add_option('--savefig', dest='savefig', action='store_true', help='save figure of similarity matrix') parser, options, args = cl_parse('matrix', args, setup) if len(args) != 1: help_and_die(parser, 'missing argument') else: fn_config = args[0] if not os.path.isfile(fn_config): die('config file missing: %s' % fn_config) conf = load(filename=fn_config) config.check(conf) if not os.path.isdir(conf.project_dir): die('project directory missing: %s' % conf.project_dir) simmat_temporal_fn = os.path.join(conf.project_dir, 'simmat_temporal.npy') if ((not options.force) and (os.path.isfile(simmat_temporal_fn))): die('similarity matrix exists: %s; use force option' % simmat_temporal_fn) catalog_ref_fn = os.path.join(conf.project_dir, 'catalog.pf') if os.path.isfile(catalog_ref_fn): allevents = model.load_events(catalog_ref_fn) else: die('catalog missing: %s' % catalog_ref_fn) if conf.sw_simmat: if not os.path.isfile(conf.sim_mat_fn): die('similarity matrix missing: %s' % conf.sim_mat_fn) if conf.sim_mat_type == 'binary': if os.path.isfile(conf.sim_mat_fn): simmat = sccluster.load_similarity_matrix(conf.sim_mat_fn) else: die('cannot read similarity matrix: %s' % conf.sim_mat_fn) else: die('ascii format for similarity matrix not yet implemented') if len(allevents) != len(simmat): print(len(allevents), len(simmat)) die('clustering stopped, number of events ' + 'differs from matrix size') new_catalog_fn = os.path.join(conf.project_dir, 'events_to_be_clustered.pf') model.dump_events(allevents, new_catalog_fn) else: if conf.metric in config.acceptable_mt_based_metrics: events = [ev for ev in allevents if ev.moment_tensor is not None] else: events = [ev for ev in allevents] new_catalog_fn = os.path.join(conf.project_dir, 'events_to_be_clustered.pf') model.dump_events(events, new_catalog_fn) simmat = sccluster.compute_similarity_matrix(events, conf.metric) sccluster.save_similarity_matrix(simmat, simmat_temporal_fn) simmat_fig_fn = os.path.join(conf.project_dir, 'simmat_temporal.' + conf.figure_format) if options.view and options.savefig: scplot.view_and_savefig_similarity_matrix(simmat, simmat_fig_fn, 'Sorted chronologically') else: if options.view: scplot.view_similarity_matrix(simmat, 'Sorted chronologically') if options.savefig: scplot.savefig_similarity_matrix(simmat, simmat_fig_fn, 'Sorted chronologically') print('Similarity matrix computed and stored as "%s"' % simmat_temporal_fn) if options.savefig: print('Similarity matrix figure saved as "%s"' % simmat_fig_fn)
def main(): parser = OptionParser(usage=usage, description=description) parser.add_option('--force', dest='force', action='store_true', default=False, help='allow recreation of output <directory>') parser.add_option('--debug', dest='debug', action='store_true', default=False, help='print debugging information to stderr') parser.add_option('--dry-run', dest='dry_run', action='store_true', default=False, help='show available stations/channels and exit ' '(do not download waveforms)') parser.add_option('--continue', dest='continue_', action='store_true', default=False, help='continue download after a accident') parser.add_option('--local-data', dest='local_data', action='append', help='add file/directory with local data') parser.add_option('--local-stations', dest='local_stations', action='append', help='add local stations file') parser.add_option('--selection', dest='selection_file', action='append', help='add local stations file') parser.add_option( '--local-responses-resp', dest='local_responses_resp', action='append', help='add file/directory with local responses in RESP format') parser.add_option('--local-responses-pz', dest='local_responses_pz', action='append', help='add file/directory with local pole-zero responses') parser.add_option( '--local-responses-stationxml', dest='local_responses_stationxml', help='add file with local response information in StationXML format') parser.add_option( '--window', dest='window', default='full', help='set time window to choose [full, p, "<time-start>,<time-end>"' '] (time format is YYYY-MM-DD HH:MM:SS)') parser.add_option( '--out-components', choices=['enu', 'rtu'], dest='out_components', default='rtu', help='set output component orientations to radial-transverse-up [rtu] ' '(default) or east-north-up [enu]') parser.add_option('--out-units', choices=['M', 'M/S', 'M/S**2'], dest='output_units', default='M', help='set output units to displacement "M" (default),' ' velocity "M/S" or acceleration "M/S**2"') parser.add_option( '--padding-factor', type=float, default=3.0, dest='padding_factor', help='extend time window on either side, in multiples of 1/<fmin_hz> ' '(default: 5)') parser.add_option( '--zero-padding', dest='zero_pad', action='store_true', default=False, help='Extend traces by zero-padding if clean restitution requires' 'longer windows') parser.add_option( '--credentials', dest='user_credentials', action='append', default=[], metavar='SITE,USER,PASSWD', help='user credentials for specific site to access restricted data ' '(this option can be repeated)') parser.add_option( '--token', dest='auth_tokens', metavar='SITE,FILENAME', action='append', default=[], help='user authentication token for specific site to access ' 'restricted data (this option can be repeated)') parser.add_option( '--sites', dest='sites', metavar='SITE1,SITE2,...', default='geofon,iris,orfeus', help='sites to query (available: %s, default: "%%default"' % ', '.join(g_sites_available)) parser.add_option( '--band-codes', dest='priority_band_code', metavar='V,L,M,B,H,S,E,...', default='B,H', help='select and prioritize band codes (default: %default)') parser.add_option( '--instrument-codes', dest='priority_instrument_code', metavar='H,L,G,...', default='H,L', help='select and prioritize instrument codes (default: %default)') parser.add_option('--radius-min', dest='radius_min', metavar='VALUE', default=0.0, type=float, help='minimum radius [km]') parser.add_option('--nstations-wanted', dest='nstations_wanted', metavar='N', type=int, help='number of stations to select initially') (options, args) = parser.parse_args(sys.argv[1:]) print('Parsed arguments:', args) if len(args) not in (10, 7, 6): parser.print_help() sys.exit(1) if options.debug: util.setup_logging(program_name, 'debug') else: util.setup_logging(program_name, 'info') if options.local_responses_pz and options.local_responses_resp: logger.critical('cannot use local responses in PZ and RESP ' 'format at the same time') sys.exit(1) n_resp_opt = 0 for resp_opt in (options.local_responses_pz, options.local_responses_resp, options.local_responses_stationxml): if resp_opt: n_resp_opt += 1 if n_resp_opt > 1: logger.critical('can only handle local responses from either PZ or ' 'RESP or StationXML. Cannot yet merge different ' 'response formats.') sys.exit(1) if options.local_responses_resp and not options.local_stations: logger.critical('--local-responses-resp can only be used ' 'when --stations is also given.') sys.exit(1) try: ename = '' magnitude = None mt = None if len(args) == 10: time = util.str_to_time(args[1] + ' ' + args[2]) lat = float(args[3]) lon = float(args[4]) depth = float(args[5]) * km iarg = 6 elif len(args) == 7: if args[2].find(':') == -1: sname_or_date = None lat = float(args[1]) lon = float(args[2]) event = None time = None else: sname_or_date = args[1] + ' ' + args[2] iarg = 3 elif len(args) == 6: sname_or_date = args[1] iarg = 2 if len(args) in (7, 6) and sname_or_date is not None: events = get_events_by_name_or_date([sname_or_date], catalog=geofon) if len(events) == 0: logger.critical('no event found') sys.exit(1) elif len(events) > 1: logger.critical('more than one event found') sys.exit(1) event = events[0] time = event.time lat = event.lat lon = event.lon depth = event.depth ename = event.name magnitude = event.magnitude mt = event.moment_tensor radius = float(args[iarg]) * km fmin = float(args[iarg + 1]) sample_rate = float(args[iarg + 2]) eventname = args[iarg + 3] cwd = str(sys.argv[1]) event_dir = op.join(cwd, 'data', 'events', eventname) output_dir = op.join(event_dir, 'waveforms') except: raise parser.print_help() sys.exit(1) if options.force and op.isdir(event_dir): if not options.continue_: shutil.rmtree(event_dir) if op.exists(event_dir) and not options.continue_: logger.critical( 'directory "%s" exists. Delete it first or use the --force option' % event_dir) sys.exit(1) util.ensuredir(output_dir) if time is not None: event = model.Event(time=time, lat=lat, lon=lon, depth=depth, name=ename, magnitude=magnitude, moment_tensor=mt) if options.window == 'full': if event is None: logger.critical('need event for --window=full') sys.exit(1) low_velocity = 1500. timewindow = VelocityWindow(low_velocity, tpad=options.padding_factor / fmin) tmin, tmax = timewindow(time, radius, depth) elif options.window == 'p': if event is None: logger.critical('need event for --window=p') sys.exit(1) phases = list(map(cake.PhaseDef, 'P p'.split())) emod = cake.load_model() tpad = options.padding_factor / fmin timewindow = PhaseWindow(emod, phases, -tpad, tpad) arrivaltimes = [] for dist in num.linspace(0, radius, 20): try: arrivaltimes.extend(timewindow(time, dist, depth)) except NoArrival: pass if not arrivaltimes: logger.error('required phase arrival not found') sys.exit(1) tmin = min(arrivaltimes) tmax = max(arrivaltimes) else: try: stmin, stmax = options.window.split(',') tmin = util.str_to_time(stmin.strip()) tmax = util.str_to_time(stmax.strip()) timewindow = FixedWindow(tmin, tmax) except ValueError: logger.critical('invalid argument to --window: "%s"' % options.window) sys.exit(1) if event is not None: event.name = eventname tfade = tfade_factor / fmin tpad = tfade tmin -= tpad tmax += tpad tinc = None priority_band_code = options.priority_band_code.split(',') for s in priority_band_code: if len(s) != 1: logger.critical('invalid band code: %s' % s) priority_instrument_code = options.priority_instrument_code.split(',') for s in priority_instrument_code: if len(s) != 1: logger.critical('invalid instrument code: %s' % s) station_query_conf = dict(latitude=lat, longitude=lon, minradius=options.radius_min * km * cake.m2d, maxradius=radius * cake.m2d, channel=','.join('%s??' % s for s in priority_band_code)) target_sample_rate = sample_rate fmax = target_sample_rate # target_sample_rate = None # priority_band_code = ['H', 'B', 'M', 'L', 'V', 'E', 'S'] priority_units = ['M/S', 'M', 'M/S**2'] # output_units = 'M' sites = [x.strip() for x in options.sites.split(',') if x.strip()] for site in sites: if site not in g_sites_available: logger.critical('unknown FDSN site: %s' % site) sys.exit(1) for s in options.user_credentials: try: site, user, passwd = s.split(',') g_user_credentials[site] = user, passwd except ValueError: logger.critical('invalid format for user credentials: "%s"' % s) sys.exit(1) for s in options.auth_tokens: try: site, token_filename = s.split(',') with open(token_filename, 'r') as f: g_auth_tokens[site] = f.read() except (ValueError, OSError, IOError): logger.critical('cannot get token from file: %s' % token_filename) sys.exit(1) fn_template0 = \ 'data_%(network)s.%(station)s.%(location)s.%(channel)s_%(tmin)s.mseed' fn_template_raw = op.join(output_dir, 'raw', fn_template0) fn_stations_raw = op.join(output_dir, 'stations.raw.txt') fn_template_rest = op.join(output_dir, 'rest', fn_template0) fn_commandline = op.join(output_dir, 'beatdown.command') ftap = (ffade_factors[0] * fmin, fmin, fmax, ffade_factors[1] * fmax) # chapter 1: download sxs = [] for site in sites: try: extra_args = { 'iris': dict(matchtimeseries=True), }.get(site, {}) extra_args.update(station_query_conf) if site == 'geonet': extra_args.update(starttime=tmin, endtime=tmax) else: extra_args.update(startbefore=tmax, endafter=tmin, includerestricted=(site in g_user_credentials or site in g_auth_tokens)) logger.info('downloading channel information (%s)' % site) sx = fdsn.station(site=site, format='text', level='channel', **extra_args) except fdsn.EmptyResult: logger.error('No stations matching given criteria. (%s)' % site) sx = None if sx is not None: sxs.append(sx) if all(sx is None for sx in sxs) and not options.local_data: sys.exit(1) nsl_to_sites = defaultdict(list) nsl_to_station = {} if options.selection_file: logger.info('using stations from stations file!') stations = [] for fn in options.selection_file: stations.extend(model.load_stations(fn)) nsls_selected = set(s.nsl() for s in stations) else: nsls_selected = None for sx, site in zip(sxs, sites): site_stations = sx.get_pyrocko_stations() for s in site_stations: nsl = s.nsl() nsl_to_sites[nsl].append(site) if nsl not in nsl_to_station: if nsls_selected: if nsl in nsls_selected: nsl_to_station[nsl] = s else: nsl_to_station[ nsl] = s # using first site with this station logger.info('number of stations found: %i' % len(nsl_to_station)) # station weeding if options.nstations_wanted: nsls_selected = None stations_all = [ nsl_to_station[nsl_] for nsl_ in sorted(nsl_to_station.keys()) ] for s in stations_all: s.set_event_relative_data(event) stations_selected = weeding.weed_stations(stations_all, options.nstations_wanted)[0] nsls_selected = set(s.nsl() for s in stations_selected) logger.info('number of stations selected: %i' % len(nsls_selected)) if tinc is None: tinc = 3600. have_data = set() if options.continue_: fns = glob.glob(fn_template_raw % starfill()) p = pile.make_pile(fns) else: fns = [] have_data_site = {} could_have_data_site = {} for site in sites: have_data_site[site] = set() could_have_data_site[site] = set() available_through = defaultdict(set) it = 0 nt = int(math.ceil((tmax - tmin) / tinc)) for it in range(nt): tmin_win = tmin + it * tinc tmax_win = min(tmin + (it + 1) * tinc, tmax) logger.info('time window %i/%i (%s - %s)' % (it + 1, nt, util.tts(tmin_win), util.tts(tmax_win))) have_data_this_window = set() if options.continue_: trs_avail = p.all(tmin=tmin_win, tmax=tmax_win, load_data=False) for tr in trs_avail: have_data_this_window.add(tr.nslc_id) for site, sx in zip(sites, sxs): if sx is None: continue selection = [] channels = sx.choose_channels( target_sample_rate=target_sample_rate, priority_band_code=priority_band_code, priority_units=priority_units, priority_instrument_code=priority_instrument_code, timespan=(tmin_win, tmax_win)) for nslc in sorted(channels.keys()): if nsls_selected is not None and nslc[:3] not in nsls_selected: continue could_have_data_site[site].add(nslc) if nslc not in have_data_this_window: channel = channels[nslc] if event: lat_, lon_ = event.lat, event.lon else: lat_, lon_ = lat, lon try: dist = orthodrome.distance_accurate50m_numpy( lat_, lon_, channel.latitude.value, channel.longitude.value) except: dist = orthodrome.distance_accurate50m_numpy( lat_, lon_, channel.latitude, channel.longitude) if event: depth_ = event.depth time_ = event.time else: depth_ = None time_ = None tmin_, tmax_ = timewindow(time_, dist, depth_) tmin_this = tmin_ - tpad tmax_this = float(tmax_ + tpad) tmin_req = max(tmin_win, tmin_this) tmax_req = min(tmax_win, tmax_this) if channel.sample_rate: try: deltat = 1.0 / int(channel.sample_rate.value) except: deltat = 1.0 / int(channel.sample_rate) else: deltat = 1.0 if tmin_req < tmax_req: logger.debug('deltat %f' % deltat) # extend time window by some samples because otherwise # sometimes gaps are produced # apparently the WS are only sensitive to full seconds # round to avoid gaps, increase safetiy window selection.append(nslc + (math.floor(tmin_req - deltat * 20.0), math.ceil(tmax_req + deltat * 20.0))) if options.dry_run: for (net, sta, loc, cha, tmin, tmax) in selection: available_through[net, sta, loc, cha].add(site) else: neach = 100 i = 0 nbatches = ((len(selection) - 1) // neach) + 1 while i < len(selection): selection_now = selection[i:i + neach] f = tempfile.NamedTemporaryFile() try: sbatch = '' if nbatches > 1: sbatch = ' (batch %i/%i)' % ( (i // neach) + 1, nbatches) logger.info('downloading data (%s)%s' % (site, sbatch)) data = fdsn.dataselect(site=site, selection=selection_now, **get_user_credentials(site)) while True: buf = data.read(1024) if not buf: break f.write(buf) f.flush() trs = io.load(f.name) for tr in trs: tr.fix_deltat_rounding_errors() logger.debug('cutting window: %f - %f' % (tmin_win, tmax_win)) logger.debug( 'available window: %f - %f, nsamples: %g' % (tr.tmin, tr.tmax, tr.ydata.size)) try: logger.debug('tmin before snap %f' % tr.tmin) tr.snap(interpolate=True) logger.debug('tmin after snap %f' % tr.tmin) tr.chop(tmin_win, tmax_win, snap=(math.floor, math.ceil), include_last=True) logger.debug( 'cut window: %f - %f, nsamles: %g' % (tr.tmin, tr.tmax, tr.ydata.size)) have_data.add(tr.nslc_id) have_data_site[site].add(tr.nslc_id) except trace.NoData: pass fns2 = io.save(trs, fn_template_raw) for fn in fns2: if fn in fns: logger.warn('overwriting file %s', fn) fns.extend(fns2) except fdsn.EmptyResult: pass except HTTPError: logger.warn('an error occurred while downloading data ' 'for channels \n %s' % '\n '.join('.'.join(x[:4]) for x in selection_now)) f.close() i += neach if options.dry_run: nslcs = sorted(available_through.keys()) all_channels = defaultdict(set) all_stations = defaultdict(set) def plural_s(x): return '' if x == 1 else 's' for nslc in nslcs: sites = tuple(sorted(available_through[nslc])) logger.info('selected: %s.%s.%s.%s from site%s %s' % (nslc + (plural_s(len(sites)), '+'.join(sites)))) all_channels[sites].add(nslc) all_stations[sites].add(nslc[:3]) nchannels_all = 0 nstations_all = 0 for sites in sorted(all_channels.keys(), key=lambda sites: (-len(sites), sites)): nchannels = len(all_channels[sites]) nstations = len(all_stations[sites]) nchannels_all += nchannels nstations_all += nstations logger.info('selected (%s): %i channel%s (%i station%s)' % ('+'.join(sites), nchannels, plural_s(nchannels), nstations, plural_s(nstations))) logger.info('selected total: %i channel%s (%i station%s)' % (nchannels_all, plural_s(nchannels_all), nstations_all, plural_s(nstations_all))) logger.info('dry run done.') sys.exit(0) for nslc in have_data: # if we are in continue mode, we have to guess where the data came from if not any(nslc in have_data_site[site] for site in sites): for site in sites: if nslc in could_have_data_site[site]: have_data_site[site].add(nslc) sxs = {} for site in sites: selection = [] for nslc in sorted(have_data_site[site]): selection.append(nslc + (tmin - tpad, tmax + tpad)) if selection: logger.info('downloading response information (%s)' % site) sxs[site] = fdsn.station(site=site, level='response', selection=selection) sxs[site].dump_xml(filename=op.join(output_dir, 'stations.%s.xml' % site)) # chapter 1.5: inject local data if options.local_data: have_data_site['local'] = set() plocal = pile.make_pile(options.local_data, fileformat='detect') logger.info( 'Importing local data from %s between %s (%f) and %s (%f)' % (options.local_data, util.time_to_str(tmin), tmin, util.time_to_str(tmax), tmax)) for traces in plocal.chopper_grouped(gather=lambda tr: tr.nslc_id, tmin=tmin, tmax=tmax, tinc=tinc): for tr in traces: if tr.nslc_id not in have_data: fns.extend(io.save(traces, fn_template_raw)) have_data_site['local'].add(tr.nslc_id) have_data.add(tr.nslc_id) sites.append('local') if options.local_responses_pz: sxs['local'] = epz.make_stationxml( epz.iload(options.local_responses_pz)) if options.local_responses_resp: local_stations = [] for fn in options.local_stations: local_stations.extend(model.load_stations(fn)) sxs['local'] = resp.make_stationxml( local_stations, resp.iload(options.local_responses_resp)) if options.local_responses_stationxml: sxs['local'] = stationxml.load_xml( filename=options.local_responses_stationxml) # chapter 1.6: dump raw data stations file nsl_to_station = {} for site in sites: if site in sxs: stations = sxs[site].get_pyrocko_stations(timespan=(tmin, tmax)) for s in stations: nsl = s.nsl() if nsl not in nsl_to_station: nsl_to_station[nsl] = s stations = [nsl_to_station[nsl_] for nsl_ in sorted(nsl_to_station.keys())] util.ensuredirs(fn_stations_raw) model.dump_stations(stations, fn_stations_raw) dump_commandline(sys.argv, fn_commandline) # chapter 2: restitution if not fns: logger.error('no data available') sys.exit(1) p = pile.make_pile(fns, show_progress=False) p.get_deltatmin() otinc = None if otinc is None: otinc = nice_seconds_floor(p.get_deltatmin() * 500000.) otinc = 3600. otmin = math.floor(p.tmin / otinc) * otinc otmax = math.ceil(p.tmax / otinc) * otinc otpad = tpad * 2 fns = [] rest_traces_b = [] win_b = None for traces_a in p.chopper_grouped(gather=lambda tr: tr.nslc_id, tmin=otmin, tmax=otmax, tinc=otinc, tpad=otpad): rest_traces_a = [] win_a = None for tr in traces_a: win_a = tr.wmin, tr.wmax if win_b and win_b[0] >= win_a[0]: fns.extend(cut_n_dump(rest_traces_b, win_b, fn_template_rest)) rest_traces_b = [] win_b = None response = None failure = [] for site in sites: try: if site not in sxs: continue logger.debug('Getting response for %s' % tr.__str__()) response = sxs[site].get_pyrocko_response( tr.nslc_id, timespan=(tr.tmin, tr.tmax), fake_input_units=options.output_units) break except stationxml.NoResponseInformation: failure.append('%s: no response information' % site) except stationxml.MultipleResponseInformation: failure.append('%s: multiple response information' % site) if response is None: failure = ', '.join(failure) else: failure = '' try: if tr.tmin > tmin and options.zero_pad: logger.warning( 'Trace too short for clean restitution in ' 'desired frequency band -> zero-padding!') tr.extend(tr.tmin - tfade, tr.tmax + tfade, 'repeat') rest_tr = tr.transfer(tfade, ftap, response, invert=True) rest_traces_a.append(rest_tr) except (trace.TraceTooShort, trace.NoData): failure = 'trace too short' if failure: logger.warn('failed to restitute trace %s.%s.%s.%s (%s)' % (tr.nslc_id + (failure, ))) if rest_traces_b: rest_traces = trace.degapper(rest_traces_b + rest_traces_a, deoverlap='crossfade_cos') fns.extend(cut_n_dump(rest_traces, win_b, fn_template_rest)) rest_traces_a = [] if win_a: for tr in rest_traces: try: rest_traces_a.append( tr.chop(win_a[0], win_a[1] + otpad, inplace=False)) except trace.NoData: pass rest_traces_b = rest_traces_a win_b = win_a fns.extend(cut_n_dump(rest_traces_b, win_b, fn_template_rest)) # chapter 3: rotated restituted traces for inspection if not event: sys.exit(0) fn_template1 = \ 'DISPL.%(network)s.%(station)s.%(location)s.%(channel)s' fn_waveforms = op.join(output_dir, 'prepared', fn_template1) fn_stations = op.join(output_dir, 'stations.prepared.txt') fn_event = op.join(event_dir, 'event.txt') fn_event_yaml = op.join(event_dir, 'event.yaml') nsl_to_station = {} for site in sites: if site in sxs: stations = sxs[site].get_pyrocko_stations(timespan=(tmin, tmax)) for s in stations: nsl = s.nsl() if nsl not in nsl_to_station: nsl_to_station[nsl] = s p = pile.make_pile(fns, show_progress=False) deltat = None if sample_rate is not None: deltat = 1.0 / sample_rate traces_beat = [] used_stations = [] for nsl, s in nsl_to_station.items(): s.set_event_relative_data(event) traces = p.all(trace_selector=lambda tr: tr.nslc_id[:3] == nsl) if options.out_components == 'rtu': pios = s.guess_projections_to_rtu(out_channels=('R', 'T', 'Z')) elif options.out_components == 'enu': pios = s.guess_projections_to_enu(out_channels=('E', 'N', 'Z')) else: assert False for (proj, in_channels, out_channels) in pios: proc = trace.project(traces, proj, in_channels, out_channels) for tr in proc: tr_beat = heart.SeismicDataset.from_pyrocko_trace(tr) traces_beat.append(tr_beat) for ch in out_channels: if ch.name == tr.channel: s.add_channel(ch) if proc: io.save(proc, fn_waveforms) used_stations.append(s) stations = list(used_stations) util.ensuredirs(fn_stations) model.dump_stations(stations, fn_stations) model.dump_events([event], fn_event) from pyrocko.guts import dump dump([event], filename=fn_event_yaml) utility.dump_objects(op.join(cwd, 'seismic_data.pkl'), outlist=[stations, traces_beat]) logger.info('prepared waveforms from %i stations' % len(stations))
def command_cluster(args): ''' Execution of command cluster ''' def setup(parser): parser.add_option('--force', dest='force', action='store_true', help='overwrite existing project directory') parser.add_option('--view', dest='view', action='store_true', help='view similarity matrix after clustering') parser.add_option( '--savefig', dest='savefig', action='store_true', help='save figure of similarity matrix after clustering') parser, options, args = cl_parse('cluster', args, setup) if len(args) != 1: help_and_die(parser, 'missing argument') else: fn_config = args[0] if not os.path.isfile(fn_config): die('config file missing: %s' % fn_config) conf = load(filename=fn_config) config.check(conf) if not os.path.isdir(conf.project_dir): die('project directory missing: %s' % conf.project_dir) resdir = os.path.join(conf.project_dir, 'clustering_results') if not (options.force): if (os.path.isdir(resdir)): die('clustering result directory exists; use force option') if options.force: if (os.path.isdir(resdir)): shutil.rmtree(resdir) os.mkdir(resdir) simmat_temporal_fn = os.path.join(conf.project_dir, 'simmat_temporal.npy') if not os.path.isfile(simmat_temporal_fn): die('similarity matrix does not exists: %s; ' + 'use seiscloud matrix first' % simmat_temporal_fn) new_catalog_fn = os.path.join(conf.project_dir, 'events_to_be_clustered.pf') if not os.path.isfile(new_catalog_fn): die('catalog of selected events does not exists: %s; ' + 'use seiscloud matrix first' % new_catalog_fn) simmat_temp = sccluster.load_similarity_matrix(simmat_temporal_fn) events = model.load_events(new_catalog_fn) eventsclusters = sccluster.dbscan(simmat_temp, conf.dbscan_nmin, conf.dbscan_eps, conf.sw_force_cluster_all) clusters = sccluster.get_clusters(events, eventsclusters) if min(eventsclusters) == -1: noise_cluster_empty = False else: noise_cluster_empty = True sccluster.save_all(events, eventsclusters, clusters, conf, resdir) simmat_clus = sccluster.get_simmat_clustered(events, eventsclusters, clusters, conf, resdir, simmat_temp) simmat_clustered_fn = os.path.join(conf.project_dir, 'simmat_clustered.npy') sccluster.save_similarity_matrix(simmat_clus, simmat_clustered_fn) print('I run seiscloud for the project in "%s"' % conf.project_dir) n_clusters = len(clusters) if not noise_cluster_empty: n_clusters = n_clusters - 1 print(str(n_clusters) + ' cluster(s) found') simmat_fig_fn = os.path.join(conf.project_dir, 'simmat_clustered.' + conf.figure_format) if options.view and options.savefig: scplot.view_and_savefig_similarity_matrix(simmat_clus, simmat_fig_fn, 'Sorted after clustering') else: if options.view: scplot.view_similarity_matrix(simmat_clus, 'Sorted after clustering') if options.savefig: scplot.savefig_similarity_matrix(simmat_clus, simmat_fig_fn, 'Sorted after clustering') for cluster in clusters: selevents = clusters[cluster] median = sccluster.get_median(selevents, conf) fn_median = os.path.join(resdir, 'median_cluster' + str(cluster) + '.pf') model.dump_events([median], fn_median) # print(cluster,len(selevents)) print('Similarity matrix after clustering computed and stored as "%s"' % simmat_clustered_fn) if options.savefig: print('Similarity matrix figure saved as "%s"' % simmat_fig_fn)
from pyrocko import util, model from pyrocko.client import catalog tmin = util.str_to_time('2011-01-01 00:00:00') # beginning time of query tmax = util.str_to_time('2011-12-31 23:59:59') # create an instance of the global CMT catalog global_cmt_catalog = catalog.GlobalCMT() # query the catalog events = global_cmt_catalog.get_events(time_range=(tmin, tmax), magmin=2., latmin=-35., latmax=-20., lonmin=-76., lonmax=-65.) print('Downloaded %s events' % len(events)) print('The last one is:') print(events[-1]) # dump events to catalog model.dump_events(events, 'northern_chile_events.txt')
def main(): parser = OptionParser(usage=usage, description=description) parser.add_option( "--force", dest="force", action="store_true", default=False, help="allow recreation of output <directory>", ) parser.add_option( "--debug", dest="debug", action="store_true", default=False, help="print debugging information to stderr", ) parser.add_option( "--dry-run", dest="dry_run", action="store_true", default=False, help="show available stations/channels and exit " "(do not download waveforms)", ) parser.add_option( "--continue", dest="continue_", action="store_true", default=False, help="continue download after a accident", ) parser.add_option( "--local-data", dest="local_data", action="append", help="add file/directory with local data", ) parser.add_option( "--local-stations", dest="local_stations", action="append", help="add local stations file", ) parser.add_option( "--local-responses-resp", dest="local_responses_resp", action="append", help="add file/directory with local responses in RESP format", ) parser.add_option( "--local-responses-pz", dest="local_responses_pz", action="append", help="add file/directory with local pole-zero responses", ) parser.add_option( "--local-responses-stationxml", dest="local_responses_stationxml", help="add file with local response information in StationXML format", ) parser.add_option( "--window", dest="window", default="full", help='set time window to choose [full, p, "<time-start>,<time-end>"' "] (time format is YYYY-MM-DD HH:MM:SS)", ) parser.add_option( "--out-components", choices=["enu", "rtu"], dest="out_components", default="rtu", help="set output component orientations to radial-transverse-up [rtu] " "(default) or east-north-up [enu]", ) parser.add_option( "--padding-factor", type=float, default=3.0, dest="padding_factor", help="extend time window on either side, in multiples of 1/<fmin_hz> " "(default: 5)", ) parser.add_option( "--credentials", dest="user_credentials", action="append", default=[], metavar="SITE,USER,PASSWD", help="user credentials for specific site to access restricted data " "(this option can be repeated)", ) parser.add_option( "--token", dest="auth_tokens", metavar="SITE,FILENAME", action="append", default=[], help="user authentication token for specific site to access " "restricted data (this option can be repeated)", ) parser.add_option( "--sites", dest="sites", metavar="SITE1,SITE2,...", # default='bgr', default="http://ws.gpi.kit.edu,bgr,http://188.246.25.142:8080", help='sites to query (available: %s, default: "%%default"' % ", ".join(g_sites_available), ) parser.add_option( "--band-codes", dest="priority_band_code", metavar="V,L,M,B,H,S,E,...", default="V,L,M,B,H,E", help="select and prioritize band codes (default: %default)", ) parser.add_option( "--instrument-codes", dest="priority_instrument_code", metavar="H,L,G,...", default="H,L,O,", help="select and prioritize instrument codes (default: %default)", ) parser.add_option( "--radius-min", dest="radius_min", metavar="VALUE", default=0.0, type=float, help="minimum radius [km]", ) parser.add_option( "--tinc", dest="tinc", metavar="VALUE", default=3600.0 * 12.0, type=float, help="length of seperate saved files in s", ) parser.add_option( "--nstations-wanted", dest="nstations_wanted", metavar="N", type=int, help="number of stations to select initially", ) (options, args) = parser.parse_args(sys.argv[1:]) if len(args) not in (9, 6, 5): parser.print_help() sys.exit(1) if options.debug: util.setup_logging(program_name, "debug") else: util.setup_logging(program_name, "info") if options.local_responses_pz and options.local_responses_resp: logger.critical("cannot use local responses in PZ and RESP " "format at the same time") sys.exit(1) n_resp_opt = 0 for resp_opt in ( options.local_responses_pz, options.local_responses_resp, options.local_responses_stationxml, ): if resp_opt: n_resp_opt += 1 if n_resp_opt > 1: logger.critical("can only handle local responses from either PZ or " "RESP or StationXML. Cannot yet merge different " "response formats.") sys.exit(1) if options.local_responses_resp and not options.local_stations: logger.critical("--local-responses-resp can only be used " "when --stations is also given.") sys.exit(1) try: ename = "" magnitude = None mt = None if len(args) == 9: time = util.str_to_time(args[0] + " " + args[1]) lat = float(args[2]) lon = float(args[3]) depth = float(args[4]) * km iarg = 5 elif len(args) == 6: if args[1].find(":") == -1: sname_or_date = None lat = float(args[0]) lon = float(args[1]) event = None time = None else: sname_or_date = args[0] + " " + args[1] iarg = 2 elif len(args) == 5: sname_or_date = args[0] iarg = 1 if len(args) in (6, 5) and sname_or_date is not None: events = get_events_by_name_or_date([sname_or_date], catalog=geofon) if len(events) == 0: logger.critical("no event found") sys.exit(1) elif len(events) > 1: logger.critical("more than one event found") sys.exit(1) event = events[0] time = event.time lat = event.lat lon = event.lon depth = event.depth ename = event.name magnitude = event.magnitude mt = event.moment_tensor radius = float(args[iarg]) * km fmin = float(args[iarg + 1]) sample_rate = float(args[iarg + 2]) eventname = args[iarg + 3] event_dir = op.join("data", "events", eventname) output_dir = op.join(event_dir, "waveforms") except: raise parser.print_help() sys.exit(1) if options.force and op.isdir(event_dir): if not options.continue_: shutil.rmtree(event_dir) if op.exists(event_dir) and not options.continue_: logger.critical( 'directory "%s" exists. Delete it first or use the --force option' % event_dir) sys.exit(1) util.ensuredir(output_dir) if time is not None: event = model.Event( time=time, lat=lat, lon=lon, depth=depth, name=ename, magnitude=magnitude, moment_tensor=mt, ) if options.window == "full": if event is None: logger.critical("need event for --window=full") sys.exit(1) low_velocity = 1500.0 timewindow = VelocityWindow(low_velocity, tpad=options.padding_factor / fmin) tmin, tmax = timewindow(time, radius, depth) elif options.window == "p": if event is None: logger.critical("need event for --window=p") sys.exit(1) phases = list(map(cake.PhaseDef, "P p".split())) emod = cake.load_model() tpad = options.padding_factor / fmin timewindow = PhaseWindow(emod, phases, -tpad, tpad) arrivaltimes = [] for dist in num.linspace(0, radius, 20): try: arrivaltimes.extend(timewindow(time, dist, depth)) except NoArrival: pass if not arrivaltimes: logger.error("required phase arrival not found") sys.exit(1) tmin = min(arrivaltimes) tmax = max(arrivaltimes) else: try: stmin, stmax = options.window.split(",") tmin = util.str_to_time(stmin.strip()) tmax = util.str_to_time(stmax.strip()) timewindow = FixedWindow(tmin, tmax) except ValueError: logger.critical('invalid argument to --window: "%s"' % options.window) sys.exit(1) if event is not None: event.name = eventname tlen = tmax - tmin tfade = tfade_factor / fmin tpad = tfade tmin -= tpad tmax += tpad priority_band_code = options.priority_band_code.split(",") for s in priority_band_code: if len(s) != 1: logger.critical("invalid band code: %s" % s) priority_instrument_code = options.priority_instrument_code.split(",") for s in priority_instrument_code: if len(s) != 1: logger.critical("invalid instrument code: %s" % s) station_query_conf = dict( latitude=lat, longitude=lon, minradius=options.radius_min * km * cake.m2d, maxradius=radius * cake.m2d, channel=",".join("?%s?" % s for s in priority_band_code), ) target_sample_rate = sample_rate fmax = target_sample_rate # target_sample_rate = None # priority_band_code = ['H', 'B', 'M', 'L', 'V', 'E', 'S'] priority_units = ["M/S", "M", "M/S**2"] output_units = "M" sites = [x.strip() for x in options.sites.split(",") if x.strip()] tinc = options.tinc # for site in sites: # if site not in g_sites_available: # logger.critical('unknown FDSN site: %s' % site) # sys.exit(1) for s in options.user_credentials: try: site, user, passwd = s.split(",") g_user_credentials[site] = user, passwd except ValueError: logger.critical('invalid format for user credentials: "%s"' % s) sys.exit(1) for s in options.auth_tokens: try: site, token_filename = s.split(",") with open(token_filename, "r") as f: g_auth_tokens[site] = f.read() except (ValueError, OSError, IOError): logger.critical("cannot get token from file: %s" % token_filename) sys.exit(1) fn_template0 = ( "data_%(network)s.%(station)s.%(location)s.%(channel)s_%(tmin)s.mseed") fn_template_raw = op.join(output_dir, "raw", fn_template0) fn_template_raw_folder = op.join(output_dir, "raw/", "traces.mseed") fn_stations_raw = op.join(output_dir, "stations.raw.txt") fn_template_rest = op.join(output_dir, "rest", fn_template0) fn_commandline = op.join(output_dir, "seigerdown.command") ftap = (ffade_factors[0] * fmin, fmin, fmax, ffade_factors[1] * fmax) # chapter 1: download sxs = [] for site in sites: try: extra_args = { "iris": dict(matchtimeseries=True), }.get(site, {}) extra_args.update(station_query_conf) if site == "geonet": extra_args.update(starttime=tmin, endtime=tmax) else: extra_args.update( startbefore=tmax, endafter=tmin, includerestricted=(site in g_user_credentials or site in g_auth_tokens), ) logger.info("downloading channel information (%s)" % site) sx = fdsn.station(site=site, format="text", level="channel", **extra_args) except fdsn.EmptyResult: logger.error("No stations matching given criteria. (%s)" % site) sx = None if sx is not None: sxs.append(sx) if all(sx is None for sx in sxs) and not options.local_data: sys.exit(1) nsl_to_sites = defaultdict(list) nsl_to_station = {} for sx, site in zip(sxs, sites): site_stations = sx.get_pyrocko_stations() for s in site_stations: nsl = s.nsl() nsl_to_sites[nsl].append(site) if nsl not in nsl_to_station: nsl_to_station[nsl] = s # using first site with this station logger.info("number of stations found: %i" % len(nsl_to_station)) # station weeding nsls_selected = None if options.nstations_wanted: stations_all = [ nsl_to_station[nsl_] for nsl_ in sorted(nsl_to_station.keys()) ] for s in stations_all: s.set_event_relative_data(event) stations_selected = weeding.weed_stations(stations_all, options.nstations_wanted)[0] nsls_selected = set(s.nsl() for s in stations_selected) logger.info("number of stations selected: %i" % len(nsls_selected)) have_data = set() if options.continue_: fns = glob.glob(fn_template_raw % starfill()) p = pile.make_pile(fns) else: fns = [] have_data_site = {} could_have_data_site = {} for site in sites: have_data_site[site] = set() could_have_data_site[site] = set() available_through = defaultdict(set) it = 0 nt = int(math.ceil((tmax - tmin) / tinc)) for it in range(nt): tmin_win = tmin + it * tinc tmax_win = min(tmin + (it + 1) * tinc, tmax) logger.info("time window %i/%i (%s - %s)" % (it + 1, nt, util.tts(tmin_win), util.tts(tmax_win))) have_data_this_window = set() if options.continue_: trs_avail = p.all(tmin=tmin_win, tmax=tmax_win, load_data=False) for tr in trs_avail: have_data_this_window.add(tr.nslc_id) for site, sx in zip(sites, sxs): if sx is None: continue selection = [] channels = sx.choose_channels( target_sample_rate=target_sample_rate, priority_band_code=priority_band_code, priority_units=priority_units, priority_instrument_code=priority_instrument_code, timespan=(tmin_win, tmax_win), ) for nslc in sorted(channels.keys()): if nsls_selected is not None and nslc[:3] not in nsls_selected: continue could_have_data_site[site].add(nslc) if nslc not in have_data_this_window: channel = channels[nslc] if event: lat_, lon_ = event.lat, event.lon else: lat_, lon_ = lat, lon dist = orthodrome.distance_accurate50m_numpy( lat_, lon_, channel.latitude.value, channel.longitude.value) if event: depth_ = event.depth time_ = event.time else: depth_ = None time_ = None tmin_, tmax_ = timewindow(time_, dist, depth_) tmin_this = tmin_ - tpad tmax_this = tmax_ + tpad tmin_req = max(tmin_win, tmin_this) tmax_req = min(tmax_win, tmax_this) if channel.sample_rate: deltat = 1.0 / channel.sample_rate.value else: deltat = 1.0 if tmin_req < tmax_req: # extend time window by some samples because otherwise # sometimes gaps are produced selection.append(nslc + (tmin_req - deltat * 10.0, tmax_req + deltat * 10.0)) if options.dry_run: for (net, sta, loc, cha, tmin, tmax) in selection: available_through[net, sta, loc, cha].add(site) else: neach = 100 i = 0 nbatches = ((len(selection) - 1) // neach) + 1 while i < len(selection): selection_now = selection[i:i + neach] f = tempfile.NamedTemporaryFile() try: sbatch = "" if nbatches > 1: sbatch = " (batch %i/%i)" % ( (i // neach) + 1, nbatches) logger.info("downloading data (%s)%s" % (site, sbatch)) data = fdsn.dataselect(site=site, selection=selection_now, **get_user_credentials(site)) while True: buf = data.read(1024) if not buf: break f.write(buf) f.flush() trs = io.load(f.name) for tr in trs: if tr.station == "7869": tr.station = "MOER" tr.network = "LE" tr.location = "" try: tr.chop(tmin_win, tmax_win) have_data.add(tr.nslc_id) have_data_site[site].add(tr.nslc_id) except trace.NoData: pass fns2 = io.save(trs, fn_template_raw) io.save(trs, fn_template_raw_folder) for fn in fns2: if fn in fns: logger.warn("overwriting file %s", fn) fns.extend(fns2) except fdsn.EmptyResult: pass except HTTPError: logger.warn("an error occurred while downloading data " "for channels \n %s" % "\n ".join(".".join(x[:4]) for x in selection_now)) f.close() i += neach if options.dry_run: nslcs = sorted(available_through.keys()) all_channels = defaultdict(set) all_stations = defaultdict(set) def plural_s(x): return "" if x == 1 else "s" for nslc in nslcs: sites = tuple(sorted(available_through[nslc])) logger.info("selected: %s.%s.%s.%s from site%s %s" % (nslc + (plural_s(len(sites)), "+".join(sites)))) all_channels[sites].add(nslc) all_stations[sites].add(nslc[:3]) nchannels_all = 0 nstations_all = 0 for sites in sorted(all_channels.keys(), key=lambda sites: (-len(sites), sites)): nchannels = len(all_channels[sites]) nstations = len(all_stations[sites]) nchannels_all += nchannels nstations_all += nstations logger.info("selected (%s): %i channel%s (%i station%s)" % ( "+".join(sites), nchannels, plural_s(nchannels), nstations, plural_s(nstations), )) logger.info("selected total: %i channel%s (%i station%s)" % ( nchannels_all, plural_s(nchannels_all), nstations_all, plural_s(nstations_all), )) logger.info("dry run done.") sys.exit(0) for nslc in have_data: # if we are in continue mode, we have to guess where the data came from if not any(nslc in have_data_site[site] for site in sites): for site in sites: if nslc in could_have_data_site[site]: have_data_site[site].add(nslc) sxs = {} for site in sites: selection = [] for nslc in sorted(have_data_site[site]): selection.append(nslc + (tmin - tpad, tmax + tpad)) if selection: logger.info("downloading response information (%s)" % site) sxs[site] = fdsn.station(site=site, level="response", selection=selection) sited = site if site == "http://192.168.11.220:8080": sited = "bgr_internal" elif site == "http://ws.gpi.kit.edu": sited = "kit" if site == "http://188.246.25.142:8080": sited = "moer" sxs[site].dump_xml(filename=op.join(output_dir, "stations.%s.xml" % sited)) # chapter 1.5: inject local data if options.local_data: have_data_site["local"] = set() plocal = pile.make_pile(options.local_data, fileformat="detect") for traces in plocal.chopper_grouped(gather=lambda tr: tr.nslc_id, tmin=tmin, tmax=tmax, tinc=tinc): for tr in traces: if tr.station == "7869": tr.station = "MOER" tr.network = "LE" tr.location = "" if tr.nslc_id not in have_data: fns.extend(io.save(traces, fn_template_raw)) have_data_site["local"].add(tr.nslc_id) have_data.add(tr.nslc_id) sites.append("local") if options.local_responses_pz: sxs["local"] = epz.make_stationxml( epz.iload(options.local_responses_pz)) if options.local_responses_resp: local_stations = [] for fn in options.local_stations: local_stations.extend(model.load_stations(fn)) sxs["local"] = resp.make_stationxml( local_stations, resp.iload(options.local_responses_resp)) if options.local_responses_stationxml: sxs["local"] = stationxml.load_xml( filename=options.local_responses_stationxml) # chapter 1.6: dump raw data stations file nsl_to_station = {} for site in sites: if site in sxs: stations = sxs[site].get_pyrocko_stations(timespan=(tmin, tmax)) for s in stations: nsl = s.nsl() if nsl not in nsl_to_station: nsl_to_station[nsl] = s stations = [nsl_to_station[nsl_] for nsl_ in sorted(nsl_to_station.keys())] util.ensuredirs(fn_stations_raw) model.dump_stations(stations, fn_stations_raw) dump_commandline(sys.argv, fn_commandline) # chapter 2: restitution if not fns: logger.error("no data available") sys.exit(1) p = pile.make_pile(fns, show_progress=False) p.get_deltatmin() otinc = None if otinc is None: otinc = nice_seconds_floor(p.get_deltatmin() * 500000.0) otinc = 3600.0 otmin = math.floor(p.tmin / otinc) * otinc otmax = math.ceil(p.tmax / otinc) * otinc otpad = tpad * 2 fns = [] rest_traces_b = [] win_b = None for traces_a in p.chopper_grouped(gather=lambda tr: tr.nslc_id, tmin=otmin, tmax=otmax, tinc=otinc, tpad=otpad): rest_traces_a = [] win_a = None for tr in traces_a: if tr.station == "7869": tr.station = "MOER" tr.network = "LE" tr.location = "" win_a = tr.wmin, tr.wmax if win_b and win_b[0] >= win_a[0]: fns.extend(cut_n_dump(rest_traces_b, win_b, fn_template_rest)) rest_traces_b = [] win_b = None response = None failure = [] for site in sites: try: if site not in sxs: continue response = sxs[site].get_pyrocko_response( tr.nslc_id, timespan=(tr.tmin, tr.tmax), fake_input_units=output_units, ) break except stationxml.NoResponseInformation: failure.append("%s: no response information" % site) except stationxml.MultipleResponseInformation: failure.append("%s: multiple response information" % site) if response is None: failure = ", ".join(failure) else: failure = "" try: rest_tr = tr.transfer(tfade, ftap, response, invert=True) rest_traces_a.append(rest_tr) except (trace.TraceTooShort, trace.NoData): failure = "trace too short" if failure: logger.warn("failed to restitute trace %s.%s.%s.%s (%s)" % (tr.nslc_id + (failure, ))) if rest_traces_b: rest_traces = trace.degapper(rest_traces_b + rest_traces_a, deoverlap="crossfade_cos") fns.extend(cut_n_dump(rest_traces, win_b, fn_template_rest)) rest_traces_a = [] if win_a: for tr in rest_traces: if tr.station == "7869": tr.station = "MOER" tr.network = "LE" tr.location = "" try: rest_traces_a.append( tr.chop(win_a[0], win_a[1] + otpad, inplace=False)) except trace.NoData: pass rest_traces_b = rest_traces_a win_b = win_a fns.extend(cut_n_dump(rest_traces_b, win_b, fn_template_rest)) # chapter 3: rotated restituted traces for inspection if not event: sys.exit(0) fn_template1 = "DISPL.%(network)s.%(station)s.%(location)s.%(channel)s" fn_waveforms = op.join(output_dir, "prepared", fn_template1) fn_stations = op.join(output_dir, "stations.prepared.txt") fn_event = op.join(event_dir, "event.txt") nsl_to_station = {} for site in sites: if site in sxs: stations = sxs[site].get_pyrocko_stations(timespan=(tmin, tmax)) for s in stations: nsl = s.nsl() if nsl not in nsl_to_station: nsl_to_station[nsl] = s p = pile.make_pile(fns, show_progress=False) deltat = None if sample_rate is not None: deltat = 1.0 / sample_rate used_stations = [] for nsl, s in nsl_to_station.items(): s.set_event_relative_data(event) traces = p.all(trace_selector=lambda tr: tr.nslc_id[:3] == nsl) keep = [] for tr in traces: if deltat is not None: try: tr.downsample_to(deltat, snap=True, allow_upsample_max=5) keep.append(tr) except util.UnavailableDecimation as e: logger.warn("Cannot downsample %s.%s.%s.%s: %s" % (tr.nslc_id + (e, ))) continue if options.out_components == "rtu": pios = s.guess_projections_to_rtu(out_channels=("R", "T", "Z")) elif options.out_components == "enu": pios = s.guess_projections_to_enu(out_channels=("E", "N", "Z")) else: assert False for (proj, in_channels, out_channels) in pios: proc = trace.project(traces, proj, in_channels, out_channels) for tr in proc: for ch in out_channels: if ch.name == tr.channel: s.add_channel(ch) if proc: io.save(proc, fn_waveforms) used_stations.append(s) stations = list(used_stations) util.ensuredirs(fn_stations) model.dump_stations(stations, fn_stations) model.dump_events([event], fn_event) logger.info("prepared waveforms from %i stations" % len(stations))