def processpyrockoWaveforms(WaveformDict, Config, Folder, network, MetaDict, Event, switch, Xcorr): WaveformDict_obs = [] obspy_compat.plant() Logfile.red('Start Processing') cfg = FilterCfg(Config) new_frequence = cfg.newFrequency() # ['new_frequence'] traces = [] for tr in WaveformDict: Logfile.add( '%s:%s -------------------------------------------------------------' % (index, i)) if Config['export_unfiltered'].capitalize() == 'True': tr = obspy_compat.to_obspy_trace(tr) writeWaveform(Folder, i, tr, 'U', network) if switch == 0: Logfile.add('bandpass filtered stream for station %s ' % (i)) tr.bandpass(4, cfg.flo, cfg.fhi) elif switch == 1: Logfile.add('bandpass filtered stream for station %s ' % (i)) tr.bandpass(4, cfg.flo2, cfg.fhi2) tr.downsample_to(new_frequence) tr = obspy_compat.to_obspy_trace(tr) traces.append(tr) stream = obspy.Stream() Wdict = stream.extend([tr]) WaveformDict_obs.append(Wdict) return WaveformDict_obs
def filterpyrockoWaveform(self, Waveform): obspy_compat.plant() Logfile.red('Filter Waveform: ') cfg = FilterCfg(self.Config) new_frequence = (cfg.newFrequency()) st = Stream() for i in Waveform: tr = obspy_compat.to_pyrocko_trace(i) tr.downsample_to(new_frequence) Logfile.red('Downsampling to %d: from %d' % (new_frequence, i.stats.sampling_rate)) if switch == 0: Logfile.add('bandpass filtered stream for station %s ' % (i)) tr.bandpass(4, cfg.flo, cfg.fhi) elif switch == 1: Logfile.add('bandpass filtered stream for station %s ' % (i)) j = obspy_compat.to_obspy_trace(tr) st.append(j) return st
def filterWaveform(self, Waveform): Logfile.red('Filter Waveform: ') cfg = FilterCfg(self.Config) new_frequence = (cfg.newFrequency()) st = Stream() for i in Waveform: Logfile.red('Downsampling to %s: from %d' % (new_frequence, i.stats.sampling_rate)) j = i.resample(new_frequence) switch = cfg.filterswitch() if switch == 1: Logfile.add('bandpass filtered \ stream for station %s ' % (i)) j.filter('bandpass', freqmin=cfg.flo(), freqmax=cfg.fhi(), corners=cfg.ns(), zerophase=bool(self.Config['zph'])) elif switch == 2: Logfile.add('bandpass filtered \ stream for station %s ' % (i)) j.filter('bandpass', freqmin=cfg.flo2(), freqmax=cfg.fhi2(), corners=cfg.ns2(), zerophase=bool(self.Config['zph'])) st.append(j) return st
def processWaveforms(WaveformDict, Config, Folder, network, MetaDict, Event, Xcorr=None): Logfile.red('Start Processing') cfg = FilterCfg(Config) new_frequence = cfg.newFrequency() # ['new_frequence'] vp, vs, rho = ttt.calcak135parameter(Event) for index, i in enumerate(WaveformDict): Logfile.add( '%s:%s -------------------------------------------------------------' % (index, i)) if Config['export_unfiltered'].capitalize() == 'True': writeWaveform(Folder, i, WaveformDict[i], 'U', network) # TODO implement gain multiplication station = getStation(i, MetaDict) psign = 1 # print 'MetaDict ',MetaDict,station,station.takeoff #needs to be implemented correctly if cfg.Int('fm') == 1: print 'Event ', Event, Event.strike, Event.dip, Event.rake azi = ttt.bearing(Event, station) bazi = ttt.backazi(station, Event) psign = ttt.dubcup(rho, vp, vs, Event.strike, Event.dip, Event.rake, azi, station.takeoff) #print 'Takeoff ',station.takeoff,' Azi ',azi,' Bazi ',bazi,' vp ',vp,' vs ',vs,' rho ',rho,' Psign ',psign msg = 'Takeoff ' + str( station.takeoff) + ' Azi ' + str(azi) + ' Bazi ' + str(bazi) msg += (' vp ' + str(vp) + ' vs ' + str(vs) + ' rho ' + str(rho) + ' Psign ' + str(psign)) Logfile.add(msg) ''' if int(Config['xcorr']) == 1: #print Xcorr[i].value if Xcorr[i].value > 0: psign = 1 else: psign = -1 ''' Logfile.add(station.getName() + ' ' + station.lat + ' ' + station.lon + ' ' + station.gain + ' PSIGN: ' + str(psign)) if psign == -1: Logfile.add('correcting polarisation for station %s ' % (i)) -1. * WaveformDict[i][0].data #remove offset WaveformDict[i].detrend(type='demean') #gain correctur gain = float(station.gain) if gain == 0.0 or gain == -1.0: gain = 1 #hs : gain missing in metafile WaveformDict[i][0].data * (1.0 / gain) #filtering switch = cfg.filterswitch() # 'filterswitch' if switch == 1: Logfile.add('bandpass filtered stream for station %s ' % (i)) WaveformDict[i].filter( 'bandpass', freqmin=cfg.flo(), # ['flo'] freqmax=cfg.fhi(), # ['fhi'] corners=cfg.ns(), # ['ns'] zerophase=bool(Config['zph'])) elif switch == 2: Logfile.add('lowpass filtered stream for station %s ' % (i)) WaveformDict[i].filter( "lowpass", freq=cfg.l_fc(), # ['l_fc'] corners=cfg.l_ns(), # ['l_ns'] zerophase=bool(Config['l_zph'])) elif switch == 3: Logfile.add('highpass filtered stream for station %s ' % (i)) WaveformDict[i].filter( "highpass", freq=cfg.h_fc(), # ['h_fc'] corners=cfg.h_ns(), # ['h_ns'] zerophase=bool(Config['h_zph'])) else: Logfile.add('no filter set for station %s ' % (i)) if Config['export_filtered'].capitalize() == 'True': writeWaveform(Folder, i, WaveformDict[i], 'F', network) #resampling j = resampleWaveform(WaveformDict[i][0], new_frequence) WaveformDict[i] = j #Logfile.add (WaveformDict[i]) if Config['export_resampled'].capitalize() == 'True': writeWaveform(Folder, i, WaveformDict[i], 'R', network) #endfor return WaveformDict
def processLoop(): #==================================get meta info========================================== C = config.Config(evpath) Origin = C.parseConfig('origin') try: Syn_in = C.parseConfig('syn') syn_in = SynthCfg(Syn_in) except: pass Config = C.parseConfig('config') cfg = ConfigObj(dict=Config) if cfg.pyrocko_download() == True: Meta = C.readpyrockostations() # elif cfg.colesseo_input() == True: scenario = guts.load(filename=cfg.colosseo_scenario_yml()) scenario_path = cfg.colosseo_scenario_yml()[:-12] Meta = C.readcolosseostations(scenario_path) else: Meta = C.readMetaInfoFile() #==================================get meta info========================================== #==================================do prerequiries======================================== Folder = C.createFolder() #C.cpSkeleton(Folder,Config) C.writeConfig(Config, Origin, Folder) filter = FilterCfg(Config) ntimes = int( (cfg.UInt('forerun') + cfg.UInt('duration')) / cfg.UInt('step')) origin = OriginCfg(Origin) if cfg.colesseo_input() == True: from pyrocko import util events = scenario.get_events() ev = events[0] origin.strike = str(ev.moment_tensor.strike1) origin.rake = str(ev.moment_tensor.rake1) origin.dip = str(ev.moment_tensor.dip1) strike = ev.moment_tensor.strike1 origin.lat = str(ev.lat) origin.lon = str(ev.lon) origin.depth = str(ev.depth / 1000.) depth = ev.depth origin.time = util.time_to_str(ev.time) time_ev = util.time_to_str(ev.time) lat = ev.lat lon = ev.lon rake = ev.moment_tensor.rake1 dip = ev.moment_tensor.dip1 Origin['strike'] = str(ev.moment_tensor.strike1) Origin['rake'] = str(ev.moment_tensor.rake1) Origin['dip'] = str(ev.moment_tensor.dip1) Origin['lat'] = str(ev.lat) Origin['lon'] = str(ev.lon) Origin['time'] = util.time_to_str(ev.time) Origin['depth'] = str(ev.depth / 1000.) ev = Event(lat, lon, depth, time_ev, strike=strike, dip=dip, rake=rake) else: default = 0 strike = origin.strike(default) # Origin.get ('strike', default) dip = origin.dip(default) # Origin.get ('dip', default) rake = origin.rake(default) # Origin.get ('rake', default) ev = Event(origin.lat(), origin.lon(), origin.depth(), origin.time(), strike=strike, dip=dip, rake=rake) filtername = filter.filterName() Logfile.add('filtername = ' + filtername) #todo crosscorrelation for all arrays before processing XDict = {} RefDict = {} SL = {} if cfg.Int('xcorr') == 1: newFreq = str(filter.newFrequency()) fobjreferenceshiftname = newFreq + '_' + filtername + '.refpkl' rp = os.path.join(Folder['semb'], fobjreferenceshiftname) fobjpickleshiftname = newFreq + '_' + filtername + '.xcorrpkl' ps = os.path.join(Folder['semb'], fobjpickleshiftname) if (os.path.isfile(rp) and os.path.getsize(rp) != 0 and os.path.isfile(ps) and os.path.getsize(ps) != 0): Logfile.add('file exits : ' + rp) Logfile.add('load refshifts') f = open(rp) RefDict = pickle.load(f) x = open(ps) XDict = pickle.load(x) xcorrnetworks = cfg.String('networks').split(',') for i in xcorrnetworks: SL[i] = len(Config[i].split('|')) else: SL = {} xcorrnetworks = cfg.String('networks').split(',') for i in xcorrnetworks: W = {} refshift = 0 network = cfg.String(i).split('|') FilterMeta = ttt.filterStations(Meta, Config, Origin, network) arrayfolder = os.path.join(Folder['semb'], i) if os.access(arrayfolder, os.F_OK) == False: os.makedirs(arrayfolder) if cfg.pyrocko_download() == True: A = Xcorr(ev, FilterMeta, evpath, Config, Syn_in, arrayfolder) else: A = Xcorr(ev, FilterMeta, evpath, Config, Syn_in, arrayfolder) print "run Xcorr" W, triggerobject = A.runXcorr() XDict[i] = W RefDict[i] = triggerobject.tdiff SL[i] = len(network) #endfor fobjrefshift = open(rp, 'w') pickle.dump(RefDict, fobjrefshift) fobjrefshift.close() output = open(ps, 'w') pickle.dump(XDict, output) output.close() for i in sorted(XDict.iterkeys()): Logfile.red('Array %s has %3d of %3d Stations left' % (i, len(XDict[i]), SL[i])) logger.info( '\033[31mFor proceeding without changes press enter or give new comma seperatet network list or quit for exit\033[0m' ) while True: nnl = raw_input("please enter your choice: ") #Logfile.add ('Choise = ' + nnl) if len(nnl) == 0: if not Basic.question('Process all networks ?'): continue Logfile.red('This networks will be used for processing: %s' % (Config['networks'])) break elif str(nnl) == 'quit': sys.exit() elif str(nnl) == 'rerun': event = os.path.join(*evpath.split('/')[-1:]) try: os.remove(rp) os.remove(ps) except: pass mainfolder = os.path.join(os.path.sep, *evpath.split('/')[:-2]) os.chdir(mainfolder) cmd = ('%s arraytool.py process %s') % (sys.executable, event) Logfile.add('cmd = ' + cmd) os.system(cmd) sys.exit() else: # Check if selected array(s) exists names = nnl.split(',') isOk = True for array in names: arrayfolder = os.path.join(Folder['semb'], array) if not os.path.isdir(arrayfolder): Logfile.error('Illegal network name ' + str(array)) isOk = False break #endfor if not isOk: continue # Illegal network : input again # use these networks Logfile.add('This networks will be used for processing: %s' % (nnl)) Config['networks'] = nnl break for i in range(3, 0, -1): time.sleep(1) Logfile.red('Start processing in %d seconds ' % (i)) wd = Origin['depth'] start, stop, step = cfg.String('depths').split(',') start = int(start) stop = int(stop) + 1 step = int(step) filters = cfg.String('filters') filters = int(filters) Logfile.add('working on ' + Config['networks']) #==================================loop over depth====================== for filterindex in xrange(0, filters): for depthindex in xrange(start, stop, step): workdepth = float(wd) + depthindex Origin['depth'] = workdepth ev = Event(Origin['lat'], Origin['lon'], Origin['depth'], Origin['time'], strike=strike, dip=dip, rake=rake) Logfile.add('WORKDEPTH: ' + str(Origin['depth'])) #==================================do prerequiries=============== #==================================loop over arrays================ ASL = [] weights = [] array_centers = [] networks = Config['networks'].split(',') counter = 1 TriggerOnset = [] Wdfs = [] for i in networks: arrayname = i arrayfolder = os.path.join(Folder['semb'], arrayname) network = Config[i].split('|') Logfile.add('network: ' + str(network)) FilterMeta = ttt.filterStations(Meta, Config, Origin, network) #if len(FilterMeta) < 3: continue #hs : wieder rein if len(FilterMeta) < 3: continue W = XDict[i] refshift = RefDict[i] FilterMeta = cmpFilterMetavsXCORR(W, FilterMeta) Logfile.add( 'BOUNDING BOX DIMX: %s DIMY: %s GRIDSPACING: %s \n' % (Config['dimx'], Config['dimy'], Config['gridspacing'])) ##############=======================PARALLEL=========================================== Logfile.red('Calculating Traveltime Grid') t1 = time.time() isParallel = False #10.12.2015 TTTGridMap = [] mint = [] maxt = [] try: f = open( '../tttgrid/tttgrid_%s_%s_%s.pkl' % (ev.time, arrayname, workdepth), 'rb') print "loading travel time grid_%s_%s_%s.pkl" % ( ev.time, arrayname, workdepth) TTTGridMap, mint, maxt = pickle.load(f) f.close() print "loading of travel time grid sucessful" except: print "loading of travel time grid unsucessful, will now calculate the grid:" if isParallel: #hs # maxp = int (Config['ncore']) maxp = 6 #hs po = multiprocessing.Pool(maxp) for i in xrange(len(FilterMeta)): po.apply_async(ttt.calcTTTAdv, (Config, FilterMeta[i], Origin, i, arrayname, W, refshift)) po.close() po.join() else: #hs+ for i in xrange(len(FilterMeta)): t1 = time.time() ttt.calcTTTAdv(Config, FilterMeta[i], Origin, i, arrayname, W, refshift) Logfile.add('ttt.calcTTTAdv : ' + str(time.time() - t1) + ' sec.') #endif #hs- assert len(FilterMeta) > 0 TTTGridMap = deserializer.deserializeTTT(len(FilterMeta)) mint, maxt = deserializer.deserializeMinTMaxT( len(FilterMeta)) f = open( '../tttgrid/tttgrid_%s_%s_%s.pkl' % (ev.time, arrayname, workdepth), 'wb') print "dumping the traveltime grid for this array" pickle.dump([TTTGridMap, mint, maxt], f) f.close() t2 = time.time() Logfile.red('%s took %0.3f s' % ('TTT', (t2 - t1))) switch = filterindex tw = times.calculateTimeWindows(mint, maxt, Config, ev, switch) if cfg.pyrocko_download() == True: if cfg.quantity() == 'displacement': Wd = waveform.readWaveformsPyrocko_restituted( FilterMeta, tw, evpath, ev) else: Wd = waveform.readWaveformsPyrocko( FilterMeta, tw, evpath, ev) # Wdf = waveform.processpyrockoWaveforms(Wd, Config, Folder, arrayname, FilterMeta, ev, switch, W) elif cfg.colesseo_input() == True: Wd = waveform.readWaveforms_colesseo( FilterMeta, tw, evpath, ev, C) else: Wd = waveform.readWaveforms(FilterMeta, tw, evpath, ev) if cfg.Bool('synthetic_test') is True: Wdf = waveform.processdummyWaveforms( Wd, Config, Folder, arrayname, FilterMeta, ev, switch, W) Wdfs.append(Wdf) else: Wdf = waveform.processWaveforms(Wd, Config, Folder, arrayname, FilterMeta, ev, switch, W) Wdfs.append(Wdf) C.writeStationFile(FilterMeta, Folder, counter) Logfile.red('%d Streams added for Processing' % (len(Wd))) t1 = time.time() f = open( '../tttgrid/tttgrid_%s_%s_%s.pkl' % (ev.time, arrayname, workdepth), 'rb') print "loading travel time grid_%s_%s_%s.pkl" % ( ev.time, arrayname, workdepth) TTTGridMap, mint, maxt = pickle.load(f) f.close() if cfg.optimize() == True: optim.solve(counter, Config, Wdf, FilterMeta, mint, maxt, TTTGridMap, Folder, Origin, ntimes, switch, ev, arrayfolder, syn_in) else: arraySemb, weight, array_center = sembCalc.doCalc( counter, Config, Wdf, FilterMeta, mint, maxt, TTTGridMap, Folder, Origin, ntimes, switch, ev, arrayfolder, syn_in) t2 = time.time() Logfile.add('CALC took %0.3f sec' % (t2 - t1)) weights.append(weight) array_centers.append(array_center) ASL.append(arraySemb) counter += 1 sembCalc.writeSembMatricesSingleArray(arraySemb, Config, Origin, arrayfolder, ntimes, switch) fileName = os.path.join(arrayfolder, 'stations.txt') Logfile.add('Write to file ' + fileName) fobjarraynetwork = open(fileName, 'w') for i in FilterMeta: fobjarraynetwork.write( ('%s %s %s\n') % (i.getName(), i.lat, i.lon)) fobjarraynetwork.close() TTTGridMAP = [] if cfg.optimize_all() == True: import optim_csemb from optim_csemb import solve sembmax = sembCalc.collectSemb(ASL, Config, Origin, Folder, ntimes, len(networks), switch) optim_csemb.solve(counter, Config, Wdf, FilterMeta, mint, maxt, TTTGridMap, Folder, Origin, ntimes, switch, ev, arrayfolder, syn_in, ASL, sembmax, evpath, XDict, RefDict, workdepth, filterindex, Wdfs) if ASL: Logfile.red('collect semblance matrices from all arrays') sembmax = sembCalc.collectSemb(ASL, Config, Origin, Folder, ntimes, len(networks), switch, array_centers) if cfg.Bool('weight_by_noise') == True: sembCalc.collectSembweighted(ASL, Config, Origin, Folder, ntimes, len(networks), switch, weights) else: Logfile.red('Nothing to do -> Finish') print "depth:" print workdepth
def processWaveforms(WaveformDict, Config, Folder, network, MetaDict, Event, switch, Xcorr): Logfile.red('Start Processing') cfg = FilterCfg(Config) new_frequence = cfg.newFrequency() for index, i in enumerate(WaveformDict): Logfile.add('%s:%s ---------------------' % (index, i)) if Config['export_unfiltered'].capitalize() is 'True': writeWaveform(Folder, i, WaveformDict[i], 'U', network) station = getStation(i, MetaDict) if cfg.Int('fm') == 1: azi = ttt.bearing(Event, station) bazi = ttt.backazi(station, Event) msg = 'Takeoff ' + str(station.takeoff) + ' Azi ' + str(azi) +\ 'Bazi ' + str(bazi) Logfile.add(msg) gain = float(station.gain) if gain == 0.0 or gain == -1.0: gain = 1 WaveformDict[i][0].data * (1.0 / gain) if switch is 0: Logfile.add('bandpass filtered stream for station %s ' % (i)) WaveformDict[i].filter('bandpass', freqmin=cfg.flo(), freqmax=cfg.fhi(), corners=cfg.ns(), zerophase=bool(Config['zph'])) elif switch is 1: Logfile.add('bandpass filtered stream for station %s ' % (i)) WaveformDict[i].filter('bandpass', freqmin=cfg.flo2(), freqmax=cfg.fhi2(), corners=cfg.ns2(), zerophase=bool(Config['zph2'])) else: Logfile.add('no filter set for station %s ' % (i)) if Config['export_filtered'].capitalize() is 'True': writeWaveform(Folder, i, WaveformDict[i], 'F', network) j = resampleWaveform(WaveformDict[i][0], new_frequence) WaveformDict[i] = j if Config['export_resampled'].capitalize() is 'True': writeWaveform(Folder, i, WaveformDict[i], 'R', network) return WaveformDict
options,args = main(sys.argv) Basic.checkExistsDir(options.eventpath, isAbort=True) Globals.setEventDir(options.eventpath) C = config.Config(options.eventpath) Origin = C.parseConfig('origin') Conf = globalConf() Config = C.parseConfig('config') filter = FilterCfg(Config) cfg = ConfigObj(dict=Config) minDist, maxDist = cfg.FloatRange('mindist', 'maxdist') ev = Event(Origin['lat'],Origin['lon'],Origin['depth'],Origin['time'] ) event = model.Event(lat=float(ev.lat), lon=float(ev.lon), depth=float(ev.depth)*1000., time=util.str_to_time(ev.time)) newFreq = float(filter.newFrequency()) options.time = Origin ['time'] options.duration = int(Conf['duration']) sdspath = os.path.join(options.eventpath,'data') model.dump_events([event], sdspath+'event.pf') tmin = util.str_to_time(ev.time)-600. tmax = util.str_to_time(ev.time)+1800. def get_stations(site, lat, lon, rmin, rmax, tmin, tmax, channel_pattern='BH*'): extra = {} if site == 'iris': extra.update(matchtimeseries=True) sx = fdsn.station( site=site, latitude=lat, longitude=lon,
def filterWaveform(self, Waveform): Logfile.red('Filter Waveform: ') cfg = FilterCfg(self.Config) #new_frequence = int (self.Config['new_frequence']) new_frequence = int(cfg.newFrequency()) st = Stream() for i in Waveform: Logfile.red('Downsampling to %d: from %d' % (new_frequence, i.stats.sampling_rate)) j = self.resampleWaveform(i, new_frequence) j.detrend(type='demean') old = True #??? if old: switch = cfg.filterswitch() # ['filterswitch'] if switch == 1: Logfile.add('bandpass filtered stream for station %s ' % (i)) j.filter( 'bandpass', freqmin=cfg.flo(), # ['flo'] freqmax=cfg.fhi(), # ['fhi'] corners=cfg.ns(), # ['ns'] zerophase=bool(self.Config['zph'])) elif switch == 2: Logfile.add('lowpass filtered stream for station %s ' % (i)) j.filter( "lowpass", freq=cfg.l_fc(), # ['l_fc'] corners=cfg.l_ns(), # ['l_ns'] zerophase=bool(self.Config['l_zph'])) elif switch == 3: Logfile.add('highpass filtered stream for station %s ' % (i)) j.filter( "highpass", freq=cfg.h_fc(), # ['h_fc'] corners=cfg.h_ns(), # ['h_ns'] zerophase=bool(self.Config['h_zph'])) else: dummy = 1 #Logfile.add ('bandpass filtered stream for station %s '% (i)) #j.filter ("bandpass", freqmin=0.4, freqmax=3, # corners=3, zerophase=False) st.append(j) else: j1 = filterWaveform_2(Config, j, i) st.append(j1) #endfor return st
def processLoop(): C = config.Config(evpath) Origin = C.parseConfig('origin') flag_rpe = False try: Syn_in = C.parseConfig('syn') syn_in = SynthCfg(Syn_in) except TypeError: pass Config = C.parseConfig('config') cfg = ConfigObj(dict=Config) phases = cfg.Str('ttphases') phases = phases.split(',') if cfg.pyrocko_download() is True: Meta = C.readpyrockostations() elif cfg.colesseo_input() is True: scenario = guts.load(filename=cfg.colosseo_scenario_yml()) scenario_path = cfg.colosseo_scenario_yml()[:-12] Meta = C.readcolosseostations(scenario_path) else: Meta = C.readMetaInfoFile() Folder = C.createFolder() C.writeConfig(Config, Origin, Folder) filter = FilterCfg(Config) if cfg.UInt('forerun') > 0: ntimes = int( (cfg.UInt('forerun') + cfg.UInt('duration')) / cfg.UInt('step')) else: ntimes = int((cfg.UInt('duration')) / cfg.UInt('step')) origin = OriginCfg(Origin) if cfg.colesseo_input() is True: from pyrocko import util events = scenario.get_events() ev = events[0] origin.strike = str(ev.moment_tensor.strike1) origin.rake = str(ev.moment_tensor.rake1) origin.dip = str(ev.moment_tensor.dip1) strike = ev.moment_tensor.strike1 origin.lat = str(ev.lat) origin.lon = str(ev.lon) origin.depth = str(ev.depth / 1000.) depth = ev.depth origin.time = util.time_to_str(ev.time) time_ev = util.time_to_str(ev.time) lat = ev.lat lon = ev.lon rake = ev.moment_tensor.rake1 dip = ev.moment_tensor.dip1 Origin['strike'] = str(ev.moment_tensor.strike1) Origin['rake'] = str(ev.moment_tensor.rake1) Origin['dip'] = str(ev.moment_tensor.dip1) Origin['lat'] = str(ev.lat) Origin['lon'] = str(ev.lon) Origin['time'] = util.time_to_str(ev.time) Origin['depth'] = str(ev.depth / 1000.) ev = Event(lat, lon, depth, time_ev, strike=strike, dip=dip, rake=rake) else: default = 0 strike = origin.strike(default) dip = origin.dip(default) rake = origin.rake(default) ev = Event(origin.lat(), origin.lon(), origin.depth(), origin.time(), strike=strike, dip=dip, rake=rake) if cfg.Bool('correct_shifts_empirical') is True: Origin_emp = C.parseConfig('origin_emp') origin_emp = OriginCfg(Origin_emp) ev_emp = Event(origin_emp.lat(), origin_emp.lon(), origin_emp.depth(), origin_emp.time(), strike=strike, dip=dip, rake=rake) filtername = filter.filterName() Logfile.add('filtername = ' + filtername) XDict = OrderedDict() RefDict = OrderedDict() SL = OrderedDict() refshifts_global = [] newFreq = str(filter.newFrequency()) xcorrnetworks = cfg.String('networks').split(',') if cfg.Int('xcorr') is 1: fobjreferenceshiftname = newFreq + '_' + filtername + '.refpkl' rp = os.path.join(Folder['semb'], fobjreferenceshiftname) fobjreferenceshiftnameemp = newFreq + '_' + filtername + 'emp' + '.refpkl' rpe = os.path.join(Folder['semb'], fobjreferenceshiftnameemp) fobjpickleshiftname = newFreq + '_' + filtername + '.xcorrpkl' ps = os.path.join(Folder['semb'], fobjpickleshiftname) if (os.path.isfile(rp) and os.path.getsize(rp) != 0 and os.path.isfile(ps) and os.path.getsize(ps) != 0): Logfile.add('xcorr/reference shift file exits : ' + rp) Logfile.add('loaded reference shift') if sys.version_info.major >= 3: f = open(rp, 'rb') else: f = open(rp) RefDict = pickle.load(f) if sys.version_info.major >= 3: x = open(ps, 'rb') else: x = open(ps) XDict = pickle.load(x) for i in xcorrnetworks: SL[i] = len(Config[i].split('|')) else: SL = {} for i in xcorrnetworks: W = {} network = cfg.String(i).split('|') FilterMeta = ttt.filterStations(Meta, Config, Origin, network) arrayfolder = os.path.join(Folder['semb'], i) if os.access(arrayfolder, os.F_OK) is False: os.makedirs(arrayfolder) if cfg.pyrocko_download() is True: # TODO check seperate xcoor nescessity A = Xcorr(ev, FilterMeta, evpath, Config, Syn_in, arrayfolder) print("run Xcorr") phase = phases[0] W, triggerobject = A.runXcorr(phase) XDict[i] = W RefDict[i] = triggerobject.tdiff SL[i] = len(network) for j in range(0, len(FilterMeta)): refshifts_global.append(triggerobject.tdiff) if sys.version_info.major >= 3: fobjrefshift = open(rp, 'wb') else: fobjrefshift = open(rp, 'w') pickle.dump(RefDict, fobjrefshift) fobjrefshift.close() if sys.version_info.major >= 3: output = open(ps, 'wb') else: output = open(ps, 'w') pickle.dump(XDict, output) output.close() else: fobjreferenceshiftname = newFreq + '_' + filtername + '.refpkl' rp = os.path.join(Folder['semb'], fobjreferenceshiftname) fobjreferenceshiftnameemp = newFreq + '_' + filtername + 'emp' + '.refpkl' rpe = os.path.join(Folder['semb'], fobjreferenceshiftnameemp) fobjpickleshiftname = newFreq + '_' + filtername + '.xcorrpkl' ps = os.path.join(Folder['semb'], fobjpickleshiftname) refshift = 0 if (os.path.isfile(rp) and os.path.getsize(rp) != 0 and os.path.isfile(ps) and os.path.getsize(ps) != 0): Logfile.add('Temporay Memory file exits : ' + rp) if sys.version_info.major >= 3: f = open(rp, 'rb') else: f = open(rp) RefDict = pickle.load(f) if sys.version_info.major >= 3: x = open(ps, 'rb') else: x = open(ps) XDict = pickle.load(x) for i in xcorrnetworks: SL[i] = len(Config[j].split('|')) network = cfg.String(i).split('|') FilterMeta = ttt.filterStations(Meta, Config, Origin, network) RefDict[i] = refshift for j in range(0, len(FilterMeta)): refshifts_global.append(refshift) else: SL = {} for i in xcorrnetworks: W = {} refshift = 0 network = cfg.String(i).split('|') FilterMeta = ttt.filterStations(Meta, Config, Origin, network) arrayfolder = os.path.join(Folder['semb'], i) if os.access(arrayfolder, os.F_OK) is False: os.makedirs(arrayfolder) if cfg.pyrocko_download() is True: # TODO check seperate xcoor nescessity A = Xcorr(ev, FilterMeta, evpath, Config, Syn_in, arrayfolder) else: A = Xcorr(ev, FilterMeta, evpath, Config, Syn_in, arrayfolder) print("run Xcorr") phase = phases[0] W, triggerobject = A.runXcorr_dummy(phase) XDict[j] = W RefDict[j] = refshift SL[j] = len(network) for j in range(0, len(FilterMeta)): refshifts_global.append(refshift) if sys.version_info.major >= 3: fobjrefshift = open(rp, 'wb') else: fobjrefshift = open(rp, 'w') pickle.dump(RefDict, fobjrefshift) fobjrefshift.close() if sys.version_info.major >= 3: output = open(ps, 'wb') else: output = open(ps, 'w') pickle.dump(XDict, output) output.close() if sys.version_info.major >= 3: for j in sorted(XDict.keys()): Logfile.red('Array %s has %3d of %3d Stations left' % (j, len(XDict[j]), SL[j])) else: for j in sorted(XDict.keys()): Logfile.red('Array %s has %3d of %3d Stations left' % (j, len(XDict[j]), SL[j])) while True: if sys.version_info.major >= 3: nnl = input("please enter your choice: ") else: nnl = raw_input("please enter your choice: ") if len(nnl) == 0: if not Basic.question('Process all networks ?'): continue Logfile.red('This networks will be used for processing: %s' % (Config['networks'])) break elif str(nnl) == 'quit': sys.exit() elif str(nnl) == 'rerun': event = os.path.join(*evpath.split('/')[-1:]) try: os.remove(rp) os.remove(ps) except Exception: pass mainfolder = os.path.join(os.path.sep, *evpath.split('/')[:-2]) os.chdir(mainfolder) cmd = ('%s arraytool.py process %s') % (sys.executable, event) Logfile.add('cmd = ' + cmd) os.system(cmd) sys.exit() else: names = nnl.split(',') isOk = True for array in names: arrayfolder = os.path.join(Folder['semb'], array) if not os.path.isdir(arrayfolder): Logfile.error('Illegal network name ' + str(array)) isOk = False break if not isOk: continue Logfile.add('This networks will be used for processing: %s' % (nnl)) Config['networks'] = nnl break for j in range(3, 0, -1): time.sleep(1) Logfile.red('Start processing in %d seconds ' % (j)) wd = Origin['depth'] start, stop, step = cfg.String('depths').split(',') start = int(start) stop = int(stop) + 1 step_depth = int(step) filters = cfg.String('filters') filters = int(filters) Logfile.add('working on ' + Config['networks']) if cfg.Bool('correct_shifts_empirical') is True: emp_loop = True else: emp_loop = False # ==================================loop over phases====================== for phase in phases: if phase is 'P': desired = 'Z' if phase is 'S': desired = 'T' # ==================================loop over filter setups===== for filterindex in xrange(0, filters): # ==================================loop over depth======= for depthindex in xrange(start, stop, step_depth): workdepth = float(wd) + depthindex Origin['depth'] = workdepth ev = Event(Origin['lat'], Origin['lon'], Origin['depth'], Origin['time'], strike=strike, dip=dip, rake=rake) Logfile.add('WORKDEPTH: ' + str(Origin['depth'])) networks = Config['networks'].split(',') ASL = [] weights = [] array_centers = [] counter = 1 stations_per_array = [] Wdfs = [] Wdfs_emp = [] FilterMetas = [] TTTgrids = OrderedDict() mints = [] maxts = [] refshifts = [] for i in networks: arrayname = i arrayfolder = os.path.join(Folder['semb'], arrayname) network = Config[i].split('|') Logfile.add('network: ' + str(network)) FilterMeta = ttt.filterStations(Meta, Config, Origin, network) W = XDict[i] refshift = RefDict[i] for j in range(0, len(FilterMeta)): if cfg.correct_shifts() is False: refshift = refshift * 0. refshifts.append(refshift) FilterMeta = cmpFilterMetavsXCORR(W, FilterMeta) Logfile.add('BOUNDING BOX DIMX: %s DIMY: %s GRIDSPACING:\ %s \n' % (Config['dimx'], Config['dimy'], Config['gridspacing'])) Logfile.red('Calculating Traveltime Grid') t1 = time.time() isParallel = False TTTGridMap = [] mint = [] maxt = [] ttt_model = cfg.Str('traveltime_model') try: if cfg.Bool('correct_shifts_empirical') is True: f = open( '../tttgrid/tttgrid%s_%s_%s_%s_%s_emp.pkl' % (phase, ttt_model, ev_emp.time, arrayname, workdepth), 'rb') print( "loading travel time grid%s_%s_%s_%s_%s_emp.pkl" % (phase, ttt_model, ev_emp.time, arrayname, workdepth)) TTTGridMap_emp, mint_emp, maxt_emp = pickle.load(f) f.close() f = open( '../tttgrid/tttgrid%s_%s_%s_%s_%s.pkl' % (phase, ttt_model, ev.time, arrayname, workdepth), 'rb') print( "loading travel time grid%s_%s_%s_%s_%s.pkl" % (phase, ttt_model, ev.time, arrayname, workdepth)) TTTGridMap, mint, maxt = pickle.load(f) f.close() print("loading of travel time grid sucessful") except Exception: print("loading of travel time grid unsucessful,\n \ will now calculate the grid:") if isParallel: maxp = 6 po = multiprocessing.Pool(maxp) for i in xrange(len(FilterMeta)): po.apply_async(ttt.calcTTTAdv, (Config, FilterMeta[i], Origin, i, arrayname, W, refshift)) po.close() po.join() else: for i in xrange(len(FilterMeta)): t1 = time.time() ttt.calcTTTAdv(Config, FilterMeta[i], Origin, i, arrayname, W, refshift, phase) Logfile.add('ttt.calcTTTAdv : ' + str(time.time() - t1) + ' sec.') assert len(FilterMeta) > 0 TTTGridMap = deserializer.deserializeTTT( len(FilterMeta)) mint, maxt = deserializer.deserializeMinTMaxT( len(FilterMeta)) f = open( '../tttgrid/tttgrid%s_%s_%s_%s_%s.pkl' % (phase, ttt_model, ev.time, arrayname, workdepth), 'wb') print("dumping the traveltime grid for this array") pickle.dump([TTTGridMap, mint, maxt], f) f.close() if cfg.Bool('correct_shifts_empirical') is True: ttt.calcTTTAdv(Config, FilterMeta[i], Origin_emp, i, arrayname, W, refshift, phase) assert len(FilterMeta) > 0 TTTGridMap_emp = deserializer.deserializeTTT( len(FilterMeta)) mint_emp, maxt_emp = deserializer.deserializeMinTMaxT( len(FilterMeta)) f = open( '../tttgrid/tttgrid%s_%s_%s_%s_%s_emp.pkl' % (phase, ttt_model, ev_emp.time, arrayname, workdepth), 'wb') print("dumping the traveltime grid for this array") pickle.dump([TTTGridMap_emp, mint_emp, maxt_emp], f) f.close() t2 = time.time() Logfile.red('%s took %0.3f s' % ('TTT', (t2 - t1))) switch = filterindex tw = times.calculateTimeWindows(mint, maxt, Config, ev, switch) if cfg.Bool('correct_shifts_empirical') is True: tw_emp = times.calculateTimeWindows( mint_emp, maxt_emp, Config, ev_emp, switch) if cfg.pyrocko_download() is True: if cfg.quantity() == 'displacement': Wd_emp = waveform.readWaveformsPyrocko_restituted( FilterMeta, tw, evpath, ev_emp, desired) elif cfg.Bool('synthetic_test') is True: Wd_emp = waveform.readWaveformsPyrockodummy( FilterMeta, tw_emp, evpath_emp, ev_emp) else: Wd_emp = waveform.readWaveformsPyrocko( FilterMeta, tw_emp, evpath_emp, ev_emp, desired) elif cfg.colesseo_input() is True: Wd_emp = waveform.readWaveforms_colesseo( FilterMeta, tw_emp, evpath_emp, ev_emp, C) else: Wd_emp = waveform.readWaveforms( FilterMeta, tw_emp, evpath_emp, ev_emp) if cfg.Bool('synthetic_test') is True\ or cfg.Bool('dynamic_filter') is True: Wdf_emp = waveform.processdummyWaveforms( Wd_emp, Config, Folder, arrayname, FilterMeta, ev_emp, switch, W) Wdfs_emp.extend(Wdf_emp) else: Wdf_emp = waveform.processWaveforms( Wd_emp, Config, Folder, arrayname, FilterMeta, ev_emp, switch, W) Wdfs_emp.extend(Wdf_emp) if cfg.pyrocko_download() is True: if cfg.quantity() == 'displacement': Wd = waveform.readWaveformsPyrocko_restituted( FilterMeta, tw, evpath, ev, desired) elif cfg.Bool('synthetic_test') is True: Wd = waveform.readWaveformsPyrockodummy( FilterMeta, tw, evpath, ev) else: Wd = waveform.readWaveformsPyrocko( FilterMeta, tw, evpath, ev, desired) elif cfg.colesseo_input() is True: Wd = waveform.readWaveforms_colesseo( FilterMeta, tw, evpath, ev, C) else: Wd = waveform.readWaveforms(FilterMeta, tw, evpath, ev) if cfg.Bool('synthetic_test') is True\ or cfg.Bool('dynamic_filter') is True: Wdf = waveform.processdummyWaveforms( Wd, Config, Folder, arrayname, FilterMeta, ev, switch, W) Wdfs.extend(Wdf) else: Wdf = waveform.processWaveforms( Wd, Config, Folder, arrayname, FilterMeta, ev, switch, W) Wdfs.extend(Wdf) C.writeStationFile(FilterMeta, Folder, counter) Logfile.red('%d Streams added for Processing' % (len(Wd))) t1 = time.time() f = open( '../tttgrid/tttgrid%s_%s_%s_%s_%s.pkl' % (phase, ttt_model, ev.time, arrayname, workdepth), 'rb') TTTGridMap, mint, maxt = pickle.load(f) f.close() if switch == 0: step = cfg.step() if switch == 1: step = cfg.step_f2() if cfg.UInt('forerun') > 0: ntimes = int( (cfg.UInt('forerun') + cfg.UInt('duration')) / step) else: ntimes = int((cfg.UInt('duration')) / step) if cfg.Bool('combine_all') is False: if cfg.optimize() is True: optim.solve(counter, Config, Wdf, FilterMeta, mint, maxt, TTTGridMap, Folder, Origin, ntimes, switch, ev, arrayfolder, syn_in, refshifts, phase, rpe + str(arrayname), flag_rpe) else: if cfg.Bool('correct_shifts_empirical') is True: if cfg.Bool('correct_shifts_empirical_run' ) is True: f = open( '../tttgrid/tttgrid%s_%s_%s_%s_%s_emp.pkl' % (phase, ttt_model, ev_emp.time, arrayname, workdepth), 'rb') TTTGridMap_emp, mint_emp, maxt_emp = pickle.load( f) f.close() flag_rpe = True arraySemb, weight, array_center = sembCalc.doCalc( counter, Config, Wdf_emp, FilterMeta, mint, maxt, TTTGridMap_emp, Folder, Origin, ntimes, switch, ev_emp, arrayfolder, syn_in, refshifts, phase, rpe + str(arrayname), flag_rpe) if sys.version_info.major >= 3: f = open(rpe + str(arrayname), 'rb') else: f = open(rpe + str(arrayname)) RefDict_empirical = pickle.load(f) refshifts = RefDict_empirical for j in range(0, len(FilterMeta)): if cfg.correct_shifts() is False: refshifts[j] = refshifts[j] * 0. flag_rpe = False arraySemb, weight, array_center = sembCalc.doCalc( counter, Config, Wdf, FilterMeta, mint, maxt, TTTGridMap, Folder, Origin, ntimes, switch, ev, arrayfolder, syn_in, refshifts, phase, rpe + str(arrayname), flag_rpe) weights.append(weight) array_centers.append(array_center) ASL.append(arraySemb) sembCalc.writeSembMatricesSingleArray( arraySemb, Config, Origin, arrayfolder, ntimes, switch, phase) fileName = os.path.join(arrayfolder, 'stations.txt') Logfile.add('Write to file ' + fileName) fobjarraynetwork = open(fileName, 'w') for i in FilterMeta: fobjarraynetwork.write( ('%s %s %s\n') % (i.getName(), i.lat, i.lon)) fobjarraynetwork.close() t2 = time.time() Logfile.add('CALC took %0.3f sec' % (t2 - t1)) counter += 1 stations_per_array.append(len(FilterMeta)) TTTgrids.update(TTTGridMap) mints.append(mint) maxts.append(maxt) FilterMetas[len(FilterMetas):] = FilterMeta TTTGridMap = [] if cfg.Bool('combine_all') is True: if cfg.pyrocko_download() is True: if cfg.Bool('synthetic_test') is True: Wd = waveform.readWaveformsPyrockodummy( FilterMetas, tw, evpath, ev) else: if cfg.quantity() == 'displacement': Wd = waveform.readWaveformsPyrocko_restituted( FilterMetas, tw, evpath, ev, desired) else: Wd = waveform.readWaveformsPyrocko( FilterMetas, tw, evpath, ev, desired) elif cfg.colesseo_input() is True: Wd = waveform.readWaveforms_colesseo( FilterMetas, tw, evpath, ev, C) else: Wd = waveform.readWaveforms(FilterMetas, tw, evpath, ev) if cfg.Bool('synthetic_test') is True: Wdf = waveform.processdummyWaveforms( Wd, Config, Folder, arrayname, FilterMetas, ev, switch, W) else: Wdf = waveform.processWaveforms( Wd, Config, Folder, arrayname, FilterMetas, ev, switch, W) mint = num.min(mints) maxt = num.max(maxts) flag_rpe = False if cfg.Bool('bootstrap_array_weights') is False: arraySemb, weight, array_center = sembCalc.doCalc( counter, Config, Wdf, FilterMetas, mint, maxt, TTTgrids, Folder, Origin, ntimes, switch, ev, arrayfolder, syn_in, refshifts_global, phase, rpe + str(arrayname), flag_rpe) ASL.append(arraySemb) weights.append(weight) array_centers.append(array_center) sembCalc.writeSembMatricesSingleArray( arraySemb, Config, Origin, arrayfolder, ntimes, switch, phase) else: nboot = cfg.Int('n_bootstrap') tmp_general = 1 for ibootstrap in range(nboot): f = rstate.uniform(0., 1., size=counter + 1) f = num.sort(f) g = f[1:] - f[:-1] k = 0 ws = [] for wss in range(0, counter - 1): for stats in range(0, stations_per_array[k]): ws.append(g[k]) k = +1 ws = num.asarray(ws) arraySemb, weight, array_center = sembCalc.doCalc( counter, Config, Wdf, FilterMetas, mint, maxt, TTTgrids, Folder, Origin, ntimes, switch, ev, arrayfolder, syn_in, refshifts_global, phase, rpe + str(arrayname), flag_rpe, bs_weights=ws) ASL.append(arraySemb) weights.append(weight) array_centers.append(array_center) sembCalc.writeSembMatricesSingleArray( arraySemb, Config, Origin, arrayfolder, ntimes, switch, phase, bootstrap=ibootstrap) if ASL: Logfile.red('collect semblance matrices from\ all arrays') sembmax, tmp = sembCalc.collectSemb( ASL, Config, Origin, Folder, ntimes, len(networks), switch, array_centers, phase, cboot=ibootstrap) tmp_general *= tmp ASL = [] sembmax, tmp = sembCalc.collectSemb( ASL, Config, Origin, Folder, ntimes, len(networks), switch, array_centers, phase, cboot=None, temp_comb=tmp_general) if cfg.optimize_all() is True: import optim_csemb sembmax, tmp = sembCalc.collectSemb( ASL, Config, Origin, Folder, ntimes, len(networks), switch) optim_csemb.solve(counter, Config, Wdf, FilterMeta, mint, maxt, TTTGridMap, Folder, Origin, ntimes, switch, ev, arrayfolder, syn_in, ASL, sembmax, evpath, XDict, RefDict, workdepth, filterindex, Wdfs) if ASL and cfg.Bool('bootstrap_array_weights') is False: Logfile.red('collect semblance matrices from all arrays') sembmax, tmp = sembCalc.collectSemb( ASL, Config, Origin, Folder, ntimes, len(networks), switch, array_centers, phase) if cfg.Bool('weight_by_noise') is True: sembCalc.collectSembweighted(ASL, Config, Origin, Folder, ntimes, len(networks), switch, weights) else: Logfile.red('Nothing to do -> Finish') print("last work depth:") print(workdepth)
def processWaveforms(WaveformDict, Config, Folder, network, MetaDict, Event, switch, Xcorr): Logfile.red('Start Processing') cfg = FilterCfg(Config) new_frequence = cfg.newFrequency() # ['new_frequence'] vp, vs, rho = ttt.calcak135parameter(Event) for index, i in enumerate(WaveformDict): Logfile.add( '%s:%s -------------------------------------------------------------' % (index, i)) if Config['export_unfiltered'].capitalize() == 'True': writeWaveform(Folder, i, WaveformDict[i], 'U', network) station = getStation(i, MetaDict) psign = 1 if cfg.Int('fm') == 1: azi = ttt.bearing(Event, station) bazi = ttt.backazi(station, Event) psign = ttt.dubcup(rho, vp, vs, Event.strike, Event.dip, Event.rake, azi, station.takeoff) msg = 'Takeoff ' + str( station.takeoff) + ' Azi ' + str(azi) + ' Bazi ' + str(bazi) msg += (' vp ' + str(vp) + ' vs ' + str(vs) + ' rho ' + str(rho) + ' Psign ' + str(psign)) Logfile.add(msg) try: Logfile.add(station.getName() + ' ' + station.lat + ' ' + station.lon + ' ' + station.gain + ' PSIGN: ' + str(psign)) except: try: print psign, station.getName( ), station.lat, station.lon, station.gain except: pass if psign == -1: Logfile.add('correcting polarisation for station %s ' % (i)) -1. * WaveformDict[i][0].data #gain correctur gain = float(station.gain) if gain == 0.0 or gain == -1.0: gain = 1 #hs : gain missing in metafile WaveformDict[i][0].data * (1.0 / gain) if switch == 0: Logfile.add('bandpass filtered stream for station %s ' % (i)) WaveformDict[i].filter( 'bandpass', freqmin=cfg.flo(), # ['flo'] freqmax=cfg.fhi(), # ['fhi'] corners=cfg.ns(), # ['ns'] zerophase=bool(Config['zph'])) elif switch == 1: Logfile.add('bandpass filtered stream for station %s ' % (i)) WaveformDict[i].filter( 'bandpass', freqmin=cfg.flo2(), # ['flo'] freqmax=cfg.fhi2(), # ['fhi'] corners=cfg.ns2(), # ['ns'] zerophase=bool(Config['zph2'])) else: Logfile.add('no filter set for station %s ' % (i)) if Config['export_filtered'].capitalize() == 'True': writeWaveform(Folder, i, WaveformDict[i], 'F', network) #resampling j = resampleWaveform(WaveformDict[i][0], new_frequence) WaveformDict[i] = j if Config['export_resampled'].capitalize() == 'True': writeWaveform(Folder, i, WaveformDict[i], 'R', network) return WaveformDict
def processLoop(): #==================================get meta info========================================== C = config.Config(evpath) Origin = C.parseConfig('origin') Config = C.parseConfig('config') Meta = C.readMetaInfoFile() #==================================get meta info========================================== #==================================do prerequiries======================================== Folder = C.createFolder() C.cpSkeleton(Folder, Config) C.writeConfig(Config, Origin, Folder) cfg = ConfigObj(dict=Config) filter = FilterCfg(Config) origin = OriginCfg(Origin) ntimes = int( (cfg.UInt('forerun') + cfg.UInt('duration')) / cfg.UInt('step')) default = 0 strike = origin.strike(default) # Origin.get ('strike', default) dip = origin.dip(default) # Origin.get ('dip', default) rake = origin.rake(default) # Origin.get ('rake', default) # ev = Event (Origin['lat'],Origin['lon'],Origin['depth'],Origin['time'], # strike = strike,dip=dip,rake=rake) ev = Event(origin.lat(), origin.lon(), origin.depth(), origin.time(), strike=strike, dip=dip, rake=rake) filtername = filter.filterName() Logfile.add('filtername = ' + filtername) #todo crosscorrelation for all arrays before processing XDict = {} RefDict = {} SL = {} if cfg.Int('xcorr') == 1: newFreq = str(filter.newFrequency()) fobjreferenceshiftname = newFreq + '_' + filtername + '.refpkl' rp = os.path.join(Folder['semb'], fobjreferenceshiftname) fobjpickleshiftname = newFreq + '_' + filtername + '.xcorrpkl' ps = os.path.join(Folder['semb'], fobjpickleshiftname) if (os.path.isfile(rp) and os.path.getsize(rp) != 0 and os.path.isfile(ps) and os.path.getsize(ps) != 0): Logfile.add('file exits : ' + rp) Logfile.add('load refshifts') f = open(rp) RefDict = pickle.load(f) x = open(ps) XDict = pickle.load(x) #xcorrnetworks = Config['networks'].split(',') xcorrnetworks = cfg.String('networks').split(',') for i in xcorrnetworks: SL[i] = len(Config[i].split('|')) else: SL = {} xcorrnetworks = cfg.String('networks').split(',') for i in xcorrnetworks: W = {} refshift = 0 network = cfg.String(i).split('|') FilterMeta = ttt.filterStations(Meta, Config, Origin, network) arrayfolder = os.path.join(Folder['semb'], i) if os.access(arrayfolder, os.F_OK) == False: os.makedirs(arrayfolder) A = Xcorr(ev, FilterMeta, evpath, Config, arrayfolder) W, triggerobject = A.runXcorr() XDict[i] = W RefDict[i] = triggerobject.tdiff SL[i] = len(network) #endfor fobjrefshift = open(rp, 'w') pickle.dump(RefDict, fobjrefshift) fobjrefshift.close() output = open(ps, 'w') pickle.dump(XDict, output) output.close() #endif for i in sorted(XDict.iterkeys()): Logfile.red('Array %s has %3d of %3d Stations left' % (i, len(XDict[i]), SL[i])) logger.info( '\033[31mFor proceeding without changes press enter or give new comma seperatet network list or quit for exit\033[0m' ) while True: nnl = raw_input("please enter your choice: ") #Logfile.add ('Choise = ' + nnl) if len(nnl) == 0: if not Basic.question('Process all networks ?'): continue Logfile.red('This networks will be used for processing: %s' % (Config['networks'])) break elif str(nnl) == 'quit': sys.exit() elif str(nnl) == 'rerun': event = os.path.join(*evpath.split('/')[-1:]) try: os.remove(rp) os.remove(ps) except: pass mainfolder = os.path.join(os.path.sep, *evpath.split('/')[:-2]) os.chdir(mainfolder) cmd = ('%s arraytool.py process %s') % (sys.executable, event) Logfile.add('cmd = ' + cmd) os.system(cmd) sys.exit() else: # Check if selected array(s) exists names = nnl.split(',') isOk = True for array in names: arrayfolder = os.path.join(Folder['semb'], array) if not os.path.isdir(arrayfolder): Logfile.error('Illegal network name ' + str(array)) isOk = False break #endfor if not isOk: continue # Illegal network : input again # use these networks Logfile.add('This networks will be used for processing: %s' % (nnl)) Config['networks'] = nnl break #endif #endwhile True for i in range(3, 0, -1): time.sleep(1) Logfile.red('Start processing in %d seconds ' % (i)) #endfor #print XDict #print RefDict #TriggerOnset.append(triggerobject) #print 'MAINTDIFF ',triggerobject.tdiff #sys.exit() wd = Origin['depth'] start, stop, step = cfg.String('depths').split(',') start = int(start) stop = int(stop) + 1 step = int(step) Logfile.add('working on ' + Config['networks']) #==================================loop over depth======================================= for depthindex in xrange(start, stop): workdepth = float(wd) + depthindex * step Origin['depth'] = workdepth ev = Event(Origin['lat'], Origin['lon'], Origin['depth'], Origin['time'], strike=strike, dip=dip, rake=rake) Logfile.add('WORKDEPTH: ' + str(Origin['depth'])) #==================================do prerequiries======================================== #==================================loop over arrays======================================= ASL = [] networks = Config['networks'].split(',') counter = 1 TriggerOnset = [] for i in networks: arrayname = i arrayfolder = os.path.join(Folder['semb'], arrayname) network = Config[i].split('|') Logfile.add('network: ' + str(network)) FilterMeta = ttt.filterStations(Meta, Config, Origin, network) #if len(FilterMeta) < 3: continue #hs : wieder rein if len(FilterMeta) < 3: continue W = XDict[i] refshift = RefDict[i] FilterMeta = cmpFilterMetavsXCORR(W, FilterMeta) #print 'W: ',W,len(W); print refshift; print 'FM ',FilterMeta,len(FilterMeta) Logfile.add( 'BOUNDING BOX DIMX: %s DIMY: %s GRIDSPACING: %s \n' % (Config['dimx'], Config['dimy'], Config['gridspacing'])) ##############=======================PARALLEL=========================================== Logfile.red('Calculating Traveltime Grid') t1 = time.time() if WINDOWS: isParallel = False #hs : parallel #else : isParallel = True #10.12.2015 else: isParallel = False #10.12.2015 if isParallel: #hs maxp = int(Config['ncore']) #maxp = 20 #hs po = multiprocessing.Pool(maxp) for i in xrange(len(FilterMeta)): po.apply_async(ttt.calcTTTAdv, (Config, FilterMeta[i], Origin, i, arrayname, W, refshift)) po.close() po.join() else: #hs+ for i in xrange(len(FilterMeta)): t1 = time.time() ttt.calcTTTAdv(Config, FilterMeta[i], Origin, i, arrayname, W, refshift) Logfile.add('ttt.calcTTTAdv : ' + str(time.time() - t1) + ' sec.') #endif #hs- assert len(FilterMeta) > 0 TTTGridMap = deserializer.deserializeTTT(len(FilterMeta)) mint, maxt = deserializer.deserializeMinTMaxT(len(FilterMeta)) t2 = time.time() Logfile.red('%s took %0.3f s' % ('TTT', (t2 - t1))) #sys.exit() ##############=======================SERIELL=========================================== tw = times.calculateTimeWindows(mint, maxt, Config, ev) Wd = waveform.readWaveforms(FilterMeta, tw, evpath, ev) Wdf = waveform.processWaveforms(Wd, Config, Folder, arrayname, FilterMeta, ev, W) #sys.exit() C.writeStationFile(FilterMeta, Folder, counter) Logfile.red('%d Streams added for Processing' % (len(Wd))) ##############========================================================================= ##############=======================PARALLEL========================================== t1 = time.time() arraySemb = sembCalc.doCalc(counter, Config, Wdf, FilterMeta, mint, maxt, TTTGridMap, Folder, Origin, ntimes) t2 = time.time() Logfile.add('CALC took %0.3f sec' % (t2 - t1)) ASL.append(arraySemb) counter += 1 sembCalc.writeSembMatricesSingleArray(arraySemb, Config, Origin, arrayfolder, ntimes) fileName = os.path.join(arrayfolder, 'stations.txt') Logfile.add('Write to file ' + fileName) fobjarraynetwork = open(fileName, 'w') for i in FilterMeta: fobjarraynetwork.write( ('%s %s %s\n') % (i.getName(), i.lat, i.lon)) fobjarraynetwork.close() if ASL: Logfile.red('collect semblance matrices from all arrays') sembCalc.collectSemb(ASL, Config, Origin, Folder, ntimes, len(networks)) #fobjtrigger = open (os.path.join(Folder['semb'],'Trigger.txt'),'w') #for i in TriggerOnset: # print i.aname,i.sname,i.ttime # fobjtrigger.write(('%s %s %s\n')%(i.aname,i.sname,i.ttime)) #fobjtrigger.close() else: Logfile.red('Nothing to do -> Finish')