def processLoop():
#==================================get meta info==========================================
C = config.Config(evpath)
Origin = C.parseConfig('origin')
try:
Syn_in = C.parseConfig('syn')
syn_in = SynthCfg(Syn_in)
except:
pass
Config = C.parseConfig('config')
cfg = ConfigObj(dict=Config)
if cfg.pyrocko_download() == True:
Meta = C.readpyrockostations() #
elif cfg.colesseo_input() == True:
scenario = guts.load(filename=cfg.colosseo_scenario_yml())
scenario_path = cfg.colosseo_scenario_yml()[:-12]
Meta = C.readcolosseostations(scenario_path)
else:
Meta = C.readMetaInfoFile()
#==================================get meta info==========================================
#==================================do prerequiries========================================
Folder = C.createFolder()
#C.cpSkeleton(Folder,Config)
C.writeConfig(Config, Origin, Folder)
filter = FilterCfg(Config)
ntimes = int(
(cfg.UInt('forerun') + cfg.UInt('duration')) / cfg.UInt('step'))
origin = OriginCfg(Origin)
if cfg.colesseo_input() == True:
from pyrocko import util
events = scenario.get_events()
ev = events[0]
origin.strike = str(ev.moment_tensor.strike1)
origin.rake = str(ev.moment_tensor.rake1)
origin.dip = str(ev.moment_tensor.dip1)
strike = ev.moment_tensor.strike1
origin.lat = str(ev.lat)
origin.lon = str(ev.lon)
origin.depth = str(ev.depth / 1000.)
depth = ev.depth
origin.time = util.time_to_str(ev.time)
time_ev = util.time_to_str(ev.time)
lat = ev.lat
lon = ev.lon
rake = ev.moment_tensor.rake1
dip = ev.moment_tensor.dip1
Origin['strike'] = str(ev.moment_tensor.strike1)
Origin['rake'] = str(ev.moment_tensor.rake1)
Origin['dip'] = str(ev.moment_tensor.dip1)
Origin['lat'] = str(ev.lat)
Origin['lon'] = str(ev.lon)
Origin['time'] = util.time_to_str(ev.time)
Origin['depth'] = str(ev.depth / 1000.)
ev = Event(lat, lon, depth, time_ev, strike=strike, dip=dip, rake=rake)
else:
default = 0
strike = origin.strike(default) # Origin.get ('strike', default)
dip = origin.dip(default) # Origin.get ('dip', default)
rake = origin.rake(default) # Origin.get ('rake', default)
ev = Event(origin.lat(),
origin.lon(),
origin.depth(),
origin.time(),
strike=strike,
dip=dip,
rake=rake)
filtername = filter.filterName()
Logfile.add('filtername = ' + filtername)
#todo crosscorrelation for all arrays before processing
XDict = {}
RefDict = {}
SL = {}
if cfg.Int('xcorr') == 1:
newFreq = str(filter.newFrequency())
fobjreferenceshiftname = newFreq + '_' + filtername + '.refpkl'
rp = os.path.join(Folder['semb'], fobjreferenceshiftname)
fobjpickleshiftname = newFreq + '_' + filtername + '.xcorrpkl'
ps = os.path.join(Folder['semb'], fobjpickleshiftname)
if (os.path.isfile(rp) and os.path.getsize(rp) != 0
and os.path.isfile(ps) and os.path.getsize(ps) != 0):
Logfile.add('file exits : ' + rp)
Logfile.add('load refshifts')
f = open(rp)
RefDict = pickle.load(f)
x = open(ps)
XDict = pickle.load(x)
xcorrnetworks = cfg.String('networks').split(',')
for i in xcorrnetworks:
SL[i] = len(Config[i].split('|'))
else:
SL = {}
xcorrnetworks = cfg.String('networks').split(',')
for i in xcorrnetworks:
W = {}
refshift = 0
network = cfg.String(i).split('|')
FilterMeta = ttt.filterStations(Meta, Config, Origin, network)
arrayfolder = os.path.join(Folder['semb'], i)
if os.access(arrayfolder, os.F_OK) == False:
os.makedirs(arrayfolder)
if cfg.pyrocko_download() == True:
A = Xcorr(ev, FilterMeta, evpath, Config, Syn_in,
arrayfolder)
else:
A = Xcorr(ev, FilterMeta, evpath, Config, Syn_in,
arrayfolder)
print "run Xcorr"
W, triggerobject = A.runXcorr()
XDict[i] = W
RefDict[i] = triggerobject.tdiff
SL[i] = len(network)
#endfor
fobjrefshift = open(rp, 'w')
pickle.dump(RefDict, fobjrefshift)
fobjrefshift.close()
output = open(ps, 'w')
pickle.dump(XDict, output)
output.close()
for i in sorted(XDict.iterkeys()):
Logfile.red('Array %s has %3d of %3d Stations left' %
(i, len(XDict[i]), SL[i]))
logger.info(
'\033[31mFor proceeding without changes press enter or give new comma seperatet network list or quit for exit\033[0m'
)
while True:
nnl = raw_input("please enter your choice: ")
#Logfile.add ('Choise = ' + nnl)
if len(nnl) == 0:
if not Basic.question('Process all networks ?'): continue
Logfile.red('This networks will be used for processing: %s' %
(Config['networks']))
break
elif str(nnl) == 'quit':
sys.exit()
elif str(nnl) == 'rerun':
event = os.path.join(*evpath.split('/')[-1:])
try:
os.remove(rp)
os.remove(ps)
except:
pass
mainfolder = os.path.join(os.path.sep, *evpath.split('/')[:-2])
os.chdir(mainfolder)
cmd = ('%s arraytool.py process %s') % (sys.executable, event)
Logfile.add('cmd = ' + cmd)
os.system(cmd)
sys.exit()
else:
# Check if selected array(s) exists
names = nnl.split(',')
isOk = True
for array in names:
arrayfolder = os.path.join(Folder['semb'], array)
if not os.path.isdir(arrayfolder):
Logfile.error('Illegal network name ' + str(array))
isOk = False
break
#endfor
if not isOk: continue # Illegal network : input again
# use these networks
Logfile.add('This networks will be used for processing: %s' %
(nnl))
Config['networks'] = nnl
break
for i in range(3, 0, -1):
time.sleep(1)
Logfile.red('Start processing in %d seconds ' % (i))
wd = Origin['depth']
start, stop, step = cfg.String('depths').split(',')
start = int(start)
stop = int(stop) + 1
step = int(step)
filters = cfg.String('filters')
filters = int(filters)
Logfile.add('working on ' + Config['networks'])
#==================================loop over depth======================
for filterindex in xrange(0, filters):
for depthindex in xrange(start, stop, step):
workdepth = float(wd) + depthindex
Origin['depth'] = workdepth
ev = Event(Origin['lat'],
Origin['lon'],
Origin['depth'],
Origin['time'],
strike=strike,
dip=dip,
rake=rake)
Logfile.add('WORKDEPTH: ' + str(Origin['depth']))
#==================================do prerequiries===============
#==================================loop over arrays================
ASL = []
weights = []
array_centers = []
networks = Config['networks'].split(',')
counter = 1
TriggerOnset = []
Wdfs = []
for i in networks:
arrayname = i
arrayfolder = os.path.join(Folder['semb'], arrayname)
network = Config[i].split('|')
Logfile.add('network: ' + str(network))
FilterMeta = ttt.filterStations(Meta, Config, Origin, network)
#if len(FilterMeta) < 3: continue #hs : wieder rein
if len(FilterMeta) < 3: continue
W = XDict[i]
refshift = RefDict[i]
FilterMeta = cmpFilterMetavsXCORR(W, FilterMeta)
Logfile.add(
'BOUNDING BOX DIMX: %s DIMY: %s GRIDSPACING: %s \n' %
(Config['dimx'], Config['dimy'], Config['gridspacing']))
##############=======================PARALLEL===========================================
Logfile.red('Calculating Traveltime Grid')
t1 = time.time()
isParallel = False #10.12.2015
TTTGridMap = []
mint = []
maxt = []
try:
f = open(
'../tttgrid/tttgrid_%s_%s_%s.pkl' %
(ev.time, arrayname, workdepth), 'rb')
print "loading travel time grid_%s_%s_%s.pkl" % (
ev.time, arrayname, workdepth)
TTTGridMap, mint, maxt = pickle.load(f)
f.close()
print "loading of travel time grid sucessful"
except:
print "loading of travel time grid unsucessful, will now calculate the grid:"
if isParallel: #hs
# maxp = int (Config['ncore'])
maxp = 6 #hs
po = multiprocessing.Pool(maxp)
for i in xrange(len(FilterMeta)):
po.apply_async(ttt.calcTTTAdv,
(Config, FilterMeta[i], Origin, i,
arrayname, W, refshift))
po.close()
po.join()
else: #hs+
for i in xrange(len(FilterMeta)):
t1 = time.time()
ttt.calcTTTAdv(Config, FilterMeta[i], Origin, i,
arrayname, W, refshift)
Logfile.add('ttt.calcTTTAdv : ' +
str(time.time() - t1) + ' sec.')
#endif #hs-
assert len(FilterMeta) > 0
TTTGridMap = deserializer.deserializeTTT(len(FilterMeta))
mint, maxt = deserializer.deserializeMinTMaxT(
len(FilterMeta))
f = open(
'../tttgrid/tttgrid_%s_%s_%s.pkl' %
(ev.time, arrayname, workdepth), 'wb')
print "dumping the traveltime grid for this array"
pickle.dump([TTTGridMap, mint, maxt], f)
f.close()
t2 = time.time()
Logfile.red('%s took %0.3f s' % ('TTT', (t2 - t1)))
switch = filterindex
tw = times.calculateTimeWindows(mint, maxt, Config, ev, switch)
if cfg.pyrocko_download() == True:
if cfg.quantity() == 'displacement':
Wd = waveform.readWaveformsPyrocko_restituted(
FilterMeta, tw, evpath, ev)
else:
Wd = waveform.readWaveformsPyrocko(
FilterMeta, tw, evpath, ev)
# Wdf = waveform.processpyrockoWaveforms(Wd, Config, Folder, arrayname, FilterMeta, ev, switch, W)
elif cfg.colesseo_input() == True:
Wd = waveform.readWaveforms_colesseo(
FilterMeta, tw, evpath, ev, C)
else:
Wd = waveform.readWaveforms(FilterMeta, tw, evpath, ev)
if cfg.Bool('synthetic_test') is True:
Wdf = waveform.processdummyWaveforms(
Wd, Config, Folder, arrayname, FilterMeta, ev, switch,
W)
Wdfs.append(Wdf)
else:
Wdf = waveform.processWaveforms(Wd, Config, Folder,
arrayname, FilterMeta, ev,
switch, W)
Wdfs.append(Wdf)
C.writeStationFile(FilterMeta, Folder, counter)
Logfile.red('%d Streams added for Processing' % (len(Wd)))
t1 = time.time()
f = open(
'../tttgrid/tttgrid_%s_%s_%s.pkl' %
(ev.time, arrayname, workdepth), 'rb')
print "loading travel time grid_%s_%s_%s.pkl" % (
ev.time, arrayname, workdepth)
TTTGridMap, mint, maxt = pickle.load(f)
f.close()
if cfg.optimize() == True:
optim.solve(counter, Config, Wdf, FilterMeta, mint, maxt,
TTTGridMap, Folder, Origin, ntimes, switch, ev,
arrayfolder, syn_in)
else:
arraySemb, weight, array_center = sembCalc.doCalc(
counter, Config, Wdf, FilterMeta, mint, maxt,
TTTGridMap, Folder, Origin, ntimes, switch, ev,
arrayfolder, syn_in)
t2 = time.time()
Logfile.add('CALC took %0.3f sec' % (t2 - t1))
weights.append(weight)
array_centers.append(array_center)
ASL.append(arraySemb)
counter += 1
sembCalc.writeSembMatricesSingleArray(arraySemb, Config,
Origin, arrayfolder,
ntimes, switch)
fileName = os.path.join(arrayfolder, 'stations.txt')
Logfile.add('Write to file ' + fileName)
fobjarraynetwork = open(fileName, 'w')
for i in FilterMeta:
fobjarraynetwork.write(
('%s %s %s\n') % (i.getName(), i.lat, i.lon))
fobjarraynetwork.close()
TTTGridMAP = []
if cfg.optimize_all() == True:
import optim_csemb
from optim_csemb import solve
sembmax = sembCalc.collectSemb(ASL, Config, Origin, Folder,
ntimes, len(networks), switch)
optim_csemb.solve(counter, Config, Wdf, FilterMeta, mint, maxt,
TTTGridMap, Folder, Origin, ntimes, switch,
ev, arrayfolder, syn_in, ASL, sembmax,
evpath, XDict, RefDict, workdepth,
filterindex, Wdfs)
if ASL:
Logfile.red('collect semblance matrices from all arrays')
sembmax = sembCalc.collectSemb(ASL, Config, Origin, Folder,
ntimes, len(networks), switch,
array_centers)
if cfg.Bool('weight_by_noise') == True:
sembCalc.collectSembweighted(ASL, Config, Origin, Folder,
ntimes, len(networks), switch,
weights)
else:
Logfile.red('Nothing to do -> Finish')
print "depth:"
print workdepth
def processLoop():
C = config.Config(evpath)
Origin = C.parseConfig('origin')
flag_rpe = False
try:
Syn_in = C.parseConfig('syn')
syn_in = SynthCfg(Syn_in)
except TypeError:
pass
Config = C.parseConfig('config')
cfg = ConfigObj(dict=Config)
phases = cfg.Str('ttphases')
phases = phases.split(',')
if cfg.pyrocko_download() is True:
Meta = C.readpyrockostations()
elif cfg.colesseo_input() is True:
scenario = guts.load(filename=cfg.colosseo_scenario_yml())
scenario_path = cfg.colosseo_scenario_yml()[:-12]
Meta = C.readcolosseostations(scenario_path)
else:
Meta = C.readMetaInfoFile()
Folder = C.createFolder()
C.writeConfig(Config, Origin, Folder)
filter = FilterCfg(Config)
if cfg.UInt('forerun') > 0:
ntimes = int(
(cfg.UInt('forerun') + cfg.UInt('duration')) / cfg.UInt('step'))
else:
ntimes = int((cfg.UInt('duration')) / cfg.UInt('step'))
origin = OriginCfg(Origin)
if cfg.colesseo_input() is True:
from pyrocko import util
events = scenario.get_events()
ev = events[0]
origin.strike = str(ev.moment_tensor.strike1)
origin.rake = str(ev.moment_tensor.rake1)
origin.dip = str(ev.moment_tensor.dip1)
strike = ev.moment_tensor.strike1
origin.lat = str(ev.lat)
origin.lon = str(ev.lon)
origin.depth = str(ev.depth / 1000.)
depth = ev.depth
origin.time = util.time_to_str(ev.time)
time_ev = util.time_to_str(ev.time)
lat = ev.lat
lon = ev.lon
rake = ev.moment_tensor.rake1
dip = ev.moment_tensor.dip1
Origin['strike'] = str(ev.moment_tensor.strike1)
Origin['rake'] = str(ev.moment_tensor.rake1)
Origin['dip'] = str(ev.moment_tensor.dip1)
Origin['lat'] = str(ev.lat)
Origin['lon'] = str(ev.lon)
Origin['time'] = util.time_to_str(ev.time)
Origin['depth'] = str(ev.depth / 1000.)
ev = Event(lat, lon, depth, time_ev, strike=strike, dip=dip, rake=rake)
else:
default = 0
strike = origin.strike(default)
dip = origin.dip(default)
rake = origin.rake(default)
ev = Event(origin.lat(),
origin.lon(),
origin.depth(),
origin.time(),
strike=strike,
dip=dip,
rake=rake)
if cfg.Bool('correct_shifts_empirical') is True:
Origin_emp = C.parseConfig('origin_emp')
origin_emp = OriginCfg(Origin_emp)
ev_emp = Event(origin_emp.lat(),
origin_emp.lon(),
origin_emp.depth(),
origin_emp.time(),
strike=strike,
dip=dip,
rake=rake)
filtername = filter.filterName()
Logfile.add('filtername = ' + filtername)
XDict = OrderedDict()
RefDict = OrderedDict()
SL = OrderedDict()
refshifts_global = []
newFreq = str(filter.newFrequency())
xcorrnetworks = cfg.String('networks').split(',')
if cfg.Int('xcorr') is 1:
fobjreferenceshiftname = newFreq + '_' + filtername + '.refpkl'
rp = os.path.join(Folder['semb'], fobjreferenceshiftname)
fobjreferenceshiftnameemp = newFreq + '_' + filtername + 'emp' + '.refpkl'
rpe = os.path.join(Folder['semb'], fobjreferenceshiftnameemp)
fobjpickleshiftname = newFreq + '_' + filtername + '.xcorrpkl'
ps = os.path.join(Folder['semb'], fobjpickleshiftname)
if (os.path.isfile(rp) and os.path.getsize(rp) != 0
and os.path.isfile(ps) and os.path.getsize(ps) != 0):
Logfile.add('xcorr/reference shift file exits : ' + rp)
Logfile.add('loaded reference shift')
if sys.version_info.major >= 3:
f = open(rp, 'rb')
else:
f = open(rp)
RefDict = pickle.load(f)
if sys.version_info.major >= 3:
x = open(ps, 'rb')
else:
x = open(ps)
XDict = pickle.load(x)
for i in xcorrnetworks:
SL[i] = len(Config[i].split('|'))
else:
SL = {}
for i in xcorrnetworks:
W = {}
network = cfg.String(i).split('|')
FilterMeta = ttt.filterStations(Meta, Config, Origin, network)
arrayfolder = os.path.join(Folder['semb'], i)
if os.access(arrayfolder, os.F_OK) is False:
os.makedirs(arrayfolder)
if cfg.pyrocko_download() is True:
# TODO check seperate xcoor nescessity
A = Xcorr(ev, FilterMeta, evpath, Config, Syn_in,
arrayfolder)
print("run Xcorr")
phase = phases[0]
W, triggerobject = A.runXcorr(phase)
XDict[i] = W
RefDict[i] = triggerobject.tdiff
SL[i] = len(network)
for j in range(0, len(FilterMeta)):
refshifts_global.append(triggerobject.tdiff)
if sys.version_info.major >= 3:
fobjrefshift = open(rp, 'wb')
else:
fobjrefshift = open(rp, 'w')
pickle.dump(RefDict, fobjrefshift)
fobjrefshift.close()
if sys.version_info.major >= 3:
output = open(ps, 'wb')
else:
output = open(ps, 'w')
pickle.dump(XDict, output)
output.close()
else:
fobjreferenceshiftname = newFreq + '_' + filtername + '.refpkl'
rp = os.path.join(Folder['semb'], fobjreferenceshiftname)
fobjreferenceshiftnameemp = newFreq + '_' + filtername + 'emp' + '.refpkl'
rpe = os.path.join(Folder['semb'], fobjreferenceshiftnameemp)
fobjpickleshiftname = newFreq + '_' + filtername + '.xcorrpkl'
ps = os.path.join(Folder['semb'], fobjpickleshiftname)
refshift = 0
if (os.path.isfile(rp) and os.path.getsize(rp) != 0
and os.path.isfile(ps) and os.path.getsize(ps) != 0):
Logfile.add('Temporay Memory file exits : ' + rp)
if sys.version_info.major >= 3:
f = open(rp, 'rb')
else:
f = open(rp)
RefDict = pickle.load(f)
if sys.version_info.major >= 3:
x = open(ps, 'rb')
else:
x = open(ps)
XDict = pickle.load(x)
for i in xcorrnetworks:
SL[i] = len(Config[j].split('|'))
network = cfg.String(i).split('|')
FilterMeta = ttt.filterStations(Meta, Config, Origin, network)
RefDict[i] = refshift
for j in range(0, len(FilterMeta)):
refshifts_global.append(refshift)
else:
SL = {}
for i in xcorrnetworks:
W = {}
refshift = 0
network = cfg.String(i).split('|')
FilterMeta = ttt.filterStations(Meta, Config, Origin, network)
arrayfolder = os.path.join(Folder['semb'], i)
if os.access(arrayfolder, os.F_OK) is False:
os.makedirs(arrayfolder)
if cfg.pyrocko_download() is True:
# TODO check seperate xcoor nescessity
A = Xcorr(ev, FilterMeta, evpath, Config, Syn_in,
arrayfolder)
else:
A = Xcorr(ev, FilterMeta, evpath, Config, Syn_in,
arrayfolder)
print("run Xcorr")
phase = phases[0]
W, triggerobject = A.runXcorr_dummy(phase)
XDict[j] = W
RefDict[j] = refshift
SL[j] = len(network)
for j in range(0, len(FilterMeta)):
refshifts_global.append(refshift)
if sys.version_info.major >= 3:
fobjrefshift = open(rp, 'wb')
else:
fobjrefshift = open(rp, 'w')
pickle.dump(RefDict, fobjrefshift)
fobjrefshift.close()
if sys.version_info.major >= 3:
output = open(ps, 'wb')
else:
output = open(ps, 'w')
pickle.dump(XDict, output)
output.close()
if sys.version_info.major >= 3:
for j in sorted(XDict.keys()):
Logfile.red('Array %s has %3d of %3d Stations left' %
(j, len(XDict[j]), SL[j]))
else:
for j in sorted(XDict.keys()):
Logfile.red('Array %s has %3d of %3d Stations left' %
(j, len(XDict[j]), SL[j]))
while True:
if sys.version_info.major >= 3:
nnl = input("please enter your choice: ")
else:
nnl = raw_input("please enter your choice: ")
if len(nnl) == 0:
if not Basic.question('Process all networks ?'):
continue
Logfile.red('This networks will be used for processing: %s' %
(Config['networks']))
break
elif str(nnl) == 'quit':
sys.exit()
elif str(nnl) == 'rerun':
event = os.path.join(*evpath.split('/')[-1:])
try:
os.remove(rp)
os.remove(ps)
except Exception:
pass
mainfolder = os.path.join(os.path.sep, *evpath.split('/')[:-2])
os.chdir(mainfolder)
cmd = ('%s arraytool.py process %s') % (sys.executable, event)
Logfile.add('cmd = ' + cmd)
os.system(cmd)
sys.exit()
else:
names = nnl.split(',')
isOk = True
for array in names:
arrayfolder = os.path.join(Folder['semb'], array)
if not os.path.isdir(arrayfolder):
Logfile.error('Illegal network name ' + str(array))
isOk = False
break
if not isOk:
continue
Logfile.add('This networks will be used for processing: %s' %
(nnl))
Config['networks'] = nnl
break
for j in range(3, 0, -1):
time.sleep(1)
Logfile.red('Start processing in %d seconds ' % (j))
wd = Origin['depth']
start, stop, step = cfg.String('depths').split(',')
start = int(start)
stop = int(stop) + 1
step_depth = int(step)
filters = cfg.String('filters')
filters = int(filters)
Logfile.add('working on ' + Config['networks'])
if cfg.Bool('correct_shifts_empirical') is True:
emp_loop = True
else:
emp_loop = False
# ==================================loop over phases======================
for phase in phases:
if phase is 'P':
desired = 'Z'
if phase is 'S':
desired = 'T'
# ==================================loop over filter setups=====
for filterindex in xrange(0, filters):
# ==================================loop over depth=======
for depthindex in xrange(start, stop, step_depth):
workdepth = float(wd) + depthindex
Origin['depth'] = workdepth
ev = Event(Origin['lat'],
Origin['lon'],
Origin['depth'],
Origin['time'],
strike=strike,
dip=dip,
rake=rake)
Logfile.add('WORKDEPTH: ' + str(Origin['depth']))
networks = Config['networks'].split(',')
ASL = []
weights = []
array_centers = []
counter = 1
stations_per_array = []
Wdfs = []
Wdfs_emp = []
FilterMetas = []
TTTgrids = OrderedDict()
mints = []
maxts = []
refshifts = []
for i in networks:
arrayname = i
arrayfolder = os.path.join(Folder['semb'], arrayname)
network = Config[i].split('|')
Logfile.add('network: ' + str(network))
FilterMeta = ttt.filterStations(Meta, Config, Origin,
network)
W = XDict[i]
refshift = RefDict[i]
for j in range(0, len(FilterMeta)):
if cfg.correct_shifts() is False:
refshift = refshift * 0.
refshifts.append(refshift)
FilterMeta = cmpFilterMetavsXCORR(W, FilterMeta)
Logfile.add('BOUNDING BOX DIMX: %s DIMY: %s GRIDSPACING:\
%s \n' % (Config['dimx'], Config['dimy'],
Config['gridspacing']))
Logfile.red('Calculating Traveltime Grid')
t1 = time.time()
isParallel = False
TTTGridMap = []
mint = []
maxt = []
ttt_model = cfg.Str('traveltime_model')
try:
if cfg.Bool('correct_shifts_empirical') is True:
f = open(
'../tttgrid/tttgrid%s_%s_%s_%s_%s_emp.pkl' %
(phase, ttt_model, ev_emp.time, arrayname,
workdepth), 'rb')
print(
"loading travel time grid%s_%s_%s_%s_%s_emp.pkl"
% (phase, ttt_model, ev_emp.time, arrayname,
workdepth))
TTTGridMap_emp, mint_emp, maxt_emp = pickle.load(f)
f.close()
f = open(
'../tttgrid/tttgrid%s_%s_%s_%s_%s.pkl' %
(phase, ttt_model, ev.time, arrayname, workdepth),
'rb')
print(
"loading travel time grid%s_%s_%s_%s_%s.pkl" %
(phase, ttt_model, ev.time, arrayname, workdepth))
TTTGridMap, mint, maxt = pickle.load(f)
f.close()
print("loading of travel time grid sucessful")
except Exception:
print("loading of travel time grid unsucessful,\n \
will now calculate the grid:")
if isParallel:
maxp = 6
po = multiprocessing.Pool(maxp)
for i in xrange(len(FilterMeta)):
po.apply_async(ttt.calcTTTAdv,
(Config, FilterMeta[i], Origin,
i, arrayname, W, refshift))
po.close()
po.join()
else:
for i in xrange(len(FilterMeta)):
t1 = time.time()
ttt.calcTTTAdv(Config, FilterMeta[i], Origin,
i, arrayname, W, refshift,
phase)
Logfile.add('ttt.calcTTTAdv : ' +
str(time.time() - t1) + ' sec.')
assert len(FilterMeta) > 0
TTTGridMap = deserializer.deserializeTTT(
len(FilterMeta))
mint, maxt = deserializer.deserializeMinTMaxT(
len(FilterMeta))
f = open(
'../tttgrid/tttgrid%s_%s_%s_%s_%s.pkl' %
(phase, ttt_model, ev.time, arrayname, workdepth),
'wb')
print("dumping the traveltime grid for this array")
pickle.dump([TTTGridMap, mint, maxt], f)
f.close()
if cfg.Bool('correct_shifts_empirical') is True:
ttt.calcTTTAdv(Config, FilterMeta[i], Origin_emp,
i, arrayname, W, refshift, phase)
assert len(FilterMeta) > 0
TTTGridMap_emp = deserializer.deserializeTTT(
len(FilterMeta))
mint_emp, maxt_emp = deserializer.deserializeMinTMaxT(
len(FilterMeta))
f = open(
'../tttgrid/tttgrid%s_%s_%s_%s_%s_emp.pkl' %
(phase, ttt_model, ev_emp.time, arrayname,
workdepth), 'wb')
print("dumping the traveltime grid for this array")
pickle.dump([TTTGridMap_emp, mint_emp, maxt_emp],
f)
f.close()
t2 = time.time()
Logfile.red('%s took %0.3f s' % ('TTT', (t2 - t1)))
switch = filterindex
tw = times.calculateTimeWindows(mint, maxt, Config, ev,
switch)
if cfg.Bool('correct_shifts_empirical') is True:
tw_emp = times.calculateTimeWindows(
mint_emp, maxt_emp, Config, ev_emp, switch)
if cfg.pyrocko_download() is True:
if cfg.quantity() == 'displacement':
Wd_emp = waveform.readWaveformsPyrocko_restituted(
FilterMeta, tw, evpath, ev_emp, desired)
elif cfg.Bool('synthetic_test') is True:
Wd_emp = waveform.readWaveformsPyrockodummy(
FilterMeta, tw_emp, evpath_emp, ev_emp)
else:
Wd_emp = waveform.readWaveformsPyrocko(
FilterMeta, tw_emp, evpath_emp, ev_emp,
desired)
elif cfg.colesseo_input() is True:
Wd_emp = waveform.readWaveforms_colesseo(
FilterMeta, tw_emp, evpath_emp, ev_emp, C)
else:
Wd_emp = waveform.readWaveforms(
FilterMeta, tw_emp, evpath_emp, ev_emp)
if cfg.Bool('synthetic_test') is True\
or cfg.Bool('dynamic_filter') is True:
Wdf_emp = waveform.processdummyWaveforms(
Wd_emp, Config, Folder, arrayname, FilterMeta,
ev_emp, switch, W)
Wdfs_emp.extend(Wdf_emp)
else:
Wdf_emp = waveform.processWaveforms(
Wd_emp, Config, Folder, arrayname, FilterMeta,
ev_emp, switch, W)
Wdfs_emp.extend(Wdf_emp)
if cfg.pyrocko_download() is True:
if cfg.quantity() == 'displacement':
Wd = waveform.readWaveformsPyrocko_restituted(
FilterMeta, tw, evpath, ev, desired)
elif cfg.Bool('synthetic_test') is True:
Wd = waveform.readWaveformsPyrockodummy(
FilterMeta, tw, evpath, ev)
else:
Wd = waveform.readWaveformsPyrocko(
FilterMeta, tw, evpath, ev, desired)
elif cfg.colesseo_input() is True:
Wd = waveform.readWaveforms_colesseo(
FilterMeta, tw, evpath, ev, C)
else:
Wd = waveform.readWaveforms(FilterMeta, tw, evpath, ev)
if cfg.Bool('synthetic_test') is True\
or cfg.Bool('dynamic_filter') is True:
Wdf = waveform.processdummyWaveforms(
Wd, Config, Folder, arrayname, FilterMeta, ev,
switch, W)
Wdfs.extend(Wdf)
else:
Wdf = waveform.processWaveforms(
Wd, Config, Folder, arrayname, FilterMeta, ev,
switch, W)
Wdfs.extend(Wdf)
C.writeStationFile(FilterMeta, Folder, counter)
Logfile.red('%d Streams added for Processing' % (len(Wd)))
t1 = time.time()
f = open(
'../tttgrid/tttgrid%s_%s_%s_%s_%s.pkl' %
(phase, ttt_model, ev.time, arrayname, workdepth),
'rb')
TTTGridMap, mint, maxt = pickle.load(f)
f.close()
if switch == 0:
step = cfg.step()
if switch == 1:
step = cfg.step_f2()
if cfg.UInt('forerun') > 0:
ntimes = int(
(cfg.UInt('forerun') + cfg.UInt('duration')) /
step)
else:
ntimes = int((cfg.UInt('duration')) / step)
if cfg.Bool('combine_all') is False:
if cfg.optimize() is True:
optim.solve(counter, Config, Wdf, FilterMeta, mint,
maxt, TTTGridMap, Folder, Origin,
ntimes, switch, ev, arrayfolder,
syn_in, refshifts, phase,
rpe + str(arrayname), flag_rpe)
else:
if cfg.Bool('correct_shifts_empirical') is True:
if cfg.Bool('correct_shifts_empirical_run'
) is True:
f = open(
'../tttgrid/tttgrid%s_%s_%s_%s_%s_emp.pkl'
% (phase, ttt_model, ev_emp.time,
arrayname, workdepth), 'rb')
TTTGridMap_emp, mint_emp, maxt_emp = pickle.load(
f)
f.close()
flag_rpe = True
arraySemb, weight, array_center = sembCalc.doCalc(
counter, Config, Wdf_emp, FilterMeta,
mint, maxt, TTTGridMap_emp, Folder,
Origin, ntimes, switch, ev_emp,
arrayfolder, syn_in, refshifts, phase,
rpe + str(arrayname), flag_rpe)
if sys.version_info.major >= 3:
f = open(rpe + str(arrayname), 'rb')
else:
f = open(rpe + str(arrayname))
RefDict_empirical = pickle.load(f)
refshifts = RefDict_empirical
for j in range(0, len(FilterMeta)):
if cfg.correct_shifts() is False:
refshifts[j] = refshifts[j] * 0.
flag_rpe = False
arraySemb, weight, array_center = sembCalc.doCalc(
counter, Config, Wdf, FilterMeta, mint, maxt,
TTTGridMap, Folder, Origin, ntimes, switch, ev,
arrayfolder, syn_in, refshifts, phase,
rpe + str(arrayname), flag_rpe)
weights.append(weight)
array_centers.append(array_center)
ASL.append(arraySemb)
sembCalc.writeSembMatricesSingleArray(
arraySemb, Config, Origin, arrayfolder, ntimes,
switch, phase)
fileName = os.path.join(arrayfolder, 'stations.txt')
Logfile.add('Write to file ' + fileName)
fobjarraynetwork = open(fileName, 'w')
for i in FilterMeta:
fobjarraynetwork.write(
('%s %s %s\n') % (i.getName(), i.lat, i.lon))
fobjarraynetwork.close()
t2 = time.time()
Logfile.add('CALC took %0.3f sec' % (t2 - t1))
counter += 1
stations_per_array.append(len(FilterMeta))
TTTgrids.update(TTTGridMap)
mints.append(mint)
maxts.append(maxt)
FilterMetas[len(FilterMetas):] = FilterMeta
TTTGridMap = []
if cfg.Bool('combine_all') is True:
if cfg.pyrocko_download() is True:
if cfg.Bool('synthetic_test') is True:
Wd = waveform.readWaveformsPyrockodummy(
FilterMetas, tw, evpath, ev)
else:
if cfg.quantity() == 'displacement':
Wd = waveform.readWaveformsPyrocko_restituted(
FilterMetas, tw, evpath, ev, desired)
else:
Wd = waveform.readWaveformsPyrocko(
FilterMetas, tw, evpath, ev, desired)
elif cfg.colesseo_input() is True:
Wd = waveform.readWaveforms_colesseo(
FilterMetas, tw, evpath, ev, C)
else:
Wd = waveform.readWaveforms(FilterMetas, tw, evpath,
ev)
if cfg.Bool('synthetic_test') is True:
Wdf = waveform.processdummyWaveforms(
Wd, Config, Folder, arrayname, FilterMetas, ev,
switch, W)
else:
Wdf = waveform.processWaveforms(
Wd, Config, Folder, arrayname, FilterMetas, ev,
switch, W)
mint = num.min(mints)
maxt = num.max(maxts)
flag_rpe = False
if cfg.Bool('bootstrap_array_weights') is False:
arraySemb, weight, array_center = sembCalc.doCalc(
counter, Config, Wdf, FilterMetas, mint, maxt,
TTTgrids, Folder, Origin, ntimes, switch, ev,
arrayfolder, syn_in, refshifts_global, phase,
rpe + str(arrayname), flag_rpe)
ASL.append(arraySemb)
weights.append(weight)
array_centers.append(array_center)
sembCalc.writeSembMatricesSingleArray(
arraySemb, Config, Origin, arrayfolder, ntimes,
switch, phase)
else:
nboot = cfg.Int('n_bootstrap')
tmp_general = 1
for ibootstrap in range(nboot):
f = rstate.uniform(0., 1., size=counter + 1)
f = num.sort(f)
g = f[1:] - f[:-1]
k = 0
ws = []
for wss in range(0, counter - 1):
for stats in range(0, stations_per_array[k]):
ws.append(g[k])
k = +1
ws = num.asarray(ws)
arraySemb, weight, array_center = sembCalc.doCalc(
counter,
Config,
Wdf,
FilterMetas,
mint,
maxt,
TTTgrids,
Folder,
Origin,
ntimes,
switch,
ev,
arrayfolder,
syn_in,
refshifts_global,
phase,
rpe + str(arrayname),
flag_rpe,
bs_weights=ws)
ASL.append(arraySemb)
weights.append(weight)
array_centers.append(array_center)
sembCalc.writeSembMatricesSingleArray(
arraySemb,
Config,
Origin,
arrayfolder,
ntimes,
switch,
phase,
bootstrap=ibootstrap)
if ASL:
Logfile.red('collect semblance matrices from\
all arrays')
sembmax, tmp = sembCalc.collectSemb(
ASL,
Config,
Origin,
Folder,
ntimes,
len(networks),
switch,
array_centers,
phase,
cboot=ibootstrap)
tmp_general *= tmp
ASL = []
sembmax, tmp = sembCalc.collectSemb(
ASL,
Config,
Origin,
Folder,
ntimes,
len(networks),
switch,
array_centers,
phase,
cboot=None,
temp_comb=tmp_general)
if cfg.optimize_all() is True:
import optim_csemb
sembmax, tmp = sembCalc.collectSemb(
ASL, Config, Origin, Folder, ntimes, len(networks),
switch)
optim_csemb.solve(counter, Config, Wdf, FilterMeta, mint,
maxt, TTTGridMap, Folder, Origin, ntimes,
switch, ev, arrayfolder, syn_in, ASL,
sembmax, evpath, XDict, RefDict,
workdepth, filterindex, Wdfs)
if ASL and cfg.Bool('bootstrap_array_weights') is False:
Logfile.red('collect semblance matrices from all arrays')
sembmax, tmp = sembCalc.collectSemb(
ASL, Config, Origin, Folder, ntimes, len(networks),
switch, array_centers, phase)
if cfg.Bool('weight_by_noise') is True:
sembCalc.collectSembweighted(ASL, Config, Origin,
Folder, ntimes,
len(networks), switch,
weights)
else:
Logfile.red('Nothing to do -> Finish')
print("last work depth:")
print(workdepth)
def writeSembMatricesSingleArray(SembList, Config, Origin, arrayfolder, ntimes,
switch):
'''
method to write semblance matrizes from one processes to file for each timestep
'''
logger.info('start write semblance matrices')
cfg = ConfigObj(dict=Config)
origin = OriginCfg(Origin)
dimX = cfg.dimX() # ('dimx')
dimY = cfg.dimY() # ('dimy')
winlen = cfg.winlen() # ('winlen')
step = cfg.step() # ('step')
latv = []
lonv = []
gridspacing = cfg.Float('gridspacing')
migpoints = dimX * dimY
o_lat = origin.lat() # float (Origin['lat'])
o_lon = origin.lon() # float (Origin['lon'])
oLatul = 0
oLonul = 0
z = 0
for i in xrange(dimX):
oLatul = o_lat - ((dimX - 1) / 2) * gridspacing + i * gridspacing
if z == 0 and i == 0:
Latul = oLatul
o = 0
for j in xrange(dimY):
oLonul = o_lon - ((dimY - 1) / 2) * gridspacing + j * gridspacing
if o == 0 and j == 0: Lonul = oLonul
latv.append(oLatul)
lonv.append(oLonul)
#endfor
rc = UTCDateTime(Origin['time'])
rcs = '%s-%s-%s_%02d:%02d:%02d' % (rc.day, rc.month, rc.year, rc.hour,
rc.minute, rc.second)
d = rc.timestamp
for a, i in enumerate(SembList):
#logger.info('timestep %d' % a)
fobj = open(
os.path.join(arrayfolder,
'%s-%s_%03d.ASC' % (switch, Origin['depth'], a)), 'w')
fobj.write('# %s , %s\n' % (d, rcs))
fobj.write('# step %ds| ntimes %d| winlen: %ds\n' %
(step, ntimes, winlen))
fobj.write('# \n')
fobj.write('# southwestlat: %.2f dlat: %f nlat: %f \n' %
(Latul, gridspacing, dimX))
fobj.write('# southwestlon: %.2f dlon: %f nlon: %f \n' %
(Lonul, gridspacing, dimY))
fobj.write('# ddepth: 0 ndepth: 1 \n')
for j in range(migpoints):
x = latv[j]
y = lonv[j]
z = origin.depth() # float(Origin['depth'])
semb = i[j]
fobj.write('%.2f %.2f %.2f %.20f\n' % (x, y, z, semb))
#endfor
fobj.close()
def processLoop():
#==================================get meta info==========================================
C = config.Config(evpath)
Origin = C.parseConfig('origin')
Config = C.parseConfig('config')
Meta = C.readMetaInfoFile()
#==================================get meta info==========================================
#==================================do prerequiries========================================
Folder = C.createFolder()
C.cpSkeleton(Folder, Config)
C.writeConfig(Config, Origin, Folder)
cfg = ConfigObj(dict=Config)
filter = FilterCfg(Config)
origin = OriginCfg(Origin)
ntimes = int(
(cfg.UInt('forerun') + cfg.UInt('duration')) / cfg.UInt('step'))
default = 0
strike = origin.strike(default) # Origin.get ('strike', default)
dip = origin.dip(default) # Origin.get ('dip', default)
rake = origin.rake(default) # Origin.get ('rake', default)
# ev = Event (Origin['lat'],Origin['lon'],Origin['depth'],Origin['time'],
# strike = strike,dip=dip,rake=rake)
ev = Event(origin.lat(),
origin.lon(),
origin.depth(),
origin.time(),
strike=strike,
dip=dip,
rake=rake)
filtername = filter.filterName()
Logfile.add('filtername = ' + filtername)
#todo crosscorrelation for all arrays before processing
XDict = {}
RefDict = {}
SL = {}
if cfg.Int('xcorr') == 1:
newFreq = str(filter.newFrequency())
fobjreferenceshiftname = newFreq + '_' + filtername + '.refpkl'
rp = os.path.join(Folder['semb'], fobjreferenceshiftname)
fobjpickleshiftname = newFreq + '_' + filtername + '.xcorrpkl'
ps = os.path.join(Folder['semb'], fobjpickleshiftname)
if (os.path.isfile(rp) and os.path.getsize(rp) != 0
and os.path.isfile(ps) and os.path.getsize(ps) != 0):
Logfile.add('file exits : ' + rp)
Logfile.add('load refshifts')
f = open(rp)
RefDict = pickle.load(f)
x = open(ps)
XDict = pickle.load(x)
#xcorrnetworks = Config['networks'].split(',')
xcorrnetworks = cfg.String('networks').split(',')
for i in xcorrnetworks:
SL[i] = len(Config[i].split('|'))
else:
SL = {}
xcorrnetworks = cfg.String('networks').split(',')
for i in xcorrnetworks:
W = {}
refshift = 0
network = cfg.String(i).split('|')
FilterMeta = ttt.filterStations(Meta, Config, Origin, network)
arrayfolder = os.path.join(Folder['semb'], i)
if os.access(arrayfolder, os.F_OK) == False:
os.makedirs(arrayfolder)
A = Xcorr(ev, FilterMeta, evpath, Config, arrayfolder)
W, triggerobject = A.runXcorr()
XDict[i] = W
RefDict[i] = triggerobject.tdiff
SL[i] = len(network)
#endfor
fobjrefshift = open(rp, 'w')
pickle.dump(RefDict, fobjrefshift)
fobjrefshift.close()
output = open(ps, 'w')
pickle.dump(XDict, output)
output.close()
#endif
for i in sorted(XDict.iterkeys()):
Logfile.red('Array %s has %3d of %3d Stations left' %
(i, len(XDict[i]), SL[i]))
logger.info(
'\033[31mFor proceeding without changes press enter or give new comma seperatet network list or quit for exit\033[0m'
)
while True:
nnl = raw_input("please enter your choice: ")
#Logfile.add ('Choise = ' + nnl)
if len(nnl) == 0:
if not Basic.question('Process all networks ?'): continue
Logfile.red('This networks will be used for processing: %s' %
(Config['networks']))
break
elif str(nnl) == 'quit':
sys.exit()
elif str(nnl) == 'rerun':
event = os.path.join(*evpath.split('/')[-1:])
try:
os.remove(rp)
os.remove(ps)
except:
pass
mainfolder = os.path.join(os.path.sep, *evpath.split('/')[:-2])
os.chdir(mainfolder)
cmd = ('%s arraytool.py process %s') % (sys.executable, event)
Logfile.add('cmd = ' + cmd)
os.system(cmd)
sys.exit()
else:
# Check if selected array(s) exists
names = nnl.split(',')
isOk = True
for array in names:
arrayfolder = os.path.join(Folder['semb'], array)
if not os.path.isdir(arrayfolder):
Logfile.error('Illegal network name ' + str(array))
isOk = False
break
#endfor
if not isOk: continue # Illegal network : input again
# use these networks
Logfile.add('This networks will be used for processing: %s' %
(nnl))
Config['networks'] = nnl
break
#endif
#endwhile True
for i in range(3, 0, -1):
time.sleep(1)
Logfile.red('Start processing in %d seconds ' % (i))
#endfor
#print XDict
#print RefDict
#TriggerOnset.append(triggerobject)
#print 'MAINTDIFF ',triggerobject.tdiff
#sys.exit()
wd = Origin['depth']
start, stop, step = cfg.String('depths').split(',')
start = int(start)
stop = int(stop) + 1
step = int(step)
Logfile.add('working on ' + Config['networks'])
#==================================loop over depth=======================================
for depthindex in xrange(start, stop):
workdepth = float(wd) + depthindex * step
Origin['depth'] = workdepth
ev = Event(Origin['lat'],
Origin['lon'],
Origin['depth'],
Origin['time'],
strike=strike,
dip=dip,
rake=rake)
Logfile.add('WORKDEPTH: ' + str(Origin['depth']))
#==================================do prerequiries========================================
#==================================loop over arrays=======================================
ASL = []
networks = Config['networks'].split(',')
counter = 1
TriggerOnset = []
for i in networks:
arrayname = i
arrayfolder = os.path.join(Folder['semb'], arrayname)
network = Config[i].split('|')
Logfile.add('network: ' + str(network))
FilterMeta = ttt.filterStations(Meta, Config, Origin, network)
#if len(FilterMeta) < 3: continue #hs : wieder rein
if len(FilterMeta) < 3: continue
W = XDict[i]
refshift = RefDict[i]
FilterMeta = cmpFilterMetavsXCORR(W, FilterMeta)
#print 'W: ',W,len(W); print refshift; print 'FM ',FilterMeta,len(FilterMeta)
Logfile.add(
'BOUNDING BOX DIMX: %s DIMY: %s GRIDSPACING: %s \n' %
(Config['dimx'], Config['dimy'], Config['gridspacing']))
##############=======================PARALLEL===========================================
Logfile.red('Calculating Traveltime Grid')
t1 = time.time()
if WINDOWS:
isParallel = False #hs : parallel
#else : isParallel = True #10.12.2015
else:
isParallel = False #10.12.2015
if isParallel: #hs
maxp = int(Config['ncore'])
#maxp = 20 #hs
po = multiprocessing.Pool(maxp)
for i in xrange(len(FilterMeta)):
po.apply_async(ttt.calcTTTAdv,
(Config, FilterMeta[i], Origin, i,
arrayname, W, refshift))
po.close()
po.join()
else: #hs+
for i in xrange(len(FilterMeta)):
t1 = time.time()
ttt.calcTTTAdv(Config, FilterMeta[i], Origin, i, arrayname,
W, refshift)
Logfile.add('ttt.calcTTTAdv : ' + str(time.time() - t1) +
' sec.')
#endif #hs-
assert len(FilterMeta) > 0
TTTGridMap = deserializer.deserializeTTT(len(FilterMeta))
mint, maxt = deserializer.deserializeMinTMaxT(len(FilterMeta))
t2 = time.time()
Logfile.red('%s took %0.3f s' % ('TTT', (t2 - t1)))
#sys.exit()
##############=======================SERIELL===========================================
tw = times.calculateTimeWindows(mint, maxt, Config, ev)
Wd = waveform.readWaveforms(FilterMeta, tw, evpath, ev)
Wdf = waveform.processWaveforms(Wd, Config, Folder, arrayname,
FilterMeta, ev, W)
#sys.exit()
C.writeStationFile(FilterMeta, Folder, counter)
Logfile.red('%d Streams added for Processing' % (len(Wd)))
##############=========================================================================
##############=======================PARALLEL==========================================
t1 = time.time()
arraySemb = sembCalc.doCalc(counter, Config, Wdf, FilterMeta, mint,
maxt, TTTGridMap, Folder, Origin,
ntimes)
t2 = time.time()
Logfile.add('CALC took %0.3f sec' % (t2 - t1))
ASL.append(arraySemb)
counter += 1
sembCalc.writeSembMatricesSingleArray(arraySemb, Config, Origin,
arrayfolder, ntimes)
fileName = os.path.join(arrayfolder, 'stations.txt')
Logfile.add('Write to file ' + fileName)
fobjarraynetwork = open(fileName, 'w')
for i in FilterMeta:
fobjarraynetwork.write(
('%s %s %s\n') % (i.getName(), i.lat, i.lon))
fobjarraynetwork.close()
if ASL:
Logfile.red('collect semblance matrices from all arrays')
sembCalc.collectSemb(ASL, Config, Origin, Folder, ntimes,
len(networks))
#fobjtrigger = open (os.path.join(Folder['semb'],'Trigger.txt'),'w')
#for i in TriggerOnset:
# print i.aname,i.sname,i.ttime
# fobjtrigger.write(('%s %s %s\n')%(i.aname,i.sname,i.ttime))
#fobjtrigger.close()
else:
Logfile.red('Nothing to do -> Finish')