def checkKeys(conf, keyList, optional=False):
if type(keyList) is str:
list1 = list(keyList)
else:
list1 = keyList
if not optional:
Basic.checkExistsKeys(conf, list1, isAbort=True)
eventDir = Globals.EventDir()
isOk = True
for key in list1:
val = conf[key]
msg = None
if key == duration:
msg = Basic.checkGreaterZero(val)
elif key in [mindist, maxdist]:
msg = Basic.checkNotNegative(val)
elif key in [keyfilefolder, metaCatalog]:
Basic.checkExistsDir(os.path.join(eventDir, val), isAbort=True)
elif key in [blacklist, mail, pwd]:
continue
if msg is not None:
isOk = Logfile.error('Key <' + key + '> in config file: ' + msg)
if not isOk:
Logfile.abort()
return True
def filterclusterStationMinimumNumber(CentroidList, StationclusterList,
Config):
newCentroidList = []
newStationclusterList = []
for i in CentroidList:
counter = 0
for j in StationclusterList:
if i.rank == j.member:
counter += 1
streamID = j.net + '.' + j.sta + '.' + j.loc + '.' + j.comp
if counter < int(Config['minclusterstation']): s1 = 'OUT'
else:
s1 = 'IN '
newCentroidList.append(i)
Logfile.red('Centroid %s %d %s %.2f %5.2f' %
(i.rank, counter, s1, i.lat, i.lon))
for i in newCentroidList:
for j in StationclusterList:
if i.rank == j.member:
newStationclusterList.append(j)
return newStationclusterList, newCentroidList
def createFolder(EventPath):
Folder = {}
Logfile.red('Create working environment')
Folder['cluster'] = os.path.join(EventPath, 'cluster')
if os.access(Folder['cluster'], os.F_OK) is False:
os.makedirs(Folder['cluster'])
if os.access(os.getcwd(), os.W_OK):
basedir = os.path.join(EventPath, 'work')
sembdir = os.path.join(basedir, 'semblance')
ascdir = os.path.join(basedir, 'asc')
mseeddir = os.path.join(basedir, 'mseed')
Folder['base'] = basedir
Folder['semb'] = sembdir
Folder['asc'] = ascdir
Folder['mseed'] = mseeddir
Folder['event'] = EventPath
else:
Logfile.abort('create Folder: No write permissions for ' + os.getcwd())
Folder['config'] = os.path.join(os.getcwd(), 'skeleton')
return Folder
def signoise(self, Waveform, ttime, path):
st = str(self.Origin.time)[:-1]
ponset = UTCDateTime(st) + ttime
winnoise_start = Waveform.stats.starttime + 20
winnoise_end = ponset - 10
winsig_start = ponset - 2
winsig_end = ponset + 10
try:
winnoise = read(path,
format="MSEED",
starttime=winnoise_start,
endtime=winnoise_end,
nearest_sample=True)
winsig = read(path,
format="MSEED",
starttime=winsig_start,
endtime=winsig_end,
nearest_sample=True)
except Exception:
Logfile.exception('signoise')
psignal = abs(winsig.max()[0])
pnoise = abs(winnoise.max()[0])
signoise = float(psignal) / float(pnoise)
return signoise
def checkExistsDir(dirName, isAbort=False):
if os.path.isdir(dirName):
return True
Logfile.error('Cannot find directory', dirName)
if isAbort:
Logfile.abort()
return False
def checkFileExists(fileName, isAbort=False):
if os.path.isfile(fileName):
return True
Logfile.fileOpenError(fileName)
if isAbort:
Logfile.abort('')
else:
return False
def formatStrings(strings, format1):
result = []
try:
for i in range(len(strings)):
result.append(format1 % (strings[i]))
except Exception:
Logfile.exception('formatStrings', 'Illegal format', abortProg=True)
return result
def execsemblance(nostat, nsamp, i, nstep, dimX,dimY, mint, new_freq, minSampleCount) :
f = [nostat, nsamp, i, nstep, dimX,dimY, mint, new_freq, minSampleCount]
args = Basic.floatToString(f, delim= ',')
prog = sys.executable + ' ' + __file__
cmd = prog + ' ' + args
Logfile.add('--------------------------------------------', cmd)
result = Basic.systemCmd(cmd)
Logfile.addLines(result)
backSemb = Basic.readVector(semb_txt)
return backSemb
def checkExistsKeys(dict, keyList, isAbort=False):
isOk = True
for key in keyList:
if not key in dict:
isOk = Logfile.error('Key <' + str(key) +
'> missing in config file')
if isOk:
return True
elif isAbort:
Logfile.abort()
return False
def loadDump(fileName):
if not os.path.isfile(fileName):
Logfile.fileOpenError(fileName)
return None
if sys.version_info.major >= 3:
f = open(fileName, 'rb')
else:
f = open(fileName)
data = pickle.load(f)
f.close()
os.remove(fileName)
return data
def createDirectory(dirName, optional=False):
if os.path.isdir(dirName):
return True
os.makedirs(dirName)
if os.path.isdir(dirName):
return True
else:
Logfile.error('Cannot open directory', dirName)
if not optional:
Logfile.abort()
return False
def init(configFileName = None):
global EventDir, ProtFileDir, ConfigDict, isDebug
ProtFileDir = os.path.join(EventDir(), 'tmp1')
if True: # not isClient:
ConfigDict = ConfigFile.readGlobalConf(configFileName)
if ConfigDict == None: return False
key = 'DEBUG_FLAG'
if not isClient:
if isDebug: Logfile.add('Debugging is on')
return True
def removeTempFiles():
dir = '/tmp'
names = os.listdir(dir)
user = getpass.getuser()
cnt = 0
for s in names:
if s.startswith('obspy-'):
os.remove(dir + '/' + s)
cnt += 1
if cnt != 0:
Logfile.add('Remove ' + str(cnt) + ' temp files obspy-*.*')
return cnt
def filterStations(StationList, Config, Origin):
F = []
cfg = ConfigObj(dict=Config)
minDist, maxDist = cfg.FloatRange('mindist', 'maxdist')
origin = DataTypes.dictToLocation(Origin)
Logfile.red('Filter stations with configured parameters')
for i in StationList:
sdelta = loc2degrees(origin, i)
if sdelta > minDist and sdelta < maxDist:
F.append(
Station(i.net, i.sta, i.loc, i.comp, i.lat, i.lon, i.ele,
i.dip, i.azi, i.gain))
Logfile.red('%d STATIONS LEFT IN LIST' % len(F))
return F
def writeWaveform(Folder, station, Stream, flag, network):
if flag == 'U':
s1 = 'unfiltered'
elif flag == 'F':
s1 = 'filtered'
elif flag == 'R':
s1 = 'resampled'
else:
Logfile.abort('writeWaveform: Illegal flag <' + flag + '>')
fname = ('%s-%s-%s.mseed') % (network, station, flag)
name = os.path.join(Folder['mseed'], fname)
Stream.write(name, format='MSEED')
Logfile.add('%s stream for station %s written ' % (s1, station))
def filterStations(StationList, cfg, Origin):
F = []
minDist = cfg.config_cluster.minDist
maxDist = cfg.config_cluster.maxDist
origin = DataTypes.dictToLocation(Origin)
Logfile.red('Filter stations with configured parameters')
for i in StationList:
sdelta = loc2degrees(origin, i)
if sdelta > minDist and sdelta < maxDist:
F.append(
Station(i.net, i.sta, i.loc, i.comp, i.lat, i.lon, i.ele,
i.dip, i.azi, i.gain))
Logfile.red('%d STATIONS LEFT IN LIST' % len(F))
return F
def compareclusterCentre(oldcluster, newcluster, Config):
counter = 0
for i in range(int(Config['maxcluster'])):
if newcluster[i].rank == -1:
continue
delta = loc2degrees(oldcluster[i], newcluster[i])
msg = str(i) + ' OLD: ' + str(oldcluster[i].lat) + ' ' + str(
oldcluster[i].lon)
msg += ' NEW: ' + str(newcluster[i].lat) + ' ' + str(
newcluster[i].lon) + ' DELTA: ' + str(delta)
Logfile.add(msg)
if delta < float(Config['centroidmindistance']):
counter += 1
return counter
def cmpFilterMetavsXCORR(XcorrMeta, StationMetaList):
FilterList = []
for i in StationMetaList:
if sys.version_info.major >= 3:
for j in sorted(XcorrMeta.keys()):
if i.getName() == j:
FilterList.append(i)
else:
for j in XcorrMeta.keys():
if i.getName() == j:
FilterList.append(i)
n1 = len(FilterList)
n2 = len(StationMetaList)
Logfile.red('Xcorr Procedure finished %d of %d stations \
left for processing' % (n1, n2))
return FilterList
def filterWaveform(self, Waveform, cfg):
Logfile.red('Filter Waveform: ')
new_frequence = (cfg.config_filter.newFrequency)
st = Stream()
for i in Waveform:
Logfile.red('Downsampling to %s: from %d' % (new_frequence,
i.stats.sampling_rate))
j = i.resample(new_frequence)
switch = cfg.config_filter.filterswitch
Logfile.add('bandpass filtered \
stream for station %s ' % (i))
j.filter('bandpass',
freqmin=cfg.config_filter.flo[switch-1],
freqmax=cfg.config_filter.fhi[switch-1],
corners=cfg.config_filter.ns[switch-1],
zerophase=False)
st.append(j)
return st
def kmean(Config, inputCentroid, FilterMeta, counter, Folder, Origin, flag):
counter += 1
Logfile.add('COUNTER ' + str(counter) + ' CUTOFF ' + Config['cutoff'])
cfg = ConfigObj(dict=Config)
scl = stationBelongTocluster(Config, inputCentroid, FilterMeta)
acounter = 1
for a in inputCentroid:
for i in scl:
if acounter == i.member:
delta = loc2degrees(i, a)
if delta > cfg.Float('initialstationdistance'):
i.member = -1
acounter += 1
if counter == cfg.UInt('cutoff'):
endcheck(inputCentroid, FilterMeta, Config, Folder, Origin, flag)
sys.exit()
nsc = calculateclusterCentre(Config, scl)
t = compareclusterCentre(inputCentroid, nsc, Config)
Logfile.add('ITERATIONSTEP: ---> ' + str(counter) +
' <-----------------------------')
while t < cfg.UInt('maxcluster'):
Logfile.add('number of arrays in KMEAN: ' + str(t))
kmean(Config, nsc, FilterMeta, counter, Folder, Origin, flag)
endcheck(inputCentroid, FilterMeta, Config, Folder, Origin, flag)
sys.exit()
def readURL(url, tmpFile=None):
lines = []
try:
datasource = urlopen(url)
while True:
line = datasource.readline()
if line == '':
break
lines.append(line)
if len(lines) == 0:
Logfile.error('Cannot read from URL:', url)
lines = []
Logfile.add(str(len(lines)) + ' lines read from URL:', url)
if tmpFile:
writeTextFile(tmpFile, lines)
except:
Logfile.exception('readURL')
return lines
def existsHTML_Page(url, text=None, withComment=False, isAbort=False):
h = httplib2.Http()
try:
resp = h.request(url, 'HEAD')
status = int(resp[0]['status'])
if status < 400:
return True # page exists
except:
status = 104 # error: [Errno 104] Connection reset by peer ???
if withComment:
s = 'HTML page not found, error = ' + str(status)
if text is None:
Logfile.error(s, url)
else:
Logfile.error(text, s, url)
if isAbort:
Logfile.abort()
return False
def printBestSolution(solution):
maxline = -100
L = []
M = []
print(solution.path)
solution.path = solution.path[1:]
Logfile.add('eventpath: ', os.path.join(solution.path, 'event.stations'))
fobj = open(os.path.join(solution.path, 'event.stations'), 'r')
for line in fobj:
line = line.split()
M.append(float(line[3]))
L.append(
BestSolution(line[0], float(line[3]), float(line[1]),
float(line[2])))
fobj.close()
maxline = max(M)
C = {}
fobjarrays = open(
os.path.join(os.path.join(*solution.path.split('/')[:-2]),
'arraycluster.dat'), 'w')
for i in range(int(maxline) + 1):
array = ''
for j in L:
if int(j.cluster) == i:
array += j.station + '|'
ar = 'r' + str(i) + '=' + array[:-1] + '\n'
C[i] = array
print(ar, len(ar))
fobjarrays.write(ar)
fobjarrays.close()
return C
def filterStations(StationList, Config, Origin, network, cfg_yaml):
F = []
cfg = ConfigObj(dict=Config)
minDist = cfg_yaml.config_cluster.minDist
maxDist = cfg_yaml.config_cluster.maxDist
origin = Location(Origin['lat'], Origin['lon'])
Logfile.red('Filter stations with configured parameters...')
for j in network:
for i in StationList:
if str(i.getcmpName()[:-2]) == str(j) or str(
i.getcmpName()[:]) == str(j):
pos = Location(i.lat, i.lon)
sdelta = loc2degrees(origin, pos)
if sdelta > minDist and sdelta < maxDist:
s = Station(i.net, i.sta, i.loc, i.comp, i.lat, i.lon,
i.ele, i.dip, i.azi, i.gain)
if s not in F:
F.append(s)
Logfile.red('%d STATIONS LEFT IN LIST' % len(F))
return F
def filterBestSolution(solution):
evp = os.path.join('/',*solution.path.split('/')[:-2])
C= Config(evp)
Conf = C.parseConfig('config')
cfg = ConfigObj(dict=Conf)
SL = []
M= []
fobj = open(os.path.join(solution.path, 'event.stations'),'r')
for s in fobj:
try:
line = s.split()
net,sta,loc,comp = line[0].split('.')
slat= line[1]
slon= line[2]
smember = line[3]
M.append(smember)
SL.append(Station(net,sta,loc,comp,lat=slat,lon=slon,member=smember))
except:
Logfile.exception('filterBestSolution', '<' + s + '>')
continue
fobj.close()
M = list(set(M))
Logfile.add('number of clusters ' + str(len(M)),
'number of stations ' + str(len(SL)))
kd = obs_kilometer2degrees(cfg.Distance('intraclusterdistance'))
Logfile.add('icd ' + str(kd))
maxdist = -1
for i in SL:
counter = 0
for k in SL:
if i.member == '8' and k.member == '8':
if i.getName() != k.getName():
delta = loc2degrees(i, k)
if delta > maxdist:
maxdist = delta
if delta < kd:
counter +=1
print(i, 'less then allowd ', counter)
print('masxdist ', maxdist)
def readMetaInfoFile(EventPath):
Logfile.red('Parsing MetaInfoFile')
try:
for i in os.listdir(EventPath):
if fnmatch.fnmatch(i, 'metainfo-*.meta'):
evfile = os.path.join(EventPath, i)
MetaL = []
Logfile.add(evfile)
fobj = open(evfile, 'r')
for i in fobj:
line = i.split()
net = line[0]
sta = line[1]
loc = line[2]
comp = line[3]
lat = line[4]
lon = line[5]
ele = line[6]
dip = line[7]
azi = line[8]
gain = line[9]
if fnmatch.fnmatch(comp, 'Z'):
MetaL.append(
Station(net, sta, loc, comp, lat, lon, ele, dip, azi,
gain))
Logfile.red('%d ENTRIES IN METAFILE FOUND' % (len(MetaL)))
except Exception:
Logfile.red('METAFILE NOT READABLE')
return MetaL
def run(self):
Logfile.init(self.runTimeLog, self.errorLog)
Logfile.setStartMsg(self.version)
if not self.extern.init():
Logfile.abort()
try:
ret = self.extern.process()
if ret:
msg = 'Palantiri finished'
else:
msg = 'Palantiri finished with error - maybe Sauron looked back?'
except KeyboardInterrupt:
msg = 'Gandalf made Pippin drop the Palantiri by Control C'
ret = False
self.extern.finish()
Logfile.showLabel(msg)
return ret
def filterWaveform(self, Waveform):
Logfile.red('Filter Waveform: ')
cfg = FilterCfg(self.Config)
new_frequence = (cfg.newFrequency())
st = Stream()
for i in Waveform:
Logfile.red('Downsampling to %s: from %d' %
(new_frequence, i.stats.sampling_rate))
j = i.resample(new_frequence)
switch = cfg.filterswitch()
if switch == 1:
Logfile.add('bandpass filtered \
stream for station %s ' % (i))
j.filter('bandpass',
freqmin=cfg.flo(),
freqmax=cfg.fhi(),
corners=cfg.ns(),
zerophase=bool(self.Config['zph']))
elif switch == 2:
Logfile.add('bandpass filtered \
stream for station %s ' % (i))
j.filter('bandpass',
freqmin=cfg.flo2(),
freqmax=cfg.fhi2(),
corners=cfg.ns2(),
zerophase=bool(self.Config['zph']))
st.append(j)
return st
def calculateTimeWindows(self, mint):
tw = {}
st = str(self.Origin.time)[:-1]
tw['start'] = UTCDateTime(UTCDateTime(st) + (mint - 100))
tw['end'] = tw['start'] + 200
tw['xcorrstart'] = UTCDateTime(UTCDateTime(st)
+ (mint - self.mintforerun))
tw['xcorrend'] = tw['xcorrstart'] + 40
Logfile.add(' ORIGIN TIME %s' % UTCDateTime(self.Origin.time))
Logfile.add(' OVERALL TIME WINDOW: %s - %s' % (tw['start'], tw['end']))
Logfile.add(' XCROSS TIME WINDOW: %s - %s' % (tw['xcorrstart'],
tw['xcorrend']))
return tw
def readWaveforms(stationList, tw, EventPath, Origin):
t2 = UTCDateTime(Origin.time)
sdspath = os.path.join(EventPath, 'data', str(t2.year))
Wdict = OrderedDict()
for i in stationList:
streamData = i.getName() + '.D.' + str(t2.year)\
+ '.' + str("%03d" % t2.julday)
entry = os.path.join(sdspath, i.net, i.sta, i.comp + '.D', streamData)
tdiff = tw['end'] - tw['start']
try:
st = read(entry,
format="MSEED",
starttime=tw['start'],
endtime=tw['end'],
nearest_sample=True)
except Exception:
Logfile.error('readWaveforms: File not found', entry)
pass
if len(st.get_gaps()) > 0:
st.merge(method=0,
fill_value='interpolate',
interpolation_samples=0)
if len(st) > 0:
trdiff = st[0].stats.endtime - st[0].stats.starttime
totaldiff = abs(trdiff - tdiff)
if totaldiff < 1:
Wdict[i.getName()] = st
Logfile.add(i.getName() + ' added to StreamList ')
else:
print(' OUT ', streamData)
Logfile.red('%d Streams added with available Data' % len(Wdict))
return Wdict