def main():
logging.basicConfig(
level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
)
logging.info("*** Starting: Compute CC ***")
# Connection to the DB
db = connect()
if len(get_filters(db, all=False)) == 0:
logging.info("NO FILTERS DEFINED, exiting")
sys.exit()
# Get Configuration
params = Params()
params.goal_sampling_rate = float(get_config(db, "cc_sampling_rate"))
params.goal_duration = float(get_config(db, "analysis_duration"))
params.overlap = float(get_config(db, "overlap"))
params.maxlag = float(get_config(db, "maxlag"))
params.min30 = float(get_config(db, "corr_duration")) * params.goal_sampling_rate
params.windsorizing = float(get_config(db, "windsorizing"))
params.resampling_method = get_config(db, "resampling_method")
params.decimation_factor = int(get_config(db, "decimation_factor"))
params.preprocess_lowpass = float(get_config(db, "preprocess_lowpass"))
params.preprocess_highpass = float(get_config(db, "preprocess_highpass"))
params.keep_all = get_config(db, "keep_all", isbool=True)
params.keep_days = get_config(db, "keep_days", isbool=True)
params.components_to_compute = get_components_to_compute(db)
logging.info("Will compute %s" % " ".join(params.components_to_compute))
while is_next_job(db, jobtype="CC"):
jobs = get_next_job(db, jobtype="CC")
stations = []
pairs = []
refs = []
for job in jobs:
refs.append(job.ref)
pairs.append(job.pair)
netsta1, netsta2 = job.pair.split(":")
stations.append(netsta1)
stations.append(netsta2)
goal_day = job.day
stations = np.unique(stations)
logging.info("New CC Job: %s (%i pairs with %i stations)" % (goal_day, len(pairs), len(stations)))
jt = time.time()
xlen = int(params.goal_duration * params.goal_sampling_rate)
if "".join(params.components_to_compute).count("R") > 0 or "".join(params.components_to_compute).count("T") > 0:
comps = ["Z", "E", "N"]
tramef_Z = np.zeros((len(stations), xlen))
tramef_E = np.zeros((len(stations), xlen))
tramef_N = np.zeros((len(stations), xlen))
basetime, tramef_Z, tramef_E, tramef_N = preprocess(
db, stations, comps, goal_day, params, tramef_Z, tramef_E, tramef_N
)
else:
comps = ["Z"]
tramef_Z = np.zeros((len(stations), xlen))
basetime, tramef_Z = preprocess(db, stations, comps, goal_day, params, tramef_Z)
# print '##### STREAMS ARE ALL PREPARED AT goal Hz #####'
dt = 1.0 / params.goal_sampling_rate
# Calculate the number of slices
slices = int(params.goal_duration * params.goal_sampling_rate / params.min30)
begins = []
ends = []
i = 0
while i <= (params.goal_duration - params.min30 / params.goal_sampling_rate):
begins.append(int(i * params.goal_sampling_rate))
ends.append(int(i * params.goal_sampling_rate + params.min30))
i += int(params.min30 / params.goal_sampling_rate * (1.0 - params.overlap))
slices = len(begins)
#
# Computing only ZZ components ? Then we can be much faster:
#
# if False:
if len(params.components_to_compute) == 1 and params.components_to_compute[0] == "ZZ":
Nfft = params.min30
if params.min30 / 2 % 2 != 0:
Nfft = params.min30 + 2
cp = cosTaper(int(params.min30), 0.04)
logging.info("Pre-Whitening Traces")
whitened_slices = np.zeros(
(len(stations), len(get_filters(db, all=False)), slices, int(Nfft)), dtype=np.complex
)
for istation, station in enumerate(stations):
for islice, (begin, end) in enumerate(zip(begins, ends)):
tmp = tramef_Z[istation, begin:end]
rmsmat = np.std(np.abs(tmp))
if params.windsorizing == -1:
tmp = np.sign(tmp)
elif params.windsorizing != 0:
indexes = np.where(np.abs(tmp) > (params.windsorizing * rmsmat))[0]
tmp[indexes] = (tmp[indexes] / np.abs(tmp[indexes])) * params.windsorizing * rmsmat
tmp *= cp
for ifilter, filter in enumerate(get_filters(db, all=False)):
whitened_slices[istation, ifilter, islice, :] = whiten(
tmp, Nfft, dt, float(filter.low), float(filter.high), plot=False
)
del tmp
del tramef_Z
logging.info("Processing CC")
for ifilter, filter in enumerate(get_filters(db, all=False)):
for pair in pairs:
orig_pair = pair
if params.keep_all:
allcorr = {}
if params.keep_days:
daycorr = np.zeros(get_maxlag_samples(db))
ndaycorr = 0
station1, station2 = pair.split(":")
pair = (np.where(stations == station1)[0][0], np.where(stations == station2)[0][0])
for islice in range(slices):
tmp = np.vstack(
(whitened_slices[pair[0], ifilter, islice], whitened_slices[pair[1], ifilter, islice])
)
corr = myCorr(tmp, np.ceil(params.maxlag / dt), plot=False)
tmptime = time.gmtime(basetime + begins[islice] / params.goal_sampling_rate)
thisdate = time.strftime("%Y-%m-%d", tmptime)
thistime = time.strftime("%Y-%m-%d %H:%M:%S", tmptime)
if not np.any(np.isnan(corr)) and not np.any(np.isinf(corr)):
if params.keep_all:
ccfid = "%s_%s_%s_%s_%s" % (station1, station2, filter.ref, "ZZ", thisdate)
if ccfid not in allcorr:
allcorr[ccfid] = {}
allcorr[ccfid][thistime] = corr
if params.keep_days:
daycorr += corr
ndaycorr += 1
if params.keep_all:
for ccfid in allcorr.keys():
export_allcorr(db, ccfid, allcorr[ccfid])
if params.keep_days:
thisdate = time.strftime("%Y-%m-%d", time.gmtime(basetime))
thistime = time.strftime("%H_%M", time.gmtime(basetime))
add_corr(
db,
station1.replace(".", "_"),
station2.replace(".", "_"),
filter.ref,
thisdate,
thistime,
params.min30 / params.goal_sampling_rate,
"ZZ",
daycorr,
params.goal_sampling_rate,
day=True,
ncorr=ndaycorr,
)
update_job(db, goal_day, orig_pair, "CC", "D")
logging.info("Job Finished. It took %.2f seconds" % (time.time() - jt))
else:
# ITERATING OVER PAIRS #####
for pair in pairs:
orig_pair = pair
logging.info("Processing pair: %s" % pair.replace(":", " vs "))
tt = time.time()
station1, station2 = pair.split(":")
pair = (np.where(stations == station1)[0][0], np.where(stations == station2)[0][0])
s1 = get_station(db, station1.split(".")[0], station1.split(".")[1])
s2 = get_station(db, station2.split(".")[0], station2.split(".")[1])
if s1.X:
X0 = s1.X
Y0 = s1.Y
c0 = s1.coordinates
X1 = s2.X
Y1 = s2.Y
c1 = s2.coordinates
if c0 == c1:
coordinates = c0
else:
coordinates = "MIX"
cplAz = np.deg2rad(azimuth(coordinates, X0, Y0, X1, Y1))
logging.debug("Azimuth=%.1f" % np.rad2deg(cplAz))
else:
# logging.debug('No Coordinates found! Skipping azimuth calculation!')
cplAz = 0.0
for components in params.components_to_compute:
if components == "ZZ":
t1 = tramef_Z[pair[0]]
t2 = tramef_Z[pair[1]]
elif components[0] == "Z":
t1 = tramef_Z[pair[0]]
t2 = tramef_E[pair[1]]
elif components[1] == "Z":
t1 = tramef_E[pair[0]]
t2 = tramef_Z[pair[1]]
else:
t1 = tramef_E[pair[0]]
t2 = tramef_E[pair[1]]
if np.all(t1 == 0) or np.all(t2 == 0):
logging.debug("%s contains empty trace(s), skipping" % components)
continue
del t1, t2
if components[0] == "Z":
t1 = tramef_Z[pair[0]]
elif components[0] == "R":
if cplAz != 0:
t1 = tramef_N[pair[0]] * np.cos(cplAz) + tramef_E[pair[0]] * np.sin(cplAz)
else:
t1 = tramef_E[pair[0]]
elif components[0] == "T":
if cplAz != 0:
t1 = tramef_N[pair[0]] * np.sin(cplAz) - tramef_E[pair[0]] * np.cos(cplAz)
else:
t1 = tramef_N[pair[0]]
if components[1] == "Z":
t2 = tramef_Z[pair[1]]
elif components[1] == "R":
if cplAz != 0:
t2 = tramef_N[pair[1]] * np.cos(cplAz) + tramef_E[pair[1]] * np.sin(cplAz)
else:
t2 = tramef_E[pair[1]]
elif components[1] == "T":
if cplAz != 0:
t2 = tramef_N[pair[1]] * np.sin(cplAz) - tramef_E[pair[1]] * np.cos(cplAz)
else:
t2 = tramef_N[pair[1]]
trames = np.vstack((t1, t2))
del t1, t2
daycorr = {}
ndaycorr = {}
allcorr = {}
for filterdb in get_filters(db, all=False):
filterid = filterdb.ref
daycorr[filterid] = np.zeros(get_maxlag_samples(db))
ndaycorr[filterid] = 0
for islice, (begin, end) in enumerate(zip(begins, ends)):
# print "Progress: %#2d/%2d"% (islice+1,slices)
trame2h = trames[:, begin:end]
rmsmat = np.std(np.abs(trame2h), axis=1)
for filterdb in get_filters(db, all=False):
filterid = filterdb.ref
low = float(filterdb.low)
high = float(filterdb.high)
rms_threshold = filterdb.rms_threshold
Nfft = int(params.min30)
if params.min30 / 2 % 2 != 0:
Nfft = params.min30 + 2
trames2hWb = np.zeros((2, int(Nfft)), dtype=np.complex)
skip = False
for i, station in enumerate(pair):
if rmsmat[i] > rms_threshold:
cp = cosTaper(len(trame2h[i]), 0.04)
trame2h[i] -= trame2h[i].mean()
if params.windsorizing == -1:
trame2h[i] = np.sign(trame2h[i])
elif params.windsorizing != 0:
indexes = np.where(np.abs(trame2h[i]) > (params.windsorizing * rmsmat[i]))[0]
# clipping at windsorizing*rms
trame2h[i][indexes] = (
(trame2h[i][indexes] / np.abs(trame2h[i][indexes]))
* params.windsorizing
* rmsmat[i]
)
trames2hWb[i] = whiten(trame2h[i] * cp, Nfft, dt, low, high, plot=False)
else:
trames2hWb[i] = np.zeros(int(Nfft))
skip = True
logging.debug("Slice is Zeros!")
if not skip:
corr = myCorr(trames2hWb, np.ceil(params.maxlag / dt), plot=False)
tmptime = time.gmtime(basetime + begin / params.goal_sampling_rate)
thisdate = time.strftime("%Y-%m-%d", tmptime)
thistime = time.strftime("%Y-%m-%d %H:%M:%S", tmptime)
if params.keep_all:
ccfid = "%s_%s_%s_%s_%s" % (station1, station2, filterid, components, thisdate)
if ccfid not in allcorr:
allcorr[ccfid] = {}
allcorr[ccfid][thistime] = corr
if params.keep_days:
if not np.any(np.isnan(corr)) and not np.any(np.isinf(corr)):
daycorr[filterid] += corr
ndaycorr[filterid] += 1
del corr, thistime, trames2hWb
if params.keep_all:
for ccfid in allcorr.keys():
export_allcorr(db, ccfid, allcorr[ccfid])
if params.keep_days:
try:
for filterdb in get_filters(db, all=False):
filterid = filterdb.ref
corr = daycorr[filterid]
ncorr = ndaycorr[filterid]
if ncorr > 0:
logging.debug(
"Saving daily CCF for filter %02i, comp %s (stack of %02i CCF)"
% (filterid, components, ncorr)
)
thisdate = time.strftime("%Y-%m-%d", time.gmtime(basetime))
thistime = time.strftime("%H_%M", time.gmtime(basetime))
add_corr(
db,
station1.replace(".", "_"),
station2.replace(".", "_"),
filterid,
thisdate,
thistime,
params.min30 / params.goal_sampling_rate,
components,
corr,
params.goal_sampling_rate,
day=True,
ncorr=ncorr,
)
del corr, ncorr
except Exception as e:
logging.debug(str(e))
del trames, daycorr, ndaycorr
logging.debug("Updating Job")
update_job(db, goal_day, orig_pair, "CC", "D")
logging.info("Finished processing this pair. It took %.2f seconds" % (time.time() - tt))
logging.info("Job Finished. It took %.2f seconds" % (time.time() - jt))
logging.info("*** Finished: Compute CC ***")
def main():
logging.basicConfig(level=logging.INFO,
format='%(asctime)s [%(levelname)s] %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
logging.info('*** Starting: Compute CC ***')
# Connection to the DB
db = connect()
if len(get_filters(db, all=False)) == 0:
logging.info("NO FILTERS DEFINED, exiting")
sys.exit()
# Get Configuration
params = Params()
params.goal_sampling_rate = float(get_config(db, "cc_sampling_rate"))
params.goal_duration = float(get_config(db, "analysis_duration"))
params.overlap = float(get_config(db, "overlap"))
params.maxlag = float(get_config(db, "maxlag"))
params.min30 = float(get_config(
db, "corr_duration")) * params.goal_sampling_rate
params.windsorizing = float(get_config(db, "windsorizing"))
params.resampling_method = get_config(db, "resampling_method")
params.decimation_factor = int(get_config(db, "decimation_factor"))
params.preprocess_lowpass = float(get_config(db, "preprocess_lowpass"))
params.preprocess_highpass = float(get_config(db, "preprocess_highpass"))
params.keep_all = get_config(db, 'keep_all', isbool=True)
params.keep_days = get_config(db, 'keep_days', isbool=True)
params.components_to_compute = get_components_to_compute(db)
logging.info("Will compute %s" % " ".join(params.components_to_compute))
while is_next_job(db, jobtype='CC'):
jobs = get_next_job(db, jobtype='CC')
stations = []
pairs = []
refs = []
for job in jobs:
refs.append(job.ref)
pairs.append(job.pair)
netsta1, netsta2 = job.pair.split(':')
stations.append(netsta1)
stations.append(netsta2)
goal_day = job.day
stations = np.unique(stations)
logging.info("New CC Job: %s (%i pairs with %i stations)" %
(goal_day, len(pairs), len(stations)))
jt = time.time()
xlen = int(params.goal_duration * params.goal_sampling_rate)
if ''.join(params.components_to_compute).count('R') > 0 or ''.join(
params.components_to_compute).count('T') > 0:
comps = ['Z', 'E', 'N']
tramef_Z = np.zeros((len(stations), xlen))
tramef_E = np.zeros((len(stations), xlen))
tramef_N = np.zeros((len(stations), xlen))
basetime, tramef_Z, tramef_E, tramef_N = preprocess(
db, stations, comps, goal_day, params, tramef_Z, tramef_E,
tramef_N)
else:
comps = ['Z']
tramef_Z = np.zeros((len(stations), xlen))
basetime, tramef_Z = preprocess(db, stations, comps, goal_day,
params, tramef_Z)
# print '##### STREAMS ARE ALL PREPARED AT goal Hz #####'
dt = 1. / params.goal_sampling_rate
# Calculate the number of slices
slices = int(params.goal_duration * params.goal_sampling_rate /
params.min30)
begins = []
ends = []
i = 0
while i <= (params.goal_duration -
params.min30 / params.goal_sampling_rate):
begins.append(int(i * params.goal_sampling_rate))
ends.append(int(i * params.goal_sampling_rate + params.min30))
i += int(params.min30 / params.goal_sampling_rate *
(1.0 - params.overlap))
slices = len(begins)
#
# Computing only ZZ components ? Then we can be much faster:
#
#if False:
if len(params.components_to_compute
) == 1 and params.components_to_compute[0] == "ZZ":
Nfft = params.min30
if params.min30 / 2 % 2 != 0:
Nfft = params.min30 + 2
cp = cosTaper(int(params.min30), 0.04)
logging.info("Pre-Whitening Traces")
whitened_slices = np.zeros(
(len(stations), len(get_filters(
db, all=False)), slices, int(Nfft)),
dtype=np.complex)
for istation, station in enumerate(stations):
for islice, (begin, end) in enumerate(zip(begins, ends)):
tmp = tramef_Z[istation, begin:end]
rmsmat = np.std(np.abs(tmp))
if params.windsorizing == -1:
tmp = np.sign(tmp)
elif params.windsorizing != 0:
indexes = np.where(
np.abs(tmp) > (params.windsorizing * rmsmat))[0]
tmp[indexes] = (tmp[indexes] / np.abs(
tmp[indexes])) * params.windsorizing * rmsmat
tmp *= cp
for ifilter, filter in enumerate(get_filters(db,
all=False)):
whitened_slices[istation, ifilter,
islice, :] = whiten(tmp,
Nfft,
dt,
float(filter.low),
float(filter.high),
plot=False)
del tmp
del tramef_Z
logging.info("Processing CC")
for ifilter, filter in enumerate(get_filters(db, all=False)):
for pair in pairs:
orig_pair = pair
if params.keep_all:
allcorr = {}
if params.keep_days:
daycorr = np.zeros(get_maxlag_samples(db, ))
ndaycorr = 0
station1, station2 = pair.split(':')
pair = (np.where(stations == station1)[0][0],
np.where(stations == station2)[0][0])
for islice in range(slices):
tmp = np.vstack(
(whitened_slices[pair[0], ifilter, islice],
whitened_slices[pair[1], ifilter, islice]))
corr = myCorr(tmp,
np.ceil(params.maxlag / dt),
plot=False)
tmptime = time.gmtime(basetime + begins[islice] /
params.goal_sampling_rate)
thisdate = time.strftime("%Y-%m-%d", tmptime)
thistime = time.strftime("%Y-%m-%d %H:%M:%S", tmptime)
if not np.any(np.isnan(corr)) and not np.any(
np.isinf(corr)):
if params.keep_all:
ccfid = "%s_%s_%s_%s_%s" % (station1, station2,
filter.ref, 'ZZ',
thisdate)
if ccfid not in allcorr:
allcorr[ccfid] = {}
allcorr[ccfid][thistime] = corr
if params.keep_days:
daycorr += corr
ndaycorr += 1
if params.keep_all:
for ccfid in allcorr.keys():
export_allcorr(db, ccfid, allcorr[ccfid])
if params.keep_days:
thisdate = time.strftime("%Y-%m-%d",
time.gmtime(basetime))
thistime = time.strftime("%H_%M",
time.gmtime(basetime))
add_corr(db,
station1.replace('.', '_'),
station2.replace('.', '_'),
filter.ref,
thisdate,
thistime,
params.min30 / params.goal_sampling_rate,
'ZZ',
daycorr,
params.goal_sampling_rate,
day=True,
ncorr=ndaycorr)
update_job(db, goal_day, orig_pair, 'CC', 'D')
logging.info("Job Finished. It took %.2f seconds" %
(time.time() - jt))
else:
# ITERATING OVER PAIRS #####
for pair in pairs:
orig_pair = pair
logging.info('Processing pair: %s' % pair.replace(':', ' vs '))
tt = time.time()
station1, station2 = pair.split(':')
pair = (np.where(stations == station1)[0][0],
np.where(stations == station2)[0][0])
s1 = get_station(db,
station1.split('.')[0],
station1.split('.')[1])
s2 = get_station(db,
station2.split('.')[0],
station2.split('.')[1])
if s1.X:
X0 = s1.X
Y0 = s1.Y
c0 = s1.coordinates
X1 = s2.X
Y1 = s2.Y
c1 = s2.coordinates
if c0 == c1:
coordinates = c0
else:
coordinates = 'MIX'
cplAz = np.deg2rad(azimuth(coordinates, X0, Y0, X1, Y1))
logging.debug("Azimuth=%.1f" % np.rad2deg(cplAz))
else:
# logging.debug('No Coordinates found! Skipping azimuth calculation!')
cplAz = 0.
for components in params.components_to_compute:
if components == "ZZ":
t1 = tramef_Z[pair[0]]
t2 = tramef_Z[pair[1]]
elif components[0] == "Z":
t1 = tramef_Z[pair[0]]
t2 = tramef_E[pair[1]]
elif components[1] == "Z":
t1 = tramef_E[pair[0]]
t2 = tramef_Z[pair[1]]
else:
t1 = tramef_E[pair[0]]
t2 = tramef_E[pair[1]]
if np.all(t1 == 0) or np.all(t2 == 0):
logging.debug("%s contains empty trace(s), skipping" %
components)
continue
del t1, t2
if components[0] == "Z":
t1 = tramef_Z[pair[0]]
elif components[0] == "R":
if cplAz != 0:
t1 = tramef_N[pair[0]] * np.cos(cplAz) +\
tramef_E[pair[0]] * np.sin(cplAz)
else:
t1 = tramef_E[pair[0]]
elif components[0] == "T":
if cplAz != 0:
t1 = tramef_N[pair[0]] * np.sin(cplAz) -\
tramef_E[pair[0]] * np.cos(cplAz)
else:
t1 = tramef_N[pair[0]]
if components[1] == "Z":
t2 = tramef_Z[pair[1]]
elif components[1] == "R":
if cplAz != 0:
t2 = tramef_N[pair[1]] * np.cos(cplAz) +\
tramef_E[pair[1]] * np.sin(cplAz)
else:
t2 = tramef_E[pair[1]]
elif components[1] == "T":
if cplAz != 0:
t2 = tramef_N[pair[1]] * np.sin(cplAz) -\
tramef_E[pair[1]] * np.cos(cplAz)
else:
t2 = tramef_N[pair[1]]
trames = np.vstack((t1, t2))
del t1, t2
daycorr = {}
ndaycorr = {}
allcorr = {}
for filterdb in get_filters(db, all=False):
filterid = filterdb.ref
daycorr[filterid] = np.zeros(get_maxlag_samples(db, ))
ndaycorr[filterid] = 0
for islice, (begin, end) in enumerate(zip(begins, ends)):
# print "Progress: %#2d/%2d"% (islice+1,slices)
trame2h = trames[:, begin:end]
rmsmat = np.std(np.abs(trame2h), axis=1)
for filterdb in get_filters(db, all=False):
filterid = filterdb.ref
low = float(filterdb.low)
high = float(filterdb.high)
rms_threshold = filterdb.rms_threshold
Nfft = int(params.min30)
if params.min30 / 2 % 2 != 0:
Nfft = params.min30 + 2
trames2hWb = np.zeros((2, int(Nfft)),
dtype=np.complex)
skip = False
for i, station in enumerate(pair):
if rmsmat[i] > rms_threshold:
cp = cosTaper(len(trame2h[i]), 0.04)
trame2h[i] -= trame2h[i].mean()
if params.windsorizing == -1:
trame2h[i] = np.sign(trame2h[i])
elif params.windsorizing != 0:
indexes = np.where(
np.abs(trame2h[i]) > (
params.windsorizing *
rmsmat[i]))[0]
# clipping at windsorizing*rms
trame2h[i][indexes] = (
trame2h[i][indexes] /
np.abs(trame2h[i][indexes])
) * params.windsorizing * rmsmat[i]
trames2hWb[i] = whiten(trame2h[i] * cp,
Nfft,
dt,
low,
high,
plot=False)
else:
trames2hWb[i] = np.zeros(int(Nfft))
skip = True
logging.debug('Slice is Zeros!')
if not skip:
corr = myCorr(trames2hWb,
np.ceil(params.maxlag / dt),
plot=False)
tmptime = time.gmtime(
basetime +
begin / params.goal_sampling_rate)
thisdate = time.strftime("%Y-%m-%d", tmptime)
thistime = time.strftime(
"%Y-%m-%d %H:%M:%S", tmptime)
if params.keep_all:
ccfid = "%s_%s_%s_%s_%s" % (
station1, station2, filterid,
components, thisdate)
if ccfid not in allcorr:
allcorr[ccfid] = {}
allcorr[ccfid][thistime] = corr
if params.keep_days:
if not np.any(np.isnan(corr)) and \
not np.any(np.isinf(corr)):
daycorr[filterid] += corr
ndaycorr[filterid] += 1
del corr, thistime, trames2hWb
if params.keep_all:
for ccfid in allcorr.keys():
export_allcorr(db, ccfid, allcorr[ccfid])
if params.keep_days:
try:
for filterdb in get_filters(db, all=False):
filterid = filterdb.ref
corr = daycorr[filterid]
ncorr = ndaycorr[filterid]
if ncorr > 0:
logging.debug(
"Saving daily CCF for filter %02i, comp %s (stack of %02i CCF)"
% (filterid, components, ncorr))
thisdate = time.strftime(
"%Y-%m-%d", time.gmtime(basetime))
thistime = time.strftime(
"%H_%M", time.gmtime(basetime))
add_corr(db,
station1.replace('.', '_'),
station2.replace('.', '_'),
filterid,
thisdate,
thistime,
params.min30 /
params.goal_sampling_rate,
components,
corr,
params.goal_sampling_rate,
day=True,
ncorr=ncorr)
del corr, ncorr
except Exception as e:
logging.debug(str(e))
del trames, daycorr, ndaycorr
logging.debug("Updating Job")
update_job(db, goal_day, orig_pair, 'CC', 'D')
logging.info(
"Finished processing this pair. It took %.2f seconds" %
(time.time() - tt))
logging.info("Job Finished. It took %.2f seconds" %
(time.time() - jt))
logging.info('*** Finished: Compute CC ***')
if windsorizing != 0:
indexes = np.where(
np.abs(trame2h[i]) > (windsorizing * rmsmat[i]))[0]
# clipping at windsorizing*rms
trame2h[i][indexes] = (trame2h[i][indexes] / np.abs(
trame2h[i][indexes])) * windsorizing * rmsmat[i]
# logging.debug('whiten')
trames2hWb[i] = whiten(
trame2h[i], Nfft, dt, low, high, plot=False)
else:
# logging.debug("Station no %d, pas de pretraitement car rms < %f ou NaN"% (i, rms_threshold))
trames2hWb[i] = np.zeros(Nfft)
corr = myCorr(trames2hWb, np.ceil(maxlag / dt), plot=False)
thisdate = time.strftime(
"%Y-%m-%d", time.gmtime(basetime + itranche * min30 / fe))
thistime = time.strftime(
"%H_%M", time.gmtime(basetime + itranche * min30 / fe))
if keep_all:
add_corr(db, station1.replace('.', '_'), station2.replace(
'.', '_'), filterid, thisdate, thistime, min30 / fe, components, corr, fe)
if keep_days:
if not np.any(np.isnan(corr)) and not np.any(np.isinf(corr)):
daycorr[filterid] += corr
ndaycorr[filterid] += 1
del corr, thistime, trames2hWb
def main():
logging.basicConfig(level=logging.INFO,
format='%(asctime)s [%(levelname)s] %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
logging.info('*** Starting: Compute SC ***')
# Connection to the DB
db = connect()
#rule out absence of filters
if len(get_filters(db, all=False)) == 0:
logging.info("NO FILTERS DEFINED, exiting")
sys.exit()
# Get Configuration
params = Params()
params.goal_sampling_rate = float(get_config(db, "cc_sampling_rate"))
params.goal_duration = float(get_config(db, "analysis_duration"))
params.overlap = float(get_config(db, "overlap"))
params.maxlag = float(get_config(db, "maxlag"))
params.min30 = float(get_config(db, "corr_duration")) * params.goal_sampling_rate
params.windsorizing = float(get_config(db, "windsorizing"))
params.resampling_method = get_config(db, "resampling_method")
params.decimation_factor = int(get_config(db, "decimation_factor"))
params.preprocess_lowpass = float(get_config(db, "preprocess_lowpass"))
params.preprocess_highpass = float(get_config(db, "preprocess_highpass"))
params.keep_all = get_config(db, 'keep_all', isbool=True)
params.keep_days = get_config(db, 'keep_days', isbool=True)
params.components_to_compute = ['Z', 'E', 'N']
##################################
logging.info("Will compute %s" % " ".join(params.components_to_compute))
##################################
stations_to_analyse = ["%s.%s" % (sta.net, sta.sta) for sta in get_stations(db, all=True)]#extract all stations
pairs = []#pair of components
##################################
#modified part to make pair list with comps
##################################
i = 0
for comp in params.components_to_compute:
for newcomp in params.components_to_compute:
if comp != newcomp and comp > newcomp:
if i == 0:
pairs = np.array(':'.join([comp, newcomp]))
i+=1
else:
pairs = np.vstack((pairs,':'.join([comp, newcomp])))
pairs = np.hstack(pairs)
#components_to_compute = ['ZE', 'NE', 'ZN']
#pairs=components_to_compute
#for station_unique in stations_to_analyse:
while is_next_job(db, jobtype='SC'):
jobs = get_next_job(db, jobtype='SC')
#logging.info("Working on station %s" %(station_unique))
stations = []
refs = []
#go through job to make job array with stations
for job in jobs:
refs.append(job.ref)
#pairs.append(job.pair)#find a way to pair comps /!\
netsta = job.pair #just 1 station in the cell?
stations.append(netsta)
#stations.append(netstacomp2)
goal_day = job.day
stations = np.unique(stations)#only 1 station in it
#print "only this station(s) here: %s, ref= %s" %(stations, refs)
logging.info("New SC Job: %s (%i pairs with %i stations)" %
(goal_day, len(pairs)*len(stations), len(stations)))
jt = time.time()
xlen = int(params.goal_duration * params.goal_sampling_rate)
comps = ['Z', 'E', 'N']
tramef_Z = np.zeros((len(stations), xlen))
tramef_E = np.zeros((len(stations), xlen))
tramef_N = np.zeros((len(stations), xlen))
basetime, tramef_Z, tramef_E, tramef_N = preprocess(db, stations, comps, goal_day, params, tramef_Z, tramef_E, tramef_N)# preprocessing
#print type(tramef_E)
# comps = ['Z']
# tramef_Z = np.zeros((len(stations), xlen))
# basetime, tramef_Z = preprocess(db, stations, comps, goal_day, params, tramef_Z)
# print type(tramef_Z)
dt = 1. / params.goal_sampling_rate
# Calculate the number of slices
slices = int(params.goal_duration * params.goal_sampling_rate / params.min30)
begins = []
ends = []
i = 0
while i <= (params.goal_duration - params.min30 / params.goal_sampling_rate):
begins.append(int(i * params.goal_sampling_rate))
ends.append(int(i * params.goal_sampling_rate + params.min30))
i += int(params.min30 / params.goal_sampling_rate * (1.0 - params.overlap))
slices = len(begins)
# ##########################################################################################################
for station in stations:
orig_pair = station
for pair in pairs:
#print "Processing pair %s for station %s - %s" %(pair, station, goal_day)
logging.info("Processing pair %s for station %s - %s" %(pair, station, goal_day))
#print type(tramef_Z)
#print tramef_Z.keys()
tt = time.time()
comp1,comp2=pair.split(':')
components=comp1+comp2
### load trames
#assign trames according to pair
station_to_analyse=np.where(stations==station)
#print "you are looking for ",station_to_analyse
if pair.split(':')[0]=='Z':
tr1=tramef_Z[station_to_analyse]
elif pair.split(':')[0]=='E':
tr1=tramef_E[station_to_analyse]
elif pair.split(':')[0]=='N':
tr1=tramef_N[station_to_analyse]
if pair.split(':')[1]=='Z':
tr2=tramef_Z[station_to_analyse]
elif pair.split(':')[1]=='E':
tr2=tramef_E[station_to_analyse]
elif pair.split(':')[1]=='N':
tr2=tramef_N[station_to_analyse]
if np.all(tr1 == 0) or np.all(tr2 == 0):
logging.debug("%s contains empty trace(s), skipping"%components)
continue
#print "tr1 est de type ",type(tr1)
if np.all(tr1 == 0) or np.all(tr2 == 0):
logging.debug("%s contains empty trace(s), skipping"%components)
continue
trames=np.vstack((tr1,tr2))
# print tr1
# print tr2
#print np.std(tr1, axis=1), np.std(tr2, axis=1)
del tr1,tr2
## islice
daycorr = {}
ndaycorr = {}
allcorr = {}
for filterdb in get_filters(db, all=False):
filterid = filterdb.ref
daycorr[filterid] = np.zeros(get_maxlag_samples(db,))
ndaycorr[filterid] = 0
baddata=False
for islice, (begin, end) in enumerate(zip(begins, ends)):
#print "Progress: %#2d/%2d"% (islice+1,slices)
trame2h = trames[:, begin:end]
#print "Here comes the TRACES!! \n",trames
rmsmat = np.std(trame2h, axis=1)
for filterdb in get_filters(db, all=False):
filterid = filterdb.ref
low = float(filterdb.low)
high = float(filterdb.high)
rms_threshold = filterdb.rms_threshold
Nfft = int(params.min30)
if params.min30 / 2 % 2 != 0:
Nfft = params.min30 + 2
trames2hWb = np.zeros((2, int(Nfft)), dtype=np.complex)
skip = False
for i, comp in enumerate(pair.split(':')):#### fix this loop, work on each comp of the pair
#print i, comp, station, goal_day
if rmsmat[i] > rms_threshold:
cp = cosTaper(len(trame2h[i]),0.04)
trame2h[i] -= trame2h[i].mean()
if params.windsorizing == -1:
trame2h[i] = np.sign(trame2h[i])
elif params.windsorizing != 0:
indexes = np.where(
np.abs(trame2h[i]) > (params.windsorizing * rmsmat[i]))[0]
# clipping at windsorizing*rms
trame2h[i][indexes] = (trame2h[i][indexes] / np.abs(
trame2h[i][indexes])) * params.windsorizing * rmsmat[i]
trames2hWb[i] = whiten(
trame2h[i]*cp, Nfft, dt, low, high, plot=False)
else:
trames2hWb[i] = np.zeros(int(Nfft))
skip = True
logging.debug('Slice is Zeros!')
if not skip:
corr = myCorr(trames2hWb, np.ceil(params.maxlag / dt), plot=False)
tmptime = time.gmtime(basetime + begin /
params.goal_sampling_rate)
thisdate = time.strftime("%Y-%m-%d", tmptime)
thistime = time.strftime("%Y-%m-%d %H:%M:%S",
tmptime)
if params.keep_all:
ccfid = "%s_%s_%s_%s_%s" % (station,station,
filterid, components,
thisdate)
if ccfid not in allcorr:
allcorr[ccfid] = {}
allcorr[ccfid][thistime] = corr
if params.keep_days:
#print "KEEP DAYS!"
if not np.any(np.isnan(corr)) and \
not np.any(np.isinf(corr)):
daycorr[filterid] += corr
ndaycorr[filterid] += 1
del corr, thistime, trames2hWb
else:
#print "NOOOOOOOOOOO! Zeros!"
baddata=True
#raw_input()
if baddata==True:
logging.info("Bad data: ",pair, station, goal_day)
badfolder=os.path.join("BAD","%s"%(station))
output=np.array([station, pair, goal_day])
if not os.path.isdir(badfolder):
logging.info("Creating dir for ", output)
os.makedirs(badfolder)
badfile=os.path.join(badfolder, "%s.txt" % str(goal_day))
np.savetxt(badfile, output, delimiter=';', fmt="%s")
if params.keep_all:
for ccfid in allcorr.keys():
export_allcorr(db, ccfid, allcorr[ccfid])
if params.keep_days:
try:
for filterdb in get_filters(db, all=False):
filterid = filterdb.ref
corr = daycorr[filterid]
ncorr = ndaycorr[filterid]
if ncorr > 0:
logging.debug(
"Saving daily CCF for filter %02i, comp %s (stack of %02i CCF)" % (filterid, components, ncorr))
thisdate = time.strftime(
"%Y-%m-%d", time.gmtime(basetime))
thistime = time.strftime(
"%H_%M", time.gmtime(basetime))
stationpair="%s_%s" %(station,station)
add_corr(
db, station.replace('.', '_'),#################
station.replace('.', '_'), filterid,
thisdate, thistime, params.min30 /
params.goal_sampling_rate,
components, corr,
params.goal_sampling_rate, day=True,
ncorr=ncorr)
del corr, ncorr
except Exception as e:
logging.debug(str(e))
del trames, daycorr, ndaycorr
logging.debug("Updating Job")
update_job(db, goal_day, orig_pair, 'SC', 'D')
logging.info("Finished processing this station. It took %.2f seconds" %
(time.time() - tt))
logging.info("Job Finished. It took %.2f seconds" % (time.time() - jt))
trame2h[i][indexes] /
np.abs(trame2h[i][indexes])
) * windsorizing * rmsmat[i]
# logging.debug('whiten')
trames2hWb[i] = whiten(trame2h[i],
min30,
dt,
low,
high,
plot=False)
else:
# logging.debug("Station no %d, pas de pretraitement car rms < %f ou NaN"% (i, rms_threshold))
trames2hWb[i] = trame2h[i]
corr = myCorr(trames2hWb, np.ceil(maxlag / dt), plot=False)
thisdate = time.strftime(
"%Y-%m-%d",
time.gmtime(basetime + itranche * min30 / fe))
thistime = time.strftime(
"%H_%M", time.gmtime(basetime + itranche * min30 / fe))
if keep_all:
add_corr(db, station1.replace('.', '_'),
station2.replace('.',
'_'), filterid, thisdate,
thistime, min30 / fe, components, corr, fe)
if keep_days:
if not np.any(np.isnan(corr)) and not np.any(
np.isinf(corr)):
daycorr[filterid] += corr
