def rm_duplicate(all_sta_avail, address):
"""
remove duplicates and give back the required list for updating
"""
id_avai_stas = []
for sta in all_sta_avail:
if sta[2] == '--' or sta[2] == ' ':
sta[2] = ''
if len(sta) == 7:
id_avai_stas.append(
'%s_%s_%s_%s_%s_%s_%s' %
(sta[0], sta[1], sta[2], sta[3], sta[4], sta[5], sta[6]))
elif len(sta) == 8:
id_avai_stas.append('%s_%s_%s_%s_%s_%s_%s_%s' %
(sta[0], sta[1], sta[2], sta[3], sta[4],
sta[5], sta[6], sta[7]))
sta_ev_saved = read_station_event(address)
id_all_saved_stas = []
for saved_sta in sta_ev_saved[0]:
id_all_saved_stas.append(
'%s_%s_%s_%s' %
(saved_sta[0], saved_sta[1], saved_sta[2], saved_sta[3]))
stas_update = copy.deepcopy(id_avai_stas)
del_num = []
for saved_sta in id_all_saved_stas:
for j in range(len(stas_update)):
if saved_sta in stas_update[j]:
del_num.append(j)
del_num.sort(reverse=True)
for dn in del_num:
del stas_update[dn]
for i in range(len(stas_update)):
stas_update[i] = stas_update[i].split('_')
stas_update.sort()
print '------------------------------------------'
print 'Info:'
print 'Number of all saved stations: %s' % len(id_all_saved_stas)
print 'Number of all available stations: %s' % len(id_avai_stas)
print 'Number of stations to update for: %s' % len(stas_update)
print '------------------------------------------'
return stas_update
def writesac_all(i, address_events):
"""
Change the format of trace(s) to SAC and fill in the header information
:param i:
:param address_events:
:return:
"""
sta_ev = read_station_event(address_events[i])
ls_saved_stas = []
for j in range(len(sta_ev[0])):
station_id = '%s.%s.%s.%s' % (sta_ev[0][j][0], sta_ev[0][j][1],
sta_ev[0][j][2], sta_ev[0][j][3])
ls_saved_stas.append(
os.path.join(address_events[i], 'BH_RAW', station_id))
for j in range(len(sta_ev[0])):
try:
st = read(ls_saved_stas[j])
st[0].write(ls_saved_stas[j], format='SAC')
tr = read(ls_saved_stas[j])[0]
if sta_ev[0][j][4]:
tr.stats.sac.stla = float(sta_ev[0][j][4])
if sta_ev[0][j][5]:
tr.stats.sac.stlo = float(sta_ev[0][j][5])
if sta_ev[0][j][6]:
tr.stats.sac.stel = float(sta_ev[0][j][6])
if sta_ev[0][j][7]:
tr.stats.sac.stdp = float(sta_ev[0][j][7])
if sta_ev[0][j][9]:
tr.stats.sac.evla = float(sta_ev[0][j][9])
if sta_ev[0][j][10]:
tr.stats.sac.evlo = float(sta_ev[0][j][10])
if sta_ev[0][j][11]:
tr.stats.sac.evdp = float(sta_ev[0][j][11])
if sta_ev[0][j][12]:
tr.stats.sac.mag = float(sta_ev[0][j][12])
tr.write(ls_saved_stas[j], format='SAC')
except Exception as e:
print '\nWARNING: %s' % e
print ls_saved_stas[j]
print '------------------'
def FDSN_ARC_merge(input_dics, clients):
"""
Call "merge_stream" function that merges the retrieved waveforms in
continuous request
:param input_dics:
:param clients:
:return:
"""
print '\n*****************************'
print '%s -- Merging the waveforms' % clients
print '*****************************'
# Following two if-conditions create address to which merging
# should be applied
# Please note that these two conditions can not happen at the same time
address = None
network_name = None
BH_file = None
if clients == 'arc':
clients_name = 'arc'
else:
clients_name = 'fdsn'
if input_dics[clients_name + '_merge_auto'] == 'Y':
period = '{0:s}_{1:s}_{2:s}_{3:s}'.\
format(input_dics['min_date'].split('T')[0],
input_dics['max_date'].split('T')[0],
str(input_dics['min_mag']),
str(input_dics['max_mag']))
address = os.path.join(input_dics['datapath'], period)
if input_dics[clients_name + '_merge'] != 'N':
address = input_dics[clients_name + '_merge']
events, address_events = quake_info(address, 'info')
ls_saved_stas_tmp = []
for i in range(len(events)):
sta_ev = read_station_event(address_events[i])
# initialize some parameters which will be used later in merge_stream
for s_ev in sta_ev[0]:
if input_dics[clients_name + '_merge_auto'] == 'Y':
if clients == s_ev[13]:
network, network_name, BH_file = \
init_merging(input_dics, s_ev[0])
station_id = '%s.%s.%s.%s' % (network, s_ev[1], s_ev[2],
s_ev[3])
ls_saved_stas_tmp.append(
os.path.join(address_events[i], BH_file, station_id))
else:
network, network_name, BH_file = \
init_merging(input_dics, s_ev[0])
station_id = '%s.%s.%s.%s' % (network, s_ev[1], s_ev[2],
s_ev[3])
ls_saved_stas_tmp.append(
os.path.join(address_events[i], BH_file, station_id))
if not input_dics['net'].startswith('_'):
pattern_sta = '%s.%s.%s.%s' % (input_dics['net'], input_dics['sta'],
input_dics['loc'], input_dics['cha'])
else:
pattern_sta = '*.%s.%s.%s' % (input_dics['sta'], input_dics['loc'],
input_dics['cha'])
ls_saved_stas = []
for saved_sta in ls_saved_stas_tmp:
if fnmatch.fnmatch(os.path.basename(saved_sta), pattern_sta):
ls_saved_stas.append(saved_sta)
if len(ls_saved_stas) != 0:
saved_stations_names = []
for ls_saved_sta in ls_saved_stas:
saved_stations_names.append(os.path.basename(ls_saved_sta))
ls_sta = list(set(saved_stations_names))
ls_address = []
for add_ev in address_events:
ls_address.append(os.path.join(add_ev, BH_file))
print 'Merging the waveforms...'
merge_stream(input_dics=input_dics,
ls_address=ls_address,
ls_sta=ls_sta,
network_name=network_name)
print 'Finish merging the waveforms'
else:
print "\nThere is no waveform to merge!"
def FDSN_ARC_IC(input_dics, clients):
"""
Call "inst_correct" function based on the request.
Group the stations that have been retrieved from specific client
Grouping is necessary in applying the instrument correction correctly
(different clients are treated differently)
:param input_dics:
:param clients:
:return:
"""
print '\n*****************************'
print '%s -- Instrument Correction' % clients
print '*****************************'
# Following two if-conditions create address to which
# instrument correction should be applied
# Please note that these two conditions can not happen at the same time
address = None
if clients == 'arc':
clients_name = 'arc'
else:
clients_name = 'fdsn'
if input_dics[clients_name + '_ic_auto'] == 'Y':
period = '{0:s}_{1:s}_{2:s}_{3:s}'.\
format(input_dics['min_date'].split('T')[0],
input_dics['max_date'].split('T')[0],
str(input_dics['min_mag']),
str(input_dics['max_mag']))
address = os.path.join(input_dics['datapath'], period)
if input_dics['fdsn_update'] != 'N':
address = os.path.join(input_dics['datapath'])
if input_dics['arc_update'] != 'N':
address = os.path.join(input_dics['datapath'])
if input_dics[clients_name + '_ic'] != 'N':
address = input_dics[clients_name + '_ic']
events, address_events = quake_info(address, 'info')
for i in range(len(events)):
sta_ev = read_station_event(address_events[i])
ls_saved_stas_tmp = []
for s_ev in sta_ev[0]:
if clients.lower() == 'all_fdsn':
if not s_ev[13].lower() == 'arc':
station_id = '%s.%s.%s.%s' % (s_ev[0], s_ev[1], s_ev[2],
s_ev[3])
ls_saved_stas_tmp.append(
os.path.join(address_events[i], 'BH_RAW', station_id))
elif clients.lower() == s_ev[13].lower():
station_id = '%s.%s.%s.%s' % (s_ev[0], s_ev[1], s_ev[2],
s_ev[3])
ls_saved_stas_tmp.append(
os.path.join(address_events[i], 'BH_RAW', station_id))
if input_dics['net'].startswith('_'):
net_pat = '*'
else:
net_pat = input_dics['net']
ls_saved_stas = []
for net_iter in net_pat.split(','):
for sta_iter in input_dics['sta'].split(','):
for loc_iter in input_dics['loc'].split(','):
for cha_iter in input_dics['cha'].split(','):
pattern_sta = '%s.%s.%s.%s' % (net_iter, sta_iter,
loc_iter, cha_iter)
for saved_sta in ls_saved_stas_tmp:
if fnmatch.fnmatch(os.path.basename(saved_sta),
pattern_sta):
ls_saved_stas.append(saved_sta)
if len(ls_saved_stas) != 0:
print '\nevent: %s/%s -- %s\n' % (i + 1, len(events), clients)
inst_correct(input_dics, ls_saved_stas, address_events[i], clients)
else:
print "There is no station in the directory to correct!"
print "Address: %s" % address_events[i]
# pass address for using in create_tar_file
return address
def FDSN_ARC_IC(input_dics, clients):
"""
Call "inst_correct" function based on the request.
Group the stations that have been retrieved from specific client
Grouping is necessary in applying the instrument correction correctly
(different clients are treated differently)
:param input_dics:
:param clients:
:return:
"""
if len(input_dics['fdsn_base_url_rest']) > 0:
clients = 'all_fdsn'
print '\n*****************************'
print '%s -- Instrument Correction' % clients
print '*****************************'
# Following two if-conditions create address to which
# instrument correction should be applied
# Please note that these two conditions can not happen at the same time
address = None
if clients == 'arc':
clients_name = 'arc'
else:
clients_name = 'fdsn'
if input_dics[clients_name + '_ic_auto'] == 'Y':
period = '{0:s}_{1:s}'.\
format(input_dics['min_date'].split('T')[0],
input_dics['max_date'].split('T')[0])
address = os.path.join(input_dics['datapath'], period)
if input_dics['fdsn_update'] != 'N':
address = os.path.join(input_dics['datapath'])
if input_dics['arc_update'] != 'N':
address = os.path.join(input_dics['datapath'])
if input_dics[clients_name + '_ic'] != 'N':
address = input_dics[clients_name + '_ic']
events, address_events = quake_info(address, 'info')
for i in range(len(events)):
sta_ev = read_station_event(address_events[i])
ls_saved_stas_tmp = []
for s_ev in sta_ev[0]:
if clients.lower() == 'all_fdsn':
if not s_ev[13].lower() == 'arc':
station_id = '%s.%s.%s.%s' % (s_ev[0], s_ev[1],
s_ev[2], s_ev[3])
ls_saved_stas_tmp.append(
os.path.join(address_events[i], 'BH_RAW', station_id))
elif clients.lower() == s_ev[13].lower():
station_id = '%s.%s.%s.%s' % (s_ev[0], s_ev[1],
s_ev[2], s_ev[3])
ls_saved_stas_tmp.append(
os.path.join(address_events[i], 'BH_RAW', station_id))
if input_dics['net'].startswith('_'):
net_pat = '*'
else:
net_pat = input_dics['net']
ls_saved_stas = []
for net_iter in net_pat.split(','):
for sta_iter in input_dics['sta'].split(','):
for loc_iter in input_dics['loc'].split(','):
for cha_iter in input_dics['cha'].split(','):
pattern_sta = '%s.%s.%s.%s' % (net_iter, sta_iter,
loc_iter, cha_iter)
for saved_sta in ls_saved_stas_tmp:
if fnmatch.fnmatch(os.path.basename(saved_sta),
pattern_sta):
ls_saved_stas.append(saved_sta)
if len(ls_saved_stas) != 0:
print '\nevent: %s/%s -- %s\n' % (i+1, len(events), clients)
inst_correct(input_dics, ls_saved_stas, address_events[i], clients)
else:
print "There is no station in the directory to correct!"
print "Address: %s" % address_events[i]
# pass address for using in create_tar_file
return address
def FDSN_ARC_merge(input_dics, clients):
"""
Call "merge_stream" function that merges the retrieved waveforms in
continuous request
:param input_dics:
:param clients:
:return:
"""
print '\n*****************************'
print '%s -- Merging the waveforms' % clients
print '*****************************'
# Following two if-conditions create address to which merging
# should be applied
# Please note that these two conditions can not happen at the same time
address = None
network_name = None
BH_file = None
if clients == 'arc':
clients_name = 'arc'
else:
clients_name = 'fdsn'
if input_dics[clients_name + '_merge_auto'] == 'Y':
period = '{0:s}_{1:s}'.\
format(input_dics['min_date'].split('T')[0],
input_dics['max_date'].split('T')[0])
address = os.path.join(input_dics['datapath'], period)
if input_dics[clients_name + '_merge'] != 'N':
address = input_dics[clients_name + '_merge']
events, address_events = quake_info(address, 'info')
ls_saved_stas_tmp = []
for i in range(len(events)):
sta_ev = read_station_event(address_events[i])
# initialize some parameters which will be used later in merge_stream
for s_ev in sta_ev[0]:
if input_dics[clients_name + '_merge_auto'] == 'Y':
if clients == s_ev[13]:
network, network_name, BH_file = \
init_merging(input_dics, s_ev[0])
station_id = '%s.%s.%s.%s' % (network, s_ev[1],
s_ev[2], s_ev[3])
ls_saved_stas_tmp.append(
os.path.join(address_events[i], BH_file, station_id))
else:
network, network_name, BH_file = \
init_merging(input_dics, s_ev[0])
station_id = '%s.%s.%s.%s' % (network, s_ev[1],
s_ev[2], s_ev[3])
ls_saved_stas_tmp.append(
os.path.join(address_events[i], BH_file, station_id))
if not input_dics['net'].startswith('_'):
pattern_sta = '%s.%s.%s.%s' % (input_dics['net'], input_dics['sta'],
input_dics['loc'], input_dics['cha'])
else:
pattern_sta = '*.%s.%s.%s' % (input_dics['sta'], input_dics['loc'],
input_dics['cha'])
ls_saved_stas = []
for saved_sta in ls_saved_stas_tmp:
if fnmatch.fnmatch(os.path.basename(saved_sta), pattern_sta):
ls_saved_stas.append(saved_sta)
if len(ls_saved_stas) != 0:
saved_stations_names = []
for ls_saved_sta in ls_saved_stas:
saved_stations_names.append(os.path.basename(ls_saved_sta))
ls_sta = list(set(saved_stations_names))
ls_address = []
for add_ev in address_events:
ls_address.append(os.path.join(add_ev, BH_file))
print 'Merging the waveforms...'
merge_stream(input_dics=input_dics, ls_address=ls_address,
ls_sta=ls_sta, network_name=network_name)
print 'Finish merging the waveforms'
else:
print "\nThere is no waveform to merge!"
def writesac_all(i, address_events):
"""
Change the format of trace(s) to SAC and fill in the header information
:param i:
:param address_events:
:return:
"""
sta_ev = read_station_event(address_events[i])
ls_saved_stas = []
for j in range(len(sta_ev[0])):
station_id = '%s.%s.%s.%s' % (sta_ev[0][j][0],
sta_ev[0][j][1],
sta_ev[0][j][2],
sta_ev[0][j][3])
ls_saved_stas.append(os.path.join(address_events[i],
'BH_RAW', station_id))
for j in range(len(sta_ev[0])):
try:
st = read(ls_saved_stas[j])
if len(st) > 1:
print "\n=== WARNING:"
print "%s" % ls_saved_stas[j]
print "probably has some gaps!"
print "It will be merged (fill_value=0)."
print "\nFor more information refer to:"
print "%s" % os.path.join(address_events[i], 'info',
'waveform_gap.txt')
print "which contains all the waveforms with gap."
st.merge(method=1, fill_value=0, interpolation_samples=0)
gap_fio = open(os.path.join(address_events[i], 'info',
'waveform_gap.txt'), 'a+')
gap_msg = '%s.%s.%s.%s\t%s\n' % (sta_ev[0][j][0],
sta_ev[0][j][1],
sta_ev[0][j][2],
sta_ev[0][j][3],
'writesac_all')
gap_fio.writelines(gap_msg)
gap_fio.close()
st[0].write(ls_saved_stas[j], format='SAC')
tr = read(ls_saved_stas[j])[0]
if sta_ev[0][j][4]:
tr.stats.sac.stla = float(sta_ev[0][j][4])
if sta_ev[0][j][5]:
tr.stats.sac.stlo = float(sta_ev[0][j][5])
if sta_ev[0][j][6]:
tr.stats.sac.stel = float(sta_ev[0][j][6])
if sta_ev[0][j][7]:
tr.stats.sac.stdp = float(sta_ev[0][j][7])
if sta_ev[0][j][9]:
tr.stats.sac.evla = float(sta_ev[0][j][9])
if sta_ev[0][j][10]:
tr.stats.sac.evlo = float(sta_ev[0][j][10])
if sta_ev[0][j][11]:
tr.stats.sac.evdp = float(sta_ev[0][j][11])
if sta_ev[0][j][12]:
tr.stats.sac.mag = float(sta_ev[0][j][12])
tr.write(ls_saved_stas[j], format='SAC')
except Exception as e:
print '\nWARNING: %s' % e
print ls_saved_stas[j]
print '------------------'