def ARC_waveform(input_dics, events, Sta_req, i, req_type): """ Gets Waveforms, Response files and meta-data from ArcLink """ t_wave_1 = datetime.now() add_event = [] if req_type == 'save': period = '{0:s}_{1:s}_{2:s}_{3:s}'.\ format(input_dics['min_date'].split('T')[0], input_dics['max_date'].split('T')[0], str(input_dics['min_mag']), str(input_dics['max_mag'])) eventpath = os.path.join(input_dics['datapath'], period) for k in range(len(events)): add_event.append(os.path.join(eventpath, events[k]['event_id'])) if not os.path.isfile(os.path.join(add_event[k], 'info', 'event.pkl')): events_fio = open(os.path.join(add_event[k], 'info', 'event.pkl'), 'w') pickle.dump(events[k], events_fio) events_fio.close() elif req_type == 'update': events, add_event = \ quake_info(input_dics['arc_update'], target='info') if input_dics['test'] == 'Y': len_req_arc = input_dics['test_num'] else: len_req_arc = len(Sta_req) ARC_serial_parallel(i, events, add_event, Sta_req, input_dics, len_req_arc) if input_dics['resample_raw']: print '\nResample RAW traces to %sHz...' % input_dics['resample_raw'], resample_all(i=i, address_events=add_event, des_sr=input_dics['resample_raw'], resample_method=input_dics['resample_method']) print 'DONE' if input_dics['SAC'] == 'Y': print '\nConverting the MSEED files to SAC...', writesac_all(i=i, address_events=add_event) print 'DONE' try: len_sta_ev_open = open(os.path.join(add_event[i], 'info', 'station_event'), 'r') len_sta_ev = len(len_sta_ev_open.readlines()) except IOError: len_sta_ev = 'Can not open station_event file: %s' \ % (os.path.join(add_event[i], 'info', 'station_event')) ARC_reporter(i, add_event, events, input_dics, Sta_req, len_sta_ev, req_type, t_wave_1)
def FDSN_update(input_dics, address): """ Initialize directories and pass the required stations (removing the duplications) for FDSN update requests :param input_dics: :param address: :return: """ print '\n*********************' print 'FDSN -- Updating Mode' print '*********************' t_update_1 = datetime.now() events, address_events = quake_info(address, 'info') for i in range(len(events)): if not input_dics['list_stas']: Stas_fdsn = FDSN_available(input_dics, events[i], address_events[i], event_number=i) else: Stas_fdsn = read_list_stas(input_dics['list_stas'], input_dics['normal_mode_syn'], input_dics['specfem3D']) if input_dics['fdsn_bulk'] != 'Y': print '\n%s-Availability for event: %s/%s ---> DONE' \ % (input_dics['fdsn_base_url'], i+1, len(events)) else: print '\nFDSN-bulkfile for event: %s/%s ---> DONE' \ % (i+1, len(events)) if Stas_fdsn != [[]]: Stas_req = rm_duplicate(Stas_fdsn, address=os.path.join(address_events[i])) else: Stas_req = None print '------------------------------' print 'There is no available station!' print '------------------------------' if not os.path.isdir(os.path.join(address_events[i], 'BH_RAW')): os.makedirs(os.path.join(address_events[i], 'BH_RAW')) if Stas_req: FDSN_waveform(input_dics, events, Stas_req, i, req_type='update') else: print '\nNo available station in %s for your request!' \ % input_dics['fdsn_base_url'] print 'Check the next event...' continue print '\nTotal time for updating FDSN: %s' % (datetime.now() - t_update_1)
def ARC_waveform(input_dics, events, Sta_req, i, req_type): """ Gets Waveforms, Response files and meta-data from ArcLink """ t_wave_1 = datetime.now() add_event = [] if req_type == 'save': period = '{0:s}_{1:s}'.\ format(input_dics['min_date'].split('T')[0], input_dics['max_date'].split('T')[0]) eventpath = os.path.join(input_dics['datapath'], period) for k in range(len(events)): add_event.append(os.path.join(eventpath, events[k]['event_id'])) if not os.path.isfile( os.path.join(add_event[k], 'info', 'event.pkl')): events_fio = open( os.path.join(add_event[k], 'info', 'event.pkl'), 'w') pickle.dump(events[k], events_fio) events_fio.close() elif req_type == 'update': events, add_event = \ quake_info(input_dics['arc_update'], target='info') if input_dics['test'] == 'Y': len_req_arc = input_dics['test_num'] else: len_req_arc = len(Sta_req) ARC_serial_parallel(i, events, add_event, Sta_req, input_dics, len_req_arc) if input_dics['resample_raw']: print '\nResample RAW traces to %sHz...' % input_dics['resample_raw'], resample_all(i=i, address_events=add_event, des_sr=input_dics['resample_raw'], resample_method=input_dics['resample_method']) print 'DONE' if input_dics['SAC'] == 'Y': print '\nConverting the MSEED files to SAC...', writesac_all(i=i, address_events=add_event) print 'DONE' try: len_sta_ev_open = open( os.path.join(add_event[i], 'info', 'station_event'), 'r') len_sta_ev = len(len_sta_ev_open.readlines()) except IOError: len_sta_ev = 'Can not open station_event file: %s' \ % (os.path.join(add_event[i], 'info', 'station_event')) ARC_reporter(i, add_event, events, input_dics, Sta_req, len_sta_ev, req_type, t_wave_1)
def ARC_update(input_dics, address): """ Initialize directories and pass the required stations (removing the duplications) for ArcLink update requests :param input_dics: :param address: :return: """ print '\n************************' print 'ArcLink -- Updating Mode' print '************************' t_update_1 = datetime.now() events, address_events = quake_info(address, 'info') for i in range(len(events)): if not input_dics['list_stas']: Stas_arc = ARC_available(input_dics, events[i], address_events[i], event_number=i) else: Stas_arc = read_list_stas(input_dics['list_stas'], input_dics['normal_mode_syn'], input_dics['specfem3D']) print '\nArcLink-Availability for event: %s/%s ---> DONE' \ % (i+1, len(events)) if Stas_arc != [[]]: Stas_req = rm_duplicate(Stas_arc, address=os.path.join(address_events[i])) else: Stas_req = None print '------------------------------' print 'There is no available station!' print '------------------------------' if not os.path.isdir(os.path.join(address_events[i], 'BH_RAW')): os.makedirs(os.path.join(address_events[i], 'BH_RAW')) if Stas_req: ARC_waveform(input_dics, events, Stas_req, i, req_type='update') else: print '\nNo available station in ArcLink for your request!' print 'Check the next event...' continue print '\nTotal time for updating ArcLink: %s' \ % (datetime.now() - t_update_1)
def FDSN_ARC_merge(input_dics, clients): """ Call "merge_stream" function that merges the retrieved waveforms in continuous request :param input_dics: :param clients: :return: """ print '\n*****************************' print '%s -- Merging the waveforms' % clients print '*****************************' # Following two if-conditions create address to which merging # should be applied # Please note that these two conditions can not happen at the same time address = None network_name = None BH_file = None if clients == 'arc': clients_name = 'arc' else: clients_name = 'fdsn' if input_dics[clients_name + '_merge_auto'] == 'Y': period = '{0:s}_{1:s}_{2:s}_{3:s}'.\ format(input_dics['min_date'].split('T')[0], input_dics['max_date'].split('T')[0], str(input_dics['min_mag']), str(input_dics['max_mag'])) address = os.path.join(input_dics['datapath'], period) if input_dics[clients_name + '_merge'] != 'N': address = input_dics[clients_name + '_merge'] events, address_events = quake_info(address, 'info') ls_saved_stas_tmp = [] for i in range(len(events)): sta_ev = read_station_event(address_events[i]) # initialize some parameters which will be used later in merge_stream for s_ev in sta_ev[0]: if input_dics[clients_name + '_merge_auto'] == 'Y': if clients == s_ev[13]: network, network_name, BH_file = \ init_merging(input_dics, s_ev[0]) station_id = '%s.%s.%s.%s' % (network, s_ev[1], s_ev[2], s_ev[3]) ls_saved_stas_tmp.append( os.path.join(address_events[i], BH_file, station_id)) else: network, network_name, BH_file = \ init_merging(input_dics, s_ev[0]) station_id = '%s.%s.%s.%s' % (network, s_ev[1], s_ev[2], s_ev[3]) ls_saved_stas_tmp.append( os.path.join(address_events[i], BH_file, station_id)) if not input_dics['net'].startswith('_'): pattern_sta = '%s.%s.%s.%s' % (input_dics['net'], input_dics['sta'], input_dics['loc'], input_dics['cha']) else: pattern_sta = '*.%s.%s.%s' % (input_dics['sta'], input_dics['loc'], input_dics['cha']) ls_saved_stas = [] for saved_sta in ls_saved_stas_tmp: if fnmatch.fnmatch(os.path.basename(saved_sta), pattern_sta): ls_saved_stas.append(saved_sta) if len(ls_saved_stas) != 0: saved_stations_names = [] for ls_saved_sta in ls_saved_stas: saved_stations_names.append(os.path.basename(ls_saved_sta)) ls_sta = list(set(saved_stations_names)) ls_address = [] for add_ev in address_events: ls_address.append(os.path.join(add_ev, BH_file)) print 'Merging the waveforms...' merge_stream(input_dics=input_dics, ls_address=ls_address, ls_sta=ls_sta, network_name=network_name) print 'Finish merging the waveforms' else: print "\nThere is no waveform to merge!"
def FDSN_waveform(input_dics, events, Sta_req, i, req_type): """ Gets Waveforms, StationXML files and meta-data from FDSN :param input_dics: :param events: :param Sta_req: :param i: :param req_type: :return: """ t_wave_1 = datetime.now() add_event = [] if req_type == 'save': period = '{0:s}_{1:s}'.\ format(input_dics['min_date'].split('T')[0], input_dics['max_date'].split('T')[0]) eventpath = os.path.join(input_dics['datapath'], period) for k in range(len(events)): add_event.append(os.path.join(eventpath, events[k]['event_id'])) events_fio = open(os.path.join(add_event[k], 'info', 'event.pkl'), 'w') pickle.dump(events[k], events_fio) events_fio.close() elif req_type == 'update': events, add_event = \ quake_info(input_dics['fdsn_update'], target='info') if input_dics['test'] == 'Y': len_req_fdsn = input_dics['test_num'] else: len_req_fdsn = len(Sta_req) if input_dics['fdsn_bulk'] == 'Y': t11 = datetime.now() try: FDSN_bulk_request(i, add_event, input_dics) except Exception as e: print 'WARNING: %s' % e print 'DONE' # Following parameter is set to 'N' to avoid # retrieving the waveforms twice # When using bulk requests, waveforms are retreived in bulk # but not response/StationXML files and not metadata input_dics['waveform'] = 'N' t22 = datetime.now() print '\nbulkdataselect request is done for event: %s/%s in %s' \ % (i+1, len(events), t22-t11) FDSN_serial_parallel(i, events, add_event, Sta_req, input_dics, len_req_fdsn) if input_dics['resample_raw']: print '\nResample RAW traces to %sHz...' % input_dics['resample_raw'], resample_all(i=i, address_events=add_event, des_sr=input_dics['resample_raw'], resample_method=input_dics['resample_method']) print 'DONE' if input_dics['SAC'] == 'Y': print '\nConverting the MSEED files to SAC...', writesac_all(i=i, address_events=add_event) print 'DONE' try: len_sta_ev_open = open(os.path.join(add_event[i], 'info', 'station_event'), 'r') len_sta_ev = len(len_sta_ev_open.readlines()) except IOError: len_sta_ev = 'Can not open station_event file: %s' \ % (os.path.join(add_event[i], 'info', 'station_event')) FDSN_reporter(i, add_event, events, input_dics, Sta_req, len_sta_ev, req_type, t_wave_1)
def FDSN_ARC_IC(input_dics, clients): """ Call "inst_correct" function based on the request. Group the stations that have been retrieved from specific client Grouping is necessary in applying the instrument correction correctly (different clients are treated differently) :param input_dics: :param clients: :return: """ print '\n*****************************' print '%s -- Instrument Correction' % clients print '*****************************' # Following two if-conditions create address to which # instrument correction should be applied # Please note that these two conditions can not happen at the same time address = None if clients == 'arc': clients_name = 'arc' else: clients_name = 'fdsn' if input_dics[clients_name + '_ic_auto'] == 'Y': period = '{0:s}_{1:s}_{2:s}_{3:s}'.\ format(input_dics['min_date'].split('T')[0], input_dics['max_date'].split('T')[0], str(input_dics['min_mag']), str(input_dics['max_mag'])) address = os.path.join(input_dics['datapath'], period) if input_dics['fdsn_update'] != 'N': address = os.path.join(input_dics['datapath']) if input_dics['arc_update'] != 'N': address = os.path.join(input_dics['datapath']) if input_dics[clients_name + '_ic'] != 'N': address = input_dics[clients_name + '_ic'] events, address_events = quake_info(address, 'info') for i in range(len(events)): sta_ev = read_station_event(address_events[i]) ls_saved_stas_tmp = [] for s_ev in sta_ev[0]: if clients.lower() == 'all_fdsn': if not s_ev[13].lower() == 'arc': station_id = '%s.%s.%s.%s' % (s_ev[0], s_ev[1], s_ev[2], s_ev[3]) ls_saved_stas_tmp.append( os.path.join(address_events[i], 'BH_RAW', station_id)) elif clients.lower() == s_ev[13].lower(): station_id = '%s.%s.%s.%s' % (s_ev[0], s_ev[1], s_ev[2], s_ev[3]) ls_saved_stas_tmp.append( os.path.join(address_events[i], 'BH_RAW', station_id)) if input_dics['net'].startswith('_'): net_pat = '*' else: net_pat = input_dics['net'] ls_saved_stas = [] for net_iter in net_pat.split(','): for sta_iter in input_dics['sta'].split(','): for loc_iter in input_dics['loc'].split(','): for cha_iter in input_dics['cha'].split(','): pattern_sta = '%s.%s.%s.%s' % (net_iter, sta_iter, loc_iter, cha_iter) for saved_sta in ls_saved_stas_tmp: if fnmatch.fnmatch(os.path.basename(saved_sta), pattern_sta): ls_saved_stas.append(saved_sta) if len(ls_saved_stas) != 0: print '\nevent: %s/%s -- %s\n' % (i + 1, len(events), clients) inst_correct(input_dics, ls_saved_stas, address_events[i], clients) else: print "There is no station in the directory to correct!" print "Address: %s" % address_events[i] # pass address for using in create_tar_file return address
def FDSN_waveform(input_dics, events, Sta_req, i, req_type): """ Gets Waveforms, StationXML files and meta-data from FDSN :param input_dics: :param events: :param Sta_req: :param i: :param req_type: :return: """ t_wave_1 = datetime.now() add_event = [] if req_type == 'save': period = '{0:s}_{1:s}'.\ format(input_dics['min_date'].split('T')[0], input_dics['max_date'].split('T')[0]) eventpath = os.path.join(input_dics['datapath'], period) for k in range(len(events)): add_event.append(os.path.join(eventpath, events[k]['event_id'])) events_fio = open(os.path.join(add_event[k], 'info', 'event.pkl'), 'w') pickle.dump(events[k], events_fio) events_fio.close() elif req_type == 'update': events, add_event = \ quake_info(input_dics['fdsn_update'], target='info') if input_dics['test'] == 'Y': len_req_fdsn = input_dics['test_num'] else: len_req_fdsn = len(Sta_req) if input_dics['fdsn_bulk'] == 'Y': t11 = datetime.now() try: FDSN_bulk_request(i, add_event, input_dics) except Exception as e: print 'WARNING: %s' % e print 'DONE' # Following parameter is set to 'N' to avoid # retrieving the waveforms twice # When using bulk requests, waveforms are retreived in bulk # but not response/StationXML files and not metadata input_dics['waveform'] = 'N' t22 = datetime.now() print '\nbulkdataselect request is done for event: %s/%s in %s' \ % (i+1, len(events), t22-t11) FDSN_serial_parallel(i, events, add_event, Sta_req, input_dics, len_req_fdsn) if input_dics['resample_raw']: print '\nResample RAW traces to %sHz...' % input_dics['resample_raw'], resample_all(i=i, address_events=add_event, des_sr=input_dics['resample_raw'], resample_method=input_dics['resample_method']) print 'DONE' if input_dics['SAC'] == 'Y': print '\nConverting the MSEED files to SAC...', writesac_all(i=i, address_events=add_event) print 'DONE' try: len_sta_ev_open = open( os.path.join(add_event[i], 'info', 'station_event'), 'r') len_sta_ev = len(len_sta_ev_open.readlines()) except IOError: len_sta_ev = 'Can not open station_event file: %s' \ % (os.path.join(add_event[i], 'info', 'station_event')) FDSN_reporter(i, add_event, events, input_dics, Sta_req, len_sta_ev, req_type, t_wave_1)
def FDSN_ARC_IC(input_dics, clients): """ Call "inst_correct" function based on the request. Group the stations that have been retrieved from specific client Grouping is necessary in applying the instrument correction correctly (different clients are treated differently) :param input_dics: :param clients: :return: """ if len(input_dics['fdsn_base_url_rest']) > 0: clients = 'all_fdsn' print '\n*****************************' print '%s -- Instrument Correction' % clients print '*****************************' # Following two if-conditions create address to which # instrument correction should be applied # Please note that these two conditions can not happen at the same time address = None if clients == 'arc': clients_name = 'arc' else: clients_name = 'fdsn' if input_dics[clients_name + '_ic_auto'] == 'Y': period = '{0:s}_{1:s}'.\ format(input_dics['min_date'].split('T')[0], input_dics['max_date'].split('T')[0]) address = os.path.join(input_dics['datapath'], period) if input_dics['fdsn_update'] != 'N': address = os.path.join(input_dics['datapath']) if input_dics['arc_update'] != 'N': address = os.path.join(input_dics['datapath']) if input_dics[clients_name + '_ic'] != 'N': address = input_dics[clients_name + '_ic'] events, address_events = quake_info(address, 'info') for i in range(len(events)): sta_ev = read_station_event(address_events[i]) ls_saved_stas_tmp = [] for s_ev in sta_ev[0]: if clients.lower() == 'all_fdsn': if not s_ev[13].lower() == 'arc': station_id = '%s.%s.%s.%s' % (s_ev[0], s_ev[1], s_ev[2], s_ev[3]) ls_saved_stas_tmp.append( os.path.join(address_events[i], 'BH_RAW', station_id)) elif clients.lower() == s_ev[13].lower(): station_id = '%s.%s.%s.%s' % (s_ev[0], s_ev[1], s_ev[2], s_ev[3]) ls_saved_stas_tmp.append( os.path.join(address_events[i], 'BH_RAW', station_id)) if input_dics['net'].startswith('_'): net_pat = '*' else: net_pat = input_dics['net'] ls_saved_stas = [] for net_iter in net_pat.split(','): for sta_iter in input_dics['sta'].split(','): for loc_iter in input_dics['loc'].split(','): for cha_iter in input_dics['cha'].split(','): pattern_sta = '%s.%s.%s.%s' % (net_iter, sta_iter, loc_iter, cha_iter) for saved_sta in ls_saved_stas_tmp: if fnmatch.fnmatch(os.path.basename(saved_sta), pattern_sta): ls_saved_stas.append(saved_sta) if len(ls_saved_stas) != 0: print '\nevent: %s/%s -- %s\n' % (i+1, len(events), clients) inst_correct(input_dics, ls_saved_stas, address_events[i], clients) else: print "There is no station in the directory to correct!" print "Address: %s" % address_events[i] # pass address for using in create_tar_file return address
def FDSN_waveform(input_dics, events, Sta_req, i, req_type): """ Gets Waveforms, StationXML files and meta-data from FDSN :param input_dics: :param events: :param Sta_req: :param i: :param req_type: :return: """ t_wave_1 = datetime.now() add_event = [] if req_type == "save": period = "{0:s}_{1:s}".format(input_dics["min_date"].split("T")[0], input_dics["max_date"].split("T")[0]) eventpath = os.path.join(input_dics["datapath"], period) for k in range(len(events)): add_event.append(os.path.join(eventpath, events[k]["event_id"])) events_fio = open(os.path.join(add_event[k], "info", "event.pkl"), "w") pickle.dump(events[k], events_fio) events_fio.close() elif req_type == "update": events, add_event = quake_info(input_dics["fdsn_update"], target="info") if input_dics["test"] == "Y": len_req_fdsn = input_dics["test_num"] else: len_req_fdsn = len(Sta_req) if input_dics["fdsn_bulk"] == "Y": t11 = datetime.now() try: FDSN_bulk_request(i, add_event, input_dics) except Exception as e: print "WARNING: %s" % e print "DONE" # Following parameter is set to 'N' to avoid # retrieving the waveforms twice # When using bulk requests, waveforms are retreived in bulk # but not response/StationXML files and not metadata input_dics["waveform"] = "N" t22 = datetime.now() print "\nbulkdataselect request is done for event: %s/%s in %s" % (i + 1, len(events), t22 - t11) FDSN_serial_parallel(i, events, add_event, Sta_req, input_dics, len_req_fdsn) if input_dics["resample_raw"]: print "\nResample RAW traces to %sHz..." % input_dics["resample_raw"], resample_all( i=i, address_events=add_event, des_sr=input_dics["resample_raw"], resample_method=input_dics["resample_method"], ) print "DONE" if input_dics["SAC"] == "Y": print "\nConverting the MSEED files to SAC...", writesac_all(i=i, address_events=add_event) print "DONE" try: len_sta_ev_open = open(os.path.join(add_event[i], "info", "station_event"), "r") len_sta_ev = len(len_sta_ev_open.readlines()) except IOError: len_sta_ev = "Can not open station_event file: %s" % (os.path.join(add_event[i], "info", "station_event")) FDSN_reporter(i, add_event, events, input_dics, Sta_req, len_sta_ev, req_type, t_wave_1)
def FDSN_ARC_merge(input_dics, clients): """ Call "merge_stream" function that merges the retrieved waveforms in continuous request :param input_dics: :param clients: :return: """ print '\n*****************************' print '%s -- Merging the waveforms' % clients print '*****************************' # Following two if-conditions create address to which merging # should be applied # Please note that these two conditions can not happen at the same time address = None network_name = None BH_file = None if clients == 'arc': clients_name = 'arc' else: clients_name = 'fdsn' if input_dics[clients_name + '_merge_auto'] == 'Y': period = '{0:s}_{1:s}'.\ format(input_dics['min_date'].split('T')[0], input_dics['max_date'].split('T')[0]) address = os.path.join(input_dics['datapath'], period) if input_dics[clients_name + '_merge'] != 'N': address = input_dics[clients_name + '_merge'] events, address_events = quake_info(address, 'info') ls_saved_stas_tmp = [] for i in range(len(events)): sta_ev = read_station_event(address_events[i]) # initialize some parameters which will be used later in merge_stream for s_ev in sta_ev[0]: if input_dics[clients_name + '_merge_auto'] == 'Y': if clients == s_ev[13]: network, network_name, BH_file = \ init_merging(input_dics, s_ev[0]) station_id = '%s.%s.%s.%s' % (network, s_ev[1], s_ev[2], s_ev[3]) ls_saved_stas_tmp.append( os.path.join(address_events[i], BH_file, station_id)) else: network, network_name, BH_file = \ init_merging(input_dics, s_ev[0]) station_id = '%s.%s.%s.%s' % (network, s_ev[1], s_ev[2], s_ev[3]) ls_saved_stas_tmp.append( os.path.join(address_events[i], BH_file, station_id)) if not input_dics['net'].startswith('_'): pattern_sta = '%s.%s.%s.%s' % (input_dics['net'], input_dics['sta'], input_dics['loc'], input_dics['cha']) else: pattern_sta = '*.%s.%s.%s' % (input_dics['sta'], input_dics['loc'], input_dics['cha']) ls_saved_stas = [] for saved_sta in ls_saved_stas_tmp: if fnmatch.fnmatch(os.path.basename(saved_sta), pattern_sta): ls_saved_stas.append(saved_sta) if len(ls_saved_stas) != 0: saved_stations_names = [] for ls_saved_sta in ls_saved_stas: saved_stations_names.append(os.path.basename(ls_saved_sta)) ls_sta = list(set(saved_stations_names)) ls_address = [] for add_ev in address_events: ls_address.append(os.path.join(add_ev, BH_file)) print 'Merging the waveforms...' merge_stream(input_dics=input_dics, ls_address=ls_address, ls_sta=ls_sta, network_name=network_name) print 'Finish merging the waveforms' else: print "\nThere is no waveform to merge!"