def test_raise_polynomial(self): with self.assertRaises(NotImplementedError) as e: read_inventory(self.arclink_xml_poly) self.assertEqual( e.exception.args[0], "responsePolynomial not" "implemented. Contact the ObsPy developers")
def setUp(self): """ Read example stationXML/sc3ml format to Inventory """ self.data_dir = os.path.join(os.path.dirname(__file__), "data") stationxml_path = os.path.join(self.data_dir, "EB_response_stationXML") sc3ml_path = os.path.join(self.data_dir, "EB_response_sc3ml") self.stationxml_inventory = read_inventory(stationxml_path, format="STATIONXML") self.sc3ml_inventory = read_inventory(sc3ml_path, format="SC3ML")
def setUp(self): """ Read example stationXML/sc3ml format to Inventory """ self.data_dir = os.path.join(os.path.dirname(os.path.abspath( inspect.getfile(inspect.currentframe()))), "data") stationxml_path = os.path.join(self.data_dir, "EB_response_stationXML") sc3ml_path = os.path.join(self.data_dir, "EB_response_sc3ml") self.stationxml_inventory = read_inventory(stationxml_path, format="STATIONXML") self.sc3ml_inventory = read_inventory(sc3ml_path, format="SC3ML")
def test_sc3ml_versions(self): """ Test multiple schema versions """ for version in ['0.5', '0.99']: filename = os.path.join(self.data_dir, 'version%s' % version) with self.assertRaises(ValueError) as e: with warnings.catch_warnings(): warnings.simplefilter("ignore") read_inventory(filename) self.assertEqual(e.exception.args[0], "Schema version not supported.")
def test_readwrite(self): """ Test calculating amplitudes """ fmin = 0.001 inv = read_inventory( str(Path(self.testing_path) / "stations_mayobs.xml"), 'STATIONXML') for net in inv: for sta in net: # print(f'{sta.code=}') ch = sta.select(channel='*[Z3]')[0] info_str = f'{sta.code}.{ch.code}' resp = ch.response # fig = resp.plot(min_freq=fmin, label=info_str, show=False) paz = PAZ.from_obspy_response(resp) # paz.plot(min_freq=fmin, label='PAZ', axes=fig.axes, sym='g.', # show=False) fname = f'{info_str}.json' paz.write_json_pz(fname) paz2 = PAZ.read_json_pz(fname) paz2.input_units = paz.input_units self.assertEqual(paz, paz2) self.assertTextFilesEqual(fname, Path(self.testing_path) / fname) os.remove(fname)
def read_STATIONXML(cls, filename, component): """ read JSON paz format """ inv = read_inventory(filename, 'STATIONXML') resp = inv.select(channel='*' + component)[0][0][0].response s_pz = [] s_others = [] stages = resp.response_stages sens = resp.instrument_sensitivity for s in stages: if isinstance(s, PolesZerosResponseStage): s_pz.append(s) else: s_others.append(s) if len(s_pz) > 1: paz_base = s_pz[0] for s in s_pz[1:]: paz_base.poles.extend(s.poles) paz_base.zeros.extend(s.zeros) paz_base.normalization_factor *= s.normalization_factor resp.response_stages = [paz_base] + s_others paz = resp.get_paz() val = cls(poles=paz.poles, zeros=paz.zeros, passband_gain=sens.value, ref_freq=sens.frequency, input_units=sens.input_units, output_units=sens.output_units) return val
def test_empty_depth(self): """ Assert depth, latitude, longitude, elevation set to 0.0 if left empty """ with warnings.catch_warnings(record=True) as w: read_inventory(os.path.join(self.data_dir, "sc3ml_empty_depth_and_id.sc3ml")) self.assertEqual(str(w[0].message), "Sensor is missing " "longitude information, " "using 0.0") self.assertEqual(str(w[1].message), "Sensor is missing " "latitude information, " "using 0.0") self.assertEqual(str(w[2].message), "Sensor is missing elevation " "information, using 0.0") self.assertEqual(str(w[3].message), "Channel is missing depth " "information, using 0.0")
def read_sta_inv(invfiles, sta): for invv in invfiles: inv = read_inventory(invv) nt0 = inv[0].select(station=sta) if bool(nt0): lat = nt0[0].latitude lon = nt0[0].longitude elev = nt0[0].elevation return lat, lon, elev
def test_sc3ml_versions(self): """ Test multiple schema versions """ for version in ['0.5', '0.6', '0.7', '0.8', '0.9']: filename = os.path.join(self.data_dir, 'version%s' % version) read_inventory(filename) for version in ['0.3', '0.10']: filename = os.path.join(self.data_dir, 'version%s' % version) with self.assertRaises(ValueError) as e: with warnings.catch_warnings(): warnings.simplefilter("ignore") read_inventory(filename) self.assertEqual(e.exception.args[0], "Schema version not supported.")
def test_channel_level(self): """ Test inventory without repsonse information up to channel level """ inv = read_inventory(os.path.join(self.data_dir, "channel_level.sc3ml")) self.assertEqual(inv[0].code, "NL") self.assertEqual(inv[0][0].code, "HGN") for cha in inv[0][0].channels: self.assertTrue(cha.code in ["BHE", "BHN", "BHZ"])
def test_stage_empty_poles_and_zeros(self): """ Tests for a case where the poles and zeros are empty see #2633 """ sc3ml_mbar_path = os.path.join(self.data_dir, "zero_poles_and_zeros.sc3ml") sc3ml_inv = read_inventory(sc3ml_mbar_path) response = sc3ml_inv[0][0][0].response zeros = response.response_stages[1].zeros poles = response.response_stages[1].poles self.assertEqual(zeros, []) self.assertEqual(poles, [])
def test_sc3ml_versions(self): """ Test multiple schema versions """ read_inventory(os.path.join(self.data_dir, "version0.7")) read_inventory(os.path.join(self.data_dir, "version0.8")) read_inventory(os.path.join(self.data_dir, "version0.9")) with self.assertRaises(ValueError) as e: with warnings.catch_warnings(): warnings.simplefilter("ignore") read_inventory(os.path.join(self.data_dir, "version0.10")) self.assertEqual(e.exception.args[0], "Schema version not supported.")
def read_sta_inv(invfile, sta): inv = read_inventory(invfile) nt0 = inv[0].select(station=sta) if nt0: lat = nt0[0].latitude lon = nt0[0].longitude elev = nt0[0].elevation print(sta, lat, lon, elev) else: lat = 999 lon = 999 elev = 999 return lat, lon, elev
def test_analogue_filter_chain(self): """ Test analogue filter chain and compare to stationXML equivalent """ arclink_inv = read_inventory(self.arclink_afc_path) station_inv = read_inventory(self.station_afc_path) arclink_resp = arclink_inv[0][0][0].response.response_stages station_resp = station_inv[0][0][0].response.response_stages for arclink_stage, station_stage in zip(arclink_resp, station_resp): self.assertEqual(arclink_stage.stage_gain, station_stage.stage_gain) self.assertEqual(arclink_stage.stage_sequence_number, station_stage.stage_sequence_number) for arc_cs, sta_cs in zip(arclink_stage.__dict__.items(), station_stage.__dict__.items()): if arc_cs[0] in ['name', 'resource_id2']: continue self.assertEqual(arc_cs, sta_cs)
def read_stationxml(cls, filename, channel, station='*'): """ read JSON paz format """ inv = read_inventory(filename, 'STATIONXML') inv = inv.select(channel=channel, station=station) nscs = [{ 'net': n.code, 'sta': s.code, 'chan': c } for n in inv for s in n for c in s] if len(nscs) > 1: nsc_strs = [ f'{x["net"]}.{x["sta"]}.{x["chan"].code}' for x in nscs ] print('Multiple channels were selected, using {nsc_strs[0]}') print('Other channels were: ' + ','.join(nsc_strs[1:])) resp = nscs[0]["chan"].response return cls.from_obspy_response(resp)
def read_inventory(path_or_file_object, format='STATIONXML', xy_from_lat_lon=False, input_projection=4326, output_projection=None, *args, **kwargs) -> Inventory: """ Read inventory file :param path_or_file_object: the path to the inventory file or a file object :param format: the format :param xy_from_lat_lon: if True convert populate the XY field by converting the latitude and longitude to UTM :param input_projection: The input projection. Applicable if xy_from_lat_lon is True (default=4326 for for latitude longitude) :param output_projection: The output projection. Has to be specified if xy_from_lat_lon is True. Default=None :param args: see obspy.core.inventory.read_inventory for more information :param kwargs: see obspy.core.inventory.read_inventory for more information :return: an inventory object :rtype: ~uquake.core.inventory.Inventory """ if xy_from_lat_lon and (output_projection is None): raise ValueError('the output projection is needed for conversion' 'from latitude-longitude to UTM') if type(path_or_file_object) is Path: path_or_file_object = str(path_or_file_object) # del kwargs['xy_from_lat_lon'] obspy_inv = inventory.read_inventory(str(path_or_file_object), format=format, *args, **kwargs) return Inventory.from_obspy_inventory_object( obspy_inv, xy_from_lat_lon=xy_from_lat_lon, output_projection=output_projection, input_projection=input_projection)
def _data(): """returns a dict with fields 'mseed', 'mseed_ACC', 'mseed_VEL', 'mseed_DISP' (all Streams. The latter three after removing the response) 'inventory' (an inventory object) and two strings: 'mseed_path' and 'inventory_path'""" folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'data') mseed_path = os.path.join(folder, 'trace_GE.APE.mseed') mseed = obspy_read(mseed_path) inv_path = os.path.join(folder, 'inventory_GE.APE.xml') s = StringIO() with open(inv_path) as _opn: s.write(_opn.read()) s.seek(0) inv_obj = read_inventory(s) ret = {'mseed': mseed, 'inventory': inv_obj, 'mseed_path': mseed_path, 'data_path': folder, 'inventory_path': inv_path} for inv_output in ['ACC', 'VEL', 'DISP']: mseed2 = remove_response(mseed, inv_obj, output=inv_output) ret['mseed_'+inv_output] = mseed2 return ret
def read_stationxml(cls, filename, channel, station='*'): """ read StationXML (returns first valid response) """ inv = read_inventory(filename, 'STATIONXML') inv = inv.select(channel=channel, station=station) nscs = [{ 'net': n.code, 'sta': s.code, 'chan': c } for n in inv for s in n for c in s] if len(nscs) > 1: nsc_strs = [ f'{x["net"]}.{x["sta"]}.{x["chan"].code}' for x in nscs ] print('Multiple channels were selected, using {nsc_strs[0]}') print('Other channels were: ' + ','.join(nsc_strs[1:])) elif len(nscs) == 0: raise NameError(f'Did not find {station}, {channel} in {filename}') resp = nscs[0]["chan"].response return cls.from_obspy_response(resp)
def test_auto_read_arclink_xml(self): arclink_inv = read_inventory(self.arclink_xml_path) self.assertIsNotNone(arclink_inv) station_inv = read_inventory(self.station_xml_path) arclink_inv = arclink_inv.select(station="GOLS") channels = ['HH1', 'HH2', 'HHZ', 'BHZ', 'BH1', 'BH2', 'LH1', 'LH2', 'LHZ'] for channel in channels: arclink_cha = arclink_inv.select(channel=channel) station_cha = station_inv.select(channel=channel) for arc, st_xml in zip(arclink_cha[0][0], station_cha[0][0]): self.assertEqual(arc.code, st_xml.code) self.assertEqual(arc.latitude, st_xml.latitude) self.assertEqual(arc.longitude, st_xml.longitude) self.assertEqual(arc.depth, st_xml.depth) self.assertEqual(arc.azimuth, st_xml.azimuth) self.assertEqual(arc.dip, st_xml.dip) self.assertEqual(arc.sample_rate, st_xml.sample_rate) self.assertEqual(arc.start_date, st_xml.start_date) self.assertEqual(arc.end_date, st_xml.end_date) # reading stationxml will ignore old StationXML 1.0 defined # StorageFormat, Arclink Inventory XML and SC3ML get it stored # in extra now with warnings.catch_warnings(record=True): self.assertEqual(st_xml.storage_format, None) self.assertEqual(arc.storage_format, None) self.assertEqual( arc.extra['format'], {'namespace': SCHEMA_NAMESPACE, 'value': None}) cdisps = "clock_drift_in_seconds_per_sample" self.assertEqual(getattr(arc, cdisps), getattr(st_xml, cdisps)) # Compare datalogger element for arc_el, st_xml_el in zip(arc.data_logger.__dict__.items(), st_xml.data_logger.__dict__ .items()): self.assertEqual(arc_el, st_xml_el) # Compare sensor element for arc_sen, st_xml_sen in zip(arc.sensor.__dict__.items(), st_xml.sensor.__dict__.items()): # Skip the type; set for stationXML not for ArclinkXML if arc_sen[0] == 'type': continue self.assertEqual(arc_sen, st_xml_sen) # Same number of response stages self.assertEqual(len(arc.response.response_stages), len(st_xml.response.response_stages)) # Check the response stages for arc_resp, sta_resp in zip(arc.response.response_stages, st_xml.response.response_stages): self.assertEqual(arc_resp.stage_gain, sta_resp.stage_gain) self.assertEqual(arc_resp.stage_sequence_number, sta_resp.stage_sequence_number) for arc_cs, sta_cs in zip(arc_resp.__dict__.items(), sta_resp.__dict__.items()): # Name can be different and resource_id is not given # in stationXML if arc_cs[0] in ['name', 'resource_id2']: continue # Floating point precision troubles.. # 0.021099999999999997 != 0.0211 if isinstance(arc_cs[1], float): self.assertAlmostEqual(arc_cs[1], sta_cs[1]) else: self.assertEqual(arc_cs, sta_cs)
def test_publicid_slash(self): v = read_inventory(os.path.join(self.data_dir, "public-id-slash.xml")) self.assertEqual(len(v[0][0][0].response.response_stages), 11)
def process(input_folder, inventory, output_file_name, min_length_sec, merge_threshold, ntraces_per_file): """ INPUT_FOLDER: Path to input folder containing miniseed files \n INVENTORY: Path to FDSNStationXML inventory containing channel-level metadata for all stations \n OUTPUT_FILE_NAME: Name of output ASDF file \n """ comm = MPI.COMM_WORLD nproc = comm.Get_size() rank = comm.Get_rank() # Read inventory inv = None try: inv = read_inventory(inventory) except Exception as e: print(e) # end try files = np.array(glob.glob(input_folder + '/*.mseed')) random.Random(nproc).shuffle(files) # files = files[:100] ustations = set() ustationInv = defaultdict(list) networklist = [] stationlist = [] for file in files: _, _, net, sta, _ = file.split('.') ustations.add('%s.%s' % (net, sta)) networklist.append(net) stationlist.append(sta) # end for networklist = np.array(networklist) stationlist = np.array(stationlist) idx = np.lexsort((networklist, stationlist)) files = files[idx] myfiles = split_list(files, nproc)[rank] if (rank == 0): for i, ustation in enumerate(ustations): net, sta = ustation.split('.') sinv = inv.select(network=net, station=sta) if (not len(sinv.networks)): print(('Missing station: %s.%s' % (net, sta))) ustationInv[ustation] = None else: ustationInv[ustation] = sinv # end if # end for # end if ustationInv = comm.bcast(ustationInv, root=0) # Extract trace-count-lists in parallel mytrccountlist = np.zeros(len(myfiles)) for ifile, file in enumerate(tqdm(myfiles)): try: st = read(file, headonly=True) mytrccountlist[ifile] = len(st) # end if except Exception as e: print(e) # end try # end for trccountlist = comm.gather(mytrccountlist, root=0) if (rank == 0): trccountlist = np.array( [item for sublist in trccountlist for item in sublist]) # Some mseed files can be problematic in terms of having way too many traces -- # e.g. 250k+ traces, each a couple of samples long, for a day mseed file. We # need to blacklist them and exclude them from the ASDF file. print(('Blacklisted %d files out of %d files; ' \ 'average trace-count %f, std: %f' % (np.sum(trccountlist > ntraces_per_file), len(trccountlist), np.mean(trccountlist), np.std(trccountlist)))) f = open(str(os.path.splitext(output_file_name)[0] + '.trccount.txt'), 'w+') for i in range(len(files)): f.write('%s\t%d\n' % (files[i], trccountlist[i])) # end for f.close() if (os.path.exists(output_file_name)): os.remove(output_file_name) ds = pyasdf.ASDFDataSet(output_file_name, compression='gzip-3', mpi=False) for ifile, file in enumerate(tqdm(files)): st = [] if (trccountlist[ifile] > ntraces_per_file): continue else: try: st = read(file) except Exception as e: print(e) continue # end try # end if if (len(st)): netsta = st[0].stats.network + '.' + st[0].stats.station if (ustationInv[netsta]): if (merge_threshold): ntraces = len(st) if (ntraces > merge_threshold): try: st = st.merge(method=1, fill_value='interpolate') except: print('Failed to merge traces. Moving along..') continue # end try print( ('Merging stream with %d traces' % (ntraces))) # end if # end if # end if for tr in st: if (tr.stats.npts == 0): continue if (min_length_sec): if (tr.stats.npts * tr.stats.delta < min_length_sec): continue # end if asdfTag = make_ASDF_tag(tr, "raw_recording").encode('ascii') try: ds.add_waveforms(tr, tag='raw_recording') except Exception as e: print(e) print('Failed to append trace:') print(tr) # end try # end for try: ds.add_stationxml(ustationInv[netsta]) except Exception as e: print(e) print('Failed to append inventory:') print((ustationInv[netsta])) # end try # end if # end for print('Closing asdf file..') del ds
f.write('Orient 1 2, ' + str(st[0].stats.starttime.julday).zfill(3) + ', ' + str(st[0].stats.starttime.hour).zfill(2) + ', ' + str(theta1) + ', ' + str(rot1(theta1)) + '\n') f.write('Orient 2 3, ' + str(st[0].stats.starttime.julday).zfill(3) + ', ' + str(st[0].stats.starttime.hour).zfill(2) + ', ' + str(theta2) + ', ' + str(rot2(theta2)) + '\n') f.write('Orient 1 3, ' + str(st[0].stats.starttime.julday).zfill(3) + ', ' + str(st[0].stats.starttime.hour).zfill(2) + ', ' + str(theta3) + ', ' + str(rot3(theta3)) + '\n') return resps = glob.glob('metadata/RESP*') for resp in resps: if 'inv' not in vars(): inv = read_inventory(resp) else: inv += read_inventory(resp) for stime, etime in zip(stimes, etimes): # This loop represents a single test (we will parse data in windows for each test) ctime = stime st = Stream() while ctime <= etime: st += read('/msd/XX_TST4/2020/' + str(ctime.julday).zfill(3) + '/*BH*') st += read('/msd/GS_ALQ2/2020/' + str(ctime.julday).zfill(3) + '/00_BH*') ctime += 24 * 60 * 60 st.merge(fill_value=0) st.trim(stime, etime) st.sort()