def main(argv): inv = read_inventory("IRIS-ALL.xml") # if os.path.exists("IRIS-ALL.pkl"): # doesn't work on CentOS for some reason # with open('IRIS-ALL.pkl', 'rb') as f: # import cPickle as pkl # inv = pkl.load(f) # else: # inv = read_inventory("IRIS-ALL.xml") # with open('IRIS-ALL.pkl', 'wb') as f: # import pickle as pkl # pkl.dump(inv, f, pkl.HIGHEST_PROTOCOL) sensorDict, responseDict = extract_unique_sensors_responses(inv) print('\nFound {0} response objects with keys: {1}'.format(len(responseDict.keys()), responseDict.keys())) # unknown stations in Indonesia are usually installed by Potsdam and we assume they have network name GE default_net = 'GE' ehb1 = read_eng('BMG.STN') ehb2 = read_eng('ISC.STN') ehb = np.unique(np.vstack((ehb1, ehb2)), axis=0) isc1 = read_isc('ehb.stn') isc2 = read_isc('iscehb.stn') isc = np.unique(np.vstack((isc1, isc2)), axis=0) catalogue = [] for i in xrange(ehb.shape[0]): filed = False xml = False stn_found = isc[isc[:, 0] == ehb[i, 0], :] min_dist = 10e10 if stn_found.shape[0] > 0: if stn_found.shape[0] > 1: for j in xrange(stn_found.shape[0]): dist = locations2degrees(np.float(stn_found[j, 2]), np.float(stn_found[j, 3]), np.float(ehb[i, 1]), np.float(ehb[i, 2])) if dist < min_dist: min_dist = dist record = stn_found[j, :] else: min_dist = locations2degrees(np.float(stn_found[0, 2]), np.float(stn_found[0, 3]), np.float(ehb[i, 1]), np.float(ehb[i, 2])) record = stn_found[0, :] # Now we try to find the same station in XML file # if min_dist > 1. or stn_found.shape[0]==0: xstn_found = inv.select(station=ehb[i, 0], channel="*HZ") if len(stn_found) == 0 and len(xstn_found) == 0: # we filed to find station anywhere and assign dummy values record = [ehb[i, 0], default_net, ehb[i, 1], ehb[i, 2], ehb[i, 3], 'SHZ', '1964-1-1 00:00:00', '2599-12-31 23:59:59'] min_dist = 0. filed = True else: # if station is found somehwere we try to iterate and see if XML has data giving it preference through adding extra value to min_dist found in ISC if len(xstn_found) > 0: # print "----------",len(xstn_found) # print xstn_found[0][0].latitude min_dist = min_dist + 0.1 for j in xrange(len(xstn_found)): dist = locations2degrees(xstn_found[j][0].latitude, xstn_found[j][0].longitude, np.float(ehb[i, 1]), np.float(ehb[i, 2])) if min_dist > dist: min_dist = dist record = xstn_found[j] # print record xml = True # last defence if stations have been done but distance between declared and found locations are more than 1 degree if min_dist > 1: record = [ehb[i, 0], default_net, ehb[i, 1], ehb[i, 2], ehb[i, 3], 'SHZ', '1964-1-1 00:00:00', '2599-12-31 23:59:59'] filed = True if xml: xml = False else: if filed: if len(record[7]) < 5: record[7] = '2599-12-31 23:59:59' catalogue.append(record) else: stn_found = isc[(isc[:, 0] == record[0]) & (isc[:, 1] == record[1]), :] for k in xrange(stn_found.shape[0]): net = Network(code=stn_found[k, 1], stations=[], description=' ') if len(stn_found[k, 7]) < 5: stn_found[k, 7] = '2599-12-31 23:59:59' catalogue.append(stn_found[k, :]) stn_found = np.unique(np.array(catalogue), axis=0) if len(stn_found[stn_found == '']) > 0 or len(stn_found[stn_found == ' ']) > 0: print "Some elements are empty, check the list" # we composed our inventory. However some stations from ISC list can be left behind. We check if some stations in ISC are forgotten lost = [] for j in xrange(isc.shape[0]): # is there any common station name? common_st = stn_found[isc[j, 0] == stn_found[:, 0]] if common_st.shape[0] > 0: # is network code the same? common_net = common_st[common_st[:, 1] == isc[j, 1]] if common_net.shape[0] < 1: # ok we found forgotten one, check the XML if len(inv.select(station=isc[j, 0], network=isc[j, 1])) <= 0: # Bingo... lost.append(isc[j, :]) else: if len(inv.select(station=isc[j, 0], network=isc[j, 1])) <= 0: # Bingo... lost.append(isc[j, :]) stn_found = np.vstack((stn_found, np.array(lost))) netDict = defaultdict(list) for k in xrange(stn_found.shape[0]): result = inv.select(network=stn_found[k, 1]) if (len(result.networks)): net = result.networks[0] net.stations = [] else: net = Network(code=stn_found[k, 1], stations=[], description=' ') # print stn_found[k, 1] if len(stn_found[k, 7]) < 5: stn_found[k, 7] = '2599-12-31 23:59:59' catalogue.append(stn_found[k, :]) sta = Station(code=stn_found[k, 0], creation_date=utcdatetime.UTCDateTime(stn_found[k, 6]), \ termination_date=utcdatetime.UTCDateTime(stn_found[k, 7]), \ site=Site(name=' '), \ latitude=np.float(stn_found[k, 2]), \ longitude=np.float(stn_found[k, 3]), \ elevation=np.float(stn_found[k, 4])) if (stn_found[k, 5] in responseDict.keys()): r = responseDict[stn_found[k, 5]] cha = Channel(code=stn_found[k, 5], \ depth=0., \ azimuth=0., \ dip=-90., \ location_code='', \ latitude=np.float(stn_found[k, 2]), \ longitude=np.float(stn_found[k, 3]), \ elevation=np.float(stn_found[k, 4]), \ # sensor=sensorDict[stn_found[k,5]], \ response=r) sta.channels.append(cha) if (type(netDict[stn_found[k, 1]]) == Network): netDict[stn_found[k, 1]].stations.append(sta) else: net.stations.append(sta) netDict[stn_found[k, 1]] = net # print 'np',stn_found[k,:] # end if our_xml = Inventory(networks=netDict.values(), source='EHB') print 'Writing output files..' for inet, net in enumerate(our_xml.networks): currInv = Inventory(networks=[net], source='EHB') currInv.write("output/station.%d.xml" % (inet), format="stationxml", validate=True) # our_xml.write("station.xml",format="stationxml", validate=True) our_xml.write("station.txt", format="stationtxt")
def load_from_excel(file_name) -> Inventory: """ Read in a multi-sheet excel file with network metadata sheets: Sites, Networks, Hubs, Stations, Components, Sites, Cables, Boreholes Organize these into a uquake Inventory object :param xls_file: path to excel file :type: xls_file: str :return: inventory :rtype: uquake.core.data.inventory.Inventory """ df_dict = pd.read_excel(file_name, sheet_name=None) source = df_dict['Sites'].iloc[0]['code'] # sender (str, optional) Name of the institution sending this message. sender = df_dict['Sites'].iloc[0]['operator'] net_code = df_dict['Networks'].iloc[0]['code'] net_descriptions = df_dict['Networks'].iloc[0]['name'] contact_name = df_dict['Networks'].iloc[0]['contact_name'] contact_email = df_dict['Networks'].iloc[0]['contact_email'] contact_phone = df_dict['Networks'].iloc[0]['contact_phone'] site_operator = df_dict['Sites'].iloc[0]['operator'] site_country = df_dict['Sites'].iloc[0]['country'] site_name = df_dict['Sites'].iloc[0]['name'] site_code = df_dict['Sites'].iloc[0]['code'] print("source=%s" % source) print("sender=%s" % sender) print("net_code=%s" % net_code) network = Network(net_code) inventory = Inventory([network], source) # obspy requirements for PhoneNumber are super specific: # So likely this will raise an error if/when someone changes the value in # Networks.contact_phone """ PhoneNumber(self, area_code, phone_number, country_code=None, description=None): :type area_code: int :param area_code: The area code. :type phone_number: str :param phone_number: The phone number minus the country and area code. Must be in the form "[0-9]+-[0-9]+", e.g. 1234-5678. :type country_code: int, optional :param country_code: The country code. """ import re phone = re.findall(r"[\d']+", contact_phone) area_code = int(phone[0]) number = "%s-%s" % (phone[1], phone[2]) phone_number = PhoneNumber(area_code=area_code, phone_number=number) person = Person(names=[contact_name], agencies=[site_operator], emails=[contact_email], phones=[phone_number]) operator = Operator(site_operator, contacts=[person]) site = Site(name=site_name, description=site_name, country=site_country) # Merge Stations+Components+Sites+Cables info into sorted stations + # channels dicts: df_dict['Stations']['station_code'] = df_dict['Stations']['code'] df_dict['Sites']['sensor_code'] = df_dict['Sites']['code'] df_dict['Components']['code_channel'] = df_dict['Components']['code'] df_dict['Components']['sensor'] = df_dict['Components']['sensor__code'] df_merge = pd.merge(df_dict['Stations'], df_dict['Sites'], left_on='code', right_on='station__code', how='inner', suffixes=('', '_channel')) df_merge2 = pd.merge(df_merge, df_dict['Components'], left_on='sensor_code', right_on='sensor__code', how='inner', suffixes=('', '_sensor')) df_merge3 = pd.merge(df_merge2, df_dict['Cable types'], left_on='cable__code', right_on='code', how='inner', suffixes=('', '_cable')) df_merge4 = pd.merge(df_merge3, df_dict['Site types'], left_on='sensor_type__model', right_on='model', how='inner', suffixes=('', '_sensor_type')) df = df_merge4.sort_values(['sensor_code', 'location_code']).fillna(0) # Need to sort by unique station codes, then look through 1-3 channels # to add stn_codes = set(df['sensor_code']) stations = [] for code in stn_codes: chan_rows = df.loc[df['sensor_code'] == code] row = chan_rows.iloc[0] station = {} # Set some keys explicitly # from ipdb import set_trace; set_trace() station['code'] = '{}'.format(row['sensor_code']) station['x'] = row['location_x_channel'] station['y'] = row['location_y_channel'] station['z'] = row['location_z_channel'] station['loc'] = np.array([station['x'], station['y'], station['z']]) station['long_name'] = "{}.{}.{:02d}".format(row['network__code'], row['station_code'], row['location_code']) # MTH: 2019/07 Seem to have moved from pF to F on Cables sheet: station['cable_capacitance_pF_per_meter'] = row['c'] * 1e12 # Set the rest (minus empty fields) directly from spreadsheet names: renamed_keys = { 'sensor_code', 'location_x', 'location_y', 'location_z', 'name' } # These keys are either redundant or specific to channel, not station: remove_keys = { 'code', 'id_channel', 'orientation_x', 'orientation_y', 'orientation_z', 'id_sensor', 'enabled_channel', 'station_id', 'id_cable' } keys = row.keys() empty_keys = keys[pd.isna(row)] keys = set(keys) - set(empty_keys) - renamed_keys - remove_keys for key in keys: station[key] = row[key] # Added keys: station['motion'] = 'VELOCITY' if row['sensor_type'].upper() == 'ACCELEROMETER': station['motion'] = 'ACCELERATION' # Attach channels: station['channels'] = [] for index, rr in chan_rows.iterrows(): chan = {} chan['cmp'] = rr['code_channel_sensor'].upper() chan['orientation'] = np.array([ rr['orientation_x'], rr['orientation_y'], rr['orientation_z'] ]) chan['x'] = row['location_x_channel'] chan['y'] = row['location_y_channel'] chan['z'] = row['location_z_channel'] chan['enabled'] = rr['enabled'] station['channels'].append(chan) stations.append(station) # from ipdb import set_trace; set_trace() # Convert these station dicts to inventory.Station objects and attach to # inventory.network: station_list = [] for station in stations: # This is where namespace is first employed: station = Station.from_station_dict(station, site_name) station.site = site station.operators = [operator] station_list.append(station) network.stations = station_list return inventory