def _download_kemmerer(self):
""" downloads both stations and waveforms """
for station in ["M17A", "M18A"]:
domain = RectangularDomain(
minlatitude=40.0,
maxlatitude=43.0,
minlongitude=-111.0,
maxlongitude=-110.0,
)
restrictions = Restrictions(
starttime=obspy.UTCDateTime("2009-04-01T00:00:00"),
endtime=obspy.UTCDateTime("2009-04-04T00:00:00"),
chunklength_in_sec=3600,
network="TA",
channel="BH?",
station=station,
reject_channels_with_gaps=False,
minimum_length=0.0,
minimum_interstation_distance_in_m=10.0,
)
MassDownloader(providers=[self._download_client]).download(
domain,
restrictions,
mseed_storage=str(self.waveform_path),
stationxml_storage=str(self.station_path),
)
def download_data(starttime,
endtime,
waveform_dir,
station_dir,
networks=None,
channels=None,
providers=None,
minimum_length=0.95):
# Rectangular domain containing parts of southern Germany.
domain = RectangularDomain(minlatitude=-90,
maxlatitude=90,
minlongitude=-180,
maxlongitude=180)
if isinstance(channels, list):
channel = ",".join(channels)
elif channels == "None" or channels is None:
channel = None
else:
raise ValueError("Unknown channels: {}".format(channels))
if isinstance(networks, list):
network = ",".join(networks)
elif networks == "None" or networks is None:
network = None
else:
raise ValueError("Unknown networks: {}".format(networks))
print("network: ", network)
print("channel: ", channel)
# Set download restrictions
restrictions = Restrictions(starttime=starttime,
endtime=endtime,
reject_channels_with_gaps=False,
minimum_length=minimum_length,
station=None,
network=network,
channel=channel,
location_priorities=["", "00", "10"],
channel_priorities=["BH[ZNE12]", "HH[ZNE12]"])
if (providers is None) or (providers == "None"):
mdl = MassDownloader()
else:
mdl = MassDownloader(providers=providers)
mdl.download(domain,
restrictions,
mseed_storage=waveform_dir,
stationxml_storage=station_dir)
def get_data(date, latitude, longitude):
resp_files = '/Users/gsilveira/Documents/MadalenaMatias/dataless/resp/'
from obspy.clients.fdsn.mass_downloader import RectangularDomain, \
Restrictions, MassDownloader
domain = RectangularDomain(
minlatitude=-55.5,
maxlatitude=latitude, #66.7,
minlongitude=-71.5,
maxlongitude=longitude)
restrictions = Restrictions(
# Get data for a whole year.
starttime=obspy.UTCDateTime(int(date[0]), int(date[1]), int(date[2]),
int(date[3]), int(date[4]), int(date[5]),
int(date[6])),
endtime=obspy.UTCDateTime(2020, 10, 1),
# Chunk it to have one file per day.
chunklength_in_sec=3600000,
# Considering the enormous amount of data associated with continuous
# requests, you might want to limit the data based on SEED identifiers.
# If the location code is specified, the location priority list is not
# used; the same is true for the channel argument and priority list.
network="GH",
station="AKOS",
location="*",
channel="*",
# The typical use case for such a data set are noise correlations where
# gaps are dealt with at a later stage.
reject_channels_with_gaps=False,
# Same is true with the minimum length. All data might be useful.
minimum_length=0.0,
# Guard against the same station having different names.
minimum_interstation_distance_in_m=100.0)
# Restrict the number of providers if you know which serve the desired
# data. If in doubt just don't specify - then all providers will be
# queried.
mdl = MassDownloader(providers=["IRIS"])
mdl.download(
domain,
restrictions,
mseed_storage=("../per_hour/{network}/Leslie/{station}/"
"{channel}.{location}.{starttime}.{endtime}.mseed"),
stationxml_storage="stations")
def download_events(cat, yeartag="2005", firstid=1):
"""
Download events to lower 48 with MassDownloader using
events in obspy Catalog object cat. Parameters for
station definitions are hard wired
"""
wf_storage = yeartag + "/{starttime}/{network}_{station}_{location}.mseed"
site_storage = "site_" + yeartag + ".dir"
mdl = MassDownloader()
domain = RectangularDomain(minlatitude=20.0,
maxlatitude=54.0,
minlongitude=-135,
maxlongitude=-55)
count_evid = 1
for event in cat:
if (count_evid >= firstid):
t0 = time.time()
o = event.preferred_origin()
# Time attribute is already a UTCDateTime object
origin_time = o.time
restrictions = Restrictions(
starttime=origin_time,
endtime=origin_time + 3600.0,
reject_channels_with_gaps=True,
minimum_length=0.95,
minimum_interstation_distance_in_m=100.0,
channel_priorities=["BH[ZNE12]"],
location_priorities=["", "00", "10", "01"])
wf_storage = ("%s/event%d" % (yeartag, count_evid))
mdl.download(domain,
restrictions,
mseed_storage=wf_storage,
stationxml_storage=site_storage)
dt = time.time() - t0
print("Event ", count_evid, " download time (s)=", dt)
count_evid += 1
eq_time = datetime.datetime.strptime(df['Origin Time (UTC)'][ii], time_format)
# use the date of the event as folder name where the downloaded data will be stored
event_date = '{:04d}-{:02d}-{:02d}'.format(eq_time.year, eq_time.month,
eq_time.day)
folder_output = folder_main + event_date + '/'
# create the folder_output
if not os.path.exists(folder_output):
os.makedirs(folder_output)
# convert time to obspy.core.utcdatetime.UTCDateTime
origin_time = obspy.UTCDateTime(eq_time)
# prepare data downloader
domain = RectangularDomain(minlongitude=statLon_min, maxlongitude=statLon_max,\
minlatitude=statLat_min, maxlatitude=statLat_max)
restrictions = Restrictions(
# Get data from 5 minutes before the event to one hour after the
# event. This defines the temporal bounds of the waveform data.
starttime=origin_time - seconds_before,
endtime=origin_time + seconds_after,
# station = 'BGAR',
network=network,
# You might not want to deal with gaps in the data. If this setting is
# True, any trace with a gap/overlap will be discarded.
reject_channels_with_gaps=True,
# And you might only want waveforms that have data for at least 95 % of
# the requested time span. Any trace that is shorter than 95 % of the
# desired total duration will be discarded.
minimum_length=0.95,
import obspy
from obspy.clients.fdsn.mass_downloader import RectangularDomain, \
Restrictions, MassDownloader
# Rectangular domain containing parts of southern Germany.
domain = RectangularDomain(minlatitude=52.,
maxlatitude=72.5,
minlongitude=-172.,
maxlongitude=-122.)
restrictions = Restrictions(
# Get data for a whole year.
starttime=obspy.UTCDateTime(2017, 6, 11),
endtime=obspy.UTCDateTime(2017, 6, 21),
# Chunk it to have one file per day.
chunklength_in_sec=86400,
# Considering the enormous amount of data associated with continuous
# requests, you might want to limit the data based on SEED identifiers.
# If the location code is specified, the location priority list is not
# used; the same is true for the channel argument and priority list.
network="TA",
station="H20K",
location="",
channel="LH?",
# The typical use case for such a data set are noise correlations where
# gaps are dealt with at a later stage.
reject_channels_with_gaps=False,
# Same is true with the minimum length. All data might be useful.
minimum_length=0.0,
# Guard against the same station having different names.
minimum_interstation_distance_in_m=100.0)
import obspy
from obspy.clients.fdsn.mass_downloader import RectangularDomain, \
Restrictions, MassDownloader
# Rectangular domain containing parts of southern Germany.
domain = RectangularDomain(minlatitude=-17.78,
maxlatitude=4.15,
minlongitude=23.00,
maxlongitude=40.2)
restrictions = Restrictions(
# Get data for a whole year.
starttime=obspy.UTCDateTime(1994, 8, 5),
endtime=obspy.UTCDateTime(1994, 8, 10),
# Chunk it to have one file per day.
chunklength_in_sec=86400,
# Considering the enormous amount of data associated with continuous
# requests, you might want to limit the data based on SEED identifiers.
# If the location code is specified, the location priority list is not
# used; the same is true for the channel argument and priority list.
channel="LH?",
# The typical use case for such a data set are noise correlations where
# gaps are dealt with at a later stage.
reject_channels_with_gaps=False,
# Same is true with the minimum length. All data might be useful.
minimum_length=0.0,
# Guard against the same station having different names.
minimum_interstation_distance_in_m=100.0)
mseed_storage = ("waveforms/{network}/{station}/"
"{channel}.{location}.{starttime}.{endtime}.mseed")
from obspy.clients.fdsn.mass_downloader import RectangularDomain
import configparser
para = configparser.ConfigParser()
para.read('para.ini')
domain = RectangularDomain(
minlatitude=para["Map Info"].getfloat("minlatitude"),
maxlatitude=para["Map Info"].getfloat("maxlatitude"),
minlongitude=para["Map Info"].getfloat("minlongitude"),
maxlongitude=para["Map Info"].getfloat("maxlongitude"))
def download_surf(self, datadir, commontime = True, fskip=True, chanrank=['LH', 'BH', 'HH'],\
channels='ZNE', vmax = 8.0, vmin=.5, verbose=False, start_date=None, end_date=None, skipinv=True, threads_per_client = 3,\
providers = None, blon = 0.05, blat = 0.05):
"""request Rayleigh wave data from
====================================================================================================================
::: input parameters :::
lon0, lat0 - center of array. If specified, all waveform will have the same starttime and endtime
min/maxDelta - minimum/maximum epicentral distance, in degree
channel - Channel code, e.g. 'BHZ'.
Last character (i.e. component) can be a wildcard (‘?’ or ‘*’) to fetch Z, N and E component.
vmin, vmax - minimum/maximum velocity for surface wave window
=====================================================================================================================
"""
if providers is None:
providers = ['BGR', 'ETH', 'GFZ', 'ICGC', 'INGV', 'IPGP',\
'IRIS', 'KNMI', 'KOERI', 'LMU', 'NCEDC', 'NIEP', 'NOA', 'ODC', 'ORFEUS',\
'RASPISHAKE', 'RESIF', 'SCEDC', 'TEXNET', 'USP']
self.get_limits_lonlat()
minlongitude = self.minlon
maxlongitude = self.maxlon
if minlongitude > 180.:
minlongitude -= 360.
if maxlongitude > 180.:
maxlongitude -= 360.
lon0 = (minlongitude + maxlongitude) / 2.
lat0 = (self.minlat + self.maxlat) / 2.
domain = RectangularDomain(minlatitude=self.minlat - blat,
maxlatitude=self.maxlat + blat,
minlongitude=minlongitude - blon,
maxlongitude=maxlongitude + blon)
try:
print(self.cat)
except AttributeError:
self.copy_catalog()
try:
stime4down = obspy.core.utcdatetime.UTCDateTime(start_date)
except:
stime4down = obspy.UTCDateTime(0)
try:
etime4down = obspy.core.utcdatetime.UTCDateTime(end_date)
except:
etime4down = obspy.UTCDateTime()
mdl = MassDownloader(providers=providers)
chantype_list = []
for chantype in chanrank:
chantype_list.append('%s[%s]' % (chantype, channels))
channel_priorities = tuple(chantype_list)
t1 = time.time()
# loop over events
for event in self.cat:
pmag = event.preferred_magnitude()
try:
magnitude = pmag.mag
evdp = porigin.depth / 1000.
except:
pass
try:
Mtype = pmag.magnitude_type
event_descrip = event.event_descriptions[
0].text + ', ' + event.event_descriptions[0].type
porigin = event.preferred_origin()
otime = porigin.time
timestr = otime.isoformat()
evlo = porigin.longitude
evla = porigin.latitude
except:
pass
if otime < stime4down or otime > etime4down:
continue
if commontime:
dist, az, baz = obspy.geodetics.gps2dist_azimuth(
evla, evlo, lat0, lon0) # distance is in m
dist = dist / 1000.
starttime = otime + dist / vmax
endtime = otime + dist / vmin
oyear = otime.year
omonth = otime.month
oday = otime.day
ohour = otime.hour
omin = otime.minute
osec = otime.second
label = '%d_%s_%d_%d_%d_%d' % (oyear, mondict[omonth], oday, ohour,
omin, osec)
eventdir = datadir + '/' + label
if not os.path.isdir(eventdir):
os.makedirs(eventdir)
event_logfname = eventdir + '/download.log'
if fskip and os.path.isfile(event_logfname):
continue
stationxml_storage = "%s/{network}/{station}.xml" % eventdir
if commontime:
restrictions = Restrictions(
# starttime and endtime
starttime=starttime,
endtime=endtime,
# You might not want to deal with gaps in the data.
reject_channels_with_gaps=True,
# And you might only want waveforms that have data for at least
# 95 % of the requested time span.
minimum_length=0.95,
# No two stations should be closer than 10 km to each other.
minimum_interstation_distance_in_m=10E3,
# Only HH or BH channels. If a station has HH channels,
# those will be downloaded, otherwise the BH. Nothing will be
# downloaded if it has neither.
channel_priorities=channel_priorities,
sanitize=True)
mseed_storage = eventdir
# # # mseed_storage = ("%s/{network}/{station}/{channel}.{location}.%s.mseed" %(datadir, label) )
mdl.download(domain,
restrictions,
mseed_storage=mseed_storage,
stationxml_storage=stationxml_storage,
threads_per_client=threads_per_client)
else:
# loop over stations
Nsta = 0
for network in self.inv:
for station in network:
netcode = network.code
stacode = station.code
staid = netcode + '.' + stacode
with warnings.catch_warnings():
warnings.simplefilter("ignore")
st_date = station.start_date
ed_date = station.end_date
if skipinv and (otime < st_date or otime > ed_date):
continue
stlo = station.longitude
stla = station.latitude
dist, az, baz = obspy.geodetics.gps2dist_azimuth(
evla, evlo, stla, stlo) # distance is in m
dist = dist / 1000.
starttime = otime + dist / vmax
endtime = otime + dist / vmin
restrictions = Restrictions(
network=netcode,
station=stacode,
# starttime and endtime
starttime=starttime,
endtime=endtime,
# You might not want to deal with gaps in the data.
reject_channels_with_gaps=True,
# And you might only want waveforms that have data for at least
# 95 % of the requested time span.
minimum_length=0.95,
# No two stations should be closer than 10 km to each other.
minimum_interstation_distance_in_m=10E3,
# Only HH or BH channels. If a station has HH channels,
# those will be downloaded, otherwise the BH. Nothing will be
# downloaded if it has neither.
channel_priorities=channel_priorities,
sanitize=True)
mseed_storage = eventdir
# mseed_storage = ("%s/{network}/{station}/{channel}.{location}.%s.mseed" %(datadir, label) )
mdl.download(domain,
restrictions,
mseed_storage=mseed_storage,
stationxml_storage=stationxml_storage,
threads_per_client=threads_per_client)
Nsta += 1
print('--- [RAYLEIGH DATA DOWNLOAD] Event: %s %s' %
(otime.isoformat(), event_descrip))
with open(event_logfname, 'w') as fid:
fid.writelines('evlo: %g, evla: %g\n' % (evlo, evla))
if commontime:
fid.writelines('distance: %g km\n' % dist)
fid.writelines('DONE\n')
return
def downloadMseeds(client_list,
stations_json,
output_dir,
start_time,
end_time,
min_lat,
max_lat,
min_lon,
max_lon,
chunck_size,
channel_list=[],
n_processor=None):
"""
Uses obspy mass downloader to get contineous waveforms from a specific client in miniseed format in variable chunck sizes.
WARNING:
usage of multiprocessing and parallel downloads heavily depends on the client. It might cause missing some data for some network.
Please test first first for some short period and if It did miss some chuncks of data for some channels then set n_processor=None
to avoid parralel downloading.
Parameters
----------
client_list : str
List of client names e.g. IRIS.
stations_json : dic
Station informations.
output_dir : str
Output directory.
start_time : str
Start dateitme for the beginging of period in "YYYY-MM-DDThh:mm:ss.f" format.
end_time : str
End dateitme in "YYYY-MM-DDThh:mm:ss.f" format.
min_lat : float
Min latitude of the region.
max_lat : float
Max latitude of the region.
min_lon : float
Min longitude of the region.
max_lon : float
Max longitude of the region.
chunk_size : int
Chunck size in day.
channel_list : str
Priority channel list.
n_processor : int
Number of CPU processors for parallel downloading.
Generates
-------
./output_name/station_name/*.mseed
"""
json_file = open(stations_json)
station_dic = json.load(json_file)
print(
f"####### There are {len(station_dic)} stations in the list. #######")
start_t = UTCDateTime(start_time)
end_t = UTCDateTime(end_time)
domain = RectangularDomain(minlatitude=min_lat,
maxlatitude=max_lat,
minlongitude=min_lon,
maxlongitude=max_lon)
mdl = MassDownloader(providers=client_list)
bg = start_t
if n_processor == None:
for st in station_dic:
print(f'======= Working on {st} station.')
_get_w(bg, st, station_dic, end_t, mdl, domain, output_dir,
chunck_size, channel_list)
else:
def process(st):
print(f'======= Working on {st} station.')
_get_w(bg, st, station_dic, end_t, mdl, domain, output_dir,
chunck_size, channel_list)
with ThreadPool(n_processor) as p:
p.map(process, [st for st in station_dic])
def GetData(self,stationdirpath='stations',datadirpath='waveforms',req_type='continuous',\
chunklength=86400,tracelen=20000, vmodel='ak135'):
'''Call obspy mass downloader to get waveform data. Chunklength refers to the trace length option
for a continuous download, tracelen is for an event-based request'''
#Currently set up to download one day worth of data in the continuous mode, 2000 seconds
#in the event-based mode
self.stationdirpath = stationdirpath
self.datadirpath = datadirpath
from obspy.clients.fdsn.mass_downloader import RectangularDomain, CircularDomain,\
Restrictions, MassDownloader
if req_type == 'continuous':
#Get data from all stations within this domain
domain = RectangularDomain(minlatitude=self.minlatitude,maxlatitude=self.maxlatitude,\
minlongitude=self.minlongitude,maxlongitude=self.maxlongitude)
#Download data in daily segements - may want to change
restrictions = Restrictions(\
starttime=self.starttime,endtime=self.endtime,\
chunklength_in_sec=chunklength,\
channel=self.channel,station=self.station,location="",\
reject_channels_with_gaps=False,\
minimum_length=0.0,minimum_interstation_distance_in_m=100.0)
#Call mass downloader to get the waveform information
mdl = MassDownloader(providers=[self.clientname])
mdl.download(domain,
restrictions,
mseed_storage=datadirpath,
stationxml_storage=stationdirpath)
elif req_type == 'event':
if self.quake_cat == None:
print(
"Stop: Must call fetchEvents first to get event catalog to download from"
)
sys.exit(1)
#Add option for non-continuous download - event/station pairing for example
#Ger data for all stations in this domain
domain = RectangularDomain(minlatitude=self.minlatitude,maxlatitude=self.maxlatitude,\
minlongitude=self.minlongitude,maxlongitude=self.maxlongitude)
for event in self.quake_cat:
cnt = 0.
print("Downloading data for event %s" % event)
#For each event, download the waveforms at all stations requested
origin_time = event.origins[0].time
vel_model = TauPyModel(model=vmodel)
#case where we only want to download data for some station-event pairs'
stations_to_exclude = []
if self.station_autoselect_flag == True:
stations_to_download = []
evlat = event.origins[0].latitude
evlon = event.origins[0].longitude
#EK changes added 04/2019
evdep = event.origins[0].depth
for network in self.inventory:
for station in network:
stlat = station.latitude
stlon = station.longitude
#EK 04/2019
#this downloads data within Short Wave Window (SWW), a cone under the station bounded by an angle, here we chose 45 deg
#calculate distance between eq and station and azimuth
ddeg = locations2degrees(evlat, evlon, stlat,
stlon)
distance_m, az, baz = gps2dist_azimuth(
evlat, evlon, stlat, stlon)
#calculate proxy for incident angle
theta = np.arctan2(distance_m, evdep)
if theta <= np.pi / 4:
#find if station has needed arrival
arrivals = vel_model.get_travel_times(
source_depth_in_km=evdep / 1000.,
distance_in_degree=ddeg,
phase_list=["s", "S"])
if len(arrivals) > 0:
#get stations you want to download
stations_to_download.append(station.code)
print(station.code,
'angle = %.2f' % np.rad2deg(theta))
print(arrivals)
cnt = cnt + 1
else:
stations_to_exclude.append(station.code)
else:
if station.code not in stations_to_exclude:
stations_to_exclude.append(station.code)
print(
"\n-------------\n%g event-station pairs found in SWW\n-------------\n"
% cnt)
print(
"\n-------------\nSelecting just the following stations for download\n-------------\n"
)
print(stations_to_download)
#this approach doesn't work, use exclude_stations flag later
#restrictions = Restrictions(starttime=origin_time,endtime=origin_time + tracelen,\
#reject_channels_with_gaps=False, minimum_length=0.95, minimum_interstation_distance_in_m=10E3,\
#channel=self.channel,location="",network=self.network,station=stations_to_download)
#case where we have single network
if self.network:
restrictions = Restrictions(starttime=origin_time,endtime=origin_time + tracelen,\
reject_channels_with_gaps=False, minimum_length=0.95, minimum_interstation_distance_in_m=10E3,\
channel=self.channel,location="",network=self.network,exclude_stations=stations_to_exclude)
#Case where we want all networks within a region (assumes that we also want all stations unless we have built
# a stations to exclude list)
else:
restrictions = Restrictions(starttime=origin_time,endtime=origin_time + tracelen,\
reject_channels_with_gaps=False, minimum_length=0.95, minimum_interstation_distance_in_m=10E3,\
channel=self.channel,exclude_stations=stations_to_exclude)
mdl = MassDownloader(providers=[self.clientname])
mdl.download(domain, restrictions, mseed_storage=datadirpath,\
stationxml_storage=stationdirpath)
with open(catalog, "r", newline='') as csvfile:
line = csv.reader(csvfile, delimiter=',')
next(line, None) # skip the headers
# loop over each event
for row in line:
origin_time = UTCDateTime(row[0])
event_fname = "".join(row[0].split("T")[0].split("-")) + "".join(
"".join("".join(
row[0].split("T")[1].split("Z")).split(".")).split(":"))
# Step 1: Data Selection
# Rectangular Domain
domain = RectangularDomain(minlatitude=35.5,
maxlatitude=36,
minlongitude=-118,
maxlongitude=-117.3)
# Restrictions
# I put all the options here. You may delete some defaults to clear the code.
restrictions = Restrictions(
# time range
starttime=origin_time - 1 * 60,
endtime=origin_time + 2 * 60,
# station start and end time (default values used)
station_starttime=None,
station_endtime=None,
# The length of one chunk in seconds (default value used)
chunklength_in_sec=None,
def FDSN_downloader(datapath,
start='20141201',
stop='20161201',
network='XH',
station='*',
channel='HH*',
**kwargs):
"""This function uses the FDSN mass data downloader to automatically
download data from the XH network deployed on the RIS from Dec 2014 - Dec
2016. More information on the Obspy mass downloader available at:
https://docs.obspy.org/packages/autogen/obspy.clients.fdsn.mass_downloader.html
Parameters
----------
datapath : str
Path to save MSEED and XML data.
start : str
Start date, in format YYYYMMDD
stop : str
Stop date, in format YYYYMMDD
network : str
2-character FDSN network code
station: str
2-character station code
channel: str
3-character channel code
"""
print("=" * 65)
print("Initiating mass download request.")
start = UTCDateTime(start)
stop = UTCDateTime(stop)
if not os.path.exists(datapath):
# os.makedirs(f'{datapath}/MSEED')
os.makedirs(os.path.join(datapath, 'MSEED'))
# os.makedirs(f'{datapath}/StationXML')
os.makedirs(os.path.join(datapath, 'StationXML'))
domain = RectangularDomain(minlatitude=-85,
maxlatitude=-75,
minlongitude=160,
maxlongitude=-130)
restrictions = Restrictions(starttime=start,
endtime=stop,
chunklength_in_sec=86400,
network=network,
station=station,
location="*",
channel=channel,
reject_channels_with_gaps=True,
minimum_length=0.0,
minimum_interstation_distance_in_m=100.0)
mdl = MassDownloader(providers=["IRIS"])
mdl.download(
domain,
restrictions,
# mseed_storage=f"{datapath}/MSEED",
mseed_storage=os.path.join(datapath, 'MSEED'),
# stationxml_storage=f"{datapath}/StationXML"
stationxml_storage=os.path.join(datapath, 'StationXML'),
)
logger = logging.getLogger("obspy.clients.fdsn.mass_downloader")
logger.setLevel(logging.DEBUG)
if not os.path.isdir(raw_wf_data_dir):
os.mkdir(raw_wf_data_dir)
sta_metadata_dir = raw_wf_data_dir+'/_station_xml/'
if not os.path.isdir(sta_metadata_dir):
os.mkdir(sta_metadata_dir)
# Read NCSS preferred origin (evaluation_mode = 'final') file, was saved as dictionary
f_ncssorigin = open(event_id+'/'+event_id+'.ncssorigin', 'r')
ncss_origin = ast.literal_eval(f_ncssorigin.readlines()[0])
print ('tbefore: ', tbefore, ' tafter: ', tafter, ' tlength: ', tafter+tbefore )
starttime=UTCDateTime(ncss_origin['otime']) - tbefore
endtime=UTCDateTime(ncss_origin['otime']) + tafter
print('starttime: ', starttime, ' endtime: ', endtime)
# Set-up domain
domain = RectangularDomain(minlatitude=min_lat, maxlatitude=max_lat,
minlongitude=min_lon, maxlongitude=max_lon)
# domain = CircularDomain(latitude=ncss_origin['latitude'], longitude=ncss_origin['longitude'],
# minradius=0.0, maxradius=maxradius)
for net_chan in net_chans:
raw_wf_data_dir = event_id+'/RAW/'+net_chan
if not os.path.isdir(raw_wf_data_dir):
os.mkdir(raw_wf_data_dir)
sta_metadata_dir = raw_wf_data_dir+'/_station_xml/'
if not os.path.isdir(sta_metadata_dir):
os.mkdir(sta_metadata_dir)
net = net_chan.split('.')[0]
chan = net_chan.split('.')[1]
print('... working on ... ', net_chan, net, chan)
# Here I try to fine-tune channel and location priorities
if net_chan in [ 'BK.BH', 'BK.HH' ]:
import obspy
from obspy.clients.fdsn.mass_downloader import RectangularDomain, Restrictions, MassDownloader
domain = RectangularDomain(minlatitude=-90,
maxlatitude=90,
minlongitude=-180,
maxlongitude=180.0)
restrictions = Restrictions(starttime=obspy.UTCDateTime(2012, 1, 1),
endtime=obspy.UTCDateTime(2014, 1, 1),
chunklength_in_sec=86400,
network="YT",
station="UPTW,DNTW,BEAR,THUR",
location="",
channel="BH*",
location_priorities=[
"01",
],
channel_priorities=["BHZ", "BHN", "BHE"],
reject_channels_with_gaps=False,
minimum_length=0.0)
mseed_storage = ("/media/Data/Data/YT/MSEED/raw/")
mdl = MassDownloader(providers=["IRIS"])
mdl.download(domain=domain,
restrictions=restrictions,
mseed_storage=mseed_storage,
stationxml_storage=("/media/Data/Data/YT/XML/"))
import obspy
from obspy.clients.fdsn.mass_downloader import RectangularDomain, \
Restrictions, MassDownloader
# Rectangular domain containing parts of southern Germany.
domain = RectangularDomain(minlatitude=30, maxlatitude=50,
minlongitude=5, maxlongitude=35)
restrictions = Restrictions(
# Get data for a whole year.
starttime=obspy.UTCDateTime(2012, 1, 1),
endtime=obspy.UTCDateTime(2013, 1, 1),
# Chunk it to have one file per day.
chunklength_in_sec=86400,
# Considering the enormous amount of data associated with continuous
# requests, you might want to limit the data based on SEED identifiers.
# If the location code is specified, the location priority list is not
# used; the same is true for the channel argument and priority list.
network="BW", station="A*", location="", channel="EH*",
# The typical use case for such a data set are noise correlations where
# gaps are dealt with at a later stage.
reject_channels_with_gaps=False,
# Same is true with the minimum length. All data might be useful.
minimum_length=0.0,
# Guard against the same station having different names.
minimum_interstation_distance_in_m=100.0)
# Restrict the number of providers if you know which serve the desired
# data. If in doubt just don't specify - then all providers will be
# queried.
'''
def data_request(cmt_filename, param_path):
# Request config_file
request_param_path = os.path.join(param_path,
"RequestParams/RequestParams.yml")
# Read the parameter file
rCparams = smart_read_yaml(request_param_path, mpi_mode=is_mpi_env())
# Earthquake and Station parameters
cmt_dir = os.path.dirname(cmt_filename)
station_dir = os.path.join(cmt_dir, "station_data")
# Get STATIONS file from CMT directory
stationsfile = os.path.join(station_dir, "STATIONS")
# Observed output dir
obsd_dir = os.path.join(cmt_dir, "seismograms", "obs")
# CMT parameter input
cmt = CMTSource.from_CMTSOLUTION_file(cmt_filename)
duration = rCparams['duration']
starttime_offset = rCparams['starttime_offset']
starttime = cmt.origin_time + starttime_offset
endtime = starttime + duration
# Get station_list from station_file in database entry
stations = read_station_file(stationsfile)
# Create list of networks to download from
networks = list(set([station[0] for station in stations]))
network_string = ",".join(networks)
# Set domain containing all locations
# Rectangular domain containing parts of southern Germany.
domain = RectangularDomain(minlatitude=-90,
maxlatitude=90,
minlongitude=-180,
maxlongitude=180)
# Set download restrictions
restrictions = Restrictions(
starttime=starttime,
endtime=endtime,
reject_channels_with_gaps=False,
minimum_length=float(rCparams['minimum_length']),
# Trace needs to be almost full length
network=network_string, # Only certain networks
channel=",".join(rCparams['channels']),
location=",".join(rCparams['locations']))
# No specified providers will result in all known ones being queried.
providers = ["IRIS"]
mdl = MassDownloader(providers=providers)
# The data will be downloaded to the ``./waveforms/`` and ``./stations/``
# folders with automatically chosen file n
stationxml_storage = os.path.join(station_dir)
waveform_storage = os.path.join(obsd_dir)
logger.info("MSEEDs: %s" % waveform_storage)
logger.info("XMLs: %s" % stationxml_storage)
mdl.download(domain,
restrictions,
mseed_storage=waveform_storage,
stationxml_storage=stationxml_storage)
#!/usr/bin/env python
# -*- coding: utf8 -*-
import os
from obspy import UTCDateTime
from obspy.clients.fdsn.mass_downloader import RectangularDomain, \
Restrictions, MassDownloader
network = "ZV"
starttime = UTCDateTime(2008, 1, 1)
endtime = UTCDateTime(2011, 1, 1)
ROOTDIR = "/run/media/seisman/Seagate Expansion Drive/DOWNLOADING/"
DATADIR = ROOTDIR + network
domain = RectangularDomain(minlatitude=18,
maxlatitude=54,
minlongitude=65,
maxlongitude=135)
restrictions = Restrictions(
starttime=starttime,
endtime=endtime,
network=network,
channel="BH?,SH?,LH?",
chunklength_in_sec=86400,
reject_channels_with_gaps=False,
minimum_length=0.0,
minimum_interstation_distance_in_m=50.0,
location_priorities=['', '00', '10', '01', '*'],
)
def retrieveData(self):
"""Retrieve data from many FDSN services, turn into StreamCollection.
Args:
event (dict):
Best dictionary matching input event, fields as above
in return of getMatchingEvents().
Returns:
StreamCollection: StreamCollection object.
"""
# Bail out if FDSNFetcher not configured
if "FDSNFetcher" not in self.config["fetchers"]:
return
fdsn_conf = self.config["fetchers"]["FDSNFetcher"]
rawdir = self.rawdir
if self.rawdir is None:
rawdir = tempfile.mkdtemp()
else:
if not os.path.isdir(rawdir):
os.makedirs(rawdir)
# use the mass downloader to retrieve data of interest from any FSDN
# service.
origin_time = UTCDateTime(self.time)
# The Obspy mass downloader has it's own logger - grab that stream
# and write it to our own log file
ldict = logging.Logger.manager.loggerDict
if OBSPY_LOGGER in ldict:
root = logging.getLogger()
fhandler = root.handlers[0]
obspy_logger = logging.getLogger(OBSPY_LOGGER)
try:
obspy_stream_handler = obspy_logger.handlers[0]
obspy_logger.removeHandler(obspy_stream_handler)
except IndexError:
pass
obspy_logger.addHandler(fhandler)
# Circular domain around the epicenter.
if fdsn_conf["domain"]["type"] == "circular":
dconf = fdsn_conf["domain"]["circular"]
if dconf["use_epicenter"]:
dconf["latitude"] = self.lat
dconf["longitude"] = self.lon
dconf.pop("use_epicenter")
domain = CircularDomain(**dconf)
elif fdsn_conf["domain"]["type"] == "rectangular":
dconf = fdsn_conf["domain"]["rectangular"]
domain = RectangularDomain(**dconf)
else:
raise ValueError(
'Domain type must be either "circular" or "rectangular".')
rconf = fdsn_conf["restrictions"]
rconf["starttime"] = origin_time - rconf["time_before"]
rconf["endtime"] = origin_time + rconf["time_after"]
rconf.pop("time_before")
rconf.pop("time_after")
restrictions = Restrictions(**rconf)
# For each of the providers, check if we have a username and password provided
# in the config. If we do, initialize the client with the username and password.
# Otherwise, use default initalization.
providers = URL_MAPPINGS
if "IRISPH5" in providers.keys():
del providers["IRISPH5"]
client_list = []
for provider_str in providers.keys():
if provider_str == GEO_NET_ARCHIVE_KEY:
dt = UTCDateTime.utcnow() - UTCDateTime(self.time)
if dt < GEONET_ARCHIVE_DAYS:
provider_str = GEONET_REALTIME_URL
try:
fdsn_config = self.config["fetchers"]["FDSNFetcher"]
if provider_str in fdsn_config:
if logging.getLevelName(root.level) == "DEBUG":
client = Client(
provider_str,
user=fdsn_config[provider_str]["user"],
password=fdsn_config[provider_str]["password"],
debug=True,
)
else:
client = Client(
provider_str,
user=fdsn_config[provider_str]["user"],
password=fdsn_config[provider_str]["password"],
)
else:
if logging.getLevelName(root.level) == "DEBUG":
client = Client(provider_str, debug=True)
else:
client = Client(provider_str)
client_list.append(client)
# If the FDSN service is down, then an FDSNException is raised
except FDSNException:
logging.warning(f"Unable to initalize client {provider_str}")
except KeyError:
logging.warning(f"Unable to initalize client {provider_str}")
if len(client_list):
# Pass off the initalized clients to the Mass Downloader
if logging.getLevelName(root.level) == "DEBUG":
for handler in root.handlers:
if hasattr(handler, "baseFilename"):
log_file = getattr(handler, "baseFilename")
sys.stdout = open(log_file, "a")
mdl = MassDownloader(providers=client_list, debug=True)
else:
try:
# Need to turn off built in logging for ObsPy>=1.3.0
mdl = MassDownloader(providers=client_list,
configure_logging=False)
except TypeError:
# For ObsPy<1.3.0 the configure_logging parameter doesn't exist
mdl = MassDownloader(providers=client_list)
logging.info("Downloading new MiniSEED files...")
# The data will be downloaded to the ``./waveforms/`` and
# ``./stations/`` folders with automatically chosen file names.
mdl.download(domain,
restrictions,
mseed_storage=rawdir,
stationxml_storage=rawdir)
sys.stdout.close()
if self.stream_collection:
seed_files = glob.glob(os.path.join(rawdir, "*.mseed"))
streams = []
for seed_file in seed_files:
try:
tstreams = read_obspy(seed_file, self.config)
except BaseException as e:
tstreams = None
fmt = 'Could not read seed file %s - "%s"'
logging.info(fmt % (seed_file, str(e)))
if tstreams is None:
continue
else:
streams += tstreams
stream_collection = StreamCollection(
streams=streams, drop_non_free=self.drop_non_free)
return stream_collection
else:
return None
def downloadMseeds(client_list,
stations_json,
output_dir,
start_time,
end_time,
min_lat,
max_lat,
min_lon,
max_lon,
chunk_size,
channel_list=[],
n_processor=None):
"""
Uses obspy mass downloader to get continuous waveforms from a specific client in miniseed format in variable chunk sizes. The minimum chunk size is 1 day.
Parameters
----------
client_list: list
List of client names e.g. ["IRIS", "SCEDC", "USGGS"].
stations_json: dic,
Station informations.
output_dir: str
Output directory.
min_lat: float
Min latitude of the region.
max_lat: float
Max latitude of the region.
min_lon: float
Min longitude of the region.
max_lon: float
Max longitude of the region.
start_time: str
Start DateTime for the beginning of the period in "YYYY-MM-DDThh:mm:ss.f" format.
end_time: str
End DateTime for the beginning of the period in "YYYY-MM-DDThh:mm:ss.f" format.
channel_list: str, default=[]
A list containing the desired channel codes. Downloads will be limited to these channels based on priority. Defaults to [] --> all channels
chunk_size: int
Chunck size in day.
n_processor: int, default=None
Number of CPU processors for parallel downloading.
Returns
----------
output_name/station_name/*.mseed: Miniseed fiels for each station.
Warning
----------
usage of multiprocessing and parallel downloads heavily depends on the client. It might cause missing some data for some network. Please test first for some short period and if It did miss some chunks of data for some channels then set n_processor=None to avoid parallel downloading.
"""
json_file = open(stations_json)
station_dic = json.load(json_file)
print(
f"####### There are {len(station_dic)} stations in the list. #######")
start_t = UTCDateTime(start_time)
end_t = UTCDateTime(end_time)
domain = RectangularDomain(minlatitude=min_lat,
maxlatitude=max_lat,
minlongitude=min_lon,
maxlongitude=max_lon)
mdl = MassDownloader(providers=client_list)
bg = start_t
if n_processor == None:
for st in station_dic:
print(f'======= Working on {st} station.')
_get_w(bg, st, station_dic, end_t, mdl, domain, output_dir,
chunk_size, channel_list)
else:
def process(st):
print(f'======= Working on {st} station.')
_get_w(bg, st, station_dic, end_t, mdl, domain, output_dir,
chunk_size, channel_list)
with ThreadPool(n_processor) as p:
p.map(process, [st for st in station_dic])
def download_rf(self, datadir, minDelta=30, maxDelta=150, fskip=1, chanrank=['BH', 'HH'], channels = 'ZNE', phase='P',\
startoffset=-30., endoffset=60.0, verbose=False, start_date=None, end_date=None, skipinv=True, threads_per_client = 3,\
providers = None, blon = 0.05, blat = 0.05):
"""request receiver function data from IRIS server
====================================================================================================================
::: input parameters :::
min/maxDelta - minimum/maximum epicentral distance, in degree
channels - Channel code, e.g. 'BHZ'.
Last character (i.e. component) can be a wildcard (‘?’ or ‘*’) to fetch Z, N and E component.
min/maxDelta - minimum/maximum epicentral distance, in degree
channel_rank - rank of channel types
phase - body wave phase to be downloaded, arrival time will be computed using taup
start/endoffset - start and end offset for downloaded data
=====================================================================================================================
"""
if providers is None:
providers = ['BGR', 'ETH', 'GFZ', 'ICGC', 'INGV', 'IPGP',\
'IRIS', 'KNMI', 'KOERI', 'LMU', 'NCEDC', 'NIEP', 'NOA', 'ODC', 'ORFEUS',\
'RASPISHAKE', 'RESIF', 'SCEDC', 'TEXNET', 'USP']
self.get_limits_lonlat()
minlongitude = self.minlon
maxlongitude = self.maxlon
if minlongitude > 180.:
minlongitude -= 360.
if maxlongitude > 180.:
maxlongitude -= 360.
lon0 = (minlongitude + maxlongitude) / 2.
lat0 = (self.minlat + self.maxlat) / 2.
domain = RectangularDomain(minlatitude=self.minlat - blat,
maxlatitude=self.maxlat + blat,
minlongitude=minlongitude - blon,
maxlongitude=maxlongitude + blon)
try:
print(self.cat)
except AttributeError:
self.copy_catalog()
try:
stime4down = obspy.core.utcdatetime.UTCDateTime(start_date)
except:
stime4down = obspy.UTCDateTime(0)
try:
etime4down = obspy.core.utcdatetime.UTCDateTime(end_date)
except:
etime4down = obspy.UTCDateTime()
mdl = MassDownloader(providers=providers)
chantype_list = []
for chantype in chanrank:
chantype_list.append('%s[%s]' % (chantype, channels))
channel_priorities = tuple(chantype_list)
t1 = time.time()
# loop over events
ievent = 0
for event in self.cat:
event_id = event.resource_id.id.split('=')[-1]
pmag = event.preferred_magnitude()
magnitude = pmag.mag
Mtype = pmag.magnitude_type
event_descrip = event.event_descriptions[
0].text + ', ' + event.event_descriptions[0].type
porigin = event.preferred_origin()
otime = porigin.time
if otime < stime4down or otime > etime4down:
continue
ievent += 1
try:
print('[%s] [DOWNLOAD BODY WAVE] ' %datetime.now().isoformat().split('.')[0] + \
'Event ' + str(ievent)+': '+ str(otime)+' '+ event_descrip+', '+Mtype+' = '+str(magnitude))
except:
print('[%s] [DOWNLOAD BODY WAVE] ' %datetime.now().isoformat().split('.')[0] + \
'Event ' + str(ievent)+': '+ str(otime)+' '+ event_descrip+', M = '+str(magnitude))
evlo = porigin.longitude
evla = porigin.latitude
try:
evdp = porigin.depth / 1000.
except:
continue
# log file existence
oyear = otime.year
omonth = otime.month
oday = otime.day
ohour = otime.hour
omin = otime.minute
osec = otime.second
label = '%d_%s_%d_%d_%d_%d' % (oyear, mondict[omonth], oday, ohour,
omin, osec)
eventdir = datadir + '/' + label
if fskip == 2 and os.path.isdir(eventdir):
continue
if not os.path.isdir(eventdir):
os.makedirs(eventdir)
event_logfname = eventdir + '/download.log'
if fskip == 1 and os.path.isfile(event_logfname):
continue
stationxml_storage = "%s/{network}/{station}.xml" % eventdir
# loop over stations
Nsta = 0
for network in self.inv:
for station in network:
netcode = network.code
stacode = station.code
staid = netcode + '.' + stacode
with warnings.catch_warnings():
warnings.simplefilter("ignore")
st_date = station.start_date
ed_date = station.end_date
if skipinv and (otime < st_date or otime > ed_date):
continue
stlo = station.longitude
stla = station.latitude
dist, az, baz = obspy.geodetics.gps2dist_azimuth(
evla, evlo, stla, stlo) # distance is in m
dist = dist / 1000.
if baz < 0.:
baz += 360.
Delta = obspy.geodetics.kilometer2degrees(dist)
if Delta < minDelta:
continue
if Delta > maxDelta:
continue
arrivals = taupmodel.get_travel_times(
source_depth_in_km=evdp,
distance_in_degree=Delta,
phase_list=[phase]) #, receiver_depth_in_km=0)
try:
arr = arrivals[0]
arrival_time = arr.time
rayparam = arr.ray_param_sec_degree
except IndexError:
continue
starttime = otime + arrival_time + startoffset
endtime = otime + arrival_time + endoffset
restrictions = Restrictions(
network=netcode,
station=stacode,
# starttime and endtime
starttime=starttime,
endtime=endtime,
# You might not want to deal with gaps in the data.
reject_channels_with_gaps=True,
# And you might only want waveforms that have data for at least
# 95 % of the requested time span.
minimum_length=0.95,
# No two stations should be closer than 10 km to each other.
minimum_interstation_distance_in_m=10E3,
channel_priorities=channel_priorities,
sanitize=True)
mseed_storage = eventdir
try:
mdl.download(domain,
restrictions,
mseed_storage=mseed_storage,
stationxml_storage=stationxml_storage,
threads_per_client=threads_per_client)
except Exception:
continue
Nsta += 1
print('--- [DOWNLOAD BODY WAVE] Event: %s %s' %
(otime.isoformat(), event_descrip))
with open(event_logfname, 'w') as fid:
fid.writelines('evlo: %g, evla: %g\n' % (evlo, evla))
fid.writelines('DONE\n')
return