def test():
# Test for channel grouping with three unique channels
streams = []
# datadir = os.path.join(homedir, '..', 'data', 'knet', 'us2000cnnl')
datafiles, origin = read_data_dir("knet", "us2000cnnl", "AOM0031801241951*")
for datafile in datafiles:
streams += read_knet(datafile)
grouped_streams = StreamCollection(streams)
assert len(grouped_streams) == 1
assert grouped_streams[0].count() == 3
# Test for channel grouping with more file types
datafiles, origin = read_data_dir(
"geonet", "us1000778i", "20161113_110313_THZ_20.V2A"
)
datafile = datafiles[0]
streams += read_geonet(datafile)
grouped_streams = StreamCollection(streams)
assert len(grouped_streams) == 2
assert grouped_streams[0].count() == 3
assert grouped_streams[1].count() == 3
# Test for warning for one channel streams
datafiles, origin = read_data_dir("knet", "us2000cnnl", "AOM0071801241951.UD")
datafile = datafiles[0]
streams += read_knet(datafile)
grouped_streams = StreamCollection(streams)
# assert "One channel stream:" in logstream.getvalue()
assert len(grouped_streams) == 3
assert grouped_streams[0].count() == 3
assert grouped_streams[1].count() == 3
assert grouped_streams[2].count() == 1
def test():
homedir = os.path.dirname(os.path.abspath(
__file__)) # where is this script?
# Test for channel grouping with three unique channels
streams = []
# datadir = os.path.join(homedir, '..', 'data', 'knet', 'us2000cnnl')
datafiles, origin = read_data_dir('knet', 'us2000cnnl',
'AOM0031801241951*')
for datafile in datafiles:
streams += read_knet(datafile)
grouped_streams = StreamCollection(streams)
assert len(grouped_streams) == 1
assert grouped_streams[0].count() == 3
# Test for channel grouping with more file types
datafiles, origin = read_data_dir('geonet',
'us1000778i',
'20161113_110313_THZ_20.V2A')
datafile = datafiles[0]
streams += read_geonet(datafile)
grouped_streams = StreamCollection(streams)
assert len(grouped_streams) == 2
assert grouped_streams[0].count() == 3
assert grouped_streams[1].count() == 3
# Test for warning for one channel streams
datafiles, origin = read_data_dir(
'knet', 'us2000cnnl', 'AOM0071801241951.UD')
datafile = datafiles[0]
streams += read_knet(datafile)
grouped_streams = StreamCollection(streams)
# assert "One channel stream:" in logstream.getvalue()
assert len(grouped_streams) == 3
assert grouped_streams[0].count() == 3
assert grouped_streams[1].count() == 3
assert grouped_streams[2].count() == 1
def test():
homedir = os.path.dirname(os.path.abspath(
__file__)) # where is this script?
# Test for channel grouping with three unique channels
streams = []
# datadir = os.path.join(homedir, '..', 'data', 'knet', 'us2000cnnl')
datafiles, origin = read_data_dir('knet', 'us2000cnnl',
'AOM0031801241951*')
for datafile in datafiles:
streams += read_knet(datafile)
grouped_streams = StreamCollection(streams)
assert len(grouped_streams) == 1
assert grouped_streams[0].count() == 3
# Test for channel grouping with more file types
datafiles, origin = read_data_dir('geonet',
'us1000778i',
'20161113_110313_THZ_20.V2A')
datafile = datafiles[0]
streams += read_geonet(datafile)
grouped_streams = StreamCollection(streams)
assert len(grouped_streams) == 2
assert grouped_streams[0].count() == 3
assert grouped_streams[1].count() == 3
# Test for warning for one channel streams
datafiles, origin = read_data_dir(
'knet', 'us2000cnnl', 'AOM0071801241951.UD')
datafile = datafiles[0]
streams += read_knet(datafile)
grouped_streams = StreamCollection(streams)
# assert "One channel stream:" in logstream.getvalue()
assert len(grouped_streams) == 3
assert grouped_streams[0].count() == 3
assert grouped_streams[1].count() == 3
assert grouped_streams[2].count() == 1
def retrieveData(self, event_dict, stations=None):
"""Retrieve data from NIED, turn into StreamCollection.
Args:
event (dict):
Best dictionary matching input event, fields as above
in return of getMatchingEvents().
Returns:
StreamCollection: StreamCollection object.
"""
rawdir = self.rawdir
if self.rawdir is None:
rawdir = tempfile.mkdtemp()
else:
if not os.path.isdir(rawdir):
os.makedirs(rawdir)
cgi_value = event_dict['cgi_value']
firstid = cgi_value.split(',')[0]
dtime = event_dict['time']
fname = dtime.strftime('%Y%m%d%H%M%S') + '.tar'
localfile = os.path.join(rawdir, fname)
url = RETRIEVE_URL
if stations == None:
dkind = ['all']
else:
dkind = stations
payload = {'formattype': ['A'],
'eqidlist': cgi_value,
'datanames': '%s;alldata' % firstid,
'datakind': dkind }
logging.info('Downloading Japanese data into %s...' % localfile)
req = requests.get(url, params=payload,
auth=(self.user, self.password))
if req.status_code != URL_ERROR_CODE:
raise urllib.error.HTTPError(req.text)
else:
with open(localfile, 'wb') as f:
for chunk in req:
f.write(chunk)
logging.info('Finished downloading into %s...' % localfile)
# open the tarball, extract the kiknet/knet gzipped tarballs
tar = tarfile.open(localfile)
names = tar.getnames()
tarballs = []
for name in names:
if 'img' in name:
continue
ppath = os.path.join(rawdir, name)
tarballs.append(ppath)
tar.extract(name, path=rawdir)
tar.close()
# remove the tar file we downloaded
os.remove(localfile)
subdirs = []
for tarball in tarballs:
tar = tarfile.open(tarball, mode='r:gz')
if 'kik' in tarball:
subdir = os.path.join(rawdir, 'kiknet')
else:
subdir = os.path.join(rawdir, 'knet')
subdirs.append(subdir)
tar.extractall(path=subdir)
tar.close()
os.remove(tarball)
streams = []
for subdir in subdirs:
gzfiles = glob.glob(os.path.join(subdir, '*.gz'))
for gzfile in gzfiles:
os.remove(gzfile)
datafiles = glob.glob(os.path.join(subdir, '*.*'))
for dfile in datafiles:
logging.info('Reading KNET/KikNet file %s...' % dfile)
streams += read_knet(dfile)
if self.rawdir is None:
shutil.rmtree(rawdir)
# Japan gives us a LOT of data, much of which is not useful as it is
# too far away. Use the following distance thresholds for different
# magnitude ranges, and trim streams that are beyond this distance.
if self.restrict_stations:
threshold_distance = None
for mag, tdistance in MAGS.items():
if self.magnitude < mag:
threshold_distance = tdistance
break
else:
threshold_distance = 99999999999999.9
newstreams = []
for stream in streams:
slat = stream[0].stats.coordinates.latitude
slon = stream[0].stats.coordinates.longitude
distance = geodetic_distance(self.lon, self.lat, slon, slat)
if distance <= threshold_distance:
newstreams.append(stream)
stream_collection = StreamCollection(streams=newstreams,
drop_non_free=self.drop_non_free)
return stream_collection
def test():
dpath = os.path.join("data", "testdata", "knet", "us2000cnnl")
datadir = pkg_resources.resource_filename("gmprocess", dpath)
knet_file1 = os.path.join(datadir, "AOM0051801241951.EW")
knet_file2 = os.path.join(datadir, "AOM0051801241951.NS")
knet_file3 = os.path.join(datadir, "AOM0051801241951.UD")
assert is_knet(knet_file1)
assert is_knet(os.path.abspath(__file__)) is False
# test a knet file with npoints % 10 == 0
stream1 = read_knet(knet_file1)[0]
stream2 = read_knet(knet_file2)[0]
stream3 = read_knet(knet_file3)[0]
np.testing.assert_almost_equal(stream1[0].max(), -37.149, decimal=2)
np.testing.assert_almost_equal(stream2[0].max(), 32.859, decimal=2)
np.testing.assert_almost_equal(stream3[0].max(), 49.000, decimal=2)
# test a file that has a number of points divisible by 8
knet_file4 = os.path.join(datadir, "AOM0011801241951.EW")
knet_file5 = os.path.join(datadir, "AOM0011801241951.NS")
knet_file6 = os.path.join(datadir, "AOM0011801241951.UD")
stream4 = read_knet(knet_file4)[0]
stream5 = read_knet(knet_file5)[0]
stream6 = read_knet(knet_file6)[0]
np.testing.assert_almost_equal(stream4[0].max(), -11.435, decimal=2)
np.testing.assert_almost_equal(stream5[0].max(), 12.412, decimal=2)
np.testing.assert_almost_equal(stream6[0].max(), -9.284, decimal=2)
# test that a file that is not knet format raises an Exception
try:
knet_files, _ = read_data_dir("geonet", "nz2018p115908",
"20161113_110256_WTMC_20.V1A")
knet_file = knet_files[0]
read_knet(knet_file)[0]
success = True
except Exception:
success = False
assert not success
# test some kiknet files
dpath = os.path.join("data", "testdata", "kiknet", "usp000a1b0")
datadir = pkg_resources.resource_filename("gmprocess", dpath)
kiknet_file1 = os.path.join(datadir, "AICH040010061330.EW2")
kiknet_file2 = os.path.join(datadir, "AICH040010061330.NS2")
kiknet_file3 = os.path.join(datadir, "AICH040010061330.UD2")
assert is_knet(knet_file1)
stream1 = read_knet(kiknet_file1)[0] # east-west
stream2 = read_knet(kiknet_file2)[0] # north-south
stream3 = read_knet(kiknet_file3)[0] # vertical
assert stream1[0].stats["channel"] == "HN2"
assert stream2[0].stats["channel"] == "HN1"
assert stream3[0].stats["channel"] == "HNZ"
ewmax = np.abs(stream1[0].data).max()
nsmax = np.abs(stream2[0].data).max()
udmax = np.abs(stream3[0].data).max()
np.testing.assert_almost_equal(ewmax, 5.020, decimal=1)
np.testing.assert_almost_equal(nsmax, 10.749, decimal=1)
np.testing.assert_almost_equal(udmax, 9.111, decimal=1)
def test():
dpath = os.path.join('data', 'testdata', 'knet', 'us2000cnnl')
datadir = pkg_resources.resource_filename('gmprocess', dpath)
knet_file1 = os.path.join(datadir, 'AOM0051801241951.EW')
knet_file2 = os.path.join(datadir, 'AOM0051801241951.NS')
knet_file3 = os.path.join(datadir, 'AOM0051801241951.UD')
assert is_knet(knet_file1)
try:
assert is_knet(os.path.abspath(__file__))
except AssertionError:
assert 1 == 1
# test a knet file with npoints % 10 == 0
stream1 = read_knet(knet_file1)[0]
stream2 = read_knet(knet_file2)[0]
stream3 = read_knet(knet_file3)[0]
np.testing.assert_almost_equal(stream1[0].max(), -37.149, decimal=2)
np.testing.assert_almost_equal(stream2[0].max(), 32.859, decimal=2)
np.testing.assert_almost_equal(stream3[0].max(), 49.000, decimal=2)
# test a file that has a number of points divisible by 8
knet_file4 = os.path.join(datadir, 'AOM0011801241951.EW')
knet_file5 = os.path.join(datadir, 'AOM0011801241951.NS')
knet_file6 = os.path.join(datadir, 'AOM0011801241951.UD')
stream4 = read_knet(knet_file4)[0]
stream5 = read_knet(knet_file5)[0]
stream6 = read_knet(knet_file6)[0]
np.testing.assert_almost_equal(stream4[0].max(), -11.435, decimal=2)
np.testing.assert_almost_equal(stream5[0].max(), 12.412, decimal=2)
np.testing.assert_almost_equal(stream6[0].max(), -9.284, decimal=2)
# test that a file that is not knet format raises an Exception
try:
knet_files, _ = read_data_dir('geonet',
'nz2018p115908',
'20161113_110256_WTMC_20.V1A')
knet_file = knet_files[0]
read_knet(knet_file)[0]
success = True
except Exception:
success = False
assert not success
# test some kiknet files
dpath = os.path.join('data', 'testdata', 'kiknet', 'usp000a1b0')
datadir = pkg_resources.resource_filename('gmprocess', dpath)
kiknet_file1 = os.path.join(datadir, 'AICH040010061330.EW2')
kiknet_file2 = os.path.join(datadir, 'AICH040010061330.NS2')
kiknet_file3 = os.path.join(datadir, 'AICH040010061330.UD2')
assert is_knet(knet_file1)
stream1 = read_knet(kiknet_file1)[0] # east-west
stream2 = read_knet(kiknet_file2)[0] # north-south
stream3 = read_knet(kiknet_file3)[0] # vertical
assert stream1[0].stats['channel'] == 'HN2'
assert stream2[0].stats['channel'] == 'HN1'
assert stream3[0].stats['channel'] == 'HNZ'
ewmax = np.abs(stream1[0].data).max()
nsmax = np.abs(stream2[0].data).max()
udmax = np.abs(stream3[0].data).max()
np.testing.assert_almost_equal(ewmax, 5.020, decimal=1)
np.testing.assert_almost_equal(nsmax, 10.749, decimal=1)
np.testing.assert_almost_equal(udmax, 9.111, decimal=1)
def retrieveData(self, event_dict):
"""Retrieve data from NIED, turn into StreamCollection.
Args:
event (dict):
Best dictionary matching input event, fields as above
in return of getMatchingEvents().
Returns:
StreamCollection: StreamCollection object.
"""
rawdir = self.rawdir
if self.rawdir is None:
rawdir = tempfile.mkdtemp()
else:
if not os.path.isdir(rawdir):
os.makedirs(rawdir)
cgi_value = event_dict['cgi_value']
firstid = cgi_value.split(',')[0]
dtime = event_dict['time']
fname = dtime.strftime('%Y%m%d%H%M%S') + '.tar'
localfile = os.path.join(rawdir, fname)
url = RETRIEVE_URL
payload = {'formattype': ['A'],
'eqidlist': cgi_value,
'datanames': '%s;alldata' % firstid,
'datakind': ['all']}
logging.info('Downloading Japanese data into %s...' % localfile)
req = requests.get(url, params=payload,
auth=(self.user, self.password))
if req.status_code != URL_ERROR_CODE:
raise urllib.error.HTTPError(req.text)
else:
with open(localfile, 'wb') as f:
for chunk in req:
f.write(chunk)
logging.info('Finished downloading into %s...' % localfile)
# open the tarball, extract the kiknet/knet gzipped tarballs
tar = tarfile.open(localfile)
names = tar.getnames()
tarballs = []
for name in names:
if 'img' in name:
continue
ppath = os.path.join(rawdir, name)
tarballs.append(ppath)
tar.extract(name, path=rawdir)
tar.close()
# remove the tar file we downloaded
os.remove(localfile)
subdirs = []
for tarball in tarballs:
tar = tarfile.open(tarball, mode='r:gz')
if 'kik' in tarball:
subdir = os.path.join(rawdir, 'kiknet')
else:
subdir = os.path.join(rawdir, 'knet')
subdirs.append(subdir)
tar.extractall(path=subdir)
tar.close()
os.remove(tarball)
streams = []
for subdir in subdirs:
gzfiles = glob.glob(os.path.join(subdir, '*.gz'))
for gzfile in gzfiles:
os.remove(gzfile)
datafiles = glob.glob(os.path.join(subdir, '*.*'))
for dfile in datafiles:
logging.info('Reading KNET/KikNet file %s...' % dfile)
streams += read_knet(dfile)
if self.rawdir is None:
shutil.rmtree(rawdir)
stream_collection = StreamCollection(streams=streams,
drop_non_free=self.drop_non_free)
return stream_collection
def retrieveData(self, event_dict):
"""Retrieve data from NIED, turn into StreamCollection.
Args:
event (dict):
Best dictionary matching input event, fields as above
in return of getMatchingEvents().
Returns:
StreamCollection: StreamCollection object.
"""
rawdir = self.rawdir
if self.rawdir is None:
rawdir = tempfile.mkdtemp()
else:
if not os.path.isdir(rawdir):
os.makedirs(rawdir)
cgi_value = event_dict["cgi_value"]
firstid = cgi_value.split(",")[0]
dtime = event_dict["time"]
fname = dtime.strftime("%Y%m%d%H%M%S") + ".tar"
localfile = os.path.join(rawdir, fname)
url = RETRIEVE_URL
payload = {
"formattype": ["A"],
"eqidlist": cgi_value,
"datanames": f"{firstid};alldata",
"datakind": ["all"],
}
logging.info(f"Downloading Japanese data into {localfile}...")
req = requests.get(url, params=payload, auth=(self.user, self.password))
logging.debug("KNET download url: %s", str(url))
logging.debug("KNET download response code: %s", req.status_code)
if req.status_code != URL_ERROR_CODE:
raise urllib.error.HTTPError(req.text)
else:
with open(localfile, "wb") as f:
for chunk in req:
f.write(chunk)
logging.info(f"Finished downloading into {localfile}...")
# open the tarball, extract the kiknet/knet gzipped tarballs
tar = tarfile.open(localfile)
names = tar.getnames()
tarballs = []
for name in names:
if "img" in name:
continue
ppath = os.path.join(rawdir, name)
tarballs.append(ppath)
tar.extract(name, path=rawdir)
tar.close()
# remove the tar file we downloaded
os.remove(localfile)
subdirs = []
for tarball in tarballs:
tar = tarfile.open(tarball, mode="r:gz")
if "kik" in tarball:
subdir = os.path.join(rawdir, "kiknet")
else:
subdir = os.path.join(rawdir, "knet")
subdirs.append(subdir)
tar.extractall(path=subdir)
tar.close()
os.remove(tarball)
for subdir in subdirs:
gzfiles = glob.glob(os.path.join(subdir, "*.gz"))
for gzfile in gzfiles:
os.remove(gzfile)
if self.stream_collection:
streams = []
for subdir in subdirs:
datafiles = glob.glob(os.path.join(subdir, "*.*"))
for dfile in datafiles:
logging.info(f"Reading KNET/KikNet file {dfile}...")
streams += read_knet(dfile)
if self.rawdir is None:
shutil.rmtree(rawdir)
# Japan gives us a LOT of data, much of which is not useful as it
# is too far away. Use the following distance thresholds for
# different magnitude ranges, and trim streams that are beyond this
# distance.
threshold_distance = None
if self.restrict_stations:
for mag, tdistance in MAGS.items():
if self.magnitude < mag:
threshold_distance = tdistance
break
newstreams = []
for stream in streams:
slat = stream[0].stats.coordinates.latitude
slon = stream[0].stats.coordinates.longitude
distance = geodetic_distance(self.lon, self.lat, slon, slat)
if distance <= threshold_distance:
newstreams.append(stream)
stream_collection = StreamCollection(
streams=newstreams, drop_non_free=self.drop_non_free
)
return stream_collection
else:
return None
