def execute(self):
"""
Write info.json metadata file.
Raises:
NotADirectoryError: When the event data directory does not exist.
FileNotFoundError: When the the shake_result HDF file does not
exist.
"""
_, data_path = get_config_paths()
datadir = os.path.join(data_path, self._eventid, 'current')
if not os.path.isdir(datadir):
os.makedirs(datadir)
# try to find the event by our event id
try:
detail = get_event_by_id(self._eventid)
dataframe, msg = _get_dyfi_dataframe(detail)
except Exception as e:
fmt = 'Could not retrieve DYFI data for %s - error "%s"'
self.logger.warning(fmt % (self._eventid, str(e)))
return
if dataframe is None:
self.logger.info(msg)
return
reference = 'USGS Did You Feel It? System'
xmlfile = os.path.join(datadir, 'dyfi_dat.xml')
dataframe_to_xml(dataframe, xmlfile, reference)
self.logger.info('Wrote %i DYFI records to %s' %
(len(dataframe), xmlfile))
def test_comcat_data():
datadir = get_datadir()
config = get_config()
# test extraction from comcat event data. The VCR files are
# example Comcat event datastreams.
eventid = 'ci14607652'
# This event has both geo_1km and geo_10km
tape_file1 = os.path.join(datadir, 'vcr_comcat_geojson.yaml')
iparser = IntensityParser(eventid=eventid, config=config, network='neic')
with vcr.use_cassette(tape_file1):
detail = get_event_by_id(eventid)
df, msg = comcat.get_dyfi_dataframe_from_comcat(iparser, detail)
df, msg = iparser.postprocess(df, 'neic')
np.testing.assert_almost_equal(df['INTENSITY'].sum(), 4510.1)
reference = 'USGS Did You Feel It? System'
tempdir = tempfile.mkdtemp(prefix='tmp.', dir=datadir)
outfile = os.path.join(tempdir, 'dyfi_dat.xml')
dataframe_to_xml(df, outfile, reference)
# For debugging save the output with this line:
# dataframe_to_xml(df, datadir + '/tmp.keepthis.xml', reference)
outfilesize = os.path.getsize(outfile)
# Longer size is for file with nresp field
assert outfilesize == 183953 or outfilesize == 172852
rmtree(tempdir)
# This event has only text data
eventid = 'ci14745580'
tape_file2 = os.path.join(datadir, 'vcr_comcat_txt.yaml')
with vcr.use_cassette(tape_file2):
detail = get_event_by_id(eventid)
df, msg = comcat.get_dyfi_dataframe_from_comcat(iparser, detail)
df, msg = iparser.postprocess(df, 'neic')
np.testing.assert_almost_equal(df['INTENSITY'].sum(), 800.4)
def test_dataframe_to_xml():
homedir = os.path.dirname(os.path.abspath(
__file__)) # where is this script?
datadir = os.path.join(homedir, '..', 'data')
amps_output = os.path.join(datadir, 'amps.csv')
df = pd.read_csv(amps_output)
outdir = tempfile.mkdtemp()
try:
xmlfile = os.path.join(outdir, 'foo_dat.xml')
dataframe_to_xml(df, xmlfile)
# HNN,psa10,0.0107
root = minidom.parse(xmlfile)
comps = root.getElementsByTagName('comp')
for comp in comps:
if comp.getAttribute('name') == 'HNN':
psa10 = comp.getElementsByTagName('psa10')[0]
value = float(psa10.getAttribute('value'))
assert value == 0.0107
except Exception:
assert 1 == 2
finally:
shutil.rmtree(outdir)
def test_comcat_file():
eventid = 'nc72282711'
datadir = get_datadir()
config = get_config()
iparser = IntensityParser(eventid=eventid, config=config, network='neic')
# Test reading a comcat file
testfile = os.path.join(datadir, 'nc72282711_dyfi_geo_10km.geojson')
df, msg = iparser.get_dyfi_dataframe_from_file(testfile)
assert len(df) == 203
np.testing.assert_equal(df['INTENSITY'].sum(), 705.3)
np.testing.assert_equal(df['NRESP'].sum(), 16202)
# Test that stddev and resp carry throuh
try:
np.testing.assert_almost_equal(df['INTENSITY_STDDEV'].sum(), 39.548)
reference = 'TEST'
tempdir = tempfile.mkdtemp(prefix='tmp.', dir=datadir)
outfile = os.path.join(tempdir, 'dyfi_dat.xml')
# Make sure output file includes stddev
dataframe_to_xml(df, outfile, reference)
# To save this file for testing:
# dataframe_to_xml(df, datadir + '/tmp.test_for_stddev.xml', reference)
with open(outfile, 'r') as f:
textdata = f.read()
assert 'stddev' in textdata
rmtree(tempdir)
except AssertionError:
msg = 'A newer version of impactutils is required to process \
intensity parameters. If you do not work with stddev, \
you can safely ignore this warning.'
rmtree(tempdir)
warnings.warn(msg)
def test_write_xml():
# where is this script?
homedir = os.path.dirname(os.path.abspath(__file__))
datadir = os.path.join(homedir, '..', 'data')
complete_file = os.path.join(datadir, 'complete_pgm.xlsx')
mmimin_file = os.path.join(datadir, 'minimum_mmi.xlsx')
shakemap_file = os.path.join(datadir, 'shakemap.xlsx')
tempdir = None
try:
tempdir = tempfile.mkdtemp()
df, reference = read_excel(complete_file)
xmlfile = os.path.join(tempdir, 'foo.xml')
dataframe_to_xml(df, xmlfile, reference=reference)
xmlfile = os.path.join(tempdir, 'bar.xml')
df_mmimin, reference = read_excel(mmimin_file)
dataframe_to_xml(df_mmimin, xmlfile, reference=reference)
xmlfile = os.path.join(tempdir, 'shakemap.xml')
df_shakemap, reference = read_excel(shakemap_file)
dataframe_to_xml(df_shakemap, xmlfile, reference=reference)
root = minidom.parse(xmlfile)
stationlist = root.getElementsByTagName('stationlist')[0]
station = stationlist.getElementsByTagName('station')[0]
comp = station.getElementsByTagName('comp')[0]
pga = comp.getElementsByTagName('pga')[0]
assert station.getAttribute('code') == "I1.8226"
assert comp.getAttribute('name') == 'H1'
tvalue = float(pga.getAttribute('value'))
np.testing.assert_almost_equal(tvalue, 0.1026)
root.unlink()
except Exception:
raise AssertionError('Could not write XML file.')
finally:
if tempdir is not None:
shutil.rmtree(tempdir)
def test_read_tables():
homedir = os.path.dirname(os.path.abspath(
__file__)) # where is this script?
datadir = os.path.join(homedir, '..', 'data')
##########################################
# these files should all read successfully
##########################################
tmpdir = tempfile.mkdtemp()
try:
complete_file = os.path.join(datadir, 'complete_pgm.xlsx')
df_complete, _ = read_excel(complete_file)
np.testing.assert_almost_equal(df_complete['H1']['PGA'].sum(), 569.17)
xmlfile = os.path.join(tmpdir, 'complete_pgm.xml')
dataframe_to_xml(df_complete, xmlfile)
pgamin_file = os.path.join(datadir, 'minimum_pga.xlsx')
df_pgamin, _ = read_excel(pgamin_file)
np.testing.assert_almost_equal(df_pgamin['UNK']['PGA'].sum(), 569.17)
xmlfile = os.path.join(tmpdir, 'minimum_pga.xml')
dataframe_to_xml(df_pgamin, xmlfile)
mmimin_file = os.path.join(datadir, 'minimum_mmi.xlsx')
df_mmimin, _ = read_excel(mmimin_file)
np.testing.assert_almost_equal(
df_mmimin['INTENSITY'].sum(), 45.199872273516036)
xmlfile = os.path.join(tmpdir, 'minimum_mmi.xml')
dataframe_to_xml(df_mmimin, xmlfile)
missing_data_file = os.path.join(datadir, 'missing_rows.xlsx')
df, _ = read_excel(missing_data_file)
assert np.isnan(df['H1']['SA(0.3)'].iloc[3])
xmlfile = os.path.join(tmpdir, 'missing_rows.xml')
dataframe_to_xml(df, xmlfile)
sm2xml_example = os.path.join(datadir, 'sm2xml_output.xlsx')
df, _ = read_excel(sm2xml_example)
np.testing.assert_almost_equal(
df['HHZ']['PGA'].sum(), 150.82342541678645)
xmlfile = os.path.join(tmpdir, 'sm2xml_output.xml')
dataframe_to_xml(df, xmlfile)
except Exception:
assert 1 == 2
finally:
shutil.rmtree(tmpdir)
##########################################
# these files should all fail
##########################################
try:
missing_file = os.path.join(datadir, 'missing_columns.xlsx')
read_excel(missing_file)
assert 1 == 2
except KeyError:
assert 1 == 1
try:
wrong_file = os.path.join(datadir, 'wrong_channels.xlsx')
read_excel(wrong_file)
assert 1 == 2
except KeyError:
assert 1 == 1
try:
nodata_file = os.path.join(datadir, 'no_data.xlsx')
read_excel(nodata_file)
assert 1 == 2
except KeyError:
assert 1 == 1
try:
emptyrow_file = os.path.join(datadir, 'empty_row.xlsx')
read_excel(emptyrow_file)
assert 1 == 2
except IndexError:
assert 1 == 1
try:
noref_file = os.path.join(datadir, 'no_reference.xlsx')
read_excel(noref_file)
assert 1 == 2
except KeyError:
assert 1 == 1