def get_quake_desc(event,lat,lon,isMainEvent):
ndeaths = event['TotalDeaths']
#summarize the exposure values
exposures = np.array([event['MMI1'],event['MMI2'],event['MMI3'],event['MMI4'],event['MMI5'],
event['MMI6'],event['MMI7'],event['MMI8'],event['MMI9+']])
exposures = np.array([round_to_nearest(exp,1000) for exp in exposures])
#get the highest two exposures greater than zero
iexp = np.where(exposures > 0)[0][::-1]
romans = ['I', 'II', 'III', 'IV', 'V', 'VI', 'VII', 'VIII', 'IX or greater']
if len(iexp) >= 2:
exposures = [exposures[iexp[1]],exposures[iexp[0]]]
ilevels = [romans[iexp[1]],romans[iexp[0]]]
expfmt = ', with estimated population exposures of %s at intensity'
expfmt = expfmt + ' %s and %s at intensity %s'
exptxt = expfmt % (commify(int(exposures[0])),ilevels[0],commify(int(exposures[1])),ilevels[1])
else:
exptxt = ''
#create string describing this most impactful event
dfmt = 'A magnitude %.1f earthquake %i km %s of this event struck %s on %s (UTC)%s'
mag = event['Magnitude']
etime = event['Time'].strftime('%B %d, %Y')
etime = re.sub(' 0',' ',etime)
country = Country()
if not event['Name']:
if event['CountryCode'] == 'UM' and event['Latitude'] > 40: #hack for persistent error in expocat
cdict = country.getCountryCode('US')
else:
cdict = country.getCountryCode(event['CountryCode'])
if cdict:
cname = cdict['Name']
else:
cname = 'in the open ocean'
else:
cname = event['Name'].replace('"','')
cdist = round(geodetic_distance(event['Lat'],event['Lon'],lat,lon))
cdir = get_compass_dir(lat,lon,event['Lat'],event['Lon'],format='long').lower()
if ndeaths and str(ndeaths) != "nan":
dfmt = dfmt + ', resulting in a reported %s %s.'
if ndeaths > 1:
dstr = 'fatalities'
else:
dstr = 'fatality'
ndeathstr = commify(int(ndeaths))
eqdesc = dfmt % (mag,cdist,cdir,cname,etime,exptxt,ndeathstr,dstr)
else:
dfmt = dfmt + ', with no reported fatalities.'
eqdesc = dfmt % (mag,cdist,cdir,cname,etime,exptxt)
return eqdesc
def __init__(self, dataframe):
"""Create an instance of a GDP object with a dataframe of countries/GDP values over time.
:param dataframe:
Pandas dataframe where rows are countries, and columns are rates for different years.
dataframe should look like this:
DataFrame({'Country Code':['AFG'],
'1960':[59.78768071],
'1961':[59.89003694],})
The above contains the GDP values for Afghanistan for 1960 and 196
"""
self._dataframe = dataframe
self._country = Country()
def __init__(self, inventory, collapse, casualty, workforce, growth):
"""Create Semi-Empirical Fatality Model object.
:param inventory:
HDFContainer, containing DataFrames named:
- 'BuildingTypes',
- 'RuralNonResidential'
- 'RuralResidential'
- 'UrbanNonResidential'
- 'UrbanResidential'
where BuildingTypes is a Dataframe with columns:
- Code: Building Code ('A','S','C1',etc.)
- ShortDescription: A short description of building type (i.e, 'adobe block').
- OperationalDescription: A (sometimes) shorter description of building type.
- LongDescription: Full description of building type, i.e. 'High-Rise Ductile Reinforced Concrete Moment Frame'
All other Dataframes have columns:
- CountryCode: Two letter ISO country code ('US',etc.)
- CountryName: Name of country.
- Building Codes, as listed and described in BuildingTypes Dataframe.
:param collapse:
HDFContainer, where first index is country (by two-letter country code), columns are:
- BuildingCode: Building code as above.
- (IMT)_6.0 -> (IMT)_9.0 Columns with collapse rates at each (IMT) interval, where (IMT) could be MMI,PGA,PGV,PSA1.0, etc.
:param casualty:
HDFContainer, where first index is country (by two-letter country code), columns are:
- BuildingCode: Building code as above.
- CasualtyDay: Casualty rate, given collapse, during the day.
- CasualtyNight: Casualty rate, given collapse, during the night.
:param workforce:
HDFContainer consisting of a single dataframe, where rows are by country, and columns are:
- CountryCode two letter ISO country code.
- CountryCode name of country.
- WorkforceTotal Fraction of the country population in the workforce.
- WorkforceAgriculture Fraction of the total workforce employed in agriculture.
- WorkforceIndustrial Fraction of the total workforce employed in industry.
- WorkforceServices Fraction of the total workforce employed in services.
:param growth:
PopulationGrowth object.
"""
self._inventory = inventory
self._collapse = collapse
self._casualty = casualty
self._workforce = workforce
self._popgrowth = growth
self._country = Country()
def toSeries(self, processtime=False):
if self.summary_alert_pending == 'pending':
alert = 'pending/%s' % self.summary_alert
elapsed_dt = (datetime.utcnow() - self.time)
else:
alert = self.summary_alert
elapsed_dt = (self.processing_time - self.time)
country = Country()
elapsed_minutes = int(elapsed_dt.days * MINS_PER_DAY +
elapsed_dt.seconds / SECS_PER_MIN)
max_dollars = 0
max_ccode = ''
for cresult in self._pagerdict['model_results']['empirical_economic'][
'country_dollars']:
if cresult['us_dollars'] >= max_dollars:
max_dollars = cresult['us_dollars']
max_ccode = cresult['country_code']
if max_ccode == 'UK':
cname = 'Unspecified'
else:
cdict = country.getCountry(max_ccode)
if cdict is not None:
cname = cdict['Name']
else:
cname = 'Unknown'
d = {
'EventID': self.id,
'Impacted Country ($)': cname,
'Version': self.version,
'EventTime': self.time,
'Lat': self.latitude,
'Lon': self.longitude,
'Depth': self.depth,
'Mag': self.magnitude,
'FatalityAlert': self.fatality_alert,
'EconomicAlert': self.economic_alert,
'SummaryAlert': alert,
'Elapsed': elapsed_minutes
}
if processtime:
d['ProcessTime'] = self.processing_time
sd = pd.Series(d)
return sd
def __init__(self,model_list,losstype='fatality'):
"""Instantiate EmpiricalLoss class.
:param model_list:
List of LognormalModel objects. The names of these will be used as keys for the getModel() method.
:param losstype:
One of 'fatality' or 'economic'.
:returns:
EmpiricalLoss instance.
"""
if losstype not in ['fatality','economic']:
raise PagerException('losstype must be one of ("fatality","economic").')
self._loss_type = losstype
self._model_dict = {}
for model in model_list:
self._model_dict[model.name] = model
self._country = Country() #object that can translate between different ISO country representations.
self._overrides = {} #dictionary of manually set rates (not necessarily lognormal)
def test():
homedir = os.path.dirname(os.path.abspath(__file__)) # where is this script?
countryfile = os.path.abspath(os.path.join(homedir, '..', '..', 'data', 'countries.csv'))
testdict = {'ISON': 840, 'ISO3': 'USA',
'ISO2': 'US', 'LongName': 'United States',
'Name': 'United States', 'Population': 324515000}
print('Test creating Country object from CSV file...')
country = Country()
print('Passed creating Country object from CSV file.')
print('Test retrieving dictionary from two letter code...')
row1 = country.getCountry('US')
assert row1 == testdict
print('Passed retrieving dictionary from two letter code...')
print('Test retrieving dictionary from three letter code...')
row2 = country.getCountry('USA')
assert row2 == testdict
print('Passed retrieving dictionary from three letter code...')
print('Test retrieving dictionary from name...')
row3 = country.getCountry('United States')
assert row3 == testdict
print('Passed retrieving dictionary from name.')
print('Test retrieving dictionary from numeric code...')
row4 = country.getCountry(840)
assert row4 == testdict
print('Passed retrieving dictionary from numeric code...')
print('Test to make sure failure is an option...')
faildict = {'Name': 'Unknown',
'LongName': 'Unknown',
'ISO2': 'UK',
'ISO3': 'UKN',
'ISON': 0,
'Population': 0}
row5 = country.getCountry('happyland')
assert row5 == faildict
print('Test failed as expected...')
# test multiple kinds of numbers...
print('Make sure all kinds of numpy numbers are supported...')
numbers = [840,840.0,
np.int16(840),np.uint16(840),
np.int32(840),np.uint32(840),
np.int64(840),np.uint64(840),
np.float32(840),np.float64(840)]
for number in numbers:
row_t = country.getCountry(number)
assert row_t == testdict
print('Passed numpy numbers test.')
def fromUNSpreadsheet(cls, excelfile, default_rate=DEFAULT_RATE):
"""Instantiate population growth rates from UN global spreadsheet.
http://esa.un.org/unpd/wpp/Download/Standard/Population/
:param excelfile:
Path to Excel file containing UN population growth rate data per country.
:param default_rate:
Value to be used for growth rate when input country codes are not found in ratedict.
:returns:
PopulationGrowth instance.
"""
re_year = '[0-9]*'
df = pd.read_excel(excelfile, header=16)
ratedict = {}
starts = []
ends = []
for col in df.columns:
matches = re.findall(re_year, col)
if len(matches) and len(matches[0]):
starts.append(int(matches[0]))
ends.append(int(matches[2]))
ccode_idx = df.columns.get_loc('Country code')
uscode = 840
usrates = None
country = Country()
for idx, row in df.iterrows():
key = row['Country code']
rates = row.iloc[ccode_idx + 1:].values / 100.0
if key == uscode:
usrates = rates.copy()
if country.getCountry(key) is None:
continue
ratedict[key] = {'start': starts[:], 'end': ends[:], 'rate': rates}
# we have three non-standard "country" codes for California, eastern US, and western US.
ratedict[902] = {'start': starts[:], 'end': ends[:], 'rate': usrates}
ratedict[903] = {'start': starts[:], 'end': ends[:], 'rate': usrates}
ratedict[904] = {'start': starts[:], 'end': ends[:], 'rate': usrates}
return cls(ratedict, default_rate=default_rate)
def __init__(self, popfile, popyear, isofile, popgrowth=None):
"""Create Exposure object, with population and country code grid files,
and a dictionary of country growth rates.
:param popfile:
Any GMT or ESRI style grid file supported by MapIO, containing population data.
:param popyear:
Integer indicating year when population data is valid.
:param isofile:
Any GMT or ESRI style grid file supported by MapIO, containing country code data (ISO 3166-1 numeric).
"""
self._popfile = popfile
self._popyear = popyear
self._isofile = isofile
self._popgrid = None
self._isogrid = None
self._shakegrid = None
if popgrowth is not None:
self._popgrowth = popgrowth
else:
self._popgrowth = PopulationGrowth.fromDefault()
self._country = Country()
def test():
homedir = os.path.dirname(
os.path.abspath(__file__)) #where is this script?
countryfile = os.path.abspath(
os.path.join(homedir, '..', '..', 'data', 'countries.csv'))
testdict = {
'ISON': 840,
'ISO3': 'USA',
'ISO2': 'US',
'LongName': 'United States',
'Name': 'United States',
'Population': 324515000
}
print('Test creating Country object from CSV file...')
country = Country()
print('Passed creating Country object from CSV file.')
print('Test retrieving dictionary from two letter code...')
row1 = country.getCountry('US')
assert row1 == testdict
print('Passed retrieving dictionary from two letter code...')
print('Test retrieving dictionary from three letter code...')
row2 = country.getCountry('USA')
assert row2 == testdict
print('Passed retrieving dictionary from three letter code...')
print('Test retrieving dictionary from name...')
row3 = country.getCountry('United States')
assert row3 == testdict
print('Passed retrieving dictionary from name.')
print('Test retrieving dictionary from numeric code...')
row4 = country.getCountry(840)
assert row4 == testdict
print('Passed retrieving dictionary from numeric code...')
print('Test to make sure failure is an option...')
faildict = {
'Name': 'Unknown',
'LongName': 'Unknown',
'ISO2': 'UK',
'ISO3': 'UKN',
'ISON': 0,
'Population': 0
}
row5 = country.getCountry('happyland')
assert row5 == faildict
print('Test failed as expected...')
def get_gdp_comment(ecodict, ecomodel, econexposure, event_year, epicode):
"""Create a comment on the GDP impact of a given event in the most impacted country.
:param ecodict:
Dictionary containing country code keys and integer population estimations of economic loss.
:param ecomodel:
Instance of the EmpiricalLoss class.
:param econexposure:
Dictionary containing country code (ISO2) keys, and values of
10 element arrays representing population exposure to MMI 1-10.
Dictionary will contain an additional key 'Total', with value of exposure across all countries.
:param event_year:
Year in which event occurred.
:param epicode:
Two letter country code of epicenter or 'UK' if not a country (usu. ocean).
:returns:
A string which indicates what fraction of the country's GDP the losses represent.
"""
# get the gdp comment
# get the G value for the economic losses
eco_gvalue = ecomodel.getCombinedG(ecodict)
# get the country code of the country with the highest losses
dccode = ''
dmax = 0
expected = ecodict['TotalDollars'] / 1e6
if ecodict['TotalDollars'] > 0:
for ccode, value in ecodict.items():
if ccode == 'TotalDollars':
continue
if value > dmax:
dmax = value
dccode = ccode
else:
# how do I compare economic exposure between countries?
# do I want to compare that, or just grab the country of epicenter?
for ccode, value in ecodict.items():
if ccode == 'TotalDollars':
continue
rates = ecomodel.getLossRates(ccode, np.arange(1, 10))
emploss = np.nansum(rates * value)
if emploss > dmax:
dmax = emploss
dccode = ccode
if dccode == '':
dccode = epicode
gdp_obj = GDP.fromDefault()
gdp, outccode = gdp_obj.getGDP(dccode, event_year)
country = Country()
print('ccode: %s, dccode: %s, outccode: %s' % (ccode, dccode, outccode))
cinfo = country.getCountry(outccode)
if cinfo != 'UK':
pop = cinfo['Population']
else:
pop = 0
T = (pop * gdp) / 1e6
if T == 0:
return ''
percent = erf(1 / np.sqrt(2))
plow = round(
np.exp(np.log(max(expected, EPS)) - eco_gvalue * invphi(percent)))
phigh = round(
np.exp(eco_gvalue * invphi(percent) + np.log(max(expected, EPS))))
if plow != 0:
ptlow = int(plow * 1e2 / T)
else:
ptlow = 0
if phigh != 0:
pthigh = int(phigh * 1e2 / T)
else:
pthigh = 0
if dccode in ['XF', 'EU', 'WU']:
cname = 'the United States'
else:
cname = cinfo['Name']
if pthigh < 1.0:
strtxt = 'Estimated economic losses are less than 1%% of GDP of %s.' % cname
else:
if ptlow < 100:
ptlow = set_num_precision(ptlow, 1)
else:
ptlow = set_num_precision(ptlow, 2)
if pthigh < 100:
pthigh = set_num_precision(pthigh, 1)
else:
pthigh = set_num_precision(pthigh, 2)
if pthigh >= 100:
strtxt = 'Estimated economic losses may exceed the GDP of %s.' % cname
else:
strtxt = 'Estimated economic losses are %i-%i%% GDP of %s.' % (
ptlow, pthigh, cname)
return strtxt
class SemiEmpiricalFatality(object):
def __init__(self, inventory, collapse, casualty, workforce, growth):
"""Create Semi-Empirical Fatality Model object.
:param inventory:
HDFContainer, containing DataFrames named:
- 'BuildingTypes',
- 'RuralNonResidential'
- 'RuralResidential'
- 'UrbanNonResidential'
- 'UrbanResidential'
where BuildingTypes is a Dataframe with columns:
- Code: Building Code ('A','S','C1',etc.)
- ShortDescription: A short description of building type (i.e, 'adobe block').
- OperationalDescription: A (sometimes) shorter description of building type.
- LongDescription: Full description of building type, i.e. 'High-Rise Ductile Reinforced Concrete Moment Frame'
All other Dataframes have columns:
- CountryCode: Two letter ISO country code ('US',etc.)
- CountryName: Name of country.
- Building Codes, as listed and described in BuildingTypes Dataframe.
:param collapse:
HDFContainer, where first index is country (by two-letter country code), columns are:
- BuildingCode: Building code as above.
- (IMT)_6.0 -> (IMT)_9.0 Columns with collapse rates at each (IMT) interval, where (IMT) could be MMI,PGA,PGV,PSA1.0, etc.
:param casualty:
HDFContainer, where first index is country (by two-letter country code), columns are:
- BuildingCode: Building code as above.
- CasualtyDay: Casualty rate, given collapse, during the day.
- CasualtyNight: Casualty rate, given collapse, during the night.
:param workforce:
HDFContainer consisting of a single dataframe, where rows are by country, and columns are:
- CountryCode two letter ISO country code.
- CountryCode name of country.
- WorkforceTotal Fraction of the country population in the workforce.
- WorkforceAgriculture Fraction of the total workforce employed in agriculture.
- WorkforceIndustrial Fraction of the total workforce employed in industry.
- WorkforceServices Fraction of the total workforce employed in services.
:param growth:
PopulationGrowth object.
"""
self._inventory = inventory
self._collapse = collapse
self._casualty = casualty
self._workforce = workforce
self._popgrowth = growth
self._country = Country()
@classmethod
def fromDefault(cls):
homedir = os.path.dirname(os.path.abspath(
__file__)) # where is this module?
inventory_file = os.path.join(
homedir, '..', 'data', 'semi_inventory.hdf')
collapse_file = os.path.join(
homedir, '..', 'data', 'semi_collapse_mmi.hdf')
casualty_file = os.path.join(
homedir, '..', 'data', 'semi_casualty.hdf')
workforce_file = os.path.join(
homedir, '..', 'data', 'semi_workforce.hdf')
return cls.fromFiles(inventory_file, collapse_file, casualty_file, workforce_file)
@classmethod
def fromFiles(cls, inventory_file, collapse_file, casualty_file, workforce_file):
"""Create SemiEmpiricalFatality object from a number of input files.
:param inventory_file:
HDF5 file containing Semi-Empirical building inventory data in an HDFContainer. (described in __init__).
:param collapse_file:
HDF5 file containing Semi-Empirical collapse rate data in an HDFContainer. (described in __init__).
:param casualty_file:
HDF5 file containing Semi-Empirical casualty rate data in an HDFContainer.(described in __init__).
:param workforce_file:
HDF5 file containing Semi-Empirical workforce data in an HDFContainer. (described in __init__).
:param growth_file:
Excel spreadsheet containing population growth rate data (described in PopulationGrowth.fromUNSpreadsheet()).
:returns:
SemiEmpiricalFatality object.
"""
# turn the inventory,collapse, and casualty spreadsheets into Panels...
inventory = HDFContainer.load(inventory_file)
collapse = HDFContainer.load(collapse_file)
casualty = HDFContainer.load(casualty_file)
workforce = HDFContainer.load(workforce_file)
# extract the one dataframe from the Panel
workforce = workforce.getDataFrame('Workforce')
workforce = workforce.set_index('CountryCode')
# read the growth spreadsheet into a PopulationGrowth object...
popgrowth = PopulationGrowth.fromDefault()
return cls(inventory, collapse, casualty, workforce, popgrowth)
def setGlobalFiles(self, popfile, popyear, urbanfile, isofile):
"""Set the global data files (population,urban/rural, country code) for use of model with ShakeMaps.
:param popfile:
File name of population grid.
:param popyear:
Year population data was collected.
:param urbanfile:
File name of urban/rural grid (rural cells indicated with a 1, urban cells with a 2).
:param isofile:
File name of numeric ISO country code grid.
:returns:
None
"""
self._popfile = popfile
self._popyear = popyear
self._urbanfile = urbanfile
self._isofile = isofile
def getBuildingDesc(self, btype, desctype='short'):
"""Get a building description given a short building type code.
:param btype:
Short building type code ('A' (adobe), 'C' (reinforced concrete), etc.)
:param desctype:
A string, one of:
- 'short': Very short descriptions ('adobe block')
- 'operational': Short description, intended for use in automatically generated sentences about building types.
- 'long': Most verbose description ('Adobe block (unbaked dried mud block) walls')
:returns:
Either a short, operational, or long description of building types.
"""
bsheet = self._inventory.getDataFrame('BuildingTypes')
bsheet = bsheet.set_index('Code')
row = bsheet.loc[btype]
if desctype == 'short':
return row['ShortDescription']
elif desctype == 'operational':
return row['OperationalDescription']
else:
return row['LongDescription']
return None
def getWorkforce(self, ccode):
"""Get the workforce data corresponding to a given country code.
:param ccode:
Two letter ISO country code.
:returns:
Pandas series containing Workforce data for given country
(WorkForceTotal,WorkForceAgriculture,WorkForceIndustrial,WorkForceServices)
"""
try:
wforce = self._workforce.loc[ccode]
except:
wforce = None
return wforce
def getCollapse(self, ccode, mmi, inventory):
"""Return the collapse rates for a given country,intensity, and inventory.
:param ccode:
Two letter ISO country code.
:param mmi:
MMI value (one of 6.0,6.5,7.0,7.5,8.0,8.5,9.0)
:param inventory:
Pandas Series containing an inventory for the given country.
:returns:
Pandas Series object containing the collapse rates for given building types, ccode, and MMI.
"""
collapse_frame = self._collapse.getDataFrame(ccode)
collapse_frame = collapse_frame.set_index('BuildingCode')
try:
idx = inventory.index.drop('Unnamed: 0')
collapse_frame = collapse_frame.loc[idx]
except Exception:
collapse_dict = inventory.to_dict()
collapse = pd.Series(collapse_dict)
for key, value in collapse_dict.items():
collapse[key] = np.nan
return collapse
mmicol = 'MMI_%s' % str(mmi)
collapse = collapse_frame[mmicol]
return collapse
def getFatalityRates(self, ccode, timeofday, inventory):
"""Return fatality rates for a given country, time of day, and inventory.
:param ccode:
Two-letter ISO country code.
:param timeofday:
One of 'day','transit', or 'night'.
:param inventory:
Pandas Series containing an inventory for the given country.
:returns:
Pandas Series object containing fatality rates for given country, time of day, and inventory.
"""
fatalframe = self._casualty.getDataFrame(ccode)
fatalframe = fatalframe.set_index('BuildingCode')
timecol = TIMES[timeofday]
if 'Unnamed: 0' in inventory.index:
idx = inventory.index.drop('Unnamed: 0')
else:
idx = inventory.index
fatrates = fatalframe.loc[idx][timecol]
return fatrates
def getInventories(self, ccode, density):
"""Return two pandas Series objects corresponding to the urban or rural inventory for given country.
:param ccode:
Two-letter ISO country code.
:param density:
One of semimodel.URBAN (2) or semimodel.RURAL (1).
:returns:
Two Pandas Series: 1) Residential Inventory and 2) Non-Residential Inventory.
"""
if density == URBAN:
resinv = self._inventory.getDataFrame('UrbanResidential')
nresinv = self._inventory.getDataFrame('UrbanNonResidential')
else:
resinv = self._inventory.getDataFrame('RuralResidential')
nresinv = self._inventory.getDataFrame('RuralNonResidential')
resinv = resinv.set_index('CountryCode')
nresinv = nresinv.set_index('CountryCode')
# we may be missing inventory for certain countries (Bonaire?). Return empty series.
if ccode not in resinv.index or ccode not in nresinv.index:
return (pd.Series(), pd.Series())
# pandas series of residential inventory
resrow = resinv.loc[ccode]
resrow = resrow.drop('CountryName')
# pandas series of non-residential inventory
nresrow = nresinv.loc[ccode]
nresrow = nresrow.drop('CountryName')
# now trim down the series to only include finite and non-zero values
resrow = resrow[resrow.notnull()]
resrow = resrow[resrow > 0]
nresrow = nresrow[nresrow.notnull()]
nresrow = nresrow[nresrow > 0]
return (resrow, nresrow)
def getLosses(self, shakefile):
"""Calculate number of fatalities using semi-empirical approach.
:param shakefile:
Path to a ShakeMap grid.xml file.
:returns:
Tuple of:
1) Total number of fatalities
2) Dictionary of residential fatalities per building type, per country.
3) Dictionary of non-residential fatalities per building type, per country.
"""
# get shakemap geodict
shakedict = ShakeGrid.getFileGeoDict(shakefile, adjust='res')
# get population geodict
popdict = get_file_geodict(self._popfile)
# get country code geodict
isodict = get_file_geodict(self._isofile)
# get urban grid geodict
urbdict = get_file_geodict(self._urbanfile)
# load all of the grids we need
if popdict == shakedict == isodict == urbdict:
# special case, probably for testing...
shakegrid = ShakeGrid.load(shakefile, adjust='res')
popgrid = read(self._popfile)
isogrid = read(self._isofile)
urbgrid = read(self._urbanfile)
else:
sampledict = popdict.getBoundsWithin(shakedict)
shakegrid = ShakeGrid.load(shakefile,
samplegeodict=sampledict,
resample=True,
method='linear',
adjust='res')
popgrid = read(self._popfile,
samplegeodict=sampledict,
resample=False)
isogrid = read(self._isofile,
samplegeodict=sampledict,
resample=True,
method='nearest',
doPadding=True,
padValue=0)
urbgrid = read(self._urbanfile,
samplegeodict=sampledict,
resample=True,
method='nearest',
doPadding=True,
padValue=RURAL)
# determine the local apparent time of day (based on longitude)
edict = shakegrid.getEventDict()
etime = edict['event_timestamp']
elon = edict['lon']
time_of_day, event_year, event_hour = get_time_of_day(etime, elon)
# round off our MMI data to nearest 0.5 (5.5 should stay 5.5, 5.4
# should become 5.5, 5.24 should become 5.0, etc.)
# TODO: Someday, make this more general to include perhaps grids of all IMT values, or
# at least the ones we have collapse data for.
mmidata = np.round(shakegrid.getLayer('mmi').getData() / 0.5) * 0.5
# get arrays from our other grids
popdata = popgrid.getData()
isodata = isogrid.getData()
urbdata = urbgrid.getData()
# modify the population values for growth rate by country
ucodes = np.unique(isodata[~np.isnan(isodata)])
for ccode in ucodes:
cidx = (isodata == ccode)
popdata[cidx] = self._popgrowth.adjustPopulation(
popdata[cidx], ccode, self._popyear, event_year)
# create a dictionary containing indoor populations by building type (in cells where MMI >= 6)
#popbystruct = get_indoor_pop(mmidata,popdata,urbdata,isodata,time_of_day)
# find all mmi values greater than 9, set them to 9
mmidata[mmidata > 9.0] = 9.0
# dictionary containers for sums of fatalities (res/nonres) by building type
res_fatal_by_ccode = {}
nonres_fatal_by_ccode = {}
# fatality sum
ntotal = 0
# loop over countries
ucodes = np.unique(isodata[~np.isnan(isodata)])
for ucode in ucodes:
if ucode == 0:
continue
res_fatal_by_btype = {}
nonres_fatal_by_btype = {}
cdict = self._country.getCountry(int(ucode))
ccode = cdict['ISO2']
# get the workforce Series data for the current country
wforce = self.getWorkforce(ccode)
if wforce is None:
logging.info('No workforce data for %s. Skipping.' %
(cdict['Name']))
continue
# loop over MMI values 6-9
for mmi in np.arange(6, 9.5, 0.5):
c1 = (mmidata == mmi)
c2 = (isodata == ucode)
if ucode > 900 and ucode != CALIFORNIA_US_CCODE:
ucode = US_CCODE
for dclass in [URBAN, RURAL]:
c3 = (urbdata == dclass)
# get the population data in those cells at MMI, in country, and density class
# I think I want an AND condition here
popcells = popdata[c1 & c2 & c3]
# get the population distribution across residential, non-residential, and outdoor.
res, nonres, outside = pop_dist(
popcells, wforce, time_of_day, dclass)
# get the inventory for urban residential
resrow, nresrow = self.getInventories(ccode, dclass)
# TODO - figure out why this is happening, make the following lines
# not necessary
if 'Unnamed: 0' in resrow:
resrow = resrow.drop('Unnamed: 0')
if 'Unnamed: 0' in nresrow:
nresrow = nresrow.drop('Unnamed: 0')
# now multiply the residential/non-residential population through the inventory data
numres = len(resrow)
numnonres = len(nresrow)
resmat = np.reshape(
resrow.values, (numres, 1)).astype(np.float32)
nresmat = np.reshape(
nresrow.values, (numnonres, 1)).astype(np.float32)
popres = np.tile(res, (numres, 1))
popnonres = np.tile(nonres, (numnonres, 1))
popresbuilding = (popres * resmat)
popnonresbuilding = (popnonres * nresmat)
# now we have the residential and non-residental population
# distributed through the building types for each cell that matches
# MMI,country, and density criteria.
# popresbuilding rows are building types, columns are population cells
# next, we get the collapse rates for these buildings
# and multiply them by the population by building.
collapse_res = self.getCollapse(ccode, mmi, resrow)
collapse_nonres = self.getCollapse(ccode, mmi, nresrow)
resrates = np.reshape(
collapse_res.values.astype(np.float32), (numres, 1))
nonresrates = np.reshape(
collapse_nonres.values.astype(np.float32), (numnonres, 1))
rescollapse = popresbuilding * resrates
nonrescollapse = popnonresbuilding * nonresrates
# get the fatality rates given collapse by building type and
# multiply through the result of collapse*population per building
resfatalcol = self.getFatalityRates(
ccode, time_of_day, resrow)
nonresfatalcol = self.getFatalityRates(
ccode, time_of_day, nresrow)
resfatal = np.reshape(
resfatalcol.values.astype(np.float32), (numres, 1))
nonresfatal = np.reshape(
nonresfatalcol.values.astype(np.float32), (numnonres, 1))
resfat = rescollapse * resfatal
nonresfat = nonrescollapse * nonresfatal
# zero out the cells where fatalities are less than 1 or nan
try:
if len(resfat) and len(resfat[0]):
resfat[np.ma.masked_less(resfat, 1).mask] = 0.0
except:
resfat[np.isnan(resfat)] = 0.0
try:
if len(nonresfat) and len(nonresfat[0]):
nonresfat[np.ma.masked_less(
nonresfat, 1).mask] = 0.0
except:
nonresfat[np.isnan(nonresfat)] = 0.0
# sum the fatalities per building through all cells
resfatbybuilding = np.nansum(resfat, axis=1)
nonresfatbybuilding = np.nansum(nonresfat, axis=1)
resfdict = dict(
zip(resrow.index, resfatbybuilding.tolist()))
nonresfdict = dict(
zip(nresrow.index, nonresfatbybuilding.tolist()))
res_fatal_by_btype = add_dicts(
res_fatal_by_btype, resfdict)
nonres_fatal_by_btype = add_dicts(
nonres_fatal_by_btype, nonresfdict)
# add the fatalities by building type to the dictionary containing fatalities by country
res_fatal_by_ccode[ccode] = res_fatal_by_btype.copy()
nonres_fatal_by_ccode[ccode] = nonres_fatal_by_btype.copy()
# increment the total number of fatalities
ntotal += int(sum(res_fatal_by_btype.values())
+ sum(nonres_fatal_by_btype.values()))
return (ntotal, res_fatal_by_ccode, nonres_fatal_by_ccode)
def main(pargs, config):
# get the users home directory
homedir = os.path.expanduser("~")
# handle cancel messages
if pargs.cancel:
# we presume that pargs.gridfile in this context is an event ID.
msg = _cancel(pargs.gridfile, config)
print(msg)
return True
# what kind of thing is gridfile?
is_file = os.path.isfile(pargs.gridfile)
is_url, url_gridfile = _is_url(pargs.gridfile)
is_pdl, pdl_gridfile = _check_pdl(pargs.gridfile, config)
if is_file:
gridfile = pargs.gridfile
elif is_url:
gridfile = url_gridfile
elif is_pdl:
gridfile = pdl_gridfile
else:
print("ShakeMap Grid file %s does not exist." % pargs.gridfile)
return False
pager_folder = os.path.join(homedir, config["output_folder"])
pager_archive = os.path.join(homedir, config["archive_folder"])
admin = PagerAdmin(pager_folder, pager_archive)
# stdout will now be logged as INFO, stderr will be logged as WARNING
mail_host = config["mail_hosts"][0]
mail_from = config["mail_from"]
developers = config["developers"]
logfile = os.path.join(pager_folder, "pager.log")
plog = PagerLogger(logfile, developers, mail_from, mail_host, debug=pargs.debug)
logger = plog.getLogger()
try:
eid = None
pager_version = None
# get all the basic event information and print it, if requested
shake_tuple = getHeaderData(gridfile)
eid = shake_tuple[1]["event_id"].lower()
etime = shake_tuple[1]["event_timestamp"]
if not len(eid):
eid = shake_tuple[0]["event_id"].lower()
network = shake_tuple[1]["event_network"].lower()
if network == "":
network = "us"
if not eid.startswith(network):
eid = network + eid
# Create a ComcatInfo object to hopefully tell us a number of things about this event
try:
ccinfo = ComCatInfo(eid)
location = ccinfo.getLocation()
tsunami = ccinfo.getTsunami()
authid, allids = ccinfo.getAssociatedIds()
authsource, othersources = ccinfo.getAssociatedSources()
except: # fail over to what we can determine locally
location = shake_tuple[1]["event_description"]
tsunami = shake_tuple[1]["magnitude"] >= TSUNAMI_MAG_THRESH
authid = eid
authsource = network
allids = []
# location field can be empty (None), which breaks a bunch of things
if location is None:
location = ""
# Check to see if user wanted to override default tsunami criteria
if pargs.tsunami != "auto":
if pargs.tsunami == "on":
tsunami = True
else:
tsunami = False
# check to see if this event is a scenario
is_scenario = False
shakemap_type = shake_tuple[0]["shakemap_event_type"]
if shakemap_type == "SCENARIO":
is_scenario = True
# if event is NOT a scenario and event time is in the future,
# flag the event as a scenario and yell about it.
if etime > datetime.datetime.utcnow():
is_scenario = True
logger.warning(
"Event origin time is in the future! Flagging this as a scenario."
)
if is_scenario:
if re.search("scenario", location.lower()) is None:
location = "Scenario " + location
# create the event directory (if it does not exist), and start logging there
logger.info("Creating event directory")
event_folder = admin.createEventFolder(authid, etime)
# Stop processing if there is a "stop" file in the event folder
stopfile = os.path.join(event_folder, "stop")
if os.path.isfile(stopfile):
fmt = '"stop" file found in %s. Stopping processing, returning with 1.'
logger.info(fmt % (event_folder))
return True
pager_version = get_pager_version(event_folder)
version_folder = os.path.join(event_folder, "version.%03d" % pager_version)
os.makedirs(version_folder)
event_logfile = os.path.join(version_folder, "event.log")
# this will turn off the global rotating log file
# and switch to the one in the version folder.
plog.setVersionHandler(event_logfile)
# Copy the grid.xml file to the version folder
# sometimes (usu when testing) the input grid isn't called grid.xml. Rename it here.
version_grid = os.path.join(version_folder, "grid.xml")
shutil.copyfile(gridfile, version_grid)
# Check to see if the tsunami flag has been previously set
tsunami_toggle = {"on": 1, "off": 0}
tsunami_file = os.path.join(event_folder, "tsunami")
if os.path.isfile(tsunami_file):
tsunami = tsunami_toggle[open(tsunami_file, "rt").read().strip()]
# get the rest of the event info
etime = shake_tuple[1]["event_timestamp"]
elat = shake_tuple[1]["lat"]
elon = shake_tuple[1]["lon"]
emag = shake_tuple[1]["magnitude"]
# get the year of the event
event_year = shake_tuple[1]["event_timestamp"].year
# find the population data collected most closely to the event_year
pop_year, popfile = _get_pop_year(
event_year, config["model_data"]["population_data"]
)
logger.info("Population year: %i Population file: %s\n" % (pop_year, popfile))
# Get exposure results
logger.info("Calculating population exposure.")
isofile = config["model_data"]["country_grid"]
expomodel = Exposure(popfile, pop_year, isofile)
exposure = None
exposure = expomodel.calcExposure(gridfile)
# incidentally grab the country code of the epicenter
numcode = expomodel._isogrid.getValue(elat, elon)
if np.isnan(numcode):
cdict = None
else:
cdict = Country().getCountry(int(numcode))
if cdict is None:
ccode = "UK"
else:
ccode = cdict["ISO2"]
logger.info("Country code at epicenter is %s" % ccode)
# get fatality results, if requested
logger.info("Calculating empirical fatalities.")
fatmodel = EmpiricalLoss.fromDefaultFatality()
fatdict = fatmodel.getLosses(exposure)
# get economic results, if requested
logger.info("Calculating economic exposure.")
econexpmodel = EconExposure(popfile, pop_year, isofile)
ecomodel = EmpiricalLoss.fromDefaultEconomic()
econexposure = econexpmodel.calcExposure(gridfile)
ecodict = ecomodel.getLosses(econexposure)
shakegrid = econexpmodel.getShakeGrid()
# Get semi-empirical losses
logger.info("Calculating semi-empirical fatalities.")
urbanfile = config["model_data"]["urban_rural_grid"]
if not os.path.isfile(urbanfile):
raise PagerException("Urban-rural grid file %s does not exist." % urbanfile)
semi = SemiEmpiricalFatality.fromDefault()
semi.setGlobalFiles(popfile, pop_year, urbanfile, isofile)
semiloss, resfat, nonresfat = semi.getLosses(gridfile)
# get all of the other components of PAGER
logger.info("Getting all comments.")
# get the fatality and economic comments
impact1, impact2 = get_impact_comments(
fatdict, ecodict, econexposure, event_year, ccode
)
# get comment describing vulnerable structures in the region.
struct_comment = get_structure_comment(resfat, nonresfat, semi)
# get the comment describing historic secondary hazards
secondary_comment = get_secondary_comment(elat, elon, emag)
# get the comment describing historical comments in the region
historical_comment = get_historical_comment(elat, elon, emag, exposure, fatdict)
# generate the probability plots
logger.info("Drawing probability plots.")
fat_probs_file, eco_probs_file = _draw_probs(
fatmodel, fatdict, ecomodel, ecodict, version_folder
)
# generate the exposure map
exposure_base = os.path.join(version_folder, "exposure")
logger.info("Generating exposure map...")
oceanfile = config["model_data"]["ocean_vectors"]
oceangrid = config["model_data"]["ocean_grid"]
cityfile = config["model_data"]["city_file"]
borderfile = config["model_data"]["border_vectors"]
shake_grid = expomodel.getShakeGrid()
pop_grid = expomodel.getPopulationGrid()
pdf_file, png_file, mapcities = draw_contour(
shake_grid,
pop_grid,
oceanfile,
oceangrid,
cityfile,
exposure_base,
borderfile,
is_scenario=is_scenario,
)
logger.info("Generated exposure map %s" % pdf_file)
# figure out whether this event has been "released".
is_released = _get_release_status(
pargs,
config,
fatmodel,
fatdict,
ecomodel,
ecodict,
shake_tuple,
event_folder,
)
# Create a data object to encapsulate everything we know about the PAGER
# results, and then serialize that to disk in the form of a number of JSON files.
logger.info("Making PAGER Data object.")
doc = PagerData()
timezone_file = config["model_data"]["timezones_file"]
elapsed = pargs.elapsed
doc.setInputs(
shakegrid,
timezone_file,
pager_version,
shakegrid.getEventDict()["event_id"],
authid,
tsunami,
location,
is_released,
elapsed=elapsed,
)
logger.info("Setting inputs.")
doc.setExposure(exposure, econexposure)
logger.info("Setting exposure.")
doc.setModelResults(
fatmodel, ecomodel, fatdict, ecodict, semiloss, resfat, nonresfat
)
logger.info("Setting comments.")
doc.setComments(
impact1, impact2, struct_comment, historical_comment, secondary_comment
)
logger.info("Setting map info.")
doc.setMapInfo(cityfile, mapcities)
logger.info("Validating.")
doc.validate()
# if we have determined that the event is a scenario (origin time is in the future)
# and the shakemap is not flagged as such, set the shakemap type in the
# pagerdata object to be 'SCENARIO'.
if is_scenario:
doc.setToScenario()
json_folder = os.path.join(version_folder, "json")
os.makedirs(json_folder)
logger.info("Saving output to JSON.")
doc.saveToJSON(json_folder)
logger.info("Saving output to XML.")
doc.saveToLegacyXML(version_folder)
logger.info("Creating onePAGER pdf...")
onepager_pdf, error = create_onepager(doc, version_folder)
if onepager_pdf is None:
raise PagerException("Could not create onePAGER output: \n%s" % error)
# copy the contents.xml file to the version folder
contentsfile = get_data_path("contents.xml")
if contentsfile is None:
raise PagerException("Could not find contents.xml file.")
shutil.copy(contentsfile, version_folder)
# send pdf as attachment to internal team of PAGER users
if not is_released and not is_scenario:
message_pager(config, onepager_pdf, doc)
# run transfer, as appropriate and as specified by config
# the PAGER product eventsource and eventsourcecode should
# match the input ShakeMap settings for these properties.
# This can possibly cause confusion if a regional ShakeMap is
# trumped with one from NEIC, but this should happen less often
# than an NEIC origin being made authoritative over a regional one.
eventsource = network
eventsourcecode = eid
res, msg = transfer(
config,
doc,
eventsourcecode,
eventsource,
version_folder,
is_scenario=is_scenario,
)
logger.info(msg)
if not res:
logger.critical('Error transferring PAGER content. "%s"' % msg)
print("Created onePAGER pdf %s" % onepager_pdf)
logger.info("Created onePAGER pdf %s" % onepager_pdf)
logger.info("Done.")
return True
except Exception as e:
f = io.StringIO()
traceback.print_exc(file=f)
msg = e
msg = "%s\n %s" % (str(msg), f.getvalue())
hostname = socket.gethostname()
msg = msg + "\n" + "Error occurred on %s\n" % (hostname)
if gridfile is not None:
msg = msg + "\n" + "Error on file: %s\n" % (gridfile)
if eid is not None:
msg = msg + "\n" + "Error on event: %s\n" % (eid)
if pager_version is not None:
msg = msg + "\n" + "Error on version: %i\n" % (pager_version)
f.close()
logger.critical(msg)
logger.info("Sent error to email")
return False
class EmpiricalLoss(object):
"""Container class for multiple LognormalModel objects.
"""
def __init__(self, model_list, losstype='fatality'):
"""Instantiate EmpiricalLoss class.
:param model_list:
List of LognormalModel objects. The names of these will be used as keys for the getModel() method.
:param losstype:
One of 'fatality' or 'economic'.
:returns:
EmpiricalLoss instance.
"""
if losstype not in ['fatality', 'economic']:
raise PagerException(
'losstype must be one of ("fatality","economic").')
self._loss_type = losstype
self._model_dict = {}
for model in model_list:
self._model_dict[model.name] = model
self._country = Country(
) #object that can translate between different ISO country representations.
self._overrides = {
} #dictionary of manually set rates (not necessarily lognormal)
def getModel(self, ccode):
"""Return the LognormalModel associated with given country code,
or a default model if country code not found.
:param ccode:
Usually two letter ISO country code.
:returns:
LognormalModel instance containing model for input country code, or a default model.
"""
ccode = ccode.upper()
default = LognormalModel('default',
DEFAULT_THETA,
DEFAULT_BETA,
DEFAULT_L2G,
alpha=DEFAULT_ALPHA)
if ccode in self._model_dict:
return self._model_dict[ccode]
else:
return default
@classmethod
def fromDefaultFatality(cls):
homedir = os.path.dirname(
os.path.abspath(__file__)) #where is this module?
fatxml = os.path.join(homedir, '..', 'data', 'fatality.xml')
return cls.fromXML(fatxml)
@classmethod
def fromDefaultEconomic(cls):
homedir = os.path.dirname(
os.path.abspath(__file__)) #where is this module?
econxml = os.path.join(homedir, '..', 'data', 'economy.xml')
return cls.fromXML(econxml)
@classmethod
def fromXML(cls, xmlfile):
"""Load country-specific models from an XML file of the form:
<?xml version="1.0" encoding="US-ASCII" standalone="yes"?>
<models vstr="2.2" type="fatality">
<model ccode="AF" theta="11.613073" beta="0.180683" gnormvalue="1.0"/>
</models>
or
<?xml version="1.0" encoding="US-ASCII" standalone="yes"?>
<models vstr="1.3" type="economic">
<model alpha="15.065400" beta="0.100000" gnormvalue="4.113200" ccode="AF"/>
</models>
:param xmlfile:
XML file containing model parameters (see above).
:returns:
EmpiricalLoss instance.
"""
root = minidom.parse(xmlfile)
rootmodels = root.getElementsByTagName('models')[0]
models = rootmodels.getElementsByTagName('model')
losstype = rootmodels.getAttribute('type')
model_list = []
for model in models:
key = model.getAttribute('ccode')
theta = float(model.getAttribute('theta'))
beta = float(model.getAttribute('beta'))
l2g = float(model.getAttribute('gnormvalue'))
if model.hasAttribute('alpha'):
alpha = float(model.getAttribute('alpha'))
else:
alpha = 1.0 #what is the appropriate default value for this?
model_list.append(
LognormalModel(key, theta, beta, l2g, alpha=alpha))
root.unlink()
return cls(model_list, losstype)
def getLossRates(self, ccode, mmirange):
"""Return loss rates for given country country code model at input MMI values.
:param ccode:
Country code (usually two letter ISO code).
:param mmirange:
Array-like range of MMI values at which loss rates will be calculated.
:returns:
Rates from LognormalModel associated with ccode, or default model (see getModel()).
"""
#mmirange is mmi value, not index
model = self.getModel(ccode)
yy = model.getLossRates(mmirange)
return yy
def getLosses(self, exposure_dict):
"""Given an input dictionary of ccode (usually ISO numeric), calculate losses per country and total losses.
:param exposure_dict:
Dictionary containing country code keys, and 10 element arrays representing population
exposures to shaking from MMI values 1-10. If loss type is economic, then this
input represents exposure *x* per capita GDP *x* alpha (a correction factor).
:returns:
Dictionary containing country code keys and integer population estimations of loss.
"""
#Get a loss dictionary
fatdict = {}
for ccode, exparray in exposure_dict.items():
#exposure array will now also have a row of Total Exposure to shaking.
if ccode.find('Total') > -1 or ccode.find('maximum') > -1:
continue
if ccode == 'UK': #unknown
continue
mmirange = np.arange(5, 10)
model = self.getModel(ccode)
if ccode in self._overrides:
rates = self.getOverrideModel(ccode)[4:9]
else:
rates = None
expo = exparray[:]
expo[8] += expo[9]
expo = expo[4:9] #should now be the same size as rates array
losses = model.getLosses(expo, mmirange, rates=rates)
fatdict[ccode] = int(losses) #TODO: int or round, or neither?
#now go through the whole list, and get the total number of losses.
total = sum(list(fatdict.values()))
if self._loss_type == 'fatality':
fatdict['TotalFatalities'] = total
else:
fatdict['TotalDollars'] = total
return fatdict
def getCombinedG(self, lossdict):
"""Get combined L2G statistic for all countries contributing to losses.
:param lossdict:
Dictionary (as retued by getLosses() method, containing keys of ISO2 country codes,
and values of loss (fatalities or dollars) for that country.
:returns:
sqrt(sum(l2g^2)) for array of all l2g values from countries that had non-zero losses.
"""
#combine g norm values from all countries that contributed to losses, or if NO losses in
#any countries, then the combined G from all of them.
g = []
has_loss = np.sum(list(lossdict.values())) > 0
for ccode, loss in lossdict.items():
if has_loss:
if loss > 0:
g.append(self.getModel(ccode).l2g)
else:
g.append(self.getModel(ccode).l2g)
g = np.array(g)
zetf = np.sqrt(np.sum(np.power(g, 2)))
if zetf > 2.5:
zetf = 2.5
return zetf
def getProbabilities(self, lossdict, G):
"""Calculate probabilities over the standard PAGER loss ranges.
:param lossdict:
Dictionary (as retued by getLosses() method, containing keys of ISO2 country codes,
and values of loss (fatalities or dollars) for that country.
:param G:
Combined G value (see getCombinedG() method).
:returns:
Ordered Dictionary of probability of losses over ranges :
- '0-1' (green alert)
- '1-10' (yellow alert)
- '10-100' (yellow alert)
- '100-1000' (orange alert)
- '1000-10000' (red alert)
- '10000-100000' (red alert)
- '100000-10000000' (red alert)
"""
ranges = OrderedDict([('0-1', 0.0), ('1-10', 0.0), ('10-100', 0.0),
('100-1000', 0.0), ('1000-10000', 0.0),
('10000-100000', 0.0), ('100000-10000000', 0.0)])
expected = np.sum(list(lossdict.values()))
if self._loss_type == 'economic':
expected = expected / 1e6 #turn USD into millions of USD
for rangekey, value in ranges.items():
rparts = rangekey.split('-')
rmin = int(rparts[0])
if len(rparts) == 1:
rmax = int(rparts[0])
else:
rmax = int(rparts[1])
#the high end of the highest red range should be a very large number (ideally infinity).
#one trillion should do it.
if rmax == 10000000:
rmax = 1e12
prob = calcEmpiricalProbFromRange(G, expected, (rmin, rmax))
ranges[rangekey] = prob
return ranges
def getAlertLevel(self, lossdict):
"""Get the alert level associated with the input losses.
:param lossdict:
Loss results dictionary as returned by getLosses() method.
:returns:
String alert level, one of ('green','yellow','orange','red').
"""
levels = [(1, 'green'), (100, 'yellow'), (1000, 'orange'),
(1e12, 'red')]
if 'TotalFatalities' in lossdict:
total = lossdict['TotalFatalities']
else:
total = lossdict['TotalDollars'] / 1e6
for i in range(0, len(levels) - 1):
lossmax, thislevel = levels[i]
if total < lossmax:
return thislevel
return 'red' #we should never get here, unless we have 1e18 USD in losses!
def overrideModel(self, ccode, rates):
"""Override the rates determined from theta,beta values with these hard-coded ones.
Once set on the instance object, these will be the preferred rates.
NB: While probably most useful for testing, this method may have real-world uses, so we are
exposing it in the interface.
:param ccode:
(Usually) two-letter ISO country code.
:param rates:
10 element (MMI 1-10) array of loss rates which will be used to calculate losses.
"""
self._overrides[ccode] = rates
def getOverrideModel(self, ccode):
"""Get the override rates for the input country code. If not set, None will be returned.
:param ccode:
ISO 2 letter country code.
:returns:
10 element (MMI 1-10) array of loss rates used to calculate losses for input country code,
or None.
"""
if ccode in self._overrides:
return self._overrides[ccode]
else:
return None
def clearOverrides(self):
"""Clear out any models that have been set manually using overrideModel().
"""
self._overrides.clear()
def getLossGrid(self, mmidata, popdata, isodata):
"""Calculate floating point losses on a grid.
:param mmidata:
Array of MMI values, dimensions (M,N).
:param popdata:
Array of population values, dimensions (M,N).
:param isodata:
Array of numeric country code values, dimensions (M,N).
:returns:
Grid of floating point loss values, dimensions (M,N).
"""
ucodes = np.unique(isodata)
fatgrid = np.zeros_like(mmidata)
#we treat MMI 10 as MMI 9 for modeling purposes...
mmidata[mmidata > 9.5] = 9.0
for isocode in ucodes:
countrydict = self._country.getCountry(int(isocode))
if countrydict is None:
ccode = 'unknown'
else:
ccode = countrydict['ISO2']
if ccode not in self._overrides:
rates = self.getLossRates(ccode, np.arange(5, 10))
else:
rates = self.getOverrideModel(ccode)[4:9]
tcidx = np.where(isodata == isocode)
cidx = np.ravel_multi_index(tcidx, isodata.shape)
for i in range(0, len(rates)):
mmi = i + 5
mmi_lower = mmi - 0.5
mmi_upper = mmi + 0.5
midx = np.ravel_multi_index(
np.where((mmidata >= mmi_lower) & (mmidata < mmi_upper)),
mmidata.shape)
idx = np.intersect1d(cidx, midx)
idx2d = np.unravel_index(idx, mmidata.shape)
fatgrid[idx2d] = popdata[idx2d] * rates[i]
return fatgrid
def getLossByShapes(self,
mmidata,
popdata,
isodata,
shapes,
geodict,
eventyear=None,
gdpobj=None):
"""Divide the losses calculated per grid cell into polygons that intersect with the grid.
:param mmidata:
Array of MMI values, dimensions (M,N).
:param popdata:
Array of population values, dimensions (M,N).
:param isodata:
Array of numeric country code values, dimensions (M,N).
:param shapes:
Sequence of GeoJSON-like polygons as returned from fiona.open().
:param eventyear:
4 digit event year, must be not None if loss type is economic.
:param gdpobj:
GDP object, containing per capita GDP data from all countries.
Must not be None if calculating economic losses.
:returns:
Tuple of:
1) modified sequence of polygons, including a new field "fatalities" or "dollars_lost".
2) Total number of losses in all polygons.
"""
lossgrid = self.getLossGrid(mmidata, popdata, isodata)
polyshapes = []
totloss = 0
if self._loss_type == 'fatality':
fieldname = 'fatalities'
else:
fieldname = 'dollars_lost'
for polyrec in shapes:
polygon = shapely.geometry.shape(polyrec['geometry'])
#overlay the polygon on top of a grid, turn polygon pixels to 1, non-polygon pixels to 0.
tgrid = Grid2D.rasterizeFromGeometry([polygon],
geodict,
fillValue=0,
burnValue=1.0,
attribute='value',
mustContainCenter=True)
#get the indices of the polygon cells
shapeidx = tgrid.getData() == 1.0
#get the sum of those cells in the loss grid
losses = np.nansum(lossgrid[shapeidx])
polyrec['properties'][fieldname] = int(losses)
polyshapes.append(polyrec)
totloss += int(losses)
return (polyshapes, totloss)
def get_quake_desc(event, lat, lon, isMainEvent):
ndeaths = event["TotalDeaths"]
# summarize the exposure values
exposures = np.array([
event["MMI1"],
event["MMI2"],
event["MMI3"],
event["MMI4"],
event["MMI5"],
event["MMI6"],
event["MMI7"],
event["MMI8"],
event["MMI9+"],
])
exposures = np.array([round_to_nearest(exp, 1000) for exp in exposures])
# get the highest two exposures greater than zero
iexp = np.where(exposures > 0)[0][::-1]
romans = [
"I", "II", "III", "IV", "V", "VI", "VII", "VIII", "IX or greater"
]
if len(iexp) >= 2:
exposures = [exposures[iexp[1]], exposures[iexp[0]]]
ilevels = [romans[iexp[1]], romans[iexp[0]]]
expfmt = ", with estimated population exposures of %s at intensity"
expfmt = expfmt + " %s and %s at intensity %s"
exptxt = expfmt % (
commify(int(exposures[0])),
ilevels[0],
commify(int(exposures[1])),
ilevels[1],
)
else:
exptxt = ""
# create string describing this most impactful event
dfmt = "A magnitude %.1f earthquake %i km %s of this event struck %s on %s (UTC)%s"
mag = event["Magnitude"]
etime = pd.Timestamp(event["Time"])
etime = etime.strftime("%B %d, %Y")
etime = re.sub(" 0", " ", etime)
country = Country()
if pd.isnull(event["Name"]):
# hack for persistent error in expocat
if event["CountryCode"] == "UM" and event["Lat"] > 40:
cdict = country.getCountry("US")
else:
cdict = country.getCountry(event["CountryCode"])
if cdict:
cname = cdict["Name"]
else:
cname = "in the open ocean"
else:
cname = event["Name"].replace('"', "")
cdist = round(geodetic_distance(event["Lat"], event["Lon"], lat, lon))
cdir = get_compass_dir(lat, lon, event["Lat"], event["Lon"],
format="long").lower()
if ndeaths and str(ndeaths) != "nan":
dfmt = dfmt + ", resulting in a reported %s %s."
if ndeaths > 1:
dstr = "fatalities"
else:
dstr = "fatality"
ndeathstr = commify(int(ndeaths))
eqdesc = dfmt % (mag, cdist, cdir, cname, etime, exptxt, ndeathstr,
dstr)
else:
dfmt = dfmt + ", with no reported fatalities."
eqdesc = dfmt % (mag, cdist, cdir, cname, etime, exptxt)
return eqdesc
class GDP(object):
def __init__(self, dataframe):
"""Create an instance of a GDP object with a dataframe of countries/GDP values over time.
:param dataframe:
Pandas dataframe where rows are countries, and columns are rates for different years.
dataframe should look like this:
DataFrame({'Country Code':['AFG'],
'1960':[59.78768071],
'1961':[59.89003694],})
The above contains the GDP values for Afghanistan for 1960 and 196
"""
self._dataframe = dataframe
self._country = Country()
@classmethod
def fromDefault(cls):
homedir = os.path.dirname(
os.path.abspath(__file__)) #where is this module?
excelfile = os.path.join(homedir, '..', 'data',
'API_NY.GDP.PCAP.CD_DS2_en_excel_v2.xls')
return cls.fromWorldBank(excelfile)
@classmethod
def fromWorldBank(cls, excelfile):
"""Read in Excel data from the World Bank containing per capita GDP values for all countries in the world.
Taken from: http://data.worldbank.org/indicator/NY.GDP.PCAP.CD
:param excelfile:
Excel spreadsheet downloaded from above source.
:returns:
GDP instance.
"""
df = pd.read_excel(excelfile, sheetname='Data', header=3)
return cls(df)
def getGDP(self, ccode, year):
"""Get the GDP value for a given country code and a particular year.
:param ccode:
Any of ISO2, ISO3, or ISON country codes.
:param year:
Year of desired GDP value. If this year is before the earliest year in the data source,
the earliest GDP value will be used. If this year is after the latest year in the data source,
the latest non-NaN GDP value will be used.
:returns:
Tuple of:
- GDP value corresponding to country code and year, unless country code is not found, in which case
a default global GDP value will be returned.
- The country code which is most applicable to the output GDP. For example, if the GDP value chosen
is the global value, then this country code will be None. If the input country code is XF (California),
then the output country code will be 'US'.
"""
#first make sure the ccode is valid...
countrydict = self._country.getCountry(ccode)
if countrydict is None:
return (GLOBAL_GDP, None)
if countrydict['ISO2'] in ['XF', 'EU', 'WU']:
ccode = 'USA'
outccode = 'US'
else:
ccode = countrydict['ISO3']
outccode = ccode
yearstr = str(year)
row = self._dataframe[self._dataframe['Country Code'] == ccode].iloc[0]
if yearstr in row:
gdp = row[yearstr]
else:
columns = row.index.tolist()
years = []
for c in columns:
res = re.search('[0-9]{4}', c)
if res is not None:
years.append(res.group())
if yearstr < min(years):
#assume that the years in the dataframe are sequential and increasing to the right
#get the first non-null GDP value
gdp = row[row.notnull()][0]
else:
#get the last non-null GDP value
gdp = row[row.notnull()][-1]
return (gdp, outccode)
def make_test_semi_model(ccode, timeofday, density, popvalue, mmi):
"""Run the semi-empirical model for a single value of input. Intended for testing purposes.
:param ccode:
Two letter ISO country code ('US', 'JP', etc.) to be used to extract inventory, collapse rates, etc.
:param timeofday:
One of 'day','night' - used to determine residential/non-residental population distribution and casualty rates.
:param density:
One of semimodel.URBAN (2) or semimodel.RURAL (1).
:param popvalue:
Scalar population value to multiply by inventory, collapse, and fatality rates.
:param mmi:
MMI value used to extract collapse rates in given country code.
:returns:
Tuple of:
1) Total number of fatalities
2) Dictionary of residential fatalities per building type, per country.
3) Dictionary of non-residential fatalities per building type, per country.
"""
country = Country()
cdict = country.getCountry(ccode)
ucode = cdict['ISON']
geodict = GeoDict({
'xmin': 0.5,
'xmax': 4.5,
'ymin': 0.5,
'ymax': 4.5,
'dx': 1.0,
'dy': 1.0,
'nx': 5,
'ny': 5
})
if timeofday == 'day':
etime = datetime(2016, 1, 1, 12, 0, 0) #noon
elif timeofday == 'transit':
etime = datetime(2016, 1, 1, 18, 0, 0) #6 pm
else:
etime = datetime(2016, 1, 1, 0, 0, 0) #midnight
eventdict = {
'event_id': '1234',
'magnitude': 7.5,
'lat': 0.0,
'lon': 0.0,
'depth': 10.0,
'event_timestamp': etime,
'event_description': 'test data',
'event_network': 'us'
}
shakedict = {
'event_id': '1234',
'shakemap_id': '1234',
'shakemap_version': 1,
'code_version': '1.0',
'process_timestamp': datetime.utcnow(),
'shakemap_originator': 'us',
'map_status': 'RELEASED',
'shakemap_event_type': 'SCENARIO'
}
uncdict = {'mmi': (1.0, 1)}
popdata = np.ones((2, 2), dtype=np.float32) * (popvalue) / 4
isodata = np.ones((2, 2), dtype=np.int16) * ucode
urbdata = np.ones((2, 2), dtype=np.int16) * density
mmidata = np.ones((2, 2), dtype=np.float32) * mmi
geodict = GeoDict({
'xmin': 0.5,
'xmax': 1.5,
'ymin': 0.5,
'ymax': 1.5,
'dx': 1.0,
'dy': 1.0,
'nx': 2,
'ny': 2
})
popgrid = GMTGrid(popdata, geodict)
isogrid = GMTGrid(isodata, geodict)
urbgrid = GMTGrid(urbdata, geodict)
popyear = 2016
layers = {'mmi': mmidata}
mmigrid = ShakeGrid(layers, geodict, eventdict, shakedict, uncdict)
popfile = isofile = urbfile = shakefile = ''
popsum = None
newresfat = None
newnresfat = None
try:
#make some temporary files
f, popfile = tempfile.mkstemp()
os.close(f)
f, isofile = tempfile.mkstemp()
os.close(f)
f, urbfile = tempfile.mkstemp()
os.close(f)
f, shakefile = tempfile.mkstemp()
os.close(f)
popgrid.save(popfile)
isogrid.save(isofile)
urbgrid.save(urbfile)
mmigrid.save(shakefile)
semi = SemiEmpiricalFatality.fromDefault()
semi.setGlobalFiles(popfile, popyear, urbfile, isofile)
t, resfat, nonresfat = semi.getLosses(shakefile)
popsum = 0
newresfat = {ccode: {}}
newnonresfat = {ccode: {}}
for key, value in resfat[ccode].items():
if value < 1:
value = np.floor(value)
newresfat[ccode][key] = value / 4.0
popsum += value / 4.0
for key, value in nonresfat[ccode].items():
newnonresfat[ccode][key] = value / 4.0
if value < 1:
value = np.floor(value)
popsum += value / 4.0
popsum = int(popsum)
finally:
files = [popfile, isofile, urbfile, shakefile]
for fname in files:
if os.path.isfile(fname):
os.remove(fname)
return (popsum, newresfat, newnonresfat)
class Exposure(object):
def __init__(self,popfile,popyear,isofile,popgrowth=None):
"""Create Exposure object, with population and country code grid files,
and a dictionary of country growth rates.
:param popfile:
Any GMT or ESRI style grid file supported by MapIO, containing population data.
:param popyear:
Integer indicating year when population data is valid.
:param isofile:
Any GMT or ESRI style grid file supported by MapIO, containing country code data (ISO 3166-1 numeric).
"""
self._popfile = popfile
self._popyear = popyear
self._isofile = isofile
self._popgrid = None
self._isogrid = None
self._shakegrid = None
if popgrowth is not None:
self._popgrowth = popgrowth
else:
self._popgrowth = PopulationGrowth.fromDefault()
self._country = Country()
self._pop_class = get_file_type(self._popfile)
self._iso_class = get_file_type(self._isofile)
def calcExposure(self,shakefile):
"""Calculate population exposure to shaking, per country, plus total exposure across all countries.
:param shakefile:
Path to ShakeMap grid.xml file.
:returns:
Dictionary containing country code (ISO2) keys, and values of
10 element arrays representing population exposure to MMI 1-10.
Dictionary will contain an additional key 'TotalExposure', with value of exposure across all countries.
Dictionary will also contain a field "maximum_border_mmi" which indicates the maximum MMI value along
any edge of the ShakeMap.
"""
#get shakemap geodict
shakedict = ShakeGrid.getFileGeoDict(shakefile,adjust='res')
#get population geodict
popdict,t = self._pop_class.getFileGeoDict(self._popfile)
#get country code geodict
isodict,t = self._iso_class.getFileGeoDict(self._isofile)
#special case for very high latitude events that may be outside the bounds
#of our population data...
if not popdict.intersects(shakedict):
expdict = {'UK':np.zeros((10,)),'TotalExposure':np.zeros((10,))}
return expdict
if popdict == shakedict == isodict:
#special case, probably for testing...
self._shakegrid = ShakeGrid.load(shakefile,adjust='res')
self._popgrid = self._pop_class.load(self._popfile)
self._isogrid = self._iso_class.load(self._isofile)
else:
sampledict = popdict.getBoundsWithin(shakedict)
self._shakegrid = ShakeGrid.load(shakefile,samplegeodict=sampledict,resample=True,
method='linear',adjust='res')
self._popgrid = self._pop_class.load(self._popfile,samplegeodict=sampledict,
resample=False,doPadding=True,padValue=np.nan)
self._isogrid = self._iso_class.load(self._isofile,samplegeodict=sampledict,
resample=True,method='nearest',doPadding=True,padValue=0)
mmidata = self._shakegrid.getLayer('mmi').getData()
popdata = self._popgrid.getData()
isodata = self._isogrid.getData()
eventyear = self._shakegrid.getEventDict()['event_timestamp'].year
#in order to avoid crazy far-future scenarios where PAGER models are probably invalid,
#check to see if the time gap between the date of population data collection and event year
#reaches either of a couple of different thresholds.
if eventyear > self._popyear:
tdiff = (eventyear - self._popyear)
if tdiff > SCENARIO_WARNING and tdiff < SCENARIO_ERROR:
msg = '''The input ShakeMap event year is more than %i years from the population date.
PAGER results for events this far in the future may not be valid.''' % SCENARIO_WARNING
warnings.warn(msg)
if tdiff > SCENARIO_ERROR:
msg = '''The input ShakeMap event year is more than %i years from the population date.
PAGER results for events this far in the future are not valid. Stopping.''' % SCENARIO_ERROR
raise PagerException(msg)
ucodes = np.unique(isodata)
for ccode in ucodes:
cidx = (isodata == ccode)
popdata[cidx] = self._popgrowth.adjustPopulation(popdata[cidx],ccode,self._popyear,eventyear)
exposure_dict = calc_exposure(mmidata,popdata,isodata)
newdict = {}
#Get rolled up exposures
total = np.zeros((10,),dtype=np.uint32)
for isocode,value in exposure_dict.items():
cdict = self._country.getCountry(int(isocode))
if cdict is None:
ccode = 'UK'
else:
ccode = cdict['ISO2']
newdict[ccode] = value
total += value
newdict['TotalExposure'] = total
#get the maximum MMI value along any of the four map edges
nrows,ncols = mmidata.shape
top = mmidata[0,0:ncols].max()
bottom = mmidata[nrows-1,0:ncols].max()
left = mmidata[0:nrows,0].max()
right = mmidata[0:nrows,ncols-1].max()
newdict['maximum_border_mmi'] = np.array([top,bottom,left,right]).max()
return newdict
def getPopulationGrid(self):
"""Return the internal population grid.
:returns:
Population grid.
"""
if self._popgrid is None:
raise PagerException('calcExposure() method must be called first.')
return self._popgrid
def getCountryGrid(self):
"""Return the Grid2D object containing ISO numeric country codes.
:returns:
Grid2D object containing ISO numeric country codes.
"""
if self._isogrid is None:
raise PagerException('calcExposure() method must be called first.')
return self._isogrid
def getShakeGrid(self):
"""Return the MultiGrid object containing ShakeMap data.
:returns:
MultiGrid object containing ShakeMap data.
"""
if self._shakegrid is None:
raise PagerException('calcExposure() method must be called first.')
return self._shakegrid
import textwrap
from urllib import request
import tempfile
import socket
import logging
# third party imports
import matplotlib
import numpy as np
# this allows us to have a non-interactive backend - essential on systems
# without a display
matplotlib.use("Agg")
COUNTRY = Country()
TIMEFMT = "%Y-%m-%d %H:%M:%S"
TSUNAMI_MAG_THRESH = 7.3
def _is_url(gridfile):
try:
fh = request.urlopen(gridfile)
tdir = tempfile.mkdtemp()
grid_file = os.path.join(tdir, "grid.xml")
data = fh.read().decode("utf-8")
fh.close()
f = open(grid_file, "wt")
f.write(data)
f.close()
return (True, grid_file)