def getEventsBeforeDate(self, beforedate):
events = []
event_folders = self.getAllEventFolders()
for event_folder in event_folders:
version_folder = self.getLastVersion(event_folder)
json_folder = os.path.join(version_folder,'json')
pdata = PagerData()
pdata.loadFromJSON(json_folder)
if pdata.time < beforedate:
events.append(pdata.id)
return events
def query(self,start_time=datetime.datetime(1800,1,1),end_time=datetime.datetime.utcnow(),
mag_threshold=0.0,alert_threshold='green',version='last',eventid=None):
levels = {'green':0,
'yellow':1,
'orange':2,
'red':3}
if eventid is not None:
all_event_folders = [self.getEventFolder(eventid)]
version = 'all'
else:
all_event_folders = self.getAllEventFolders()
event_data = []
do_process_time = False
if eventid is not None:
do_process_time = True
df = pd.DataFrame(columns=PagerData.getSeriesColumns(processtime=do_process_time))
jsonfolders = []
for event_folder in all_event_folders:
vnums = self.getVersionNumbers(event_folder)
if version == 'first':
vnum = vnums[0]
jsonfolders.append(os.path.join(event_folder,'version.%03d' % vnum,'json'))
elif version == 'last':
vnum = vnums[-1]
jsonfolders.append(os.path.join(event_folder,'version.%03d' % vnum,'json'))
elif version == 'eight':
for vnum in vnums:
jsonfolder = os.path.join(event_folder,'version.%03d' % vnum,'json')
pdata = PagerData()
try:
pdata.loadFromJSON(jsonfolder)
except:
continue
if pdata.processing_time >= pdata.time + datetime.timedelta(seconds=EIGHT_HOURS):
break
jsonfolders.append(jsonfolder)
elif version == 'all':
for vnum in vnums:
jsonfolder = os.path.join(event_folder,'version.%03d' % vnum,'json')
jsonfolders.append(jsonfolder)
else:
raise PagerException('version option "%s" not supported.' % version)
broken = []
for jsonfolder in jsonfolders:
pdata = PagerData()
vnum = 1000
while vnum > 1:
try:
pdata.loadFromJSON(jsonfolder)
vnum = 0
except:
#handle the case where the most recent version of the event has some
#sort of error causing it to miss
root,jsonfolder = os.path.split(jsonfolder)
root2,vfolder = os.path.split(root)
vt,vnums = vfolder.split('.')
vnum = int(vnums) - 1
jsonfolder = os.path.join(root2,'%s.%03d' % (vt,vnum),'json')
if not pdata._is_validated:
broken.append(jsonfolder)
meetsLevel = levels[pdata.summary_alert] >= levels[alert_threshold]
meetsMag = pdata.magnitude >= mag_threshold
if pdata.time >= start_time and pdata.time <= end_time and meetsLevel and meetsMag:
row = pdata.toSeries(processtime=do_process_time)
df = df.append(row,ignore_index=True)
df.Version = df.Version.astype(int)
df.Elapsed = df.Elapsed.astype(int)
df = df.sort_values('EventTime')
df = df.set_index('EventID')
return (df,broken)
def test():
homedir = os.path.dirname(os.path.abspath(
__file__)) # where is this script?
fatfile = os.path.join(homedir, '..', 'data', 'fatality.xml')
ecofile = os.path.join(homedir, '..', 'data', 'economy.xml')
cityfile = os.path.join(homedir, '..', 'data', 'cities1000.txt')
event = 'northridge'
shakefile = os.path.join(homedir, '..', 'data',
'eventdata', event, '%s_grid.xml' % event)
popfile = os.path.join(homedir, '..', 'data',
'eventdata', event, '%s_gpw.flt' % event)
isofile = os.path.join(homedir, '..', 'data',
'eventdata', event, '%s_isogrid.bil' % event)
urbanfile = os.path.join(homedir, '..', 'data',
'eventdata', 'northridge', 'northridge_urban.bil')
oceanfile = os.path.join(
homedir, '..', 'data', 'eventdata', 'northridge', 'northridge_ocean.json')
oceangridfile = os.path.join(
homedir, '..', 'data', 'eventdata', 'northridge', 'northridge_ocean.bil')
timezonefile = os.path.join(
homedir, '..', 'data', 'eventdata', 'northridge', 'northridge_timezone.shp')
invfile = os.path.join(homedir, '..', 'data', 'semi_inventory.hdf')
colfile = os.path.join(homedir, '..', 'data', 'semi_collapse_mmi.hdf')
casfile = os.path.join(homedir, '..', 'data', 'semi_casualty.hdf')
workfile = os.path.join(homedir, '..', 'data', 'semi_workforce.hdf')
tdir = tempfile.mkdtemp()
basename = os.path.join(tdir, 'output')
exp = Exposure(popfile, 2012, isofile)
results = exp.calcExposure(shakefile)
shakegrid = exp.getShakeGrid()
popgrid = exp.getPopulationGrid()
pdffile, pngfile, mapcities = draw_contour(
shakegrid, popgrid, oceanfile, oceangridfile, cityfile, basename)
shutil.rmtree(tdir)
popyear = 2012
shake_tuple = getHeaderData(shakefile)
tsunami = shake_tuple[1]['magnitude'] >= TSUNAMI_MAG_THRESH
semi = SemiEmpiricalFatality.fromDefault()
semi.setGlobalFiles(popfile, popyear, urbanfile, isofile)
semiloss, resfat, nonresfat = semi.getLosses(shakefile)
popgrowth = PopulationGrowth.fromDefault()
econexp = EconExposure(popfile, 2012, isofile)
fatmodel = EmpiricalLoss.fromDefaultFatality()
expobject = Exposure(popfile, 2012, isofile, popgrowth)
expdict = expobject.calcExposure(shakefile)
fatdict = fatmodel.getLosses(expdict)
econexpdict = econexp.calcExposure(shakefile)
ecomodel = EmpiricalLoss.fromDefaultEconomic()
ecodict = ecomodel.getLosses(expdict)
shakegrid = econexp.getShakeGrid()
pagerversion = 1
cities = Cities.loadFromGeoNames(cityfile)
impact1 = '''Red alert level for economic losses. Extensive damage is probable
and the disaster is likely widespread. Estimated economic losses are less
than 1% of GDP of Italy. Past events with this alert level have required
a national or international level response.'''
impact2 = '''Orange alert level for shaking-related fatalities. Significant
casualties are likely.'''
structcomment = '''Overall, the population in this region resides in structures
that are a mix of vulnerable and earthquake resistant construction. The predominant
vulnerable building types are unreinforced brick with mud and mid-rise nonductile
concrete frame with infill construction.'''
histeq = [1, 2, 3]
struct_comment = '''Overall, the population in this region resides
in structures that are resistant to earthquake
shaking, though some vulnerable structures
exist.'''
secondary_comment = '''Recent earthquakes in this area have caused secondary hazards
such as landslides that might have contributed to losses.'''
hist_comment = ''''A magnitude 7.1 earthquake 240 km east of this event struck Reventador: Ecuador
on March 6, 1987 (UTC), with estimated population exposures of 14,000 at intensity VIII and 2,000
at intensity IX or greater, resulting in a reported 5,000 fatalities.'''.replace('\n', '')
location = 'At the top of the world.'
is_released = True
doc = PagerData()
eventcode = shakegrid.getEventDict()['event_id']
versioncode = eventcode
doc.setInputs(shakegrid, timezonefile, pagerversion,
versioncode, eventcode, tsunami, location, is_released)
doc.setExposure(expdict, econexpdict)
doc.setModelResults(fatmodel, ecomodel,
fatdict, ecodict,
semiloss, resfat, nonresfat)
doc.setComments(impact1, impact2, struct_comment,
hist_comment, secondary_comment)
doc.setMapInfo(cityfile, mapcities)
doc.validate()
# let's test the property methods
tdoc(doc, shakegrid, impact1, impact2,
expdict, struct_comment, hist_comment)
# see if we can save this to a bunch of files then read them back in
try:
tdir = tempfile.mkdtemp()
doc.saveToJSON(tdir)
newdoc = PagerData()
newdoc.loadFromJSON(tdir)
tdoc(newdoc, shakegrid, impact1, impact2,
expdict, struct_comment, hist_comment)
# test the xml saving method
xmlfile = doc.saveToLegacyXML(tdir)
except Exception as e:
assert 1 == 2
finally:
shutil.rmtree(tdir)
def query(self, start_time=datetime.datetime(1800, 1, 1), end_time=datetime.datetime.utcnow(),
mag_threshold=0.0, alert_threshold='green', version='last', eventid=None):
"""Query PAGER file for events matching input parameters.
:param start_time:
Datetime indicating the minimum date/time for the search.
:param end_time:
Datetime indicating the maximum date/time for the search.
:param mag_thresh:
Minimum magnitude threshold.
:param alert_threshold:
Minimum alert level threshold ('green','yellow','orange','red').
:param version:
Which version(s) to select from events:
- 'all' Get all versions.
- 'last' Get last version.
- 'eight' Get first version that was created more than 8 hours after origin time.
:param eventid:
Return version(s) for specific event ID.
:returns:
Pandas dataframe containing columns:
- 'EventID' - event ID
- 'Impacted Country ($)' Country with largest dollar losses.
- 'Version' - Version number
- 'EventTime' - Origin Time
- 'Lat' - Origin latitude.
- 'Lon' - Origin longitude.
- 'Depth' - Origin depth.
- 'Mag' - Event magnitude.
- 'MaxMMI' - Maximum MMI value (felt by at least 1000 people)
- 'FatalityAlert' - Fatality alert level ('green','yellow','orange','red')
- 'EconomicAlert' - Economic alert level ('green','yellow','orange','red')
- 'SummaryAlert' - Summary alert level ('green','yellow','orange','red')
- 'Elapsed' - Elapsed time (minutes) between origin time and version.
"""
levels = {'green': 0,
'yellow': 1,
'orange': 2,
'red': 3}
if eventid is not None:
all_event_folders = [self.getEventFolder(eventid)]
version = 'all'
else:
all_event_folders = self.getAllEventFolders()
event_data = []
do_process_time = False
if eventid is not None:
do_process_time = True
df = pd.DataFrame(columns=PagerData.getSeriesColumns(
processtime=do_process_time))
jsonfolders = []
for event_folder in all_event_folders:
vnums = self.getVersionNumbers(event_folder)
if version == 'first':
vnum = vnums[0]
jsonfolders.append(os.path.join(
event_folder, 'version.%03d' % vnum, 'json'))
elif version == 'last':
vnum = vnums[-1]
jsonfolders.append(os.path.join(
event_folder, 'version.%03d' % vnum, 'json'))
elif version == 'eight':
for vnum in vnums:
jsonfolder = os.path.join(
event_folder, 'version.%03d' % vnum, 'json')
pdata = PagerData()
try:
pdata.loadFromJSON(jsonfolder)
except:
continue
if pdata.processing_time >= pdata.time + datetime.timedelta(seconds=EIGHT_HOURS):
break
jsonfolders.append(jsonfolder)
elif version == 'all':
for vnum in vnums:
jsonfolder = os.path.join(
event_folder, 'version.%03d' % vnum, 'json')
jsonfolders.append(jsonfolder)
else:
raise PagerException(
'version option "%s" not supported.' % version)
broken = []
for jsonfolder in jsonfolders:
pdata = PagerData()
vnum = 1000
while vnum > 1:
try:
pdata.loadFromJSON(jsonfolder)
vnum = 0
except:
# handle the case where the most recent version of the event has some
# sort of error causing it to miss
root, jsonfolder = os.path.split(jsonfolder)
root2, vfolder = os.path.split(root)
vt, vnums = vfolder.split('.')
vnum = int(vnums) - 1
jsonfolder = os.path.join(
root2, '%s.%03d' % (vt, vnum), 'json')
if not pdata._is_validated:
broken.append(jsonfolder)
try:
meetsLevel = levels[pdata.summary_alert] >= levels[alert_threshold]
except Exception as e:
x = 1
meetsMag = pdata.magnitude >= mag_threshold
if pdata.time >= start_time and pdata.time <= end_time and meetsLevel and meetsMag:
row = pdata.toSeries(processtime=do_process_time)
df = df.append(row, ignore_index=True)
df.Version = df.Version.astype(int)
df.Elapsed = df.Elapsed.astype(int)
df = df.sort_values('EventTime')
df = df.set_index('EventID')
return (df, broken)