def test_region_serialization():
memurl = 'sqlite://'
schemadir = get_data_path('schema')
session = emailschema.create_db(memurl, schemadir)
ca_coords = [[[-125.771484, 41.957448], [-112.412109, 42.672339],
[-112.060547, 31.791221], [-126.298828, 31.416944],
[-125.771484, 41.957448]]]
lon, lat = (-119.443359, 37.149371)
regiondict = {
'type': 'Feature',
'properties': {
'code': 'US_States-California',
'desc': 'California'
},
'geometry': {
'type': 'Polygon',
'coordinates': ca_coords
}
}
region = emailschema.Region()
region.fromDict(session, regiondict)
assert region.containsPoint(lat, lon) == True
regiondict2 = region.toDict()
assert regiondict == regiondict2
session.close()
def test_create_db(userfile=None, orgfile=None):
memurl = 'sqlite://'
#create a user in a dictionary
threshold = {'alertscheme': 'eis', 'value': 'red'}
region = {'name': 'UNSOUEU'}
profile = {'regions': [region], 'thresholds': [threshold]}
address = {
'email': '*****@*****.**',
'is_primary': True,
'priority': 1,
'format': 'long',
'profiles': [profile]
}
userdict = {
'lastname': 'Luthor',
'firstname': 'Alexander',
'createdon': datetime.utcnow().strftime(emailschema.TIME_FORMAT),
'org': 'USGS',
'addresses': [address]
}
tdir = tempfile.mkdtemp()
if userfile is None:
userfile = os.path.join(tdir, 'users.json')
f = open(userfile, 'wt')
users = [userdict]
json.dump(users, f)
f.close()
try:
schemadir = get_data_path('schema')
session = emailschema.create_db(memurl,
schemadir,
users_jsonfile=userfile,
orgs_jsonfile=orgfile)
print('Testing contents of in-memory database...')
assert session.query(emailschema.User).count() == 1
assert session.query(emailschema.Region).count() == 47
print('Passed test.')
except:
shutil.rmtree(tdir)
def test_delete_cascade():
memurl = 'sqlite://'
schemadir = get_data_path('schema')
session = emailschema.create_db(memurl, schemadir)
#create a user in a dictionary
threshold = {'alertscheme': 'eis', 'value': 'red'}
region = {'name': 'UN_Regions-UNSOUEU'}
profile = {'regions': [region], 'thresholds': [threshold]}
address = {
'email': '*****@*****.**',
'is_primary': True,
'priority': 1,
'format': 'long',
'profiles': [profile]
}
userdict = {
'lastname': 'Luthor',
'firstname': 'Alexander',
'createdon': datetime.utcnow().strftime(emailschema.TIME_FORMAT),
'org': 'USGS',
'addresses': [address]
}
users_before_add = session.query(emailschema.User).count()
addresses_before_add = session.query(emailschema.Address).count()
print('Testing deleting users...')
assert users_before_add == 0
assert addresses_before_add == 0
print('No users before insert.')
user = emailschema.User()
#inflate the user from the dictionary
user.fromDict(session, userdict)
session.add(user)
users_after_add = session.query(emailschema.User).count()
addresses_after_add = session.query(emailschema.Address).count()
assert users_after_add == 1
assert addresses_after_add == 1
print('One user, one address after insert.')
session.delete(user)
session.commit()
users_after_delete = session.query(emailschema.User).count()
addresses_after_delete = session.query(emailschema.Address).count()
assert users_after_delete == 0
assert addresses_after_delete == 0
print('No users, no addresses after deleting user.')
#test deleting cascades with events
event = emailschema.Event(eventcode='us2017abcd')
version = emailschema.Version(versioncode='us2017abcd',
time=datetime.utcnow(),
country='US',
lat=34.15,
lon=-118.13,
depth=10.0,
magnitude=6.5,
number=1,
fatlevel=1,
ecolevel=2,
summarylevel=2,
released=True,
was_pending=False,
processtime=datetime.utcnow(),
maxmmi=7.1)
print('Test cascade deletes with events and versions...')
events_before_add = session.query(emailschema.Event).count()
versions_before_add = session.query(emailschema.Version).count()
assert events_before_add == 0
assert versions_before_add == 0
event.versions.append(version)
session.add(event)
session.commit()
events_after_add = session.query(emailschema.Event).count()
versions_after_add = session.query(emailschema.Version).count()
assert events_after_add == 1
assert versions_after_add == 1
session.delete(event)
session.commit()
events_after_delete = session.query(emailschema.Event).count()
versions_after_delete = session.query(emailschema.Version).count()
assert events_after_delete == 0
assert versions_after_delete == 0
session.close()
def test_get_polygon():
memurl = 'sqlite://'
schemadir = get_data_path('schema')
session = emailschema.create_db(memurl, schemadir)
POINTS = {
'UNEASTAFR': [(-18.319329, 36.408691)],
'UNNORAFR': [(29.051368, 20.478516)],
'UNWESTAS': [(23.216107, 45.263672)],
'UNWESTAFR': [(10.806328, -11.513672)],
'UNSEAS': [(1.202915, 116.279297)],
'UNNOREU': [(61.76013, 14.853516)],
'UNCENTAMER': [(11.817621, -84.177246)],
'UNMIC': [(13.444304, 144.793731)],
'UNMIDAFR': [(-9.115656, 17.094727)],
'UNSOUAMER': [(-21.314964, -59.150391)],
'UNSOUEU': [(40.101185, -2.504883)],
'UNWESTEU': [(47.567261, 3.999023)],
'UNMEL': [(-9.975613, 149.128418)],
'UNAUSNZ': [(-23.427969, 134.912109)],
'UNCARIB': [(18.847812, -70.466309)],
'UNSOUAFR': [(-31.814563, 24.038086)],
'UNEASTEU': [(50.392761, 25.708008)],
'UNEASTAS': [(29.59973, 111.577148)],
'UNNORAMER': [(38.505191, -100.019531)],
'UNCENTAS': [(40.033924, 66.225586)],
'UNSOUAS': [(14.067317, 77.607422)],
'UNPOL': [(-21.178986, -175.198242)],
'FEMA01': [(44.585577, -69.147949)],
'FEMA05': [(46.342188, -88.791504)],
'FEMA10': [(44.80425, -120.651855), (65.924072, -151.347656)],
'FEMA07': [(38.577158, -97.668457)],
'FEMA02': [(42.645071, -74.992676)],
'FEMA08': [(43.384092, -107.556152)],
'FEMA09': [(37.434522, -120.695801), (13.444304, 144.793731),
(-14.30164, -170.696181)],
'FEMA03': [(37.747915, -78.112793)],
'FEMA06': [(30.215168, -98.195801)],
'FEMA04': [(33.703207, -84.276123)],
'SWAN': [(22.093275, -4.262695), (-29.71191, 21.181641)],
'LAC': [(-26.613086, -61.083984)],
'EAP': [(-28.042895, 140.449219), (-17.751956, -149.315186)],
'SA': [(17.595594, 76.311035)],
'ECA': [(6.389001, 42.84668)],
'EMCA': [(49.772396, 16.479492), (55.463285, -105.732422)],
'Cont_US': [(37.974515, -104.501953)],
'US_Terr': [(19.639354, -155.577393)],
'Not_US': [(-10.541821, 25.136719)],
'USNORTHCOM': [(25.433353, -103.535156)],
'USEUCOM': [(51.940032, 10.700684)],
'USPACOM': [(21.441245, -157.922974)],
'USSOUTHCOM': [(-1.331972, -60.073242)],
'USAFRICOM': [(14.551684, 21.269531)],
'USCENTCOM': [(21.767152, 49.350586)]
}
for regioncode, plist in POINTS.items():
region = session.query(emailschema.Region).filter(
emailschema.Region.name == regioncode).first()
if region is None:
raise Exception('Could not find region %s in database!' %
regioncode)
for point in plist:
lat, lon = point
print('Checking region %s...' % regioncode)
if not region.containsPoint(lat, lon):
raise Exception(
'Region %s does not contain point (%.4f,%.4f).' %
(regioncode, lat, lon))
session.close()
def test_user_serialization():
memurl = 'sqlite://'
schemadir = get_data_path('schema')
session = emailschema.create_db(memurl, schemadir)
#create a user in a dictionary
threshold = {'alertscheme': 'eis', 'value': 'red'}
region = {'name': 'UN_Regions-UNSOUEU'}
profile = {'regions': [region], 'thresholds': [threshold]}
address = {
'email': '*****@*****.**',
'is_primary': True,
'priority': 1,
'format': 'long',
'profiles': [profile]
}
userdict = {
'lastname': 'Luthor',
'firstname': 'Alexander',
'createdon': datetime.utcnow().strftime(emailschema.TIME_FORMAT),
'org': 'USGS',
'addresses': [address]
}
user = emailschema.User()
#inflate the user from the dictionary
user.fromDict(session, userdict)
#deflate the user into a dictionary
userdict2 = user.toDict()
#make sure the input/output dictionaries have the same content
assert userdict['lastname'] == userdict2['lastname']
assert userdict['firstname'] == userdict2['firstname']
assert userdict['createdon'] == userdict2['createdon']
assert userdict['org'] == userdict2['org']
assert userdict['addresses'][0]['email'] == userdict2['addresses'][0][
'email']
assert userdict['addresses'][0]['is_primary'] == userdict2['addresses'][0][
'is_primary']
assert userdict['addresses'][0]['priority'] == userdict2['addresses'][0][
'priority']
assert userdict['addresses'][0]['format'] == userdict2['addresses'][0][
'format']
rname = userdict['addresses'][0]['profiles'][0]['regions'][0]['name']
rname2 = userdict2['addresses'][0]['profiles'][0]['regions'][0]['name']
assert rname == rname2
tname = userdict['addresses'][0]['profiles'][0]['thresholds'][0][
'alertscheme']
tname2 = userdict2['addresses'][0]['profiles'][0]['thresholds'][0][
'alertscheme']
assert tname == tname2
tvalue = userdict['addresses'][0]['profiles'][0]['thresholds'][0]['value']
tvalue2 = userdict2['addresses'][0]['profiles'][0]['thresholds'][0][
'value']
assert tvalue == tvalue2
session.close()
def main(pargs, config):
# get the users home directory
homedir = os.path.expanduser("~")
# handle cancel messages
if pargs.cancel:
# we presume that pargs.gridfile in this context is an event ID.
msg = _cancel(pargs.gridfile, config)
print(msg)
return True
# what kind of thing is gridfile?
is_file = os.path.isfile(pargs.gridfile)
is_url, url_gridfile = _is_url(pargs.gridfile)
is_pdl, pdl_gridfile = _check_pdl(pargs.gridfile, config)
if is_file:
gridfile = pargs.gridfile
elif is_url:
gridfile = url_gridfile
elif is_pdl:
gridfile = pdl_gridfile
else:
print("ShakeMap Grid file %s does not exist." % pargs.gridfile)
return False
pager_folder = os.path.join(homedir, config["output_folder"])
pager_archive = os.path.join(homedir, config["archive_folder"])
admin = PagerAdmin(pager_folder, pager_archive)
# stdout will now be logged as INFO, stderr will be logged as WARNING
mail_host = config["mail_hosts"][0]
mail_from = config["mail_from"]
developers = config["developers"]
logfile = os.path.join(pager_folder, "pager.log")
plog = PagerLogger(logfile, developers, mail_from, mail_host, debug=pargs.debug)
logger = plog.getLogger()
try:
eid = None
pager_version = None
# get all the basic event information and print it, if requested
shake_tuple = getHeaderData(gridfile)
eid = shake_tuple[1]["event_id"].lower()
etime = shake_tuple[1]["event_timestamp"]
if not len(eid):
eid = shake_tuple[0]["event_id"].lower()
network = shake_tuple[1]["event_network"].lower()
if network == "":
network = "us"
if not eid.startswith(network):
eid = network + eid
# Create a ComcatInfo object to hopefully tell us a number of things about this event
try:
ccinfo = ComCatInfo(eid)
location = ccinfo.getLocation()
tsunami = ccinfo.getTsunami()
authid, allids = ccinfo.getAssociatedIds()
authsource, othersources = ccinfo.getAssociatedSources()
except: # fail over to what we can determine locally
location = shake_tuple[1]["event_description"]
tsunami = shake_tuple[1]["magnitude"] >= TSUNAMI_MAG_THRESH
authid = eid
authsource = network
allids = []
# location field can be empty (None), which breaks a bunch of things
if location is None:
location = ""
# Check to see if user wanted to override default tsunami criteria
if pargs.tsunami != "auto":
if pargs.tsunami == "on":
tsunami = True
else:
tsunami = False
# check to see if this event is a scenario
is_scenario = False
shakemap_type = shake_tuple[0]["shakemap_event_type"]
if shakemap_type == "SCENARIO":
is_scenario = True
# if event is NOT a scenario and event time is in the future,
# flag the event as a scenario and yell about it.
if etime > datetime.datetime.utcnow():
is_scenario = True
logger.warning(
"Event origin time is in the future! Flagging this as a scenario."
)
if is_scenario:
if re.search("scenario", location.lower()) is None:
location = "Scenario " + location
# create the event directory (if it does not exist), and start logging there
logger.info("Creating event directory")
event_folder = admin.createEventFolder(authid, etime)
# Stop processing if there is a "stop" file in the event folder
stopfile = os.path.join(event_folder, "stop")
if os.path.isfile(stopfile):
fmt = '"stop" file found in %s. Stopping processing, returning with 1.'
logger.info(fmt % (event_folder))
return True
pager_version = get_pager_version(event_folder)
version_folder = os.path.join(event_folder, "version.%03d" % pager_version)
os.makedirs(version_folder)
event_logfile = os.path.join(version_folder, "event.log")
# this will turn off the global rotating log file
# and switch to the one in the version folder.
plog.setVersionHandler(event_logfile)
# Copy the grid.xml file to the version folder
# sometimes (usu when testing) the input grid isn't called grid.xml. Rename it here.
version_grid = os.path.join(version_folder, "grid.xml")
shutil.copyfile(gridfile, version_grid)
# Check to see if the tsunami flag has been previously set
tsunami_toggle = {"on": 1, "off": 0}
tsunami_file = os.path.join(event_folder, "tsunami")
if os.path.isfile(tsunami_file):
tsunami = tsunami_toggle[open(tsunami_file, "rt").read().strip()]
# get the rest of the event info
etime = shake_tuple[1]["event_timestamp"]
elat = shake_tuple[1]["lat"]
elon = shake_tuple[1]["lon"]
emag = shake_tuple[1]["magnitude"]
# get the year of the event
event_year = shake_tuple[1]["event_timestamp"].year
# find the population data collected most closely to the event_year
pop_year, popfile = _get_pop_year(
event_year, config["model_data"]["population_data"]
)
logger.info("Population year: %i Population file: %s\n" % (pop_year, popfile))
# Get exposure results
logger.info("Calculating population exposure.")
isofile = config["model_data"]["country_grid"]
expomodel = Exposure(popfile, pop_year, isofile)
exposure = None
exposure = expomodel.calcExposure(gridfile)
# incidentally grab the country code of the epicenter
numcode = expomodel._isogrid.getValue(elat, elon)
if np.isnan(numcode):
cdict = None
else:
cdict = Country().getCountry(int(numcode))
if cdict is None:
ccode = "UK"
else:
ccode = cdict["ISO2"]
logger.info("Country code at epicenter is %s" % ccode)
# get fatality results, if requested
logger.info("Calculating empirical fatalities.")
fatmodel = EmpiricalLoss.fromDefaultFatality()
fatdict = fatmodel.getLosses(exposure)
# get economic results, if requested
logger.info("Calculating economic exposure.")
econexpmodel = EconExposure(popfile, pop_year, isofile)
ecomodel = EmpiricalLoss.fromDefaultEconomic()
econexposure = econexpmodel.calcExposure(gridfile)
ecodict = ecomodel.getLosses(econexposure)
shakegrid = econexpmodel.getShakeGrid()
# Get semi-empirical losses
logger.info("Calculating semi-empirical fatalities.")
urbanfile = config["model_data"]["urban_rural_grid"]
if not os.path.isfile(urbanfile):
raise PagerException("Urban-rural grid file %s does not exist." % urbanfile)
semi = SemiEmpiricalFatality.fromDefault()
semi.setGlobalFiles(popfile, pop_year, urbanfile, isofile)
semiloss, resfat, nonresfat = semi.getLosses(gridfile)
# get all of the other components of PAGER
logger.info("Getting all comments.")
# get the fatality and economic comments
impact1, impact2 = get_impact_comments(
fatdict, ecodict, econexposure, event_year, ccode
)
# get comment describing vulnerable structures in the region.
struct_comment = get_structure_comment(resfat, nonresfat, semi)
# get the comment describing historic secondary hazards
secondary_comment = get_secondary_comment(elat, elon, emag)
# get the comment describing historical comments in the region
historical_comment = get_historical_comment(elat, elon, emag, exposure, fatdict)
# generate the probability plots
logger.info("Drawing probability plots.")
fat_probs_file, eco_probs_file = _draw_probs(
fatmodel, fatdict, ecomodel, ecodict, version_folder
)
# generate the exposure map
exposure_base = os.path.join(version_folder, "exposure")
logger.info("Generating exposure map...")
oceanfile = config["model_data"]["ocean_vectors"]
oceangrid = config["model_data"]["ocean_grid"]
cityfile = config["model_data"]["city_file"]
borderfile = config["model_data"]["border_vectors"]
shake_grid = expomodel.getShakeGrid()
pop_grid = expomodel.getPopulationGrid()
pdf_file, png_file, mapcities = draw_contour(
shake_grid,
pop_grid,
oceanfile,
oceangrid,
cityfile,
exposure_base,
borderfile,
is_scenario=is_scenario,
)
logger.info("Generated exposure map %s" % pdf_file)
# figure out whether this event has been "released".
is_released = _get_release_status(
pargs,
config,
fatmodel,
fatdict,
ecomodel,
ecodict,
shake_tuple,
event_folder,
)
# Create a data object to encapsulate everything we know about the PAGER
# results, and then serialize that to disk in the form of a number of JSON files.
logger.info("Making PAGER Data object.")
doc = PagerData()
timezone_file = config["model_data"]["timezones_file"]
elapsed = pargs.elapsed
doc.setInputs(
shakegrid,
timezone_file,
pager_version,
shakegrid.getEventDict()["event_id"],
authid,
tsunami,
location,
is_released,
elapsed=elapsed,
)
logger.info("Setting inputs.")
doc.setExposure(exposure, econexposure)
logger.info("Setting exposure.")
doc.setModelResults(
fatmodel, ecomodel, fatdict, ecodict, semiloss, resfat, nonresfat
)
logger.info("Setting comments.")
doc.setComments(
impact1, impact2, struct_comment, historical_comment, secondary_comment
)
logger.info("Setting map info.")
doc.setMapInfo(cityfile, mapcities)
logger.info("Validating.")
doc.validate()
# if we have determined that the event is a scenario (origin time is in the future)
# and the shakemap is not flagged as such, set the shakemap type in the
# pagerdata object to be 'SCENARIO'.
if is_scenario:
doc.setToScenario()
json_folder = os.path.join(version_folder, "json")
os.makedirs(json_folder)
logger.info("Saving output to JSON.")
doc.saveToJSON(json_folder)
logger.info("Saving output to XML.")
doc.saveToLegacyXML(version_folder)
logger.info("Creating onePAGER pdf...")
onepager_pdf, error = create_onepager(doc, version_folder)
if onepager_pdf is None:
raise PagerException("Could not create onePAGER output: \n%s" % error)
# copy the contents.xml file to the version folder
contentsfile = get_data_path("contents.xml")
if contentsfile is None:
raise PagerException("Could not find contents.xml file.")
shutil.copy(contentsfile, version_folder)
# send pdf as attachment to internal team of PAGER users
if not is_released and not is_scenario:
message_pager(config, onepager_pdf, doc)
# run transfer, as appropriate and as specified by config
# the PAGER product eventsource and eventsourcecode should
# match the input ShakeMap settings for these properties.
# This can possibly cause confusion if a regional ShakeMap is
# trumped with one from NEIC, but this should happen less often
# than an NEIC origin being made authoritative over a regional one.
eventsource = network
eventsourcecode = eid
res, msg = transfer(
config,
doc,
eventsourcecode,
eventsource,
version_folder,
is_scenario=is_scenario,
)
logger.info(msg)
if not res:
logger.critical('Error transferring PAGER content. "%s"' % msg)
print("Created onePAGER pdf %s" % onepager_pdf)
logger.info("Created onePAGER pdf %s" % onepager_pdf)
logger.info("Done.")
return True
except Exception as e:
f = io.StringIO()
traceback.print_exc(file=f)
msg = e
msg = "%s\n %s" % (str(msg), f.getvalue())
hostname = socket.gethostname()
msg = msg + "\n" + "Error occurred on %s\n" % (hostname)
if gridfile is not None:
msg = msg + "\n" + "Error on file: %s\n" % (gridfile)
if eid is not None:
msg = msg + "\n" + "Error on event: %s\n" % (eid)
if pager_version is not None:
msg = msg + "\n" + "Error on version: %i\n" % (pager_version)
f.close()
logger.critical(msg)
logger.info("Sent error to email")
return False
