def main():
config = utils.get_config_params("config.ini")
version = config.get("es", "version")
table = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"geometry": {
"type": "geo_shape"
}
}
},
}),
view="opendrr_sauid_hexgrid_unclipped_{}".format(version),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM boundaries."SAUID_HexGrid_unclipped" \
ORDER BY "SAUID_HexGrid_unclipped"."sauid" \
LIMIT {limit} \
OFFSET {offset}',
)
table.postgis2es()
return
def main():
config = utils.get_config_params("config.ini")
version = config.get("es", "version")
# args = parse_args()
table = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"coordinates": {
"type": "geo_point"
},
"geometry": {
"type": "geo_shape"
},
}
},
}),
view="opendrr_psra_uhs_{}".format(version),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM results_psra_national.psra_uhs \
ORDER BY psra_uhs."geom" \
LIMIT {limit} \
OFFSET {offset}',
)
table.postgis2es()
return
def main():
config = utils.get_config_params("config.ini")
version = config.get("es", "version")
dsraTable = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
"mappings": {"properties": {"geometry": {"type": "geo_shape"}}},
}
),
view="opendrr_shakemap_scenario_extents_{}".format(version),
sqlquerystring="SELECT *, ST_AsGeoJSON(geom) \
FROM gmf.shakemap_scenario_extents \
LIMIT {limit} \
OFFSET {offset}",
)
dsraTable.postgis2es()
return
def main():
config = utils.get_config_params("config.ini")
version = config.get("es", "version")
# sauid level aggregation
table = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"geometry": {
"type": "geo_shape"
}
}
},
}),
view="opendrr_nhsl_physical_exposure_indicators_s_{}".format(version),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom_poly) \
FROM \
results_nhsl_physical_exposure.nhsl_physical_exposure_indicators_s \
ORDER BY nhsl_physical_exposure_indicators_s."Sauid" \
LIMIT {limit} \
OFFSET {offset}',
)
table.postgis2es()
# building level aggregation
table = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"coordinates": {
"type": "geo_point"
},
"geometry": {
"type": "geo_shape"
},
}
},
}),
view="opendrr_nhsl_physical_exposure_indicators_b_{}".format(version),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom_point) \
FROM \
results_nhsl_physical_exposure.nhsl_physical_exposure_indicators_b \
ORDER BY nhsl_physical_exposure_indicators_b."BldgID" \
LIMIT {limit} \
OFFSET {offset}',
)
table.postgis2es()
# hexgrid 1km aggregation
table = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"geometry": {
"type": "geo_shape"
}
}
},
}),
view="opendrr_nhsl_physical_exposure_indicators_hexgrid_1km_{}".format(
version),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM \
results_nhsl_physical_exposure.nhsl_physical_exposure_indicators_hexgrid_1km \
ORDER BY nhsl_physical_exposure_indicators_hexgrid_1km."gridid_1" \
LIMIT {limit} \
OFFSET {offset}',
)
table.postgis2es()
# hexgrid 1km unclipped aggregation
table = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"geometry": {
"type": "geo_shape"
}
}
},
}),
view="opendrr_nhsl_physical_exposure_indicators_hexgrid_1km_uc_{}".
format(version),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM \
results_nhsl_physical_exposure.nhsl_physical_exposure_indicators_hexgrid_1km_uc \
ORDER BY nhsl_physical_exposure_indicators_hexgrid_1km_uc."gridid_1" \
LIMIT {limit} \
OFFSET {offset}',
)
table.postgis2es()
# hexgrid 5km aggregation
table = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"geometry": {
"type": "geo_shape"
}
}
},
}),
view="opendrr_nhsl_physical_exposure_indicators_hexgrid_5km_{}".format(
version),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM \
results_nhsl_physical_exposure.nhsl_physical_exposure_indicators_hexgrid_5km \
ORDER BY nhsl_physical_exposure_indicators_hexgrid_5km."gridid_5" \
LIMIT {limit} \
OFFSET {offset}',
)
table.postgis2es()
# hexgrid 5km unclipped aggregation
table = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"geometry": {
"type": "geo_shape"
}
}
},
}),
view="opendrr_nhsl_physical_exposure_indicators_hexgrid_5km_uc_{}".
format(version),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM \
results_nhsl_physical_exposure.nhsl_physical_exposure_indicators_hexgrid_5km_uc \
ORDER BY nhsl_physical_exposure_indicators_hexgrid_5km_uc."gridid_5" \
LIMIT {limit} \
OFFSET {offset}',
)
table.postgis2es()
# hexgrid 10km aggregation
table = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"geometry": {
"type": "geo_shape"
}
}
},
}),
view="opendrr_nhsl_physical_exposure_indicators_hexgrid_10km_{}".
format(version),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM \
results_nhsl_physical_exposure.nhsl_physical_exposure_indicators_hexgrid_10km \
ORDER BY nhsl_physical_exposure_indicators_hexgrid_10km."gridid_10" \
LIMIT {limit} \
OFFSET {offset}',
)
table.postgis2es()
# hexgrid 10km unclipped aggregation
table = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"geometry": {
"type": "geo_shape"
}
}
},
}),
view="opendrr_nhsl_physical_exposure_indicators_hexgrid_10km_uc_{}".
format(version),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM \
results_nhsl_physical_exposure.nhsl_physical_exposure_indicators_hexgrid_10km_uc \
ORDER BY nhsl_physical_exposure_indicators_hexgrid_10km_uc."gridid_10" \
LIMIT {limit} \
OFFSET {offset}',
)
table.postgis2es()
# hexgrid 25km aggregation
table = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"geometry": {
"type": "geo_shape"
}
}
},
}),
view="opendrr_nhsl_physical_exposure_indicators_hexgrid_25km_{}".
format(version),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM \
results_nhsl_physical_exposure.nhsl_physical_exposure_indicators_hexgrid_25km \
ORDER BY nhsl_physical_exposure_indicators_hexgrid_25km."gridid_25" \
LIMIT {limit} \
OFFSET {offset}',
)
table.postgis2es()
# hexgrid 25km unclipped aggregation
table = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"geometry": {
"type": "geo_shape"
}
}
},
}),
view="opendrr_nhsl_physical_exposure_indicators_hexgrid_25km_uc_{}".
format(version),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM \
results_nhsl_physical_exposure.nhsl_physical_exposure_indicators_hexgrid_25km_uc \
ORDER BY nhsl_physical_exposure_indicators_hexgrid_25km_uc."gridid_25" \
LIMIT {limit} \
OFFSET {offset}',
)
table.postgis2es()
# hexgrid 50km unclipped aggregation
table = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"geometry": {
"type": "geo_shape"
}
}
},
}),
view="opendrr_nhsl_physical_exposure_indicators_hexgrid_50km_uc_{}".
format(version),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM \
results_nhsl_physical_exposure.nhsl_physical_exposure_indicators_hexgrid_50km_uc \
ORDER BY nhsl_physical_exposure_indicators_hexgrid_50km_uc."gridid_50" \
LIMIT {limit} \
OFFSET {offset}',
)
table.postgis2es()
# hexgrid 100km unclipped aggregation
table = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"geometry": {
"type": "geo_shape"
}
}
},
}),
view="opendrr_nhsl_physical_exposure_indicators_hexgrid_100km_uc_{}".
format(version),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM \
results_nhsl_physical_exposure.nhsl_physical_exposure_indicators_hexgrid_100km_uc \
ORDER BY nhsl_physical_exposure_indicators_hexgrid_100km_uc."gridid_100" \
LIMIT {limit} \
OFFSET {offset}',
)
table.postgis2es()
# hexgrid global fabric
# table = utils.PostGISdataset(
# utils.PostGISConnection(),
# utils.ESConnection(settings={
# 'settings': {
# 'number_of_shards': 1,
# 'number_of_replicas': 0
# },
# 'mappings': {
# 'properties': {
# 'geometry': {
# 'type': 'geo_shape'
# }
# }
# }
# }),
# view="opendrr_nhsl_physical_exposure_indicators_hexgrid_global_fabric",
# sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
# FROM \
# results_nhsl_physical_exposure.nhsl_physical_exposure_indicators_hexgrid_global_fabric \
# ORDER BY nhsl_physical_exposure_indicators_hexgrid_global_fabric."gridid" \
# LIMIT {limit} \
# OFFSET {offset}'
# )
# table.postgis2es()
return
def main():
args = parse_args()
config = utils.get_config_params("config.ini")
version = config.get("es", "version")
if args.aggregation.lower() == "sauid":
aggregation = args.aggregation[0].lower()
else:
aggregation = args.aggregation
# index settings
if args.geometry == "geom_poly" or "geom":
table = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
"mappings": {"properties": {"geometry": {"type": "geo_shape"}}},
}
),
view="opendrr_nhsl_social_fabric_indicators_{agg}_{version}".format(
**{"agg": aggregation, "version": version}
),
sqlquerystring='SELECT *, ST_AsGeoJSON({geom}) \
FROM \
results_nhsl_social_fabric.nhsl_social_fabric_indicators_{agg} \
ORDER BY "{sort_field}" \
LIMIT {{limit}} \
OFFSET {{offset}}'.format(
**{
"geom": args.geometry,
"agg": aggregation,
"sort_field": args.sortfield,
}
),
)
elif args.geometry == "geom_point":
table = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
"mappings": {
"properties": {
"coordinates": {"type": "geo_point"},
"geometry": {"type": "geo_shape"},
}
},
}
),
view="opendrr_nhsl_social_fabric_indicators_{agg}_{version}".format(
**{"agg": args.aggregation[0].lower(), "version": version}
),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom_point) \
FROM \
results_nhsl_social_fabric.nhsl_social_fabric_indicators_{agg} \
ORDER BY "{sort_field}" \
LIMIT {{limit}} \
OFFSET {{offset}}'.format(
**{"agg": aggregation, "sort_field": args.sortfield}
),
)
table.postgis2es()
return
raise Exception
elif cfg.get_service_date == "on_demand":
is_changes_applied = utils.validate_graph_changes_applied(coverage_name)
if not is_changes_applied:
raise Exception
# If it's up - delete the old gtfs and osm files - only from AWS machines
if is_changes_applied and utils.is_aws_machine():
utils.delete_file_from_host(Path(os.getcwd()).parent / osm_file_path / osm_file_name)
utils.delete_file_from_host(Path(os.getcwd()).parent / gtfs_file_path / gtfs_file_name)
# config variables to be moved to config-file downstrem
default_coverage_name, coverage_name, navitia_docker_compose_file_path, navitia_docker_compose_file_name, \
navitia_docker_compose_default_file_name, gtfs_file_path, gtfs_zip_file_name = utils.get_config_params()
try:
# Get the docker service client
docker_client = utils.get_docker_service_client()
containers = docker_client.containers.list(filters={"name": "worker"})
if len(containers) == 0:
_log.error("Navitia docker containers are down, bringing them up with default coverage for processing")
utils.start_navitia_with_single_coverage(navitia_docker_compose_file_path,
navitia_docker_compose_default_file_name,
default_coverage_name)
containers = docker_client.containers.list(filters={"name": "worker"})
# Get the worker container
def main():
args = parse_args()
config = utils.get_config_params("config.ini")
version = config.get("es", "version")
# Create shakemap object and load to ES
dsraTable = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
"mappings": {
"properties": {
"coordinates": {"type": "geo_point"},
"geometry": {"type": "geo_shape"},
}
},
}
),
view="opendrr_dsra_{eqScenario}_shakemap_{version}".format(
**{"eqScenario": args.eqScenario, "version": version}
).lower(),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM results_dsra_{eqScenario}.dsra_{eqScenario}_shakemap \
ORDER BY dsra_{eqScenario}_shakemap."SiteID" \
LIMIT {{limit}} \
OFFSET {{offset}}'.format(
**{"eqScenario": args.eqScenario}
),
)
dsraTable.postgis2es()
# Create load 1km shakemap hexgrid
dsraTable = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
"mappings": {"properties": {"geometry": {"type": "geo_shape"}}},
}
),
view="opendrr_dsra_{eqScenario}_shakemap_hexgrid_1km_{version}".format(
**{"eqScenario": args.eqScenario, "version": version}
).lower(),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM results_dsra_{eqScenario}.dsra_{eqScenario}_sm_hg_1 \
ORDER BY dsra_{eqScenario}_sm_hg_1."gridid_1" \
LIMIT {{limit}} \
OFFSET {{offset}}'.format(
**{"eqScenario": args.eqScenario}
),
)
dsraTable.postgis2es()
# Create load 1km shakemap hexgrid unclipped
dsraTable = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
"mappings": {"properties": {"geometry": {"type": "geo_shape"}}},
}
),
view="opendrr_dsra_{eqScenario}_shakemap_hexgrid_1km_uc_{version}".format(
**{"eqScenario": args.eqScenario, "version": version}
).lower(),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM results_dsra_{eqScenario}.dsra_{eqScenario}_sm_hg_1_uc \
ORDER BY dsra_{eqScenario}_sm_hg_1_uc."gridid_1" \
LIMIT {{limit}} \
OFFSET {{offset}}'.format(
**{"eqScenario": args.eqScenario}
),
)
dsraTable.postgis2es()
# Create load 5km shakemap hexgrid
dsraTable = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
"mappings": {"properties": {"geometry": {"type": "geo_shape"}}},
}
),
view="opendrr_dsra_{eqScenario}_shakemap_hexgrid_5km_{version}".format(
**{"eqScenario": args.eqScenario, "version": version}
).lower(),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM results_dsra_{eqScenario}.dsra_{eqScenario}_sm_hg_5 \
ORDER BY dsra_{eqScenario}_sm_hg_5."gridid_5" \
LIMIT {{limit}} \
OFFSET {{offset}}'.format(
**{"eqScenario": args.eqScenario}
),
)
dsraTable.postgis2es()
# Create load 5km shakemap hexgrid unclipped
dsraTable = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
"mappings": {"properties": {"geometry": {"type": "geo_shape"}}},
}
),
view="opendrr_dsra_{eqScenario}_shakemap_hexgrid_5km_uc_{version}".format(
**{"eqScenario": args.eqScenario, "version": version}
).lower(),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM results_dsra_{eqScenario}.dsra_{eqScenario}_sm_hg_5_uc \
ORDER BY dsra_{eqScenario}_sm_hg_5_uc."gridid_5" \
LIMIT {{limit}} \
OFFSET {{offset}}'.format(
**{"eqScenario": args.eqScenario}
),
)
dsraTable.postgis2es()
# Create load 10km shakemap hexgrid
dsraTable = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
"mappings": {"properties": {"geometry": {"type": "geo_shape"}}},
}
),
view="opendrr_dsra_{eqScenario}_shakemap_hexgrid_10km_{version}".format(
**{"eqScenario": args.eqScenario, "version": version}
).lower(),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM results_dsra_{eqScenario}.dsra_{eqScenario}_sm_hg_10 \
ORDER BY dsra_{eqScenario}_sm_hg_10."gridid_10" \
LIMIT {{limit}} \
OFFSET {{offset}}'.format(
**{"eqScenario": args.eqScenario}
),
)
dsraTable.postgis2es()
# Create load 10km shakemap hexgrid unclipped
dsraTable = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
"mappings": {"properties": {"geometry": {"type": "geo_shape"}}},
}
),
view="opendrr_dsra_{eqScenario}_shakemap_hexgrid_10km_uc_{version}".format(
**{"eqScenario": args.eqScenario, "version": version}
).lower(),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM results_dsra_{eqScenario}.dsra_{eqScenario}_sm_hg_10_uc \
ORDER BY dsra_{eqScenario}_sm_hg_10_uc."gridid_10" \
LIMIT {{limit}} \
OFFSET {{offset}}'.format(
**{"eqScenario": args.eqScenario}
),
)
dsraTable.postgis2es()
# Create load 25km shakemap hexgrid
dsraTable = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
"mappings": {"properties": {"geometry": {"type": "geo_shape"}}},
}
),
view="opendrr_dsra_{eqScenario}_shakemap_hexgrid_25km_{version}".format(
**{"eqScenario": args.eqScenario, "version": version}
).lower(),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM results_dsra_{eqScenario}.dsra_{eqScenario}_sm_hg_25 \
ORDER BY dsra_{eqScenario}_sm_hg_25."gridid_25" \
LIMIT {{limit}} \
OFFSET {{offset}}'.format(
**{"eqScenario": args.eqScenario}
),
)
dsraTable.postgis2es()
# Create load 25km shakemap hexgrid unclipped
dsraTable = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
"mappings": {"properties": {"geometry": {"type": "geo_shape"}}},
}
),
view="opendrr_dsra_{eqScenario}_shakemap_hexgrid_25km_uc_{version}".format(
**{"eqScenario": args.eqScenario, "version": version}
).lower(),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM results_dsra_{eqScenario}.dsra_{eqScenario}_sm_hg_25_uc \
ORDER BY dsra_{eqScenario}_sm_hg_25_uc."gridid_25" \
LIMIT {{limit}} \
OFFSET {{offset}}'.format(
**{"eqScenario": args.eqScenario}
),
)
dsraTable.postgis2es()
# Create load 50km shakemap hexgrid unclipped
dsraTable = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
"mappings": {"properties": {"geometry": {"type": "geo_shape"}}},
}
),
view="opendrr_dsra_{eqScenario}_shakemap_hexgrid_50km_uc_{version}".format(
**{"eqScenario": args.eqScenario, "version": version}
).lower(),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM results_dsra_{eqScenario}.dsra_{eqScenario}_sm_hg_50_uc \
ORDER BY dsra_{eqScenario}_sm_hg_50_uc."gridid_50" \
LIMIT {{limit}} \
OFFSET {{offset}}'.format(
**{"eqScenario": args.eqScenario}
),
)
dsraTable.postgis2es()
# Create load 100km shakemap hexgrid unclipped
dsraTable = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
"mappings": {"properties": {"geometry": {"type": "geo_shape"}}},
}
),
view="opendrr_dsra_{eqScenario}_shakemap_hexgrid_100km_uc_{version}".format(
**{"eqScenario": args.eqScenario, "version": version}
).lower(),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM results_dsra_{eqScenario}.dsra_{eqScenario}_sm_hg_100_uc \
ORDER BY dsra_{eqScenario}_sm_hg_100_uc."gridid_100" \
LIMIT {{limit}} \
OFFSET {{offset}}'.format(
**{"eqScenario": args.eqScenario}
),
)
dsraTable.postgis2es()
return
#!/usr/bin/python
import time
import praw
import traceback
import re
import logger
import utils
import lookup
import profiler
import formatter
config = utils.get_config_params('reddit_bot')
# get the login info
with open(config['login_file']) as f:
for l in f:
setting = ''.join(l.split()).split(':')
config[setting[0]] = setting[1]
# posts we've searched
# key: hour the post was posted (0 for 12:00 AM - 12:59, etc)
# value: list of post_id
# this structure lets us easily remove lists of posts that are too old
_searched = {}
# submissions we've replied to
# key: submission_id
# value: list of dictionaries of hero info we have profiled
# ensures we don't post the same profile more than once per thread
_replied_to = {}
import sqlite3
import utils
import logger
config = utils.get_config_params('formatter')
def format_gear(gear):
""" given a dictionary of gear { item_slot: { item_info } }, return
a formatted string to send to a reddit post """
items = {}
intro = '\n\n###### \n\n****\n**Equipped Gear:**\n\n'
try:
# open a db connection
db = sqlite3.connect(config['database'])
db.row_factory = sqlite3.Row
db_cur = db.cursor()
table = config['item_table']
query = 'SELECT * FROM {t} WHERE name=\'{s}\''
# get the max order here (useful to remove secondary effects, etc)
max_order = int(config['max_order'])
for slot in gear:
name = _create_url(gear[slot]['name'], gear[slot]['url'])
i_type = gear[slot]['type']
item_info = u'> **{n} ({t})**'.format(n=name, t=i_type)
# because blizzard randomly uses a non-standard apostrophe sometimes
item_info = item_info.replace(u'\u2019', '\'')
import time
import sqlite3
import lookup
import utils
import logger
config = utils.get_config_params('profiler')
def get_gear(hero_data, region='us'):
""" given a dictionary of hero data, get the gear and return it in a
dictionary of { item_slot: { item_info } } use region to get proper
urls """
gear = {}
try:
# open a db connection
db = sqlite3.connect(config['database'])
db_cur = db.cursor()
table = config['item_table']
for slot, item in hero_data['items'].iteritems():
item_url = item['tooltipParams']
item_data = lookup.item_lookup(item_url, region=region)
if not item_data:
logger.error('Unable to load item {i}'.format(i=item_url))
utils.close_db(db)
return None
gear[slot] = {
import time
import traceback
import utils
config = utils.get_config_params('logging')
def get_file_path():
""" Returns the file path for the log file """
file_path = (config['log_location'], config['log_file'])
return '{path}/{file}'.format(path=file_path[0], file=file_path[1])
def debug(entry):
""" Log entry if log_level is set to debug or lower """
if _get_log_level() <= 0:
_log('DEBUG - {entry}'.format(entry=entry))
def info(entry):
""" Log entry if log_level is set to info or lower """
if _get_log_level() <= 1:
_log('INFO - {entry}'.format(entry=entry))
def warn(entry):
""" Log entry if log_level is set to warn or lower """
if _get_log_level() <= 2:
_log('WARN - {entry}'.format(entry=entry))
def error(entry):
def main(args):
""" Gets all eval.txt files for models in a given directory and compiles them into a single document (as well as single csv).
Also prints information on the best-performing model. """
eval_writer = open(os.path.join(args.models_dir, 'cumulative_evals-'+args.model_iteration+'.txt'), 'w')
eval_csv_writer = open(os.path.join(args.models_dir, 'cumulative_evals-'+args.model_iteration+'.csv'), 'w')
csv_header_written = False
csv_header = 'saved model path, best epoch, validation loss, validation perplexity, '
sys.stdout = Logger(os.path.join(args.models_dir, 'cumulative_evals-'+args.model_iteration+'.log'))
best_validation_loss = None
best_model_path = None
best_config = None
best_eval = None
validation_losses = []
perplexities = []
epochs = []
all_params = {}
modified_eval_types = ['simple_terms', 'colors', 'spatial_relations', 'dialogue', 'synonym']
agreement_types = ['next_placements', 'all_placements', 'all_removals']
eval_statistics = {}
eval_terms = [x.strip() for x in args.eval_terms.split(',')]
for config_file in glob(args.models_dir+'/**/config.txt', recursive=True):
_, config_params = get_config_params(config_file)
for param in config_params:
value = config_params[param]
if config_params[param] is None:
value = "None"
all_params[param] = type(value)
# iterate over model directories that have successfully been trained and evaluated
for config_file in glob(args.models_dir+'/**/config.txt', recursive=True):
print('Accumulating evals for model:', '/'.join(config_file.split('/')[:-1]))
if not os.path.exists(config_file.replace('config.txt','eval-'+args.model_iteration+'.txt')):
continue
model_path = ('/'.join(os.path.abspath(config_file).split("/")[:-1]))
content = '='*89+'\nsaved model path: '+model_path+'\n'
csv = model_path+','
# get the configuration of parameters used for this particular model
config_content, config_params = get_config_params(config_file)
with open(config_file.replace('config.txt','eval-'+args.model_iteration+'.txt'), 'r') as f:
eval_content = f.read()
content += '\n'+config_content+'\n'+eval_content
eval_writer.write(content+'\n')
# parse the model's eval.txt file
eval_values = {}
for line in eval_content.split('\n'):
if line.startswith('Best model found at') or line.startswith('Final model found at'):
epoch = int(line.split()[-1].replace('.',''))
eval_values["epoch"] = epoch
epochs.append(epoch)
elif line.startswith('Loss:'):
validation_loss = float(line.split()[-1])
eval_values["validation_loss"] = validation_loss
if not best_validation_loss or validation_loss <= best_validation_loss:
best_validation_loss = validation_loss
best_model_path = model_path
best_config = config_params
best_eval = eval_values
validation_losses.append(validation_loss)
elif line.startswith('Perplexity:'):
perplexity = float(line.split()[-1])
eval_values["validation_perplexity"] = perplexity
perplexities.append(perplexity)
if not eval_values.get('validation_perplexity'):
eval_values['validation_perplexity'] = -1
perplexities.append(-1)
csv += str(eval_values["epoch"])+',' + str(eval_values['validation_loss'])+',' + str(eval_values['validation_perplexity']*args.perplexity_factor)+','
for split in ['val', 'test']:
args_sfx = '-'+args.model_iteration+'-'+split+('' if not args.development_mode else '-development_mode')
args_sfx += '-multinomial' if args.decoding_strategy == 'multinomial' else '-beam_'+str(args.beam_size)+('-gamma_'+str(args.gamma) if args.gamma else '')
if not csv_header_written:
csv_header += 'mean utterance length ('+split+'), std dev ('+split+'), bleu-1 ('+split+'), bleu-2 ('+split+'), bleu-3 ('+split+'), bleu-4 ('+split+'), '
for eval_type in modified_eval_types:
csv_header += eval_type+'-bleu ('+split+'), '
for agreement_type in agreement_types:
csv_header += agreement_type+' agreement ('+split+'), '
if os.path.exists(config_file.replace('config.txt', 'mul-std-bleu'+args_sfx+'.csv')):
with open(config_file.replace('config.txt', 'mul-std-bleu'+args_sfx+'.csv')) as f:
mul_std_bleu = f.readline().strip()
csv += mul_std_bleu+','
mul_std_bleu = mul_std_bleu.split(',')
if 'mean utterance length ('+split+')' in eval_terms:
eval_values['mean utterance length ('+split+')'] = float(mul_std_bleu[0])
if 'std dev ('+split+')' in eval_terms:
eval_values['std dev ('+split+')'] = float(mul_std_bleu[1])
for i in range(4):
if 'bleu-'+str(i+1)+' ('+split+')' in eval_terms:
eval_values['bleu-'+str(i+1)+' ('+split+')'] = float(mul_std_bleu[i+2])
args_sfx = '-multinomial' if args.decoding_strategy == 'multinomial' else '-beam_'+str(args.beam_size)+('-gamma_'+str(args.gamma) if args.gamma else '')
generated_sentences_file = config_file.replace('config.txt', 'generated_sentences-best-'+split+args_sfx+'.txt')
parser = argparse.ArgumentParser()
parser.add_argument('generated_sentences_file', help='file of sentences generated by a model')
parser.add_argument('--simple_terms_file', default='../data/lexicons/simple-terms-redux.txt')
parser.add_argument('--colors_file', default='../data/lexicons/colors.txt')
parser.add_argument('--spatial_relations_file', default='../data/lexicons/spatial-relations.txt')
parser.add_argument('--dialogue_file', default='../data/lexicons/dialogue.txt')
parser.add_argument('--shapes_file', default='../data/lexicons/shapes.txt')
parser.add_argument('--synonyms_file', default='../data/lexicons/synonym_substitutions.json')
parser.add_argument('--with_simple_synonyms', default=True, action='store_true')
parser.add_argument('--with_utterance_synonyms', default=True, action='store_true')
parser.add_argument('--num_synonym_references', default=4)
parser.add_argument('--suppress_printing', default=True)
parser.add_argument('--output_file', default=None)
modified_bleu_args = parser.parse_args([generated_sentences_file])
modified_bleu_scores, agreements = evaluation_analysis.main(modified_bleu_args)
for eval_type in modified_eval_types:
header_str = eval_type+'-bleu ('+split+')'
if header_str in eval_terms:
eval_values[header_str] = modified_bleu_scores[eval_type]
csv += str(modified_bleu_scores[eval_type])+','
for agreement_type in agreement_types:
header_str = agreement_type+' agreement ('+split+')'
if header_str in eval_terms:
eval_values[header_str] = agreements[agreement_type]
csv += str(agreements[agreement_type])+','
else:
for i in range(9+len(modified_eval_types)):
csv += '-1,'
for param in all_params:
if config_params.get(param) is None:
default_value = '-1'
if all_params[param] == bool:
default_value = 'False'
elif all_params[param] == str:
default_value = 'None'
config_params[param] = default_value
pair_values = {'num_encoder_hidden_layers': None, 'num_decoder_hidden_layers': None, 'dropout_rnn': None, 'dropout_nae': None, 'dropout_counter': None}
for param, value in sorted(config_params.items()):
if param == 'load_dataset' or param == 'ignore_diff' or param == 'data_dir' or param == 'gold_configs_dir' or param == 'vocab_dir' or param == 'date_dir' or param == 'teacher_forcing_ratio' or param == 'seed' or param == 'num_workers' or param == 'strict' or param == 'suppress_logs' or param == 'visualize':
continue
if args.model_type == 'cnn_3d':
if 'pretrained' in param or param == 'add_builder_utterances' or param == 'augment_dataset' or param == 'augmentation_factor' or param == 'pretrained_and_augmented' or param == 'exactly_k' or param == 'strict' or param == 'ignore_diff' or 'world_state' in param or 'block_embedding' in param:
continue
if isinstance(value, str) and ',' in value:
value = '"'+value+'"'
csv += str(value)+','
if not csv_header_written:
csv_header += param+','
if param in pair_values and value is not None:
pair_values[param] = str(value)
elif param != 'hyperparameter_file':
append_eval_values(eval_statistics, param, value, eval_values)
append_eval_values(eval_statistics, 'encoder_decoder_layers', pair_values['num_encoder_hidden_layers']+','+pair_values['num_decoder_hidden_layers'], eval_values)
if pair_values['dropout_rnn'] is not None and pair_values['dropout_rnn'] != 'None':
header = 'dropout_rnn_counter'
pair_value = pair_values['dropout_counter']
if 'next_actions' in args.model_type:
header = 'dropout_rnn_nae'
pair_value = pair_values['dropout_nae']
append_eval_values(eval_statistics, header, pair_values['dropout_rnn']+','+pair_value, eval_values)
if not csv_header_written:
eval_csv_writer.write(csv_header[:-1]+'\n')
csv_header_written = True
eval_csv_writer.write(csv[:-1]+'\n')
if not best_model_path:
print("Error: no best model was found -- check that the model directories include an eval.txt file.")
sys.exit(0)
eval_writer.write('='*89)
eval_writer.close()
eval_csv_writer.close()
# print details of best overall model found
print("Best model found:", model_path)
for param in best_config:
print(param.ljust(25), best_config[param])
for value in best_eval:
print(value.ljust(25), best_eval[value])
# print statistics of min/max/std over all models
print('\nEpochs at which best models were found:')
print('\tmin:', np.min(epochs))
print('\tmax:', np.max(epochs))
print('\tstd:', np.std(epochs))
print('\nValidation losses:')
print('\tmin:', np.min(validation_losses))
print('\tmax:', np.max(validation_losses))
print('\tstd:', np.std(validation_losses))
print('\nPerplexities:')
print('\tmin:', np.min(perplexities))
print('\tmax:', np.max(perplexities))
print('\tstd:', np.std(perplexities))
print("\nWrote cumulative evaluation log to", os.path.join(args.models_dir, 'cumulative_evals-'+args.model_iteration+'.txt'))
print("Wrote cumulative evaluation csv to", os.path.join(args.models_dir, 'cumulative_evals-'+args.model_iteration+'.csv'), '\n')
for param in list(eval_statistics.keys()):
for value in list(eval_statistics[param].keys()):
if len(eval_statistics[param][value]) < len(eval_terms):
eval_statistics[param].pop(value)
if len(eval_statistics[param]) < 2:
eval_statistics.pop(param)
for param in eval_statistics:
print('Parameter:', param)
for value in eval_statistics[param]:
print('\tValue:', value)
for eval_term in eval_terms:
eval_values = eval_statistics[param][value][eval_term]
if len(eval_values) < 1:
continue
max_value = "{0:.3f}".format(max(eval_values))
mean = "{0:.3f}".format(np.mean(eval_values))
std = "{0:.3f}".format(np.std(eval_values))
print('\t\t', eval_term.ljust(30), 'max: '+max_value+' mean: '+mean+' std: '+std)
print()
sys.stdout = sys.__stdout__
import os
from uuid import uuid4
from utils import get_config_params, _run, clean_environ, init_logger
import logging
logger = logging.getLogger(__name__)
init_logger(logger)
VYOS_SHELL_API = get_config_params('bin', 'shell_api_path')
VYOS_SBIN_DIR = get_config_params('bin', 'vyos_sbin_dir')
VYOS_SAVE_SCRIPT = 'vyatta-save-config.pl'
# Create/Get the logger object
#logger = init_logger()
class SessionAlreadyExists(Exception): pass
class SetupSessionFailed(Exception): pass
class OperationFailed(Exception): pass
class SessionNotExists(Exception): pass
class Session(object):
"""
Return the session instance if exists. Else, create new one.
SessionAlreadyExists exception raised on the second instantiation.
"""
_ref = None
def __new__(cls, *args, **kw):
if cls._ref is not None:
raise SessionAlreadyExists('A session exist already !')
cls._ref = super(Session, cls).__new__(cls, *args, **kw)
return cls._ref
#!/usr/bin/python3
# =================================================================
# SPDX-License-Identifier: MIT
#
# Copyright (C) 2020-2021 Government of Canada
#
# Main Authors: Drew Rotheram <*****@*****.**>
# Joost van Ulden <*****@*****.**>
# =================================================================
import argparse
import utils
config = utils.get_config_params("config.ini")
version = config.get("es", "version")
def main():
# args = parse_args()
table = utils.PostGISTable(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"coordinates": {
import os
from uuid import uuid4
from utils import get_config_params, _run, clean_environ, init_logger
import logging
logger = logging.getLogger(__name__)
init_logger(logger)
VYOS_SHELL_API = get_config_params('bin', 'shell_api_path')
VYOS_SBIN_DIR = get_config_params('bin', 'vyos_sbin_dir')
VYOS_SAVE_SCRIPT = 'vyatta-save-config.pl'
# Create/Get the logger object
#logger = init_logger()
class SessionAlreadyExists(Exception):
pass
class SetupSessionFailed(Exception):
pass
class OperationFailed(Exception):
pass
class SessionNotExists(Exception):
pass
def main():
args = parse_args()
config = utils.get_config_params("config.ini")
version = config.get("es", "version")
# Create building level aggregation object and load to ES
dsraTable = utils.PostGISPointDataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"coordinates": {
"type": "geo_point"
},
"geometry": {
"type": "geo_shape"
},
}
},
}),
view="opendrr_dsra_{eqScenario}_indicators_b_{version}".format(
**{
"eqScenario": args.eqScenario,
"version": version
}).lower(),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom_point) \
FROM results_dsra_{eqScenario}.dsra_{eqScenario}_indicators_b \
ORDER BY dsra_{eqScenario}_indicators_b."AssetID" \
LIMIT {{limit}} \
OFFSET {{offset}}'.format(**{"eqScenario": args.eqScenario}),
)
dsraTable.postgis2es()
# Create Sauid level aggregation object and load to ES
dsraTable = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"geometry": {
"type": "geo_shape"
}
}
},
}),
view="opendrr_dsra_{eqScenario}_indicators_s_{version}".format(
**{
"eqScenario": args.eqScenario,
"version": version
}).lower(),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom_poly) \
FROM results_dsra_{eqScenario}.dsra_{eqScenario}_indicators_s \
ORDER BY dsra_{eqScenario}_indicators_s."Sauid" \
LIMIT {{limit}} \
OFFSET {{offset}}'.format(**{"eqScenario": args.eqScenario}),
)
dsraTable.postgis2es()
# Create CSD level aggregation object and load to ES
dsraTable = utils.PostGISdataset(
utils.PostGISConnection(),
utils.ESConnection(
settings={
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"geometry": {
"type": "geo_shape"
}
}
},
}),
view="opendrr_dsra_{eqScenario}_indicators_csd_{version}".format(
**{
"eqScenario": args.eqScenario,
"version": version
}).lower(),
sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
FROM results_dsra_{eqScenario}.dsra_{eqScenario}_indicators_csd \
ORDER BY dsra_{eqScenario}_indicators_csd."csduid" \
LIMIT {{limit}} \
OFFSET {{offset}}'.format(**{"eqScenario": args.eqScenario}),
)
dsraTable.postgis2es()
return
