def send_problems_report(problem_message_records):
"""Send report of problems that affect metadata syncing.
Args:
problem_message_records (dict): Mapping of provlem messages to collection of
records where that problem occurred during sync.
"""
body_template = """
<p>Problems were found that prevent certain metadata records from syncing
between Geoportal and the datasets.<br />
</p>
<table style="width:100%">
{}{}
</table>
"""
table_header = ("<tr><th>Count</th><th>ID</th><th>Title</th>" +
"<th>Problem</th><th>Dataset Path</th></tr>")
row_template = "<tr><td>{}</td><td>{}</td><td>{}</td><td>{}</td><td>{}</td></tr>"
if problem_message_records:
LOG.warning("Found syncing problem records: sending report.")
table_rows = []
i = 0
for message, records in sorted(problem_message_records.items()):
for record in sorted(records):
i += 1
table_rows.append(
row_template.format(i, record.id, record.title, message,
record.dataset_path))
KWARGS_PROBLEMS_REPORT_MESSAGE["body"] = body_template.format(
table_header, table_rows)
send_email(**KWARGS_PROBLEMS_REPORT_MESSAGE)
else:
LOG.info("No syncing problem records found. Not sending report.")
def publication_issues_message_etl():
"""Send message of issues that affect address publication."""
field_names = ["description", "postcomm", "address", "address_id"]
issues = sorted(
arcetl.attributes.as_iters(
dataset.TILLAMOOK_ADDRESS_POINT_ISSUES.path(),
field_names,
dataset_where_sql="ok_to_publish = 0",
))
LOG.warning("Found validation publication issues: sending email.")
table_header = "<tr>{}</tr>".format("".join("<th>{}</th>".format(column)
for column in field_names))
row_template = "<tr><td>{}</td><td>{}</td><td>{}</td><td>{}</td></tr>"
table_rows = "".join(row_template.format(*issue) for issue in issues)
KWARGS_ISSUES_MESSAGE["body"] = KWARGS_ISSUES_MESSAGE["body"].format(
table_header, table_rows)
send_email(**KWARGS_ISSUES_MESSAGE)
def warehouse_issues():
"""Run check for issues in RLIDGeo update."""
all_dataset_kwargs = {}
for tag in DATASET_KWARGS:
all_dataset_kwargs.update(DATASET_KWARGS[tag])
datasets = {
"existing":
sorted(
name.split(".", 1)[-1]
for name in arcetl.workspace.dataset_names(database.RLIDGEO.path)),
"listed":
sorted(all_dataset_kwargs),
"unlisted": [],
"listed_not_exist": [],
}
datasets["existing_lower"] = {
name.lower()
for name in datasets["existing"]
}
datasets["listed_lower"] = {name.lower() for name in datasets["listed"]}
for dataset_name in datasets["existing"]:
if dataset_name.lower() not in datasets["listed_lower"]:
datasets["unlisted"].append(dataset_name)
for dataset_name in datasets["listed"]:
if dataset_name.lower() not in datasets["existing_lower"]:
datasets["listed_not_exist"].append(dataset_name)
message_body = ""
if datasets["unlisted"]:
message_body += "<h2>Datasets not listed in DATASET_KWARGS</h2><ul>{}</ul>".format(
"".join("<li>{}</li>".format(item)
for item in datasets["unlisted"]))
if datasets["listed_not_exist"]:
message_body += "<h2>Datasets listed in DATASET_KWARGS that do not exist</h2><ul>{}</ul>".format(
"".join("<li>{}</li>".format(item)
for item in datasets["listed_not_exist"]))
if message_body:
LOG.info("Found update issues: sending email.")
send_email(subject="RLIDGeo Update Issues",
body=message_body,
body_format="HTML",
**KWARGS_ISSUES_MESSAGE)
else:
LOG.info("No update issues found.")
def send_publication_issues_message():
"""Send message of issues that affect address publication."""
keys = ["description", "city_name", "concat_address", "geofeat_id"]
issues = sorted(
arcetl.attributes.as_iters(
dataset.ADDRESS_ISSUES.path(),
field_names=keys,
dataset_where_sql="update_publication = 0",
))
table_header = "<tr>{}</tr>".format("".join("<th>{}</th>".format(key)
for key in keys))
row_template = "<tr><td>{}</td><td>{}</td><td>{}</td><td>{}</td></tr>"
if issues:
LOG.warning("Found validation publication issues: sending email.")
table_rows = "".join(row_template.format(*issue) for issue in issues)
KWARGS_ISSUES_MESSAGE["body"] = KWARGS_ISSUES_MESSAGE["body"].format(
table_header, table_rows)
send_email(**KWARGS_ISSUES_MESSAGE)
else:
LOG.info("No validation publication issues found. Not sending email.")
def send_new_lincom_address_message():
"""Send message for any recently-added address in the Lincoln PSAP area."""
keys = ["city_name", "concat_address", "geofeat_id", "initial_create_date"]
addresses = sorted(addr for addr in arcetl.attributes.as_iters(
dataset.SITE_ADDRESS.path("pub"),
field_names=keys,
dataset_where_sql="psap_code = 'LI' ",
) if (datetime.datetime.now() - addr[-1]).days < 15)
table_header = "<tr>{}</tr>".format("".join("<th>{}</th>".format(key)
for key in keys))
row_template = "<tr><td>{}</td><td>{}</td><td>{}</td><td>{}</td></tr>"
if addresses:
LOG.warning("Found new addresses in Lincoln PSAP area: sending email.")
table_rows = "".join(row_template.format(*addr) for addr in addresses)
KWARGS_NEW_LINCOLN_MESSAGE["body"] = KWARGS_NEW_LINCOLN_MESSAGE[
"body"].format(table_header, table_rows)
send_email(**KWARGS_NEW_LINCOLN_MESSAGE)
else:
LOG.info(
"No new addresses in Lincoln PSAP area found. Not sending email.")
def msag_update():
"""Run update for the Master Street Address Guide dataset in RLIDGeo warehouse."""
LOG.info("Start: Update MSAG dataset in RLIDGeo warehouse.")
unretired_where_sql = "expiration_date is null"
# Do not update MSAG if there will be a significant change.
count = {
"current":
arcetl.features.count(dataset.MSAG_RANGE.path("current")),
"previous":
arcetl.features.count(dataset.MSAG_RANGE.path("master"),
dataset_where_sql=unretired_where_sql),
}
if abs(count["current"] - count["previous"]) > count["previous"] * 0.01:
send_email(subject="RLIDGeo MSAG Update Issues",
body="""
<p>
Greater than 1% change in number of un-expired ranges; not applying
changes.
</p>
""",
body_format="HTML",
**KWARGS_ISSUES_MESSAGE)
msag_keys = [
"msag_id",
"emergency_service_number",
"parity_code",
"parity",
"from_house_number",
"to_house_number",
"pre_direction_code",
"street_name",
"street_type_code",
"city_code",
"city_name",
"effective_date",
"shape@",
]
msag_id_range = {
feature["msag_id"]: feature
for feature in arcetl.attributes.as_dicts(
dataset.MSAG_RANGE.path("current"), field_names=msag_keys)
}
feature_count = Counter(deleted=0)
most_recent_date = max(val["effective_date"]
for val in msag_id_range.values())
master_cursor = arcpy.da.UpdateCursor(
dataset.MSAG_RANGE.path("master"),
field_names=["msag_id", "expiration_date", "shape@"],
where_clause=unretired_where_sql,
)
# Collect IDs for add-check later.
master_ids = set()
LOG.info("Start: Update geometry & expiration dates.")
with master_cursor:
for msag_id, expiration_date, geom in master_cursor:
master_ids.add(msag_id)
if msag_id not in msag_id_range:
LOG.info("Range (ID=%s) not in current: set expiration date.",
msag_id)
expiration_date = most_recent_date
elif geom is None or not geom.equals(
msag_id_range[msag_id]["shape@"]):
LOG.info("Range (ID=%s) changed geometry.", msag_id)
geom = msag_id_range[msag_id]["shape@"]
else:
feature_count["unchanged"] += 1
continue
master_cursor.updateRow((msag_id, expiration_date, geom))
feature_count["altered"] += 1
LOG.info("End: Update.")
LOG.info("Start: Insert new ranges from current MSAG.")
master_cursor = arcpy.da.InsertCursor(dataset.MSAG_RANGE.path("master"),
field_names=msag_keys)
with master_cursor:
for msag_id, range_ in msag_id_range.items():
if msag_id not in master_ids:
LOG.info("Range (ID=%s) new in current: adding.", msag_id)
master_cursor.insertRow(tuple(range_[key]
for key in msag_keys))
feature_count["inserted"] += 1
LOG.info("End: Insert.")
record_dataset_update(
os.path.basename(dataset.MSAG_RANGE.path("master")),
feature_count,
metadata_path=dataset.METADATA_DATASET_UPDATE.path("RLIDGeo"),
)
LOG.info("End: Update.")