def snapshot_etl():
"""Run ETL for snapshot of the RLIDGeo geodatabase."""
name = "RLIDGeo_" + datestamp()
xml_path = arcetl.workspace.create_geodatabase_xml_backup(
geodatabase_path=database.RLIDGEO.path,
output_path=os.path.join(tempfile.gettempdir(), name + ".xml"),
include_data=False,
include_metadata=True,
)
snapshot_path = arcetl.workspace.create_file_geodatabase(
geodatabase_path=os.path.join(path.REGIONAL_DATA, "history",
name + ".gdb"),
xml_workspace_path=xml_path,
include_xml_data=False,
)
os.remove(xml_path)
# Push datasets to snapshot (ignore certain patterns).
for name in arcetl.workspace.dataset_names(database.RLIDGEO.path):
copy_name = name.split(".")[-1]
if any(pattern.lower() in copy_name.lower()
for pattern in IGNORE_PATTERNS_RLIDGEO_SNAPSHOT):
# Need to delete ignored dataset in snapshot (created by the XML).
arcetl.dataset.delete(os.path.join(snapshot_path, copy_name))
continue
transform.etl_dataset(
source_path=os.path.join(database.RLIDGEO.path, name),
output_path=os.path.join(snapshot_path, copy_name),
)
arcetl.workspace.compress(snapshot_path)
def oem_tillamook_delivery_etl():
"""Run ETL for OEM-Tillamook delivery."""
name = "OEM_Tillamook"
gdb_path = os.path.join(PATH["tillamook_deliverables"], name + ".gdb")
for dataset_name, kwargs in OEM_TILLAMOOK_DATASET_KWARGS.items():
kwargs["output_path"] = os.path.join(gdb_path, dataset_name)
transform.etl_dataset(**kwargs)
zip_name = "{}_{}.zip".format(name, datestamp())
zip_path = os.path.join(PATH["tillamook_deliverables"], zip_name)
path.archive_directory(
directory_path=gdb_path,
archive_path=zip_path,
directory_as_base=True,
archive_exclude_patterns=[".lock"],
)
send_links_email(urls=[zip_path], **OEM_TILLAMOOK_MESSAGE_KWARGS)
def lcso_cad_datasets_etl():
"""Run ETL for LSCO CAD delivery datasets."""
for dataset_name, kwargs in DATASET_KWARGS.items():
kwargs["output_path"] = os.path.join(DELIVERABLES_PATH, dataset_name + ".shp")
transform.etl_dataset(**kwargs)
zip_name = "LCSO_CAD_{}.zip".format(datestamp())
zip_path = os.path.join(path.RLID_MAPS_WWW_SHARE, "Download", zip_name)
conn = credential.UNCPathCredential(
path.RLID_MAPS_WWW_SHARE, **credential.CPA_MAP_SERVER
)
with conn:
path.archive_directory(
directory_path=DELIVERABLES_PATH,
archive_path=zip_path,
directory_as_base=False,
archive_exclude_patterns=[".lock", ".zip"],
)
zip_url = url.RLID_MAPS + "Download/" + zip_name
send_links_email(urls=[zip_url], **MESSAGE_KWARGS)
def tillamook_delivery_etl():
"""Run ETL for Tillamook delivery."""
name = "Tillamook"
gdb_path = os.path.join(PATH["tillamook_deliverables"], name + ".gdb")
for dataset_name, kwargs in chain(TILLAMOOK_DATASET_KWARGS.items(),
TILLAMOOK_GIS_DATASET_KWARGS.items()):
kwargs["output_path"] = os.path.join(gdb_path, dataset_name)
transform.etl_dataset(**kwargs)
zip_name = "{}_{}.zip".format(name, datestamp())
zip_path = os.path.join(path.RLID_MAPS_WWW_SHARE, "Download", zip_name)
conn = credential.UNCPathCredential(path.RLID_MAPS_WWW_SHARE,
**credential.CPA_MAP_SERVER)
with conn:
path.archive_directory(
directory_path=gdb_path,
archive_path=zip_path,
directory_as_base=True,
archive_exclude_patterns=[".lock"],
)
zip_url = url.RLID_MAPS + "Download/" + zip_name
send_message_tillamook(zip_url,
metadata_where_sql="in_tillamook = 1",
**TILLAMOOK_MESSAGE_KWARGS)