issue.setdefault("maint_notes")
issue.setdefault("maint_init_date", datetime.datetime.now())
etl.transform(
arcetl.features.insert_from_dicts,
insert_features=issues,
field_names=issues[0].keys(),
)
etl.update(
dataset.ADDRESS_ISSUES.path(),
id_field_names=["site_address_gfid", "description"],
)
# Jobs.
NIGHTLY_JOB = Job("Address_Issues_Nightly", etls=[issues_update])
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
available_names = {
key
for key in list(globals()) if not key.startswith("__")
}
pipeline_names = args.parse_args().pipelines
if pipeline_names and available_names.issuperset(pipeline_names):
pipelines = [globals()[arg] for arg in args.parse_args().pipelines]
"""Run update for current license usage."""
LOG.info("Start: Collect license usage from FlexNet License Manager.")
session = database.CPA_ADMIN.create_session()
names = (name
for name, in session.query(LicenseArcGISDesktop.internal_name))
for name in names:
session.add_all(
LicenseUsage(**usage) for usage in license_usage_info(name))
session.commit()
session.close()
LOG.info("End: Collect.")
# Jobs.
FIVE_MINUTE_JOB = Job("License_Usage", etls=[license_usage_update])
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
available_names = {
key
for key in list(globals()) if not key.startswith("__")
}
pipeline_names = args.parse_args().pipelines
if pipeline_names and available_names.issuperset(pipeline_names):
pipelines = [globals()[arg] for arg in args.parse_args().pipelines]
def locators_update():
"""Run update for map server locators/geocoders."""
for name in LOCATOR_NAMES:
locator_path = os.path.join(GEOCODE_PATH, name)
package_path = os.path.join(GEOCODE_PATH, "packaged", name + ".gcpk")
arcetl.workspace.build_locator(locator_path)
##TODO: Create arcetl.workspace.package_locator function, then use here.
old_overwrite_output = arcpy.env.overwriteOutput
arcpy.env.overwriteOutput = True
arcpy.PackageLocator_management(locator_path, package_path)
arcpy.env.overwriteOutput = old_overwrite_output
# Jobs.
WEEKLY_JOB = Job("Regional_Data_Warehouse_Weekly",
etls=[datasets_update, locators_update])
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
available_names = {
key
for key in list(globals()) if not key.startswith("__")
}
pipeline_names = args.parse_args().pipelines
if pipeline_names and available_names.issuperset(pipeline_names):
pipelines = [globals()[arg] for arg in args.parse_args().pipelines]
for _dataset in DATASETS:
arcetl.features.update_from_dicts(
dataset_path=_dataset.path("pub"),
update_features=source_rows(snapshot_db_path,
_dataset.path("source")),
id_field_names=_dataset.id_field_names,
field_names=_dataset.field_names,
delete_missing_features=False,
use_edit_session=False,
)
LOG.info("End: Update.")
# Jobs.
NIGHTLY_JOB = Job("CLMPO_GBF_Nightly", etls=[gbf_pub_update])
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
# Collect pipeline objects.
if args.parse_args().pipelines:
pipelines = [globals()[arg] for arg in args.parse_args().pipelines]
else:
pipelines = []
# Execute.
for pipeline in pipelines:
etl.load(dataset.ZONING_COUNTY.path("pub"))
##TODO: Auto-generate LCOGGeo.lcagadm.ZoningOverlay from overX fields & aliases. Also move to ETL_Load_A.
# Jobs.
BOUNDARY_DATASETS_JOB = Job(
"Planning_Development_Boundary_Datasets",
etls=[
# Pass 1.
metro_plan_boundary_etl,
nodal_development_area_etl,
plan_designation_city_etl,
willamette_river_greenway_etl,
zoning_city_etl,
# Pass 2.
plan_designation_county_etl,
zoning_county_etl,
# Pass 3.
plan_designation_etl,
zoning_etl,
],
)
TAXLOT_ZONING_JOB = Job("Taxlot_Zoning_Dataset", etls=[taxlot_zoning_etl])
# Execution.
def main():
]
transform.update_attributes_by_values(etl, value_kwargs)
# Build values: Concatenations.
etl.transform(
arcetl.attributes.update_by_function,
field_name="full_name",
function=concatenate_arguments,
field_as_first_arg=False,
arg_field_names=["predir", "name", "type", "sufdir"],
)
etl.load(dataset.TILLAMOOK_ROAD_CENTERLINE.path("pub"))
# Jobs.
NIGHTLY_JOB = Job("OEM_Tillamook_Datasets_Nightly",
etls=[production_datasets_etl])
WEEKLY_JOB = Job(
"OEM_Tillamook_Datasets_Weekly",
etls=[
emergency_service_zone_etl,
# Must addresses & roads after emergency service zones.
address_point_etl,
road_centerline_etl,
publication_issues_message_etl,
# Must run after addresses.
msag_ranges_current_etl,
# Must run after current MSAG ranges.
msag_update,
# Must run after *all* dataset ETLs.
metadata_tillamook_ecd_etl,
left join Report_Ordered as sorted
on report.row_rank - 1 = sorted.row_rank;
""",
]
kwargs = {
"dataset": "Addressing.dbo.SiteAddress_evw",
"report": "Addressing.dbo.Report_SiteAddress_MaintSummary",
}
for sql in sql_statements:
arcetl.workspace.execute_sql(sql.format(**kwargs),
database.ADDRESSING.path)
# Jobs.
MONTHLY_JOB = Job("Address_Reports_Monthly",
etls=[site_address_maint_summary_etl])
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
available_names = {
key
for key in list(globals()) if not key.startswith("__")
}
pipeline_names = args.parse_args().pipelines
if pipeline_names and available_names.issuperset(pipeline_names):
pipelines = [globals()[arg] for arg in args.parse_args().pipelines]
"datasets": ["info", "address"],
},
]
for kwargs in update_kwargs:
for key in kwargs["datasets"]:
LOG.info("Update %s in %s.", kwargs["field_names"][1],
dataset_path[key])
arcetl.features.update_from_iters(dataset_path[key],
id_field_names=["geofeat_id"],
delete_missing_features=False,
**kwargs)
# Jobs.
NIGHTLY_JOB = Job("Address_Assess_Tax_Info_Nightly",
etls=[assess_tax_info_update])
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
available_names = {
key
for key in list(globals()) if not key.startswith("__")
}
pipeline_names = args.parse_args().pipelines
if pipeline_names and available_names.issuperset(pipeline_names):
pipelines = [globals()[arg] for arg in args.parse_args().pipelines]
join_dataset_path=hydrants_copy.path,
join_field_name=name,
on_field_pairs=[("facility_intid", "hydrant_id")],
)
# Remove features without a near-hydrant (should not happen).
etl.transform(
arcetl.features.delete, dataset_where_sql="facility_intid is null"
)
etl.load(dataset.SITE_ADDRESS_CLOSEST_HYDRANT.path())
# Jobs.
WEEKLY_JOB = Job(
"Service_Facility_Datasets_Weekly",
etls=[closest_hydrant_etl, closest_fire_station_etl]
)
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
available_names = {key for key in list(globals()) if not key.startswith("__")}
pipeline_names = args.parse_args().pipelines
if pipeline_names and available_names.issuperset(pipeline_names):
pipelines = [globals()[arg] for arg in args.parse_args().pipelines]
for pipeline in pipelines:
# "one_way",
# "flow",
# ],
# **kwargs
# )
# arcetl.attributes.update_by_function(
# field_name="mailcity", function=city_name_case, **kwargs
# )
# update.force_title_case(field_names=["county"], **kwargs)
# Jobs.
NIGHTLY_JOB = Job(
"Production_Datasets_Nightly",
etls=[
proposed_street_name_update, mailing_city_area_etl,
production_update_etl
],
)
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
available_names = {
key
for key in list(globals()) if not key.startswith("__")
}
-S {name}: server instance name.
-E: trusted connection.
-d {name}: database name.
-b: terminate batch job if errors.
-Q "{string}": query string.
"""
call_string = " ".join([
"sqlcmd.exe -S {} -E -d RLID -b -Q",
'"exec dbo.proc_load_GIS @as_return_msg = null, @ai_return_code = null;"',
])
subprocess.check_call(call_string.format("gisql113"))
# Jobs.
WEEKLY_JOB = Job("RLID_GIS_Load_Weekly", etls=[load_rlid_gis])
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
available_names = {
key
for key in list(globals()) if not key.startswith("__")
}
pipeline_names = args.parse_args().pipelines
if pipeline_names and available_names.issuperset(pipeline_names):
pipelines = [globals()[arg] for arg in args.parse_args().pipelines]
def general_land_use_codes_etl():
"""Run ETL for general land use codes."""
with arcetl.ArcETL("General Land Use Codes") as etl:
etl.extract(dataset.LAND_USE_CODES_USE_CODES.path("maint"))
etl.load(dataset.LAND_USE_CODES_USE_CODES.path("pub"))
# Jobs.
WEEKLY_JOB = Job(
"Land_Use_Datasets",
etls=[
# Pass 1.
building_etl,
detailed_land_use_codes_etl,
general_land_use_codes_etl,
# Pass 2.
land_use_area_etl,
],
)
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
available_names = {
key
)
with conn:
path.archive_directory(
directory_path=DELIVERABLES_PATH,
archive_path=zip_path,
directory_as_base=False,
archive_exclude_patterns=[".lock", ".zip"],
)
zip_url = url.RLID_MAPS + "Download/" + zip_name
send_links_email(urls=[zip_url], **MESSAGE_KWARGS)
# Jobs.
MONTHLY_JOB = Job("LCSO_CAD_Delivery", etls=[lcso_cad_datasets_etl])
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
pipelines = [globals()[arg] for arg in args.parse_args().pipelines]
for pipeline in pipelines:
execute_pipeline(pipeline)
if __name__ == "__main__":
message_body += "<h2>Datasets listed in DATASET_KWARGS that do not exist</h2><ul>{}</ul>".format(
"".join("<li>{}</li>".format(item)
for item in datasets["listed_not_exist"]))
if message_body:
LOG.info("Found update issues: sending email.")
send_email(subject="RLIDGeo Update Issues",
body=message_body,
body_format="HTML",
**KWARGS_ISSUES_MESSAGE)
else:
LOG.info("No update issues found.")
# Jobs.
MONTHLY_JOB = Job("RLIDGeo_Monthly", etls=[snapshot_etl])
WEEKLY_JOB = Job(
"RLIDGeo_Weekly",
etls=[
datasets_primary_update,
datasets_secondary_update,
msag_update,
warehouse_issues,
],
)
# Execution.
def main():
arcetl.features.delete_by_id,
delete_ids=ids["hold"],
id_field_names="site_address_gfid",
)
LOG.info("%s addresses held from publication", len(ids["hold"]))
LOG.info("%s addresses rolled-back from publication",
len(ids["rollback"]))
if any([ids["hold"], ids["rollback"]]):
send_publication_issues_message()
etl.load(dataset.SITE_ADDRESS.path("pub"))
send_new_lincom_address_message()
# Jobs.
WEEKLY_JOB = Job("Address_Datasets_Weekly",
etls=[site_address_etl, facility_etl])
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
available_names = {
key
for key in list(globals()) if not key.startswith("__")
}
pipeline_names = args.parse_args().pipelines
if pipeline_names and available_names.issuperset(pipeline_names):
pipelines = [globals()[arg] for arg in args.parse_args().pipelines]
result_response = session.post(url=url.DEQ_WEB +
'wq/onsite/sdsresults.asp',
params=FORM_PAYLOAD,
headers={'Referer': form_response.url})
csv_relpath = re.search(CSV_HREF_PATTERN,
result_response.text).group(0).split('"')[1]
csv_url = requests.compat.urljoin(url.DEQ_WEB, csv_relpath)
csv_response = session.get(url=csv_url,
headers={'Referer': result_response.url})
with open(LP_DEQ_CSV_PATH, 'wb') as csvfile:
csvfile.write(csv_response.content)
# Jobs.
INPUT_HOURLY_JOB = Job('ePermitting_Input_Hourly', etls=(lp_deq_etl, ))
# Execution.
DEFAULT_PIPELINES = (INPUT_HOURLY_JOB, )
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument('pipelines', nargs='*', help="Pipeline(s) to run")
# Collect pipeline objects.
if args.parse_args().pipelines:
pipelines = tuple(globals()[arg]
for arg in args.parse_args().pipelines)
else:
This script should only be used for updating geodatabase datasets & other managed
data stores. Purely file-based formats like shapefiles are best updated via
`file_datasets_etl`, for reasons related to locking mechanisms.
"""
conn = credential.UNCPathCredential(DATA_PATH, **credential.CPA_MAP_SERVER)
with conn:
for kwargs in DATASET_KWARGS_WEEKLY:
if kwargs.get("source_path"):
transform.etl_dataset(**kwargs)
# Jobs.
DAILY_JOB = Job("GIMAP_Datasets_Daily", etls=[daily_datasets_etl])
WEEKLY_01_JOB = Job("GIMAP_Datasets_Weekly_01", etls=[file_datasets_etl])
WEEKLY_02_JOB = Job(
"GIMAP_Datasets_Weekly_02",
etls=[rlidgeo_datasets_etl, weekly_datasets_etl, locators_etl],
)
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
LOG.info("Starting compression of geodatabases.")
for geodatabase in randomized(database.GISRV106_DATABASES):
if not geodatabase.compress:
continue
arcetl.workspace.compress(geodatabase.path)
LOG.info("Geodatabases compression complete.")
# Jobs.
NIGHTLY_JOB = Job(
"Geodatabase_Maintenance_Nightly",
etls=[
geodatabase_compress_etl,
geodatabase_backup_schema_etl,
geodatabase_backup_datasets_etl,
geodatabase_backup_build_etl,
],
)
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
available_names = {
key
for key in list(globals()) if not key.startswith("__")
result_key = update_tax_map(
staging_path, rlid_path, release_date, archive_previous=True
)
count[result_key] += 1
document.log_state_counts(count, documents_type="tax maps")
# Finally, update tax map repository currency date (if we placed any).
if count["updated"]:
rlid_data_currency_setter("Tax Maps", max(file_name_release_date.values()))
elapsed(start_time, LOG)
LOG.info("END SCRIPT: Update")
# Jobs.
DAILY_JOB = Job("RLID_Documents_Daily", etls=[tax_maps_staging_update, tax_maps_update])
NIGHTLY_JOB = Job(
"RLID_Documents_Nightly",
etls=[property_cards_staging_update, property_cards_update],
)
WEEKLY_JOB = Job(
"RLID_Documents_Weekly",
etls=[petition_documents_update, plat_maps_update, tax_maps_not_in_source_etl],
)
# Execution.
WEEKLY_JOB = Job(
"Boundary_Datasets_Weekly",
etls=[
# City boundaries.
annexation_history_etl,
incorporated_city_limits_etl,
ugb_etl,
ugb_line_etl,
# Education boundaries.
elementary_school_area_etl,
elementary_school_line_etl,
high_school_area_etl,
high_school_line_etl,
middle_school_area_etl,
middle_school_line_etl,
school_district_etl,
# Election boundaries.
city_ward_etl,
county_commissioner_dist_etl,
election_precinct_etl,
epud_subdistrict_etl,
eweb_commissioner_etl,
lcc_board_zone_etl,
swc_district_etl,
state_representative_dist_etl,
state_senator_district_etl,
# Other/miscellaneous boundaries.
zip_code_area_etl,
],
)
)
# Assign effective date for new ranges.
etl.transform(
arcetl.attributes.update_by_function,
field_name="effective_date",
function=datetime.date.today,
field_as_first_arg=False,
dataset_where_sql="effective_date is null",
)
etl.load(dataset.MSAG_RANGE.path("current"))
# Jobs.
WEEKLY_JOB = Job("MSAG_Weekly", etls=[msag_ranges_current_etl])
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
available_names = {key for key in list(globals()) if not key.startswith("__")}
pipeline_names = args.parse_args().pipelines
if pipeline_names and available_names.issuperset(pipeline_names):
pipelines = [globals()[arg] for arg in args.parse_args().pipelines]
for pipeline in pipelines:
execute_pipeline(pipeline)
"""Run ETL for extracts forEugene Parcel Database CESQL024 (dev: CESQL023).
Confirmed by a message from Barry Bogart (2014-10-27), the parcel database is only
in use to support the legacy app Special Assessments/Accounts Receivable (SPAARS).
Barry: "SPAARS is an older app that will presumably be replaced before too many
years from now, but I am not aware of any active project at this time."
"""
for table_name, sql in EXTRACT_TABLE_QUERY_SQL.items():
file_path = os.path.join(path.REGIONAL_STAGING, "EugeneParcelDB",
table_name + ".txt")
extract_database_file(file_path, sql)
# Jobs.
WEEKLY_JOB = Job("Eugene_Parcel_Database_Weekly",
etls=[eugene_parcel_database_etl])
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
available_names = {
key
for key in list(globals()) if not key.startswith("__")
}
pipeline_names = args.parse_args().pipelines
if pipeline_names and available_names.issuperset(pipeline_names):
pipelines = [globals()[arg] for arg in args.parse_args().pipelines]
features = ((maptaxlot, box.esri_geometry)
for maptaxlot, box in bound_boxes)
with arcetl.ArcETL("Taxlot Focus Boxes - {}".format(
scale.title())) as etl:
etl.init_schema(dataset.TAXLOT_FOCUS_BOX.path(scale))
etl.transform(
arcetl.features.insert_from_iters,
insert_features=features,
field_names=["maptaxlot", "shape@"],
)
etl.load(dataset.TAXLOT_FOCUS_BOX.path(scale))
# Jobs.
WEEKLY_JOB = Job("Spatial_Reference_Datasets_Weekly",
etls=[taxlot_focus_box_etl])
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
available_names = {
key
for key in list(globals()) if not key.startswith("__")
}
pipeline_names = args.parse_args().pipelines
if pipeline_names and available_names.issuperset(pipeline_names):
pipelines = [globals()[arg] for arg in args.parse_args().pipelines]
arg_field_names=["approx_acres", "approx_taxlot_acres"],
)
# Remove minimal overlays.
etl.transform(arcetl.features.delete,
dataset_where_sql="taxlot_area_ratio <= 0.001")
etl.load(dataset.TAXLOT_FIRE_PROTECTION.path())
# Jobs.
BOUNDARY_DATASETS_JOB = Job(
"Public_Safety_Boundary_Datasets",
etls=[
# Pass 1.
ambulance_service_area_etl,
fire_protection_area_etl,
psap_area_etl,
# Pass 2.
emergency_service_zone_etl,
],
)
TAXLOT_FIRE_PROTECTION_JOB = Job("Taxlot_Fire_Protection_Dataset",
etls=[taxlot_fire_protection_etl])
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
with csvfile:
csvwriter = csv.writer(csvfile)
csvwriter.writerow(["document_id", "document_path", "check_time"])
for doc_path in rlid_record_paths():
if not os.path.exists(doc_path):
doc_id = os.path.splitext(os.path.basename(doc_path))[0]
csvwriter.writerow((doc_id, doc_path, check_time))
missing_count += 1
LOG.info("Found %s missing documents.", missing_count)
LOG.info("End: Compile.")
elapsed(start_time, LOG)
# Jobs.
HOURLY_JOB = Job("RLID_Documents_Deeds_Records_Hourly",
etls=[deeds_records_update])
WEEKLY_JOB = Job("RLID_Documents_Deeds_Records_Weekly",
etls=[missing_in_rlid_etl])
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
available_names = {
key
for key in list(globals()) if not key.startswith("__")
}
"""
conn = credential.UNCPathCredential(path.RLID_MAPS_DATA_SHARE,
**credential.CPA_MAP_SERVER)
with conn:
for gdb_relpath in sorted(KWARGS_MONTHLY_DATASETS):
LOG.info("Update datasets in %s", gdb_relpath)
gdb_path = os.path.join(DATA_PATH, gdb_relpath)
for kwargs in KWARGS_MONTHLY_DATASETS[gdb_relpath]:
kwargs['output_path'] = os.path.join(gdb_path,
kwargs['output_name'])
transform.etl_dataset(**kwargs)
# Jobs.
NIGHTLY_JOB = Job('OEM_Tillamook_Service_Datasets_Nightly',
etls=(service_datasets_nightly_etl, ))
MONTHLY_JOB = Job('OEM_Tillamook_Service_Datasets_Monthly',
etls=(service_datasets_monthly_etl, ))
# Execution.
DEFAULT_PIPELINES = ()
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument('pipelines', nargs='*', help="Pipeline(s) to run")
# Collect pipeline objects.
pipelines = (tuple(globals()[arg] for arg in args.parse_args().pipelines)
##TODO: Check dicts for counts. If all/most of a column is None, throw error & don't write.
def postal_info_update():
"""Run update for address postal info dataset."""
arcetl.features.update_from_dicts(
dataset.ADDRESS_POSTAL_INFO.path(),
update_features=postal_info_rows,
id_field_names=dataset.ADDRESS_POSTAL_INFO.id_field_names,
field_names=dataset.ADDRESS_POSTAL_INFO.field_names,
)
# Jobs.
WEEKLY_JOB = Job("Address_Postal_Info_Weekly",
etls=[address_workfile_etl, postal_info_update])
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
available_names = {
key
for key in list(globals()) if not key.startswith("__")
}
pipeline_names = args.parse_args().pipelines
if pipeline_names and available_names.issuperset(pipeline_names):
pipelines = [globals()[arg] for arg in args.parse_args().pipelines]
directory_as_base=True,
archive_exclude_patterns=[".lock"],
)
zip_url = url.RLID_MAPS + "Download/" + zip_name
send_message_tillamook(zip_url,
metadata_where_sql="in_tillamook = 1",
**TILLAMOOK_MESSAGE_KWARGS)
# Jobs.
MONTHLY_JOB = Job(
"OEM_Deliveries_Monthly",
etls=[
tillamook_911_delivery_etl,
tillamook_delivery_etl,
oem_lane_delivery_etl,
oem_tillamook_delivery_etl,
],
)
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
pipelines = [globals()[arg] for arg in args.parse_args().pipelines]
for pipeline in pipelines:
execute_pipeline(pipeline)
}
if owner not in owners:
owners.append(owner)
LOG.info("End: Collect.")
etl.init_schema(dataset.TAXLOT_OWNER.path("pub"))
etl.transform(
arcetl.features.insert_from_dicts,
insert_features=owners,
field_names=rlid_field_name.keys(),
)
etl.load(dataset.TAXLOT_OWNER.path("pub"))
# Jobs.
DAILY_JOB = Job("Assess_Tax_Datasets_Daily", etls=[comparable_sale_taxlot_etl])
WEEKLY_JOB = Job(
"Assess_Tax_Datasets_Weekly",
etls=[
plat_etl,
plss_dlc_etl,
plss_quarter_section_etl,
plss_section_etl,
plss_township_etl,
tax_code_area_etl,
taxlot_owner_etl,
],
)
# Execution.
# conn = credential.UNCPathCredential(path.XX_SHARE, **credential.XX_SHARE)
# with conn, arcetl.ArcETL("##TODO: Update Name") as etl:
with arcetl.ArcETL("##TODO: Update Name") as etl:
##TODO: Add extract keyword arguments (if necessary).
etl.extract("##TODO: dataset_path")
##TODO: Add transform keyword arguments (if necessary).
etl.transform("##TODO: transformation (e.g. arcetl.features.dissolve)")
##TODO: Add load keyword arguments (if necessary).
etl.update("##TODO: dataset_path", "##TODO: id_field_names")
# Jobs.
# Match name to ETL-Job metadata table (case-insensitive). etls must be an iterable.
TEMPLATE_JOB = Job("Job_Name", etls=[template_etl])
# Execution.
def main():
"""Script execution code."""
args = argparse.ArgumentParser()
args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
available_names = {key for key in list(globals()) if not key.startswith("__")}
pipeline_names = args.parse_args().pipelines
if pipeline_names and available_names.issuperset(pipeline_names):
pipelines = [globals()[arg] for arg in args.parse_args().pipelines]
for pipeline in pipelines:
execute_pipeline(pipeline)