def erase(dataset_path, erase_dataset_path, **kwargs):
"""Erase feature geometry where it overlaps erase-dataset geometry.
Args:
dataset_path (str): Path of the dataset.
erase_dataset_path (str): Path of the dataset defining the erase-area.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
erase_where_sql (str): SQL where-clause for erase-dataset subselection.
tolerance (float): Tolerance for coincidence, in dataset's units.
use_edit_session (bool): Flag to perform updates in an edit session. Default is
False.
log_level (str): Level to log the function at. Default is 'info'.
Returns:
str: Path of the dataset updated.
"""
kwargs.setdefault('dataset_where_sql')
kwargs.setdefault('erase_where_sql')
kwargs.setdefault('tolerance')
kwargs.setdefault('use_edit_session', False)
log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info'))
log(
"Start: Erase features in %s where overlapping %s.",
dataset_path,
erase_dataset_path,
)
meta = {'dataset': arcobj.dataset_metadata(dataset_path)}
view = {
'dataset': arcobj.DatasetView(dataset_path,
kwargs['dataset_where_sql']),
'erase': arcobj.DatasetView(erase_dataset_path,
kwargs['erase_where_sql']),
}
temp_output_path = unique_path('output')
with view['dataset'], view['erase']:
arcpy.analysis.Erase(
in_features=view['dataset'].name,
erase_features=view['erase'].name,
out_feature_class=temp_output_path,
cluster_tolerance=kwargs['tolerance'],
)
session = arcobj.Editor(meta['dataset']['workspace_path'],
kwargs['use_edit_session'])
with session:
delete(dataset_path,
dataset_where_sql=kwargs['dataset_where_sql'],
log_level=None)
insert_from_path(dataset_path,
insert_dataset_path=temp_output_path,
log_level=None)
dataset.delete(temp_output_path, log_level=None)
log("End: Erase.")
return dataset_path
def is_valid(dataset_path):
"""Check whether dataset exists/is valid.
Args:
dataset_path (str): Path of the dataset.
Returns:
bool: True if dataset is valid, False otherwise.
"""
valid = (dataset_path and arcpy.Exists(dataset_path)
and dataset_metadata(dataset_path)["is_table"])
return valid
def project(dataset_path, output_path, spatial_reference_item=4326, **kwargs):
"""Project dataset features to a new dataset.
Args:
dataset_path (str): Path of the dataset.
output_path (str): Path of the output dataset.
spatial_reference_item: Item from which the spatial reference of the output
geometry will be derived. Default is 4326 (EPSG code for unprojected WGS84).
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
log_level (str): Level to log the function at. Default is "info".
Returns:
str: Path of the converted dataset.
"""
kwargs.setdefault("dataset_where_sql")
meta = {"spatial": spatial_reference_metadata(spatial_reference_item)}
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Project %s to srid=%s in %s.",
dataset_path,
meta["spatial"]["object"].factoryCode,
output_path,
)
meta["dataset"] = dataset_metadata(dataset_path)
"""Project tool cannot output to an in-memory workspace (will throw error 000944).
This is not a bug. Esri"s Project documentation (as of v10.6) specifically states:
"The in_memory workspace is not supported as a location to write the output
dataset."
https://desktop.arcgis.com/en/arcmap/latest/tools/data-management-toolbox/project.htm
https://pro.arcgis.com/en/pro-app/tool-reference/data-management/project.htm
To avoid all this ado, using create to clone a (reprojected)
dataset & insert features into it.
"""
dataset.create(
dataset_path=output_path,
field_metadata_list=meta["dataset"]["user_fields"],
geometry_type=meta["dataset"]["geometry_type"],
spatial_reference_item=meta["spatial"]["object"],
log_level=None,
)
features.insert_from_path(
dataset_path=output_path,
insert_dataset_path=dataset_path,
field_names=meta["dataset"]["user_fields"],
insert_where_sql=kwargs["dataset_where_sql"],
log_level=None,
)
log("End: Project.")
return output_path
def keep_by_location(dataset_path, location_dataset_path, **kwargs):
"""Keep features where geometry overlaps location-dataset geometry.
Args:
dataset_path (str): Path of the dataset.
location_dataset_path (str): Path of location-dataset.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
location_where_sql (str): SQL where-clause for location-dataset subselection.
use_edit_session (bool): Flag to perform updates in an edit session. Default is
False.
log_level (str): Level to log the function at. Default is 'info'.
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault('dataset_where_sql')
kwargs.setdefault('location_where_sql')
kwargs.setdefault('use_edit_session', False)
log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info'))
log(
"Start: Keep features in %s where overlapping %s.",
dataset_path,
location_dataset_path,
)
meta = {'dataset': arcobj.dataset_metadata(dataset_path)}
session = arcobj.Editor(meta['dataset']['workspace_path'],
kwargs['use_edit_session'])
view = {
'dataset':
arcobj.DatasetView(dataset_path, kwargs['dataset_where_sql']),
'location':
arcobj.DatasetView(location_dataset_path,
kwargs['location_where_sql']),
}
with session, view['dataset'], view['location']:
arcpy.management.SelectLayerByLocation(
in_layer=view['dataset'].name,
overlap_type='intersect',
select_features=view['location'].name,
selection_type='new_selection',
)
arcpy.management.SelectLayerByLocation(
in_layer=view['dataset'].name, selection_type='switch_selection')
feature_count = delete(view['dataset'].name, log_level=None)
for key in ['deleted', 'unchanged']:
log("%s features %s.", feature_count[key], key)
log("End: Keep.")
return feature_count
def delete_by_id(dataset_path, delete_ids, id_field_names, **kwargs):
"""Delete features in dataset with given IDs.
Note:
There is no guarantee that the ID field(s) are unique.
Use ArcPy cursor token names for object IDs and geometry objects/properties.
Args:
dataset_path (str): Path of the dataset.
delete_ids (iter): Collection of feature IDs.
id_field_names (iter, str): Name(s) of the ID field/key(s).
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
use_edit_session (bool): Flag to perform updates in an edit session. Default is
False.
log_level (str): Level to log the function at. Default is 'info'.
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault('use_edit_session', False)
log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info'))
log("Start: Delete features in %s with given IDs.", dataset_path)
meta = {'dataset': arcobj.dataset_metadata(dataset_path)}
keys = {'id': list(contain(id_field_names))}
if inspect.isgeneratorfunction(delete_ids):
delete_ids = delete_ids()
ids = {'delete': {tuple(contain(_id)) for _id in delete_ids}}
feature_count = Counter()
session = arcobj.Editor(meta['dataset']['workspace_path'],
kwargs['use_edit_session'])
##TODO: Preserve possible generators!
if ids['delete']:
cursor = arcpy.da.UpdateCursor(dataset_path, field_names=keys['id'])
with session, cursor:
for row in cursor:
_id = tuple(row)
if _id in ids['delete']:
cursor.deleteRow()
feature_count['deleted'] += 1
else:
feature_count['unchanged'] += 1
for key in ['deleted', 'unchanged']:
log("%s features %s.", feature_count[key], key)
log("End: Delete.")
return feature_count
def update_by_value(dataset_path, field_name, value, **kwargs):
"""Update attribute values by assigning a given value.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
value (object): Static value to assign.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("use_edit_session", True)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s by given value.", field_name, dataset_path
)
meta = {"dataset": dataset_metadata(dataset_path)}
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=[field_name],
where_clause=kwargs["dataset_where_sql"],
)
update_action_count = Counter()
with session, cursor:
for [old_value] in cursor:
if same_value(old_value, value):
update_action_count["unchanged"] += 1
else:
try:
cursor.updateRow([value])
update_action_count["altered"] += 1
except RuntimeError:
LOG.error("Offending value is %s", value)
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
def copy(dataset_path, output_path, **kwargs):
"""Copy features into a new dataset.
Args:
dataset_path (str): Path of the dataset.
output_path (str): Path of output dataset.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
schema_only (bool): Copy only the schema--omitting data--if True. Default is
False.
overwrite (bool): Overwrite the output dataset if it exists, if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
Raises:
ValueError: If dataset type not supported.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("schema_only", False)
kwargs.setdefault("overwrite", False)
if kwargs["schema_only"]:
kwargs["dataset_where_sql"] = "0=1"
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log("Start: Copy dataset %s to %s.", dataset_path, output_path)
meta = {"dataset": dataset_metadata(dataset_path)}
view = DatasetView(dataset_path, kwargs["dataset_where_sql"])
with view:
if meta["dataset"]["is_spatial"]:
exec_copy = arcpy.management.CopyFeatures
elif meta["dataset"]["is_table"]:
exec_copy = arcpy.management.CopyRows
else:
raise ValueError(
"{} unsupported dataset type.".format(dataset_path))
if kwargs["overwrite"] and arcpy.Exists(output_path):
delete(output_path, log_level=None)
exec_copy(view.name, output_path)
log("End: Copy.")
return Counter(copied=feature_count(output_path))
def update_by_expression(dataset_path, field_name, expression, **kwargs):
"""Update attribute values using a (single) code-expression.
Wraps arcpy.management.CalculateField.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
expression (str): Python string expression to evaluate values from.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
str: Name of the field updated.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s using expression: `%s`.",
field_name,
dataset_path,
expression,
)
meta = {"dataset": dataset_metadata(dataset_path)}
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
dataset_view = DatasetView(dataset_path, kwargs["dataset_where_sql"])
with session, dataset_view:
arcpy.management.CalculateField(
in_table=dataset_view.name,
field=field_name,
expression=expression,
expression_type="python_9.3",
)
log("End: Update.")
return field_name
def as_dicts(dataset_path, field_names=None, **kwargs):
"""Generate mappings of feature attribute name to value.
Notes:
Use ArcPy cursor token names for object IDs and geometry objects/properties.
Args:
dataset_path (str): Path of the dataset.
field_names (iter): Collection of field names. Names will be the keys in the
dictionary mapping to their values. If value is None, all attributes fields
will be used.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
spatial_reference_item: Item from which the spatial reference of the output
geometry will be derived.
Yields:
dict.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("spatial_reference_item")
meta = {"spatial": spatial_reference_metadata(kwargs["spatial_reference_item"])}
if field_names is None:
meta["dataset"] = dataset_metadata(dataset_path)
keys = {"feature": [key for key in meta["dataset"]["field_names_tokenized"]]}
else:
keys = {"feature": list(contain(field_names))}
cursor = arcpy.da.SearchCursor(
in_table=dataset_path,
field_names=keys["feature"],
where_clause=kwargs["dataset_where_sql"],
spatial_reference=meta["spatial"]["object"],
)
with cursor:
for feature in cursor:
yield dict(zip(cursor.fields, feature))
def insert_from_iters(dataset_path, insert_features, field_names, **kwargs):
"""Insert features into dataset from iterables.
Args:
dataset_path (str): Path of the dataset.
insert_features (iter of iter): Collection of iterables representing features.
field_names (iter): Collection of field names to insert. These must match the
order of their attributes in the insert_features items.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
use_edit_session (bool): Flag to perform updates in an edit session. Default is
False.
log_level (str): Level to log the function at. Default is 'info'.
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault('use_edit_session', False)
log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info'))
log("Start: Insert features into %s from iterables.", dataset_path)
meta = {'dataset': arcobj.dataset_metadata(dataset_path)}
keys = {'row': list(contain(field_names))}
if inspect.isgeneratorfunction(insert_features):
insert_features = insert_features()
session = arcobj.Editor(meta['dataset']['workspace_path'],
kwargs['use_edit_session'])
cursor = arcpy.da.InsertCursor(dataset_path, field_names=keys['row'])
feature_count = Counter()
with session, cursor:
for row in insert_features:
cursor.insertRow(tuple(row))
feature_count['inserted'] += 1
log("%s features inserted.", feature_count['inserted'])
log("End: Insert.")
return feature_count
def insert_from_path(dataset_path,
insert_dataset_path,
field_names=None,
**kwargs):
"""Insert features into dataset from another dataset.
Args:
dataset_path (str): Path of the dataset.
insert_dataset_path (str): Path of dataset to insert features from.
field_names (iter): Collection of field names to insert. Listed field must be
present in both datasets. If field_names is None, all fields will be
inserted.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
insert_where_sql (str): SQL where-clause for insert-dataset subselection.
use_edit_session (bool): Flag to perform updates in an edit session. Default is
False.
log_level (str): Level to log the function at. Default is 'info'.
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault('insert_where_sql')
kwargs.setdefault('use_edit_session', False)
log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info'))
log("Start: Insert features into %s from %s.", dataset_path,
insert_dataset_path)
meta = {
'dataset': arcobj.dataset_metadata(dataset_path),
'insert': arcobj.dataset_metadata(insert_dataset_path),
}
if field_names is None:
keys = set.intersection(*(set(
name.lower() for name in _meta['field_names_tokenized'])
for _meta in meta.values()))
else:
keys = set(name.lower() for name in contain(field_names))
# OIDs & area/length "fields" have no business being part of an insert.
# Geometry itself is handled separately in append function.
for _meta in meta.values():
for key in chain(*_meta['field_token'].items()):
keys.discard(key)
append_kwargs = {
'inputs': unique_name('view'),
'target': dataset_path,
'schema_type': 'no_test',
'field_mapping': arcpy.FieldMappings(),
}
# Create field maps.
# ArcGIS Pro's no-test append is case-sensitive (verified 1.0-1.1.1).
# Avoid this problem by using field mapping.
# BUG-000090970 - ArcGIS Pro 'No test' field mapping in Append tool does not auto-
# map to the same field name if naming convention differs.
for key in keys:
field_map = arcpy.FieldMap()
field_map.addInputField(insert_dataset_path, key)
append_kwargs['field_mapping'].addFieldMap(field_map)
view = arcobj.DatasetView(
insert_dataset_path,
kwargs['insert_where_sql'],
view_name=append_kwargs['inputs'],
# Must be nonspatial to append to nonspatial table.
force_nonspatial=(not meta['dataset']['is_spatial']),
)
session = arcobj.Editor(meta['dataset']['workspace_path'],
kwargs['use_edit_session'])
with view, session:
arcpy.management.Append(**append_kwargs)
feature_count = Counter({'inserted': view.count})
log("%s features inserted.", feature_count['inserted'])
log("End: Insert.")
return feature_count
def eliminate_interior_rings(dataset_path,
max_area=None,
max_percent_total_area=None,
**kwargs):
"""Eliminate interior rings of polygon features.
Note:
If no value if provided for either max_area or max_percent_total_area, (nearly)
all interior rings will be removed. Technically, max_percent_total_area will be
set to 99.9999.
Args:
dataset_path (str): Path of the dataset.
max_area (float, str): Maximum area which parts smaller than are eliminated.
Numeric area will be in dataset's units. String area will be formatted as
'{number} {unit}'.
max_percent_total_area (float): Maximum percent of total area which parts
smaller than are eliminated.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
use_edit_session (bool): Flag to perform updates in an edit session. Default is
False.
log_level (str): Level to log the function at. Default is 'info'.
Returns:
str: Path of the dataset updated.
"""
kwargs.setdefault('dataset_where_sql')
kwargs.setdefault('use_edit_session', False)
log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info'))
log("Start: Eliminate interior rings in %s.", dataset_path)
# Only set max_percent_total_area default if neither it or area defined.
if all([max_area is None, max_percent_total_area is None]):
max_percent_total_area = 99.9999
if all([max_area is not None, max_percent_total_area is not None]):
condition = 'area_or_percent'
elif max_area is not None:
condition = 'area'
else:
condition = 'percent'
meta = {'dataset': arcobj.dataset_metadata(dataset_path)}
view = {
'dataset': arcobj.DatasetView(dataset_path,
kwargs['dataset_where_sql'])
}
temp_output_path = unique_path('output')
with view['dataset']:
arcpy.management.EliminatePolygonPart(
in_features=view['dataset'].name,
out_feature_class=temp_output_path,
condition=condition,
part_area=max_area,
part_area_percent=max_percent_total_area,
part_option='contained_only',
)
session = arcobj.Editor(meta['dataset']['workspace_path'],
kwargs['use_edit_session'])
with session:
delete(dataset_path,
dataset_where_sql=kwargs['dataset_where_sql'],
log_level=None)
insert_from_path(dataset_path,
insert_dataset_path=temp_output_path,
log_level=None)
dataset.delete(temp_output_path, log_level=None)
log("End: Eliminate.")
return dataset_path
def dissolve(dataset_path,
dissolve_field_names=None,
multipart=True,
**kwargs):
"""Dissolve geometry of features that share values in given fields.
Args:
dataset_path (str): Path of the dataset.
dissolve_field_names (iter): Iterable of field names to dissolve on.
multipart (bool): Flag to allow multipart features in output.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
unsplit_lines (bool): Flag to merge line features when endpoints meet without
crossing features. Default is False.
dataset_where_sql (str): SQL where-clause for dataset subselection.
tolerance (float): Tolerance for coincidence, in dataset's units.
use_edit_session (bool): Flag to perform updates in an edit session. Default is
False.
log_level (str): Level to log the function at. Default is 'info'.
Returns:
str: Path of the dataset updated.
"""
kwargs.setdefault('unsplit_lines', False)
kwargs.setdefault('dataset_where_sql')
kwargs.setdefault('use_edit_session', False)
log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info'))
log(
"Start: Dissolve features in %s on fields: %s.",
dataset_path,
dissolve_field_names,
)
meta = {
'dataset': arcobj.dataset_metadata(dataset_path),
'orig_tolerance': arcpy.env.XYTolerance,
}
keys = {'dissolve': tuple(contain(dissolve_field_names))}
view = {
'dataset': arcobj.DatasetView(dataset_path,
kwargs['dataset_where_sql'])
}
temp_output_path = unique_path('output')
with view['dataset']:
if 'tolerance' in kwargs:
arcpy.env.XYTolerance = kwargs['tolerance']
arcpy.management.Dissolve(
in_features=view['dataset'].name,
out_feature_class=temp_output_path,
dissolve_field=keys['dissolve'],
multi_part=multipart,
unsplit_lines=kwargs['unsplit_lines'],
)
if 'tolerance' in kwargs:
arcpy.env.XYTolerance = meta['orig_tolerance']
session = arcobj.Editor(meta['dataset']['workspace_path'],
kwargs['use_edit_session'])
with session:
delete(dataset_path,
dataset_where_sql=kwargs['dataset_where_sql'],
log_level=None)
insert_from_path(dataset_path,
insert_dataset_path=temp_output_path,
log_level=None)
dataset.delete(temp_output_path, log_level=None)
log("End: Dissolve.")
return dataset_path
def update_by_feature_match(
dataset_path, field_name, id_field_names, update_type, **kwargs
):
"""Update attribute values by aggregating info about matching features.
Note: Currently, the sort_order update type uses functionality that only works with
datasets contained in databases.
Valid update_type codes:
"flag_value": Apply the flag_value argument value to matched features.
"match_count": Apply the count of matched features.
"sort_order": Apply the position of the feature sorted with matches.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
id_field_names (iter): Field names used to identify a feature.
update_type (str): Code indicating what values to apply to matched features.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
flag_value: Value to apply to matched features. Only used when update_type is
"flag_value".
sort_field_names (iter): Iterable of field names used to sort matched features.
Only affects output when update_type="sort_order".
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s"
+ " by feature-matching %s on identifiers (%s).",
field_name,
dataset_path,
update_type.replace("_", " "),
id_field_names,
)
if update_type not in ["flag_value", "match_count", "sort_order"]:
raise ValueError("Invalid update_type.")
for _type, kwarg in {
"flag_value": "flag_value",
"sort_order": "sort_field_names",
}.items():
if update_type == _type and kwarg not in kwargs:
raise TypeError(
"""{} is required keyword argument when update_type == "{}", .""".format(
_type, kwarg
)
)
meta = {"dataset": dataset_metadata(dataset_path)}
keys = {
"id": list(contain(id_field_names)),
"sort": list(contain(kwargs.get("sort_field_names", []))),
}
keys["feature"] = keys["id"] + [field_name]
matcher = FeatureMatcher(dataset_path, keys["id"], kwargs["dataset_where_sql"])
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=keys["feature"],
where_clause=kwargs["dataset_where_sql"],
sql_clause=(
(None, "order by " + ", ".join(keys["sort"]))
if update_type == "sort_order"
else None
),
)
update_action_count = Counter()
with session, cursor:
for feature in cursor:
value = {
"id": feature[0] if len(keys["id"]) == 1 else tuple(feature[:-1]),
"old": feature[-1],
}
if update_type == "flag_value":
if matcher.is_duplicate(value["id"]):
value["new"] = kwargs["flag_value"]
else:
value["new"] = value["old"]
elif update_type == "match_count":
value["new"] = matcher.match_count(value["id"])
elif update_type == "sort_order":
value["new"] = matcher.increment_assigned(value["id"])
if same_value(value["old"], value["new"]):
update_action_count["unchanged"] += 1
else:
try:
cursor.updateRow(feature[:-1] + [value["new"]])
update_action_count["altered"] += 1
except RuntimeError:
LOG.error("Offending value is %s", value["new"])
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
def update_by_mapping(dataset_path, field_name, mapping, key_field_names, **kwargs):
"""Update attribute values by finding them in a mapping.
Note: Mapping key must be a tuple if an iterable.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
mapping: Mapping to get values from.
key_field_names (iter): Fields names whose values will comprise the mapping key.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
default_value: Value to return from mapping if key value on feature not
present. Default is None.
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("default_value")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s by mapping with key in %s.",
field_name,
dataset_path,
key_field_names,
)
meta = {"dataset": dataset_metadata(dataset_path)}
keys = {"map": list(contain(key_field_names))}
keys["feature"] = keys["map"] + [field_name]
if isinstance(mapping, EXEC_TYPES):
mapping = mapping()
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=keys["feature"],
where_clause=kwargs["dataset_where_sql"],
)
update_action_count = Counter()
with session, cursor:
for feature in cursor:
value = {
"map_key": feature[0] if len(keys["map"]) == 1 else tuple(feature[:-1]),
"old": feature[-1],
}
value["new"] = mapping.get(value["map_key"], kwargs["default_value"])
if same_value(value["old"], value["new"]):
update_action_count["unchanged"] += 1
else:
try:
cursor.updateRow(feature[:-1] + [value["new"]])
update_action_count["altered"] += 1
except RuntimeError:
LOG.error("Offending value is %s", value["new"])
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
def update_by_function(dataset_path, field_name, function, **kwargs):
"""Update attribute values by passing them to a function.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
function (types.FunctionType): Function to get values from.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
field_as_first_arg (bool): True if field value will be the first positional
argument. Default is True.
arg_field_names (iter): Field names whose values will be the positional
arguments (not including primary field).
kwarg_field_names (iter): Field names whose names & values will be the method
keyword arguments.
dataset_where_sql (str): SQL where-clause for dataset subselection.
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("field_as_first_arg", True)
kwargs.setdefault("arg_field_names", [])
kwargs.setdefault("kwarg_field_names", [])
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s by function %s.",
field_name,
dataset_path,
function,
)
meta = {"dataset": dataset_metadata(dataset_path)}
keys = {
"args": list(contain(kwargs["arg_field_names"])),
"kwargs": list(contain(kwargs["kwarg_field_names"])),
}
keys["feature"] = keys["args"] + keys["kwargs"] + [field_name]
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=keys["feature"],
where_clause=kwargs["dataset_where_sql"],
)
update_action_count = Counter()
with session, cursor:
for feature in cursor:
value = {
"old": feature[-1],
"args": feature[: len(keys["args"])],
"kwargs": dict(zip(keys["kwargs"], feature[len(keys["args"]) : -1])),
}
if kwargs["field_as_first_arg"]:
value["args"] = [value["old"]] + value["args"]
value["new"] = function(*value["args"], **value["kwargs"])
if same_value(value["old"], value["new"]):
update_action_count["unchanged"] += 1
else:
try:
cursor.updateRow(feature[:-1] + [value["new"]])
update_action_count["altered"] += 1
except RuntimeError:
LOG.error("Offending value is %s", value["new"])
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
def adjust_for_shapefile(dataset_path, **kwargs):
"""Adjust features to meet shapefile requirements.
Note:
Shapefiles cannot have null-values. Nulls will be replaced with the values
provided in the null replacement keyword arguments.
Shapefiles only have dates in the date/datetime field. Times will be truncated
in the adjustment.
Args:
dataset_path (str): Path of the dataset.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
date_null_replacement (datetime.date): Replacement value for null-values in
date fields. Default is datetime.date.min.
integer_null_replacement (int): Replacement value for null-values in integer
fields. Default is 0.
numeric_null_replacement (float): Replacement value for null-values in numeric
fields. Default is 0.0.
string_null_replacement (str): Replacement value for null-values in string
fields. Default is "".
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
str: Path of the adjusted dataset.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log("Start: Adjust features for shapefile output in %s.", dataset_path)
replacement_value = {
"date": kwargs.setdefault("date_null_replacement", datetime.date.min),
"double": kwargs.setdefault("numeric_null_replacement", 0.0),
"single": kwargs.setdefault("numeric_null_replacement", 0.0),
"integer": kwargs.setdefault("integer_null_replacement", 0),
"smallinteger": kwargs.setdefault("integer_null_replacement", 0),
"string": kwargs.setdefault("string_null_replacement", ""),
# Shapefile loader handles non-user fields seperately.
# "geometry", "oid",
}
meta = {"dataset": dataset_metadata(dataset_path)}
session = Editor(meta["dataset"]["workspace_path"],
kwargs["use_edit_session"])
with session:
for field in meta["dataset"]["user_fields"]:
if field["type"].lower() not in replacement_value:
log("Skipping %s field: type cannot transfer to shapefile.",
field["name"])
continue
else:
log("Adjusting values in %s field.", field["name"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=[field["name"]],
where_clause=kwargs["dataset_where_sql"],
)
with cursor:
for (old_value, ) in cursor:
if old_value is None:
new_value = replacement_value[field["type"].lower()]
try:
cursor.updateRow([new_value])
except RuntimeError:
LOG.error("Offending value is %s", new_value)
raise RuntimeError
log("End: Adjust.")
return dataset_path
def update_by_geometry(dataset_path, field_name, geometry_properties, **kwargs):
"""Update attribute values by cascading through geometry properties.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
geometry_properties (iter): Geometry property names in object-access order to
retrieve the update value.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
spatial_reference_item: Item from which the spatial reference for the output
geometry property will be derived. Default is the update dataset.
use_edit_session (bool): Updates are done in an edit session if True. If not
not specified or None, the spatial reference of the dataset is used.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("spatial_reference_item")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s by geometry properties %s.",
field_name,
dataset_path,
geometry_properties,
)
meta = {
"dataset": dataset_metadata(dataset_path),
"spatial": spatial_reference_metadata(kwargs["spatial_reference_item"]),
}
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=["shape@", field_name],
where_clause=kwargs["dataset_where_sql"],
spatial_reference=meta["spatial"]["object"],
)
update_action_count = Counter()
with session, cursor:
for feature in cursor:
value = {"geometry": feature[0], "old": feature[-1]}
value["new"] = property_value(
value["geometry"],
GEOMETRY_PROPERTY_TRANSFORM,
*contain(geometry_properties)
)
if same_value(value["old"], value["new"]):
update_action_count["unchanged"] += 1
else:
try:
cursor.updateRow([value["geometry"], value["new"]])
update_action_count["altered"] += 1
except RuntimeError:
LOG.error("Offending value is %s", value["new"])
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
def update_from_path(dataset_path,
update_dataset_path,
id_field_names,
field_names=None,
**kwargs):
"""Update features in dataset from another dataset.
Args:
dataset_path (str): Path of the dataset.
update_dataset_path (str): Path of dataset to update features from.
id_field_names (iter, str): Name(s) of the ID field/key(s).
field_names (iter): Collection of field names/keys to check & update. Listed
field must be present in both datasets. If field_names is None, all fields
will be inserted.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection. WARNING:
defining this has major effects: filtered features will not be considered
for updating or deletion, and duplicates in the update features will be
inserted as if novel.
update_where_sql (str): SQL where-clause for update-dataset subselection.
chunk_where_sqls (iter): Collection of SQL where-clauses for updating between
the datasets in chunks.
delete_missing_features (bool): True if update should delete features missing
from update_features, False otherwise. Default is True.
use_edit_session (bool): Flag to perform updates in an edit session. Default is
True.
log_level (str): Level to log the function at. Default is 'info'.
Returns:
collections.Counter: Counts for each feature action.
"""
for key in ['dataset_where_sql', 'update_where_sql']:
kwargs.setdefault(key)
if not kwargs[key]:
kwargs[key] = "1=1"
kwargs.setdefault('subset_where_sqls', ["1=1"])
kwargs.setdefault('delete_missing_features', True)
kwargs.setdefault('use_edit_session', True)
log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info'))
log("Start: Update features in %s from %s.", dataset_path,
update_dataset_path)
meta = {
'dataset': arcobj.dataset_metadata(dataset_path),
'update': arcobj.dataset_metadata(update_dataset_path),
}
if field_names is None:
field_names = (
set(name.lower()
for name in meta['dataset']['field_names_tokenized'])
& set(name.lower()
for name in meta['update']['field_names_tokenized']))
else:
field_names = set(name.lower() for name in field_names)
# OIDs & area/length "fields" have no business being part of an update.
for key in ['oid@', 'shape@area', 'shape@length']:
field_names.discard(key)
keys = {
'id': list(contain(id_field_names)),
'attr': list(contain(field_names))
}
keys['row'] = keys['id'] + keys['attr']
feature_count = Counter()
for kwargs['subset_where_sql'] in contain(kwargs['subset_where_sqls']):
if not kwargs['subset_where_sql'] == "1=1":
log("Subset: `%s`", kwargs['subset_where_sql'])
iters = attributes.as_iters(
update_dataset_path,
keys['row'],
dataset_where_sql=(
"({update_where_sql}) and ({subset_where_sql})".format(
**kwargs)))
view = arcobj.DatasetView(
dataset_path,
dataset_where_sql=(
"({dataset_where_sql}) and ({subset_where_sql})".format(
**kwargs)),
field_names=keys['row'])
with view:
feature_count.update(
update_from_iters(
dataset_path=view.name,
update_features=iters,
id_field_names=keys['id'],
field_names=keys['row'],
delete_missing_features=kwargs['delete_missing_features'],
use_edit_session=kwargs['use_edit_session'],
log_level=None,
))
for key in UPDATE_TYPES:
log("%s features %s.", feature_count[key], key)
log("End: Update.")
return feature_count
def add_index(dataset_path, field_names, **kwargs):
"""Add index to dataset fields.
Note:
Index names can only be applied to non-spatial indexes for geodatabase feature
classes and tables.
There is a limited length allowed for index names; longer names will be
truncated without warning.
Args:
dataset_path (str): Path of the dataset.
field_names (iter): Collection of participating field names.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
index_name (str): Name for index. Optional; see note.
is_ascending (bool): Build index in ascending order if True. Default is False.
is_unique (bool): Build index with unique constraint if True. Default is False.
fail_on_lock_ok (bool): If True, indicate success even if dataset locks prevent
adding index. Default is False.
log_level (str): Level to log the function at. Default is "info".
Returns:
str: Path of the dataset receiving the index.
Raises:
RuntimeError: If more than one field and any are geometry-types.
arcpy.ExecuteError: If dataset lock prevents adding index.
"""
field_names = [name.lower() for name in contain(field_names)]
kwargs.setdefault("index_name", "ndx_" + "_".join(field_names))
kwargs.setdefault("is_ascending", False)
kwargs.setdefault("is_unique", False)
kwargs.setdefault("fail_on_lock_ok", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log("Start: Add index to field(s) %s on %s.", field_names, dataset_path)
meta = {"dataset": dataset_metadata(dataset_path)}
meta["field_types"] = {
field["type"].lower()
for field in meta["dataset"]["fields"]
if field["name"].lower() in field_names
}
if "geometry" in meta["field_types"]:
if len(field_names) > 1:
raise RuntimeError("Cannot create a composite spatial index.")
exec_add = arcpy.management.AddSpatialIndex
add_kwargs = {"in_features": dataset_path}
else:
exec_add = arcpy.management.AddIndex
add_kwargs = {
"in_table": dataset_path,
"fields": field_names,
"index_name": kwargs["index_name"],
"unique": kwargs["is_unique"],
"ascending": kwargs["is_ascending"],
}
try:
exec_add(**add_kwargs)
except arcpy.ExecuteError as error:
if error.message.startswith("ERROR 000464"):
LOG.warning("Lock on %s prevents adding index.", dataset_path)
if not kwargs["fail_on_lock_ok"]:
raise
log("End: Add.")
return dataset_path
def update_by_overlay(
dataset_path, field_name, overlay_dataset_path, overlay_field_name, **kwargs
):
"""Update attribute values by finding overlay feature value.
Note:
Since only one value will be selected in the overlay, operations with multiple
overlaying features will respect the geoprocessing environment merge rule. This
rule generally defaults to the value of the "first" feature.
Only one overlay flag at a time can be used (e.g. "overlay_most_coincident",
"overlay_central_coincident"). If multiple are set to True, the first one
referenced in the code will be used. If no overlay flags are set, the operation
will perform a basic intersection check, and the result will be at the whim of
the geoprocessing environment merge rule for the update field.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
overlay_dataset_path (str): Path of the overlay-dataset.
overlay_field_name (str): Name of the overlay-field.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
overlay_central_coincident (bool): Overlay will use the centrally-coincident
value if True. Default is False.
overlay_most_coincident (bool): Overlay will use the most coincident value if
True. Default is False.
overlay_where_sql (str): SQL where-clause for overlay dataset subselection.
replacement_value: Value to replace a present overlay-field value with.
tolerance (float): Tolerance for coincidence, in units of the dataset.
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("overlay_central_coincident", False)
kwargs.setdefault("overlay_most_coincident", False)
kwargs.setdefault("overlay_where_sql")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s by overlay values in %s on %s.",
field_name,
dataset_path,
overlay_field_name,
overlay_dataset_path,
)
meta = {
"dataset": dataset_metadata(dataset_path),
"original_tolerance": arcpy.env.XYTolerance,
}
join_kwargs = {"join_operation": "join_one_to_many", "join_type": "keep_all"}
if kwargs["overlay_central_coincident"]:
join_kwargs["match_option"] = "have_their_center_in"
##TODO: Implement overlay_most_coincident.
elif kwargs["overlay_most_coincident"]:
raise NotImplementedError("overlay_most_coincident not yet implemented.")
# else:
# join_kwargs["match_option"] = "intersect"
dataset_view = DatasetView(dataset_path, kwargs["dataset_where_sql"])
overlay_copy = TempDatasetCopy(
overlay_dataset_path,
kwargs["overlay_where_sql"],
field_names=[overlay_field_name],
)
with dataset_view, overlay_copy:
# Avoid field name collisions with neutral name.
overlay_copy.field_name = dataset.rename_field(
overlay_copy.path,
overlay_field_name,
new_field_name=unique_name(overlay_field_name),
log_level=None,
)
if "tolerance" in kwargs:
arcpy.env.XYTolerance = kwargs["tolerance"]
# Create temp output of the overlay.
temp_output_path = unique_path("output")
arcpy.analysis.SpatialJoin(
target_features=dataset_view.name,
join_features=overlay_copy.path,
out_feature_class=temp_output_path,
**join_kwargs
)
if "tolerance" in kwargs:
arcpy.env.XYTolerance = meta["original_tolerance"]
# Push overlay (or replacement) value from output to update field.
if "replacement_value" in kwargs and kwargs["replacement_value"] is not None:
function = lambda x: kwargs["replacement_value"] if x else None
else:
function = lambda x: x
update_by_function(
temp_output_path,
field_name,
function,
field_as_first_arg=False,
arg_field_names=[overlay_copy.field_name],
log_level=None,
)
# Update values in original dataset.
update_action_count = update_by_joined_value(
dataset_path,
field_name,
join_dataset_path=temp_output_path,
join_field_name=field_name,
on_field_pairs=[(meta["dataset"]["oid_field_name"], "target_fid")],
dataset_where_sql=kwargs["dataset_where_sql"],
use_edit_session=kwargs["use_edit_session"],
log_level=None,
)
dataset.delete(temp_output_path, log_level=None)
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
def update_by_node_ids(dataset_path, from_id_field_name, to_id_field_name, **kwargs):
"""Update attribute values by passing them to a function.
Args:
dataset_path (str): Path of the dataset.
from_id_field_name (str): Name of the from-ID field.
to_id_field_name (str): Name of the to-ID field.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s & %s on %s by node IDs.",
from_id_field_name,
to_id_field_name,
dataset_path,
)
meta = {"dataset": dataset_metadata(dataset_path)}
keys = {"feature": ["oid@", from_id_field_name, to_id_field_name]}
oid_node = id_node_map(
dataset_path, from_id_field_name, to_id_field_name, update_nodes=True
)
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=keys["feature"],
where_clause=kwargs["dataset_where_sql"],
)
update_action_count = Counter()
with session, cursor:
for feature in cursor:
value = {"oid": feature[0], "old_nodes": feature[1:]}
value["new_nodes"] = [
oid_node[value["oid"]]["from"],
oid_node[value["oid"]]["to"],
]
if same_feature(value["old_nodes"], value["new_nodes"]):
update_action_count["unchanged"] += 1
else:
try:
cursor.updateRow([value["oid"]] + value["new_nodes"])
update_action_count["altered"] += 1
except RuntimeError:
LOG.error("Offending value one of %s", value["new_nodes"])
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
def polygons_to_lines(dataset_path, output_path, topological=False, **kwargs):
"""Convert geometry from polygons to lines.
Note:
If topological is set to True, shared outlines will be a single, separate
feature. Note that one cannot pass attributes to a topological transformation
(as the values would not apply to all adjacent features).
If an id field name is specified, the output dataset will identify the input
features that defined the line feature with the name & values from the provided
field. This option will be ignored if the output is non-topological lines, as
the field will pass over with the rest of the attributes.
Args:
dataset_path (str): Path of the dataset.
output_path (str): Path of the output dataset.
topological (bool): Flag to indicate lines should be topological, or merged
where lines overlap.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
id_field_name (str): Name of the field to apply ID to lines from.
tolerance (float): Tolerance for coincidence, in units of the dataset.
log_level (str): Level to log the function at. Default is "info".
Returns:
str: Path of the converted dataset.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("id_field_name")
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log("Start: Convert polgyons in %s to lines in %s.", dataset_path, output_path)
meta = {
"dataset": dataset_metadata(dataset_path),
"orig_tolerance": arcpy.env.XYTolerance,
}
view = DatasetView(dataset_path, kwargs["dataset_where_sql"])
with view:
if "tolerance" in kwargs:
arcpy.env.XYTolerance = kwargs["tolerance"]
arcpy.management.PolygonToLine(
in_features=view.name,
out_feature_class=output_path,
neighbor_option=topological,
)
if "tolerance" in kwargs:
arcpy.env.XYTolerance = meta["orig_tolerance"]
if topological:
for side in ["left", "right"]:
meta[side] = {"oid_key": side.upper() + "_FID"}
if kwargs["id_field_name"]:
meta[side]["id_field"] = next(
field
for field in meta["dataset"]["fields"]
if field["name"].lower() == kwargs["id_field_name"].lower()
)
meta[side]["id_field"]["name"] = side + "_" + kwargs["id_field_name"]
# Cannot create an OID-type field, so force to long.
if meta[side]["id_field"]["type"].lower() == "oid":
meta[side]["id_field"]["type"] = "long"
dataset.add_field_from_metadata(
output_path, meta[side]["id_field"], log_level=None
)
attributes.update_by_joined_value(
output_path,
field_name=meta[side]["id_field"]["name"],
join_dataset_path=dataset_path,
join_field_name=kwargs["id_field_name"],
on_field_pairs=[
(meta[side]["oid_key"], meta["dataset"]["oid_field_name"])
],
log_level=None,
)
dataset.delete_field(output_path, meta[side]["oid_key"], log_level=None)
else:
dataset.delete_field(output_path, "ORIG_FID", log_level=None)
log("End: Convert.")
return output_path
def update_from_iters(dataset_path, update_features, id_field_names,
field_names, **kwargs):
"""Update features in dataset from iterables.
Note:
There is no guarantee that the ID field(s) are unique.
Use ArcPy cursor token names for object IDs and geometry objects/properties.
Args:
dataset_path (str): Path of the dataset.
update_features (iter of dict): Collection of iterables representing features.
id_field_names (iter, str): Name(s) of the ID field/key(s). *All* ID fields
must also be in field_names.
field_names (iter): Collection of field names/keys to check & update.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
delete_missing_features (bool): True if update should delete features missing
from update_features, False otherwise. Default is True.
use_edit_session (bool): Flag to perform updates in an edit session. Default is
True.
log_level (str): Level to log the function at. Default is 'info'.
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault('delete_missing_features', True)
kwargs.setdefault('use_edit_session', True)
log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info'))
log("Start: Update features in %s from iterables.", dataset_path)
meta = {'dataset': arcobj.dataset_metadata(dataset_path)}
keys = {
'id': list(contain(id_field_names)),
'feat': list(contain(field_names))
}
if not set(keys['id']).issubset(keys['feat']):
raise ValueError("id_field_names must be a subset of field_names.")
ids = {
'dataset': {
tuple(freeze_values(*_id))
for _id in attributes.as_iters(dataset_path, keys['id'])
}
}
if inspect.isgeneratorfunction(update_features):
update_features = update_features()
feats = {'insert': set(), 'id_update': dict()}
for feat in update_features:
feat = tuple(freeze_values(*feat))
_id = tuple(feat[keys['feat'].index(key)] for key in keys['id'])
if _id not in ids['dataset']:
feats['insert'].add(feat)
else:
feats['id_update'][_id] = feat
if kwargs['delete_missing_features']:
ids['delete'] = {
_id
for _id in ids['dataset'] if _id not in feats['id_update']
}
else:
ids['delete'] = set()
feature_count = Counter()
session = arcobj.Editor(meta['dataset']['workspace_path'],
kwargs['use_edit_session'])
if ids['delete'] or feats['id_update']:
cursor = arcpy.da.UpdateCursor(dataset_path, field_names=keys['feat'])
with session, cursor:
for feat in cursor:
_id = tuple(
freeze_values(*(feat[keys['feat'].index(key)]
for key in keys['id'])))
if _id in ids['delete']:
cursor.deleteRow()
feature_count['deleted'] += 1
continue
elif (_id in feats['id_update'] and
not arcobj.same_feature(feat, feats['id_update'][_id])):
cursor.updateRow(feats['id_update'][_id])
feature_count['altered'] += 1
else:
feature_count['unchanged'] += 1
if feats['insert']:
cursor = arcpy.da.InsertCursor(dataset_path, field_names=keys['feat'])
with session, cursor:
for feat in feats['insert']:
try:
cursor.insertRow(feat)
except RuntimeError:
LOG.error(
"Feature failed to write to cursor. Offending row:")
for key, val in zip(keys['feat'], feat):
LOG.error("%s: %s", key, val)
raise
feature_count['inserted'] += 1
for key in UPDATE_TYPES:
log("%s features %s.", feature_count[key], key)
log("End: Update.")
return feature_count
def update_by_joined_value(
dataset_path,
field_name,
join_dataset_path,
join_field_name,
on_field_pairs,
**kwargs
):
"""Update attribute values by referencing a joinable field.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
join_dataset_path (str): Path of the join-dataset.
join_field_name (str): Name of the join-field.
on_field_pairs (iter): Field name pairs used to to determine join.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s by joined values in %s on %s.",
field_name,
dataset_path,
join_field_name,
join_dataset_path,
)
meta = {"dataset": dataset_metadata(dataset_path)}
keys = {
"dataset_id": list(pair[0] for pair in on_field_pairs),
"join_id": list(pair[1] for pair in on_field_pairs),
}
keys["feature"] = keys["dataset_id"] + [field_name]
join_value = id_map(
join_dataset_path, id_field_names=keys["join_id"], field_names=join_field_name
)
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=keys["feature"],
where_clause=kwargs["dataset_where_sql"],
)
update_action_count = Counter()
with session, cursor:
for feature in cursor:
value = {
"id": (
feature[0] if len(keys["dataset_id"]) == 1 else tuple(feature[:-1])
),
"old": feature[-1],
}
value["new"] = join_value.get(value["id"])
if same_value(value["old"], value["new"]):
update_action_count["unchanged"] += 1
else:
try:
cursor.updateRow(feature[:-1] + [value["new"]])
update_action_count["altered"] += 1
except RuntimeError:
LOG.error("Offending value is %s", value["new"])
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
def delete(dataset_path, **kwargs):
"""Delete features in the dataset.
Args:
dataset_path (str): Path of the dataset.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
use_edit_session (bool): Flag to perform updates in an edit session. Default is
False.
log_level (str): Level to log the function at. Default is 'info'.
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault('dataset_where_sql')
kwargs.setdefault('use_edit_session', False)
log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info'))
if kwargs['dataset_where_sql']:
log(
"Start: Delete features from %s where `%s`.",
dataset_path,
kwargs['dataset_where_sql'],
)
else:
log("Start: Delete features from %s.", dataset_path)
meta = {'dataset': arcobj.dataset_metadata(dataset_path)}
truncate_error_codes = [
# "Only supports Geodatabase tables and feature classes."
'ERROR 000187',
# "Operation not supported on a versioned table."
'ERROR 001259',
# "Operation not supported on table {table name}."
'ERROR 001260',
# Operation not supported on a feature class in a controller dataset.
'ERROR 001395',
]
feature_count = Counter()
# Can use (faster) truncate when no sub-selection or edit session.
run_truncate = (kwargs['dataset_where_sql'] is None
and kwargs['use_edit_session'] is False)
if run_truncate:
feature_count['deleted'] = dataset.feature_count(dataset_path)
feature_count['unchanged'] = 0
try:
arcpy.management.TruncateTable(in_table=dataset_path)
except arcpy.ExecuteError:
# Avoid arcpy.GetReturnCode(); error code position inconsistent.
# Search messages for 'ERROR ######' instead.
if any(code in arcpy.GetMessages()
for code in truncate_error_codes):
LOG.debug("Truncate unsupported; will try deleting rows.")
run_truncate = False
else:
raise
if not run_truncate:
view = {
'dataset':
arcobj.DatasetView(dataset_path, kwargs['dataset_where_sql'])
}
session = arcobj.Editor(meta['dataset']['workspace_path'],
kwargs['use_edit_session'])
with view['dataset'], session:
feature_count['deleted'] = view['dataset'].count
arcpy.management.DeleteRows(in_rows=view['dataset'].name)
feature_count['unchanged'] = dataset.feature_count(dataset_path)
for key in ['deleted', 'unchanged']:
log("%s features %s.", feature_count[key], key)
log("End: Delete.")
return feature_count
def update_by_unique_id(dataset_path, field_name, **kwargs):
"""Update attribute values by assigning a unique ID.
Existing IDs are preserved, if unique.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("use_edit_session", True)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s by assigning unique IDs.",
field_name,
dataset_path,
)
meta = {
"dataset": dataset_metadata(dataset_path),
"field": field_metadata(dataset_path, field_name),
}
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=[field_name],
where_clause=kwargs["dataset_where_sql"],
)
with session:
used_ids = set()
# First run will clear duplicate IDs & gather used IDs.
with cursor:
for [id_value] in cursor:
if id_value in used_ids:
cursor.updateRow([None])
else:
used_ids.add(id_value)
id_pool = unique_ids(
data_type=python_type(meta["field"]["type"]),
string_length=meta["field"].get("length"),
)
# Second run will fill in missing IDs.
update_action_count = Counter()
with cursor:
for [id_value] in cursor:
if id_value is not None:
update_action_count["unchanged"] += 1
else:
id_value = next(id_pool)
while id_value in used_ids:
id_value = next(id_pool)
try:
cursor.updateRow([id_value])
update_action_count["altered"] += 1
used_ids.add(id_value)
except RuntimeError:
LOG.error("Offending value is %s", id_value)
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
