def update_from_dicts(dataset_path, update_features, id_field_names,
field_names, **kwargs):
"""Update features in dataset from dictionaries.
Note:
There is no guarantee that the ID field(s) are unique.
Use ArcPy cursor token names for object IDs and geometry objects/properties.
Args:
dataset_path (str): Path of the dataset.
update_features (iter of dict): Collection of dictionaries representing
features.
id_field_names (iter, str): Name(s) of the ID field/key(s).
field_names (iter): Collection of field names/keys to check & update.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
delete_missing_features (bool): True if update should delete features missing
from update_features, False otherwise. Default is True.
use_edit_session (bool): Flag to perform updates in an edit session. Default is
True.
log_level (str): Level to log the function at. Default is 'info'.
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault('delete_missing_features', True)
kwargs.setdefault('use_edit_session', True)
log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info'))
log("Start: Update features in %s from dictionaries.", dataset_path)
keys = {
'id': list(contain(id_field_names)),
'attr': list(contain(field_names))
}
keys['row'] = keys['id'] + keys['attr']
if inspect.isgeneratorfunction(update_features):
update_features = update_features()
iters = ((feature[key] for key in keys['row'])
for feature in update_features)
feature_count = update_from_iters(
dataset_path,
update_features=iters,
id_field_names=keys['id'],
field_names=keys['row'],
delete_missing_features=kwargs['delete_missing_features'],
use_edit_session=kwargs['use_edit_session'],
log_level=None,
)
for key in UPDATE_TYPES:
log("%s features %s.", feature_count[key], key)
log("End: Update.")
return feature_count
def id_map(dataset_path, id_field_names, field_names, **kwargs):
"""Return mapping of feature ID to attribute or list of attributes.
Notes:
There is no guarantee that the ID value(s) are unique.
Use ArcPy cursor token names for object IDs and geometry objects/properties.
Args:
dataset_path (str): Path of the dataset.
id_field_names (iter, str): Name(s) of the ID field(s).
field_names (iter, str): Name(s) of the field(s).
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
spatial_reference_item: Item from which the spatial reference of the output
geometry will be derived.
Returns:
dict.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("spatial_reference_item")
meta = {"spatial": spatial_reference_metadata(kwargs["spatial_reference_item"])}
keys = {
"id": list(contain(id_field_names)),
"attribute": list(contain(field_names)),
}
cursor = arcpy.da.SearchCursor(
in_table=dataset_path,
field_names=keys["id"] + keys["attribute"],
where_clause=kwargs["dataset_where_sql"],
spatial_reference=meta["spatial"]["object"],
)
id_attributes = {}
with cursor:
for feature in cursor:
value = {
"id": feature[0]
if len(keys["id"]) == 1
else feature[: len(keys["id"])],
"attributes": (
feature[len(keys["id"])]
if len(keys["attribute"]) == 1
else feature[len(keys["id"]) :]
),
}
id_attributes[value["id"]] = value["attributes"]
return id_attributes
def delete_by_id(dataset_path, delete_ids, id_field_names, **kwargs):
"""Delete features in dataset with given IDs.
Note:
There is no guarantee that the ID field(s) are unique.
Use ArcPy cursor token names for object IDs and geometry objects/properties.
Args:
dataset_path (str): Path of the dataset.
delete_ids (iter): Collection of feature IDs.
id_field_names (iter, str): Name(s) of the ID field/key(s).
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
use_edit_session (bool): Flag to perform updates in an edit session. Default is
False.
log_level (str): Level to log the function at. Default is 'info'.
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault('use_edit_session', False)
log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info'))
log("Start: Delete features in %s with given IDs.", dataset_path)
meta = {'dataset': arcobj.dataset_metadata(dataset_path)}
keys = {'id': list(contain(id_field_names))}
if inspect.isgeneratorfunction(delete_ids):
delete_ids = delete_ids()
ids = {'delete': {tuple(contain(_id)) for _id in delete_ids}}
feature_count = Counter()
session = arcobj.Editor(meta['dataset']['workspace_path'],
kwargs['use_edit_session'])
##TODO: Preserve possible generators!
if ids['delete']:
cursor = arcpy.da.UpdateCursor(dataset_path, field_names=keys['id'])
with session, cursor:
for row in cursor:
_id = tuple(row)
if _id in ids['delete']:
cursor.deleteRow()
feature_count['deleted'] += 1
else:
feature_count['unchanged'] += 1
for key in ['deleted', 'unchanged']:
log("%s features %s.", feature_count[key], key)
log("End: Delete.")
return feature_count
def as_iters(dataset_path, field_names, **kwargs):
"""Generate iterables of feature attribute values.
Notes:
Use ArcPy cursor token names for object IDs and geometry objects/properties.
Args:
dataset_path (str): Path of the dataset.
field_names (iter): Collection of field names. The order of the names in the
collection will determine where its value will fall in the generated item.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
spatial_reference_item: Item from which the spatial reference of the output
geometry will be derived.
iter_type: Iterable type to yield. Default is tuple.
Yields:
iter.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("spatial_reference_item")
kwargs.setdefault("iter_type", tuple)
meta = {"spatial": spatial_reference_metadata(kwargs["spatial_reference_item"])}
keys = {"feature": list(contain(field_names))}
cursor = arcpy.da.SearchCursor(
in_table=dataset_path,
field_names=keys["feature"],
where_clause=kwargs["dataset_where_sql"],
spatial_reference=meta["spatial"]["object"],
)
with cursor:
for feature in cursor:
yield kwargs["iter_type"](feature)
def assigned_count(self, id_values):
"""Return the assigned count for features with the given identifier.
Args:
id_values (iter): Feature identifier values.
"""
return self.assigned[tuple(contain(id_values))]
def is_duplicate(self, id_values):
"""Return True if more than one feature has given ID.
Args:
id_values (iter): Feature identifier values.
"""
return self.matched[tuple(contain(id_values))] > 1
def match_count(self, id_values):
"""Return match count for features with given ID.
Args:
id_values (iter): Feature identifier values.
"""
return self.matched[tuple(contain(id_values))]
def increment_assigned(self, id_values):
"""Increment assigned count for given feature ID.
Args:
id_values (iter): Feature identifier values.
"""
_id = tuple(contain(id_values))
self.assigned[_id] += 1
return self.assigned[_id]
def rows_to_csvfile(rows, output_path, field_names, header=False, **kwargs):
"""Write collection of rows to a CSV-file.
Note: Rows can be represented by either dictionaries or sequences.
Args:
rows (iter): Collection of dictionaries or sequences representing rows.
output_path (str): Path of the output dataset.
field_names (iter): Collection of the field names, in the desired order or
output.
header (bool): Write a header in the CSV output if True.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
file_mode (str): Code indicating the file mode for writing. Default is "wb".
log_level (str): Level to log the function at. Default is "info".
Returns:
str: Path of the CSV-file.
"""
kwargs.setdefault("file_mode", "wb")
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log("Start: Convert rows to CSVfile %s.", output_path)
field_names = list(contain(field_names))
with open(output_path, kwargs["file_mode"]) as csvfile:
for index, row in enumerate(rows):
if index == 0:
if isinstance(row, dict):
writer = csv.DictWriter(csvfile, field_names)
if header:
writer.writeheader()
elif isinstance(row, Sequence):
writer = csv.writer(csvfile)
if header:
writer.writerow(field_names)
else:
raise TypeError("Rows must be dictionaries or sequences.")
writer.writerow(row)
log("End: Write.")
return output_path
def as_dicts(dataset_path, field_names=None, **kwargs):
"""Generate mappings of feature attribute name to value.
Notes:
Use ArcPy cursor token names for object IDs and geometry objects/properties.
Args:
dataset_path (str): Path of the dataset.
field_names (iter): Collection of field names. Names will be the keys in the
dictionary mapping to their values. If value is None, all attributes fields
will be used.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
spatial_reference_item: Item from which the spatial reference of the output
geometry will be derived.
Yields:
dict.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("spatial_reference_item")
meta = {"spatial": spatial_reference_metadata(kwargs["spatial_reference_item"])}
if field_names is None:
meta["dataset"] = dataset_metadata(dataset_path)
keys = {"feature": [key for key in meta["dataset"]["field_names_tokenized"]]}
else:
keys = {"feature": list(contain(field_names))}
cursor = arcpy.da.SearchCursor(
in_table=dataset_path,
field_names=keys["feature"],
where_clause=kwargs["dataset_where_sql"],
spatial_reference=meta["spatial"]["object"],
)
with cursor:
for feature in cursor:
yield dict(zip(cursor.fields, feature))
def insert_from_iters(dataset_path, insert_features, field_names, **kwargs):
"""Insert features into dataset from iterables.
Args:
dataset_path (str): Path of the dataset.
insert_features (iter of iter): Collection of iterables representing features.
field_names (iter): Collection of field names to insert. These must match the
order of their attributes in the insert_features items.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
use_edit_session (bool): Flag to perform updates in an edit session. Default is
False.
log_level (str): Level to log the function at. Default is 'info'.
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault('use_edit_session', False)
log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info'))
log("Start: Insert features into %s from iterables.", dataset_path)
meta = {'dataset': arcobj.dataset_metadata(dataset_path)}
keys = {'row': list(contain(field_names))}
if inspect.isgeneratorfunction(insert_features):
insert_features = insert_features()
session = arcobj.Editor(meta['dataset']['workspace_path'],
kwargs['use_edit_session'])
cursor = arcpy.da.InsertCursor(dataset_path, field_names=keys['row'])
feature_count = Counter()
with session, cursor:
for row in insert_features:
cursor.insertRow(tuple(row))
feature_count['inserted'] += 1
log("%s features inserted.", feature_count['inserted'])
log("End: Insert.")
return feature_count
def insert_from_dicts(dataset_path, insert_features, field_names, **kwargs):
"""Insert features into dataset from dictionaries.
Args:
dataset_path (str): Path of the dataset.
insert_features (iter of dict): Collection of dictionaries representing
features.
field_names (iter): Collection of field names/keys to insert.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
use_edit_session (bool): Flag to perform updates in an edit session. Default is
False.
log_level (str): Level to log the function at. Default is 'info'.
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault('use_edit_session', False)
log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info'))
log("Start: Insert features into %s from dictionaries.", dataset_path)
keys = {'row': list(contain(field_names))}
if inspect.isgeneratorfunction(insert_features):
insert_features = insert_features()
iters = ((feature[key] for key in keys['row'])
for feature in insert_features)
feature_count = insert_from_iters(
dataset_path,
iters,
field_names,
use_edit_session=kwargs['use_edit_session'],
log_level=None,
)
log("%s features inserted.", feature_count['inserted'])
log("End: Insert.")
return feature_count
def insert_from_path(dataset_path,
insert_dataset_path,
field_names=None,
**kwargs):
"""Insert features into dataset from another dataset.
Args:
dataset_path (str): Path of the dataset.
insert_dataset_path (str): Path of dataset to insert features from.
field_names (iter): Collection of field names to insert. Listed field must be
present in both datasets. If field_names is None, all fields will be
inserted.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
insert_where_sql (str): SQL where-clause for insert-dataset subselection.
use_edit_session (bool): Flag to perform updates in an edit session. Default is
False.
log_level (str): Level to log the function at. Default is 'info'.
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault('insert_where_sql')
kwargs.setdefault('use_edit_session', False)
log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info'))
log("Start: Insert features into %s from %s.", dataset_path,
insert_dataset_path)
meta = {
'dataset': arcobj.dataset_metadata(dataset_path),
'insert': arcobj.dataset_metadata(insert_dataset_path),
}
if field_names is None:
keys = set.intersection(*(set(
name.lower() for name in _meta['field_names_tokenized'])
for _meta in meta.values()))
else:
keys = set(name.lower() for name in contain(field_names))
# OIDs & area/length "fields" have no business being part of an insert.
# Geometry itself is handled separately in append function.
for _meta in meta.values():
for key in chain(*_meta['field_token'].items()):
keys.discard(key)
append_kwargs = {
'inputs': unique_name('view'),
'target': dataset_path,
'schema_type': 'no_test',
'field_mapping': arcpy.FieldMappings(),
}
# Create field maps.
# ArcGIS Pro's no-test append is case-sensitive (verified 1.0-1.1.1).
# Avoid this problem by using field mapping.
# BUG-000090970 - ArcGIS Pro 'No test' field mapping in Append tool does not auto-
# map to the same field name if naming convention differs.
for key in keys:
field_map = arcpy.FieldMap()
field_map.addInputField(insert_dataset_path, key)
append_kwargs['field_mapping'].addFieldMap(field_map)
view = arcobj.DatasetView(
insert_dataset_path,
kwargs['insert_where_sql'],
view_name=append_kwargs['inputs'],
# Must be nonspatial to append to nonspatial table.
force_nonspatial=(not meta['dataset']['is_spatial']),
)
session = arcobj.Editor(meta['dataset']['workspace_path'],
kwargs['use_edit_session'])
with view, session:
arcpy.management.Append(**append_kwargs)
feature_count = Counter({'inserted': view.count})
log("%s features inserted.", feature_count['inserted'])
log("End: Insert.")
return feature_count
def dissolve(dataset_path,
dissolve_field_names=None,
multipart=True,
**kwargs):
"""Dissolve geometry of features that share values in given fields.
Args:
dataset_path (str): Path of the dataset.
dissolve_field_names (iter): Iterable of field names to dissolve on.
multipart (bool): Flag to allow multipart features in output.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
unsplit_lines (bool): Flag to merge line features when endpoints meet without
crossing features. Default is False.
dataset_where_sql (str): SQL where-clause for dataset subselection.
tolerance (float): Tolerance for coincidence, in dataset's units.
use_edit_session (bool): Flag to perform updates in an edit session. Default is
False.
log_level (str): Level to log the function at. Default is 'info'.
Returns:
str: Path of the dataset updated.
"""
kwargs.setdefault('unsplit_lines', False)
kwargs.setdefault('dataset_where_sql')
kwargs.setdefault('use_edit_session', False)
log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info'))
log(
"Start: Dissolve features in %s on fields: %s.",
dataset_path,
dissolve_field_names,
)
meta = {
'dataset': arcobj.dataset_metadata(dataset_path),
'orig_tolerance': arcpy.env.XYTolerance,
}
keys = {'dissolve': tuple(contain(dissolve_field_names))}
view = {
'dataset': arcobj.DatasetView(dataset_path,
kwargs['dataset_where_sql'])
}
temp_output_path = unique_path('output')
with view['dataset']:
if 'tolerance' in kwargs:
arcpy.env.XYTolerance = kwargs['tolerance']
arcpy.management.Dissolve(
in_features=view['dataset'].name,
out_feature_class=temp_output_path,
dissolve_field=keys['dissolve'],
multi_part=multipart,
unsplit_lines=kwargs['unsplit_lines'],
)
if 'tolerance' in kwargs:
arcpy.env.XYTolerance = meta['orig_tolerance']
session = arcobj.Editor(meta['dataset']['workspace_path'],
kwargs['use_edit_session'])
with session:
delete(dataset_path,
dataset_where_sql=kwargs['dataset_where_sql'],
log_level=None)
insert_from_path(dataset_path,
insert_dataset_path=temp_output_path,
log_level=None)
dataset.delete(temp_output_path, log_level=None)
log("End: Dissolve.")
return dataset_path
def add_index(dataset_path, field_names, **kwargs):
"""Add index to dataset fields.
Note:
Index names can only be applied to non-spatial indexes for geodatabase feature
classes and tables.
There is a limited length allowed for index names; longer names will be
truncated without warning.
Args:
dataset_path (str): Path of the dataset.
field_names (iter): Collection of participating field names.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
index_name (str): Name for index. Optional; see note.
is_ascending (bool): Build index in ascending order if True. Default is False.
is_unique (bool): Build index with unique constraint if True. Default is False.
fail_on_lock_ok (bool): If True, indicate success even if dataset locks prevent
adding index. Default is False.
log_level (str): Level to log the function at. Default is "info".
Returns:
str: Path of the dataset receiving the index.
Raises:
RuntimeError: If more than one field and any are geometry-types.
arcpy.ExecuteError: If dataset lock prevents adding index.
"""
field_names = [name.lower() for name in contain(field_names)]
kwargs.setdefault("index_name", "ndx_" + "_".join(field_names))
kwargs.setdefault("is_ascending", False)
kwargs.setdefault("is_unique", False)
kwargs.setdefault("fail_on_lock_ok", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log("Start: Add index to field(s) %s on %s.", field_names, dataset_path)
meta = {"dataset": dataset_metadata(dataset_path)}
meta["field_types"] = {
field["type"].lower()
for field in meta["dataset"]["fields"]
if field["name"].lower() in field_names
}
if "geometry" in meta["field_types"]:
if len(field_names) > 1:
raise RuntimeError("Cannot create a composite spatial index.")
exec_add = arcpy.management.AddSpatialIndex
add_kwargs = {"in_features": dataset_path}
else:
exec_add = arcpy.management.AddIndex
add_kwargs = {
"in_table": dataset_path,
"fields": field_names,
"index_name": kwargs["index_name"],
"unique": kwargs["is_unique"],
"ascending": kwargs["is_ascending"],
}
try:
exec_add(**add_kwargs)
except arcpy.ExecuteError as error:
if error.message.startswith("ERROR 000464"):
LOG.warning("Lock on %s prevents adding index.", dataset_path)
if not kwargs["fail_on_lock_ok"]:
raise
log("End: Add.")
return dataset_path
def update_from_iters(dataset_path, update_features, id_field_names,
field_names, **kwargs):
"""Update features in dataset from iterables.
Note:
There is no guarantee that the ID field(s) are unique.
Use ArcPy cursor token names for object IDs and geometry objects/properties.
Args:
dataset_path (str): Path of the dataset.
update_features (iter of dict): Collection of iterables representing features.
id_field_names (iter, str): Name(s) of the ID field/key(s). *All* ID fields
must also be in field_names.
field_names (iter): Collection of field names/keys to check & update.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
delete_missing_features (bool): True if update should delete features missing
from update_features, False otherwise. Default is True.
use_edit_session (bool): Flag to perform updates in an edit session. Default is
True.
log_level (str): Level to log the function at. Default is 'info'.
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault('delete_missing_features', True)
kwargs.setdefault('use_edit_session', True)
log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info'))
log("Start: Update features in %s from iterables.", dataset_path)
meta = {'dataset': arcobj.dataset_metadata(dataset_path)}
keys = {
'id': list(contain(id_field_names)),
'feat': list(contain(field_names))
}
if not set(keys['id']).issubset(keys['feat']):
raise ValueError("id_field_names must be a subset of field_names.")
ids = {
'dataset': {
tuple(freeze_values(*_id))
for _id in attributes.as_iters(dataset_path, keys['id'])
}
}
if inspect.isgeneratorfunction(update_features):
update_features = update_features()
feats = {'insert': set(), 'id_update': dict()}
for feat in update_features:
feat = tuple(freeze_values(*feat))
_id = tuple(feat[keys['feat'].index(key)] for key in keys['id'])
if _id not in ids['dataset']:
feats['insert'].add(feat)
else:
feats['id_update'][_id] = feat
if kwargs['delete_missing_features']:
ids['delete'] = {
_id
for _id in ids['dataset'] if _id not in feats['id_update']
}
else:
ids['delete'] = set()
feature_count = Counter()
session = arcobj.Editor(meta['dataset']['workspace_path'],
kwargs['use_edit_session'])
if ids['delete'] or feats['id_update']:
cursor = arcpy.da.UpdateCursor(dataset_path, field_names=keys['feat'])
with session, cursor:
for feat in cursor:
_id = tuple(
freeze_values(*(feat[keys['feat'].index(key)]
for key in keys['id'])))
if _id in ids['delete']:
cursor.deleteRow()
feature_count['deleted'] += 1
continue
elif (_id in feats['id_update'] and
not arcobj.same_feature(feat, feats['id_update'][_id])):
cursor.updateRow(feats['id_update'][_id])
feature_count['altered'] += 1
else:
feature_count['unchanged'] += 1
if feats['insert']:
cursor = arcpy.da.InsertCursor(dataset_path, field_names=keys['feat'])
with session, cursor:
for feat in feats['insert']:
try:
cursor.insertRow(feat)
except RuntimeError:
LOG.error(
"Feature failed to write to cursor. Offending row:")
for key, val in zip(keys['feat'], feat):
LOG.error("%s: %s", key, val)
raise
feature_count['inserted'] += 1
for key in UPDATE_TYPES:
log("%s features %s.", feature_count[key], key)
log("End: Update.")
return feature_count
def update_by_geometry(dataset_path, field_name, geometry_properties, **kwargs):
"""Update attribute values by cascading through geometry properties.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
geometry_properties (iter): Geometry property names in object-access order to
retrieve the update value.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
spatial_reference_item: Item from which the spatial reference for the output
geometry property will be derived. Default is the update dataset.
use_edit_session (bool): Updates are done in an edit session if True. If not
not specified or None, the spatial reference of the dataset is used.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("spatial_reference_item")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s by geometry properties %s.",
field_name,
dataset_path,
geometry_properties,
)
meta = {
"dataset": dataset_metadata(dataset_path),
"spatial": spatial_reference_metadata(kwargs["spatial_reference_item"]),
}
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=["shape@", field_name],
where_clause=kwargs["dataset_where_sql"],
spatial_reference=meta["spatial"]["object"],
)
update_action_count = Counter()
with session, cursor:
for feature in cursor:
value = {"geometry": feature[0], "old": feature[-1]}
value["new"] = property_value(
value["geometry"],
GEOMETRY_PROPERTY_TRANSFORM,
*contain(geometry_properties)
)
if same_value(value["old"], value["new"]):
update_action_count["unchanged"] += 1
else:
try:
cursor.updateRow([value["geometry"], value["new"]])
update_action_count["altered"] += 1
except RuntimeError:
LOG.error("Offending value is %s", value["new"])
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
def update_by_function(dataset_path, field_name, function, **kwargs):
"""Update attribute values by passing them to a function.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
function (types.FunctionType): Function to get values from.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
field_as_first_arg (bool): True if field value will be the first positional
argument. Default is True.
arg_field_names (iter): Field names whose values will be the positional
arguments (not including primary field).
kwarg_field_names (iter): Field names whose names & values will be the method
keyword arguments.
dataset_where_sql (str): SQL where-clause for dataset subselection.
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("field_as_first_arg", True)
kwargs.setdefault("arg_field_names", [])
kwargs.setdefault("kwarg_field_names", [])
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s by function %s.",
field_name,
dataset_path,
function,
)
meta = {"dataset": dataset_metadata(dataset_path)}
keys = {
"args": list(contain(kwargs["arg_field_names"])),
"kwargs": list(contain(kwargs["kwarg_field_names"])),
}
keys["feature"] = keys["args"] + keys["kwargs"] + [field_name]
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=keys["feature"],
where_clause=kwargs["dataset_where_sql"],
)
update_action_count = Counter()
with session, cursor:
for feature in cursor:
value = {
"old": feature[-1],
"args": feature[: len(keys["args"])],
"kwargs": dict(zip(keys["kwargs"], feature[len(keys["args"]) : -1])),
}
if kwargs["field_as_first_arg"]:
value["args"] = [value["old"]] + value["args"]
value["new"] = function(*value["args"], **value["kwargs"])
if same_value(value["old"], value["new"]):
update_action_count["unchanged"] += 1
else:
try:
cursor.updateRow(feature[:-1] + [value["new"]])
update_action_count["altered"] += 1
except RuntimeError:
LOG.error("Offending value is %s", value["new"])
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
def update_by_feature_match(
dataset_path, field_name, id_field_names, update_type, **kwargs
):
"""Update attribute values by aggregating info about matching features.
Note: Currently, the sort_order update type uses functionality that only works with
datasets contained in databases.
Valid update_type codes:
"flag_value": Apply the flag_value argument value to matched features.
"match_count": Apply the count of matched features.
"sort_order": Apply the position of the feature sorted with matches.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
id_field_names (iter): Field names used to identify a feature.
update_type (str): Code indicating what values to apply to matched features.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
flag_value: Value to apply to matched features. Only used when update_type is
"flag_value".
sort_field_names (iter): Iterable of field names used to sort matched features.
Only affects output when update_type="sort_order".
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s"
+ " by feature-matching %s on identifiers (%s).",
field_name,
dataset_path,
update_type.replace("_", " "),
id_field_names,
)
if update_type not in ["flag_value", "match_count", "sort_order"]:
raise ValueError("Invalid update_type.")
for _type, kwarg in {
"flag_value": "flag_value",
"sort_order": "sort_field_names",
}.items():
if update_type == _type and kwarg not in kwargs:
raise TypeError(
"""{} is required keyword argument when update_type == "{}", .""".format(
_type, kwarg
)
)
meta = {"dataset": dataset_metadata(dataset_path)}
keys = {
"id": list(contain(id_field_names)),
"sort": list(contain(kwargs.get("sort_field_names", []))),
}
keys["feature"] = keys["id"] + [field_name]
matcher = FeatureMatcher(dataset_path, keys["id"], kwargs["dataset_where_sql"])
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=keys["feature"],
where_clause=kwargs["dataset_where_sql"],
sql_clause=(
(None, "order by " + ", ".join(keys["sort"]))
if update_type == "sort_order"
else None
),
)
update_action_count = Counter()
with session, cursor:
for feature in cursor:
value = {
"id": feature[0] if len(keys["id"]) == 1 else tuple(feature[:-1]),
"old": feature[-1],
}
if update_type == "flag_value":
if matcher.is_duplicate(value["id"]):
value["new"] = kwargs["flag_value"]
else:
value["new"] = value["old"]
elif update_type == "match_count":
value["new"] = matcher.match_count(value["id"])
elif update_type == "sort_order":
value["new"] = matcher.increment_assigned(value["id"])
if same_value(value["old"], value["new"]):
update_action_count["unchanged"] += 1
else:
try:
cursor.updateRow(feature[:-1] + [value["new"]])
update_action_count["altered"] += 1
except RuntimeError:
LOG.error("Offending value is %s", value["new"])
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
def id_node_map(
dataset_path,
from_id_field_name,
to_id_field_name,
id_field_names=("oid@",),
**kwargs
):
"""Return mapping of feature ID to from- & to-node ID dictionary.
Notes:
From & to IDs must be same attribute type.
Default output format: `{feature_id: {"from": from_node_id, "to": to_node_id}}`
Args:
dataset_path (str): Path of the dataset.
from_id_field_name (str): Name of from-ID field.
to_id_field_name (str): Name of to-ID field.
id_field_names (iter, str): Name(s) of the ID field(s).
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
field_names_as_keys (bool): Use of node ID field names as keys in the map-value
if True; use "from" and "to" if False. Default is False.
update_nodes (bool): Update nodes based on feature geometries if True. Default
is False.
Returns:
dict: Mapping of feature IDs to node-end ID dictionaries.
`{feature_id: {"from": from_node_id, "to": to_node_id}}`
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("field_names_as_keys", False)
kwargs.setdefault("update_nodes", False)
keys = {
"id": list(contain(id_field_names)),
"node": {
"from": from_id_field_name if kwargs["field_names_as_keys"] else "from",
"to": to_id_field_name if kwargs["field_names_as_keys"] else "to",
},
}
keys["feature"] = [from_id_field_name, to_id_field_name] + keys["id"]
id_nodes = defaultdict(dict)
# If updating nodes, need to gather geometry/coordinates.
if kwargs["update_nodes"]:
coordinate_node = coordinate_node_map(
dataset_path, from_id_field_name, to_id_field_name, keys["id"], **kwargs
)
for node in coordinate_node.values():
for end in ["from", "to"]:
for feature_id in node["ids"][end]:
id_nodes[feature_id][keys["node"][end]] = node["node_id"]
else:
for feature in as_iters(
dataset_path,
field_names=keys["feature"],
dataset_where_sql=kwargs["dataset_where_sql"],
):
from_node_id, to_node_id = feature[:2]
feature_id = feature[2:]
if len(keys["id"]) == 1:
feature_id = feature_id[0]
id_nodes[feature_id][keys["node"]["from"]] = from_node_id
id_nodes[feature_id][keys["node"]["to"]] = to_node_id
return id_nodes
def coordinate_node_map(
dataset_path,
from_id_field_name,
to_id_field_name,
id_field_names=("oid@",),
**kwargs
):
"""Return mapping of coordinates to node-info dictionary.
Notes:
From & to IDs must be same attribute type.
Default output format:
{(x, y): {"node_id": <id>, "ids": {"from": set(), "to": set()}}}
Args:
dataset_path (str): Path of the dataset.
from_id_field_name (str): Name of the from-ID field.
to_id_field_name (str): Name of the to-ID field.
id_field_names (iter, str): Name(s) of the ID field(s).
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
update_nodes (bool): Update nodes based on feature geometries if True. Default
is False.
Returns:
dict.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("update_nodes", False)
meta = {
"from_id_field": field_metadata(dataset_path, from_id_field_name),
"to_id_field": field_metadata(dataset_path, to_id_field_name),
}
if meta["from_id_field"]["type"] != meta["to_id_field"]["type"]:
raise ValueError("From- and to-ID fields must be of same type.")
keys = {"id": list(contain(id_field_names))}
keys["feature"] = ["shape@", from_id_field_name, to_id_field_name] + keys["id"]
coordinate_node = {}
for feature in as_iters(
dataset_path, keys["feature"], dataset_where_sql=kwargs["dataset_where_sql"]
):
_id = tuple(feature[3:])
if len(keys["id"]) == 1:
_id = _id[0]
geom = feature[0]
node_id = {"from": feature[1], "to": feature[2]}
coordinate = {
"from": (geom.firstPoint.X, geom.firstPoint.Y),
"to": (geom.lastPoint.X, geom.lastPoint.Y),
}
for end in ["from", "to"]:
if coordinate[end] not in coordinate_node:
# Create new coordinate-node.
coordinate_node[coordinate[end]] = {
"node_id": node_id[end],
"ids": defaultdict(set),
}
# Assign new ID if current is missing.
if coordinate_node[coordinate[end]]["node_id"] is None:
coordinate_node[coordinate[end]]["node_id"] = node_id[end]
# Assign lower ID if different than current.
else:
coordinate_node[coordinate[end]]["node_id"] = min(
coordinate_node[coordinate[end]]["node_id"], node_id[end]
)
# Add feature ID to end-ID set.
coordinate_node[coordinate[end]]["ids"][end].add(_id)
if kwargs["update_nodes"]:
coordinate_node = _update_coordinate_node_map(
coordinate_node, meta["from_id_field"]
)
return coordinate_node
def update_from_path(dataset_path,
update_dataset_path,
id_field_names,
field_names=None,
**kwargs):
"""Update features in dataset from another dataset.
Args:
dataset_path (str): Path of the dataset.
update_dataset_path (str): Path of dataset to update features from.
id_field_names (iter, str): Name(s) of the ID field/key(s).
field_names (iter): Collection of field names/keys to check & update. Listed
field must be present in both datasets. If field_names is None, all fields
will be inserted.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection. WARNING:
defining this has major effects: filtered features will not be considered
for updating or deletion, and duplicates in the update features will be
inserted as if novel.
update_where_sql (str): SQL where-clause for update-dataset subselection.
chunk_where_sqls (iter): Collection of SQL where-clauses for updating between
the datasets in chunks.
delete_missing_features (bool): True if update should delete features missing
from update_features, False otherwise. Default is True.
use_edit_session (bool): Flag to perform updates in an edit session. Default is
True.
log_level (str): Level to log the function at. Default is 'info'.
Returns:
collections.Counter: Counts for each feature action.
"""
for key in ['dataset_where_sql', 'update_where_sql']:
kwargs.setdefault(key)
if not kwargs[key]:
kwargs[key] = "1=1"
kwargs.setdefault('subset_where_sqls', ["1=1"])
kwargs.setdefault('delete_missing_features', True)
kwargs.setdefault('use_edit_session', True)
log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info'))
log("Start: Update features in %s from %s.", dataset_path,
update_dataset_path)
meta = {
'dataset': arcobj.dataset_metadata(dataset_path),
'update': arcobj.dataset_metadata(update_dataset_path),
}
if field_names is None:
field_names = (
set(name.lower()
for name in meta['dataset']['field_names_tokenized'])
& set(name.lower()
for name in meta['update']['field_names_tokenized']))
else:
field_names = set(name.lower() for name in field_names)
# OIDs & area/length "fields" have no business being part of an update.
for key in ['oid@', 'shape@area', 'shape@length']:
field_names.discard(key)
keys = {
'id': list(contain(id_field_names)),
'attr': list(contain(field_names))
}
keys['row'] = keys['id'] + keys['attr']
feature_count = Counter()
for kwargs['subset_where_sql'] in contain(kwargs['subset_where_sqls']):
if not kwargs['subset_where_sql'] == "1=1":
log("Subset: `%s`", kwargs['subset_where_sql'])
iters = attributes.as_iters(
update_dataset_path,
keys['row'],
dataset_where_sql=(
"({update_where_sql}) and ({subset_where_sql})".format(
**kwargs)))
view = arcobj.DatasetView(
dataset_path,
dataset_where_sql=(
"({dataset_where_sql}) and ({subset_where_sql})".format(
**kwargs)),
field_names=keys['row'])
with view:
feature_count.update(
update_from_iters(
dataset_path=view.name,
update_features=iters,
id_field_names=keys['id'],
field_names=keys['row'],
delete_missing_features=kwargs['delete_missing_features'],
use_edit_session=kwargs['use_edit_session'],
log_level=None,
))
for key in UPDATE_TYPES:
log("%s features %s.", feature_count[key], key)
log("End: Update.")
return feature_count
def update_by_mapping(dataset_path, field_name, mapping, key_field_names, **kwargs):
"""Update attribute values by finding them in a mapping.
Note: Mapping key must be a tuple if an iterable.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
mapping: Mapping to get values from.
key_field_names (iter): Fields names whose values will comprise the mapping key.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
default_value: Value to return from mapping if key value on feature not
present. Default is None.
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("default_value")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s by mapping with key in %s.",
field_name,
dataset_path,
key_field_names,
)
meta = {"dataset": dataset_metadata(dataset_path)}
keys = {"map": list(contain(key_field_names))}
keys["feature"] = keys["map"] + [field_name]
if isinstance(mapping, EXEC_TYPES):
mapping = mapping()
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=keys["feature"],
where_clause=kwargs["dataset_where_sql"],
)
update_action_count = Counter()
with session, cursor:
for feature in cursor:
value = {
"map_key": feature[0] if len(keys["map"]) == 1 else tuple(feature[:-1]),
"old": feature[-1],
}
value["new"] = mapping.get(value["map_key"], kwargs["default_value"])
if same_value(value["old"], value["new"]):
update_action_count["unchanged"] += 1
else:
try:
cursor.updateRow(feature[:-1] + [value["new"]])
update_action_count["altered"] += 1
except RuntimeError:
LOG.error("Offending value is %s", value["new"])
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
