def update_by_value(
dataset_path: Union[Path, str],
field_name: str,
*,
value: Any,
dataset_where_sql: Optional[str] = None,
use_edit_session: bool = False,
log_level: int = logging.INFO,
) -> Counter:
"""Update attribute values by assigning a given value.
Args:
dataset_path: Path to dataset.
field_name: Name of field.
value: Value to assign.
dataset_where_sql: SQL where-clause for dataset subselection.
use_edit_session: True if edits are to be made in an edit session.
log_level: Level to log the function at.
Returns:
Attribute counts for each update-state.
Raises:
RuntimeError: If attribute cannot be updated.
"""
dataset_path = Path(dataset_path)
LOG.log(
log_level,
"Start: Update attributes in `%s.%s` by given value.",
dataset_path,
field_name,
)
cursor = arcpy.da.UpdateCursor(
# ArcPy2.8.0: Convert to str.
in_table=str(dataset_path),
field_names=[field_name],
where_clause=dataset_where_sql,
)
session = Editing(Dataset(dataset_path).workspace_path, use_edit_session)
states = Counter()
with session, cursor:
for (old_value, ) in cursor:
if same_value(old_value, value):
states["unchanged"] += 1
else:
try:
cursor.updateRow([value])
states["altered"] += 1
except RuntimeError as error:
raise RuntimeError(
f"Update cursor failed: Offending value: `{value}`"
) from error
log_entity_states("attributes", states, logger=LOG, log_level=log_level)
LOG.log(log_level, "End: Update.")
return states
def update_rows(
dataset_path: Union[Path, str],
*,
field_name: str,
id_field_names: Iterable[str],
cmp_dataset_path: Union[Path, str],
cmp_field_name: Optional[str] = None,
cmp_id_field_names: Optional[Iterable[str]] = None,
cmp_date: Optional[Union[date, _datetime]] = None,
date_initiated_field_name: str = "date_initiated",
date_expired_field_name: str = "date_expired",
use_edit_session: bool = False,
log_level: int = logging.INFO,
) -> Counter:
"""Add field value changes to tracking dataset from comparison dataset.
Args:
dataset_path: Path to tracking dataset.
field_name: Name of field with tracked attribute.
id_field_names: Names of the feature ID fields.
cmp_dataset_path: Path to comparison dataset.
cmp_field_name: Name of field with tracked attribute in comparison dataset. If
set to None, will assume same as field_name.
cmp_id_field_names: Names of the feature ID fields in comparison dataset. If set
to None, will assume same as field_name.
cmp_date: Date to mark comparison change. If set to None, will set to the date
of execution.
date_initiated_field_name: Name of tracking-row-inititated date field.
date_expired_field_name: Name of tracking-row-expired date field.
use_edit_session: True if edits are to be made in an edit session.
log_level: Level to log the function at.
Returns:
Feature counts for each update-state.
"""
dataset_path = Path(dataset_path)
cmp_dataset_path = Path(cmp_dataset_path)
LOG.log(
log_level,
"Start: Update tracking rows in `%s` from `%s`.",
dataset_path,
cmp_dataset_path,
)
id_field_names = list(id_field_names)
if cmp_field_name is None:
cmp_field_name = field_name
cmp_id_field_names = (id_field_names if cmp_id_field_names is None else
list(cmp_id_field_names))
if cmp_date is None:
cmp_date = date.today()
current_where_sql = f"{date_expired_field_name} IS NULL"
id_current_value = {
row[:-1]: row[-1]
for row in features.as_tuples(
dataset_path,
field_names=id_field_names + [field_name],
dataset_where_sql=current_where_sql,
)
}
id_cmp_value = {
row[:-1]: row[-1]
for row in features.as_tuples(cmp_dataset_path,
field_names=cmp_id_field_names +
[cmp_field_name])
}
changed_ids = set()
expired_ids = {_id for _id in id_current_value if _id not in id_cmp_value}
new_rows = []
for _id, value in id_cmp_value.items():
if _id not in id_current_value:
new_rows.append(_id + (value, cmp_date))
elif not same_value(value, id_current_value[_id]):
changed_ids.add(_id)
new_rows.append(_id + (value, cmp_date))
# ArcPy2.8.0: Convert Path to str.
cursor = arcpy.da.UpdateCursor(
in_table=str(dataset_path),
field_names=id_field_names + [field_name, date_expired_field_name],
where_clause=current_where_sql,
)
session = Editing(Dataset(dataset_path).workspace_path, use_edit_session)
states = Counter()
with session, cursor:
for row in cursor:
_id = tuple(row[:len(id_field_names)])
if _id in changed_ids or _id in expired_ids:
cursor.updateRow(_id + (row[-2], cmp_date))
else:
states["unchanged"] += 1
features.insert_from_iters(
dataset_path,
field_names=id_field_names + [field_name, date_initiated_field_name],
source_features=new_rows,
use_edit_session=use_edit_session,
log_level=logging.DEBUG,
)
states["changed"] = len(changed_ids)
states["expired"] = len(expired_ids)
log_entity_states("features", states, logger=LOG, log_level=log_level)
LOG.log(log_level, "End: Update.")
return states
def consolidate_rows(
dataset_path: Union[Path, str],
*,
field_name: str,
id_field_names: Iterable[str],
date_initiated_field_name: str = "date_initiated",
date_expired_field_name: str = "date_expired",
use_edit_session: bool = False,
log_level: int = logging.INFO,
) -> Counter:
"""Consolidate tracking dataset rows where the value does not actually change.
Useful for quick-loaded point-in-time values, or for processing hand-altered rows.
Args:
dataset_path: Path to tracking dataset.
field_name: Name of field with tracked attribute.
id_field_names: Names of the feature ID fields.
date_initiated_field_name: Name of tracking-row-inititated date field.
date_expired_field_name: Name of tracking-row-expired date field.
use_edit_session: True if edits are to be made in an edit session.
log_level: Level to log the function at.
Returns:
Feature counts for each update-state.
"""
dataset_path = Path(dataset_path)
LOG.log(log_level, "Start: Consolidate tracking rows in `%s`.",
dataset_path)
id_field_names = list(id_field_names)
field_names = id_field_names + [
date_initiated_field_name,
date_expired_field_name,
field_name,
]
id_rows = defaultdict(list)
for row in features.as_dicts(dataset_path, field_names=field_names):
_id = tuple(row[name] for name in id_field_names)
id_rows[_id].append(row)
for _id in list(id_rows):
rows = sorted(id_rows[_id], key=itemgetter(date_initiated_field_name))
for i, row in enumerate(rows):
if i == 0 or row[date_initiated_field_name] is None:
continue
date_initiated = row[date_initiated_field_name]
value = row[field_name]
previous_row = rows[i - 1]
previous_value = previous_row[field_name]
previous_date_expired = previous_row[date_expired_field_name]
if same_value(value, previous_value) and same_value(
date_initiated, previous_date_expired):
# Move previous row date initiated to current row & clear from previous.
row[date_initiated_field_name] = previous_row[
date_initiated_field_name]
previous_row[date_initiated_field_name] = None
id_rows[_id] = [
row for row in rows if row[date_initiated_field_name] is not None
]
states = features.update_from_dicts(
dataset_path,
field_names=field_names,
# In tracking dataset, ID is ID + date_initiated.
id_field_names=id_field_names + [date_initiated_field_name],
source_features=chain(*id_rows.values()),
use_edit_session=use_edit_session,
log_level=logging.DEBUG,
)
log_entity_states("tracking rows", states, logger=LOG, log_level=log_level)
LOG.log(log_level, "End: Consolidate.")
return states
def update_by_overlay_count(
dataset_path: Union[Path, str],
field_name: str,
*,
overlay_dataset_path: Union[Path, str],
dataset_where_sql: Optional[str] = None,
overlay_where_sql: Optional[str] = None,
tolerance: Optional[float] = None,
use_edit_session: bool = False,
log_level: int = logging.INFO,
) -> Counter:
"""Update attribute values by count of overlay features.
Args:
dataset_path: Path to dataset.
field_name: Name of field.
overlay_dataset_path: Path to overlay-dataset.
Keyword Args:
dataset_where_sql: SQL where-clause for dataset subselection.
overlay_where_sql: SQL where-clause for overlay-dataset subselection.
tolerance: Tolerance for coincidence, in units of the dataset. If set to None,
will use the default tolerance for the workspace of the dataset.
use_edit_session: True if edits are to be made in an edit session.
log_level: Level to log the function at.
Returns:
Attribute counts for each update-state.
Raises:
RuntimeError: If attribute cannot be updated.
"""
dataset_path = Path(dataset_path)
overlay_dataset_path = Path(overlay_dataset_path)
LOG.log(
log_level,
"Start: Update attributes in `%s.%s` by overlay feature counts from `%s`.",
dataset_path,
field_name,
overlay_dataset_path,
)
original_tolerance = arcpy.env.XYTolerance
view = DatasetView(dataset_path,
field_names=[],
dataset_where_sql=dataset_where_sql)
overlay_view = DatasetView(
overlay_dataset_path,
field_names=[],
dataset_where_sql=overlay_where_sql,
)
with view, overlay_view:
if tolerance is not None:
arcpy.env.XYTolerance = tolerance
temp_output_path = unique_path("output")
arcpy.analysis.SpatialJoin(
target_features=view.name,
join_features=overlay_view.name,
# ArcPy2.8.0: Convert to str.
out_feature_class=str(temp_output_path),
join_operation="JOIN_ONE_TO_ONE",
join_type="KEEP_COMMON",
match_option="INTERSECT",
)
arcpy.env.XYTolerance = original_tolerance
cursor = arcpy.da.SearchCursor(
# ArcPy2.8.0: Convert to str.
in_table=str(temp_output_path),
field_names=["TARGET_FID", "Join_Count"],
)
with cursor:
oid_overlay_count = dict(cursor)
# ArcPy2.8.0: Convert to str.
arcpy.management.Delete(str(temp_output_path))
cursor = arcpy.da.UpdateCursor(
# ArcPy2.8.0: Convert to str.
in_table=str(dataset_path),
field_names=["OID@", field_name],
where_clause=dataset_where_sql,
)
session = Editing(Dataset(dataset_path).workspace_path, use_edit_session)
states = Counter()
with session, cursor:
for feature in cursor:
oid = feature[0]
old_value = feature[1]
new_value = oid_overlay_count.get(oid, 0)
if same_value(old_value, new_value):
states["unchanged"] += 1
else:
try:
cursor.updateRow([oid, new_value])
states["altered"] += 1
except RuntimeError as error:
raise RuntimeError(
f"Update cursor failed: Offending value: `{new_value}`"
) from error
log_entity_states("attributes", states, logger=LOG, log_level=log_level)
LOG.log(log_level, "End: Update.")
return states
def update_by_mapping(
dataset_path: Union[Path, str],
field_name: str,
*,
mapping: Union[Mapping, FunctionType],
key_field_names: Iterable[str],
dataset_where_sql: Optional[str] = None,
default_value: Any = None,
use_edit_session: bool = False,
log_level: int = logging.INFO,
) -> Counter:
"""Update attribute values by finding them in a mapping.
Notes:
Mapping key must be a tuple if an iterable.
Args:
dataset_path: Path to dataset.
field_name: Name of field.
mapping: Mapping to get values from.
key_field_names: Names of mapping key fields.
dataset_where_sql: SQL where-clause for dataset subselection.
default_value: Value to assign mapping if key value not in mapping.
use_edit_session: True if edits are to be made in an edit session.
log_level: Level to log the function at.
Returns:
Attribute counts for each update-state.
Raises:
RuntimeError: If attribute cannot be updated.
"""
dataset_path = Path(dataset_path)
key_field_names = list(key_field_names)
LOG.log(
log_level,
"Start: Update attributes in `%s.%s` by mapping with key in `%s`.",
dataset_path,
field_name,
key_field_names,
)
if isinstance(mapping, EXECUTABLE_TYPES):
mapping = mapping()
cursor = arcpy.da.UpdateCursor(
# ArcPy2.8.0: Convert to str.
in_table=str(dataset_path),
field_names=key_field_names + [field_name],
where_clause=dataset_where_sql,
)
session = Editing(Dataset(dataset_path).workspace_path, use_edit_session)
states = Counter()
with session, cursor:
for feature in cursor:
key = feature[0] if len(key_field_names) == 1 else tuple(
feature[:-1])
old_value = feature[-1]
new_value = mapping.get(key, default_value)
if same_value(old_value, new_value):
states["unchanged"] += 1
else:
try:
cursor.updateRow(feature[:-1] + [new_value])
states["altered"] += 1
except RuntimeError as error:
raise RuntimeError(
f"Update cursor failed: Offending value: `{new_value}`"
) from error
log_entity_states("attributes", states, logger=LOG, log_level=log_level)
LOG.log(log_level, "End: Update.")
return states
def update_by_joined_value(
dataset_path: Union[Path, str],
field_name: str,
*,
key_field_names: Iterable[str],
join_dataset_path: Union[Path, str],
join_field_name: str,
join_key_field_names: Iterable[str],
dataset_where_sql: Optional[str] = None,
join_dataset_where_sql: Optional[str] = None,
use_edit_session: bool = False,
log_level: int = logging.INFO,
) -> Counter:
"""Update attribute values by referencing a joinable field in another dataset.
key_field_names & join_key_field_names must be the same length & same order.
Args:
dataset_path: Path to dataset.
field_name: Name of field.
key_field_names: Names of relationship key fields.
join_dataset_path: Path to join-dataset.
join_field_name: Name of join-field.
join_key_field_names: Names of relationship key fields on join-dataset.
dataset_where_sql: SQL where-clause for dataset subselection.
join_dataset_where_sql: SQL where-clause for join-dataset subselection.
use_edit_session: Updates are done in an edit session if True.
use_edit_session: True if edits are to be made in an edit session.
log_level: Level to log the function at.
Returns:
Attribute counts for each update-state.
Raises:
AttributeError: If key_field_names & join_key_field_names have different length.
RuntimeError: If attribute cannot be updated.
"""
dataset_path = Path(dataset_path)
join_dataset_path = Path(join_dataset_path)
LOG.log(
log_level,
"Start: Update attributes in `%s.%s` by joined values in `%s.%s`.",
dataset_path,
field_name,
join_dataset_path,
join_field_name,
)
key_field_names = list(key_field_names)
join_key_field_names = list(join_key_field_names)
if len(key_field_names) != len(join_key_field_names):
raise AttributeError(
"key_field_names & join_key_field_names not same length.")
cursor = arcpy.da.SearchCursor(
# ArcPy2.8.0: Convert to str.
in_table=str(join_dataset_path),
field_names=join_key_field_names + [join_field_name],
where_clause=join_dataset_where_sql,
)
with cursor:
id_join_value = {feature[:-1]: feature[-1] for feature in cursor}
cursor = arcpy.da.UpdateCursor(
# ArcPy2.8.0: Convert to str.
in_table=str(dataset_path),
field_names=key_field_names + [field_name],
where_clause=dataset_where_sql,
)
session = Editing(Dataset(dataset_path).workspace_path, use_edit_session)
states = Counter()
with session, cursor:
for feature in cursor:
old_value = feature[-1]
new_value = id_join_value.get(tuple(feature[:-1]))
if same_value(old_value, new_value):
states["unchanged"] += 1
else:
try:
cursor.updateRow(feature[:-1] + [new_value])
states["altered"] += 1
except RuntimeError as error:
raise RuntimeError(
f"Update cursor failed: Offending value: `{new_value}`"
) from error
log_entity_states("attributes", states, logger=LOG, log_level=log_level)
LOG.log(log_level, "End: Update.")
return states
def update_by_function(
dataset_path: Union[Path, str],
field_name: str,
*,
function: FunctionType,
field_as_first_arg: bool = True,
arg_field_names: Iterable[str] = (),
kwarg_field_names: Iterable[str] = (),
dataset_where_sql: Optional[str] = None,
spatial_reference_item: SpatialReferenceSourceItem = None,
use_edit_session: bool = False,
log_level: int = logging.INFO,
) -> Counter:
"""Update attribute values by passing them to a function.
Args:
dataset_path: Path to dataset.
field_name: Name of field.
function: Function to return values from.
field_as_first_arg: True if field value will be the first positional argument.
arg_field_names: Field names whose values will be the function positional
arguments (not including primary field).
kwarg_field_names: Field names whose names & values will be the function keyword
arguments.
dataset_where_sql: SQL where-clause for dataset subselection.
spatial_reference_item: Item from which the spatial reference for any geometry
properties will be set to. If set to None, will use spatial reference of
the dataset.
use_edit_session: True if edits are to be made in an edit session.
log_level: Level to log the function at.
Returns:
Attribute counts for each update-state.
Raises:
RuntimeError: If attribute cannot be updated.
"""
dataset_path = Path(dataset_path)
LOG.log(
log_level,
"Start: Update attributes in `%s.%s` by function `%s`.",
dataset_path,
field_name,
# Partials show all the pre-loaded arg & kwarg values, which is cumbersome.
"partial version of function {}".format(function.func) if isinstance(
function, partial) else "function `{}`".format(function),
)
arg_field_names = list(arg_field_names)
kwarg_field_names = list(kwarg_field_names)
cursor = arcpy.da.UpdateCursor(
# ArcPy2.8.0: Convert to str.
in_table=str(dataset_path),
field_names=arg_field_names + kwarg_field_names + [field_name],
where_clause=dataset_where_sql,
spatial_reference=SpatialReference(spatial_reference_item).object,
)
session = Editing(Dataset(dataset_path).workspace_path, use_edit_session)
states = Counter()
with session, cursor:
for feature in cursor:
old_value = feature[-1]
args = feature[:len(arg_field_names)]
if field_as_first_arg:
args = [old_value] + args
kwargs = dict(
zip(kwarg_field_names, feature[len(arg_field_names):-1]))
new_value = function(*args, **kwargs)
if same_value(old_value, new_value):
states["unchanged"] += 1
else:
try:
cursor.updateRow(feature[:-1] + [new_value])
states["altered"] += 1
except RuntimeError as error:
raise RuntimeError(
f"Update cursor failed: Offending value: `{new_value}`"
) from error
log_entity_states("attributes", states, logger=LOG, log_level=log_level)
LOG.log(log_level, "End: Update.")
return states
def update_by_field(
dataset_path: Union[Path, str],
field_name: str,
*,
source_field_name: str,
dataset_where_sql: Optional[str] = None,
spatial_reference_item: SpatialReferenceSourceItem = None,
use_edit_session: bool = False,
log_level: int = logging.INFO,
) -> Counter:
"""Update attribute values with values from another field.
Args:
dataset_path: Path to dataset.
field_name: Name of field.
source_field_name: Name of field to get values from.
dataset_where_sql: SQL where-clause for dataset subselection.
spatial_reference_item: Item from which the spatial reference for any geometry
properties will be set to. If set to None, will use spatial reference of
the dataset.
use_edit_session: True if edits are to be made in an edit session.
log_level: Level to log the function at.
Returns:
Attribute counts for each update-state.
Raises:
RuntimeError: If attribute cannot be updated.
"""
dataset_path = Path(dataset_path)
LOG.log(
log_level,
"Start: Update attributes in `%s.%s` by field `%s`.",
dataset_path,
field_name,
source_field_name,
)
cursor = arcpy.da.UpdateCursor(
# ArcPy2.8.0: Convert to str.
in_table=str(dataset_path),
field_names=[field_name, source_field_name],
where_clause=dataset_where_sql,
spatial_reference=SpatialReference(spatial_reference_item).object,
)
session = Editing(Dataset(dataset_path).workspace_path, use_edit_session)
states = Counter()
with session, cursor:
for old_value, new_value in cursor:
if same_value(old_value, new_value):
states["unchanged"] += 1
else:
try:
cursor.updateRow([new_value, new_value])
states["altered"] += 1
except RuntimeError as error:
raise RuntimeError(
f"Update cursor failed: Offending value: `{new_value}`"
) from error
log_entity_states("attributes", states, logger=LOG, log_level=log_level)
LOG.log(log_level, "End: Update.")
return states