def update_by_value(dataset_path, field_name, value, **kwargs):
"""Update attribute values by assigning a given value.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
value (object): Static value to assign.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("use_edit_session", True)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s by given value.", field_name, dataset_path
)
meta = {"dataset": dataset_metadata(dataset_path)}
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=[field_name],
where_clause=kwargs["dataset_where_sql"],
)
update_action_count = Counter()
with session, cursor:
for [old_value] in cursor:
if same_value(old_value, value):
update_action_count["unchanged"] += 1
else:
try:
cursor.updateRow([value])
update_action_count["altered"] += 1
except RuntimeError:
LOG.error("Offending value is %s", value)
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
def update_by_expression(dataset_path, field_name, expression, **kwargs):
"""Update attribute values using a (single) code-expression.
Wraps arcpy.management.CalculateField.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
expression (str): Python string expression to evaluate values from.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
str: Name of the field updated.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s using expression: `%s`.",
field_name,
dataset_path,
expression,
)
meta = {"dataset": dataset_metadata(dataset_path)}
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
dataset_view = DatasetView(dataset_path, kwargs["dataset_where_sql"])
with session, dataset_view:
arcpy.management.CalculateField(
in_table=dataset_view.name,
field=field_name,
expression=expression,
expression_type="python_9.3",
)
log("End: Update.")
return field_name
def adjust_for_shapefile(dataset_path, **kwargs):
"""Adjust features to meet shapefile requirements.
Note:
Shapefiles cannot have null-values. Nulls will be replaced with the values
provided in the null replacement keyword arguments.
Shapefiles only have dates in the date/datetime field. Times will be truncated
in the adjustment.
Args:
dataset_path (str): Path of the dataset.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
date_null_replacement (datetime.date): Replacement value for null-values in
date fields. Default is datetime.date.min.
integer_null_replacement (int): Replacement value for null-values in integer
fields. Default is 0.
numeric_null_replacement (float): Replacement value for null-values in numeric
fields. Default is 0.0.
string_null_replacement (str): Replacement value for null-values in string
fields. Default is "".
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
str: Path of the adjusted dataset.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log("Start: Adjust features for shapefile output in %s.", dataset_path)
replacement_value = {
"date": kwargs.setdefault("date_null_replacement", datetime.date.min),
"double": kwargs.setdefault("numeric_null_replacement", 0.0),
"single": kwargs.setdefault("numeric_null_replacement", 0.0),
"integer": kwargs.setdefault("integer_null_replacement", 0),
"smallinteger": kwargs.setdefault("integer_null_replacement", 0),
"string": kwargs.setdefault("string_null_replacement", ""),
# Shapefile loader handles non-user fields seperately.
# "geometry", "oid",
}
meta = {"dataset": dataset_metadata(dataset_path)}
session = Editor(meta["dataset"]["workspace_path"],
kwargs["use_edit_session"])
with session:
for field in meta["dataset"]["user_fields"]:
if field["type"].lower() not in replacement_value:
log("Skipping %s field: type cannot transfer to shapefile.",
field["name"])
continue
else:
log("Adjusting values in %s field.", field["name"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=[field["name"]],
where_clause=kwargs["dataset_where_sql"],
)
with cursor:
for (old_value, ) in cursor:
if old_value is None:
new_value = replacement_value[field["type"].lower()]
try:
cursor.updateRow([new_value])
except RuntimeError:
LOG.error("Offending value is %s", new_value)
raise RuntimeError
log("End: Adjust.")
return dataset_path
def update_by_node_ids(dataset_path, from_id_field_name, to_id_field_name, **kwargs):
"""Update attribute values by passing them to a function.
Args:
dataset_path (str): Path of the dataset.
from_id_field_name (str): Name of the from-ID field.
to_id_field_name (str): Name of the to-ID field.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s & %s on %s by node IDs.",
from_id_field_name,
to_id_field_name,
dataset_path,
)
meta = {"dataset": dataset_metadata(dataset_path)}
keys = {"feature": ["oid@", from_id_field_name, to_id_field_name]}
oid_node = id_node_map(
dataset_path, from_id_field_name, to_id_field_name, update_nodes=True
)
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=keys["feature"],
where_clause=kwargs["dataset_where_sql"],
)
update_action_count = Counter()
with session, cursor:
for feature in cursor:
value = {"oid": feature[0], "old_nodes": feature[1:]}
value["new_nodes"] = [
oid_node[value["oid"]]["from"],
oid_node[value["oid"]]["to"],
]
if same_feature(value["old_nodes"], value["new_nodes"]):
update_action_count["unchanged"] += 1
else:
try:
cursor.updateRow([value["oid"]] + value["new_nodes"])
update_action_count["altered"] += 1
except RuntimeError:
LOG.error("Offending value one of %s", value["new_nodes"])
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
def update_by_mapping(dataset_path, field_name, mapping, key_field_names, **kwargs):
"""Update attribute values by finding them in a mapping.
Note: Mapping key must be a tuple if an iterable.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
mapping: Mapping to get values from.
key_field_names (iter): Fields names whose values will comprise the mapping key.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
default_value: Value to return from mapping if key value on feature not
present. Default is None.
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("default_value")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s by mapping with key in %s.",
field_name,
dataset_path,
key_field_names,
)
meta = {"dataset": dataset_metadata(dataset_path)}
keys = {"map": list(contain(key_field_names))}
keys["feature"] = keys["map"] + [field_name]
if isinstance(mapping, EXEC_TYPES):
mapping = mapping()
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=keys["feature"],
where_clause=kwargs["dataset_where_sql"],
)
update_action_count = Counter()
with session, cursor:
for feature in cursor:
value = {
"map_key": feature[0] if len(keys["map"]) == 1 else tuple(feature[:-1]),
"old": feature[-1],
}
value["new"] = mapping.get(value["map_key"], kwargs["default_value"])
if same_value(value["old"], value["new"]):
update_action_count["unchanged"] += 1
else:
try:
cursor.updateRow(feature[:-1] + [value["new"]])
update_action_count["altered"] += 1
except RuntimeError:
LOG.error("Offending value is %s", value["new"])
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
def update_by_joined_value(
dataset_path,
field_name,
join_dataset_path,
join_field_name,
on_field_pairs,
**kwargs
):
"""Update attribute values by referencing a joinable field.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
join_dataset_path (str): Path of the join-dataset.
join_field_name (str): Name of the join-field.
on_field_pairs (iter): Field name pairs used to to determine join.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s by joined values in %s on %s.",
field_name,
dataset_path,
join_field_name,
join_dataset_path,
)
meta = {"dataset": dataset_metadata(dataset_path)}
keys = {
"dataset_id": list(pair[0] for pair in on_field_pairs),
"join_id": list(pair[1] for pair in on_field_pairs),
}
keys["feature"] = keys["dataset_id"] + [field_name]
join_value = id_map(
join_dataset_path, id_field_names=keys["join_id"], field_names=join_field_name
)
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=keys["feature"],
where_clause=kwargs["dataset_where_sql"],
)
update_action_count = Counter()
with session, cursor:
for feature in cursor:
value = {
"id": (
feature[0] if len(keys["dataset_id"]) == 1 else tuple(feature[:-1])
),
"old": feature[-1],
}
value["new"] = join_value.get(value["id"])
if same_value(value["old"], value["new"]):
update_action_count["unchanged"] += 1
else:
try:
cursor.updateRow(feature[:-1] + [value["new"]])
update_action_count["altered"] += 1
except RuntimeError:
LOG.error("Offending value is %s", value["new"])
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
def update_by_geometry(dataset_path, field_name, geometry_properties, **kwargs):
"""Update attribute values by cascading through geometry properties.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
geometry_properties (iter): Geometry property names in object-access order to
retrieve the update value.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
spatial_reference_item: Item from which the spatial reference for the output
geometry property will be derived. Default is the update dataset.
use_edit_session (bool): Updates are done in an edit session if True. If not
not specified or None, the spatial reference of the dataset is used.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("spatial_reference_item")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s by geometry properties %s.",
field_name,
dataset_path,
geometry_properties,
)
meta = {
"dataset": dataset_metadata(dataset_path),
"spatial": spatial_reference_metadata(kwargs["spatial_reference_item"]),
}
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=["shape@", field_name],
where_clause=kwargs["dataset_where_sql"],
spatial_reference=meta["spatial"]["object"],
)
update_action_count = Counter()
with session, cursor:
for feature in cursor:
value = {"geometry": feature[0], "old": feature[-1]}
value["new"] = property_value(
value["geometry"],
GEOMETRY_PROPERTY_TRANSFORM,
*contain(geometry_properties)
)
if same_value(value["old"], value["new"]):
update_action_count["unchanged"] += 1
else:
try:
cursor.updateRow([value["geometry"], value["new"]])
update_action_count["altered"] += 1
except RuntimeError:
LOG.error("Offending value is %s", value["new"])
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
def update_by_function(dataset_path, field_name, function, **kwargs):
"""Update attribute values by passing them to a function.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
function (types.FunctionType): Function to get values from.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
field_as_first_arg (bool): True if field value will be the first positional
argument. Default is True.
arg_field_names (iter): Field names whose values will be the positional
arguments (not including primary field).
kwarg_field_names (iter): Field names whose names & values will be the method
keyword arguments.
dataset_where_sql (str): SQL where-clause for dataset subselection.
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("field_as_first_arg", True)
kwargs.setdefault("arg_field_names", [])
kwargs.setdefault("kwarg_field_names", [])
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s by function %s.",
field_name,
dataset_path,
function,
)
meta = {"dataset": dataset_metadata(dataset_path)}
keys = {
"args": list(contain(kwargs["arg_field_names"])),
"kwargs": list(contain(kwargs["kwarg_field_names"])),
}
keys["feature"] = keys["args"] + keys["kwargs"] + [field_name]
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=keys["feature"],
where_clause=kwargs["dataset_where_sql"],
)
update_action_count = Counter()
with session, cursor:
for feature in cursor:
value = {
"old": feature[-1],
"args": feature[: len(keys["args"])],
"kwargs": dict(zip(keys["kwargs"], feature[len(keys["args"]) : -1])),
}
if kwargs["field_as_first_arg"]:
value["args"] = [value["old"]] + value["args"]
value["new"] = function(*value["args"], **value["kwargs"])
if same_value(value["old"], value["new"]):
update_action_count["unchanged"] += 1
else:
try:
cursor.updateRow(feature[:-1] + [value["new"]])
update_action_count["altered"] += 1
except RuntimeError:
LOG.error("Offending value is %s", value["new"])
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
def update_by_feature_match(
dataset_path, field_name, id_field_names, update_type, **kwargs
):
"""Update attribute values by aggregating info about matching features.
Note: Currently, the sort_order update type uses functionality that only works with
datasets contained in databases.
Valid update_type codes:
"flag_value": Apply the flag_value argument value to matched features.
"match_count": Apply the count of matched features.
"sort_order": Apply the position of the feature sorted with matches.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
id_field_names (iter): Field names used to identify a feature.
update_type (str): Code indicating what values to apply to matched features.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
flag_value: Value to apply to matched features. Only used when update_type is
"flag_value".
sort_field_names (iter): Iterable of field names used to sort matched features.
Only affects output when update_type="sort_order".
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("use_edit_session", False)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s"
+ " by feature-matching %s on identifiers (%s).",
field_name,
dataset_path,
update_type.replace("_", " "),
id_field_names,
)
if update_type not in ["flag_value", "match_count", "sort_order"]:
raise ValueError("Invalid update_type.")
for _type, kwarg in {
"flag_value": "flag_value",
"sort_order": "sort_field_names",
}.items():
if update_type == _type and kwarg not in kwargs:
raise TypeError(
"""{} is required keyword argument when update_type == "{}", .""".format(
_type, kwarg
)
)
meta = {"dataset": dataset_metadata(dataset_path)}
keys = {
"id": list(contain(id_field_names)),
"sort": list(contain(kwargs.get("sort_field_names", []))),
}
keys["feature"] = keys["id"] + [field_name]
matcher = FeatureMatcher(dataset_path, keys["id"], kwargs["dataset_where_sql"])
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=keys["feature"],
where_clause=kwargs["dataset_where_sql"],
sql_clause=(
(None, "order by " + ", ".join(keys["sort"]))
if update_type == "sort_order"
else None
),
)
update_action_count = Counter()
with session, cursor:
for feature in cursor:
value = {
"id": feature[0] if len(keys["id"]) == 1 else tuple(feature[:-1]),
"old": feature[-1],
}
if update_type == "flag_value":
if matcher.is_duplicate(value["id"]):
value["new"] = kwargs["flag_value"]
else:
value["new"] = value["old"]
elif update_type == "match_count":
value["new"] = matcher.match_count(value["id"])
elif update_type == "sort_order":
value["new"] = matcher.increment_assigned(value["id"])
if same_value(value["old"], value["new"]):
update_action_count["unchanged"] += 1
else:
try:
cursor.updateRow(feature[:-1] + [value["new"]])
update_action_count["altered"] += 1
except RuntimeError:
LOG.error("Offending value is %s", value["new"])
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count
def update_by_unique_id(dataset_path, field_name, **kwargs):
"""Update attribute values by assigning a unique ID.
Existing IDs are preserved, if unique.
Args:
dataset_path (str): Path of the dataset.
field_name (str): Name of the field.
**kwargs: Arbitrary keyword arguments. See below.
Keyword Args:
dataset_where_sql (str): SQL where-clause for dataset subselection.
use_edit_session (bool): Updates are done in an edit session if True. Default is
False.
log_level (str): Level to log the function at. Default is "info".
Returns:
collections.Counter: Counts for each feature action.
"""
kwargs.setdefault("dataset_where_sql")
kwargs.setdefault("use_edit_session", True)
log = leveled_logger(LOG, kwargs.setdefault("log_level", "info"))
log(
"Start: Update attributes in %s on %s by assigning unique IDs.",
field_name,
dataset_path,
)
meta = {
"dataset": dataset_metadata(dataset_path),
"field": field_metadata(dataset_path, field_name),
}
session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"])
cursor = arcpy.da.UpdateCursor(
in_table=dataset_path,
field_names=[field_name],
where_clause=kwargs["dataset_where_sql"],
)
with session:
used_ids = set()
# First run will clear duplicate IDs & gather used IDs.
with cursor:
for [id_value] in cursor:
if id_value in used_ids:
cursor.updateRow([None])
else:
used_ids.add(id_value)
id_pool = unique_ids(
data_type=python_type(meta["field"]["type"]),
string_length=meta["field"].get("length"),
)
# Second run will fill in missing IDs.
update_action_count = Counter()
with cursor:
for [id_value] in cursor:
if id_value is not None:
update_action_count["unchanged"] += 1
else:
id_value = next(id_pool)
while id_value in used_ids:
id_value = next(id_pool)
try:
cursor.updateRow([id_value])
update_action_count["altered"] += 1
used_ids.add(id_value)
except RuntimeError:
LOG.error("Offending value is %s", id_value)
raise
for action, count in sorted(update_action_count.items()):
log("%s attributes %s.", count, action)
log("End: Update.")
return update_action_count