def update_by_value(dataset_path, field_name, value, **kwargs): """Update attribute values by assigning a given value. Args: dataset_path (str): Path of the dataset. field_name (str): Name of the field. value (object): Static value to assign. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: dataset_where_sql (str): SQL where-clause for dataset subselection. use_edit_session (bool): Updates are done in an edit session if True. Default is False. log_level (str): Level to log the function at. Default is "info". Returns: collections.Counter: Counts for each feature action. """ kwargs.setdefault("dataset_where_sql") kwargs.setdefault("use_edit_session", True) log = leveled_logger(LOG, kwargs.setdefault("log_level", "info")) log( "Start: Update attributes in %s on %s by given value.", field_name, dataset_path ) meta = {"dataset": dataset_metadata(dataset_path)} session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"]) cursor = arcpy.da.UpdateCursor( in_table=dataset_path, field_names=[field_name], where_clause=kwargs["dataset_where_sql"], ) update_action_count = Counter() with session, cursor: for [old_value] in cursor: if same_value(old_value, value): update_action_count["unchanged"] += 1 else: try: cursor.updateRow([value]) update_action_count["altered"] += 1 except RuntimeError: LOG.error("Offending value is %s", value) raise for action, count in sorted(update_action_count.items()): log("%s attributes %s.", count, action) log("End: Update.") return update_action_count
def update_by_expression(dataset_path, field_name, expression, **kwargs): """Update attribute values using a (single) code-expression. Wraps arcpy.management.CalculateField. Args: dataset_path (str): Path of the dataset. field_name (str): Name of the field. expression (str): Python string expression to evaluate values from. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: dataset_where_sql (str): SQL where-clause for dataset subselection. use_edit_session (bool): Updates are done in an edit session if True. Default is False. log_level (str): Level to log the function at. Default is "info". Returns: str: Name of the field updated. """ kwargs.setdefault("dataset_where_sql") kwargs.setdefault("use_edit_session", False) log = leveled_logger(LOG, kwargs.setdefault("log_level", "info")) log( "Start: Update attributes in %s on %s using expression: `%s`.", field_name, dataset_path, expression, ) meta = {"dataset": dataset_metadata(dataset_path)} session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"]) dataset_view = DatasetView(dataset_path, kwargs["dataset_where_sql"]) with session, dataset_view: arcpy.management.CalculateField( in_table=dataset_view.name, field=field_name, expression=expression, expression_type="python_9.3", ) log("End: Update.") return field_name
def adjust_for_shapefile(dataset_path, **kwargs): """Adjust features to meet shapefile requirements. Note: Shapefiles cannot have null-values. Nulls will be replaced with the values provided in the null replacement keyword arguments. Shapefiles only have dates in the date/datetime field. Times will be truncated in the adjustment. Args: dataset_path (str): Path of the dataset. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: dataset_where_sql (str): SQL where-clause for dataset subselection. date_null_replacement (datetime.date): Replacement value for null-values in date fields. Default is datetime.date.min. integer_null_replacement (int): Replacement value for null-values in integer fields. Default is 0. numeric_null_replacement (float): Replacement value for null-values in numeric fields. Default is 0.0. string_null_replacement (str): Replacement value for null-values in string fields. Default is "". use_edit_session (bool): Updates are done in an edit session if True. Default is False. log_level (str): Level to log the function at. Default is "info". Returns: str: Path of the adjusted dataset. """ kwargs.setdefault("dataset_where_sql") kwargs.setdefault("use_edit_session", False) log = leveled_logger(LOG, kwargs.setdefault("log_level", "info")) log("Start: Adjust features for shapefile output in %s.", dataset_path) replacement_value = { "date": kwargs.setdefault("date_null_replacement", datetime.date.min), "double": kwargs.setdefault("numeric_null_replacement", 0.0), "single": kwargs.setdefault("numeric_null_replacement", 0.0), "integer": kwargs.setdefault("integer_null_replacement", 0), "smallinteger": kwargs.setdefault("integer_null_replacement", 0), "string": kwargs.setdefault("string_null_replacement", ""), # Shapefile loader handles non-user fields seperately. # "geometry", "oid", } meta = {"dataset": dataset_metadata(dataset_path)} session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"]) with session: for field in meta["dataset"]["user_fields"]: if field["type"].lower() not in replacement_value: log("Skipping %s field: type cannot transfer to shapefile.", field["name"]) continue else: log("Adjusting values in %s field.", field["name"]) cursor = arcpy.da.UpdateCursor( in_table=dataset_path, field_names=[field["name"]], where_clause=kwargs["dataset_where_sql"], ) with cursor: for (old_value, ) in cursor: if old_value is None: new_value = replacement_value[field["type"].lower()] try: cursor.updateRow([new_value]) except RuntimeError: LOG.error("Offending value is %s", new_value) raise RuntimeError log("End: Adjust.") return dataset_path
def update_by_node_ids(dataset_path, from_id_field_name, to_id_field_name, **kwargs): """Update attribute values by passing them to a function. Args: dataset_path (str): Path of the dataset. from_id_field_name (str): Name of the from-ID field. to_id_field_name (str): Name of the to-ID field. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: dataset_where_sql (str): SQL where-clause for dataset subselection. use_edit_session (bool): Updates are done in an edit session if True. Default is False. log_level (str): Level to log the function at. Default is "info". Returns: collections.Counter: Counts for each feature action. """ kwargs.setdefault("dataset_where_sql") kwargs.setdefault("use_edit_session", False) log = leveled_logger(LOG, kwargs.setdefault("log_level", "info")) log( "Start: Update attributes in %s & %s on %s by node IDs.", from_id_field_name, to_id_field_name, dataset_path, ) meta = {"dataset": dataset_metadata(dataset_path)} keys = {"feature": ["oid@", from_id_field_name, to_id_field_name]} oid_node = id_node_map( dataset_path, from_id_field_name, to_id_field_name, update_nodes=True ) session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"]) cursor = arcpy.da.UpdateCursor( in_table=dataset_path, field_names=keys["feature"], where_clause=kwargs["dataset_where_sql"], ) update_action_count = Counter() with session, cursor: for feature in cursor: value = {"oid": feature[0], "old_nodes": feature[1:]} value["new_nodes"] = [ oid_node[value["oid"]]["from"], oid_node[value["oid"]]["to"], ] if same_feature(value["old_nodes"], value["new_nodes"]): update_action_count["unchanged"] += 1 else: try: cursor.updateRow([value["oid"]] + value["new_nodes"]) update_action_count["altered"] += 1 except RuntimeError: LOG.error("Offending value one of %s", value["new_nodes"]) raise for action, count in sorted(update_action_count.items()): log("%s attributes %s.", count, action) log("End: Update.") return update_action_count
def update_by_mapping(dataset_path, field_name, mapping, key_field_names, **kwargs): """Update attribute values by finding them in a mapping. Note: Mapping key must be a tuple if an iterable. Args: dataset_path (str): Path of the dataset. field_name (str): Name of the field. mapping: Mapping to get values from. key_field_names (iter): Fields names whose values will comprise the mapping key. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: dataset_where_sql (str): SQL where-clause for dataset subselection. default_value: Value to return from mapping if key value on feature not present. Default is None. use_edit_session (bool): Updates are done in an edit session if True. Default is False. log_level (str): Level to log the function at. Default is "info". Returns: collections.Counter: Counts for each feature action. """ kwargs.setdefault("dataset_where_sql") kwargs.setdefault("default_value") kwargs.setdefault("use_edit_session", False) log = leveled_logger(LOG, kwargs.setdefault("log_level", "info")) log( "Start: Update attributes in %s on %s by mapping with key in %s.", field_name, dataset_path, key_field_names, ) meta = {"dataset": dataset_metadata(dataset_path)} keys = {"map": list(contain(key_field_names))} keys["feature"] = keys["map"] + [field_name] if isinstance(mapping, EXEC_TYPES): mapping = mapping() session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"]) cursor = arcpy.da.UpdateCursor( in_table=dataset_path, field_names=keys["feature"], where_clause=kwargs["dataset_where_sql"], ) update_action_count = Counter() with session, cursor: for feature in cursor: value = { "map_key": feature[0] if len(keys["map"]) == 1 else tuple(feature[:-1]), "old": feature[-1], } value["new"] = mapping.get(value["map_key"], kwargs["default_value"]) if same_value(value["old"], value["new"]): update_action_count["unchanged"] += 1 else: try: cursor.updateRow(feature[:-1] + [value["new"]]) update_action_count["altered"] += 1 except RuntimeError: LOG.error("Offending value is %s", value["new"]) raise for action, count in sorted(update_action_count.items()): log("%s attributes %s.", count, action) log("End: Update.") return update_action_count
def update_by_joined_value( dataset_path, field_name, join_dataset_path, join_field_name, on_field_pairs, **kwargs ): """Update attribute values by referencing a joinable field. Args: dataset_path (str): Path of the dataset. field_name (str): Name of the field. join_dataset_path (str): Path of the join-dataset. join_field_name (str): Name of the join-field. on_field_pairs (iter): Field name pairs used to to determine join. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: dataset_where_sql (str): SQL where-clause for dataset subselection. use_edit_session (bool): Updates are done in an edit session if True. Default is False. log_level (str): Level to log the function at. Default is "info". Returns: collections.Counter: Counts for each feature action. """ kwargs.setdefault("dataset_where_sql") kwargs.setdefault("use_edit_session", False) log = leveled_logger(LOG, kwargs.setdefault("log_level", "info")) log( "Start: Update attributes in %s on %s by joined values in %s on %s.", field_name, dataset_path, join_field_name, join_dataset_path, ) meta = {"dataset": dataset_metadata(dataset_path)} keys = { "dataset_id": list(pair[0] for pair in on_field_pairs), "join_id": list(pair[1] for pair in on_field_pairs), } keys["feature"] = keys["dataset_id"] + [field_name] join_value = id_map( join_dataset_path, id_field_names=keys["join_id"], field_names=join_field_name ) session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"]) cursor = arcpy.da.UpdateCursor( in_table=dataset_path, field_names=keys["feature"], where_clause=kwargs["dataset_where_sql"], ) update_action_count = Counter() with session, cursor: for feature in cursor: value = { "id": ( feature[0] if len(keys["dataset_id"]) == 1 else tuple(feature[:-1]) ), "old": feature[-1], } value["new"] = join_value.get(value["id"]) if same_value(value["old"], value["new"]): update_action_count["unchanged"] += 1 else: try: cursor.updateRow(feature[:-1] + [value["new"]]) update_action_count["altered"] += 1 except RuntimeError: LOG.error("Offending value is %s", value["new"]) raise for action, count in sorted(update_action_count.items()): log("%s attributes %s.", count, action) log("End: Update.") return update_action_count
def update_by_geometry(dataset_path, field_name, geometry_properties, **kwargs): """Update attribute values by cascading through geometry properties. Args: dataset_path (str): Path of the dataset. field_name (str): Name of the field. geometry_properties (iter): Geometry property names in object-access order to retrieve the update value. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: dataset_where_sql (str): SQL where-clause for dataset subselection. spatial_reference_item: Item from which the spatial reference for the output geometry property will be derived. Default is the update dataset. use_edit_session (bool): Updates are done in an edit session if True. If not not specified or None, the spatial reference of the dataset is used. log_level (str): Level to log the function at. Default is "info". Returns: collections.Counter: Counts for each feature action. """ kwargs.setdefault("dataset_where_sql") kwargs.setdefault("spatial_reference_item") kwargs.setdefault("use_edit_session", False) log = leveled_logger(LOG, kwargs.setdefault("log_level", "info")) log( "Start: Update attributes in %s on %s by geometry properties %s.", field_name, dataset_path, geometry_properties, ) meta = { "dataset": dataset_metadata(dataset_path), "spatial": spatial_reference_metadata(kwargs["spatial_reference_item"]), } session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"]) cursor = arcpy.da.UpdateCursor( in_table=dataset_path, field_names=["shape@", field_name], where_clause=kwargs["dataset_where_sql"], spatial_reference=meta["spatial"]["object"], ) update_action_count = Counter() with session, cursor: for feature in cursor: value = {"geometry": feature[0], "old": feature[-1]} value["new"] = property_value( value["geometry"], GEOMETRY_PROPERTY_TRANSFORM, *contain(geometry_properties) ) if same_value(value["old"], value["new"]): update_action_count["unchanged"] += 1 else: try: cursor.updateRow([value["geometry"], value["new"]]) update_action_count["altered"] += 1 except RuntimeError: LOG.error("Offending value is %s", value["new"]) raise for action, count in sorted(update_action_count.items()): log("%s attributes %s.", count, action) log("End: Update.") return update_action_count
def update_by_function(dataset_path, field_name, function, **kwargs): """Update attribute values by passing them to a function. Args: dataset_path (str): Path of the dataset. field_name (str): Name of the field. function (types.FunctionType): Function to get values from. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: field_as_first_arg (bool): True if field value will be the first positional argument. Default is True. arg_field_names (iter): Field names whose values will be the positional arguments (not including primary field). kwarg_field_names (iter): Field names whose names & values will be the method keyword arguments. dataset_where_sql (str): SQL where-clause for dataset subselection. use_edit_session (bool): Updates are done in an edit session if True. Default is False. log_level (str): Level to log the function at. Default is "info". Returns: collections.Counter: Counts for each feature action. """ kwargs.setdefault("field_as_first_arg", True) kwargs.setdefault("arg_field_names", []) kwargs.setdefault("kwarg_field_names", []) kwargs.setdefault("dataset_where_sql") kwargs.setdefault("use_edit_session", False) log = leveled_logger(LOG, kwargs.setdefault("log_level", "info")) log( "Start: Update attributes in %s on %s by function %s.", field_name, dataset_path, function, ) meta = {"dataset": dataset_metadata(dataset_path)} keys = { "args": list(contain(kwargs["arg_field_names"])), "kwargs": list(contain(kwargs["kwarg_field_names"])), } keys["feature"] = keys["args"] + keys["kwargs"] + [field_name] session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"]) cursor = arcpy.da.UpdateCursor( in_table=dataset_path, field_names=keys["feature"], where_clause=kwargs["dataset_where_sql"], ) update_action_count = Counter() with session, cursor: for feature in cursor: value = { "old": feature[-1], "args": feature[: len(keys["args"])], "kwargs": dict(zip(keys["kwargs"], feature[len(keys["args"]) : -1])), } if kwargs["field_as_first_arg"]: value["args"] = [value["old"]] + value["args"] value["new"] = function(*value["args"], **value["kwargs"]) if same_value(value["old"], value["new"]): update_action_count["unchanged"] += 1 else: try: cursor.updateRow(feature[:-1] + [value["new"]]) update_action_count["altered"] += 1 except RuntimeError: LOG.error("Offending value is %s", value["new"]) raise for action, count in sorted(update_action_count.items()): log("%s attributes %s.", count, action) log("End: Update.") return update_action_count
def update_by_feature_match( dataset_path, field_name, id_field_names, update_type, **kwargs ): """Update attribute values by aggregating info about matching features. Note: Currently, the sort_order update type uses functionality that only works with datasets contained in databases. Valid update_type codes: "flag_value": Apply the flag_value argument value to matched features. "match_count": Apply the count of matched features. "sort_order": Apply the position of the feature sorted with matches. Args: dataset_path (str): Path of the dataset. field_name (str): Name of the field. id_field_names (iter): Field names used to identify a feature. update_type (str): Code indicating what values to apply to matched features. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: dataset_where_sql (str): SQL where-clause for dataset subselection. flag_value: Value to apply to matched features. Only used when update_type is "flag_value". sort_field_names (iter): Iterable of field names used to sort matched features. Only affects output when update_type="sort_order". use_edit_session (bool): Updates are done in an edit session if True. Default is False. log_level (str): Level to log the function at. Default is "info". Returns: collections.Counter: Counts for each feature action. """ kwargs.setdefault("dataset_where_sql") kwargs.setdefault("use_edit_session", False) log = leveled_logger(LOG, kwargs.setdefault("log_level", "info")) log( "Start: Update attributes in %s on %s" + " by feature-matching %s on identifiers (%s).", field_name, dataset_path, update_type.replace("_", " "), id_field_names, ) if update_type not in ["flag_value", "match_count", "sort_order"]: raise ValueError("Invalid update_type.") for _type, kwarg in { "flag_value": "flag_value", "sort_order": "sort_field_names", }.items(): if update_type == _type and kwarg not in kwargs: raise TypeError( """{} is required keyword argument when update_type == "{}", .""".format( _type, kwarg ) ) meta = {"dataset": dataset_metadata(dataset_path)} keys = { "id": list(contain(id_field_names)), "sort": list(contain(kwargs.get("sort_field_names", []))), } keys["feature"] = keys["id"] + [field_name] matcher = FeatureMatcher(dataset_path, keys["id"], kwargs["dataset_where_sql"]) session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"]) cursor = arcpy.da.UpdateCursor( in_table=dataset_path, field_names=keys["feature"], where_clause=kwargs["dataset_where_sql"], sql_clause=( (None, "order by " + ", ".join(keys["sort"])) if update_type == "sort_order" else None ), ) update_action_count = Counter() with session, cursor: for feature in cursor: value = { "id": feature[0] if len(keys["id"]) == 1 else tuple(feature[:-1]), "old": feature[-1], } if update_type == "flag_value": if matcher.is_duplicate(value["id"]): value["new"] = kwargs["flag_value"] else: value["new"] = value["old"] elif update_type == "match_count": value["new"] = matcher.match_count(value["id"]) elif update_type == "sort_order": value["new"] = matcher.increment_assigned(value["id"]) if same_value(value["old"], value["new"]): update_action_count["unchanged"] += 1 else: try: cursor.updateRow(feature[:-1] + [value["new"]]) update_action_count["altered"] += 1 except RuntimeError: LOG.error("Offending value is %s", value["new"]) raise for action, count in sorted(update_action_count.items()): log("%s attributes %s.", count, action) log("End: Update.") return update_action_count
def update_by_unique_id(dataset_path, field_name, **kwargs): """Update attribute values by assigning a unique ID. Existing IDs are preserved, if unique. Args: dataset_path (str): Path of the dataset. field_name (str): Name of the field. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: dataset_where_sql (str): SQL where-clause for dataset subselection. use_edit_session (bool): Updates are done in an edit session if True. Default is False. log_level (str): Level to log the function at. Default is "info". Returns: collections.Counter: Counts for each feature action. """ kwargs.setdefault("dataset_where_sql") kwargs.setdefault("use_edit_session", True) log = leveled_logger(LOG, kwargs.setdefault("log_level", "info")) log( "Start: Update attributes in %s on %s by assigning unique IDs.", field_name, dataset_path, ) meta = { "dataset": dataset_metadata(dataset_path), "field": field_metadata(dataset_path, field_name), } session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"]) cursor = arcpy.da.UpdateCursor( in_table=dataset_path, field_names=[field_name], where_clause=kwargs["dataset_where_sql"], ) with session: used_ids = set() # First run will clear duplicate IDs & gather used IDs. with cursor: for [id_value] in cursor: if id_value in used_ids: cursor.updateRow([None]) else: used_ids.add(id_value) id_pool = unique_ids( data_type=python_type(meta["field"]["type"]), string_length=meta["field"].get("length"), ) # Second run will fill in missing IDs. update_action_count = Counter() with cursor: for [id_value] in cursor: if id_value is not None: update_action_count["unchanged"] += 1 else: id_value = next(id_pool) while id_value in used_ids: id_value = next(id_pool) try: cursor.updateRow([id_value]) update_action_count["altered"] += 1 used_ids.add(id_value) except RuntimeError: LOG.error("Offending value is %s", id_value) raise for action, count in sorted(update_action_count.items()): log("%s attributes %s.", count, action) log("End: Update.") return update_action_count