def update_from_dicts(dataset_path, update_features, id_field_names, field_names, **kwargs): """Update features in dataset from dictionaries. Note: There is no guarantee that the ID field(s) are unique. Use ArcPy cursor token names for object IDs and geometry objects/properties. Args: dataset_path (str): Path of the dataset. update_features (iter of dict): Collection of dictionaries representing features. id_field_names (iter, str): Name(s) of the ID field/key(s). field_names (iter): Collection of field names/keys to check & update. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: delete_missing_features (bool): True if update should delete features missing from update_features, False otherwise. Default is True. use_edit_session (bool): Flag to perform updates in an edit session. Default is True. log_level (str): Level to log the function at. Default is 'info'. Returns: collections.Counter: Counts for each feature action. """ kwargs.setdefault('delete_missing_features', True) kwargs.setdefault('use_edit_session', True) log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info')) log("Start: Update features in %s from dictionaries.", dataset_path) keys = { 'id': list(contain(id_field_names)), 'attr': list(contain(field_names)) } keys['row'] = keys['id'] + keys['attr'] if inspect.isgeneratorfunction(update_features): update_features = update_features() iters = ((feature[key] for key in keys['row']) for feature in update_features) feature_count = update_from_iters( dataset_path, update_features=iters, id_field_names=keys['id'], field_names=keys['row'], delete_missing_features=kwargs['delete_missing_features'], use_edit_session=kwargs['use_edit_session'], log_level=None, ) for key in UPDATE_TYPES: log("%s features %s.", feature_count[key], key) log("End: Update.") return feature_count
def id_map(dataset_path, id_field_names, field_names, **kwargs): """Return mapping of feature ID to attribute or list of attributes. Notes: There is no guarantee that the ID value(s) are unique. Use ArcPy cursor token names for object IDs and geometry objects/properties. Args: dataset_path (str): Path of the dataset. id_field_names (iter, str): Name(s) of the ID field(s). field_names (iter, str): Name(s) of the field(s). **kwargs: Arbitrary keyword arguments. See below. Keyword Args: dataset_where_sql (str): SQL where-clause for dataset subselection. spatial_reference_item: Item from which the spatial reference of the output geometry will be derived. Returns: dict. """ kwargs.setdefault("dataset_where_sql") kwargs.setdefault("spatial_reference_item") meta = {"spatial": spatial_reference_metadata(kwargs["spatial_reference_item"])} keys = { "id": list(contain(id_field_names)), "attribute": list(contain(field_names)), } cursor = arcpy.da.SearchCursor( in_table=dataset_path, field_names=keys["id"] + keys["attribute"], where_clause=kwargs["dataset_where_sql"], spatial_reference=meta["spatial"]["object"], ) id_attributes = {} with cursor: for feature in cursor: value = { "id": feature[0] if len(keys["id"]) == 1 else feature[: len(keys["id"])], "attributes": ( feature[len(keys["id"])] if len(keys["attribute"]) == 1 else feature[len(keys["id"]) :] ), } id_attributes[value["id"]] = value["attributes"] return id_attributes
def delete_by_id(dataset_path, delete_ids, id_field_names, **kwargs): """Delete features in dataset with given IDs. Note: There is no guarantee that the ID field(s) are unique. Use ArcPy cursor token names for object IDs and geometry objects/properties. Args: dataset_path (str): Path of the dataset. delete_ids (iter): Collection of feature IDs. id_field_names (iter, str): Name(s) of the ID field/key(s). **kwargs: Arbitrary keyword arguments. See below. Keyword Args: use_edit_session (bool): Flag to perform updates in an edit session. Default is False. log_level (str): Level to log the function at. Default is 'info'. Returns: collections.Counter: Counts for each feature action. """ kwargs.setdefault('use_edit_session', False) log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info')) log("Start: Delete features in %s with given IDs.", dataset_path) meta = {'dataset': arcobj.dataset_metadata(dataset_path)} keys = {'id': list(contain(id_field_names))} if inspect.isgeneratorfunction(delete_ids): delete_ids = delete_ids() ids = {'delete': {tuple(contain(_id)) for _id in delete_ids}} feature_count = Counter() session = arcobj.Editor(meta['dataset']['workspace_path'], kwargs['use_edit_session']) ##TODO: Preserve possible generators! if ids['delete']: cursor = arcpy.da.UpdateCursor(dataset_path, field_names=keys['id']) with session, cursor: for row in cursor: _id = tuple(row) if _id in ids['delete']: cursor.deleteRow() feature_count['deleted'] += 1 else: feature_count['unchanged'] += 1 for key in ['deleted', 'unchanged']: log("%s features %s.", feature_count[key], key) log("End: Delete.") return feature_count
def as_iters(dataset_path, field_names, **kwargs): """Generate iterables of feature attribute values. Notes: Use ArcPy cursor token names for object IDs and geometry objects/properties. Args: dataset_path (str): Path of the dataset. field_names (iter): Collection of field names. The order of the names in the collection will determine where its value will fall in the generated item. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: dataset_where_sql (str): SQL where-clause for dataset subselection. spatial_reference_item: Item from which the spatial reference of the output geometry will be derived. iter_type: Iterable type to yield. Default is tuple. Yields: iter. """ kwargs.setdefault("dataset_where_sql") kwargs.setdefault("spatial_reference_item") kwargs.setdefault("iter_type", tuple) meta = {"spatial": spatial_reference_metadata(kwargs["spatial_reference_item"])} keys = {"feature": list(contain(field_names))} cursor = arcpy.da.SearchCursor( in_table=dataset_path, field_names=keys["feature"], where_clause=kwargs["dataset_where_sql"], spatial_reference=meta["spatial"]["object"], ) with cursor: for feature in cursor: yield kwargs["iter_type"](feature)
def assigned_count(self, id_values): """Return the assigned count for features with the given identifier. Args: id_values (iter): Feature identifier values. """ return self.assigned[tuple(contain(id_values))]
def is_duplicate(self, id_values): """Return True if more than one feature has given ID. Args: id_values (iter): Feature identifier values. """ return self.matched[tuple(contain(id_values))] > 1
def match_count(self, id_values): """Return match count for features with given ID. Args: id_values (iter): Feature identifier values. """ return self.matched[tuple(contain(id_values))]
def increment_assigned(self, id_values): """Increment assigned count for given feature ID. Args: id_values (iter): Feature identifier values. """ _id = tuple(contain(id_values)) self.assigned[_id] += 1 return self.assigned[_id]
def rows_to_csvfile(rows, output_path, field_names, header=False, **kwargs): """Write collection of rows to a CSV-file. Note: Rows can be represented by either dictionaries or sequences. Args: rows (iter): Collection of dictionaries or sequences representing rows. output_path (str): Path of the output dataset. field_names (iter): Collection of the field names, in the desired order or output. header (bool): Write a header in the CSV output if True. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: file_mode (str): Code indicating the file mode for writing. Default is "wb". log_level (str): Level to log the function at. Default is "info". Returns: str: Path of the CSV-file. """ kwargs.setdefault("file_mode", "wb") log = leveled_logger(LOG, kwargs.setdefault("log_level", "info")) log("Start: Convert rows to CSVfile %s.", output_path) field_names = list(contain(field_names)) with open(output_path, kwargs["file_mode"]) as csvfile: for index, row in enumerate(rows): if index == 0: if isinstance(row, dict): writer = csv.DictWriter(csvfile, field_names) if header: writer.writeheader() elif isinstance(row, Sequence): writer = csv.writer(csvfile) if header: writer.writerow(field_names) else: raise TypeError("Rows must be dictionaries or sequences.") writer.writerow(row) log("End: Write.") return output_path
def as_dicts(dataset_path, field_names=None, **kwargs): """Generate mappings of feature attribute name to value. Notes: Use ArcPy cursor token names for object IDs and geometry objects/properties. Args: dataset_path (str): Path of the dataset. field_names (iter): Collection of field names. Names will be the keys in the dictionary mapping to their values. If value is None, all attributes fields will be used. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: dataset_where_sql (str): SQL where-clause for dataset subselection. spatial_reference_item: Item from which the spatial reference of the output geometry will be derived. Yields: dict. """ kwargs.setdefault("dataset_where_sql") kwargs.setdefault("spatial_reference_item") meta = {"spatial": spatial_reference_metadata(kwargs["spatial_reference_item"])} if field_names is None: meta["dataset"] = dataset_metadata(dataset_path) keys = {"feature": [key for key in meta["dataset"]["field_names_tokenized"]]} else: keys = {"feature": list(contain(field_names))} cursor = arcpy.da.SearchCursor( in_table=dataset_path, field_names=keys["feature"], where_clause=kwargs["dataset_where_sql"], spatial_reference=meta["spatial"]["object"], ) with cursor: for feature in cursor: yield dict(zip(cursor.fields, feature))
def insert_from_iters(dataset_path, insert_features, field_names, **kwargs): """Insert features into dataset from iterables. Args: dataset_path (str): Path of the dataset. insert_features (iter of iter): Collection of iterables representing features. field_names (iter): Collection of field names to insert. These must match the order of their attributes in the insert_features items. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: use_edit_session (bool): Flag to perform updates in an edit session. Default is False. log_level (str): Level to log the function at. Default is 'info'. Returns: collections.Counter: Counts for each feature action. """ kwargs.setdefault('use_edit_session', False) log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info')) log("Start: Insert features into %s from iterables.", dataset_path) meta = {'dataset': arcobj.dataset_metadata(dataset_path)} keys = {'row': list(contain(field_names))} if inspect.isgeneratorfunction(insert_features): insert_features = insert_features() session = arcobj.Editor(meta['dataset']['workspace_path'], kwargs['use_edit_session']) cursor = arcpy.da.InsertCursor(dataset_path, field_names=keys['row']) feature_count = Counter() with session, cursor: for row in insert_features: cursor.insertRow(tuple(row)) feature_count['inserted'] += 1 log("%s features inserted.", feature_count['inserted']) log("End: Insert.") return feature_count
def insert_from_dicts(dataset_path, insert_features, field_names, **kwargs): """Insert features into dataset from dictionaries. Args: dataset_path (str): Path of the dataset. insert_features (iter of dict): Collection of dictionaries representing features. field_names (iter): Collection of field names/keys to insert. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: use_edit_session (bool): Flag to perform updates in an edit session. Default is False. log_level (str): Level to log the function at. Default is 'info'. Returns: collections.Counter: Counts for each feature action. """ kwargs.setdefault('use_edit_session', False) log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info')) log("Start: Insert features into %s from dictionaries.", dataset_path) keys = {'row': list(contain(field_names))} if inspect.isgeneratorfunction(insert_features): insert_features = insert_features() iters = ((feature[key] for key in keys['row']) for feature in insert_features) feature_count = insert_from_iters( dataset_path, iters, field_names, use_edit_session=kwargs['use_edit_session'], log_level=None, ) log("%s features inserted.", feature_count['inserted']) log("End: Insert.") return feature_count
def insert_from_path(dataset_path, insert_dataset_path, field_names=None, **kwargs): """Insert features into dataset from another dataset. Args: dataset_path (str): Path of the dataset. insert_dataset_path (str): Path of dataset to insert features from. field_names (iter): Collection of field names to insert. Listed field must be present in both datasets. If field_names is None, all fields will be inserted. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: insert_where_sql (str): SQL where-clause for insert-dataset subselection. use_edit_session (bool): Flag to perform updates in an edit session. Default is False. log_level (str): Level to log the function at. Default is 'info'. Returns: collections.Counter: Counts for each feature action. """ kwargs.setdefault('insert_where_sql') kwargs.setdefault('use_edit_session', False) log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info')) log("Start: Insert features into %s from %s.", dataset_path, insert_dataset_path) meta = { 'dataset': arcobj.dataset_metadata(dataset_path), 'insert': arcobj.dataset_metadata(insert_dataset_path), } if field_names is None: keys = set.intersection(*(set( name.lower() for name in _meta['field_names_tokenized']) for _meta in meta.values())) else: keys = set(name.lower() for name in contain(field_names)) # OIDs & area/length "fields" have no business being part of an insert. # Geometry itself is handled separately in append function. for _meta in meta.values(): for key in chain(*_meta['field_token'].items()): keys.discard(key) append_kwargs = { 'inputs': unique_name('view'), 'target': dataset_path, 'schema_type': 'no_test', 'field_mapping': arcpy.FieldMappings(), } # Create field maps. # ArcGIS Pro's no-test append is case-sensitive (verified 1.0-1.1.1). # Avoid this problem by using field mapping. # BUG-000090970 - ArcGIS Pro 'No test' field mapping in Append tool does not auto- # map to the same field name if naming convention differs. for key in keys: field_map = arcpy.FieldMap() field_map.addInputField(insert_dataset_path, key) append_kwargs['field_mapping'].addFieldMap(field_map) view = arcobj.DatasetView( insert_dataset_path, kwargs['insert_where_sql'], view_name=append_kwargs['inputs'], # Must be nonspatial to append to nonspatial table. force_nonspatial=(not meta['dataset']['is_spatial']), ) session = arcobj.Editor(meta['dataset']['workspace_path'], kwargs['use_edit_session']) with view, session: arcpy.management.Append(**append_kwargs) feature_count = Counter({'inserted': view.count}) log("%s features inserted.", feature_count['inserted']) log("End: Insert.") return feature_count
def dissolve(dataset_path, dissolve_field_names=None, multipart=True, **kwargs): """Dissolve geometry of features that share values in given fields. Args: dataset_path (str): Path of the dataset. dissolve_field_names (iter): Iterable of field names to dissolve on. multipart (bool): Flag to allow multipart features in output. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: unsplit_lines (bool): Flag to merge line features when endpoints meet without crossing features. Default is False. dataset_where_sql (str): SQL where-clause for dataset subselection. tolerance (float): Tolerance for coincidence, in dataset's units. use_edit_session (bool): Flag to perform updates in an edit session. Default is False. log_level (str): Level to log the function at. Default is 'info'. Returns: str: Path of the dataset updated. """ kwargs.setdefault('unsplit_lines', False) kwargs.setdefault('dataset_where_sql') kwargs.setdefault('use_edit_session', False) log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info')) log( "Start: Dissolve features in %s on fields: %s.", dataset_path, dissolve_field_names, ) meta = { 'dataset': arcobj.dataset_metadata(dataset_path), 'orig_tolerance': arcpy.env.XYTolerance, } keys = {'dissolve': tuple(contain(dissolve_field_names))} view = { 'dataset': arcobj.DatasetView(dataset_path, kwargs['dataset_where_sql']) } temp_output_path = unique_path('output') with view['dataset']: if 'tolerance' in kwargs: arcpy.env.XYTolerance = kwargs['tolerance'] arcpy.management.Dissolve( in_features=view['dataset'].name, out_feature_class=temp_output_path, dissolve_field=keys['dissolve'], multi_part=multipart, unsplit_lines=kwargs['unsplit_lines'], ) if 'tolerance' in kwargs: arcpy.env.XYTolerance = meta['orig_tolerance'] session = arcobj.Editor(meta['dataset']['workspace_path'], kwargs['use_edit_session']) with session: delete(dataset_path, dataset_where_sql=kwargs['dataset_where_sql'], log_level=None) insert_from_path(dataset_path, insert_dataset_path=temp_output_path, log_level=None) dataset.delete(temp_output_path, log_level=None) log("End: Dissolve.") return dataset_path
def add_index(dataset_path, field_names, **kwargs): """Add index to dataset fields. Note: Index names can only be applied to non-spatial indexes for geodatabase feature classes and tables. There is a limited length allowed for index names; longer names will be truncated without warning. Args: dataset_path (str): Path of the dataset. field_names (iter): Collection of participating field names. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: index_name (str): Name for index. Optional; see note. is_ascending (bool): Build index in ascending order if True. Default is False. is_unique (bool): Build index with unique constraint if True. Default is False. fail_on_lock_ok (bool): If True, indicate success even if dataset locks prevent adding index. Default is False. log_level (str): Level to log the function at. Default is "info". Returns: str: Path of the dataset receiving the index. Raises: RuntimeError: If more than one field and any are geometry-types. arcpy.ExecuteError: If dataset lock prevents adding index. """ field_names = [name.lower() for name in contain(field_names)] kwargs.setdefault("index_name", "ndx_" + "_".join(field_names)) kwargs.setdefault("is_ascending", False) kwargs.setdefault("is_unique", False) kwargs.setdefault("fail_on_lock_ok", False) log = leveled_logger(LOG, kwargs.setdefault("log_level", "info")) log("Start: Add index to field(s) %s on %s.", field_names, dataset_path) meta = {"dataset": dataset_metadata(dataset_path)} meta["field_types"] = { field["type"].lower() for field in meta["dataset"]["fields"] if field["name"].lower() in field_names } if "geometry" in meta["field_types"]: if len(field_names) > 1: raise RuntimeError("Cannot create a composite spatial index.") exec_add = arcpy.management.AddSpatialIndex add_kwargs = {"in_features": dataset_path} else: exec_add = arcpy.management.AddIndex add_kwargs = { "in_table": dataset_path, "fields": field_names, "index_name": kwargs["index_name"], "unique": kwargs["is_unique"], "ascending": kwargs["is_ascending"], } try: exec_add(**add_kwargs) except arcpy.ExecuteError as error: if error.message.startswith("ERROR 000464"): LOG.warning("Lock on %s prevents adding index.", dataset_path) if not kwargs["fail_on_lock_ok"]: raise log("End: Add.") return dataset_path
def update_from_iters(dataset_path, update_features, id_field_names, field_names, **kwargs): """Update features in dataset from iterables. Note: There is no guarantee that the ID field(s) are unique. Use ArcPy cursor token names for object IDs and geometry objects/properties. Args: dataset_path (str): Path of the dataset. update_features (iter of dict): Collection of iterables representing features. id_field_names (iter, str): Name(s) of the ID field/key(s). *All* ID fields must also be in field_names. field_names (iter): Collection of field names/keys to check & update. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: delete_missing_features (bool): True if update should delete features missing from update_features, False otherwise. Default is True. use_edit_session (bool): Flag to perform updates in an edit session. Default is True. log_level (str): Level to log the function at. Default is 'info'. Returns: collections.Counter: Counts for each feature action. """ kwargs.setdefault('delete_missing_features', True) kwargs.setdefault('use_edit_session', True) log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info')) log("Start: Update features in %s from iterables.", dataset_path) meta = {'dataset': arcobj.dataset_metadata(dataset_path)} keys = { 'id': list(contain(id_field_names)), 'feat': list(contain(field_names)) } if not set(keys['id']).issubset(keys['feat']): raise ValueError("id_field_names must be a subset of field_names.") ids = { 'dataset': { tuple(freeze_values(*_id)) for _id in attributes.as_iters(dataset_path, keys['id']) } } if inspect.isgeneratorfunction(update_features): update_features = update_features() feats = {'insert': set(), 'id_update': dict()} for feat in update_features: feat = tuple(freeze_values(*feat)) _id = tuple(feat[keys['feat'].index(key)] for key in keys['id']) if _id not in ids['dataset']: feats['insert'].add(feat) else: feats['id_update'][_id] = feat if kwargs['delete_missing_features']: ids['delete'] = { _id for _id in ids['dataset'] if _id not in feats['id_update'] } else: ids['delete'] = set() feature_count = Counter() session = arcobj.Editor(meta['dataset']['workspace_path'], kwargs['use_edit_session']) if ids['delete'] or feats['id_update']: cursor = arcpy.da.UpdateCursor(dataset_path, field_names=keys['feat']) with session, cursor: for feat in cursor: _id = tuple( freeze_values(*(feat[keys['feat'].index(key)] for key in keys['id']))) if _id in ids['delete']: cursor.deleteRow() feature_count['deleted'] += 1 continue elif (_id in feats['id_update'] and not arcobj.same_feature(feat, feats['id_update'][_id])): cursor.updateRow(feats['id_update'][_id]) feature_count['altered'] += 1 else: feature_count['unchanged'] += 1 if feats['insert']: cursor = arcpy.da.InsertCursor(dataset_path, field_names=keys['feat']) with session, cursor: for feat in feats['insert']: try: cursor.insertRow(feat) except RuntimeError: LOG.error( "Feature failed to write to cursor. Offending row:") for key, val in zip(keys['feat'], feat): LOG.error("%s: %s", key, val) raise feature_count['inserted'] += 1 for key in UPDATE_TYPES: log("%s features %s.", feature_count[key], key) log("End: Update.") return feature_count
def update_by_geometry(dataset_path, field_name, geometry_properties, **kwargs): """Update attribute values by cascading through geometry properties. Args: dataset_path (str): Path of the dataset. field_name (str): Name of the field. geometry_properties (iter): Geometry property names in object-access order to retrieve the update value. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: dataset_where_sql (str): SQL where-clause for dataset subselection. spatial_reference_item: Item from which the spatial reference for the output geometry property will be derived. Default is the update dataset. use_edit_session (bool): Updates are done in an edit session if True. If not not specified or None, the spatial reference of the dataset is used. log_level (str): Level to log the function at. Default is "info". Returns: collections.Counter: Counts for each feature action. """ kwargs.setdefault("dataset_where_sql") kwargs.setdefault("spatial_reference_item") kwargs.setdefault("use_edit_session", False) log = leveled_logger(LOG, kwargs.setdefault("log_level", "info")) log( "Start: Update attributes in %s on %s by geometry properties %s.", field_name, dataset_path, geometry_properties, ) meta = { "dataset": dataset_metadata(dataset_path), "spatial": spatial_reference_metadata(kwargs["spatial_reference_item"]), } session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"]) cursor = arcpy.da.UpdateCursor( in_table=dataset_path, field_names=["shape@", field_name], where_clause=kwargs["dataset_where_sql"], spatial_reference=meta["spatial"]["object"], ) update_action_count = Counter() with session, cursor: for feature in cursor: value = {"geometry": feature[0], "old": feature[-1]} value["new"] = property_value( value["geometry"], GEOMETRY_PROPERTY_TRANSFORM, *contain(geometry_properties) ) if same_value(value["old"], value["new"]): update_action_count["unchanged"] += 1 else: try: cursor.updateRow([value["geometry"], value["new"]]) update_action_count["altered"] += 1 except RuntimeError: LOG.error("Offending value is %s", value["new"]) raise for action, count in sorted(update_action_count.items()): log("%s attributes %s.", count, action) log("End: Update.") return update_action_count
def update_by_function(dataset_path, field_name, function, **kwargs): """Update attribute values by passing them to a function. Args: dataset_path (str): Path of the dataset. field_name (str): Name of the field. function (types.FunctionType): Function to get values from. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: field_as_first_arg (bool): True if field value will be the first positional argument. Default is True. arg_field_names (iter): Field names whose values will be the positional arguments (not including primary field). kwarg_field_names (iter): Field names whose names & values will be the method keyword arguments. dataset_where_sql (str): SQL where-clause for dataset subselection. use_edit_session (bool): Updates are done in an edit session if True. Default is False. log_level (str): Level to log the function at. Default is "info". Returns: collections.Counter: Counts for each feature action. """ kwargs.setdefault("field_as_first_arg", True) kwargs.setdefault("arg_field_names", []) kwargs.setdefault("kwarg_field_names", []) kwargs.setdefault("dataset_where_sql") kwargs.setdefault("use_edit_session", False) log = leveled_logger(LOG, kwargs.setdefault("log_level", "info")) log( "Start: Update attributes in %s on %s by function %s.", field_name, dataset_path, function, ) meta = {"dataset": dataset_metadata(dataset_path)} keys = { "args": list(contain(kwargs["arg_field_names"])), "kwargs": list(contain(kwargs["kwarg_field_names"])), } keys["feature"] = keys["args"] + keys["kwargs"] + [field_name] session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"]) cursor = arcpy.da.UpdateCursor( in_table=dataset_path, field_names=keys["feature"], where_clause=kwargs["dataset_where_sql"], ) update_action_count = Counter() with session, cursor: for feature in cursor: value = { "old": feature[-1], "args": feature[: len(keys["args"])], "kwargs": dict(zip(keys["kwargs"], feature[len(keys["args"]) : -1])), } if kwargs["field_as_first_arg"]: value["args"] = [value["old"]] + value["args"] value["new"] = function(*value["args"], **value["kwargs"]) if same_value(value["old"], value["new"]): update_action_count["unchanged"] += 1 else: try: cursor.updateRow(feature[:-1] + [value["new"]]) update_action_count["altered"] += 1 except RuntimeError: LOG.error("Offending value is %s", value["new"]) raise for action, count in sorted(update_action_count.items()): log("%s attributes %s.", count, action) log("End: Update.") return update_action_count
def update_by_feature_match( dataset_path, field_name, id_field_names, update_type, **kwargs ): """Update attribute values by aggregating info about matching features. Note: Currently, the sort_order update type uses functionality that only works with datasets contained in databases. Valid update_type codes: "flag_value": Apply the flag_value argument value to matched features. "match_count": Apply the count of matched features. "sort_order": Apply the position of the feature sorted with matches. Args: dataset_path (str): Path of the dataset. field_name (str): Name of the field. id_field_names (iter): Field names used to identify a feature. update_type (str): Code indicating what values to apply to matched features. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: dataset_where_sql (str): SQL where-clause for dataset subselection. flag_value: Value to apply to matched features. Only used when update_type is "flag_value". sort_field_names (iter): Iterable of field names used to sort matched features. Only affects output when update_type="sort_order". use_edit_session (bool): Updates are done in an edit session if True. Default is False. log_level (str): Level to log the function at. Default is "info". Returns: collections.Counter: Counts for each feature action. """ kwargs.setdefault("dataset_where_sql") kwargs.setdefault("use_edit_session", False) log = leveled_logger(LOG, kwargs.setdefault("log_level", "info")) log( "Start: Update attributes in %s on %s" + " by feature-matching %s on identifiers (%s).", field_name, dataset_path, update_type.replace("_", " "), id_field_names, ) if update_type not in ["flag_value", "match_count", "sort_order"]: raise ValueError("Invalid update_type.") for _type, kwarg in { "flag_value": "flag_value", "sort_order": "sort_field_names", }.items(): if update_type == _type and kwarg not in kwargs: raise TypeError( """{} is required keyword argument when update_type == "{}", .""".format( _type, kwarg ) ) meta = {"dataset": dataset_metadata(dataset_path)} keys = { "id": list(contain(id_field_names)), "sort": list(contain(kwargs.get("sort_field_names", []))), } keys["feature"] = keys["id"] + [field_name] matcher = FeatureMatcher(dataset_path, keys["id"], kwargs["dataset_where_sql"]) session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"]) cursor = arcpy.da.UpdateCursor( in_table=dataset_path, field_names=keys["feature"], where_clause=kwargs["dataset_where_sql"], sql_clause=( (None, "order by " + ", ".join(keys["sort"])) if update_type == "sort_order" else None ), ) update_action_count = Counter() with session, cursor: for feature in cursor: value = { "id": feature[0] if len(keys["id"]) == 1 else tuple(feature[:-1]), "old": feature[-1], } if update_type == "flag_value": if matcher.is_duplicate(value["id"]): value["new"] = kwargs["flag_value"] else: value["new"] = value["old"] elif update_type == "match_count": value["new"] = matcher.match_count(value["id"]) elif update_type == "sort_order": value["new"] = matcher.increment_assigned(value["id"]) if same_value(value["old"], value["new"]): update_action_count["unchanged"] += 1 else: try: cursor.updateRow(feature[:-1] + [value["new"]]) update_action_count["altered"] += 1 except RuntimeError: LOG.error("Offending value is %s", value["new"]) raise for action, count in sorted(update_action_count.items()): log("%s attributes %s.", count, action) log("End: Update.") return update_action_count
def id_node_map( dataset_path, from_id_field_name, to_id_field_name, id_field_names=("oid@",), **kwargs ): """Return mapping of feature ID to from- & to-node ID dictionary. Notes: From & to IDs must be same attribute type. Default output format: `{feature_id: {"from": from_node_id, "to": to_node_id}}` Args: dataset_path (str): Path of the dataset. from_id_field_name (str): Name of from-ID field. to_id_field_name (str): Name of to-ID field. id_field_names (iter, str): Name(s) of the ID field(s). **kwargs: Arbitrary keyword arguments. See below. Keyword Args: dataset_where_sql (str): SQL where-clause for dataset subselection. field_names_as_keys (bool): Use of node ID field names as keys in the map-value if True; use "from" and "to" if False. Default is False. update_nodes (bool): Update nodes based on feature geometries if True. Default is False. Returns: dict: Mapping of feature IDs to node-end ID dictionaries. `{feature_id: {"from": from_node_id, "to": to_node_id}}` """ kwargs.setdefault("dataset_where_sql") kwargs.setdefault("field_names_as_keys", False) kwargs.setdefault("update_nodes", False) keys = { "id": list(contain(id_field_names)), "node": { "from": from_id_field_name if kwargs["field_names_as_keys"] else "from", "to": to_id_field_name if kwargs["field_names_as_keys"] else "to", }, } keys["feature"] = [from_id_field_name, to_id_field_name] + keys["id"] id_nodes = defaultdict(dict) # If updating nodes, need to gather geometry/coordinates. if kwargs["update_nodes"]: coordinate_node = coordinate_node_map( dataset_path, from_id_field_name, to_id_field_name, keys["id"], **kwargs ) for node in coordinate_node.values(): for end in ["from", "to"]: for feature_id in node["ids"][end]: id_nodes[feature_id][keys["node"][end]] = node["node_id"] else: for feature in as_iters( dataset_path, field_names=keys["feature"], dataset_where_sql=kwargs["dataset_where_sql"], ): from_node_id, to_node_id = feature[:2] feature_id = feature[2:] if len(keys["id"]) == 1: feature_id = feature_id[0] id_nodes[feature_id][keys["node"]["from"]] = from_node_id id_nodes[feature_id][keys["node"]["to"]] = to_node_id return id_nodes
def coordinate_node_map( dataset_path, from_id_field_name, to_id_field_name, id_field_names=("oid@",), **kwargs ): """Return mapping of coordinates to node-info dictionary. Notes: From & to IDs must be same attribute type. Default output format: {(x, y): {"node_id": <id>, "ids": {"from": set(), "to": set()}}} Args: dataset_path (str): Path of the dataset. from_id_field_name (str): Name of the from-ID field. to_id_field_name (str): Name of the to-ID field. id_field_names (iter, str): Name(s) of the ID field(s). **kwargs: Arbitrary keyword arguments. See below. Keyword Args: dataset_where_sql (str): SQL where-clause for dataset subselection. update_nodes (bool): Update nodes based on feature geometries if True. Default is False. Returns: dict. """ kwargs.setdefault("dataset_where_sql") kwargs.setdefault("update_nodes", False) meta = { "from_id_field": field_metadata(dataset_path, from_id_field_name), "to_id_field": field_metadata(dataset_path, to_id_field_name), } if meta["from_id_field"]["type"] != meta["to_id_field"]["type"]: raise ValueError("From- and to-ID fields must be of same type.") keys = {"id": list(contain(id_field_names))} keys["feature"] = ["shape@", from_id_field_name, to_id_field_name] + keys["id"] coordinate_node = {} for feature in as_iters( dataset_path, keys["feature"], dataset_where_sql=kwargs["dataset_where_sql"] ): _id = tuple(feature[3:]) if len(keys["id"]) == 1: _id = _id[0] geom = feature[0] node_id = {"from": feature[1], "to": feature[2]} coordinate = { "from": (geom.firstPoint.X, geom.firstPoint.Y), "to": (geom.lastPoint.X, geom.lastPoint.Y), } for end in ["from", "to"]: if coordinate[end] not in coordinate_node: # Create new coordinate-node. coordinate_node[coordinate[end]] = { "node_id": node_id[end], "ids": defaultdict(set), } # Assign new ID if current is missing. if coordinate_node[coordinate[end]]["node_id"] is None: coordinate_node[coordinate[end]]["node_id"] = node_id[end] # Assign lower ID if different than current. else: coordinate_node[coordinate[end]]["node_id"] = min( coordinate_node[coordinate[end]]["node_id"], node_id[end] ) # Add feature ID to end-ID set. coordinate_node[coordinate[end]]["ids"][end].add(_id) if kwargs["update_nodes"]: coordinate_node = _update_coordinate_node_map( coordinate_node, meta["from_id_field"] ) return coordinate_node
def update_from_path(dataset_path, update_dataset_path, id_field_names, field_names=None, **kwargs): """Update features in dataset from another dataset. Args: dataset_path (str): Path of the dataset. update_dataset_path (str): Path of dataset to update features from. id_field_names (iter, str): Name(s) of the ID field/key(s). field_names (iter): Collection of field names/keys to check & update. Listed field must be present in both datasets. If field_names is None, all fields will be inserted. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: dataset_where_sql (str): SQL where-clause for dataset subselection. WARNING: defining this has major effects: filtered features will not be considered for updating or deletion, and duplicates in the update features will be inserted as if novel. update_where_sql (str): SQL where-clause for update-dataset subselection. chunk_where_sqls (iter): Collection of SQL where-clauses for updating between the datasets in chunks. delete_missing_features (bool): True if update should delete features missing from update_features, False otherwise. Default is True. use_edit_session (bool): Flag to perform updates in an edit session. Default is True. log_level (str): Level to log the function at. Default is 'info'. Returns: collections.Counter: Counts for each feature action. """ for key in ['dataset_where_sql', 'update_where_sql']: kwargs.setdefault(key) if not kwargs[key]: kwargs[key] = "1=1" kwargs.setdefault('subset_where_sqls', ["1=1"]) kwargs.setdefault('delete_missing_features', True) kwargs.setdefault('use_edit_session', True) log = leveled_logger(LOG, kwargs.setdefault('log_level', 'info')) log("Start: Update features in %s from %s.", dataset_path, update_dataset_path) meta = { 'dataset': arcobj.dataset_metadata(dataset_path), 'update': arcobj.dataset_metadata(update_dataset_path), } if field_names is None: field_names = ( set(name.lower() for name in meta['dataset']['field_names_tokenized']) & set(name.lower() for name in meta['update']['field_names_tokenized'])) else: field_names = set(name.lower() for name in field_names) # OIDs & area/length "fields" have no business being part of an update. for key in ['oid@', 'shape@area', 'shape@length']: field_names.discard(key) keys = { 'id': list(contain(id_field_names)), 'attr': list(contain(field_names)) } keys['row'] = keys['id'] + keys['attr'] feature_count = Counter() for kwargs['subset_where_sql'] in contain(kwargs['subset_where_sqls']): if not kwargs['subset_where_sql'] == "1=1": log("Subset: `%s`", kwargs['subset_where_sql']) iters = attributes.as_iters( update_dataset_path, keys['row'], dataset_where_sql=( "({update_where_sql}) and ({subset_where_sql})".format( **kwargs))) view = arcobj.DatasetView( dataset_path, dataset_where_sql=( "({dataset_where_sql}) and ({subset_where_sql})".format( **kwargs)), field_names=keys['row']) with view: feature_count.update( update_from_iters( dataset_path=view.name, update_features=iters, id_field_names=keys['id'], field_names=keys['row'], delete_missing_features=kwargs['delete_missing_features'], use_edit_session=kwargs['use_edit_session'], log_level=None, )) for key in UPDATE_TYPES: log("%s features %s.", feature_count[key], key) log("End: Update.") return feature_count
def update_by_mapping(dataset_path, field_name, mapping, key_field_names, **kwargs): """Update attribute values by finding them in a mapping. Note: Mapping key must be a tuple if an iterable. Args: dataset_path (str): Path of the dataset. field_name (str): Name of the field. mapping: Mapping to get values from. key_field_names (iter): Fields names whose values will comprise the mapping key. **kwargs: Arbitrary keyword arguments. See below. Keyword Args: dataset_where_sql (str): SQL where-clause for dataset subselection. default_value: Value to return from mapping if key value on feature not present. Default is None. use_edit_session (bool): Updates are done in an edit session if True. Default is False. log_level (str): Level to log the function at. Default is "info". Returns: collections.Counter: Counts for each feature action. """ kwargs.setdefault("dataset_where_sql") kwargs.setdefault("default_value") kwargs.setdefault("use_edit_session", False) log = leveled_logger(LOG, kwargs.setdefault("log_level", "info")) log( "Start: Update attributes in %s on %s by mapping with key in %s.", field_name, dataset_path, key_field_names, ) meta = {"dataset": dataset_metadata(dataset_path)} keys = {"map": list(contain(key_field_names))} keys["feature"] = keys["map"] + [field_name] if isinstance(mapping, EXEC_TYPES): mapping = mapping() session = Editor(meta["dataset"]["workspace_path"], kwargs["use_edit_session"]) cursor = arcpy.da.UpdateCursor( in_table=dataset_path, field_names=keys["feature"], where_clause=kwargs["dataset_where_sql"], ) update_action_count = Counter() with session, cursor: for feature in cursor: value = { "map_key": feature[0] if len(keys["map"]) == 1 else tuple(feature[:-1]), "old": feature[-1], } value["new"] = mapping.get(value["map_key"], kwargs["default_value"]) if same_value(value["old"], value["new"]): update_action_count["unchanged"] += 1 else: try: cursor.updateRow(feature[:-1] + [value["new"]]) update_action_count["altered"] += 1 except RuntimeError: LOG.error("Offending value is %s", value["new"]) raise for action, count in sorted(update_action_count.items()): log("%s attributes %s.", count, action) log("End: Update.") return update_action_count