def transform(self, transformation, **kwargs):
"""Run transform operation as defined in the workspace.
Args:
transformation: Function or method used to perform a transformation upon
the transform-dataset.
**kwargs: Arbitrary keyword arguments; passed through to the transformation.
Returns:
arcetl.etl.ArcETL: Reference to the instance.
"""
# Unless otherwise stated, dataset path is self.transform_path.
kwargs.setdefault("dataset_path", self.transform_path)
# Add output_path to kwargs if needed.
if "output_path" in funcsigs.signature(transformation).parameters:
kwargs.setdefault(
"output_path",
unique_path(getattr(transformation, "__name__", "transform")),
)
transformation(**kwargs)
# If there"s a new output, replace old transform.
if "output_path" in funcsigs.signature(transformation).parameters:
if dataset.is_valid(self.transform_path):
dataset.delete(self.transform_path, log_level=None)
self.transform_path = kwargs["output_path"]
return self
def close(self):
"""Clean up instance."""
LOG.info("Closing ArcETL instance for %s.", self.name)
# Clear the transform dataset.
if self.transform_path and dataset.is_valid(self.transform_path):
dataset.delete(self.transform_path, log_level=None)
self.transform_path = None
LOG.info("Closed.")
def load(
self, dataset_path, load_where_sql=None, preserve_features=False, **kwargs
):
"""Load features from transform- to load-dataset.
Args:
dataset_path (str): Path of dataset to load.
load_where_sql (str): SQL where-clause for subselection from the
transform-dataset.
preserve_features (bool): Keep current features in load-dataset if True;
remove them before adding transform-features if False.
Keyword Args:
use_edit_session (bool): Updates are done in an edit session if True.
Default is False.
Returns:
arcetl.etl.ArcETL: Reference to the instance.
"""
kwargs.setdefault("use_edit_session", False)
LOG.info("Start: Load %s.", dataset_path)
# Load to an existing dataset.
if dataset.is_valid(dataset_path):
feature_action_count = Counter()
if not preserve_features:
feature_action_count.update(
features.delete(dataset_path, log_level=None, **kwargs)
)
feature_action_count.update(
features.insert_from_path(
dataset_path,
insert_dataset_path=self.transform_path,
insert_where_sql=load_where_sql,
use_edit_session=kwargs["use_edit_session"],
log_level=None,
)
)
# Load to a new dataset.
else:
feature_action_count = dataset.copy(
self.transform_path,
output_path=dataset_path,
dataset_where_sql=load_where_sql,
log_level=None,
)
for action, count in sorted(feature_action_count.items()):
LOG.info("%s features %s.", count, action)
LOG.info("End: Load.")
return self
def load_diffs(self, dataset_path, preserve_features=False, **kwargs):
"""Load diff features to dataset.
Args:
dataset_path (str): Path of dataset to load.
preserve_features (bool): Flag to indicate whether to remove features in
the load-dataset before adding the transformed features.
Keyword Args:
use_edit_session (bool): Updates are done in an edit session if True.
Default is False.
Returns:
collections.Counter: Counts for each update type.
"""
kwargs.setdefault("use_edit_session", False)
LOG.info("Start: Load diffs to %s.", dataset_path)
feature_count = Counter()
if dataset.is_valid(dataset_path):
if not preserve_features:
feature_count.update(
features.delete(dataset_path, log_level=None, **kwargs)
)
else:
##TODO: Either drop spatial part of diff, or add geoms to self.diffs.values().
dataset.create(
dataset_path,
field_metadata_list=self._diff_field_metas,
geometry_type="polygon",
spatial_reference_item=self._dataset["init"]["spatial_reference"],
log_level=None,
)
feature_count.update(
features.insert_from_dicts(
dataset_path,
insert_features=chain(*self.diffs.values()),
field_names=self._keys["diff"],
log_level=None,
**kwargs
)
)
for key in ["deleted", "inserted"]:
LOG.info("%s features %s.", feature_count[key], key)
LOG.info("End: Load.")
return feature_count