def unified_dataset(self): """Unified dataset for this project. :return: Unified dataset for this project. :rtype: :class:`~tamr_unify_client.models.dataset.resource.Dataset` """ alias = self.api_path + "/unifiedDataset" resource_json = self.client.get(alias).successful().json() return Dataset.from_json(self.client, resource_json, alias)
def published_clusters(self): """Published record clusters generated by Unify's pair-matching model. Call :func:`~tamr_unify_client.models.dataset.resource.Dataset.refresh` from this dataset to republish clusters according to the latest clustering. :returns: The published clusters represented as a dataset. :rtype: :class:`~tamr_unify_client.models.dataset.resource.Dataset` """ alias = self.api_path + "/publishedClusters" return Dataset(self.client, None, alias)
def pairs(self): """Record pairs generated by Unify's binning model. Pairs are displayed on the "Pairs" page in the Unify UI. Call :func:`~tamr_unify_client.models.dataset.resource.Dataset.refresh` from this dataset to regenerate pairs according to the latest binning model. :returns: The record pairs represented as a dataset. :rtype: :class:`~tamr_unify_client.models.dataset.resource.Dataset` """ alias = self.api_path + "/recordPairs" return Dataset(self.client, None, alias)
def create(self, creation_spec): """ Create a Dataset in Unify :param creation_spec: Dataset creation specification should be formatted as specified in the `Public Docs for Creating a Dataset <https://docs.tamr.com/reference#create-a-dataset>`_. :type creation_spec: dict[str, str] :returns: The created Dataset :rtype: :class:`~tamr_unify_client.models.dataset.resource.Dataset` """ data = self.client.post(self.api_path, json=creation_spec).successful().json() return Dataset.from_json(self, data)
def record_clusters(self): """Record Clusters as a dataset. Unify clusters labeled pairs using pairs model. These clusters populate the cluster review page and get transient cluster ids, rather than published cluster ids (i.e., "Permanent Ids") Call :func:`~tamr_unify_client.models.dataset.resource.Dataset.refresh` from this dataset to generate clusters based on to the latest pair-matching model. :returns: The record clusters represented as a dataset. :rtype: :class:`~tamr_unify_client.models.dataset.resource.Dataset` """ alias = self.api_path + "/recordClusters" return Dataset(self.client, None, alias)
def high_impact_pairs(self): """High-impact pairs as a dataset. Unify labels pairs as "high-impact" if labeling these pairs would help it learn most quickly (i.e. "Active learning"). High-impact pairs are displayed with a ⚡ lightning bolt icon on the "Pairs" page in the Unify UI. Call :func:`~tamr_unify_client.models.dataset.resource.Dataset.refresh` from this dataset to produce new high-impact pairs according to the latest pair-matching model. :returns: The high-impact pairs represented as a dataset. :rtype: :class:`~tamr_unify_client.models.dataset.resource.Dataset` """ alias = self.api_path + "/highImpactPairs" return Dataset(self.client, None, alias)
def test_feature_to_record(self): feature = {"type": "Feature", "id": "1"} actual = Dataset._feature_to_record(feature, ["pk"], "geo") expected = {"pk": "1"} self.assertEqual(expected, actual) feature = { "type": "Feature", "id": "1", "geometry": { "type": "Point", "coordinates": [0, 0] }, } actual = Dataset._feature_to_record(feature, ["pk"], "geo") expected = {"pk": "1", "geo": {"point": [0, 0]}} self.assertEqual(expected, actual) feature = { "type": "Feature", "id": "1", "geometry": { "type": "MultiPoint", "coordinates": [[0, 0], [1, 1]] }, } actual = Dataset._feature_to_record(feature, ["pk"], "geo") expected = {"pk": "1", "geo": {"multiPoint": [[0, 0], [1, 1]]}} self.assertEqual(expected, actual) feature = { "type": "Feature", "id": "1", "geometry": { "type": "LineString", "coordinates": [[0, 0], [1, 1]] }, } actual = Dataset._feature_to_record(feature, ["pk"], "geo") expected = {"pk": "1", "geo": {"lineString": [[0, 0], [1, 1]]}} self.assertEqual(expected, actual) feature = { "type": "Feature", "id": "1", "geometry": { "type": "MultiLineString", "coordinates": [[[0, 0], [1, 1], [2, 2]]], }, } actual = Dataset._feature_to_record(feature, ["pk"], "geo") expected = { "pk": "1", "geo": { "multiLineString": [[[0, 0], [1, 1], [2, 2]]] } } self.assertEqual(expected, actual) feature = { "type": "Feature", "id": "1", "geometry": { "type": "Polygon", "coordinates": [[[0, 0], [1, 1], [2, 2]]] }, } actual = Dataset._feature_to_record(feature, ["pk"], "geo") expected = {"pk": "1", "geo": {"polygon": [[[0, 0], [1, 1], [2, 2]]]}} self.assertEqual(expected, actual) feature = { "type": "Feature", "id": "1", "geometry": { "type": "MultiPolygon", "coordinates": [[[[0, 0], [1, 1], [2, 2]]]], }, } actual = Dataset._feature_to_record(feature, ["pk"], "geo") expected = { "pk": "1", "geo": { "multiPolygon": [[[[0, 0], [1, 1], [2, 2]]]] } } self.assertEqual(expected, actual) feature = {"type": "Feature", "id": "1", "geometry": None} actual = Dataset._feature_to_record(feature, ["pk"], "geo") expected = {"pk": "1"} self.assertEqual(expected, actual) feature = { "type": "Feature", "id": "1", "bbox": [0, 0, 1, 1], "geometry": { "type": "Point", "coordinates": [0, 0] }, } actual = Dataset._feature_to_record(feature, ["pk"], "geo") expected = {"pk": "1", "geo": {"point": [0, 0]}, "bbox": [0, 0, 1, 1]} self.assertEqual(expected, actual) feature = { "type": "Feature", "id": "1", "bbox": None, "geometry": { "type": "Point", "coordinates": [0, 0] }, } actual = Dataset._feature_to_record(feature, ["pk"], "geo") expected = {"pk": "1", "geo": {"point": [0, 0]}} self.assertEqual(expected, actual) feature = { "type": "Feature", "id": "1", "bbox": [0, 0, 1, 1], "geometry": { "type": "Point", "coordinates": [0, 0] }, "properties": { "prop1": "val1", "prop2": "val2" }, } actual = Dataset._feature_to_record(feature, ["pk"], "geo") expected = { "pk": "1", "geo": { "point": [0, 0] }, "bbox": [0, 0, 1, 1], "prop1": "val1", "prop2": "val2", } self.assertEqual(expected, actual) feature = { "type": "Feature", "id": "1", "bbox": [0, 0, 1, 1], "geometry": { "type": "Point", "coordinates": [0, 0] }, "properties": None, } actual = Dataset._feature_to_record(feature, ["pk"], "geo") expected = {"pk": "1", "geo": {"point": [0, 0]}, "bbox": [0, 0, 1, 1]} self.assertEqual(expected, actual) feature = { "type": "Feature", "id": "1", "bbox": [0, 0, 1, 1], "geometry": { "type": "Point", "coordinates": [0, 0] }, # Properties with names that conflict with # the props in the key or geometry # get ignored "properties": { "pk": "val1", "geo": "val2", "bbox": "val3" }, } actual = Dataset._feature_to_record(feature, ["pk"], "geo") expected = {"pk": "1", "geo": {"point": [0, 0]}, "bbox": [0, 0, 1, 1]} self.assertEqual(expected, actual) feature = { "type": "Feature", "id": ["1", "2"], "geometry": { "type": "Point", "coordinates": [0, 0] }, } actual = Dataset._feature_to_record(feature, ["pk1", "pk2"], "geo") expected = {"pk1": "1", "pk2": "2", "geo": {"point": [0, 0]}} self.assertEqual(expected, actual) class NotAFeature: @property def __geo_interface__(self): return { "type": "Feature", "id": "1", "geometry": { "type": "Point", "coordinates": [0, 0] }, } naf = NotAFeature() actual = Dataset._feature_to_record(naf, ["pk"], "geo") expected = {"pk": "1", "geo": {"point": [0, 0]}} self.assertEqual(expected, actual)
def test_record_to_feature(self): empty_record = {"id": "1"} def key_value_single(rec): return rec["id"] actual = Dataset._record_to_feature(empty_record, key_value_single, ["id"], "geom") expected = {"type": "Feature", "id": "1"} self.assertEqual(expected, actual) record_with_point = {"id": "1", "geom": {"point": [1, 1]}} actual = Dataset._record_to_feature(record_with_point, key_value_single, ["id"], "geom") expected = { "type": "Feature", "id": "1", "geometry": { "type": "Point", "coordinates": [1, 1] }, } self.assertEqual(expected, actual) record_with_multi_point = {"id": "1", "geom": {"multiPoint": [[1, 1]]}} actual = Dataset._record_to_feature(record_with_multi_point, key_value_single, ["id"], "geom") expected = { "type": "Feature", "id": "1", "geometry": { "type": "MultiPoint", "coordinates": [[1, 1]] }, } self.assertEqual(expected, actual) record_with_line = { "id": "1", "geom": { "lineString": [[1, 1], [2, 2]] } } actual = Dataset._record_to_feature(record_with_line, key_value_single, ["id"], "geom") expected = { "type": "Feature", "id": "1", "geometry": { "type": "LineString", "coordinates": [[1, 1], [2, 2]] }, } self.assertEqual(expected, actual) record_with_multi_line = { "id": "1", "geom": { "multiLineString": [[[1, 1], [2, 2]]] }, } actual = Dataset._record_to_feature(record_with_multi_line, key_value_single, ["id"], "geom") expected = { "type": "Feature", "id": "1", "geometry": { "type": "MultiLineString", "coordinates": [[[1, 1], [2, 2]]] }, } self.assertEqual(expected, actual) record_with_polygon = { "id": "1", "geom": { "polygon": [[[1, 1], [2, 2], [3, 3]]] }, } actual = Dataset._record_to_feature(record_with_polygon, key_value_single, ["id"], "geom") expected = { "type": "Feature", "id": "1", "geometry": { "type": "Polygon", "coordinates": [[[1, 1], [2, 2], [3, 3]]] }, } self.assertEqual(expected, actual) record_with_multi_polygon = { "id": "1", "geom": { "multiPolygon": [[[[1, 1], [2, 2], [3, 3]]]] }, } actual = Dataset._record_to_feature(record_with_multi_polygon, key_value_single, ["id"], "geom") expected = { "type": "Feature", "id": "1", "geometry": { "type": "MultiPolygon", "coordinates": [[[[1, 1], [2, 2], [3, 3]]]], }, } self.assertEqual(expected, actual) record_with_full_geo = { "id": "1", "geom": { "point": None, "multiPoint": None, "lineString": None, "multiLineString": None, "polygon": None, "multiPolygon": [[[[1, 1], [2, 2], [3, 3]]]], }, } actual = Dataset._record_to_feature(record_with_full_geo, key_value_single, ["id"], "geom") expected = { "type": "Feature", "id": "1", "geometry": { "type": "MultiPolygon", "coordinates": [[[[1, 1], [2, 2], [3, 3]]]], }, } self.assertEqual(expected, actual) record_with_null_geo = { "id": "1", "geom": { "point": None, "multiPoint": None, "lineString": None, "multiLineString": None, "polygon": None, "multiPolygon": None, }, } actual = Dataset._record_to_feature(record_with_null_geo, key_value_single, ["id"], "geom") expected = {"geometry": None, "type": "Feature", "id": "1"} self.assertEqual(expected, actual) record_with_bbox = {"id": "1", "bbox": [[0, 0], [1, 1]]} actual = Dataset._record_to_feature(record_with_bbox, key_value_single, ["id"], "geom") expected = {"type": "Feature", "id": "1", "bbox": [[0, 0], [1, 1]]} self.assertEqual(expected, actual) record_with_props = {"id": "1", "p1": "v1", "p2": "v2"} actual = Dataset._record_to_feature(record_with_props, key_value_single, ["id"], "geom") expected = { "type": "Feature", "id": "1", "properties": { "p1": "v1", "p2": "v2" }, } self.assertEqual(expected, actual) def key_value_composite(rec): return [rec[v] for v in ["id1", "id2"]] record_with_composite_key = {"id1": "1", "id2": "2"} actual = Dataset._record_to_feature(record_with_composite_key, key_value_composite, ["id1", "id2"], "geom") expected = {"type": "Feature", "id": ["1", "2"]} self.assertEqual(expected, actual) record_with_everything = { "id1": "1", "id2": "2", "bbox": [[0, 0], [1, 1]], "name": "record with everything", "geom": { "point": None, "multiPoint": None, "lineString": None, "multiLineString": None, "polygon": [[[0, 0], [0, 1], [1, 1], [1, 0], [0, 0]]], "multiPolygon": None, }, "alternate_geom": { "point": [1, 1], "multiPoint": None, "lineString": None, "multiLineString": None, "polygon": None, "multiPolygon": None, }, } actual = Dataset._record_to_feature(record_with_everything, key_value_composite, ["id1", "id2"], "geom") expected = { "type": "Feature", "id": ["1", "2"], "bbox": [[0, 0], [1, 1]], "properties": { "name": "record with everything", "alternate_geom": { "point": [1, 1], "multiPoint": None, "lineString": None, "multiLineString": None, "polygon": None, "multiPolygon": None, }, }, "geometry": { "type": "Polygon", "coordinates": [[[0, 0], [0, 1], [1, 1], [1, 0], [0, 0]]], }, } self.assertEqual(expected, actual) record_without_geo = {"id": "1", "prop1": "val1"} actual = Dataset._record_to_feature(record_without_geo, key_value_single, ["id"], None) expected = { "type": "Feature", "id": "1", "properties": { "prop1": "val1" } } self.assertEqual(expected, actual)