def test_apply_changes():
s = fake.session()
dataset_json = utils.load_json("dataset.json")
dataset_url = tc.URL(path="projects/1/unifiedDataset")
unified_dataset = tc.dataset.unified._from_json(dataset_url, dataset_json)
operation_json = utils.load_json("operation_pending.json")
operation_url = tc.URL(path="operations/1")
url = tc.URL(path="projects/1/unifiedDataset:refresh")
responses.add(responses.POST, str(url), json=operation_json)
response = tc.dataset.unified._apply_changes_async(s, unified_dataset)
assert response == tc.operation._from_json(operation_url, operation_json)
def test_operation_check_success():
s = fake.session()
url = tc.URL(path="operations/1")
op_json = utils.load_json("operation_succeeded.json")
op = tc.operation._from_json(url, op_json)
tc.operation.check(s, op)
def test_upsert_index_as_primary_key():
s = utils.session()
dataset = utils.dataset()
url = tc.URL(path="datasets/1:updateRecords")
updates = [
tc.record._create_command(record, primary_key_name="primary_key")
for record in _records_with_keys_json_2
]
snoop: Dict = {}
responses.add_callback(
responses.POST,
str(url),
partial(
utils.capture_payload, snoop=snoop, status=200, response_json=_response_json
),
)
df = pd.DataFrame(
_records_json_2,
index=[record["primary_key"] for record in _records_with_keys_json_2],
)
df.index.name = "primary_key"
response = tc.dataframe.upsert(s, dataset, df, primary_key_name="primary_key")
assert response == _response_json
assert snoop["payload"] == utils.stringify(updates)
def test_upsert():
s = utils.session()
dataset = utils.dataset()
url = tc.URL(path="datasets/1:updateRecords")
updates = [
tc.record._create_command(record, primary_key_name="primary_key")
for record in _records_json
]
snoop: Dict = {}
responses.add_callback(
responses.POST,
str(url),
partial(utils.capture_payload,
snoop=snoop,
status=200,
response_json=_response_json),
)
response = tc.record.upsert(s,
dataset,
_records_json,
primary_key_name="primary_key")
assert response == _response_json
assert snoop["payload"] == utils.stringify(updates)
def test_create():
s = utils.session()
dataset = utils.dataset()
attrs = tuple([
tc.SubAttribute(
name=str(i),
is_nullable=True,
type=tc.attribute.type.Array(tc.attribute.type.STRING),
) for i in range(4)
])
attrs_url = tc.URL(path=dataset.url.path + "/attributes")
url = replace(attrs_url, path=attrs_url.path + "/attr")
attr_json = utils.load_json("attribute.json")
responses.add(responses.POST, str(attrs_url), json=attr_json)
attr = tc.attribute.create(
s,
dataset,
name="attr",
is_nullable=False,
type=tc.attribute.type.Record(attributes=attrs),
)
assert attr == tc.attribute._from_json(url, attr_json)
def attribute() -> tc.Attribute:
return tc.Attribute(
url=tc.URL(path="datasets/1/attributes/RowNum"),
name="RowNum",
type=tc.attribute.type.DEFAULT,
description="Synthetic row number",
is_nullable=False,
)
def test_from_resource_id_not_found():
s = utils.session()
instance = utils.instance()
url = tc.URL(path="projects/1")
responses.add(responses.GET, str(url), status=404)
with pytest.raises(tc.project.NotFound):
tc.project.from_resource_id(s, instance, "1")
def test_operation_from_json():
url = tc.URL(path="operations/1")
operation_json = utils.load_json("operation_succeeded.json")
op = tc.operation._from_json(url, operation_json)
assert op.url == url
assert op.type == operation_json["type"]
assert op.description == operation_json["description"]
assert op.status == operation_json["status"]
assert tc.operation.succeeded(op)
def test_delete():
s = utils.session()
url = tc.URL(path="datasets/1/attributes/RowNum")
attr_json = utils.load_json("attributes.json")[0]
attr = tc.attribute._from_json(url, attr_json)
responses.add(responses.DELETE, str(attr.url), status=204)
tc.attribute.delete(s, attr)
def test_json():
"""original -> to_json -> from_json -> original"""
attrs_json = utils.load_json("attributes.json")
dataset_id = 1
for attr_json in attrs_json:
attr_id = attr_json["name"]
url = tc.URL(f"datasets/{dataset_id}/attributes/{attr_id}")
attr = tc.attribute._from_json(url, attr_json)
assert attr == tc.attribute._from_json(url, tc.attribute.to_json(attr))
def test_from_project_dataset_not_found():
s = fake.session()
instance = fake.instance()
project = fake.mastering_project()
url = tc.URL(path="projects/1/unifiedDataset")
responses.add(responses.GET, str(url), status=404)
with pytest.raises(tc.dataset.unified.NotFound):
tc.dataset.unified.from_project(s, instance, project)
def test_from_resource_id():
s = utils.session()
dataset = utils.dataset()
url = tc.URL(path=dataset.url.path + "/attributes/attr")
attr_json = utils.load_json("attribute.json")
responses.add(responses.GET, str(url), json=attr_json)
attr = tc.attribute.from_resource_id(s, dataset, "attr")
assert attr == tc.attribute._from_json(url, attr_json)
def test_delete_attribute_not_found():
s = utils.session()
url = tc.URL(path="datasets/1/attributes/RowNum")
attr_json = utils.load_json("attributes.json")[0]
attr = tc.attribute._from_json(url, attr_json)
responses.add(responses.PUT, str(attr.url), status=404)
with pytest.raises(tc.attribute.NotFound):
attr = tc.attribute.update(s, attr)
def test_from_json():
attrs_json = utils.load_json("attributes.json")
dataset_id = 1
for attr_json in attrs_json:
attr_id = attr_json["name"]
url = tc.URL(path=f"datasets/{dataset_id}/attributes/{attr_id}")
attr = tc.attribute._from_json(url, attr_json)
assert attr.name == attr_json["name"]
assert attr.description == attr_json["description"]
assert attr.is_nullable == attr_json["isNullable"]
def test_operation_failed_success():
s = fake.session()
url = tc.URL(path="operations/1")
op_json = utils.load_json("operation_failed.json")
op = tc.operation._from_json(url, op_json)
with pytest.raises(tc.operation.Failed) as exc_info:
tc.operation.check(s, op)
err_msg = str(exc_info.value)
assert str(url) in err_msg
assert op.status is not None and str(op.status["state"]) in err_msg
def test_stream():
s = utils.session()
dataset = utils.dataset()
url = tc.URL(path="datasets/1/records")
responses.add(
responses.GET, str(url), body="\n".join(json.dumps(x) for x in _records_json)
)
records = tc.record.stream(s, dataset)
assert list(records) == _records_json
def test_from_resource_id_mastering():
s = utils.session()
instance = utils.instance()
project_json = utils.load_json("mastering_project.json")
url = tc.URL(path="projects/1")
responses.add(responses.GET, str(url), json=project_json)
project = tc.project.from_resource_id(s, instance, "1")
assert isinstance(project, tc.MasteringProject)
assert project.name == "proj"
assert project.description == "Mastering Project"
def test_from_resource_id():
s = utils.session()
instance = utils.instance()
dataset_json = utils.load_json("dataset.json")
url = tc.URL(path="datasets/1")
responses.add(responses.GET, str(url), json=dataset_json)
dataset = tc.dataset.from_resource_id(s, instance, "1")
assert dataset.name == "dataset 1 name"
assert dataset.description == "dataset 1 description"
assert dataset.key_attribute_names == ("tamr_id", )
def test_operation_by_url():
s = fake.session()
url = tc.URL(path="operations/1")
operation_json = utils.load_json("operation_succeeded.json")
responses.add(responses.GET, str(url), json=operation_json)
op = tc.operation._by_url(s, url)
assert op.url == url
assert op.type == operation_json["type"]
assert op.description == operation_json["description"]
assert op.status == operation_json["status"]
assert tc.operation.succeeded(op)
def test_from_project():
s = fake.session()
instance = fake.instance()
project = fake.mastering_project()
dataset_json = utils.load_json("dataset.json")
url = tc.URL(path="projects/1/unifiedDataset")
responses.add(responses.GET, str(url), json=dataset_json)
unified_dataset = tc.dataset.unified.from_project(s, instance, project)
assert unified_dataset.name == "dataset 1 name"
assert unified_dataset.description == "dataset 1 description"
assert unified_dataset.key_attribute_names == ("tamr_id", )
def test_operation_poll():
s = fake.session()
url = tc.URL(path="operations/1")
pending_operation_json = utils.load_json("operation_pending.json")
op1 = tc.operation._from_json(url, pending_operation_json)
succeeded_operation_json = utils.load_json("operation_succeeded.json")
responses.add(responses.GET, str(url), json=succeeded_operation_json)
op2 = tc.operation.poll(s, op1)
assert op2.url == op1.url
assert not tc.operation.succeeded(op1)
assert tc.operation.succeeded(op2)
def test_update():
s = utils.session()
url = tc.URL(path="datasets/1/attributes/RowNum")
attr_json = utils.load_json("attributes.json")[0]
attr = tc.attribute._from_json(url, attr_json)
updated_attr_json = utils.load_json("updated_attribute.json")
responses.add(responses.PUT, str(attr.url), json=updated_attr_json)
updated_attr = tc.attribute.update(
s, attr, description=updated_attr_json["description"])
assert updated_attr == replace(
attr, description=updated_attr_json["description"])
def test_ndjson():
s = utils.session()
records = [{"a": 1}, {"b": 2}, {"c": 3}]
url = tc.URL(path="datasets/1/records")
responses.add(responses.GET,
str(url),
body="\n".join(json.dumps(x) for x in records))
r = s.get(str(url))
ndjson = list(tc.response.ndjson(r))
assert len(ndjson) == 3
for record in ndjson:
assert record in records
def test_operation_from_response():
s = fake.session()
instance = fake.instance()
url = tc.URL(path="operations/1")
operation_json = utils.load_json("operation_succeeded.json")
responses.add(responses.GET, str(url), json=operation_json)
r = s.get(str(url))
op = tc.operation._from_response(instance, r)
assert op.url == url
assert op.type == operation_json["type"]
assert op.description == operation_json["description"]
assert op.status == operation_json["status"]
assert tc.operation.succeeded(op)
def test_operation_from_response_noop():
s = fake.session()
instance = fake.instance()
url = tc.URL(path="operations/2")
responses.add(responses.GET, str(url), status=204)
url_dummy = tc.URL(path="operations/-1")
responses.add(responses.GET, str(url_dummy), status=404)
r = s.get(str(url))
op2 = tc.operation._from_response(instance, r)
assert op2.url is not None
assert op2.type == "NOOP"
assert op2.description is not None
assert op2.status is not None
assert op2.status["state"] == "SUCCEEDED"
assert tc.operation.succeeded(op2)
op2w = tc.operation.wait(s, op2)
assert tc.operation.succeeded(op2w)
with pytest.raises(tc.operation.NotFound):
tc.operation.poll(s, op2w)
def test_by_resource_id():
s = fake.session()
instance = fake.instance()
url = tc.URL(path="operations/1")
operation_json = utils.load_json("operation_succeeded.json")
responses.add(responses.GET, str(url), json=operation_json)
resource_id = "1"
op = tc.operation.by_resource_id(s, instance, resource_id)
assert op.url == url
assert op.type == operation_json["type"]
assert op.description == operation_json["description"]
assert op.status == operation_json["status"]
assert tc.operation.succeeded(op)
def test_from_json_unrecognized_project_type():
instance = fake.instance()
url = tc.URL("project/1", instance)
data: tc._types.JsonDict = {
"id": "unify://unified-data/v1/projects/1",
"name": "project 1",
"description": "A project of unknown type",
"type": "UNKNOWN",
"unifiedDatasetName": "",
"relativeId": "projects/1",
"externalId": "58bdbe72-3c08-427d-97bd-45b16d92c79c",
}
project = tc.project._from_json(url, data)
assert isinstance(project, tc.UnknownProject)
assert project.name == "project 1"
assert project.description == "A project of unknown type"
def test_poll():
s = fake.session()
data = {
"id": "unify://unified-data/v1/backups/2020-08-17_21-32-10-961",
"relativeId": "2020-08-17_21-32-10-961",
"user": "******",
"backupPath": "/home/ubuntu/tamr/backups/2020-08-17_21-32-10-961",
"state": "RUNNING",
"stage": "",
"errorMessage": "",
"created": "2020-08-17_21-32-10-961",
"lastModified": "2020-08-17_21-51-57-600",
}
backup = tc.backup._from_json(
url=tc.URL(path="backups/2020-08-17_21-32-10-961"), data=data)
tc.backup.poll(session=s, backup=backup)
def from_resource_id(session: tc.Session, instance: tc.Instance,
id: str) -> Dataset:
"""Get dataset by resource ID
Fetches dataset from Tamr server
Args:
instance: Tamr instance containing this dataset
id: Dataset ID
Raises:
DatasetNotFound: If no dataset could be found at the specified URL.
Corresponds to a 404 HTTP error.
requests.HTTPError: If any other HTTP error is encountered.
"""
url = tc.URL(instance=instance, path=f"datasets/{id}")
return _from_url(session, url)
def test_cancel():
s = fake.session()
data = {
"id":
"unify://unified-data/v1/restore/restore-2020-08-19_20-01-20-233",
"relativeId": "restore-2020-08-19_20-01-20-233",
"user": "******",
"backupPath": "/home/ubuntu/tamr/backups/2020-08-17_22-07-11-100",
"state": "RUNNING",
"stage": "",
"errorMessage": "",
"created": "2020-08-19_20-01-20-233",
"lastModified": "2020-08-19_20-02-19-351",
}
restore = tc.restore._from_json(url=tc.URL(path="instance/restore"),
data=data)
tc.restore.cancel(session=s, restore=restore)
