def test_apply_changes(): s = fake.session() dataset_json = utils.load_json("dataset.json") dataset_url = tc.URL(path="projects/1/unifiedDataset") unified_dataset = tc.dataset.unified._from_json(dataset_url, dataset_json) operation_json = utils.load_json("operation_pending.json") operation_url = tc.URL(path="operations/1") url = tc.URL(path="projects/1/unifiedDataset:refresh") responses.add(responses.POST, str(url), json=operation_json) response = tc.dataset.unified._apply_changes_async(s, unified_dataset) assert response == tc.operation._from_json(operation_url, operation_json)
def test_operation_check_success(): s = fake.session() url = tc.URL(path="operations/1") op_json = utils.load_json("operation_succeeded.json") op = tc.operation._from_json(url, op_json) tc.operation.check(s, op)
def test_upsert_index_as_primary_key(): s = utils.session() dataset = utils.dataset() url = tc.URL(path="datasets/1:updateRecords") updates = [ tc.record._create_command(record, primary_key_name="primary_key") for record in _records_with_keys_json_2 ] snoop: Dict = {} responses.add_callback( responses.POST, str(url), partial( utils.capture_payload, snoop=snoop, status=200, response_json=_response_json ), ) df = pd.DataFrame( _records_json_2, index=[record["primary_key"] for record in _records_with_keys_json_2], ) df.index.name = "primary_key" response = tc.dataframe.upsert(s, dataset, df, primary_key_name="primary_key") assert response == _response_json assert snoop["payload"] == utils.stringify(updates)
def test_upsert(): s = utils.session() dataset = utils.dataset() url = tc.URL(path="datasets/1:updateRecords") updates = [ tc.record._create_command(record, primary_key_name="primary_key") for record in _records_json ] snoop: Dict = {} responses.add_callback( responses.POST, str(url), partial(utils.capture_payload, snoop=snoop, status=200, response_json=_response_json), ) response = tc.record.upsert(s, dataset, _records_json, primary_key_name="primary_key") assert response == _response_json assert snoop["payload"] == utils.stringify(updates)
def test_create(): s = utils.session() dataset = utils.dataset() attrs = tuple([ tc.SubAttribute( name=str(i), is_nullable=True, type=tc.attribute.type.Array(tc.attribute.type.STRING), ) for i in range(4) ]) attrs_url = tc.URL(path=dataset.url.path + "/attributes") url = replace(attrs_url, path=attrs_url.path + "/attr") attr_json = utils.load_json("attribute.json") responses.add(responses.POST, str(attrs_url), json=attr_json) attr = tc.attribute.create( s, dataset, name="attr", is_nullable=False, type=tc.attribute.type.Record(attributes=attrs), ) assert attr == tc.attribute._from_json(url, attr_json)
def attribute() -> tc.Attribute: return tc.Attribute( url=tc.URL(path="datasets/1/attributes/RowNum"), name="RowNum", type=tc.attribute.type.DEFAULT, description="Synthetic row number", is_nullable=False, )
def test_from_resource_id_not_found(): s = utils.session() instance = utils.instance() url = tc.URL(path="projects/1") responses.add(responses.GET, str(url), status=404) with pytest.raises(tc.project.NotFound): tc.project.from_resource_id(s, instance, "1")
def test_operation_from_json(): url = tc.URL(path="operations/1") operation_json = utils.load_json("operation_succeeded.json") op = tc.operation._from_json(url, operation_json) assert op.url == url assert op.type == operation_json["type"] assert op.description == operation_json["description"] assert op.status == operation_json["status"] assert tc.operation.succeeded(op)
def test_delete(): s = utils.session() url = tc.URL(path="datasets/1/attributes/RowNum") attr_json = utils.load_json("attributes.json")[0] attr = tc.attribute._from_json(url, attr_json) responses.add(responses.DELETE, str(attr.url), status=204) tc.attribute.delete(s, attr)
def test_json(): """original -> to_json -> from_json -> original""" attrs_json = utils.load_json("attributes.json") dataset_id = 1 for attr_json in attrs_json: attr_id = attr_json["name"] url = tc.URL(f"datasets/{dataset_id}/attributes/{attr_id}") attr = tc.attribute._from_json(url, attr_json) assert attr == tc.attribute._from_json(url, tc.attribute.to_json(attr))
def test_from_project_dataset_not_found(): s = fake.session() instance = fake.instance() project = fake.mastering_project() url = tc.URL(path="projects/1/unifiedDataset") responses.add(responses.GET, str(url), status=404) with pytest.raises(tc.dataset.unified.NotFound): tc.dataset.unified.from_project(s, instance, project)
def test_from_resource_id(): s = utils.session() dataset = utils.dataset() url = tc.URL(path=dataset.url.path + "/attributes/attr") attr_json = utils.load_json("attribute.json") responses.add(responses.GET, str(url), json=attr_json) attr = tc.attribute.from_resource_id(s, dataset, "attr") assert attr == tc.attribute._from_json(url, attr_json)
def test_delete_attribute_not_found(): s = utils.session() url = tc.URL(path="datasets/1/attributes/RowNum") attr_json = utils.load_json("attributes.json")[0] attr = tc.attribute._from_json(url, attr_json) responses.add(responses.PUT, str(attr.url), status=404) with pytest.raises(tc.attribute.NotFound): attr = tc.attribute.update(s, attr)
def test_from_json(): attrs_json = utils.load_json("attributes.json") dataset_id = 1 for attr_json in attrs_json: attr_id = attr_json["name"] url = tc.URL(path=f"datasets/{dataset_id}/attributes/{attr_id}") attr = tc.attribute._from_json(url, attr_json) assert attr.name == attr_json["name"] assert attr.description == attr_json["description"] assert attr.is_nullable == attr_json["isNullable"]
def test_operation_failed_success(): s = fake.session() url = tc.URL(path="operations/1") op_json = utils.load_json("operation_failed.json") op = tc.operation._from_json(url, op_json) with pytest.raises(tc.operation.Failed) as exc_info: tc.operation.check(s, op) err_msg = str(exc_info.value) assert str(url) in err_msg assert op.status is not None and str(op.status["state"]) in err_msg
def test_stream(): s = utils.session() dataset = utils.dataset() url = tc.URL(path="datasets/1/records") responses.add( responses.GET, str(url), body="\n".join(json.dumps(x) for x in _records_json) ) records = tc.record.stream(s, dataset) assert list(records) == _records_json
def test_from_resource_id_mastering(): s = utils.session() instance = utils.instance() project_json = utils.load_json("mastering_project.json") url = tc.URL(path="projects/1") responses.add(responses.GET, str(url), json=project_json) project = tc.project.from_resource_id(s, instance, "1") assert isinstance(project, tc.MasteringProject) assert project.name == "proj" assert project.description == "Mastering Project"
def test_from_resource_id(): s = utils.session() instance = utils.instance() dataset_json = utils.load_json("dataset.json") url = tc.URL(path="datasets/1") responses.add(responses.GET, str(url), json=dataset_json) dataset = tc.dataset.from_resource_id(s, instance, "1") assert dataset.name == "dataset 1 name" assert dataset.description == "dataset 1 description" assert dataset.key_attribute_names == ("tamr_id", )
def test_operation_by_url(): s = fake.session() url = tc.URL(path="operations/1") operation_json = utils.load_json("operation_succeeded.json") responses.add(responses.GET, str(url), json=operation_json) op = tc.operation._by_url(s, url) assert op.url == url assert op.type == operation_json["type"] assert op.description == operation_json["description"] assert op.status == operation_json["status"] assert tc.operation.succeeded(op)
def test_from_project(): s = fake.session() instance = fake.instance() project = fake.mastering_project() dataset_json = utils.load_json("dataset.json") url = tc.URL(path="projects/1/unifiedDataset") responses.add(responses.GET, str(url), json=dataset_json) unified_dataset = tc.dataset.unified.from_project(s, instance, project) assert unified_dataset.name == "dataset 1 name" assert unified_dataset.description == "dataset 1 description" assert unified_dataset.key_attribute_names == ("tamr_id", )
def test_operation_poll(): s = fake.session() url = tc.URL(path="operations/1") pending_operation_json = utils.load_json("operation_pending.json") op1 = tc.operation._from_json(url, pending_operation_json) succeeded_operation_json = utils.load_json("operation_succeeded.json") responses.add(responses.GET, str(url), json=succeeded_operation_json) op2 = tc.operation.poll(s, op1) assert op2.url == op1.url assert not tc.operation.succeeded(op1) assert tc.operation.succeeded(op2)
def test_update(): s = utils.session() url = tc.URL(path="datasets/1/attributes/RowNum") attr_json = utils.load_json("attributes.json")[0] attr = tc.attribute._from_json(url, attr_json) updated_attr_json = utils.load_json("updated_attribute.json") responses.add(responses.PUT, str(attr.url), json=updated_attr_json) updated_attr = tc.attribute.update( s, attr, description=updated_attr_json["description"]) assert updated_attr == replace( attr, description=updated_attr_json["description"])
def test_ndjson(): s = utils.session() records = [{"a": 1}, {"b": 2}, {"c": 3}] url = tc.URL(path="datasets/1/records") responses.add(responses.GET, str(url), body="\n".join(json.dumps(x) for x in records)) r = s.get(str(url)) ndjson = list(tc.response.ndjson(r)) assert len(ndjson) == 3 for record in ndjson: assert record in records
def test_operation_from_response(): s = fake.session() instance = fake.instance() url = tc.URL(path="operations/1") operation_json = utils.load_json("operation_succeeded.json") responses.add(responses.GET, str(url), json=operation_json) r = s.get(str(url)) op = tc.operation._from_response(instance, r) assert op.url == url assert op.type == operation_json["type"] assert op.description == operation_json["description"] assert op.status == operation_json["status"] assert tc.operation.succeeded(op)
def test_operation_from_response_noop(): s = fake.session() instance = fake.instance() url = tc.URL(path="operations/2") responses.add(responses.GET, str(url), status=204) url_dummy = tc.URL(path="operations/-1") responses.add(responses.GET, str(url_dummy), status=404) r = s.get(str(url)) op2 = tc.operation._from_response(instance, r) assert op2.url is not None assert op2.type == "NOOP" assert op2.description is not None assert op2.status is not None assert op2.status["state"] == "SUCCEEDED" assert tc.operation.succeeded(op2) op2w = tc.operation.wait(s, op2) assert tc.operation.succeeded(op2w) with pytest.raises(tc.operation.NotFound): tc.operation.poll(s, op2w)
def test_by_resource_id(): s = fake.session() instance = fake.instance() url = tc.URL(path="operations/1") operation_json = utils.load_json("operation_succeeded.json") responses.add(responses.GET, str(url), json=operation_json) resource_id = "1" op = tc.operation.by_resource_id(s, instance, resource_id) assert op.url == url assert op.type == operation_json["type"] assert op.description == operation_json["description"] assert op.status == operation_json["status"] assert tc.operation.succeeded(op)
def test_from_json_unrecognized_project_type(): instance = fake.instance() url = tc.URL("project/1", instance) data: tc._types.JsonDict = { "id": "unify://unified-data/v1/projects/1", "name": "project 1", "description": "A project of unknown type", "type": "UNKNOWN", "unifiedDatasetName": "", "relativeId": "projects/1", "externalId": "58bdbe72-3c08-427d-97bd-45b16d92c79c", } project = tc.project._from_json(url, data) assert isinstance(project, tc.UnknownProject) assert project.name == "project 1" assert project.description == "A project of unknown type"
def test_poll(): s = fake.session() data = { "id": "unify://unified-data/v1/backups/2020-08-17_21-32-10-961", "relativeId": "2020-08-17_21-32-10-961", "user": "******", "backupPath": "/home/ubuntu/tamr/backups/2020-08-17_21-32-10-961", "state": "RUNNING", "stage": "", "errorMessage": "", "created": "2020-08-17_21-32-10-961", "lastModified": "2020-08-17_21-51-57-600", } backup = tc.backup._from_json( url=tc.URL(path="backups/2020-08-17_21-32-10-961"), data=data) tc.backup.poll(session=s, backup=backup)
def from_resource_id(session: tc.Session, instance: tc.Instance, id: str) -> Dataset: """Get dataset by resource ID Fetches dataset from Tamr server Args: instance: Tamr instance containing this dataset id: Dataset ID Raises: DatasetNotFound: If no dataset could be found at the specified URL. Corresponds to a 404 HTTP error. requests.HTTPError: If any other HTTP error is encountered. """ url = tc.URL(instance=instance, path=f"datasets/{id}") return _from_url(session, url)
def test_cancel(): s = fake.session() data = { "id": "unify://unified-data/v1/restore/restore-2020-08-19_20-01-20-233", "relativeId": "restore-2020-08-19_20-01-20-233", "user": "******", "backupPath": "/home/ubuntu/tamr/backups/2020-08-17_22-07-11-100", "state": "RUNNING", "stage": "", "errorMessage": "", "created": "2020-08-19_20-01-20-233", "lastModified": "2020-08-19_20-02-19-351", } restore = tc.restore._from_json(url=tc.URL(path="instance/restore"), data=data) tc.restore.cancel(session=s, restore=restore)