def _run_custom(
project: Project,
*,
run_update_golden_records: bool = False,
run_publish_golden_records: bool = False,
) -> List[Operation]:
"""Executes specified steps of a golden records project.
Args:
project: The target golden records project
run_update_golden_records: Whether refresh should be called on the draft golden records
dataset
run_publish_golden_records: Whether refresh should be called on the published golden
records dataset
Returns:
The operations that were run
Raises:
TypeError: if the `project` is not a Golden Record project
"""
version.enforce_after_or_equal(project.client,
compare_version="2020.004.0")
if ProjectType[project.type] != ProjectType.GOLDEN_RECORDS:
error_msg = f"Cannot use as a golden records project. Project type: {project.type}"
LOGGER.error(error_msg)
raise TypeError(error_msg)
completed_operations = []
if run_update_golden_records:
LOGGER.info(
f"Updating the draft golden records for project {project.name} "
f"(id={project.resource_id}).")
resp = project.client.post(
f"/api/versioned/v1/projects/{project.resource_id}/goldenRecords:refresh"
).successful()
op = Operation.from_response(client=project.client, response=resp)
op = op.wait()
operation.enforce_success(op)
completed_operations.append(op)
if run_publish_golden_records:
LOGGER.info(
f"Publishing golden records for project {project.name} (id={project.resource_id})."
)
resp = project.client.post(
f"/api/versioned/v1/projects/{project.resource_id}/publishedGoldenRecords:refresh"
f"?validate=true&version=CURRENT").successful()
op = Operation.from_response(client=project.client, response=resp)
op = op.wait()
operation.enforce_success(op)
completed_operations.append(op)
return completed_operations
def _collect_operation_calls(*,
response: Response,
poll_interval_seconds: int = 3) -> List[Response]:
"""If the provided response is an Operation, wait for the operation to complete and
return responses related to that operation.
Args:
response: A previous Response generated from the same Tamr client
poll_interval_seconds: Time interval (in seconds) between subsequent polls
Returns:
Responses related to polling the operation
"""
client = utils.client._from_response(response)
op = Operation.from_response(client, response)
LOGGER.info(f"Waiting for operation to complete: {op}")
request_while_pending = client.get(
endpoint=f"/api/versioned/v1/operations/{op.resource_id}")
while op.state == "PENDING":
op = op.poll()
sleep(poll_interval_seconds)
request_while_running = client.get(
endpoint=f"/api/versioned/v1/operations/{op.resource_id}")
op.wait()
request_when_complete = client.get(
endpoint=f"/api/versioned/v1/operations/{op.resource_id}")
return [
request_while_pending, request_while_running, request_when_complete
]
def test_operation_from_response(client):
responses.add(responses.GET, full_url(client, "operations/1"), json=op_1_json)
op1 = Operation.from_response(client, client.get("operations/1").successful())
assert op1.resource_id == "1"
assert op1.succeeded
def refresh(self, **options):
"""Brings dataset up-to-date if needed, taking whatever actions are required.
:param ``**options``: Options passed to underlying :class:`~tamr_unify_client.operation.Operation` .
See :func:`~tamr_unify_client.operation.Operation.apply_options` .
:returns: The refresh operation.
:rtype: :class:`~tamr_unify_client.operation.Operation`
"""
response = self.client.post(self.api_path + ":refresh").successful()
op = Operation.from_response(self.client, response)
return op.apply_options(**options)
def from_resource_id(tamr: Client, *, job_id: Union[int, str]) -> Operation:
"""Create an operation from a job id
Args:
tamr: A Tamr client
job_id: A job ID
Returns:
A Tamr operation
"""
job_response = tamr.get(f"/api/versioned/v1/operations/{job_id}")
return Operation.from_response(tamr, job_response)
def create_profile(self, **options):
"""Create a profile for this dataset.
If a profile already exists, the existing profile will be brought
up to date.
:param ``**options``: Options passed to underlying :class:`~tamr_unify_client.operation.Operation` .
See :func:`~tamr_unify_client.operation.Operation.apply_options` .
:return: The operation to create the profile.
:rtype: :class:`~tamr_unify_client.operation.Operation`
"""
response = self.client.post(self.api_path + "/profile:refresh").successful()
op = Operation.from_response(self.client, response)
return op.apply_options(**options)
def refresh(self, **options):
"""Updates the dataset profile if needed.
The dataset profile is updated on the server; you will need to call
:func:`~tamr_unify_client.dataset.resource.Dataset.profile`
to retrieve the updated profile.
:param ``**options``: Options passed to underlying :class:`~tamr_unify_client.operation.Operation` .
See :func:`~tamr_unify_client.operation.Operation.apply_options` .
:returns: The refresh operation.
:rtype: :class:`~tamr_unify_client.operation.Operation`
"""
response = self.client.post(self.api_path + ":refresh").successful()
op = Operation.from_response(self.client, response)
return op.apply_options(**options)
def refresh(self, **options):
"""Updates the estimated pair counts if needed.
The pair count estimates are updated on the server; you will need to call
:func:`~tamr_unify_client.mastering.project.MasteringProject.estimate_pairs`
to retrieve the updated estimate.
:param ``**options``: Options passed to underlying :class:`~tamr_unify_client.operation.Operation` .
See :func:`~tamr_unify_client.operation.Operation.apply_options` .
:returns: The refresh operation.
:rtype: :class:`~tamr_unify_client.operation.Operation`
"""
response = self.client.post(self.api_path + ":refresh").successful()
op = Operation.from_response(self.client, response)
return op.apply_options(**options)
def test_operation_from_response_noop(client):
responses.add(responses.GET, full_url(client, "operations/2"), status=204)
responses.add(responses.GET, full_url(client, "operations/-1"), status=404)
op2 = Operation.from_response(client, client.get("operations/2").successful())
assert op2.api_path is not None
assert op2.relative_id is not None
assert op2.resource_id is not None
assert op2.type == "NOOP"
assert op2.description is not None
assert op2.status is not None
assert op2.state == "SUCCEEDED"
assert op2.succeeded
op2a = op2.apply_options(asynchronous=True)
assert op2a.succeeded
op2w = op2a.wait()
assert op2w.succeeded
with pytest.raises(HTTPError):
op2w.poll()