def test_refresh(self, stub):
message = job_pb2.Job(id="foo")
refresh_message = job_pb2.Job(id="foo", status=job_pb2.STATUS_UNKNOWN)
job = Job(message)
stub.return_value.GetJob.return_value = refresh_message
job.refresh()
stub.return_value.GetJob.assert_called_with(
job_pb2.GetJobRequest(id=job.id), timeout=Client.DEFAULT_TIMEOUT)
assert job._message == refresh_message
def test_wait_for_result_success(self, stub):
id_ = "foo"
message = job_pb2.Job(id=id_)
j = Job(message)
j._load_result = mock.Mock()
status = job_pb2.STATUS_SUCCESS
stub.return_value.GetJob.return_value = job_pb2.Job(id=id_,
status=status)
j._wait_for_result()
j._load_result.assert_called_once_with()
assert j._message.status == status
def test_cancel(self, stub):
message = job_pb2.Job(id="foo")
cancel_message = job_pb2.Job(id="foo",
status=job_pb2.STATUS_FAILURE,
terminated=True)
job = Job(message)
stub.return_value.CancelJob.return_value = cancel_message
job.cancel()
stub.return_value.CancelJob.assert_called_with(
job_pb2.CancelJobRequest(id=job.id),
timeout=Client.DEFAULT_TIMEOUT)
assert job._message == cancel_message
def test_wait_for_result_failure(self, stub):
id_ = "foo"
message = job_pb2.Job(id=id_)
j = Job(message)
status = job_pb2.STATUS_FAILURE
stub.return_value.GetJob.return_value = job_pb2.Job(id=id_,
status=status)
with pytest.raises(Exception):
# TODO(justin) fix exception type
j._wait_for_result()
assert j._message.status == status
def test_execute(self, stub):
obj = types.Int(1)
parameters = {"foo": types.Str("bar")}
job = Job.build(obj, parameters, channel="foo")
new_message = job_pb2.Job(
id="foo",
parameters=job._message.parameters,
serialized_graft=job._message.serialized_graft,
serialized_typespec=job._message.serialized_typespec,
type=job._message.type,
channel="foo",
)
stub.return_value.CreateJob.return_value = new_message
job.execute()
job.execute(
) # if it has an id short circuit execution, create should only be called once
stub.return_value.CreateJob.assert_called_once_with(
job_pb2.CreateJobRequest(
parameters=job._message.parameters,
serialized_graft=job._message.serialized_graft,
serialized_typespec=job._message.serialized_typespec,
type=job._message.type,
channel="foo",
),
timeout=Client.DEFAULT_TIMEOUT,
)
assert job._message is new_message
def test_download_result(self, stub):
job = Job(
job_pb2.Job(
id="foo",
status=job_pb2.STATUS_SUCCESS,
error=job_pb2.JobError(code=errors_pb2.ERROR_NONE),
))
result = {}
buffer = pa.serialize(result,
context=serialization_context).to_buffer()
codec = "lz4"
responses.add(
responses.GET,
Job.BUCKET_PREFIX.format(job.id),
body=pa.compress(buffer, codec=codec, asbytes=True),
headers={
"x-goog-meta-codec": codec,
"x-goog-meta-decompressed_size": str(len(buffer)),
},
status=200,
)
assert job._download_result() == result
def test_instantiate(self, stub, client):
message = job_pb2.Job(id="foo")
job = Job(message, client=client)
assert job._message == message
if client is not None:
assert job._client == client
else:
assert isinstance(job._client, Client)
def test_unmarshal_primitive(self, stub):
marshalled = (1, 2, True, None)
job = Job(
job_pb2.Job(id="foo",
status=job_pb2.STATUS_SUCCESS,
type=types_pb2.List))
result = job._unmarshal(marshalled)
assert result == list(marshalled)
def test_wait_for_result_terminated(self, stub):
id_ = "foo"
message = job_pb2.Job(id=id_)
j = Job(message)
status = job_pb2.STATUS_FAILURE
stub.return_value.GetJob.return_value = job_pb2.Job(
id=id_,
status=status,
stage=job_pb2.STAGE_DONE,
error=job_pb2.JobError(code=errors_pb2.ERROR_TERMINATED),
)
with pytest.raises(Exception):
# TODO(justin) fix exception type
j._wait_for_result()
assert j._message.status == status
def test_load_result_error(self, stub):
message = job_pb2.Job(
id="foo",
status=job_pb2.STATUS_FAILURE,
error=job_pb2.JobError(code=errors_pb2.ERROR_INVALID),
)
job = Job(message)
with pytest.raises(JobInvalid):
job._load_result()
def test_watch(self, stub):
id_ = "foo"
message = job_pb2.Job(id=id_)
job = Job(message)
stub.return_value.WatchJob.return_value = [
job_pb2.Job(id=id_,
stage=job_pb2.STAGE_PENDING,
status=job_pb2.STATUS_UNKNOWN),
job_pb2.Job(id=id_,
stage=job_pb2.STAGE_RUNNING,
status=job_pb2.STATUS_UNKNOWN),
job_pb2.Job(id=id_,
stage=job_pb2.STAGE_DONE,
status=job_pb2.STATUS_SUCCESS),
]
state_messages = [state._message for state in job.watch()]
assert state_messages == stub.return_value.WatchJob.return_value
def test_init():
message = job_pb2.Job(id="foo",
error=job_pb2.JobError(
code=errors_pb2.ERROR_DEADLINE, message="bar"))
job = Job(message)
e = JobComputeError(job)
assert e.code == job.error.code
assert e.message == job.error.message
assert e.id == job.id
def test_get(self, stub, client):
id_ = "foo"
message = job_pb2.Job(id=id_)
stub.return_value.GetJob.return_value = message
job = Job.get(id_, client=client)
assert job._message == message
stub.return_value.GetJob.assert_called_with(
job_pb2.GetJobRequest(id=id_), timeout=Client.DEFAULT_TIMEOUT)
if client is not None:
assert job._client == client
else:
assert isinstance(job._client, Client)
def test_wait_for_result_timeout(self, stub):
id_ = "foo"
status = job_pb2.STATUS_UNKNOWN
message = job_pb2.Job(id=id_,
status=status,
stage=job_pb2.STAGE_PENDING)
j = Job(message)
stub.return_value.GetJob.return_value = message
with pytest.raises(Exception):
# TODO(justin) fix exception type
j._wait_for_result(1e-4)
assert j._message.status == status
stub.return_value.GetJob.assert_called()
def test_unmarshal_image(self, stub):
marshalled = {
"ndarray": {
"red": []
},
"properties": {
"foo": "bar",
"geometry": {
"type": "Point",
"coordinates": [0, 0]
},
},
"bandinfo": {
"red": {}
},
"geocontext": {
"geometry": None,
"crs": "EPSG:4326",
"bounds": (-98, 40, -90, 44),
},
}
job = Job(
job_pb2.Job(id="foo",
status=job_pb2.STATUS_SUCCESS,
type=types_pb2.Image))
result = job._unmarshal(marshalled)
# NOTE(gabe): we check the class name, versus `isinstance(result, results.ImageResult)`,
# because importing results in this test would register its unmarshallers,
# and part of what we're testing for is that the unmarshallers are getting registered correctly.
assert result.__class__.__name__ == "ImageResult"
assert result.ndarray == marshalled["ndarray"]
assert result.properties == marshalled["properties"]
assert result.bandinfo == marshalled["bandinfo"]
assert result.geocontext == marshalled["geocontext"]
def build(cls, proxy_object, parameters, channel=None, client=None):
"""
Build a new `Job` for computing a proxy object under certain parameters.
Does not actually trigger computation; call `Job.execute` on the result to do so.
Parameters
----------
proxy_object: Proxytype
Proxy object to compute
parameters: dict[str, Proxytype]
Python dictionary of parameter names and values
channel: str or None, optional
Channel name to submit the `Job` to.
If None, uses the default channel for this client
(``descarteslabs.workflows.__channel__``).
Channels are different versions of the backend,
to allow for feature changes without breaking existing code.
Not all clients are compatible with all channels.
This client is only guaranteed to work with its default channel,
whose name can be found under ``descarteslabs.workflows.__channel__``.
client : `.workflows.client.Client`, optional
Allows you to use a specific client instance with non-default
auth and parameters
Returns
-------
Job
The job waiting to be executed.
Example
-------
>>> from descarteslabs.workflows import Job, Int, parameter
>>> my_int = Int(1) + parameter("other_int", Int)
>>> job = Job.build(my_int, {"other_int": 10})
>>> # the job does not execute until `.execute` is called
>>> job.execute() # doctest: +SKIP
"""
if channel is None:
# NOTE(gabe): we look up the variable from the `_channel` package here,
# rather than importing it directly at the top,
# so it can easily be changed during an interactive session.
channel = _channel.__channel__
if client is None:
client = Client()
typespec = serialize_typespec(type(proxy_object))
result_type = _typespec_to_unmarshal_str(typespec)
# ^ this also preemptively checks whether the result type is something we'll know how to unmarshal
parameters = parameters_to_grafts(**parameters)
message = job_pb2.Job(
parameters=json.dumps(parameters),
serialized_graft=json.dumps(proxy_object.graft),
serialized_typespec=json.dumps(typespec),
type=types_pb2.ResultType.Value(result_type),
channel=channel,
)
instance = cls(message, client)
instance._object = proxy_object
return instance
