def test_refresh(self, stub): message = job_pb2.Job(id="foo") refresh_message = job_pb2.Job(id="foo", status=job_pb2.STATUS_UNKNOWN) job = Job(message) stub.return_value.GetJob.return_value = refresh_message job.refresh() stub.return_value.GetJob.assert_called_with( job_pb2.GetJobRequest(id=job.id), timeout=Client.DEFAULT_TIMEOUT) assert job._message == refresh_message
def test_wait_for_result_success(self, stub): id_ = "foo" message = job_pb2.Job(id=id_) j = Job(message) j._load_result = mock.Mock() status = job_pb2.STATUS_SUCCESS stub.return_value.GetJob.return_value = job_pb2.Job(id=id_, status=status) j._wait_for_result() j._load_result.assert_called_once_with() assert j._message.status == status
def test_cancel(self, stub): message = job_pb2.Job(id="foo") cancel_message = job_pb2.Job(id="foo", status=job_pb2.STATUS_FAILURE, terminated=True) job = Job(message) stub.return_value.CancelJob.return_value = cancel_message job.cancel() stub.return_value.CancelJob.assert_called_with( job_pb2.CancelJobRequest(id=job.id), timeout=Client.DEFAULT_TIMEOUT) assert job._message == cancel_message
def test_wait_for_result_failure(self, stub): id_ = "foo" message = job_pb2.Job(id=id_) j = Job(message) status = job_pb2.STATUS_FAILURE stub.return_value.GetJob.return_value = job_pb2.Job(id=id_, status=status) with pytest.raises(Exception): # TODO(justin) fix exception type j._wait_for_result() assert j._message.status == status
def test_execute(self, stub): obj = types.Int(1) parameters = {"foo": types.Str("bar")} job = Job.build(obj, parameters, channel="foo") new_message = job_pb2.Job( id="foo", parameters=job._message.parameters, serialized_graft=job._message.serialized_graft, serialized_typespec=job._message.serialized_typespec, type=job._message.type, channel="foo", ) stub.return_value.CreateJob.return_value = new_message job.execute() job.execute( ) # if it has an id short circuit execution, create should only be called once stub.return_value.CreateJob.assert_called_once_with( job_pb2.CreateJobRequest( parameters=job._message.parameters, serialized_graft=job._message.serialized_graft, serialized_typespec=job._message.serialized_typespec, type=job._message.type, channel="foo", ), timeout=Client.DEFAULT_TIMEOUT, ) assert job._message is new_message
def test_download_result(self, stub): job = Job( job_pb2.Job( id="foo", status=job_pb2.STATUS_SUCCESS, error=job_pb2.JobError(code=errors_pb2.ERROR_NONE), )) result = {} buffer = pa.serialize(result, context=serialization_context).to_buffer() codec = "lz4" responses.add( responses.GET, Job.BUCKET_PREFIX.format(job.id), body=pa.compress(buffer, codec=codec, asbytes=True), headers={ "x-goog-meta-codec": codec, "x-goog-meta-decompressed_size": str(len(buffer)), }, status=200, ) assert job._download_result() == result
def test_instantiate(self, stub, client): message = job_pb2.Job(id="foo") job = Job(message, client=client) assert job._message == message if client is not None: assert job._client == client else: assert isinstance(job._client, Client)
def test_unmarshal_primitive(self, stub): marshalled = (1, 2, True, None) job = Job( job_pb2.Job(id="foo", status=job_pb2.STATUS_SUCCESS, type=types_pb2.List)) result = job._unmarshal(marshalled) assert result == list(marshalled)
def test_wait_for_result_terminated(self, stub): id_ = "foo" message = job_pb2.Job(id=id_) j = Job(message) status = job_pb2.STATUS_FAILURE stub.return_value.GetJob.return_value = job_pb2.Job( id=id_, status=status, stage=job_pb2.STAGE_DONE, error=job_pb2.JobError(code=errors_pb2.ERROR_TERMINATED), ) with pytest.raises(Exception): # TODO(justin) fix exception type j._wait_for_result() assert j._message.status == status
def test_load_result_error(self, stub): message = job_pb2.Job( id="foo", status=job_pb2.STATUS_FAILURE, error=job_pb2.JobError(code=errors_pb2.ERROR_INVALID), ) job = Job(message) with pytest.raises(JobInvalid): job._load_result()
def test_watch(self, stub): id_ = "foo" message = job_pb2.Job(id=id_) job = Job(message) stub.return_value.WatchJob.return_value = [ job_pb2.Job(id=id_, stage=job_pb2.STAGE_PENDING, status=job_pb2.STATUS_UNKNOWN), job_pb2.Job(id=id_, stage=job_pb2.STAGE_RUNNING, status=job_pb2.STATUS_UNKNOWN), job_pb2.Job(id=id_, stage=job_pb2.STAGE_DONE, status=job_pb2.STATUS_SUCCESS), ] state_messages = [state._message for state in job.watch()] assert state_messages == stub.return_value.WatchJob.return_value
def test_init(): message = job_pb2.Job(id="foo", error=job_pb2.JobError( code=errors_pb2.ERROR_DEADLINE, message="bar")) job = Job(message) e = JobComputeError(job) assert e.code == job.error.code assert e.message == job.error.message assert e.id == job.id
def test_get(self, stub, client): id_ = "foo" message = job_pb2.Job(id=id_) stub.return_value.GetJob.return_value = message job = Job.get(id_, client=client) assert job._message == message stub.return_value.GetJob.assert_called_with( job_pb2.GetJobRequest(id=id_), timeout=Client.DEFAULT_TIMEOUT) if client is not None: assert job._client == client else: assert isinstance(job._client, Client)
def test_wait_for_result_timeout(self, stub): id_ = "foo" status = job_pb2.STATUS_UNKNOWN message = job_pb2.Job(id=id_, status=status, stage=job_pb2.STAGE_PENDING) j = Job(message) stub.return_value.GetJob.return_value = message with pytest.raises(Exception): # TODO(justin) fix exception type j._wait_for_result(1e-4) assert j._message.status == status stub.return_value.GetJob.assert_called()
def test_unmarshal_image(self, stub): marshalled = { "ndarray": { "red": [] }, "properties": { "foo": "bar", "geometry": { "type": "Point", "coordinates": [0, 0] }, }, "bandinfo": { "red": {} }, "geocontext": { "geometry": None, "crs": "EPSG:4326", "bounds": (-98, 40, -90, 44), }, } job = Job( job_pb2.Job(id="foo", status=job_pb2.STATUS_SUCCESS, type=types_pb2.Image)) result = job._unmarshal(marshalled) # NOTE(gabe): we check the class name, versus `isinstance(result, results.ImageResult)`, # because importing results in this test would register its unmarshallers, # and part of what we're testing for is that the unmarshallers are getting registered correctly. assert result.__class__.__name__ == "ImageResult" assert result.ndarray == marshalled["ndarray"] assert result.properties == marshalled["properties"] assert result.bandinfo == marshalled["bandinfo"] assert result.geocontext == marshalled["geocontext"]
def build(cls, proxy_object, parameters, channel=None, client=None): """ Build a new `Job` for computing a proxy object under certain parameters. Does not actually trigger computation; call `Job.execute` on the result to do so. Parameters ---------- proxy_object: Proxytype Proxy object to compute parameters: dict[str, Proxytype] Python dictionary of parameter names and values channel: str or None, optional Channel name to submit the `Job` to. If None, uses the default channel for this client (``descarteslabs.workflows.__channel__``). Channels are different versions of the backend, to allow for feature changes without breaking existing code. Not all clients are compatible with all channels. This client is only guaranteed to work with its default channel, whose name can be found under ``descarteslabs.workflows.__channel__``. client : `.workflows.client.Client`, optional Allows you to use a specific client instance with non-default auth and parameters Returns ------- Job The job waiting to be executed. Example ------- >>> from descarteslabs.workflows import Job, Int, parameter >>> my_int = Int(1) + parameter("other_int", Int) >>> job = Job.build(my_int, {"other_int": 10}) >>> # the job does not execute until `.execute` is called >>> job.execute() # doctest: +SKIP """ if channel is None: # NOTE(gabe): we look up the variable from the `_channel` package here, # rather than importing it directly at the top, # so it can easily be changed during an interactive session. channel = _channel.__channel__ if client is None: client = Client() typespec = serialize_typespec(type(proxy_object)) result_type = _typespec_to_unmarshal_str(typespec) # ^ this also preemptively checks whether the result type is something we'll know how to unmarshal parameters = parameters_to_grafts(**parameters) message = job_pb2.Job( parameters=json.dumps(parameters), serialized_graft=json.dumps(proxy_object.graft), serialized_typespec=json.dumps(typespec), type=types_pb2.ResultType.Value(result_type), channel=channel, ) instance = cls(message, client) instance._object = proxy_object return instance