def test_execute(self, stub):
obj = types.Int(1)
parameters = {"foo": types.Str("bar")}
job = Job.build(obj, parameters, channel="foo")
new_message = job_pb2.Job(
id="foo",
parameters=job._message.parameters,
serialized_graft=job._message.serialized_graft,
typespec=job._message.typespec,
type=job._message.type,
channel="foo",
)
stub.return_value.CreateJob.return_value = new_message
job.execute()
job.execute(
) # if it has an id short circuit execution, create should only be called once
stub.return_value.CreateJob.assert_called_once_with(
job_pb2.CreateJobRequest(
parameters=job._message.parameters,
serialized_graft=job._message.serialized_graft,
typespec=job._message.typespec,
type=job._message.type,
channel="foo",
),
timeout=Client.DEFAULT_TIMEOUT,
)
assert job._message is new_message
def test_create(self, stub):
obj = types.Int(1)
parameters = {"foo": types.Str("bar")}
typespec = cereal.serialize_typespec(type(obj))
create_job_request_message = job_pb2.CreateJobRequest(
parameters=json.dumps(parameters_to_grafts(**parameters)),
serialized_graft=json.dumps(obj.graft),
typespec=typespec,
type=types_pb2.ResultType.Value(cereal.typespec_to_unmarshal_str(typespec)),
no_cache=False,
channel=_channel.__channel__,
)
message = job_pb2.Job(
id="foo",
parameters=create_job_request_message.parameters,
serialized_graft=create_job_request_message.serialized_graft,
typespec=create_job_request_message.typespec,
type=create_job_request_message.type,
no_cache=create_job_request_message.no_cache,
channel=create_job_request_message.channel,
)
stub.return_value.CreateJob.return_value = message
job = Job(obj, parameters)
stub.return_value.CreateJob.assert_called_once_with(
create_job_request_message,
timeout=Client.DEFAULT_TIMEOUT,
metadata=(("x-wf-channel", create_job_request_message.channel),),
)
assert job._message is message
def execute(self):
"""
Asynchronously submit a job for execution.
After submission, ``self.id`` will be the ID of the running job.
This method is idempotent: calling it multiple times on the same `Job` object
will only trigger execution once.
Example
-------
>>> from descarteslabs.workflows import Job, Int
>>> job = Job.build(Int(1), {})
>>> job.execute() # doctest: +SKIP
"""
if self.id is not None:
return
message = self._client.api["CreateJob"](
job_pb2.CreateJobRequest(
parameters=self._message.parameters,
serialized_graft=self._message.serialized_graft,
typespec=self._message.typespec,
type=self._message.type,
channel=self._message.channel,
),
timeout=self._client.DEFAULT_TIMEOUT,
)
self._message = message
def __init__(self, proxy_object, parameters, client=None, cache=True):
"""
Creates a new `Job` to compute the provided proxy object with the given
parameters.
Parameters
----------
proxy_object: Proxytype
Proxy object to compute
parameters: dict[str, Proxytype]
Python dictionary of parameter names and values
client : `.workflows.client.Client`, optional
Allows you to use a specific client instance with non-default
auth and parameters
cache : bool, default True
Whether to use the cache for this job.
Returns
-------
Job
The job that's executing.
Example
-------
>>> from descarteslabs.workflows import Job, Int, parameter
>>> my_int = Int(1) + parameter("other_int", Int)
>>> job = Job(my_int, {"other_int": 10}) # doctest: +SKIP
>>> job.stage # doctest: +SKIP
QUEUED
"""
if client is None:
client = get_global_grpc_client()
typespec = serialize_typespec(type(proxy_object))
result_type = typespec_to_unmarshal_str(typespec)
# ^ this also preemptively checks whether the result type is something we'll know how to unmarshal
parameters = parameters_to_grafts(**parameters)
message = client.api["CreateJob"](
job_pb2.CreateJobRequest(
parameters=json.dumps(parameters),
serialized_graft=json.dumps(proxy_object.graft),
typespec=typespec,
type=types_pb2.ResultType.Value(result_type),
no_cache=not cache,
channel=_channel.__channel__,
),
timeout=client.DEFAULT_TIMEOUT,
)
self._message = message
self._client = client
self._object = proxy_object
def test_create(self, stub):
obj = types.Int(1)
parameters = {"foo": types.Str("bar")}
typespec = cereal.serialize_typespec(type(obj))
format_proto = user_format_to_proto({
"type": "pyarrow",
"compression": "brotli"
})
destination_proto = user_destination_to_proto({"type": "download"})
create_job_request_message = job_pb2.CreateJobRequest(
parameters=json.dumps(parameters_to_grafts(**parameters)),
serialized_graft=json.dumps(obj.graft),
typespec=typespec,
type=types_pb2.ResultType.Value(
cereal.typespec_to_unmarshal_str(typespec)),
format=format_proto,
destination=destination_proto,
no_cache=False,
channel=_channel.__channel__,
)
message = job_pb2.Job(
id="foo",
parameters=create_job_request_message.parameters,
serialized_graft=create_job_request_message.serialized_graft,
typespec=create_job_request_message.typespec,
type=create_job_request_message.type,
format=create_job_request_message.format,
destination=create_job_request_message.destination,
no_cache=create_job_request_message.no_cache,
channel=create_job_request_message.channel,
)
stub.return_value.CreateJob.return_value = message
job = Job(
obj,
parameters,
format={
"type": "pyarrow",
"compression": "brotli"
},
destination="download",
)
stub.return_value.CreateJob.assert_called_once_with(
create_job_request_message,
timeout=Client.DEFAULT_TIMEOUT,
metadata=(("x-wf-channel", create_job_request_message.channel), ),
)
assert job._message is message
def __init__(
self,
obj,
geoctx=None,
format="pyarrow",
destination="download",
cache=True,
_ruster=None,
_trace=False,
client=None,
**arguments,
):
"""
Creates a new `Job` to compute the provided proxy object with the given
arguments.
Parameters
----------
obj: Proxytype
Proxy object to compute, or list/tuple of proxy objects.
If it depends on parameters, ``obj`` is first converted
to a `.Function` that takes those parameters.
geoctx: `~.workflows.types.geospatial.GeoContext`, or None
The GeoContext parameter under which to run the computation.
Almost all computations will require a `~.workflows.types.geospatial.GeoContext`,
but for operations that only involve non-geospatial types,
this parameter is optional.
format: str or dict, default "pyarrow"
The serialization format for the result.
destination: str or dict, default "download"
The destination for the result.
cache: bool, default True
Whether to use the cache for this job.
client: `.workflows.client.Client`, optional
Allows you to use a specific client instance with non-default
auth and parameters
**arguments: Any
Values for all parameters that ``obj`` depends on
(or arguments that ``obj`` takes, if it's a `.Function`).
Can be given as Proxytypes, or as Python objects like numbers,
lists, and dicts that can be promoted to them.
These arguments cannot depend on any parameters.
Example
-------
>>> from descarteslabs.workflows import Job, Int, parameter
>>> my_int = Int(1) + parameter("other_int", Int)
>>> job = Job(my_int, other_int=10) # doctest: +SKIP
>>> job.stage # doctest: +SKIP
QUEUED
"""
if client is None:
client = get_global_grpc_client()
if geoctx is not None:
try:
geoctx = GeoContext._promote(geoctx)
except ProxyTypeError as e:
raise TypeError(f"Invalid GeoContext {geoctx!r}: {e}")
obj, argument_grafts, typespec, result_type = to_computable(
obj, arguments)
format_proto = user_format_to_proto(format)
destination_proto = user_destination_to_proto(destination)
message = client.api["CreateJob"](
job_pb2.CreateJobRequest(
serialized_graft=json.dumps(obj.graft),
typespec=typespec,
arguments={
name: json.dumps(arg)
for name, arg in argument_grafts.items()
},
geoctx_graft=json.dumps(geoctx.graft)
if geoctx is not None else None,
no_ruster=_ruster is False,
channel=client._wf_channel,
client_version=__version__,
no_cache=not cache,
trace=_trace,
type=types_pb2.ResultType.Value(result_type),
format=format_proto,
destination=destination_proto,
),
timeout=client.DEFAULT_TIMEOUT,
)
self._message = message
self._client = client
self._object = obj
self._arguments = None