def test_ready_skipped(self):
task = UploadTask(group_id, upload_id=upload_id, client=self.client)
self.mock_response(
responses.GET,
{"data": {"id": upload_id, "attributes": {"status": "PENDING"}}},
)
self.mock_response(
responses.GET,
{"data": {"id": upload_id, "attributes": {"status": "PENDING"}}},
)
self.mock_response(
responses.GET,
{
"data": {
"id": task_id,
"attributes": {"status": "SUCCESS", "load": {"state": "SKIPPED"}},
}
},
)
assert not task.ready
assert task.status == "PENDING"
assert task.tuid is None
assert task.ready
assert task.status == "SUCCESS"
assert task.tuid == task_id
def test_results(self):
task = UploadTask(group_id, upload_id=upload_id, client=self.client)
self.mock_response(
responses.GET,
{
"data": {
"id": upload_id,
"attributes": {
"status": "SUCCESS",
"result": {
"errors": ["invalid geometry"],
"input_features": 1,
"input_rows": 1,
},
"load": {
"state": "DONE",
"errors": ["some BQ error"],
"output_rows": 1,
},
},
}
},
)
assert task.ready
assert task.status == "SUCCESS"
assert task.error_rows == 2
assert len(task.errors) == 2
assert task.input_features == 1
assert task.input_rows == 1
assert task.output_rows == 1
def upload(self, file_ref, max_errors=0):
"""
Asynchonously add features from a file of
`Newline Delimited JSON <https://github.com/ndjson/ndjson-spec>`_
features. The file itself will be uploaded synchronously,
but loading the features is done asynchronously.
Parameters
----------
file_ref : io.IOBase or str
An open file object, or a path to the file to upload.
max_errors : int
The maximum number of errors permitted before declaring failure.
Returns
-------
`UploadTask`
The upload task. The details may take time to become available
so asking for them before they're available will block
until the details are available.
"""
upload_id = self.vector_client.upload_features(file_ref, self.id)
return UploadTask(self.id,
upload_id=upload_id,
client=self.vector_client)
def test_results(self):
task = UploadTask(group_id, upload_id=upload_id, client=self.client)
self.mock_response(responses.GET, {
'data': {
'id': upload_id,
'attributes': {
'status': 'SUCCESS',
'result': {
'errors': ['invalid geometry'],
'input_features': 1,
'input_rows': 1
},
'load': {
'state': 'DONE',
'errors': ['some BQ error'],
'output_rows': 1
},
}}})
self.assertTrue(task.ready)
self.assertEqual(task.status, 'SUCCESS')
self.assertEqual(task.error_rows, 2)
self.assertEqual(len(task.errors), 2)
self.assertEqual(task.input_features, 1)
self.assertEqual(task.input_rows, 1)
self.assertEqual(task.output_rows, 1)
def upload(self, file_ref, max_errors=0, correct_winding_order=False):
"""
Asynchonously add features from a file of
`Newline Delimited JSON <https://github.com/ndjson/ndjson-spec>`_
features. The file itself will be uploaded synchronously,
but loading the features is done asynchronously.
Parameters
----------
file_ref : io.IOBase or str
An open file object, or a path to the file to upload.
max_errors : int
The maximum number of errors permitted before declaring failure.
correct_winding_order : bool
A Boolean specifying whether to correct Polygon and MultiPolygon
features that do not follow counter-clockwise winding order.
Returns
-------
`UploadTask`
The upload task. The details may take time to become available
so asking for them before they're available will block
until the details are available.
"""
upload_id = self.vector_client.upload_features(
file_ref,
self.id,
max_errors=max_errors,
correct_winding_order=correct_winding_order)
return UploadTask(self.id,
upload_id=upload_id,
client=self.vector_client)
def upload(self, file_ref, max_errors=0, fix_geometry="accept"):
"""
Asynchronously add features from a file of
`Newline Delimited JSON <https://github.com/ndjson/ndjson-spec>`_
features. The file itself will be uploaded synchronously,
but loading the features is done asynchronously.
Parameters
----------
file_ref : io.IOBase or str
An open file object, or a path to the file to upload.
max_errors : int
The maximum number of errors permitted before declaring failure.
fix_geometry : str
String specifying how to handle certain problem geometries, including those
which do not follow counter-clockwise winding order (which is required by the
GeoJSON spec but not many popular tools). Allowed values are ``reject`` (reject
invalid geometries with an error), ``fix`` (correct invalid geometries if
possible and use this corrected value when creating the feature), and ``accept``
(the default) which will correct the geometry for internal use but retain the
original geometry in the results.
Returns
-------
:class:`UploadTask <descarteslabs.common.tasks.uploadtask.UploadTask>`
The upload task. The details may take time to become available
so asking for them before they're available will block
until the details are available.
Raises
------
~descarteslabs.client.exceptions.NotFoundError
Raised if the product cannot be found.
~descarteslabs.client.exceptions.RateLimitError
Raised when too many requests have been made within a given time period.
~descarteslabs.client.exceptions.ServerError
Raised when a unknown error occurred on the server.
Example
-------
>>> from descarteslabs.vectors import FeatureCollection, Feature
>>> fc = FeatureCollection('my-vector-product-id') # doctest: +SKIP
>>> task = fc.upload("/path/to/features.ndjson") # doctest: +SKIP
"""
upload_id = self.vector_client.upload_features(
file_ref,
self.id,
max_errors=max_errors,
fix_geometry=fix_geometry)
return UploadTask(self.id,
upload_id=upload_id,
client=self.vector_client)
def test_ready_skipped(self):
task = UploadTask(group_id, upload_id=upload_id, client=self.client)
self.mock_response(responses.GET,
{'data': {'id': upload_id, 'attributes': {'status': 'PENDING'}}})
self.mock_response(responses.GET,
{'data': {'id': upload_id, 'attributes': {'status': 'PENDING'}}})
self.mock_response(responses.GET,
{'data': {'id': task_id, 'attributes': {'status': 'SUCCESS', 'load': {'state': 'SKIPPED'}}}})
self.assertFalse(task.ready)
self.assertEqual(task.status, 'PENDING')
self.assertIsNone(task.tuid)
self.assertTrue(task.ready)
self.assertEqual(task.status, 'SUCCESS')
self.assertEqual(task.tuid, task_id)
def test_ready_failure(self):
task = UploadTask(group_id, upload_id=upload_id, client=self.client)
self.mock_response(responses.GET,
{'data': {'id': upload_id, 'attributes': {'status': 'PENDING'}}})
self.mock_response(responses.GET,
{'data': {'id': upload_id, 'attributes': {'status': 'PENDING'}}})
self.mock_response(responses.GET,
{'data': {'id': task_id, 'attributes': {'status': 'FAILURE'}}})
self.assertFalse(task.ready)
self.assertEqual(task.status, 'PENDING')
self.assertIsNone(task.tuid)
self.assertTrue(task.ready)
self.assertEqual(task.status, 'FAILURE')
self.assertEqual(task.tuid, task_id)
def list_uploads(self):
"""
Get all the upload tasks for this product.
Returns
-------
list(`UploadTask`)
The list of tasks for the product.
"""
results = []
for result in self.vector_client.get_upload_results(self.id):
results.append(
UploadTask(self.id,
tuid=result.id,
result_attrs=result.attributes,
client=self.vector_client))
return results
def test_ready_failure(self):
task = UploadTask(group_id, upload_id=upload_id, client=self.client)
self.mock_response(
responses.GET,
{"data": {"id": upload_id, "attributes": {"status": "PENDING"}}},
)
self.mock_response(
responses.GET,
{"data": {"id": upload_id, "attributes": {"status": "PENDING"}}},
)
self.mock_response(
responses.GET,
{"data": {"id": task_id, "attributes": {"status": "FAILURE"}}},
)
assert not task.ready
assert task.status == "PENDING"
assert task.tuid is None
assert task.ready
assert task.status == "FAILURE"
assert task.tuid == task_id
def test_from_guid_tuid(self):
task = UploadTask(group_id, tuid=task_id, client=self.client)
assert task.tuid == task_id
def test_from_guid_upload_id(self):
task = UploadTask(group_id, upload_id=upload_id, client=self.client)
assert task.upload_id == upload_id
def list_uploads(self, pending=True):
"""
Get all the upload tasks for this product.
Parameters
----------
pending : bool
If ``True`` then include pending/currently running upload tasks in the result,
otherwise only include complete upload tasks. Defaults to ``True``.
Returns
-------
list(:class:`UploadTask <descarteslabs.common.tasks.uploadtask.UploadTask>`)
The list of tasks for the product.
Raises
------
~descarteslabs.client.exceptions.NotFoundError
Raised if the product cannot be found.
~descarteslabs.client.exceptions.RateLimitError
Raised when too many requests have been made within a given time period.
~descarteslabs.client.exceptions.ServerError
Raised when a unknown error occurred on the server.
Example
-------
>>> from descarteslabs.vectors import FeatureCollection, Feature
>>> fc = FeatureCollection('my-vector-product-id') # doctest: +SKIP
>>> task = fc.upload("/path/to/features.ndjson") # doctest: +SKIP
>>> uploads = fc.list_uploads() # doctest: +SKIP
"""
results = []
for result in self.vector_client.get_upload_results(self.id,
pending=pending):
# PENDING tasks aren't really tasks yet, and the id is the upload_id, not the task_id
if result.attributes and result.attributes.get(
"status") == "PENDING":
results.append(
UploadTask(
self.id,
upload_id=result.id,
result_attrs=result.attributes,
client=self.vector_client,
))
else:
upload = UploadTask(
self.id,
tuid=result.id,
result_attrs=result.attributes,
client=self.vector_client,
)
# get_upload_results does not (and cannot) include the task result,
# so force an update
try:
upload.get_result(wait=False)
except TransientResultError:
pass
results.append(upload)
return results
def test_from_guid_tuid(self):
task = UploadTask(group_id, tuid=task_id, client=self.client)
self.assertEqual(task.tuid, task_id)