def test_task(self):
client = APIClient()
node_odm = start_processing_node()
user = User.objects.get(username="******")
self.assertFalse(user.is_superuser)
other_user = User.objects.get(username="******")
project = Project.objects.create(owner=user, name="test project")
other_project = Project.objects.create(owner=other_user,
name="another test project")
other_task = Task.objects.create(project=other_project)
# Start processing node
# Create processing node
pnode = ProcessingNode.objects.create(hostname="localhost", port=11223)
# Verify that it's working
self.assertTrue(pnode.api_version is not None)
# task creation via file upload
image1 = open("app/fixtures/tiny_drone_image.jpg", 'rb')
image2 = open("app/fixtures/tiny_drone_image_2.jpg", 'rb')
# Not authenticated?
res = client.post("/api/projects/{}/tasks/".format(project.id),
{'images': [image1, image2]},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_403_FORBIDDEN)
client.login(username="******", password="******")
# Cannot create a task for a project that does not exist
res = client.post("/api/projects/0/tasks/",
{'images': [image1, image2]},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot create a task for a project for which we have no access to
res = client.post("/api/projects/{}/tasks/".format(other_project.id),
{'images': [image1, image2]},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot create a task without images
res = client.post("/api/projects/{}/tasks/".format(project.id),
{'images': []},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
# Cannot create a task with just 1 image
res = client.post("/api/projects/{}/tasks/".format(project.id),
{'images': image1},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
# Normal case with images[], name and processing node parameter
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task',
'processing_node': pnode.id
},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
multiple_param_task = Task.objects.latest('created_at')
self.assertTrue(multiple_param_task.name == 'test_task')
self.assertTrue(multiple_param_task.processing_node.id == pnode.id)
# Cannot create a task with images[], name, but invalid processing node parameter
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task',
'processing_node': 9999
},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
# Normal case with images[] parameter
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'auto_processing_node': 'false'
},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
# Should have returned the id of the newly created task
task = Task.objects.latest('created_at')
self.assertTrue('id' in res.data)
self.assertTrue(str(task.id) == res.data['id'])
# Two images should have been uploaded
self.assertTrue(ImageUpload.objects.filter(task=task).count() == 2)
# No processing node is set
self.assertTrue(task.processing_node is None)
# tiles.json should not be accessible at this point
tile_types = ['orthophoto', 'dsm', 'dtm']
for tile_type in tile_types:
res = client.get("/api/projects/{}/tasks/{}/{}/tiles.json".format(
project.id, task.id, tile_type))
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
# Neither should an individual tile
# Z/X/Y coords are chosen based on node-odm test dataset for orthophoto_tiles/
res = client.get(
"/api/projects/{}/tasks/{}/orthophoto/tiles/16/16020/42443.png".
format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot access a tiles.json we have no access to
res = client.get(
"/api/projects/{}/tasks/{}/orthophoto/tiles.json".format(
other_project.id, other_task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot access an individual tile we have no access to
res = client.get(
"/api/projects/{}/tasks/{}/orthophoto/tiles/16/16020/42443.png".
format(other_project.id, other_task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot download assets (they don't exist yet)
for asset in list(task.ASSETS_MAP.keys()):
res = client.get("/api/projects/{}/tasks/{}/download/{}".format(
project.id, task.id, asset))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot access raw assets (they don't exist yet)
res = client.get(
"/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif"
.format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot assign processing node to a task we have no access to
res = client.patch(
"/api/projects/{}/tasks/{}/".format(other_project.id,
other_task.id),
{'processing_node': pnode.id})
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
testWatch.clear()
# No UUID at this point
self.assertTrue(len(task.uuid) == 0)
# Assign processing node to task via API
res = client.patch(
"/api/projects/{}/tasks/{}/".format(project.id, task.id),
{'processing_node': pnode.id})
self.assertTrue(res.status_code == status.HTTP_200_OK)
# On update scheduler.processing_pending_tasks should have been called in the background
testWatch.wait_until_call("app.scheduler.process_pending_tasks",
timeout=5)
# Processing should have started and a UUID is assigned
task.refresh_from_db()
self.assertTrue(task.status in [
status_codes.RUNNING, status_codes.COMPLETED
]) # Sometimes the task finishes and we can't test for RUNNING state
self.assertTrue(len(task.uuid) > 0)
time.sleep(DELAY)
# Calling process pending tasks should finish the process
scheduler.process_pending_tasks()
task.refresh_from_db()
self.assertTrue(task.status == status_codes.COMPLETED)
# Can download assets
for asset in list(task.ASSETS_MAP.keys()):
res = client.get("/api/projects/{}/tasks/{}/download/{}".format(
project.id, task.id, asset))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# A textured mesh archive file should exist
self.assertTrue(
os.path.exists(
task.assets_path(
task.ASSETS_MAP["textured_model.zip"]["deferred_path"])))
# Can download raw assets
res = client.get(
"/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif"
.format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Can access tiles.json
for tile_type in tile_types:
res = client.get("/api/projects/{}/tasks/{}/{}/tiles.json".format(
project.id, task.id, tile_type))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Bounds are what we expect them to be
# (4 coords in lat/lon)
tiles = json.loads(res.content.decode("utf-8"))
self.assertTrue(len(tiles['bounds']) == 4)
self.assertTrue(round(tiles['bounds'][0], 7) == -91.9945132)
# Can access individual tiles
for tile_type in tile_types:
res = client.get(
"/api/projects/{}/tasks/{}/{}/tiles/16/16020/42443.png".format(
project.id, task.id, tile_type))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Another user does not have access to the resources
other_client = APIClient()
other_client.login(username="******", password="******")
def accessResources(expectedStatus):
for tile_type in tile_types:
res = other_client.get(
"/api/projects/{}/tasks/{}/{}/tiles.json".format(
project.id, task.id, tile_type))
self.assertTrue(res.status_code == expectedStatus)
res = other_client.get(
"/api/projects/{}/tasks/{}/{}/tiles/16/16020/42443.png".format(
project.id, task.id, tile_type))
self.assertTrue(res.status_code == expectedStatus)
accessResources(status.HTTP_404_NOT_FOUND)
# Original owner enables sharing
res = client.patch(
"/api/projects/{}/tasks/{}/".format(project.id, task.id),
{'public': True})
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Now other user can acccess resources
accessResources(status.HTTP_200_OK)
# User logs out
other_client.logout()
# He can still access the resources as anonymous
accessResources(status.HTTP_200_OK)
# Other user still does not have access to certain parts of the API
res = other_client.get("/api/projects/{}/tasks/{}/".format(
project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_403_FORBIDDEN)
# Restart a task
testWatch.clear()
res = client.post("/api/projects/{}/tasks/{}/restart/".format(
project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
testWatch.wait_until_call("app.scheduler.process_pending_tasks",
timeout=5)
task.refresh_from_db()
self.assertTrue(
task.status in [status_codes.RUNNING, status_codes.COMPLETED])
# Cancel a task
testWatch.clear()
res = client.post("/api/projects/{}/tasks/{}/cancel/".format(
project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
testWatch.wait_until_call("app.scheduler.process_pending_tasks",
timeout=5)
# Should have been canceled
task.refresh_from_db()
self.assertTrue(task.status == status_codes.CANCELED)
# Remove a task
res = client.post("/api/projects/{}/tasks/{}/remove/".format(
project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
testWatch.wait_until_call("app.scheduler.process_pending_tasks",
2,
timeout=5)
# Has been removed along with assets
self.assertFalse(Task.objects.filter(pk=task.id).exists())
self.assertFalse(ImageUpload.objects.filter(task=task).exists())
task_assets_path = os.path.join(
settings.MEDIA_ROOT, task_directory_path(task.id, task.project.id))
self.assertFalse(os.path.exists(task_assets_path))
testWatch.clear()
testWatch.intercept("app.scheduler.process_pending_tasks")
# Create a task, then kill the processing node
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task_offline',
'processing_node': pnode.id,
'auto_processing_node': 'false'
},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
task = Task.objects.get(pk=res.data['id'])
# Stop processing node
node_odm.terminate()
task.refresh_from_db()
self.assertTrue(task.last_error is None)
scheduler.process_pending_tasks()
# Processing should fail and set an error
task.refresh_from_db()
self.assertTrue(task.last_error is not None)
self.assertTrue(task.status == status_codes.FAILED)
# Now bring it back online
node_odm = start_processing_node()
# Restart
res = client.post("/api/projects/{}/tasks/{}/restart/".format(
project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
task.refresh_from_db()
self.assertTrue(task.pending_action == pending_actions.RESTART)
# After processing, the task should have restarted, and have no UUID or status
scheduler.process_pending_tasks()
task.refresh_from_db()
self.assertTrue(task.status is None)
self.assertTrue(len(task.uuid) == 0)
# Another step and it should have acquired a UUID
scheduler.process_pending_tasks()
task.refresh_from_db()
self.assertTrue(
task.status in [status_codes.RUNNING, status_codes.COMPLETED])
self.assertTrue(len(task.uuid) > 0)
# Another step and it should be completed
time.sleep(DELAY)
scheduler.process_pending_tasks()
task.refresh_from_db()
self.assertTrue(task.status == status_codes.COMPLETED)
# Test connection, timeout errors
res = client.post("/api/projects/{}/tasks/{}/restart/".format(
project.id, task.id))
def connTimeout(*args, **kwargs):
raise requests.exceptions.ConnectTimeout("Simulated timeout")
testWatch.intercept("nodeodm.api_client.task_output", connTimeout)
scheduler.process_pending_tasks()
# Timeout errors should be handled by retrying again at a later time
# and not fail
task.refresh_from_db()
self.assertTrue(task.last_error is None)
# Reassigning the task to another project should move its assets
self.assertTrue(
os.path.exists(full_task_directory_path(task.id, project.id)))
self.assertTrue(len(task.imageupload_set.all()) == 2)
for image in task.imageupload_set.all():
self.assertTrue(
'project/{}/'.format(project.id) in image.image.path)
task.project = other_project
task.save()
task.refresh_from_db()
self.assertFalse(
os.path.exists(full_task_directory_path(task.id, project.id)))
self.assertTrue(
os.path.exists(full_task_directory_path(task.id,
other_project.id)))
for image in task.imageupload_set.all():
self.assertTrue(
'project/{}/'.format(other_project.id) in image.image.path)
node_odm.terminate()
# Restart node-odm as to not generate orthophotos
testWatch.clear()
node_odm = start_processing_node("--test_skip_orthophotos")
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task_no_orthophoto',
'processing_node': pnode.id,
'auto_processing_node': 'false'
},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
scheduler.process_pending_tasks()
time.sleep(DELAY)
scheduler.process_pending_tasks()
task = Task.objects.get(pk=res.data['id'])
self.assertTrue(task.status == status_codes.COMPLETED)
# Orthophoto files/directories should be missing
self.assertFalse(
os.path.exists(
task.assets_path("odm_orthophoto", "odm_orthophoto.tif")))
self.assertFalse(os.path.exists(task.assets_path("orthophoto_tiles")))
# orthophoto_extent should be none
self.assertTrue(task.orthophoto_extent is None)
# but other extents should be populated
self.assertTrue(task.dsm_extent is not None)
self.assertTrue(task.dtm_extent is not None)
self.assertTrue(os.path.exists(task.assets_path("dsm_tiles")))
self.assertTrue(os.path.exists(task.assets_path("dtm_tiles")))
# Can access only tiles of available assets
res = client.get("/api/projects/{}/tasks/{}/dsm/tiles.json".format(
project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
res = client.get("/api/projects/{}/tasks/{}/dtm/tiles.json".format(
project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
res = client.get(
"/api/projects/{}/tasks/{}/orthophoto/tiles.json".format(
project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
# Available assets should be missing orthophoto.tif type
# but others such as textured_model.zip should be available
res = client.get("/api/projects/{}/tasks/{}/".format(
project.id, task.id))
self.assertFalse('orthophoto.tif' in res.data['available_assets'])
self.assertTrue('textured_model.zip' in res.data['available_assets'])
image1.close()
image2.close()
node_odm.terminate()
def test_task(self):
client = APIClient()
with start_processing_node():
user = User.objects.get(username="******")
self.assertFalse(user.is_superuser)
other_user = User.objects.get(username="******")
project = Project.objects.create(owner=user, name="test project")
other_project = Project.objects.create(owner=other_user,
name="another test project")
other_task = Task.objects.create(project=other_project)
# Start processing node
# Create processing node
pnode = ProcessingNode.objects.create(hostname="localhost",
port=11223)
# Verify that it's working
self.assertTrue(pnode.api_version is not None)
# task creation via file upload
image1 = open("app/fixtures/tiny_drone_image.jpg", 'rb')
image2 = open("app/fixtures/tiny_drone_image_2.jpg", 'rb')
multispec_image = open(
"app/fixtures/tiny_drone_image_multispec.tif", 'rb')
img1 = Image.open("app/fixtures/tiny_drone_image.jpg")
# Not authenticated?
res = client.post("/api/projects/{}/tasks/".format(project.id),
{'images': [image1, image2]},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_403_FORBIDDEN)
image1.seek(0)
image2.seek(0)
client.login(username="******", password="******")
# Cannot create a task for a project that does not exist
res = client.post("/api/projects/0/tasks/",
{'images': [image1, image2]},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
image1.seek(0)
image2.seek(0)
# Cannot create a task for a project for which we have no access to
res = client.post("/api/projects/{}/tasks/".format(
other_project.id), {'images': [image1, image2]},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
image1.seek(0)
image2.seek(0)
# Cannot create a task without images
res = client.post("/api/projects/{}/tasks/".format(project.id),
{'images': []},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
# Cannot create a task with just 1 image
res = client.post("/api/projects/{}/tasks/".format(project.id),
{'images': image1},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
image1.seek(0)
# Normal case with images[], name and processing node parameter
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task',
'processing_node': pnode.id
},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
multiple_param_task = Task.objects.latest('created_at')
self.assertTrue(multiple_param_task.name == 'test_task')
self.assertTrue(multiple_param_task.processing_node.id == pnode.id)
self.assertEqual(multiple_param_task.import_url, "")
image1.seek(0)
image2.seek(0)
# Uploaded images should be the same size as originals
with Image.open(
multiple_param_task.task_path(
"tiny_drone_image.jpg")) as im:
self.assertTrue(im.size == img1.size)
# Normal case with images[], GCP, name and processing node parameter and resize_to option
testWatch.clear()
gcp = open("app/fixtures/gcp.txt", 'r')
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2, multispec_image, gcp],
'name': 'test_task',
'processing_node': pnode.id,
'resize_to': img1.size[0] / 2.0
},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
resized_task = Task.objects.latest('created_at')
image1.seek(0)
image2.seek(0)
gcp.seek(0)
multispec_image.seek(0)
# Uploaded images should have been resized
with Image.open(
resized_task.task_path("tiny_drone_image.jpg")) as im:
self.assertTrue(im.size[0] == img1.size[0] / 2.0)
# Except the multispectral image
with Image.open(
resized_task.task_path(
"tiny_drone_image_multispec.tif")) as im:
self.assertTrue(im.size[0] == img1.size[0])
# GCP should have been scaled
with open(resized_task.task_path("gcp.txt")) as f:
lines = list(map(lambda l: l.strip(), f.readlines()))
[x, y, z, px, py, imagename, *extras] = lines[1].split(' ')
self.assertTrue(
imagename == "tiny_drone_image.JPG") # case insensitive
self.assertTrue(float(px) == 2.0) # scaled by half
self.assertTrue(float(py) == 3.0) # scaled by half
self.assertTrue(float(x) == 576529.22) # Didn't change
[x, y, z, px, py, imagename, *extras] = lines[5].split(' ')
self.assertTrue(imagename == "missing_image.jpg")
self.assertTrue(float(px) == 8.0) # Didn't change
self.assertTrue(float(py) == 8.0) # Didn't change
# Resize progress is 100%
resized_task.refresh_from_db()
self.assertEqual(resized_task.resize_progress, 1.0)
# Upload progress is 100%
self.assertEqual(resized_task.upload_progress, 1.0)
# Upload progress callback has been called
self.assertTrue(
testWatch.get_calls_count("Task.process.callback") > 0)
# This is not a partial task
self.assertFalse(resized_task.partial)
# Case with malformed GCP file option
with open("app/fixtures/gcp_malformed.txt", 'r') as malformed_gcp:
res = client.post("/api/projects/{}/tasks/".format(
project.id), {
'images': [image1, image2, malformed_gcp],
'name': 'test_task',
'processing_node': pnode.id,
'resize_to': img1.size[0] / 2.0
},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
malformed_gcp_task = Task.objects.latest('created_at')
# We just pass it along, it will get errored out during processing
# But we shouldn't fail.
with open(malformed_gcp_task.task_path(
"gcp_malformed.txt")) as f:
lines = list(map(lambda l: l.strip(), f.readlines()))
self.assertTrue(lines[1] == "<O_O>")
image1.seek(0)
image2.seek(0)
# Cannot create a task with images[], name, but invalid processing node parameter
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task',
'processing_node': 9999
},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
image1.seek(0)
image2.seek(0)
# Normal case with images[] parameter
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'auto_processing_node': 'false'
},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
image1.seek(0)
image2.seek(0)
# Should have returned the id of the newly created task
task = Task.objects.latest('created_at')
self.assertTrue('id' in res.data)
self.assertTrue(str(task.id) == res.data['id'])
# Progress is at 0%
self.assertEqual(task.running_progress, 0.0)
# Two images should have been uploaded
self.assertTrue(ImageUpload.objects.filter(task=task).count() == 2)
# Can_rerun_from should be an empty list
self.assertTrue(len(res.data['can_rerun_from']) == 0)
# processing_node_name should be null
self.assertTrue(res.data['processing_node_name'] is None)
# No processing node is set
self.assertTrue(task.processing_node is None)
# tiles.json, bounds, metadata should not be accessible at this point
tile_types = ['orthophoto', 'dsm', 'dtm']
endpoints = ['tiles.json', 'bounds', 'metadata']
for ep in endpoints:
for tile_type in tile_types:
res = client.get("/api/projects/{}/tasks/{}/{}/{}".format(
project.id, task.id, tile_type, ep))
self.assertEqual(res.status_code,
status.HTTP_404_NOT_FOUND)
# Neither should an individual tile
# Z/X/Y coords are chosen based on node-odm test dataset for orthophoto_tiles/
res = client.get(
"/api/projects/{}/tasks/{}/orthophoto/tiles/16/16020/42443.png"
.format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot access a tiles.json we have no access to
res = client.get(
"/api/projects/{}/tasks/{}/orthophoto/tiles.json".format(
other_project.id, other_task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot access an individual tile we have no access to
res = client.get(
"/api/projects/{}/tasks/{}/orthophoto/tiles/16/16020/42443.png"
.format(other_project.id, other_task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot download assets (they don't exist yet)
for asset in list(task.ASSETS_MAP.keys()):
res = client.get(
"/api/projects/{}/tasks/{}/download/{}".format(
project.id, task.id, asset))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot access raw assets (they don't exist yet)
res = client.get(
"/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif"
.format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot assign processing node to a task we have no access to
res = client.patch(
"/api/projects/{}/tasks/{}/".format(other_project.id,
other_task.id),
{'processing_node': pnode.id})
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# No UUID at this point
self.assertTrue(len(task.uuid) == 0)
# Assign processing node to task via API
res = client.patch(
"/api/projects/{}/tasks/{}/".format(project.id, task.id),
{'processing_node': pnode.id})
self.assertTrue(res.status_code == status.HTTP_200_OK)
# On update worker.tasks.process_pending_tasks should have been called in the background
# (during tests this is sync)
# Processing should have started and a UUID is assigned
# Calling process pending tasks should finish the process
# and invoke the plugins completed signal
task.refresh_from_db()
self.assertTrue(task.status in [
status_codes.RUNNING, status_codes.COMPLETED
]) # Sometimes this finishes before we get here
self.assertTrue(len(task.uuid) > 0)
with catch_signal(task_completed) as handler:
retry_count = 0
while task.status != status_codes.COMPLETED:
worker.tasks.process_pending_tasks()
time.sleep(DELAY)
task.refresh_from_db()
retry_count += 1
if retry_count > 10:
break
self.assertEqual(task.status, status_codes.COMPLETED)
# Progress is 100%
self.assertTrue(task.running_progress == 1.0)
handler.assert_any_call(
sender=Task,
task_id=task.id,
signal=task_completed,
)
# Processing node should have a "rerun_from" option
pnode_rerun_from_opts = list(
filter(lambda d: 'name' in d and d['name'] == 'rerun-from',
pnode.available_options))[0]
self.assertTrue(len(pnode_rerun_from_opts['domain']) > 0)
# The can_rerun_from field of a task should now be populated
# with the same values as the "rerun_from" domain values of
# the processing node
res = client.get("/api/projects/{}/tasks/{}/".format(
project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
self.assertTrue(
pnode_rerun_from_opts['domain'] == res.data['can_rerun_from'])
# processing_node_name should be the name of the pnode
self.assertEqual(res.data['processing_node_name'], str(pnode))
# Can download assets
for asset in list(task.ASSETS_MAP.keys()):
res = client.get(
"/api/projects/{}/tasks/{}/download/{}".format(
project.id, task.id, asset))
self.assertEqual(res.status_code, status.HTTP_200_OK)
# We can stream downloads
res = client.get(
"/api/projects/{}/tasks/{}/download/{}?_force_stream=1".format(
project.id, task.id,
list(task.ASSETS_MAP.keys())[0]))
self.assertTrue(res.status_code == status.HTTP_200_OK)
self.assertTrue(res.has_header('_stream'))
# The tif files are valid Cloud Optimized GeoTIFF
self.assertTrue(
valid_cogeo(task.assets_path(
task.ASSETS_MAP["orthophoto.tif"])))
self.assertTrue(
valid_cogeo(task.assets_path(task.ASSETS_MAP["dsm.tif"])))
self.assertTrue(
valid_cogeo(task.assets_path(task.ASSETS_MAP["dtm.tif"])))
# A textured mesh archive file should exist
self.assertTrue(
os.path.exists(
task.assets_path(task.ASSETS_MAP["textured_model.zip"]
["deferred_path"])))
# Tiles archives should have been created
self.assertTrue(
os.path.exists(
task.assets_path(
task.ASSETS_MAP["dsm_tiles.zip"]["deferred_path"])))
self.assertTrue(
os.path.exists(
task.assets_path(
task.ASSETS_MAP["dtm_tiles.zip"]["deferred_path"])))
self.assertTrue(
os.path.exists(
task.assets_path(task.ASSETS_MAP["orthophoto_tiles.zip"]
["deferred_path"])))
# Can download raw assets
res = client.get(
"/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif"
.format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Can access tiles.json, bounds and metadata
for ep in endpoints:
for tile_type in tile_types:
res = client.get("/api/projects/{}/tasks/{}/{}/{}".format(
project.id, task.id, tile_type, ep))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Bounds are what we expect them to be
# (4 coords in lat/lon)
res = client.get(
"/api/projects/{}/tasks/{}/orthophoto/tiles.json".format(
project.id, task.id))
tiles = json.loads(res.content.decode("utf-8"))
self.assertTrue(len(tiles['bounds']) == 4)
self.assertTrue(round(tiles['bounds'][0], 7) == -91.9945132)
res = client.get(
"/api/projects/{}/tasks/{}/orthophoto/bounds".format(
project.id, task.id))
bounds = json.loads(res.content.decode("utf-8"))
self.assertTrue(len(bounds['bounds']) == 4)
self.assertTrue(round(bounds['bounds'][0], 7) == -91.9945132)
# Metadata checks for orthophoto
res = client.get(
"/api/projects/{}/tasks/{}/orthophoto/metadata".format(
project.id, task.id))
self.assertEqual(res.status_code, status.HTTP_200_OK)
metadata = json.loads(res.content.decode("utf-8"))
fields = [
'bounds', 'minzoom', 'maxzoom', 'statistics', 'algorithms',
'color_maps', 'tiles', 'scheme', 'name'
]
for f in fields:
self.assertTrue(f in metadata)
self.assertEqual(metadata['minzoom'], 17 - ZOOM_EXTRA_LEVELS)
self.assertEqual(metadata['maxzoom'], 17 + ZOOM_EXTRA_LEVELS)
# Colormaps and algorithms should be empty lists
self.assertEqual(metadata['algorithms'], [])
self.assertEqual(metadata['color_maps'], [])
# Address key is removed
self.assertFalse('address' in metadata)
# Scheme is xyz
self.assertEqual(metadata['scheme'], 'xyz')
# Tiles URL has no extra params
self.assertTrue(metadata['tiles'][0].endswith('.png'))
# Histogram stats are available (3 bands for orthophoto)
self.assertTrue(len(metadata['statistics']) == 3)
for b in ['1', '2', '3']:
self.assertEqual(len(metadata['statistics'][b]['histogram']),
2)
self.assertEqual(
len(metadata['statistics'][b]['histogram'][0]), 255)
self.assertTrue('max' in metadata['statistics'][b])
self.assertTrue('min' in metadata['statistics'][b])
# Metadata with invalid formula
res = client.get(
"/api/projects/{}/tasks/{}/orthophoto/metadata?formula=INVALID"
.format(project.id, task.id))
self.assertEqual(res.status_code, status.HTTP_400_BAD_REQUEST)
# Metadata with a valid formula but invalid bands
res = client.get(
"/api/projects/{}/tasks/{}/orthophoto/metadata?formula=NDVI&bands=ABC"
.format(project.id, task.id))
self.assertEqual(res.status_code, status.HTTP_400_BAD_REQUEST)
# Medatata with valid formula and bands
res = client.get(
"/api/projects/{}/tasks/{}/orthophoto/metadata?formula=NDVI&bands=RGN"
.format(project.id, task.id))
self.assertEqual(res.status_code, status.HTTP_200_OK)
metadata = json.loads(res.content.decode("utf-8"))
# Colormaps and algorithms are populated
self.assertTrue(len(metadata['algorithms']) > 0)
self.assertTrue(len(metadata['color_maps']) > 0)
# Algorithms have valid keys
for k in ['id', 'filters', 'expr', 'help']:
for a in metadata['algorithms']:
self.assertTrue(k in a)
self.assertTrue(len(a['filters']) > 0)
# Colormap is for algorithms
self.assertEqual(
len([
x for x in metadata['color_maps'] if x['key'] == 'rdylgn'
]), 1)
self.assertEqual(
len([x for x in metadata['color_maps'] if x['key'] == 'jet']),
0)
# Formula parameters are copied to tile URL
self.assertTrue(
metadata['tiles'][0].endswith('?formula=NDVI&bands=RGN'))
# Histogram stats are available (1 band)
self.assertTrue(len(metadata['statistics']) == 1)
# Medatata with valid formula and bands that specifies a scale range
res = client.get(
"/api/projects/{}/tasks/{}/orthophoto/metadata?formula=VARI".
format(project.id, task.id))
self.assertEqual(res.status_code, status.HTTP_200_OK)
metadata = json.loads(res.content.decode("utf-8"))
self.assertTrue(len(metadata['statistics']) == 1)
# Min/max values have been replaced
self.assertEqual(metadata['statistics']['1']['min'],
algos['VARI']['range'][0])
self.assertEqual(metadata['statistics']['1']['max'],
algos['VARI']['range'][1])
tile_path = {
'orthophoto': '17/32042/46185',
'dsm': '18/64083/92370',
'dtm': '18/64083/92370'
}
# Metadata for DSM/DTM
for tile_type in ['dsm', 'dtm']:
res = client.get(
"/api/projects/{}/tasks/{}/{}/metadata".format(
project.id, task.id, tile_type))
self.assertEqual(res.status_code, status.HTTP_200_OK)
metadata = json.loads(res.content.decode("utf-8"))
# Colormaps are populated
self.assertTrue(len(metadata['color_maps']) > 0)
# Colormaps are for elevation
self.assertEqual(
len([
x for x in metadata['color_maps']
if x['key'] == 'rdylgn'
]), 0)
self.assertEqual(
len([
x for x in metadata['color_maps'] if x['key'] == 'jet'
]), 1)
# Algorithms are empty
self.assertEqual(len(metadata['algorithms']), 0)
# Min/max values are what we expect them to be
self.assertEqual(len(metadata['statistics']), 1)
self.assertEqual(round(metadata['statistics']['1']['min'], 2),
156.92)
self.assertEqual(round(metadata['statistics']['1']['max'], 2),
164.88)
# Can access individual tiles
for tile_type in tile_types:
res = client.get(
"/api/projects/{}/tasks/{}/{}/tiles/{}.png".format(
project.id, task.id, tile_type, tile_path[tile_type]))
self.assertEqual(res.status_code, status.HTTP_200_OK)
with Image.open(io.BytesIO(res.content)) as i:
self.assertEqual(i.width, 256)
self.assertEqual(i.height, 256)
# Can access retina tiles
for tile_type in tile_types:
res = client.get(
"/api/projects/{}/tasks/{}/{}/tiles/{}@2x.png".format(
project.id, task.id, tile_type, tile_path[tile_type]))
self.assertEqual(res.status_code, status.HTTP_200_OK)
with Image.open(io.BytesIO(res.content)) as i:
self.assertEqual(i.width, 512)
self.assertEqual(i.height, 512)
# Cannot access tile 0/0/0
res = client.get(
"/api/projects/{}/tasks/{}/orthophoto/tiles/0/0/0.png".format(
project.id, task.id))
self.assertEqual(res.status_code, status.HTTP_404_NOT_FOUND)
# Cannot access zoom levels outside of the allowed zoom levels
res = client.get(
"/api/projects/{}/tasks/{}/orthophoto/tiles/14/32042/46185.png"
.format(project.id, task.id))
self.assertEqual(res.status_code, status.HTTP_404_NOT_FOUND)
res = client.get(
"/api/projects/{}/tasks/{}/orthophoto/tiles/20/32042/46185.png"
.format(project.id, task.id))
self.assertEqual(res.status_code, status.HTTP_404_NOT_FOUND)
# Can access hillshade, formulas, bands, rescale, color_map
params = [
("dsm", "color_map=jet&hillshade=3&rescale=150,170",
status.HTTP_200_OK),
("dsm", "color_map=jet&hillshade=0&rescale=150,170",
status.HTTP_200_OK),
("dsm", "color_map=invalid&rescale=150,170",
status.HTTP_400_BAD_REQUEST),
("dsm", "color_map=jet&rescale=invalid",
status.HTTP_400_BAD_REQUEST),
("dsm", "color_map=jet&rescale=150,170&hillshade=invalid",
status.HTTP_400_BAD_REQUEST),
("dtm", "hillshade=3", status.HTTP_200_OK),
("dtm", "hillshade=99999999999999999999999999999999999",
status.HTTP_200_OK),
("dtm", "hillshade=-9999999999999999999999999999999999",
status.HTTP_200_OK),
("dtm", "hillshade=0", status.HTTP_200_OK),
("orthophoto", "hillshade=3", status.HTTP_400_BAD_REQUEST),
("orthophoto", "formula=NDVI&bands=RGN", status.HTTP_200_OK),
("orthophoto", "formula=VARI&bands=RGN",
status.HTTP_400_BAD_REQUEST),
("orthophoto", "formula=VARI&bands=RGB", status.HTTP_200_OK),
("orthophoto", "formula=VARI&bands=invalid",
status.HTTP_400_BAD_REQUEST),
("orthophoto", "formula=invalid&bands=RGB",
status.HTTP_400_BAD_REQUEST),
("orthophoto",
"formula=NDVI&bands=RGN&color_map=rdylgn&rescale=-1,1",
status.HTTP_200_OK),
("orthophoto",
"formula=NDVI&bands=RGN&color_map=rdylgn&rescale=1,-1",
status.HTTP_200_OK),
("orthophoto", "formula=NDVI&bands=RGN&color_map=invalid",
status.HTTP_400_BAD_REQUEST),
]
for k in algos:
a = algos[k]
filters = get_camera_filters_for(a)
self.assertTrue(len(filters) > 0, "%s has filters" % k)
for f in filters:
params.append(
("orthophoto",
"formula={}&bands={}&color_map=rdylgn".format(k, f),
status.HTTP_200_OK))
for tile_type, url, sc in params:
res = client.get(
"/api/projects/{}/tasks/{}/{}/tiles/{}.png?{}".format(
project.id, task.id, tile_type, tile_path[tile_type],
url))
self.assertEqual(res.status_code, sc)
# Another user does not have access to the resources
other_client = APIClient()
other_client.login(username="******", password="******")
def accessResources(expectedStatus):
for tile_type in tile_types:
res = other_client.get(
"/api/projects/{}/tasks/{}/{}/tiles.json".format(
project.id, task.id, tile_type))
self.assertEqual(res.status_code, expectedStatus)
res = other_client.get(
"/api/projects/{}/tasks/{}/{}/tiles/{}.png".format(
project.id, task.id, tile_type, tile_path[tile_type]))
self.assertEqual(res.status_code, expectedStatus)
res = other_client.get("/api/projects/{}/tasks/{}/".format(
project.id, task.id))
self.assertEqual(res.status_code, expectedStatus)
accessResources(status.HTTP_404_NOT_FOUND)
# Original owner enables sharing
res = client.patch(
"/api/projects/{}/tasks/{}/".format(project.id, task.id),
{'public': True})
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Now other user can acccess resources
accessResources(status.HTTP_200_OK)
# He cannot change a task
res = other_client.patch(
"/api/projects/{}/tasks/{}/".format(project.id, task.id),
{'name': "Changed! Uh oh"})
self.assertEqual(res.status_code, status.HTTP_404_NOT_FOUND)
# User logs out
other_client.logout()
# He can still access the resources as anonymous
accessResources(status.HTTP_200_OK)
# Restart a task
testWatch.clear()
res = client.post("/api/projects/{}/tasks/{}/restart/".format(
project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# process_task is called in the background
task.refresh_from_db()
self.assertTrue(
task.status in [status_codes.RUNNING, status_codes.COMPLETED])
# Should return without issues
task.check_if_canceled()
# Cancel a task
res = client.post("/api/projects/{}/tasks/{}/cancel/".format(
project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# task is processed right away
# Should have been canceled
task.refresh_from_db()
self.assertTrue(task.status == status_codes.CANCELED)
self.assertTrue(task.pending_action is None)
# Manually set pending action
task.pending_action = pending_actions.CANCEL
task.save()
# Should raise TaskInterruptedException
self.assertRaises(TaskInterruptedException, task.check_if_canceled)
# Restore
task.pending_action = None
task.save()
# Remove a task and verify that it calls the proper plugins signals
with catch_signal(task_removing) as h1:
with catch_signal(task_removed) as h2:
res = client.post(
"/api/projects/{}/tasks/{}/remove/".format(
project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
h1.assert_called_once_with(sender=Task,
task_id=task.id,
signal=task_removing)
h2.assert_called_once_with(sender=Task,
task_id=task.id,
signal=task_removed)
# task is processed right away
# Has been removed along with assets
self.assertFalse(Task.objects.filter(pk=task.id).exists())
self.assertFalse(ImageUpload.objects.filter(task=task).exists())
task_assets_path = os.path.join(
settings.MEDIA_ROOT,
task_directory_path(task.id, task.project.id))
self.assertFalse(os.path.exists(task_assets_path))
# Create a task
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task_offline',
'processing_node': pnode.id,
'auto_processing_node': 'false'
},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
task = Task.objects.get(pk=res.data['id'])
image1.seek(0)
image2.seek(0)
# Processing should fail and set an error
task.refresh_from_db()
self.assertTrue(task.last_error is not None)
self.assertTrue(task.status == status_codes.FAILED)
# Now bring it back online
with start_processing_node():
# Restart
res = client.post("/api/projects/{}/tasks/{}/restart/".format(
project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
task.refresh_from_db()
# After processing, the task should have restarted, and have no UUID or status
self.assertTrue(task.status is None)
self.assertTrue(len(task.uuid) == 0)
# Another step and it should have acquired a UUID
worker.tasks.process_pending_tasks()
task.refresh_from_db()
self.assertTrue(
task.status in [status_codes.RUNNING, status_codes.COMPLETED])
self.assertTrue(len(task.uuid) > 0)
# Another step and it should be completed
time.sleep(DELAY)
worker.tasks.process_pending_tasks()
task.refresh_from_db()
self.assertTrue(task.status == status_codes.COMPLETED)
# Test rerun-from clearing mechanism:
# 1 .Set some task options, including rerun_from
task.options = [{
'name': 'mesh-size',
'value': 1000
}, {
'name': 'rerun-from',
'value': 'odm_meshing'
}]
task.save()
# 2. Remove the task directly from node-odm (simulate a task purge)
self.assertTrue(task.processing_node.remove_task(task.uuid))
# 3. Restart the task
res = client.post("/api/projects/{}/tasks/{}/restart/".format(
project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# 4. Check that the rerun_from parameter has been cleared
# but the other parameters are still set
task.refresh_from_db()
self.assertTrue(len(task.uuid) == 0)
self.assertTrue(
len(
list(
filter(lambda d: d['name'] == 'rerun-from',
task.options))) == 0)
self.assertTrue(
len(
list(
filter(lambda d: d['name'] == 'mesh-size',
task.options))) == 1)
# Test connection, timeout errors
def connTimeout(*args, **kwargs):
raise requests.exceptions.ConnectTimeout("Simulated timeout")
testWatch.intercept("nodeodm.api_client.task_output", connTimeout)
worker.tasks.process_pending_tasks()
# Timeout errors should be handled by retrying again at a later time
# and not fail
task.refresh_from_db()
self.assertTrue(task.last_error is None)
# Reassigning the task to another project should move its assets
self.assertTrue(
os.path.exists(full_task_directory_path(task.id, project.id)))
self.assertTrue(len(task.imageupload_set.all()) == 2)
for image in task.imageupload_set.all():
self.assertTrue(
'project/{}/'.format(project.id) in image.image.path)
task.project = other_project
task.save()
task.refresh_from_db()
self.assertFalse(
os.path.exists(full_task_directory_path(task.id, project.id)))
self.assertTrue(
os.path.exists(
full_task_directory_path(task.id, other_project.id)))
for image in task.imageupload_set.all():
self.assertTrue(
'project/{}/'.format(other_project.id) in image.image.path)
# Restart node-odm as to not generate orthophotos
testWatch.clear()
with start_processing_node("--test_skip_orthophotos"):
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task_no_orthophoto',
'processing_node': pnode.id,
'auto_processing_node': 'false'
},
format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
worker.tasks.process_pending_tasks()
time.sleep(DELAY)
worker.tasks.process_pending_tasks()
task = Task.objects.get(pk=res.data['id'])
self.assertTrue(task.status == status_codes.COMPLETED)
# Orthophoto files/directories should be missing
self.assertFalse(
os.path.exists(
task.assets_path("odm_orthophoto", "odm_orthophoto.tif")))
self.assertFalse(
os.path.exists(task.assets_path("orthophoto_tiles")))
# orthophoto_extent should be none
self.assertTrue(task.orthophoto_extent is None)
# but other extents should be populated
self.assertTrue(task.dsm_extent is not None)
self.assertTrue(task.dtm_extent is not None)
self.assertTrue(os.path.exists(task.assets_path("dsm_tiles")))
self.assertTrue(os.path.exists(task.assets_path("dtm_tiles")))
# Can access only tiles of available assets
res = client.get("/api/projects/{}/tasks/{}/dsm/tiles.json".format(
project.id, task.id))
self.assertEqual(res.status_code, status.HTTP_200_OK)
res = client.get("/api/projects/{}/tasks/{}/dtm/tiles.json".format(
project.id, task.id))
self.assertEqual(res.status_code, status.HTTP_200_OK)
res = client.get(
"/api/projects/{}/tasks/{}/orthophoto/tiles.json".format(
project.id, task.id))
self.assertEqual(res.status_code, status.HTTP_404_NOT_FOUND)
# Available assets should be missing orthophoto.tif type
# but others such as textured_model.zip should be available
res = client.get("/api/projects/{}/tasks/{}/".format(
project.id, task.id))
self.assertFalse('orthophoto.tif' in res.data['available_assets'])
self.assertFalse(
'orthophoto_tiles.zip' in res.data['available_assets'])
self.assertTrue(
'textured_model.zip' in res.data['available_assets'])
image1.close()
image2.close()
multispec_image.close()
gcp.close()
def test_task(self):
client = APIClient()
node_odm = start_processing_node()
user = User.objects.get(username="******")
self.assertFalse(user.is_superuser)
other_user = User.objects.get(username="******")
project = Project.objects.create(
owner=user,
name="test project"
)
other_project = Project.objects.create(
owner=other_user,
name="another test project"
)
other_task = Task.objects.create(project=other_project)
# Start processing node
# Create processing node
pnode = ProcessingNode.objects.create(hostname="localhost", port=11223)
# Verify that it's working
self.assertTrue(pnode.api_version is not None)
# task creation via file upload
image1 = open("app/fixtures/tiny_drone_image.jpg", 'rb')
image2 = open("app/fixtures/tiny_drone_image_2.jpg", 'rb')
img1 = Image.open("app/fixtures/tiny_drone_image.jpg")
# Not authenticated?
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2]
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_403_FORBIDDEN);
image1.seek(0)
image2.seek(0)
client.login(username="******", password="******")
# Cannot create a task for a project that does not exist
res = client.post("/api/projects/0/tasks/", {
'images': [image1, image2]
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
image1.seek(0)
image2.seek(0)
# Cannot create a task for a project for which we have no access to
res = client.post("/api/projects/{}/tasks/".format(other_project.id), {
'images': [image1, image2]
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
image1.seek(0)
image2.seek(0)
# Cannot create a task without images
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': []
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
# Cannot create a task with just 1 image
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': image1
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
image1.seek(0)
# Normal case with images[], name and processing node parameter
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task',
'processing_node': pnode.id
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
multiple_param_task = Task.objects.latest('created_at')
self.assertTrue(multiple_param_task.name == 'test_task')
self.assertTrue(multiple_param_task.processing_node.id == pnode.id)
self.assertEqual(multiple_param_task.import_url, "")
image1.seek(0)
image2.seek(0)
# Uploaded images should be the same size as originals
with Image.open(multiple_param_task.task_path("tiny_drone_image.jpg")) as im:
self.assertTrue(im.size == img1.size)
# Normal case with images[], GCP, name and processing node parameter and resize_to option
testWatch.clear()
gcp = open("app/fixtures/gcp.txt", 'r')
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2, gcp],
'name': 'test_task',
'processing_node': pnode.id,
'resize_to': img1.size[0] / 2.0
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
resized_task = Task.objects.latest('created_at')
image1.seek(0)
image2.seek(0)
gcp.seek(0)
# Uploaded images should have been resized
with Image.open(resized_task.task_path("tiny_drone_image.jpg")) as im:
self.assertTrue(im.size[0] == img1.size[0] / 2.0)
# GCP should have been scaled
with open(resized_task.task_path("gcp.txt")) as f:
lines = list(map(lambda l: l.strip(), f.readlines()))
[x, y, z, px, py, imagename, *extras] = lines[1].split(' ')
self.assertTrue(imagename == "tiny_drone_image.JPG") # case insensitive
self.assertTrue(float(px) == 2.0) # scaled by half
self.assertTrue(float(py) == 3.0) # scaled by half
self.assertTrue(float(x) == 576529.22) # Didn't change
[x, y, z, px, py, imagename, *extras] = lines[5].split(' ')
self.assertTrue(imagename == "missing_image.jpg")
self.assertTrue(float(px) == 8.0) # Didn't change
self.assertTrue(float(py) == 8.0) # Didn't change
# Resize progress is 100%
resized_task.refresh_from_db()
self.assertEqual(resized_task.resize_progress, 1.0)
# Upload progress is 100%
self.assertEqual(resized_task.upload_progress, 1.0)
# Upload progress callback has been called
self.assertTrue(testWatch.get_calls_count("Task.process.callback") > 0)
# Case with malformed GCP file option
with open("app/fixtures/gcp_malformed.txt", 'r') as malformed_gcp:
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2, malformed_gcp],
'name': 'test_task',
'processing_node': pnode.id,
'resize_to': img1.size[0] / 2.0
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
malformed_gcp_task = Task.objects.latest('created_at')
# We just pass it along, it will get errored out during processing
# But we shouldn't fail.
with open(malformed_gcp_task.task_path("gcp_malformed.txt")) as f:
lines = list(map(lambda l: l.strip(), f.readlines()))
self.assertTrue(lines[1] == "<O_O>")
image1.seek(0)
image2.seek(0)
# Cannot create a task with images[], name, but invalid processing node parameter
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task',
'processing_node': 9999
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
image1.seek(0)
image2.seek(0)
# Normal case with images[] parameter
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'auto_processing_node': 'false'
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
image1.seek(0)
image2.seek(0)
# Should have returned the id of the newly created task
task = Task.objects.latest('created_at')
self.assertTrue('id' in res.data)
self.assertTrue(str(task.id) == res.data['id'])
# Progress is at 0%
self.assertEqual(task.running_progress, 0.0)
# Two images should have been uploaded
self.assertTrue(ImageUpload.objects.filter(task=task).count() == 2)
# Can_rerun_from should be an empty list
self.assertTrue(len(res.data['can_rerun_from']) == 0)
# processing_node_name should be null
self.assertTrue(res.data['processing_node_name'] is None)
# No processing node is set
self.assertTrue(task.processing_node is None)
# tiles.json should not be accessible at this point
tile_types = ['orthophoto', 'dsm', 'dtm']
for tile_type in tile_types:
res = client.get("/api/projects/{}/tasks/{}/{}/tiles.json".format(project.id, task.id, tile_type))
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
# Neither should an individual tile
# Z/X/Y coords are chosen based on node-odm test dataset for orthophoto_tiles/
res = client.get("/api/projects/{}/tasks/{}/orthophoto/tiles/16/16020/42443.png".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot access a tiles.json we have no access to
res = client.get("/api/projects/{}/tasks/{}/orthophoto/tiles.json".format(other_project.id, other_task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot access an individual tile we have no access to
res = client.get("/api/projects/{}/tasks/{}/orthophoto/tiles/16/16020/42443.png".format(other_project.id, other_task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot download assets (they don't exist yet)
for asset in list(task.ASSETS_MAP.keys()):
res = client.get("/api/projects/{}/tasks/{}/download/{}".format(project.id, task.id, asset))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot access raw assets (they don't exist yet)
res = client.get("/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot assign processing node to a task we have no access to
res = client.patch("/api/projects/{}/tasks/{}/".format(other_project.id, other_task.id), {
'processing_node': pnode.id
})
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# No UUID at this point
self.assertTrue(len(task.uuid) == 0)
# Assign processing node to task via API
res = client.patch("/api/projects/{}/tasks/{}/".format(project.id, task.id), {
'processing_node': pnode.id
})
self.assertTrue(res.status_code == status.HTTP_200_OK)
# On update worker.tasks.process_pending_tasks should have been called in the background
# (during tests this is sync)
# Processing should have started and a UUID is assigned
# Calling process pending tasks should finish the process
# and invoke the plugins completed signal
task.refresh_from_db()
self.assertTrue(task.status in [status_codes.RUNNING, status_codes.COMPLETED]) # Sometimes this finishes before we get here
self.assertTrue(len(task.uuid) > 0)
with catch_signal(task_completed) as handler:
retry_count = 0
while task.status != status_codes.COMPLETED:
worker.tasks.process_pending_tasks()
time.sleep(DELAY)
task.refresh_from_db()
retry_count += 1
if retry_count > 10:
break
self.assertEqual(task.status, status_codes.COMPLETED)
# Progress is 100%
self.assertTrue(task.running_progress == 1.0)
handler.assert_any_call(
sender=Task,
task_id=task.id,
signal=task_completed,
)
# Processing node should have a "rerun_from" option
pnode_rerun_from_opts = list(filter(lambda d: 'name' in d and d['name'] == 'rerun-from', pnode.available_options))[0]
self.assertTrue(len(pnode_rerun_from_opts['domain']) > 0)
# The can_rerun_from field of a task should now be populated
# with the same values as the "rerun_from" domain values of
# the processing node
res = client.get("/api/projects/{}/tasks/{}/".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
self.assertTrue(pnode_rerun_from_opts['domain'] == res.data['can_rerun_from'])
# processing_node_name should be the name of the pnode
self.assertEqual(res.data['processing_node_name'], str(pnode))
# Can download assets
for asset in list(task.ASSETS_MAP.keys()):
res = client.get("/api/projects/{}/tasks/{}/download/{}".format(project.id, task.id, asset))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# We can stream downloads
res = client.get("/api/projects/{}/tasks/{}/download/{}?_force_stream=1".format(project.id, task.id, list(task.ASSETS_MAP.keys())[0]))
self.assertTrue(res.status_code == status.HTTP_200_OK)
self.assertTrue(res.has_header('_stream'))
# A textured mesh archive file should exist
self.assertTrue(os.path.exists(task.assets_path(task.ASSETS_MAP["textured_model.zip"]["deferred_path"])))
# Tiles archives should have been created
self.assertTrue(os.path.exists(task.assets_path(task.ASSETS_MAP["dsm_tiles.zip"]["deferred_path"])))
self.assertTrue(os.path.exists(task.assets_path(task.ASSETS_MAP["dtm_tiles.zip"]["deferred_path"])))
self.assertTrue(os.path.exists(task.assets_path(task.ASSETS_MAP["orthophoto_tiles.zip"]["deferred_path"])))
# Can download raw assets
res = client.get("/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Can access tiles.json
for tile_type in tile_types:
res = client.get("/api/projects/{}/tasks/{}/{}/tiles.json".format(project.id, task.id, tile_type))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Bounds are what we expect them to be
# (4 coords in lat/lon)
tiles = json.loads(res.content.decode("utf-8"))
self.assertTrue(len(tiles['bounds']) == 4)
self.assertTrue(round(tiles['bounds'][0], 7) == -91.9945132)
# Can access individual tiles
for tile_type in tile_types:
res = client.get("/api/projects/{}/tasks/{}/{}/tiles/16/16020/42443.png".format(project.id, task.id, tile_type))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Another user does not have access to the resources
other_client = APIClient()
other_client.login(username="******", password="******")
def accessResources(expectedStatus):
for tile_type in tile_types:
res = other_client.get("/api/projects/{}/tasks/{}/{}/tiles.json".format(project.id, task.id, tile_type))
self.assertTrue(res.status_code == expectedStatus)
res = other_client.get("/api/projects/{}/tasks/{}/{}/tiles/16/16020/42443.png".format(project.id, task.id, tile_type))
self.assertTrue(res.status_code == expectedStatus)
res = other_client.get("/api/projects/{}/tasks/{}/".format(project.id, task.id))
self.assertTrue(res.status_code == expectedStatus)
accessResources(status.HTTP_404_NOT_FOUND)
# Original owner enables sharing
res = client.patch("/api/projects/{}/tasks/{}/".format(project.id, task.id), {
'public': True
})
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Now other user can acccess resources
accessResources(status.HTTP_200_OK)
# He cannot change a task
res = other_client.patch("/api/projects/{}/tasks/{}/".format(project.id, task.id), {
'name': "Changed! Uh oh"
})
self.assertEqual(res.status_code, status.HTTP_404_NOT_FOUND)
# User logs out
other_client.logout()
# He can still access the resources as anonymous
accessResources(status.HTTP_200_OK)
# Restart a task
testWatch.clear()
res = client.post("/api/projects/{}/tasks/{}/restart/".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# process_task is called in the background
task.refresh_from_db()
self.assertTrue(task.status in [status_codes.RUNNING, status_codes.COMPLETED])
# Should return without issues
task.check_if_canceled()
# Cancel a task
res = client.post("/api/projects/{}/tasks/{}/cancel/".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# task is processed right away
# Should have been canceled
task.refresh_from_db()
self.assertTrue(task.status == status_codes.CANCELED)
self.assertTrue(task.pending_action is None)
# Manually set pending action
task.pending_action = pending_actions.CANCEL
task.save()
# Should raise TaskInterruptedException
self.assertRaises(TaskInterruptedException, task.check_if_canceled)
# Restore
task.pending_action = None
task.save()
# Remove a task and verify that it calls the proper plugins signals
with catch_signal(task_removing) as h1:
with catch_signal(task_removed) as h2:
res = client.post("/api/projects/{}/tasks/{}/remove/".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
h1.assert_called_once_with(sender=Task, task_id=task.id, signal=task_removing)
h2.assert_called_once_with(sender=Task, task_id=task.id, signal=task_removed)
# task is processed right away
# Has been removed along with assets
self.assertFalse(Task.objects.filter(pk=task.id).exists())
self.assertFalse(ImageUpload.objects.filter(task=task).exists())
task_assets_path = os.path.join(settings.MEDIA_ROOT, task_directory_path(task.id, task.project.id))
self.assertFalse(os.path.exists(task_assets_path))
# Stop processing node
node_odm.terminate()
# Create a task
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task_offline',
'processing_node': pnode.id,
'auto_processing_node': 'false'
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
task = Task.objects.get(pk=res.data['id'])
image1.seek(0)
image2.seek(0)
# Processing should fail and set an error
task.refresh_from_db()
self.assertTrue(task.last_error is not None)
self.assertTrue(task.status == status_codes.FAILED)
# Now bring it back online
node_odm = start_processing_node()
# Restart
res = client.post("/api/projects/{}/tasks/{}/restart/".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
task.refresh_from_db()
# After processing, the task should have restarted, and have no UUID or status
self.assertTrue(task.status is None)
self.assertTrue(len(task.uuid) == 0)
# Another step and it should have acquired a UUID
worker.tasks.process_pending_tasks()
task.refresh_from_db()
self.assertTrue(task.status in [status_codes.RUNNING, status_codes.COMPLETED])
self.assertTrue(len(task.uuid) > 0)
# Another step and it should be completed
time.sleep(DELAY)
worker.tasks.process_pending_tasks()
task.refresh_from_db()
self.assertTrue(task.status == status_codes.COMPLETED)
# Test rerun-from clearing mechanism:
# 1 .Set some task options, including rerun_from
task.options = [{'name': 'mesh-size', 'value':1000},
{'name': 'rerun-from', 'value': 'odm_meshing'}]
task.save()
# 2. Remove the task directly from node-odm (simulate a task purge)
self.assertTrue(task.processing_node.remove_task(task.uuid))
# 3. Restart the task
res = client.post("/api/projects/{}/tasks/{}/restart/".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# 4. Check that the rerun_from parameter has been cleared
# but the other parameters are still set
task.refresh_from_db()
self.assertTrue(len(task.uuid) == 0)
self.assertTrue(len(list(filter(lambda d: d['name'] == 'rerun-from', task.options))) == 0)
self.assertTrue(len(list(filter(lambda d: d['name'] == 'mesh-size', task.options))) == 1)
# Test connection, timeout errors
def connTimeout(*args, **kwargs):
raise requests.exceptions.ConnectTimeout("Simulated timeout")
testWatch.intercept("nodeodm.api_client.task_output", connTimeout)
worker.tasks.process_pending_tasks()
# Timeout errors should be handled by retrying again at a later time
# and not fail
task.refresh_from_db()
self.assertTrue(task.last_error is None)
# Reassigning the task to another project should move its assets
self.assertTrue(os.path.exists(full_task_directory_path(task.id, project.id)))
self.assertTrue(len(task.imageupload_set.all()) == 2)
for image in task.imageupload_set.all():
self.assertTrue('project/{}/'.format(project.id) in image.image.path)
task.project = other_project
task.save()
task.refresh_from_db()
self.assertFalse(os.path.exists(full_task_directory_path(task.id, project.id)))
self.assertTrue(os.path.exists(full_task_directory_path(task.id, other_project.id)))
for image in task.imageupload_set.all():
self.assertTrue('project/{}/'.format(other_project.id) in image.image.path)
node_odm.terminate()
# Restart node-odm as to not generate orthophotos
testWatch.clear()
node_odm = start_processing_node("--test_skip_orthophotos")
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task_no_orthophoto',
'processing_node': pnode.id,
'auto_processing_node': 'false'
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
worker.tasks.process_pending_tasks()
time.sleep(DELAY)
worker.tasks.process_pending_tasks()
task = Task.objects.get(pk=res.data['id'])
self.assertTrue(task.status == status_codes.COMPLETED)
# Orthophoto files/directories should be missing
self.assertFalse(os.path.exists(task.assets_path("odm_orthophoto", "odm_orthophoto.tif")))
self.assertFalse(os.path.exists(task.assets_path("orthophoto_tiles")))
# orthophoto_extent should be none
self.assertTrue(task.orthophoto_extent is None)
# but other extents should be populated
self.assertTrue(task.dsm_extent is not None)
self.assertTrue(task.dtm_extent is not None)
self.assertTrue(os.path.exists(task.assets_path("dsm_tiles")))
self.assertTrue(os.path.exists(task.assets_path("dtm_tiles")))
# Can access only tiles of available assets
res = client.get("/api/projects/{}/tasks/{}/dsm/tiles.json".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
res = client.get("/api/projects/{}/tasks/{}/dtm/tiles.json".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
res = client.get("/api/projects/{}/tasks/{}/orthophoto/tiles.json".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
# Available assets should be missing orthophoto.tif type
# but others such as textured_model.zip should be available
res = client.get("/api/projects/{}/tasks/{}/".format(project.id, task.id))
self.assertFalse('orthophoto.tif' in res.data['available_assets'])
self.assertFalse('orthophoto_tiles.zip' in res.data['available_assets'])
self.assertTrue('textured_model.zip' in res.data['available_assets'])
image1.close()
image2.close()
gcp.close()
node_odm.terminate()
def test_task(self):
client = APIClient()
node_odm = start_processing_node()
user = User.objects.get(username="******")
self.assertFalse(user.is_superuser)
other_user = User.objects.get(username="******")
project = Project.objects.create(
owner=user,
name="test project"
)
other_project = Project.objects.create(
owner=other_user,
name="another test project"
)
other_task = Task.objects.create(project=other_project)
# Start processing node
# Create processing node
pnode = ProcessingNode.objects.create(hostname="localhost", port=11223)
# Verify that it's working
self.assertTrue(pnode.api_version is not None)
# task creation via file upload
image1 = open("app/fixtures/tiny_drone_image.jpg", 'rb')
image2 = open("app/fixtures/tiny_drone_image_2.jpg", 'rb')
img1 = Image.open("app/fixtures/tiny_drone_image.jpg")
# Not authenticated?
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2]
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_403_FORBIDDEN);
image1.seek(0)
image2.seek(0)
client.login(username="******", password="******")
# Cannot create a task for a project that does not exist
res = client.post("/api/projects/0/tasks/", {
'images': [image1, image2]
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
image1.seek(0)
image2.seek(0)
# Cannot create a task for a project for which we have no access to
res = client.post("/api/projects/{}/tasks/".format(other_project.id), {
'images': [image1, image2]
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
image1.seek(0)
image2.seek(0)
# Cannot create a task without images
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': []
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
# Cannot create a task with just 1 image
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': image1
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
image1.seek(0)
# Normal case with images[], name and processing node parameter
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task',
'processing_node': pnode.id
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
multiple_param_task = Task.objects.latest('created_at')
self.assertTrue(multiple_param_task.name == 'test_task')
self.assertTrue(multiple_param_task.processing_node.id == pnode.id)
image1.seek(0)
image2.seek(0)
# Uploaded images should be the same size as originals
with Image.open(multiple_param_task.task_path("tiny_drone_image.jpg")) as im:
self.assertTrue(im.size == img1.size)
# Normal case with images[], GCP, name and processing node parameter and resize_to option
gcp = open("app/fixtures/gcp.txt", 'r')
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2, gcp],
'name': 'test_task',
'processing_node': pnode.id,
'resize_to': img1.size[0] / 2.0
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
resized_task = Task.objects.latest('created_at')
image1.seek(0)
image2.seek(0)
gcp.seek(0)
# Uploaded images should have been resized
with Image.open(resized_task.task_path("tiny_drone_image.jpg")) as im:
self.assertTrue(im.size[0] == img1.size[0] / 2.0)
# GCP should have been scaled
with open(resized_task.task_path("gcp.txt")) as f:
lines = list(map(lambda l: l.strip(), f.readlines()))
[x, y, z, px, py, imagename, *extras] = lines[1].split(' ')
self.assertTrue(imagename == "tiny_drone_image.JPG") # case insensitive
self.assertTrue(float(px) == 2.0) # scaled by half
self.assertTrue(float(py) == 3.0) # scaled by half
self.assertTrue(float(x) == 576529.22) # Didn't change
[x, y, z, px, py, imagename, *extras] = lines[5].split(' ')
self.assertTrue(imagename == "missing_image.jpg")
self.assertTrue(float(px) == 8.0) # Didn't change
self.assertTrue(float(py) == 8.0) # Didn't change
# Case with malformed GCP file option
with open("app/fixtures/gcp_malformed.txt", 'r') as malformed_gcp:
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2, malformed_gcp],
'name': 'test_task',
'processing_node': pnode.id,
'resize_to': img1.size[0] / 2.0
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
malformed_gcp_task = Task.objects.latest('created_at')
# We just pass it along, it will get errored out during processing
# But we shouldn't fail.
with open(malformed_gcp_task.task_path("gcp_malformed.txt")) as f:
lines = list(map(lambda l: l.strip(), f.readlines()))
self.assertTrue(lines[1] == "<O_O>")
image1.seek(0)
image2.seek(0)
# Cannot create a task with images[], name, but invalid processing node parameter
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task',
'processing_node': 9999
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
image1.seek(0)
image2.seek(0)
# Normal case with images[] parameter
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'auto_processing_node': 'false'
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
image1.seek(0)
image2.seek(0)
# Should have returned the id of the newly created task
task = Task.objects.latest('created_at')
self.assertTrue('id' in res.data)
self.assertTrue(str(task.id) == res.data['id'])
# Two images should have been uploaded
self.assertTrue(ImageUpload.objects.filter(task=task).count() == 2)
# Can_rerun_from should be an empty list
self.assertTrue(len(res.data['can_rerun_from']) == 0)
# No processing node is set
self.assertTrue(task.processing_node is None)
# tiles.json should not be accessible at this point
tile_types = ['orthophoto', 'dsm', 'dtm']
for tile_type in tile_types:
res = client.get("/api/projects/{}/tasks/{}/{}/tiles.json".format(project.id, task.id, tile_type))
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
# Neither should an individual tile
# Z/X/Y coords are chosen based on node-odm test dataset for orthophoto_tiles/
res = client.get("/api/projects/{}/tasks/{}/orthophoto/tiles/16/16020/42443.png".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot access a tiles.json we have no access to
res = client.get("/api/projects/{}/tasks/{}/orthophoto/tiles.json".format(other_project.id, other_task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot access an individual tile we have no access to
res = client.get("/api/projects/{}/tasks/{}/orthophoto/tiles/16/16020/42443.png".format(other_project.id, other_task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot download assets (they don't exist yet)
for asset in list(task.ASSETS_MAP.keys()):
res = client.get("/api/projects/{}/tasks/{}/download/{}".format(project.id, task.id, asset))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot access raw assets (they don't exist yet)
res = client.get("/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot assign processing node to a task we have no access to
res = client.patch("/api/projects/{}/tasks/{}/".format(other_project.id, other_task.id), {
'processing_node': pnode.id
})
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# No UUID at this point
self.assertTrue(len(task.uuid) == 0)
# Assign processing node to task via API
res = client.patch("/api/projects/{}/tasks/{}/".format(project.id, task.id), {
'processing_node': pnode.id
})
self.assertTrue(res.status_code == status.HTTP_200_OK)
# On update worker.tasks.process_pending_tasks should have been called in the background
# (during tests this is sync)
# Processing should have started and a UUID is assigned
task.refresh_from_db()
self.assertTrue(task.status in [status_codes.RUNNING, status_codes.COMPLETED]) # Sometimes the task finishes and we can't test for RUNNING state
self.assertTrue(len(task.uuid) > 0)
# Processing node should have a "rerun_from" option
pnode_rerun_from_opts = list(filter(lambda d: 'name' in d and d['name'] == 'rerun-from', pnode.available_options))[0]
self.assertTrue(len(pnode_rerun_from_opts['domain']) > 0)
# The can_rerun_from field of a task should now be populated
# with the same values as the "rerun_from" domain values of
# the processing node
res = client.get("/api/projects/{}/tasks/{}/".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
self.assertTrue(pnode_rerun_from_opts['domain'] == res.data['can_rerun_from'])
time.sleep(DELAY)
# Calling process pending tasks should finish the process
# and invoke the plugins completed signal
with catch_signal(task_completed) as handler:
worker.tasks.process_pending_tasks()
task.refresh_from_db()
self.assertTrue(task.status == status_codes.COMPLETED)
handler.assert_called_with(
sender=Task,
task_id=task.id,
signal=task_completed,
)
# Can download assets
for asset in list(task.ASSETS_MAP.keys()):
res = client.get("/api/projects/{}/tasks/{}/download/{}".format(project.id, task.id, asset))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# A textured mesh archive file should exist
self.assertTrue(os.path.exists(task.assets_path(task.ASSETS_MAP["textured_model.zip"]["deferred_path"])))
# Can download raw assets
res = client.get("/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Can access tiles.json
for tile_type in tile_types:
res = client.get("/api/projects/{}/tasks/{}/{}/tiles.json".format(project.id, task.id, tile_type))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Bounds are what we expect them to be
# (4 coords in lat/lon)
tiles = json.loads(res.content.decode("utf-8"))
self.assertTrue(len(tiles['bounds']) == 4)
self.assertTrue(round(tiles['bounds'][0], 7) == -91.9945132)
# Can access individual tiles
for tile_type in tile_types:
res = client.get("/api/projects/{}/tasks/{}/{}/tiles/16/16020/42443.png".format(project.id, task.id, tile_type))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Another user does not have access to the resources
other_client = APIClient()
other_client.login(username="******", password="******")
def accessResources(expectedStatus):
for tile_type in tile_types:
res = other_client.get("/api/projects/{}/tasks/{}/{}/tiles.json".format(project.id, task.id, tile_type))
self.assertTrue(res.status_code == expectedStatus)
res = other_client.get("/api/projects/{}/tasks/{}/{}/tiles/16/16020/42443.png".format(project.id, task.id, tile_type))
self.assertTrue(res.status_code == expectedStatus)
accessResources(status.HTTP_404_NOT_FOUND)
# Original owner enables sharing
res = client.patch("/api/projects/{}/tasks/{}/".format(project.id, task.id), {
'public': True
})
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Now other user can acccess resources
accessResources(status.HTTP_200_OK)
# User logs out
other_client.logout()
# He can still access the resources as anonymous
accessResources(status.HTTP_200_OK)
# Other user still does not have access to certain parts of the API
res = other_client.get("/api/projects/{}/tasks/{}/".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_403_FORBIDDEN)
# Restart a task
testWatch.clear()
res = client.post("/api/projects/{}/tasks/{}/restart/".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# process_task is called in the background
task.refresh_from_db()
self.assertTrue(task.status in [status_codes.RUNNING, status_codes.COMPLETED])
# Cancel a task
res = client.post("/api/projects/{}/tasks/{}/cancel/".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# task is processed right away
# Should have been canceled
task.refresh_from_db()
self.assertTrue(task.status == status_codes.CANCELED)
# Remove a task and verify that it calls the proper plugins signals
with catch_signal(task_removing) as h1:
with catch_signal(task_removed) as h2:
res = client.post("/api/projects/{}/tasks/{}/remove/".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
h1.assert_called_once_with(sender=Task, task_id=task.id, signal=task_removing)
h2.assert_called_once_with(sender=Task, task_id=task.id, signal=task_removed)
# task is processed right away
# Has been removed along with assets
self.assertFalse(Task.objects.filter(pk=task.id).exists())
self.assertFalse(ImageUpload.objects.filter(task=task).exists())
task_assets_path = os.path.join(settings.MEDIA_ROOT, task_directory_path(task.id, task.project.id))
self.assertFalse(os.path.exists(task_assets_path))
# Stop processing node
node_odm.terminate()
# Create a task
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task_offline',
'processing_node': pnode.id,
'auto_processing_node': 'false'
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
task = Task.objects.get(pk=res.data['id'])
image1.seek(0)
image2.seek(0)
# Processing should fail and set an error
task.refresh_from_db()
self.assertTrue(task.last_error is not None)
self.assertTrue(task.status == status_codes.FAILED)
# Now bring it back online
node_odm = start_processing_node()
# Restart
res = client.post("/api/projects/{}/tasks/{}/restart/".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
task.refresh_from_db()
# After processing, the task should have restarted, and have no UUID or status
self.assertTrue(task.status is None)
self.assertTrue(len(task.uuid) == 0)
# Another step and it should have acquired a UUID
worker.tasks.process_pending_tasks()
task.refresh_from_db()
self.assertTrue(task.status in [status_codes.RUNNING, status_codes.COMPLETED])
self.assertTrue(len(task.uuid) > 0)
# Another step and it should be completed
time.sleep(DELAY)
worker.tasks.process_pending_tasks()
task.refresh_from_db()
self.assertTrue(task.status == status_codes.COMPLETED)
# Test rerun-from clearing mechanism:
# 1 .Set some task options, including rerun_from
task.options = [{'name': 'mesh-size', 'value':1000},
{'name': 'rerun-from', 'value': 'odm_meshing'}]
task.save()
# 2. Remove the task directly from node-odm (simulate a task purge)
self.assertTrue(task.processing_node.remove_task(task.uuid))
# 3. Restart the task
res = client.post("/api/projects/{}/tasks/{}/restart/".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# 4. Check that the rerun_from parameter has been cleared
# but the other parameters are still set
task.refresh_from_db()
self.assertTrue(len(task.uuid) == 0)
self.assertTrue(len(list(filter(lambda d: d['name'] == 'rerun-from', task.options))) == 0)
self.assertTrue(len(list(filter(lambda d: d['name'] == 'mesh-size', task.options))) == 1)
# Test connection, timeout errors
def connTimeout(*args, **kwargs):
raise requests.exceptions.ConnectTimeout("Simulated timeout")
testWatch.intercept("nodeodm.api_client.task_output", connTimeout)
worker.tasks.process_pending_tasks()
# Timeout errors should be handled by retrying again at a later time
# and not fail
task.refresh_from_db()
self.assertTrue(task.last_error is None)
# Reassigning the task to another project should move its assets
self.assertTrue(os.path.exists(full_task_directory_path(task.id, project.id)))
self.assertTrue(len(task.imageupload_set.all()) == 2)
for image in task.imageupload_set.all():
self.assertTrue('project/{}/'.format(project.id) in image.image.path)
task.project = other_project
task.save()
task.refresh_from_db()
self.assertFalse(os.path.exists(full_task_directory_path(task.id, project.id)))
self.assertTrue(os.path.exists(full_task_directory_path(task.id, other_project.id)))
for image in task.imageupload_set.all():
self.assertTrue('project/{}/'.format(other_project.id) in image.image.path)
node_odm.terminate()
# Restart node-odm as to not generate orthophotos
testWatch.clear()
node_odm = start_processing_node("--test_skip_orthophotos")
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task_no_orthophoto',
'processing_node': pnode.id,
'auto_processing_node': 'false'
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
worker.tasks.process_pending_tasks()
time.sleep(DELAY)
worker.tasks.process_pending_tasks()
task = Task.objects.get(pk=res.data['id'])
self.assertTrue(task.status == status_codes.COMPLETED)
# Orthophoto files/directories should be missing
self.assertFalse(os.path.exists(task.assets_path("odm_orthophoto", "odm_orthophoto.tif")))
self.assertFalse(os.path.exists(task.assets_path("orthophoto_tiles")))
# orthophoto_extent should be none
self.assertTrue(task.orthophoto_extent is None)
# but other extents should be populated
self.assertTrue(task.dsm_extent is not None)
self.assertTrue(task.dtm_extent is not None)
self.assertTrue(os.path.exists(task.assets_path("dsm_tiles")))
self.assertTrue(os.path.exists(task.assets_path("dtm_tiles")))
# Can access only tiles of available assets
res = client.get("/api/projects/{}/tasks/{}/dsm/tiles.json".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
res = client.get("/api/projects/{}/tasks/{}/dtm/tiles.json".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
res = client.get("/api/projects/{}/tasks/{}/orthophoto/tiles.json".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
# Available assets should be missing orthophoto.tif type
# but others such as textured_model.zip should be available
res = client.get("/api/projects/{}/tasks/{}/".format(project.id, task.id))
self.assertFalse('orthophoto.tif' in res.data['available_assets'])
self.assertTrue('textured_model.zip' in res.data['available_assets'])
image1.close()
image2.close()
gcp.close()
node_odm.terminate()