def test_auto_update_node_info(self):
with start_processing_node():
online_node = ProcessingNode.objects.create(hostname="localhost",
port=11223)
self.assertTrue(
online_node.last_refreshed != None,
"Last refreshed info is here (update_node_info() was called)")
def test_find_best_available_node_and_is_online(self):
with start_processing_node():
# Fixtures are all offline
self.assertTrue(ProcessingNode.find_best_available_node() is None)
# Bring one online
pnode = ProcessingNode.objects.get(pk=1)
self.assertFalse(pnode.is_online())
pnode.last_refreshed = timezone.now()
pnode.queue_count = 2
pnode.save()
self.assertTrue(pnode.is_online())
self.assertTrue(
ProcessingNode.find_best_available_node().id == pnode.id)
# Bring another online with lower queue count
another_pnode = ProcessingNode.objects.get(pk=2)
another_pnode.last_refreshed = pnode.last_refreshed
another_pnode.queue_count = 1
another_pnode.save()
self.assertTrue(ProcessingNode.find_best_available_node().id ==
another_pnode.id)
# Bring it offline
another_pnode.last_refreshed -= timedelta(
minutes=settings.NODE_OFFLINE_MINUTES)
another_pnode.save()
self.assertFalse(another_pnode.is_online())
# Best choice now is original processing node
self.assertTrue(
ProcessingNode.find_best_available_node().id == pnode.id)
def test_online_processing_node(self):
with start_processing_node():
online_node = ProcessingNode.objects.get(pk=1)
self.assertTrue(
str(online_node) == "localhost:11223",
"Formatting string works")
self.assertTrue(online_node.last_refreshed == None,
"Last refreshed not yet set")
self.assertTrue(
len(online_node.available_options) == 0,
"Available options not yet set")
self.assertTrue(online_node.api_version == "",
"API version is not set")
self.assertTrue(online_node.update_node_info(),
"Could update info")
self.assertTrue(online_node.last_refreshed is not None,
"Last refreshed is set")
self.assertTrue(
len(online_node.available_options) > 0,
"Available options are set")
self.assertTrue(online_node.api_version != "",
"API version is set")
self.assertTrue(online_node.max_images is None,
"No max images limit is set")
self.assertTrue(
isinstance(online_node.get_available_options_json(),
six.string_types), "Available options json works")
self.assertTrue(
isinstance(online_node.get_available_options_json(pretty=True),
six.string_types),
"Available options json works with pretty")
def test_info(self):
with start_processing_node():
info = self.api_client.info()
self.assertTrue(isinstance(info.version, six.string_types),
"Found version string")
self.assertTrue(isinstance(info.task_queue_count, int),
"Found task queue count")
self.assertTrue(info.max_images is None, "Found task max images")
def test_options(self):
with start_processing_node():
options = self.api_client.options()
self.assertTrue(len(options) > 0, "Found options")
def test_token_auth(self):
def wait_for_status(api,
uuid,
status,
num_retries=10,
error_description="Failed to wait for status"):
retries = 0
while True:
try:
task_info = api.get_task(uuid).info()
if task_info.status.value == status:
return True
except (NodeResponseError, NodeServerError):
pass
time.sleep(0.5)
retries += 1
if retries >= num_retries:
self.assertTrue(False, error_description)
return False
with start_processing_node(['--token', 'test_token']):
api = Node("localhost", 11223, "test_token")
online_node = ProcessingNode.objects.get(pk=3)
self.assertTrue(online_node.update_node_info(),
"Could update info")
# Cannot call info(), options() without tokens
api.token = "invalid"
self.assertRaises(NodeResponseError, api.info)
self.assertRaises(NodeResponseError, api.options)
# Cannot call create_task() without token
import glob
self.assertRaises(NodeResponseError, api.create_task,
glob.glob("nodeodm/fixtures/test_images/*.JPG"))
# Can call create_task() with token
api.token = "test_token"
res = api.create_task(
glob.glob("nodeodm/fixtures/test_images/*.JPG"))
uuid = res.uuid
self.assertTrue(uuid != None)
# Can call task_info() with token
task_info = api.get_task(uuid).info()
self.assertTrue(isinstance(task_info.date_created, datetime))
# Cannot call task_info() without token
api.token = "invalid"
try:
api.get_task(uuid).info()
except NodeResponseError as e:
self.assertTrue('token does not match' in str(e))
# Here we are waiting for the task to be completed
api.token = "test_token"
wait_for_status(api, uuid, status_codes.COMPLETED, 10,
"Could not download assets")
# Cannot download assets without token
api.token = "invalid"
task = api.get_task(uuid)
self.assertRaises(NodeResponseError, task.download_assets,
settings.MEDIA_TMP)
api.token = "test_token"
asset_archive = task.download_zip(settings.MEDIA_TMP)
self.assertTrue(os.path.exists(asset_archive))
os.unlink(asset_archive)
# Cannot get task output without token
api.token = "invalid"
self.assertRaises(NodeResponseError, task.output, 0)
api.token = "test_token"
res = task.output()
self.assertTrue(isinstance(res, list))
# Cannot restart task without token
online_node.token = "invalid"
self.assertRaises(NodeResponseError, online_node.restart_task,
uuid)
online_node.token = "test_token"
self.assertTrue(online_node.restart_task(uuid))
# Cannot cancel task without token
online_node.token = "invalid"
self.assertRaises(NodeResponseError, online_node.cancel_task, uuid)
online_node.token = "test_token"
self.assertTrue(online_node.cancel_task(uuid))
# Wait for task to be canceled
wait_for_status(api, uuid, status_codes.CANCELED, 5,
"Could not cancel task")
# Cannot delete task without token
online_node.token = "invalid"
self.assertRaises(NodeResponseError, online_node.remove_task,
"invalid token")
online_node.token = "test_token"
self.assertTrue(online_node.remove_task(uuid))
def test_client_api_and_task_methods(self):
def wait_for_status(api,
uuid,
status,
num_retries=10,
error_description="Failed to wait for status"):
retries = 0
while True:
try:
task_info = api.get_task(uuid).info()
if task_info.status.value == status:
return True
except (NodeServerError, NodeResponseError):
pass
time.sleep(0.5)
retries += 1
if retries >= num_retries:
self.assertTrue(False, error_description)
return False
with start_processing_node():
api = Node("localhost", 11223)
online_node = ProcessingNode.objects.get(pk=1)
# Can call info(), options()
self.assertTrue(type(api.info().version) == str)
self.assertTrue(len(api.options()) > 0)
# Can call new_task()
import glob
res = api.create_task(
glob.glob("nodeodm/fixtures/test_images/*.JPG"),
{'force-ccd': 6.16}, "test")
uuid = res.uuid
self.assertTrue(uuid != None)
# Can call task_info()
task = api.get_task(uuid)
task_info = task.info()
self.assertTrue(isinstance(task_info.date_created, datetime))
self.assertTrue(isinstance(task_info.uuid, str))
# Can download assets?
# Here we are waiting for the task to be completed
wait_for_status(api, uuid, status_codes.COMPLETED, 10,
"Could not download assets")
asset = api.get_task(uuid).download_zip(settings.MEDIA_TMP)
self.assertTrue(os.path.exists(asset))
# task_output
self.assertTrue(isinstance(api.get_task(uuid).output(0), list))
self.assertTrue(
isinstance(online_node.get_task_console_output(uuid, 0), list))
self.assertRaises(NodeResponseError,
online_node.get_task_console_output,
"wrong-uuid", 0)
# Can restart task
self.assertTrue(online_node.restart_task(uuid))
self.assertRaises(NodeResponseError, online_node.restart_task,
"wrong-uuid")
wait_for_status(api, uuid, status_codes.COMPLETED, 10,
"Could not restart task")
# Can restart task by passing options
self.assertTrue(
online_node.restart_task(uuid, [{
'name': 'mesh-size',
'value': 12345
}, {
'name': 'invalid',
'value': True
}]))
wait_for_status(api, uuid, status_codes.COMPLETED, 10,
"Could not restart task with options")
# Verify that options have been updated after restarting the task
task_info = api.get_task(uuid).info()
self.assertTrue(len(
task_info.options) == 2) # pc-ept has been added
self.assertTrue(task_info.options[0]['name'] == 'mesh-size')
self.assertTrue(task_info.options[0]['value'] == 12345)
# Can cancel task (should work even if we completed the task)
self.assertTrue(online_node.cancel_task(uuid))
self.assertRaises(NodeResponseError, online_node.cancel_task,
"wrong-uuid")
# Wait for task to be canceled
wait_for_status(api, uuid, status_codes.CANCELED, 5,
"Could not remove task")
self.assertTrue(online_node.remove_task(uuid))
self.assertRaises(NodeResponseError, online_node.remove_task,
"wrong-uuid")
# Cannot delete task again
self.assertRaises(NodeResponseError, online_node.remove_task, uuid)
# Task has been deleted
self.assertRaises(NodeResponseError, online_node.get_task_info,
uuid)
# Test URL building for HTTPS
sslApi = Node("localhost", 443, 'abc')
self.assertEqual(sslApi.url('/info'),
'https://localhost/info?token=abc')
def test_task(self):
client = APIClient()
with start_processing_node():
user = User.objects.get(username="******")
self.assertFalse(user.is_superuser)
project = Project.objects.create(owner=user, name="test project")
image1 = open("app/fixtures/tiny_drone_image.jpg", 'rb')
image2 = open("app/fixtures/tiny_drone_image_2.jpg", 'rb')
# Create processing node
pnode = ProcessingNode.objects.create(hostname="localhost",
port=11223)
client.login(username="******", password="******")
# Create task
res = client.post("/api/projects/{}/tasks/".format(project.id),
{'images': [image1, image2]},
format="multipart")
image1.close()
image2.close()
task = Task.objects.get(id=res.data['id'])
# Wait for completion
c = 0
while c < 10:
worker.tasks.process_pending_tasks()
task.refresh_from_db()
if task.status == status_codes.COMPLETED:
break
c += 1
time.sleep(1)
self.assertEqual(task.status, status_codes.COMPLETED)
# Download task assets
task_uuid = task.uuid
res = client.get(
"/api/projects/{}/tasks/{}/download/all.zip".format(
project.id, task.id))
self.assertEqual(res.status_code, status.HTTP_200_OK)
if not os.path.exists(settings.MEDIA_TMP):
os.mkdir(settings.MEDIA_TMP)
assets_path = os.path.join(settings.MEDIA_TMP, "all.zip")
with open(assets_path, 'wb') as f:
f.write(res.content)
remove_perm('change_project', user, project)
assets_file = open(assets_path, 'rb')
# Cannot import unless we have permission
res = client.post("/api/projects/{}/tasks/import".format(
project.id), {'file': [assets_file]},
format="multipart")
self.assertEqual(res.status_code, status.HTTP_404_NOT_FOUND)
assign_perm('change_project', user, project)
# Import with file upload method
assets_file.seek(0)
res = client.post("/api/projects/{}/tasks/import".format(
project.id), {'file': [assets_file]},
format="multipart")
self.assertEqual(res.status_code, status.HTTP_201_CREATED)
assets_file.close()
file_import_task = Task.objects.get(id=res.data['id'])
# Wait for completion
c = 0
while c < 10:
worker.tasks.process_pending_tasks()
file_import_task.refresh_from_db()
if file_import_task.status == status_codes.COMPLETED:
break
c += 1
time.sleep(1)
self.assertEqual(file_import_task.import_url, "file://all.zip")
self.assertEqual(file_import_task.images_count, 1)
self.assertEqual(file_import_task.processing_node, None)
self.assertEqual(file_import_task.auto_processing_node, False)
# Can access assets
res = client.get(
"/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif"
.format(project.id, file_import_task.id))
self.assertEqual(res.status_code, status.HTTP_200_OK)
self.assertTrue(
valid_cogeo(
file_import_task.assets_path(
task.ASSETS_MAP["orthophoto.tif"])))
self.assertTrue(
valid_cogeo(
file_import_task.assets_path(task.ASSETS_MAP["dsm.tif"])))
self.assertTrue(
valid_cogeo(
file_import_task.assets_path(task.ASSETS_MAP["dtm.tif"])))
# Set task public so we can download from it without auth
file_import_task.public = True
file_import_task.save()
# Import with URL method
assets_import_url = "http://{}:{}/task/{}/download/all.zip".format(
pnode.hostname, pnode.port, task_uuid)
res = client.post(
"/api/projects/{}/tasks/import".format(project.id),
{'url': assets_import_url})
self.assertEqual(res.status_code, status.HTTP_201_CREATED)
url_task = Task.objects.get(id=res.data['id'])
# Wait for completion
c = 0
while c < 10:
worker.tasks.process_pending_tasks()
url_task.refresh_from_db()
if url_task.status == status_codes.COMPLETED:
break
c += 1
time.sleep(1)
self.assertEqual(url_task.import_url, assets_import_url)
self.assertEqual(url_task.images_count, 1)
# Import corrupted file
assets_import_url = "http://{}:{}/task/{}/download/orthophoto.tif".format(
pnode.hostname, pnode.port, task_uuid)
res = client.post(
"/api/projects/{}/tasks/import".format(project.id),
{'url': assets_import_url})
self.assertEqual(res.status_code, status.HTTP_201_CREATED)
corrupted_task = Task.objects.get(id=res.data['id'])
# Wait for completion
c = 0
while c < 10:
worker.tasks.process_pending_tasks()
corrupted_task.refresh_from_db()
if corrupted_task.status == status_codes.FAILED:
break
c += 1
time.sleep(1)
self.assertEqual(corrupted_task.status, status_codes.FAILED)
self.assertTrue("Invalid" in corrupted_task.last_error)