def execute_remote_task(
self,
done,
seed_files=[],
seed_touch_files=[],
outputs=[],
):
"""
Run a task by creating a seed file with all files in seed_files, optionally
creating empty files (for flag checks) specified in seed_touch_files
and returning the results specified in outputs. Yeah it's pretty cool!
"""
seed_file = self.create_seed_payload(seed_files,
touch_files=seed_touch_files)
# Find all images
images = glob.glob(self.path("images/**"))
# Add GCP (optional)
if os.path.exists(self.path("gcp_list.txt")):
images.append(self.path("gcp_list.txt"))
# Add seed file
images.append(seed_file)
class nonloc:
last_update = 0
def print_progress(percentage):
if (time.time() - nonloc.last_update >=
2) or int(percentage) == 100:
log.ODM_INFO("LRE: Upload of %s at [%s%%]" %
(self, int(percentage)))
nonloc.last_update = time.time()
# Upload task
task = self.node.create_task(images,
get_submodel_args_dict(config.config()),
progress_callback=print_progress,
skip_post_processing=True,
outputs=outputs)
self.remote_task = task
# Cleanup seed file
os.remove(seed_file)
# Keep track of tasks for cleanup
self.params['tasks'].append(task)
# Check status
info = task.info()
if info.status in [TaskStatus.RUNNING, TaskStatus.COMPLETED]:
def monitor():
class nonloc:
status_callback_calls = 0
last_update = 0
def status_callback(info):
# If a task switches from RUNNING to QUEUED, then we need to
# stop the process and re-add the task to the queue.
if info.status == TaskStatus.QUEUED:
log.ODM_WARNING(
"LRE: %s (%s) turned from RUNNING to QUEUED. Re-adding to back of the queue."
% (self, task.uuid))
raise NodeTaskLimitReachedException(
"Delayed task limit reached")
elif info.status == TaskStatus.RUNNING:
# Print a status message once in a while
nonloc.status_callback_calls += 1
if nonloc.status_callback_calls > 30:
log.ODM_INFO("LRE: %s (%s) is still running" %
(self, task.uuid))
nonloc.status_callback_calls = 0
try:
def print_progress(percentage):
if (time.time() - nonloc.last_update >=
2) or int(percentage) == 100:
log.ODM_INFO("LRE: Download of %s at [%s%%]" %
(self, int(percentage)))
nonloc.last_update = time.time()
task.wait_for_completion(status_callback=status_callback)
log.ODM_INFO("LRE: Downloading assets for %s" % self)
task.download_assets(self.project_path,
progress_callback=print_progress)
log.ODM_INFO(
"LRE: Downloaded and extracted assets for %s" % self)
done()
except exceptions.TaskFailedError as e:
# Try to get output
try:
output_lines = task.output()
# Save to file
error_log_path = self.path("error.log")
with open(error_log_path, 'w') as f:
f.write('\n'.join(output_lines) + '\n')
msg = "(%s) failed with task output: %s\nFull log saved at %s" % (
task.uuid, "\n".join(
output_lines[-10:]), error_log_path)
done(exceptions.TaskFailedError(msg))
except:
log.ODM_WARNING(
"LRE: Could not retrieve task output for %s (%s)" %
(self, task.uuid))
done(e)
except Exception as e:
done(e)
# Launch monitor thread and return
t = threading.Thread(target=monitor)
self.params['threads'].append(t)
t.start()
elif info.status == TaskStatus.QUEUED:
raise NodeTaskLimitReachedException("Task limit reached")
else:
raise Exception("Could not send task to node, task status is %s" %
str(info.status))
def test_get_submodel_argv_dict(self):
# Base
args = config.config(["--project-path", "/datasets"])
self.assertEqual(get_submodel_args_dict(args),
{'orthophoto-cutline': True, 'skip-3dmodel': True, 'dem-euclidean-map': True})