def test_upload_port(self):
"""
Test single task and port
"""
new_path = os.path.join(self.test_path, "input")
test_task_def = MY_BASIC_APP_WF_DEF["tasks"][0]
task_run_name = "upload_test_run"
# prefix is {task_name}/{port_name}
test_prefix = "/".join(["not_provided", task_run_name, test_task_def["inputs"][0]["name"]])
exp_results = [
"data/readme.txt",
"data/subsub/test.rst",
"imgs/tester.png",
# '.DS_Store',
"README.md",
"app.py",
# '__pycache__/app.cpython-27-PYTEST.pyc'
]
with MyBasicApp() as app:
app.task.second_port.value = new_path
app.task.run_name = task_run_name
port_service = PortService(app.task, storage_service=self.test_storage)
port_service.upload_input_ports()
new_task = port_service.task
self.assertTrue(new_task.is_valid(remote=True))
bucket_contents = self.test_storage.client.list_objects(Bucket=self.test_bucket)["Contents"]
print([key["Key"] for key in bucket_contents])
count = 0
for key in bucket_contents:
self.assertEqual(len(bucket_contents), len(exp_results))
for result in exp_results:
if key["Key"].endswith(result):
self.assertTrue(True)
count += 1
self.assertIn(self.test_account_id, key["Key"])
self.assertIn(test_prefix, key["Key"])
self.assertEqual(count, len(bucket_contents))
self.assertTrue(new_task.second_port.value.startswith(self.test_storage.location))
self.assertTrue(new_task.second_port.value.endswith(test_prefix))
def test_port_file_search(self):
test_prefix = "My/Test/Prefix"
source_files = PortService._get_port_files(self.test_path, test_prefix)
exp_result = (
os.path.join(self.test_path, "input", "data", "readme.txt"),
"%s/%s" % (test_prefix, "input/data/readme.txt"),
)
self.assertIn(exp_result, source_files)
def test_upload_ports(self):
"""
Test multiple tasks and ports
"""
new_path = os.path.join(self.test_path, "input")
with MyBasicApp() as app:
app.task.second_port.value = new_path
app.task.run_name = "uploadss_test_run"
port_service = PortService(app.task, storage_service=self.test_storage)
port_service.upload_input_ports()
new_task = port_service.task
self.assertTrue(new_task.is_valid(remote=True))
bucket_contents = self.test_storage.client.list_objects(Bucket=self.test_bucket)["Contents"]
count_files = [files for root, dirs, files in os.walk(new_path)]
self.assertEqual(len(bucket_contents), sum([len(x) for x in count_files]))
cnt_port1 = 0
cnt_port2 = 0
cnt_port3 = 0
for key in bucket_contents:
if "port4" in key["Key"]:
raise ValueError("Key name is incorrect %s" % key["Key"])
if "port1" in key["Key"]:
cnt_port1 += 1
if "port2" in key["Key"]:
cnt_port2 += 1
if "port3" in key["Key"]:
cnt_port3 += 1
self.assertTrue(cnt_port1 == cnt_port2 == cnt_port3)
def _run_app(self):
"""
Method for running a custom Application Templates.
NOTES:
* The default name of the application is app.py. So this function is going to look
for app.py, unless the --file option is provide with a different file name.
* The generated source bundle will package everything in the work_path. If large files
not required for the application source, they need to be ignored. Use a file called "pkg_ignore"
to identify folders and files to ignore.
USAGE: cloud-harness run <file_name> [--remote] [--verbose] [--upload] [--download] [--dry-run]
"""
is_remote_run = self._arguments.get('--remote')
filename = self._arguments.get('<file_name>')
upload_ports = self._arguments.get('--upload')
download_ports = self._arguments.get('--download')
is_verbose = self._arguments.get('--verbose')
# A dry run means, allow port sot be pushed up, but don't allow execution and monitoring.
is_dry_run = self._arguments.get('--dry-run')
if download_ports: # TODO temporary until implemented.
raise NotImplementedError("Downloading of output ports is not implemented yet.")
# Check if the filename passed is actually a class object (gbdxtools functionality)
if not isinstance(filename, str) and issubclass(filename, TaskTemplate):
template_class = filename
template_file = inspect.getfile(template_class)
config_file = self._write_config_file(template_file)
else:
template_file = self._get_template_abs_path(filename)
if not os.path.isfile(template_file):
raise ValueError('The location %s does not exist' % template_file)
config_file = self._write_config_file(template_file)
template_class = self._get_class(template_file)
with template_class() as template:
if is_remote_run:
task = template.task
# Set the source bundle directory to where the tempalte_file is.
task.source_bundle.value = os.path.join(os.path.dirname(template_file), 'tmp_%s' % str(uuid.uuid4()))
# Create a task service object
task_service = TaskService()
task_service.delete_task(task.name)
printer(task_service.register_task(task.json()))
task.run_name = '{task_name}_src'.format(
task_name=task.name,
# timestamp=datetime.utcnow().strftime('%Y_%m_%d_%H')
)
src_bundle_dir = task.source_bundle.value
# Create source bundle to be executed on the GBDX platform
self._archive_source(os.path.dirname(src_bundle_dir), src_bundle_dir)
port_service = PortService(task)
if upload_ports:
# Push all port data to S3
port_service.upload_input_ports()
else:
# Only push source bundle port
port_service.upload_input_ports(port_list=[self.SOURCE_BUNDLE_PORT])
# Delete source bundle directory and config after upload.
shutil.rmtree(src_bundle_dir)
os.remove(config_file)
# Build task json to run remotely
self.task = port_service.task
# Validate task
task.is_valid(remote=True)
workflow = Workflow(self.task)
if is_verbose:
printer(template.task.json())
temp_wf = workflow.json
printer(temp_wf)
if not is_dry_run:
try:
workflow.execute()
printer(workflow.id)
except Exception as e:
printer(e.message)
template.reason = "Execution Failed: %s" % e.message
return
# Monitor events of workflow
is_done = workflow.monitor_run()
if is_done:
template.reason = "Execution Completed"
else:
template.reason = "Execution Failed during Run"
if download_ports:
# port_service.download_output_port()
pass
# Note: This may be temporary while working with gbdxtools
# Delete task after run
task_service.delete_task(task.name)
else:
# Validate task
template.task.is_valid()
if is_verbose:
printer(template.task.json())
all_ports = template.task.ports[0] + template.task.ports[1]
printer([port.__str__() for port in all_ports])
if not is_dry_run:
# Run Task Locally
try:
template.invoke()
except Exception as e:
template.reason = "Failed Exception: %s" % e
template.reason = "Execution Completed"
else:
template.reason = "Execution Skipped"
