def test_upload_port(self): """ Test single task and port """ new_path = os.path.join(self.test_path, "input") test_task_def = MY_BASIC_APP_WF_DEF["tasks"][0] task_run_name = "upload_test_run" # prefix is {task_name}/{port_name} test_prefix = "/".join(["not_provided", task_run_name, test_task_def["inputs"][0]["name"]]) exp_results = [ "data/readme.txt", "data/subsub/test.rst", "imgs/tester.png", # '.DS_Store', "README.md", "app.py", # '__pycache__/app.cpython-27-PYTEST.pyc' ] with MyBasicApp() as app: app.task.second_port.value = new_path app.task.run_name = task_run_name port_service = PortService(app.task, storage_service=self.test_storage) port_service.upload_input_ports() new_task = port_service.task self.assertTrue(new_task.is_valid(remote=True)) bucket_contents = self.test_storage.client.list_objects(Bucket=self.test_bucket)["Contents"] print([key["Key"] for key in bucket_contents]) count = 0 for key in bucket_contents: self.assertEqual(len(bucket_contents), len(exp_results)) for result in exp_results: if key["Key"].endswith(result): self.assertTrue(True) count += 1 self.assertIn(self.test_account_id, key["Key"]) self.assertIn(test_prefix, key["Key"]) self.assertEqual(count, len(bucket_contents)) self.assertTrue(new_task.second_port.value.startswith(self.test_storage.location)) self.assertTrue(new_task.second_port.value.endswith(test_prefix))
def test_port_file_search(self): test_prefix = "My/Test/Prefix" source_files = PortService._get_port_files(self.test_path, test_prefix) exp_result = ( os.path.join(self.test_path, "input", "data", "readme.txt"), "%s/%s" % (test_prefix, "input/data/readme.txt"), ) self.assertIn(exp_result, source_files)
def test_upload_ports(self): """ Test multiple tasks and ports """ new_path = os.path.join(self.test_path, "input") with MyBasicApp() as app: app.task.second_port.value = new_path app.task.run_name = "uploadss_test_run" port_service = PortService(app.task, storage_service=self.test_storage) port_service.upload_input_ports() new_task = port_service.task self.assertTrue(new_task.is_valid(remote=True)) bucket_contents = self.test_storage.client.list_objects(Bucket=self.test_bucket)["Contents"] count_files = [files for root, dirs, files in os.walk(new_path)] self.assertEqual(len(bucket_contents), sum([len(x) for x in count_files])) cnt_port1 = 0 cnt_port2 = 0 cnt_port3 = 0 for key in bucket_contents: if "port4" in key["Key"]: raise ValueError("Key name is incorrect %s" % key["Key"]) if "port1" in key["Key"]: cnt_port1 += 1 if "port2" in key["Key"]: cnt_port2 += 1 if "port3" in key["Key"]: cnt_port3 += 1 self.assertTrue(cnt_port1 == cnt_port2 == cnt_port3)
def _run_app(self): """ Method for running a custom Application Templates. NOTES: * The default name of the application is app.py. So this function is going to look for app.py, unless the --file option is provide with a different file name. * The generated source bundle will package everything in the work_path. If large files not required for the application source, they need to be ignored. Use a file called "pkg_ignore" to identify folders and files to ignore. USAGE: cloud-harness run <file_name> [--remote] [--verbose] [--upload] [--download] [--dry-run] """ is_remote_run = self._arguments.get('--remote') filename = self._arguments.get('<file_name>') upload_ports = self._arguments.get('--upload') download_ports = self._arguments.get('--download') is_verbose = self._arguments.get('--verbose') # A dry run means, allow port sot be pushed up, but don't allow execution and monitoring. is_dry_run = self._arguments.get('--dry-run') if download_ports: # TODO temporary until implemented. raise NotImplementedError("Downloading of output ports is not implemented yet.") # Check if the filename passed is actually a class object (gbdxtools functionality) if not isinstance(filename, str) and issubclass(filename, TaskTemplate): template_class = filename template_file = inspect.getfile(template_class) config_file = self._write_config_file(template_file) else: template_file = self._get_template_abs_path(filename) if not os.path.isfile(template_file): raise ValueError('The location %s does not exist' % template_file) config_file = self._write_config_file(template_file) template_class = self._get_class(template_file) with template_class() as template: if is_remote_run: task = template.task # Set the source bundle directory to where the tempalte_file is. task.source_bundle.value = os.path.join(os.path.dirname(template_file), 'tmp_%s' % str(uuid.uuid4())) # Create a task service object task_service = TaskService() task_service.delete_task(task.name) printer(task_service.register_task(task.json())) task.run_name = '{task_name}_src'.format( task_name=task.name, # timestamp=datetime.utcnow().strftime('%Y_%m_%d_%H') ) src_bundle_dir = task.source_bundle.value # Create source bundle to be executed on the GBDX platform self._archive_source(os.path.dirname(src_bundle_dir), src_bundle_dir) port_service = PortService(task) if upload_ports: # Push all port data to S3 port_service.upload_input_ports() else: # Only push source bundle port port_service.upload_input_ports(port_list=[self.SOURCE_BUNDLE_PORT]) # Delete source bundle directory and config after upload. shutil.rmtree(src_bundle_dir) os.remove(config_file) # Build task json to run remotely self.task = port_service.task # Validate task task.is_valid(remote=True) workflow = Workflow(self.task) if is_verbose: printer(template.task.json()) temp_wf = workflow.json printer(temp_wf) if not is_dry_run: try: workflow.execute() printer(workflow.id) except Exception as e: printer(e.message) template.reason = "Execution Failed: %s" % e.message return # Monitor events of workflow is_done = workflow.monitor_run() if is_done: template.reason = "Execution Completed" else: template.reason = "Execution Failed during Run" if download_ports: # port_service.download_output_port() pass # Note: This may be temporary while working with gbdxtools # Delete task after run task_service.delete_task(task.name) else: # Validate task template.task.is_valid() if is_verbose: printer(template.task.json()) all_ports = template.task.ports[0] + template.task.ports[1] printer([port.__str__() for port in all_ports]) if not is_dry_run: # Run Task Locally try: template.invoke() except Exception as e: template.reason = "Failed Exception: %s" % e template.reason = "Execution Completed" else: template.reason = "Execution Skipped"