def upload_input_ports(self, port_list=None, exclude_list=None): """ Takes the workflow value for each port and does the following: * If local filesystem -> Uploads locally files to s3. S3 location will be as follows: gbd-customer-data/<acct_id>/<workflow_name>/<task_name>/<port_name>/ * If S3 url -> do nothing. :returns the update workflow with S3 urls. """ input_ports = self._task.input_ports for port in input_ports: # If port list is not None, then only allow port names in the list if port_list and port.name not in port_list: continue # Exclude ports as provided if exclude_list and port.name in exclude_list: continue # port_value = port.get('value', None) # Check if the port value is a valid file system location if not port.value or not os.path.isabs(port.value) or not os.path.isdir(port.value): continue # The prefix for each key that is uploaded, not including the the acct id. prefix = '{run_name}/{port}'.format( run_name=self._task.run_name, # task=self._task.name, port=port.name ) port_files = self._get_port_files(port.value, prefix) # Update the port value with an S3 url port.value = '%s/%s' % (self.s3_root, prefix) if len(port_files) == 0: printer('Port %s is empty, push to S3 skipped' % port.name) else: self.storage.upload(port_files) printer('Port %s pushed to account storage, %s files' % (port.name, len(port_files)))
def _create_app(self): """ Method for creating a new Application Template. USAGE: cloud-harness create <dir_name> [--destination=<path>] """ template_path = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), self.TEMPLATE_FOLDER, self.TEMPLATE_FILENAME ) new_dir = self._arguments['<dir_name>'] # Make new application directory override_destination = self._arguments.get('--destination', None) if override_destination is not None: if override_destination == '': raise ValueError('Destination path is empty') # Check if the new destination is abs and exists. if os.path.isabs(override_destination) and os.path.isdir(override_destination): new_dir = os.path.join(override_destination, new_dir) else: # Create a path from the cwd, then check if it is valid and exists. override_path = os.path.join(os.getcwd(), override_destination) if not os.path.isabs(override_path) or not os.path.isdir(override_path): raise ValueError('New path parameter %s is not a directory' % override_destination) new_dir = os.path.join(override_path, new_dir) else: if os.path.isabs(new_dir) or os.path.sep in new_dir: raise ValueError("Directory name is invalid") # No override, put the folder in the cwd. new_dir = os.path.join(os.getcwd(), new_dir) os.makedirs(new_dir) new_file_path = os.path.join(new_dir, self.DEFAULT_NEW_APP_FILENAME) # Copy the template the new application location. shutil.copyfile(template_path, new_file_path) printer('New Application created at %s' % new_file_path)
def __upload_file(self, file_tuple, folder): key_name = '%s/%s' % (folder, file_tuple[1]) printer('%s uploaded to location %s' % (file_tuple[0], file_tuple[1])) self.s3.upload_file(file_tuple[0], self.bucket, key_name)
def _run_app(self): """ Method for running a custom Application Templates. NOTES: * The default name of the application is app.py. So this function is going to look for app.py, unless the --file option is provide with a different file name. * The generated source bundle will package everything in the work_path. If large files not required for the application source, they need to be ignored. Use a file called "pkg_ignore" to identify folders and files to ignore. USAGE: cloud-harness run <file_name> [--remote] [--verbose] [--upload] [--download] [--dry-run] """ is_remote_run = self._arguments.get('--remote') filename = self._arguments.get('<file_name>') upload_ports = self._arguments.get('--upload') download_ports = self._arguments.get('--download') is_verbose = self._arguments.get('--verbose') # A dry run means, allow port sot be pushed up, but don't allow execution and monitoring. is_dry_run = self._arguments.get('--dry-run') if download_ports: # TODO temporary until implemented. raise NotImplementedError("Downloading of output ports is not implemented yet.") # Check if the filename passed is actually a class object (gbdxtools functionality) if not isinstance(filename, str) and issubclass(filename, TaskTemplate): template_class = filename template_file = inspect.getfile(template_class) config_file = self._write_config_file(template_file) else: template_file = self._get_template_abs_path(filename) if not os.path.isfile(template_file): raise ValueError('The location %s does not exist' % template_file) config_file = self._write_config_file(template_file) template_class = self._get_class(template_file) with template_class() as template: if is_remote_run: task = template.task # Set the source bundle directory to where the tempalte_file is. task.source_bundle.value = os.path.join(os.path.dirname(template_file), 'tmp_%s' % str(uuid.uuid4())) # Create a task service object task_service = TaskService() task_service.delete_task(task.name) printer(task_service.register_task(task.json())) task.run_name = '{task_name}_src'.format( task_name=task.name, # timestamp=datetime.utcnow().strftime('%Y_%m_%d_%H') ) src_bundle_dir = task.source_bundle.value # Create source bundle to be executed on the GBDX platform self._archive_source(os.path.dirname(src_bundle_dir), src_bundle_dir) port_service = PortService(task) if upload_ports: # Push all port data to S3 port_service.upload_input_ports() else: # Only push source bundle port port_service.upload_input_ports(port_list=[self.SOURCE_BUNDLE_PORT]) # Delete source bundle directory and config after upload. shutil.rmtree(src_bundle_dir) os.remove(config_file) # Build task json to run remotely self.task = port_service.task # Validate task task.is_valid(remote=True) workflow = Workflow(self.task) if is_verbose: printer(template.task.json()) temp_wf = workflow.json printer(temp_wf) if not is_dry_run: try: workflow.execute() printer(workflow.id) except Exception as e: printer(e.message) template.reason = "Execution Failed: %s" % e.message return # Monitor events of workflow is_done = workflow.monitor_run() if is_done: template.reason = "Execution Completed" else: template.reason = "Execution Failed during Run" if download_ports: # port_service.download_output_port() pass # Note: This may be temporary while working with gbdxtools # Delete task after run task_service.delete_task(task.name) else: # Validate task template.task.is_valid() if is_verbose: printer(template.task.json()) all_ports = template.task.ports[0] + template.task.ports[1] printer([port.__str__() for port in all_ports]) if not is_dry_run: # Run Task Locally try: template.invoke() except Exception as e: template.reason = "Failed Exception: %s" % e template.reason = "Execution Completed" else: template.reason = "Execution Skipped"