def describe_component(name): plasma_config = get_config() if os.path.exists(plasma_config['paths']['components_path'] + name): readme_file = plasma_config['paths'][ 'components_path'] + name + '/README' with open(readme_file, 'r') as readme: print(readme.read()) else: print('\n> component not found locally\n')
def describe_workflow(name): plasma_config = get_config() logger.debug('Executing describe workflow') if os.path.exists(plasma_config['paths']['workflows_path'] + name): readme_file = plasma_config['paths'][ 'workflows_path'] + name + '/README' with open(readme_file, 'r') as readme: print(readme.read()) else: print('\n> workflow description not found.\n')
def list_components(): plasma_config = get_config() components_path = plasma_config['paths']['components_path'] components = os.listdir(components_path) if components: print('\n> listing local components ') for item in components: if os.path.isdir(components_path + item): print('\t- ' + item) print() else: print('\n> no components have been downloaded\n')
def verify_components(workflow): plasma_config = get_config() logger.info('verifying components') components_path = plasma_config['paths']['components_path'] local_components = os.listdir(components_path) workflow_components = list(workflow.workflow['workflow'].keys()) missing_components = set(workflow_components) - set(local_components) if (len(missing_components) == 0): return True else: logger.error('unable to execute workflow') logger.error('component not found : ' + ', '.join(missing_components)) return False
def list_workflows(): plasma_config = get_config() logger.debug('Executing list workflows') workflows_path = plasma_config['paths']['workflows_path'] workflows = os.listdir(workflows_path) if workflows: print('\n> listing workflows ') for item in workflows: if item.endswith('.yml'): print('\t- ' + item) print() else: print('\n> no workflows have been created\n')
def execute_step(step): plasma_config = get_config() try: logger.info('executing step :' + step['component']) component_name = step['component'] component_path = plasma_config['paths']['components_path'] + \ component_name+'/component.py' component = component_loader(component_name, component_path) logging.getLogger(component_name).setLevel(logging.ERROR) output = component.main(step) return output except Exception as e: logger.error('failed to execute step : %s > %s' % (step['component'], step['operation'])) logger.error(e)
def download_component(name): plasma_config = get_config() print('\n> downloading ' + name + ' from the component registry') response = requests.get(api_url + '/download/' + name).json() component = response.get('data') if component: response = requests.get(component['url']) print('> extracting zip file') zip_file = zipfile.ZipFile(io.BytesIO(response.content)) zip_file.extractall(plasma_config['paths']['components_path']) print('> component stored at ' + plasma_config['components_path']) print('> download complete\n') else: print('\n> component not found') print()
def setup_virtual_environment(requirements_path, workflow_name): plasma_config = get_config() try: logger.info('setting up virtual environment') venv_path = plasma_config['paths'][ 'data_path'] + workflow_name + '_venv' venv.create(venv_path) logger.info('activating virtual environment') output = os.system('bash ' + venv_path + '/bin/activate') logger.info('installing dependencies') output = subprocess.check_output( ['pip3', 'install', '-r', requirements_path]) return True except Exception as e: logger.error("unable to setup virtual environment") logger.error(e) exit(1)
def run_workflow(workflow_name): config = get_config() workflow_path = config['paths']['workflows_path'] + workflow_name workflow = Workflow(workflow_path) workflow_valid = workflow.validate() if workflow_valid: components = verify_components(workflow) if not components: logger.error('components declared in workflow are missing') exit(1) #requirements = generate_workflow_requirements(workflow) #virtual_environment = setup_virtual_environment(requirements, workflow_name) state = execute_workflow(workflow.steps) if state is True: logger.info('workflow executed') else: logger.info('failed to execute workflow') print('\n') else: logger.error('invalid workflow') exit(1)
def generate_workflow_requirements(workflow): plasma_config = get_config() workflow_name = workflow.name logger.info('generating workflow requirements') try: components = list(workflow.workflow['workflow'].keys()) requirements_list = [] for component in components: requirements_path = plasma_config['paths']['components_path'] + \ component+'/requirements.txt' with open(requirements_path, 'r') as requirements_file: requirements = requirements_file.read().split('\n') requirements = [x for x in requirements if x] requirements_list += requirements workflow_requirements_path = plasma_config['paths']['data_path'] + \ workflow_name+'.requirements' with open(workflow_requirements_path, 'w') as file: requirements_string = '\n'.join(requirements_list) file.write(requirements_string) return workflow_requirements_path except Exception as e: logger.error('failed to generate requirements file') logger.error(e) exit(1)