def describe_component(name):
plasma_config = get_config()
if os.path.exists(plasma_config['paths']['components_path'] + name):
readme_file = plasma_config['paths'][
'components_path'] + name + '/README'
with open(readme_file, 'r') as readme:
print(readme.read())
else:
print('\n> component not found locally\n')
def describe_workflow(name):
plasma_config = get_config()
logger.debug('Executing describe workflow')
if os.path.exists(plasma_config['paths']['workflows_path'] + name):
readme_file = plasma_config['paths'][
'workflows_path'] + name + '/README'
with open(readme_file, 'r') as readme:
print(readme.read())
else:
print('\n> workflow description not found.\n')
def list_components():
plasma_config = get_config()
components_path = plasma_config['paths']['components_path']
components = os.listdir(components_path)
if components:
print('\n> listing local components ')
for item in components:
if os.path.isdir(components_path + item):
print('\t- ' + item)
print()
else:
print('\n> no components have been downloaded\n')
def verify_components(workflow):
plasma_config = get_config()
logger.info('verifying components')
components_path = plasma_config['paths']['components_path']
local_components = os.listdir(components_path)
workflow_components = list(workflow.workflow['workflow'].keys())
missing_components = set(workflow_components) - set(local_components)
if (len(missing_components) == 0):
return True
else:
logger.error('unable to execute workflow')
logger.error('component not found : ' + ', '.join(missing_components))
return False
def list_workflows():
plasma_config = get_config()
logger.debug('Executing list workflows')
workflows_path = plasma_config['paths']['workflows_path']
workflows = os.listdir(workflows_path)
if workflows:
print('\n> listing workflows ')
for item in workflows:
if item.endswith('.yml'):
print('\t- ' + item)
print()
else:
print('\n> no workflows have been created\n')
def execute_step(step):
plasma_config = get_config()
try:
logger.info('executing step :' + step['component'])
component_name = step['component']
component_path = plasma_config['paths']['components_path'] + \
component_name+'/component.py'
component = component_loader(component_name, component_path)
logging.getLogger(component_name).setLevel(logging.ERROR)
output = component.main(step)
return output
except Exception as e:
logger.error('failed to execute step : %s > %s' %
(step['component'], step['operation']))
logger.error(e)
def download_component(name):
plasma_config = get_config()
print('\n> downloading ' + name + ' from the component registry')
response = requests.get(api_url + '/download/' + name).json()
component = response.get('data')
if component:
response = requests.get(component['url'])
print('> extracting zip file')
zip_file = zipfile.ZipFile(io.BytesIO(response.content))
zip_file.extractall(plasma_config['paths']['components_path'])
print('> component stored at ' + plasma_config['components_path'])
print('> download complete\n')
else:
print('\n> component not found')
print()
def setup_virtual_environment(requirements_path, workflow_name):
plasma_config = get_config()
try:
logger.info('setting up virtual environment')
venv_path = plasma_config['paths'][
'data_path'] + workflow_name + '_venv'
venv.create(venv_path)
logger.info('activating virtual environment')
output = os.system('bash ' + venv_path + '/bin/activate')
logger.info('installing dependencies')
output = subprocess.check_output(
['pip3', 'install', '-r', requirements_path])
return True
except Exception as e:
logger.error("unable to setup virtual environment")
logger.error(e)
exit(1)
def run_workflow(workflow_name):
config = get_config()
workflow_path = config['paths']['workflows_path'] + workflow_name
workflow = Workflow(workflow_path)
workflow_valid = workflow.validate()
if workflow_valid:
components = verify_components(workflow)
if not components:
logger.error('components declared in workflow are missing')
exit(1)
#requirements = generate_workflow_requirements(workflow)
#virtual_environment = setup_virtual_environment(requirements, workflow_name)
state = execute_workflow(workflow.steps)
if state is True:
logger.info('workflow executed')
else:
logger.info('failed to execute workflow')
print('\n')
else:
logger.error('invalid workflow')
exit(1)
def generate_workflow_requirements(workflow):
plasma_config = get_config()
workflow_name = workflow.name
logger.info('generating workflow requirements')
try:
components = list(workflow.workflow['workflow'].keys())
requirements_list = []
for component in components:
requirements_path = plasma_config['paths']['components_path'] + \
component+'/requirements.txt'
with open(requirements_path, 'r') as requirements_file:
requirements = requirements_file.read().split('\n')
requirements = [x for x in requirements if x]
requirements_list += requirements
workflow_requirements_path = plasma_config['paths']['data_path'] + \
workflow_name+'.requirements'
with open(workflow_requirements_path, 'w') as file:
requirements_string = '\n'.join(requirements_list)
file.write(requirements_string)
return workflow_requirements_path
except Exception as e:
logger.error('failed to generate requirements file')
logger.error(e)
exit(1)
