def prepare_workflow_execution(recursive=False, **kwargs):
"""Set parameters for the workflow execution
and run the workflow."""
# Set the logging levels.
level = 'ACTION_INFO'
if kwargs['quiet']:
level = 'INFO'
if kwargs['debug']:
level = 'DEBUG'
log.setLevel(level)
if kwargs['log_file']:
logging.add_log(log, kwargs['log_file'])
# Remove the unnecessary kwargs.
kwargs.pop('quiet')
kwargs.pop('debug')
kwargs.pop('log_file')
# Run the workflow accordingly as recursive/CI and Non-CI.
if recursive:
for wfile in pu.find_recursive_wfile():
kwargs['wfile'] = wfile
run_workflow(**kwargs)
else:
run_workflow(**kwargs)
def setUp(self):
log.setLevel('CRITICAL')
self.Popen = MockPopen()
replacer = Replacer()
replacer.replace('popper.runner_host.Popen', self.Popen)
self.addCleanup(replacer.restore)
self.repo = tempfile.mkdtemp()
def tearDownClass(self):
log.setLevel('NOTSET')
# return to where we were before this test
os.chdir(self.curr_dir)
self.repo.close()
def setUp(self):
log.setLevel("CRITICAL")
config.load_kube_config()
c = Configuration()
c.assert_hostname = False
Configuration.set_default(c)
self._kclient = core_v1_api.CoreV1Api()
_, active_context = config.list_kube_config_contexts()
def setUp(self):
log.setLevel("CRITICAL")
self.Popen = MockPopen()
replacer = Replacer()
replacer.replace("popper.runner_host.Popen", self.Popen)
assert_replacer = Replacer()
assert_replacer.replace(
"popper.runner_slurm.assert_executable_exists",
mock_assert_executable_exists,
)
self.addCleanup(replacer.restore)
self.addCleanup(assert_replacer.restore)
def setUpClass(self):
log.setLevel('CRITICAL')
self.tempdir = tempfile.mkdtemp()
self.curr_dir = os.getcwd()
self.repo = git.Repo.clone_from('https://github.com/popperized/bin',
os.path.join(self.tempdir, 'bin'))
self.repodir = os.path.join(self.tempdir, 'bin')
os.chdir(self.repodir)
self.gitdir = os.path.join(self.repodir, '.git')
self.gotdor = os.path.join(self.repodir, '.got')
def setUp(self):
if os.environ.get('POPPER_TEST_MODE') == 'with-git':
self.with_git = True
else:
self.with_git = False
log.setLevel('CRITICAL')
if os.path.exists('/tmp/test_folder'):
shutil.rmtree('/tmp/test_folder')
os.makedirs('/tmp/test_folder')
os.chdir('/tmp/test_folder')
scm.clone('https://github.com', 'cplee', 'github-actions-demo',
os.path.join(os.getcwd(), 'github-actions-demo'))
if not self.with_git:
shutil.rmtree('/tmp/test_folder/github-actions-demo/.git')
os.chdir('/tmp/test_folder/github-actions-demo')
def setUp(self):
os.makedirs('/tmp/test_folder')
os.chdir('/tmp/test_folder')
log.setLevel('CRITICAL')
workflow = """
workflow "sample" {
resolves = "sample action"
}
action "sample action" {
uses = "sh"
args = ["echo", "Hello"]
}
"""
pu.write_file('/tmp/test_folder/a.workflow', workflow)
self.wf = Workflow('/tmp/test_folder/a.workflow')
self.wf.parse()
WorkflowRunner.instantiate_runners('docker', self.wf,
'/tmp/test_folder', False, False,
'12345')
def cli(ctx, action, wfile, workspace, reuse, recursive, quiet, debug, dry_run,
parallel, log_file):
"""Executes one or more pipelines and reports on their status.
"""
popper.scm.get_git_root_folder()
level = 'ACTION_INFO'
if quiet:
level = 'INFO'
if debug:
level = 'DEBUG'
log.setLevel(level)
if log_file:
logging.add_log(log, log_file)
if recursive:
wfile_list = pu.find_recursive_wfile()
if not wfile_list:
log.fail("Recursive search couldn't find any .workflow files ")
for wfile in wfile_list:
log.info("Found and running workflow at " + wfile)
run_pipeline(action, wfile, workspace, reuse, dry_run, parallel)
else:
run_pipeline(action, wfile, workspace, reuse, dry_run, parallel)
def cli(ctx, action, wfile, skip_clone, skip_pull, skip, workspace, reuse,
recursive, quiet, debug, dry_run, parallel, log_file,
with_dependencies, on_failure):
"""Executes one or more pipelines and reports on their status.
"""
popper.scm.get_git_root_folder()
level = 'ACTION_INFO'
if quiet:
level = 'INFO'
if debug:
level = 'DEBUG'
log.setLevel(level)
if log_file:
logging.add_log(log, log_file)
if os.environ.get('CI') == 'true':
log.info("Running in CI environment.")
if recursive:
log.warning('When CI variable is set, --recursive is ignored.')
wfile_list = pu.find_recursive_wfile()
wfile_list = workflows_from_commit_message(wfile_list)
else:
if recursive:
if action:
log.fail(
"An 'action' argument and the --recursive flag cannot be "
"both given.")
wfile_list = pu.find_recursive_wfile()
else:
wfile_list = [wfile]
if not wfile_list:
log.fail("No workflow to execute.")
for wfile in wfile_list:
wfile = pu.find_default_wfile(wfile)
log.info("Found and running workflow at " + wfile)
run_pipeline(action, wfile, skip_clone, skip_pull, skip, workspace,
reuse, dry_run, parallel, with_dependencies, on_failure)
def setUp(self):
os.makedirs('/tmp/test_folder')
os.chdir('/tmp/test_folder')
log.setLevel('CRITICAL')
workflow = """
workflow "sample" {
resolves = "sample action"
}
action "sample action" {
uses = "popperized/bin/sh@master"
args = ["echo", "Hello"]
}
"""
pu.write_file('/tmp/test_folder/a.workflow', workflow)
self.wf = Workflow('/tmp/test_folder/a.workflow')
self.wf.parse()
WorkflowRunner.download_actions(self.wf, False, False, '12345')
WorkflowRunner.instantiate_runners('vagrant', self.wf,
'/tmp/test_folder', False, False,
'12345')
self.runner = self.wf.action['sample action']['runner']
VagrantRunner.setup_vagrant_cache('12345')
def tearDown(self):
log.setLevel("NOTSET")
def setUp(self):
log.setLevel("CRITICAL")
self.maxDiff = None
def tearDown(self):
os.chdir('/tmp')
shutil.rmtree('/tmp/test_folder')
log.setLevel('NOTSET')
def tearDown(self):
os.chdir('/tmp')
shutil.rmtree('/tmp/test_folder')
shutil.rmtree(os.path.join(os.environ['HOME'], '.cache/.popper'))
log.setLevel('NOTSET')
def setUp(self):
os.makedirs('/tmp/test_folder')
os.chdir('/tmp/test_folder')
log.setLevel('CRITICAL')
def setUp(self):
log.setLevel("CRITICAL")
def setUp(self):
log.setLevel('CRITICAL')
def setUpClass(self):
log.setLevel("CRITICAL")
def tearDown(self):
self._kclient.api_client.rest_client.pool_manager.clear()
self._kclient.api_client.close()
log.setLevel("NOTSET")
def cli(
ctx,
step,
wfile,
debug,
dry_run,
log_file,
quiet,
reuse,
engine,
resource_manager,
skip,
skip_pull,
skip_clone,
substitution,
allow_loose,
workspace,
conf,
):
"""Runs a Popper workflow. Only executes STEP if given.
To specify a container engine to use other than docker, use the --engine/-e
flag. For executing on a resource manager such as SLURM or Kubernetes, use
the --resource-manager/-r flag. Alternatively, a configuration file can be
given (--conf flag) that can specify container options, resource manager
options, or both (see "Workflow Syntax and Execution Runtime" section of
the Popper documentation for more).
If the container engine (-e) or resource manager (-r) are specified with a
flag and a configuration file is given as well, the values passed via the
flags are given preference over those contained in the configuration file.
"""
# set the logging levels.
level = "STEP_INFO"
if quiet:
level = "INFO"
if debug:
level = "DEBUG"
log.setLevel(level)
if dry_run:
logging.msg_prefix = "DRYRUN: "
if log_file:
# also log to a file
logging.add_log(log, log_file)
# check conflicting flags and fail if needed
if skip and step:
log.fail("`--skip` can not be used when STEP argument is passed.")
# invoke wf factory; handles formats, validations, filtering
wf = WorkflowParser.parse(
wfile,
step=step,
skipped_steps=skip,
substitutions=substitution,
allow_loose=allow_loose,
)
config = ConfigLoader.load(
engine_name=engine,
resman_name=resource_manager,
config_file=conf,
reuse=reuse,
dry_run=dry_run,
skip_pull=skip_pull,
skip_clone=skip_clone,
workspace_dir=workspace,
)
with WorkflowRunner(config) as runner:
try:
runner.run(wf)
except Exception as e:
log.debug(traceback.format_exc())
log.fail(e)
def setUp(self):
log.setLevel("CRITICAL")
self.Popen = MockPopen()
replacer = Replacer()
replacer.replace("popper.runner_host.Popen", self.Popen)
self.addCleanup(replacer.restore)
def tearDown(self):
log.setLevel('NOTSET')
def setUp(self):
log.setLevel('CRITICAL')
self.maxDiff = None
