def prepare_workflow_execution(recursive=False, **kwargs): """Set parameters for the workflow execution and run the workflow.""" # Set the logging levels. level = 'ACTION_INFO' if kwargs['quiet']: level = 'INFO' if kwargs['debug']: level = 'DEBUG' log.setLevel(level) if kwargs['log_file']: logging.add_log(log, kwargs['log_file']) # Remove the unnecessary kwargs. kwargs.pop('quiet') kwargs.pop('debug') kwargs.pop('log_file') # Run the workflow accordingly as recursive/CI and Non-CI. if recursive: for wfile in pu.find_recursive_wfile(): kwargs['wfile'] = wfile run_workflow(**kwargs) else: run_workflow(**kwargs)
def setUp(self): log.setLevel('CRITICAL') self.Popen = MockPopen() replacer = Replacer() replacer.replace('popper.runner_host.Popen', self.Popen) self.addCleanup(replacer.restore) self.repo = tempfile.mkdtemp()
def tearDownClass(self): log.setLevel('NOTSET') # return to where we were before this test os.chdir(self.curr_dir) self.repo.close()
def setUp(self): log.setLevel("CRITICAL") config.load_kube_config() c = Configuration() c.assert_hostname = False Configuration.set_default(c) self._kclient = core_v1_api.CoreV1Api() _, active_context = config.list_kube_config_contexts()
def setUp(self): log.setLevel("CRITICAL") self.Popen = MockPopen() replacer = Replacer() replacer.replace("popper.runner_host.Popen", self.Popen) assert_replacer = Replacer() assert_replacer.replace( "popper.runner_slurm.assert_executable_exists", mock_assert_executable_exists, ) self.addCleanup(replacer.restore) self.addCleanup(assert_replacer.restore)
def setUpClass(self): log.setLevel('CRITICAL') self.tempdir = tempfile.mkdtemp() self.curr_dir = os.getcwd() self.repo = git.Repo.clone_from('https://github.com/popperized/bin', os.path.join(self.tempdir, 'bin')) self.repodir = os.path.join(self.tempdir, 'bin') os.chdir(self.repodir) self.gitdir = os.path.join(self.repodir, '.git') self.gotdor = os.path.join(self.repodir, '.got')
def setUp(self): if os.environ.get('POPPER_TEST_MODE') == 'with-git': self.with_git = True else: self.with_git = False log.setLevel('CRITICAL') if os.path.exists('/tmp/test_folder'): shutil.rmtree('/tmp/test_folder') os.makedirs('/tmp/test_folder') os.chdir('/tmp/test_folder') scm.clone('https://github.com', 'cplee', 'github-actions-demo', os.path.join(os.getcwd(), 'github-actions-demo')) if not self.with_git: shutil.rmtree('/tmp/test_folder/github-actions-demo/.git') os.chdir('/tmp/test_folder/github-actions-demo')
def setUp(self): os.makedirs('/tmp/test_folder') os.chdir('/tmp/test_folder') log.setLevel('CRITICAL') workflow = """ workflow "sample" { resolves = "sample action" } action "sample action" { uses = "sh" args = ["echo", "Hello"] } """ pu.write_file('/tmp/test_folder/a.workflow', workflow) self.wf = Workflow('/tmp/test_folder/a.workflow') self.wf.parse() WorkflowRunner.instantiate_runners('docker', self.wf, '/tmp/test_folder', False, False, '12345')
def cli(ctx, action, wfile, workspace, reuse, recursive, quiet, debug, dry_run, parallel, log_file): """Executes one or more pipelines and reports on their status. """ popper.scm.get_git_root_folder() level = 'ACTION_INFO' if quiet: level = 'INFO' if debug: level = 'DEBUG' log.setLevel(level) if log_file: logging.add_log(log, log_file) if recursive: wfile_list = pu.find_recursive_wfile() if not wfile_list: log.fail("Recursive search couldn't find any .workflow files ") for wfile in wfile_list: log.info("Found and running workflow at " + wfile) run_pipeline(action, wfile, workspace, reuse, dry_run, parallel) else: run_pipeline(action, wfile, workspace, reuse, dry_run, parallel)
def cli(ctx, action, wfile, skip_clone, skip_pull, skip, workspace, reuse, recursive, quiet, debug, dry_run, parallel, log_file, with_dependencies, on_failure): """Executes one or more pipelines and reports on their status. """ popper.scm.get_git_root_folder() level = 'ACTION_INFO' if quiet: level = 'INFO' if debug: level = 'DEBUG' log.setLevel(level) if log_file: logging.add_log(log, log_file) if os.environ.get('CI') == 'true': log.info("Running in CI environment.") if recursive: log.warning('When CI variable is set, --recursive is ignored.') wfile_list = pu.find_recursive_wfile() wfile_list = workflows_from_commit_message(wfile_list) else: if recursive: if action: log.fail( "An 'action' argument and the --recursive flag cannot be " "both given.") wfile_list = pu.find_recursive_wfile() else: wfile_list = [wfile] if not wfile_list: log.fail("No workflow to execute.") for wfile in wfile_list: wfile = pu.find_default_wfile(wfile) log.info("Found and running workflow at " + wfile) run_pipeline(action, wfile, skip_clone, skip_pull, skip, workspace, reuse, dry_run, parallel, with_dependencies, on_failure)
def setUp(self): os.makedirs('/tmp/test_folder') os.chdir('/tmp/test_folder') log.setLevel('CRITICAL') workflow = """ workflow "sample" { resolves = "sample action" } action "sample action" { uses = "popperized/bin/sh@master" args = ["echo", "Hello"] } """ pu.write_file('/tmp/test_folder/a.workflow', workflow) self.wf = Workflow('/tmp/test_folder/a.workflow') self.wf.parse() WorkflowRunner.download_actions(self.wf, False, False, '12345') WorkflowRunner.instantiate_runners('vagrant', self.wf, '/tmp/test_folder', False, False, '12345') self.runner = self.wf.action['sample action']['runner'] VagrantRunner.setup_vagrant_cache('12345')
def tearDown(self): log.setLevel("NOTSET")
def setUp(self): log.setLevel("CRITICAL") self.maxDiff = None
def tearDown(self): os.chdir('/tmp') shutil.rmtree('/tmp/test_folder') log.setLevel('NOTSET')
def tearDown(self): os.chdir('/tmp') shutil.rmtree('/tmp/test_folder') shutil.rmtree(os.path.join(os.environ['HOME'], '.cache/.popper')) log.setLevel('NOTSET')
def setUp(self): os.makedirs('/tmp/test_folder') os.chdir('/tmp/test_folder') log.setLevel('CRITICAL')
def setUp(self): log.setLevel("CRITICAL")
def setUp(self): log.setLevel('CRITICAL')
def setUpClass(self): log.setLevel("CRITICAL")
def tearDown(self): self._kclient.api_client.rest_client.pool_manager.clear() self._kclient.api_client.close() log.setLevel("NOTSET")
def cli( ctx, step, wfile, debug, dry_run, log_file, quiet, reuse, engine, resource_manager, skip, skip_pull, skip_clone, substitution, allow_loose, workspace, conf, ): """Runs a Popper workflow. Only executes STEP if given. To specify a container engine to use other than docker, use the --engine/-e flag. For executing on a resource manager such as SLURM or Kubernetes, use the --resource-manager/-r flag. Alternatively, a configuration file can be given (--conf flag) that can specify container options, resource manager options, or both (see "Workflow Syntax and Execution Runtime" section of the Popper documentation for more). If the container engine (-e) or resource manager (-r) are specified with a flag and a configuration file is given as well, the values passed via the flags are given preference over those contained in the configuration file. """ # set the logging levels. level = "STEP_INFO" if quiet: level = "INFO" if debug: level = "DEBUG" log.setLevel(level) if dry_run: logging.msg_prefix = "DRYRUN: " if log_file: # also log to a file logging.add_log(log, log_file) # check conflicting flags and fail if needed if skip and step: log.fail("`--skip` can not be used when STEP argument is passed.") # invoke wf factory; handles formats, validations, filtering wf = WorkflowParser.parse( wfile, step=step, skipped_steps=skip, substitutions=substitution, allow_loose=allow_loose, ) config = ConfigLoader.load( engine_name=engine, resman_name=resource_manager, config_file=conf, reuse=reuse, dry_run=dry_run, skip_pull=skip_pull, skip_clone=skip_clone, workspace_dir=workspace, ) with WorkflowRunner(config) as runner: try: runner.run(wf) except Exception as e: log.debug(traceback.format_exc()) log.fail(e)
def setUp(self): log.setLevel("CRITICAL") self.Popen = MockPopen() replacer = Replacer() replacer.replace("popper.runner_host.Popen", self.Popen) self.addCleanup(replacer.restore)
def tearDown(self): log.setLevel('NOTSET')
def setUp(self): log.setLevel('CRITICAL') self.maxDiff = None