def analyze(project, logger): """ Applies the flake8 script to the sources of the given project. """ logger.info("Executing flake8 on project sources.") verbose = project.get_property("verbose") project.set_property_if_unset("flake8_verbose_output", verbose) command = ExternalCommandBuilder('flake8', project) command.use_argument('--ignore={0}').formatted_with_truthy_property('flake8_ignore') command.use_argument('--max-line-length={0}').formatted_with_property('flake8_max_line_length') command.use_argument('--filename={0}').formatted_with_truthy_property('flake8_include_patterns') command.use_argument('--exclude={0}').formatted_with_truthy_property('flake8_exclude_patterns') command.use_argument('--max-complexity={0}').formatted_with_truthy_property('flake8_max_complexity') include_test_sources = project.get_property("flake8_include_test_sources") include_scripts = project.get_property("flake8_include_scripts") result = command.run_on_production_source_files(logger, include_test_sources=include_test_sources, include_scripts=include_scripts, include_dirs_only=True) count_of_warnings = len(result.report_lines) count_of_errors = len(result.error_report_lines) if count_of_errors > 0: logger.error('Errors while running flake8. See %s for full details:\n%s' % ( result.error_report_file, tail_log(result.error_report_file))) if count_of_warnings > 0: if project.get_property("flake8_break_build"): error_message = "flake8 found {0} warning(s)".format(count_of_warnings) raise BuildFailedException(error_message) else: logger.warn("flake8 found %d warning(s).", count_of_warnings)
def analyze(project, logger): """ Applies the frosted script to the sources of the given project. """ logger.info("Executing frosted on project sources.") verbose = project.get_property("verbose") project.set_property_if_unset("frosted_verbose_output", verbose) command = ExternalCommandBuilder('frosted', project) for ignored_error_code in project.get_property('frosted_ignore', []): command.use_argument('--ignore={0}'.format(ignored_error_code)) include_test_sources = project.get_property("frosted_include_test_sources") include_scripts = project.get_property("frosted_include_scripts") result = command.run_on_production_source_files( logger, include_test_sources=include_test_sources, include_scripts=include_scripts) count_of_warnings = len(result.report_lines) count_of_errors = len(result.error_report_lines) if count_of_errors > 0: logger.error('Errors while running frosted, see {0}'.format( result.error_report_file)) if count_of_warnings > 0: if project.get_property("frosted_break_build"): error_message = "frosted found {0} warning(s)".format( count_of_warnings) raise BuildFailedException(error_message) else: logger.warn("frosted found %d warning(s).", count_of_warnings)
def analyze(project, logger): """ Applies the frosted script to the sources of the given project. """ logger.info("Executing frosted on project sources.") verbose = project.get_property("verbose") project.set_property_if_unset("frosted_verbose_output", verbose) command = ExternalCommandBuilder("frosted", project) for ignored_error_code in project.get_property("frosted_ignore", []): command.use_argument("--ignore={0}".format(ignored_error_code)) include_test_sources = project.get_property("frosted_include_test_sources") include_scripts = project.get_property("frosted_include_scripts") result = command.run_on_production_source_files( logger, include_test_sources=include_test_sources, include_scripts=include_scripts ) count_of_warnings = len(result.report_lines) count_of_errors = len(result.error_report_lines) if count_of_errors > 0: logger.error("Errors while running frosted, see {0}".format(result.error_report_file)) if count_of_warnings > 0: if project.get_property("frosted_break_build"): error_message = "frosted found {0} warning(s)".format(count_of_warnings) raise BuildFailedException(error_message) else: logger.warn("frosted found %d warning(s).", count_of_warnings)
def _exec_cmd( project: Project, logger: Logger, reactor: Reactor, program: str, *arguments: str, message: str = None, error: str = None, report_file: str = None, verbose_property: str = None, force_log: bool = False, ) -> Optional[ExternalCommandResult]: report_folder = _make_folder(project, "$dir_reports", "docker") report_file = report_file or "_".join([program, *arguments]) command = ExternalCommandBuilder(program, project, reactor) for argument in arguments: command.use_argument(argument) if message: logger.info(message) result = command.run(f"{report_folder}/{report_file}") if result.exit_code == 0: return result is_verbose = project.get_property("verbose", False) is_verbose_property = verbose_property and project.get_property( verbose_property, False) if force_log or is_verbose or is_verbose_property: logger.error(result.error_report_lines) if error: raise Exception(error)
def positive_test(project, logger): print("Running a postive test") command = ExternalCommandBuilder('hydra', project) command.use_argument('positive-test') result = command.run_on_production_source_files(logger) if result.exit_code: raise BuildFailedException("Exit code is set")
def analyze(project, logger): """ Applies the flake8 script to the sources of the given project. """ logger.info("Executing flake8 on project sources.") verbose = project.get_property("verbose") project.set_property_if_unset("flake8_verbose_output", verbose) command = ExternalCommandBuilder('flake8', project) command.use_argument('--ignore={0}').formatted_with_truthy_property('flake8_ignore') command.use_argument('--max-line-length={0}').formatted_with_property('flake8_max_line_length') command.use_argument('--exclude={0}').formatted_with_truthy_property('flake8_exclude_patterns') include_test_sources = project.get_property("flake8_include_test_sources") include_scripts = project.get_property("flake8_include_scripts") result = command.run_on_production_source_files(logger, include_test_sources=include_test_sources, include_scripts=include_scripts) count_of_warnings = len(result.report_lines) count_of_errors = len(result.error_report_lines) if count_of_errors > 0: logger.error('Errors while running flake8, see {0}'.format(result.error_report_file)) if count_of_warnings > 0: if project.get_property("flake8_break_build"): error_message = "flake8 found {0} warning(s)".format(count_of_warnings) raise BuildFailedException(error_message) else: logger.warn("flake8 found %d warning(s).", count_of_warnings)
class ExternalCommandExecutionTests(unittest.TestCase): def setUp(self): self.project = Project('/base/dir') self.command = ExternalCommandBuilder('command-name', self.project) self.command.use_argument('--foo').use_argument('--bar') @patch('pybuilder.pluginhelper.external_command.read_file') @patch('pybuilder.pluginhelper.external_command.execute_tool_on_source_files') def test_should_execute_external_command_on_production_source_files(self, execution, read): execution.return_value = 0, '/tmp/reports/command-name' logger = Mock() self.command.run_on_production_source_files(logger) execution.assert_called_with( include_test_sources=False, include_scripts=False, project=self.project, logger=logger, command_and_arguments=['command-name', '--foo', '--bar'], name='command-name') @patch('pybuilder.pluginhelper.external_command.read_file') @patch('pybuilder.pluginhelper.external_command.execute_tool_on_source_files') def test_should_execute_external_command_on_production_and_test_source_files(self, execution, read): execution.return_value = 0, '/tmp/reports/command-name' logger = Mock() self.command.run_on_production_and_test_source_files(logger) execution.assert_called_with( include_test_sources=True, include_scripts=False, project=self.project, logger=logger, command_and_arguments=['command-name', '--foo', '--bar'], name='command-name') @patch('pybuilder.pluginhelper.external_command.read_file') @patch('pybuilder.pluginhelper.external_command.execute_tool_on_source_files') def test_should_execute_external_command_and_return_execution_result(self, execution, read): execution.return_value = 0, '/tmp/reports/command-name' read.side_effect = lambda argument: { '/tmp/reports/command-name': ['Running...', 'OK all done!'], '/tmp/reports/command-name.err': ['Oh no! I am not python8 compatible!', 'I will explode now.'] }[argument] logger = Mock() result = self.command.run_on_production_source_files(logger) self.assertEqual(result.exit_code, 0) self.assertEqual(result.report_file, '/tmp/reports/command-name') self.assertEqual(read.call_args_list[0], call('/tmp/reports/command-name')) self.assertEqual(result.report_lines, ['Running...', 'OK all done!']) self.assertEqual(result.error_report_file, '/tmp/reports/command-name.err') self.assertEqual(read.call_args_list[1], call('/tmp/reports/command-name.err')) self.assertEqual(result.error_report_lines, ['Oh no! I am not python8 compatible!', 'I will explode now.'])
def _run_push_cmd(project, remote_img, logger): logger.info("Pushing remote docker image - {}".format(remote_img)) report_dir = prepare_reports_directory(project) command = ExternalCommandBuilder('docker', project) command.use_argument('push') command.use_argument('{0}').formatted_with(remote_img) res = command.run("{}/{}".format(report_dir, 'docker_push_tag')) if res.exit_code > 0: logger.info(res.error_report_lines) raise Exception( "Error pushing image to remote registry - {}".format(remote_img))
def cyclomatic_complexity(project, logger): try: command = ExternalCommandBuilder('radon', project) command.use_argument('cc') command.use_argument('-a') result = command.run_on_production_source_files(logger) if len(result.error_report_lines) > 0: logger.error('Errors while running radon, see {0}'.format(result.error_report_file)) for line in result.report_lines[:-1]: logger.debug(line.strip()) if not result.report_lines: return average_complexity_line = result.report_lines[-1].strip() logger.info(average_complexity_line) except Exception as exception: print('ERROR: unable to execute cyclomatic complexity due to: {}'.format(str(exception)))
def cyclomatic_complexity(project, logger): command = ExternalCommandBuilder('radon', project) command.use_argument('cc') command.use_argument('-a') result = command.run_on_production_source_files(logger) if len(result.error_report_lines) > 0: logger.error('Errors while running radon, see {0}'.format( result.error_report_file)) for line in result.report_lines[:-1]: logger.debug(line.strip()) average_complexity_line = result.report_lines[-1].strip() logger.info(average_complexity_line)
def _run_tag_cmd(project, local_img, remote_img, logger): logger.info("Tagging local docker image {} - {}".format( local_img, remote_img)) report_dir = prepare_reports_directory(project) command = ExternalCommandBuilder('docker', project) command.use_argument('tag') command.use_argument('{0}').formatted_with(local_img) command.use_argument('{0}').formatted_with(remote_img) command.run("{}/{}".format(report_dir, 'docker_push_tag'))
class ExternalCommandBuilderTests(unittest.TestCase): def setUp(self): self.project = Project('/base/dir') self.reactor = Mock() pyb_env = Mock() self.reactor.python_env_registry = {"pybuilder": pyb_env} self.command = ExternalCommandBuilder('command-name', self.project, self.reactor) def test_should_only_use_command_name_by_default(self): self.assertEqual(self.command.as_string, 'command-name') def test_should_add_unconditional_argument_to_command(self): self.command.use_argument('--foo=bar') self.assertEqual(self.command.as_string, 'command-name --foo=bar') def test_should_add_conditional_argument_when_property_is_truthy(self): self.project.set_property('verbose', True) self.command.use_argument('--verbose').only_if_property_is_truthy( 'verbose') self.assertEqual(self.command.as_string, 'command-name --verbose') def test_should_not_add_conditional_argument_when_property_is_falsy(self): self.project.set_property('verbose', False) self.command.use_argument('--verbose').only_if_property_is_truthy( 'verbose') self.assertEqual(self.command.as_string, 'command-name') def test_should_add_conditional_argument_when_property_is_truthy_after_unconditional_argument( self): self.project.set_property('verbose', True) self.command.use_argument('--cool').use_argument( '--verbose').only_if_property_is_truthy('verbose') self.assertEqual(self.command.as_string, 'command-name --cool --verbose') def test_should_not_add_conditional_argument_when_property_is_falsy_after_unconditional_argument( self): self.project.set_property('verbose', False) self.command.use_argument('--cool').use_argument( '--verbose').only_if_property_is_truthy('verbose') self.assertEqual(self.command.as_string, 'command-name --cool') def test_should_format_unconditional_argument_with_property_when_given( self): self.project.set_property('name', 'value') self.command.use_argument('--name={0}').formatted_with_property('name') self.assertEqual(self.command.as_string, 'command-name --name=value') def test_should_format_unconditional_argument_with_value_when_given(self): self.command.use_argument('--name={0}').formatted_with('value') self.assertEqual(self.command.as_string, 'command-name --name=value') def test_should_include_conditional_argument_with_formatting_when_property_is_falsy( self): self.project.set_property('name', 'value') self.command.use_argument('--name={0}').formatted_with_property( 'name').only_if_property_is_truthy('name') self.assertEqual(self.command.as_string, 'command-name --name=value') def test_should_omit_conditional_argument_with_formatting_when_property_is_falsy( self): self.project.set_property('name', 'value') self.project.set_property('falsy', None) self.command.use_argument('--name={0}').formatted_with_property( 'name').only_if_property_is_truthy('falsy') self.assertEqual(self.command.as_string, 'command-name') def test_should_include_conditional_argument_with_truthy_formatting(self): self.project.set_property('name', 'value') self.command.use_argument('--name={0}').formatted_with_truthy_property( 'name') self.assertEqual(self.command.as_string, 'command-name --name=value') def test_should_omit_conditional_argument_with_falsy_formatting(self): self.project.set_property('name', None) self.command.use_argument('--name={0}').formatted_with_truthy_property( 'name') self.assertEqual(self.command.as_string, 'command-name')
class ExternalCommandExecutionTests(unittest.TestCase): def setUp(self): self.project = Project('/base/dir') self.reactor = Mock() pyb_env = Mock() self.reactor.python_env_registry = {"pybuilder": pyb_env} self.reactor.pybuilder_venv = pyb_env self.command = ExternalCommandBuilder('command-name', self.project, self.reactor) self.command.use_argument('--foo').use_argument('--bar') @patch("pybuilder.pluginhelper.external_command.read_file") def test_should_execute_external_command(self, _): self.command.run("any-outfile-name") self.reactor.pybuilder_venv.execute_command.assert_called_with( ['command-name', '--foo', '--bar'], 'any-outfile-name') @patch('pybuilder.pluginhelper.external_command.read_file') @patch( 'pybuilder.pluginhelper.external_command.execute_tool_on_source_files') def test_should_execute_external_command_on_production_source_files_dirs_only( self, execution, read): execution.return_value = 0, '/tmp/reports/command-name' logger = Mock() self.command.run_on_production_source_files(logger, include_dirs_only=True) execution.assert_called_with( python_env=self.reactor.pybuilder_venv, include_dirs_only=True, include_test_sources=False, include_scripts=False, project=self.project, logger=logger, command_and_arguments=['command-name', '--foo', '--bar'], name='command-name') @patch('pybuilder.pluginhelper.external_command.read_file') @patch( 'pybuilder.pluginhelper.external_command.execute_tool_on_source_files') def test_should_execute_external_command_on_production_source_files( self, execution, read): execution.return_value = 0, '/tmp/reports/command-name' logger = Mock() self.command.run_on_production_source_files(logger) execution.assert_called_with( python_env=self.reactor.pybuilder_venv, include_dirs_only=False, include_test_sources=False, include_scripts=False, project=self.project, logger=logger, command_and_arguments=['command-name', '--foo', '--bar'], name='command-name') @patch('pybuilder.pluginhelper.external_command.read_file') @patch( 'pybuilder.pluginhelper.external_command.execute_tool_on_source_files') def test_should_execute_external_command_on_production_and_test_source_files( self, execution, read): execution.return_value = 0, '/tmp/reports/command-name' logger = Mock() self.command.run_on_production_and_test_source_files(logger) execution.assert_called_with( python_env=self.reactor.pybuilder_venv, include_dirs_only=False, include_test_sources=True, include_scripts=False, project=self.project, logger=logger, command_and_arguments=['command-name', '--foo', '--bar'], name='command-name') @patch('pybuilder.pluginhelper.external_command.read_file') @patch( 'pybuilder.pluginhelper.external_command.execute_tool_on_source_files') def test_should_execute_external_command_and_return_execution_result( self, execution, read): execution.return_value = 0, '/tmp/reports/command-name' read.side_effect = lambda argument: { '/tmp/reports/command-name': ['Running...', 'OK all done!'], '/tmp/reports/command-name.err': ['Oh no! I am not python8 compatible!', 'I will explode now.'] }[argument] logger = Mock() result = self.command.run_on_production_source_files(logger) self.assertEqual(result.exit_code, 0) self.assertEqual(result.report_file, '/tmp/reports/command-name') self.assertEqual(read.call_args_list[0], call('/tmp/reports/command-name')) self.assertEqual(result.report_lines, ['Running...', 'OK all done!']) self.assertEqual(result.error_report_file, '/tmp/reports/command-name.err') self.assertEqual(read.call_args_list[1], call('/tmp/reports/command-name.err')) self.assertEqual( result.error_report_lines, ['Oh no! I am not python8 compatible!', 'I will explode now.'])
class ExternalCommandBuilderTests(unittest.TestCase): def setUp(self): self.project = Project('/base/dir') self.command = ExternalCommandBuilder('command-name', self.project) def test_should_only_use_command_name_by_default(self): self.assertEqual(self.command.as_string, 'command-name') def test_should_add_unconditional_argument_to_command(self): self.command.use_argument('--foo=bar') self.assertEqual(self.command.as_string, 'command-name --foo=bar') def test_should_add_conditional_argument_when_property_is_truthy(self): self.project.set_property('verbose', True) self.command.use_argument('--verbose').only_if_property_is_truthy('verbose') self.assertEqual(self.command.as_string, 'command-name --verbose') def test_should_not_add_conditional_argument_when_property_is_falsy(self): self.project.set_property('verbose', False) self.command.use_argument('--verbose').only_if_property_is_truthy('verbose') self.assertEqual(self.command.as_string, 'command-name') def test_should_add_conditional_argument_when_property_is_truthy_after_unconditional_argument(self): self.project.set_property('verbose', True) self.command.use_argument('--cool').use_argument('--verbose').only_if_property_is_truthy('verbose') self.assertEqual(self.command.as_string, 'command-name --cool --verbose') def test_should_not_add_conditional_argument_when_property_is_falsy_after_unconditional_argument(self): self.project.set_property('verbose', False) self.command.use_argument('--cool').use_argument('--verbose').only_if_property_is_truthy('verbose') self.assertEqual(self.command.as_string, 'command-name --cool') def test_should_format_unconditional_argument_with_property_when_given(self): self.project.set_property('name', 'value') self.command.use_argument('--name={0}').formatted_with_property('name') self.assertEqual(self.command.as_string, 'command-name --name=value') def test_should_format_unconditional_argument_with_value_when_given(self): self.command.use_argument('--name={0}').formatted_with('value') self.assertEqual(self.command.as_string, 'command-name --name=value') def test_should_include_conditional_argument_with_formatting_when_property_is_falsy(self): self.project.set_property('name', 'value') self.command.use_argument('--name={0}').formatted_with_property('name').only_if_property_is_truthy('name') self.assertEqual(self.command.as_string, 'command-name --name=value') def test_should_omit_conditional_argument_with_formatting_when_property_is_falsy(self): self.project.set_property('name', 'value') self.project.set_property('falsy', None) self.command.use_argument('--name={0}').formatted_with_property('name').only_if_property_is_truthy('falsy') self.assertEqual(self.command.as_string, 'command-name') def test_should_include_conditional_argument_with_truthy_formatting(self): self.project.set_property('name', 'value') self.command.use_argument('--name={0}').formatted_with_truthy_property('name') self.assertEqual(self.command.as_string, 'command-name --name=value') def test_should_omit_conditional_argument_with_falsy_formatting(self): self.project.set_property('name', None) self.command.use_argument('--name={0}').formatted_with_truthy_property('name') self.assertEqual(self.command.as_string, 'command-name')
def analyze(project, logger, reactor): """ Applies the flake8 script to the sources of the given project. """ logger.info("Executing flake8 on project sources.") verbose = project.get_property("verbose") project.set_property_if_unset("flake8_verbose_output", verbose) command = ExternalCommandBuilder("flake8", project, reactor) command.use_argument("--ignore={0}").formatted_with_truthy_property( "flake8_ignore") command.use_argument("--extend-ignore={0}").formatted_with_truthy_property( "flake8_extend_ignore") command.use_argument("--max-line-length={0}").formatted_with_property( "flake8_max_line_length") command.use_argument("--filename={0}").formatted_with_truthy_property( "flake8_include_patterns") command.use_argument("--exclude={0}").formatted_with_truthy_property( "flake8_exclude_patterns") command.use_argument( "--max-complexity={0}").formatted_with_truthy_property( "flake8_max_complexity") include_test_sources = project.get_property("flake8_include_test_sources") include_scripts = project.get_property("flake8_include_scripts") result = command.run_on_production_source_files( logger, include_test_sources=include_test_sources, include_scripts=include_scripts, include_dirs_only=True) count_of_warnings = len(result.report_lines) count_of_errors = len(result.error_report_lines) if count_of_errors > 0: logger.error( "Errors while running flake8. See %s for full details:\n%s" % (result.error_report_file, tail_log(result.error_report_file))) if count_of_warnings > 0: if project.get_property("flake8_break_build"): error_message = "flake8 found {0} warning(s)".format( count_of_warnings) raise BuildFailedException(error_message) else: logger.warn("flake8 found %d warning(s).", count_of_warnings)
def _create_ecr_registry(fq_artifact, project): command = ExternalCommandBuilder('aws', project) command.use_argument('ecr') command.use_argument('describe-repositories') command.use_argument('--repository-names') command.use_argument('{0}').formatted_with(fq_artifact) res = command.run("{}/{}".format(prepare_reports_directory(project), 'docker_ecr_registry_discover')) if res.exit_code > 0: command = ExternalCommandBuilder('aws', project) command.use_argument('ecr') command.use_argument('create-repository') command.use_argument('--repository-name') command.use_argument('{0}').formatted_with(fq_artifact) res = command.run("{}/{}".format(prepare_reports_directory(project), 'docker_ecr_registry_create')) if res.exit_code > 0: raise Exception("Unable to create ecr registry")
def _ecr_login(project, registry): command = ExternalCommandBuilder('aws', project) command.use_argument('ecr') command.use_argument('get-authorization-token') command.use_argument('--output') command.use_argument('text') command.use_argument('--query') command.use_argument('authorizationData[].authorizationToken') res = command.run("{}/{}".format(prepare_reports_directory(project), 'docker_ecr_get_token')) if res.exit_code > 0: raise Exception("Error getting token") pass_token = base64.b64decode(res.report_lines[0]) split = pass_token.split(":") command = ExternalCommandBuilder('docker', project) command.use_argument('login') command.use_argument('-u') command.use_argument('{0}').formatted_with(split[0]) command.use_argument('-p') command.use_argument('{0}').formatted_with(split[1]) command.use_argument('{0}').formatted_with(registry) res = command.run("{}/{}".format(prepare_reports_directory(project), 'docker_ecr_docker_login')) if res.exit_code > 0: raise Exception("Error authenticating")
def do_docker_package(project, logger): project.set_property_if_unset("docker_package_build_dir", "src/main/docker") project.set_property_if_unset("docker_package_build_image", project.name) project.set_property_if_unset("docker_package_build_version", project.version) report_dir = prepare_reports_directory(project) dist_dir = prepare_dist_directory(project) assert_can_execute(["docker", "--version"], prerequisite="docker", caller="docker_package") # is true if user set verbose in build.py or from command line verbose = project.get_property("verbose") project.set_property_if_unset("docker_package_verbose_output", verbose) temp_build_img = 'pyb-temp-{}:{}'.format(project.name, project.version) build_img = get_build_img(project) logger.info("Executing primary stage docker build for image - {}.".format( build_img)) # docker build --build-arg buildVersion=${BUILD_NUMBER} -t ${BUILD_IMG} src/ command = ExternalCommandBuilder('docker', project) command.use_argument('build') command.use_argument('--build-arg') command.use_argument('buildVersion={0}').formatted_with_property( 'docker_package_build_version') command.use_argument('-t') command.use_argument('{0}').formatted_with(temp_build_img) command.use_argument('{0}').formatted_with_property( 'docker_package_build_dir') result = command.run("{}/{}".format(report_dir, 'docker_package_build')) if result.exit_code != 0: logger.error(result.error_report_lines) raise Exception("Error building primary stage docker image") write_docker_build_file(project=project, logger=logger, build_image=temp_build_img, dist_dir=dist_dir) copy_dist_file(project=project, dist_dir=dist_dir, logger=logger) logger.info( "Executing secondary stage docker build for image - {}.".format( build_img)) command = ExternalCommandBuilder('docker', project) command.use_argument('build') command.use_argument('-t') command.use_argument('{0}').formatted_with(build_img) command.use_argument('{0}').formatted_with(dist_dir) result = command.run("{}/{}".format(report_dir, 'docker_package_img')) if result.exit_code != 0: logger.error(result.error_report_lines) raise Exception("Error building docker image") logger.info( "Finished build docker image - {} - with dist file - {}".format( build_img, dist_dir))