def uninstall_pkg(pkg_name): """ Uninstall rpm/deb package by name :param pkg_name: name of pkg to uninstall :type: String :return: Flag whether pkg uninstalled :rtype: bool """ if not is_pkg_installed(pkg_name): return True configure_logger('package_manager.uninstall_pkg') log = logging.getLogger() cmd = _CMD_PATTERN["UNINSTALL"].get( get_os_name()).format(pkg_name=pkg_name) err, out = cmd_exec(cmd) if err == 0: log.debug(out) return True log.info(out) return False
def __init__(self, build_artifacts_dir, tests_artifacts_dir, tests_artifacts_url, root_dir): """ :param build_artifacts_dir: Path to build artifacts :type build_artifacts_dir: pathlib.Path :param tests_artifacts_dir: Path to tests artifacts :type tests_artifacts_dir: pathlib.Path :param tests_artifacts_url: URL to tests artifacts :type tests_artifacts_url: String :param root_dir: Path to workdir for unpacking build artifacts :type root_dir: pathlib.Path """ # Clean workdir and re-create it self._remove(str(root_dir)) self._mkdir(str(root_dir)) self.build_artifacts_dir = build_artifacts_dir self.tests_artifacts_dir = tests_artifacts_dir self.tests_artifacts_url = tests_artifacts_url self.root_dir = root_dir self.env = os.environ.copy() # Path to dispatcher lib should be in the libraries search path self.env['LD_LIBRARY_PATH'] = self.dispatcher_dir configure_logger(logs_path=self.test_adapter_log_dir / 'test_adapter.log') self.log = logging.getLogger(self.test_adapter_log_name)
def main(): """ Arguments parser and test executor """ parser = argparse.ArgumentParser( prog="run_test.py", formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('id', help="Id of a test") args = parser.parse_args() configure_logger() test_info = TESTS.get(args.id, None) if not test_info: test_info.log.error(f'{args.id} does not exist') exit(ErrorCode.CRITICAL) os.environ['DISPLAY'] = ":0.0" test = Test(args.id, test_info) result = test.run() test.log.info('#' * 80) if not result: test.log.error('TEST FAILED') else: test.log.info('TEST PASSED') test.log.info('#' * 80) exit(not result)
def main(): """ Arguments parser and manifest creator """ argparse.ArgumentParser() parser = argparse.ArgumentParser( prog=pathlib.Path(__file__).name, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("--root-dir", metavar="PATH", default='.', help=f"Path where repository will be stored, " f"by default it is current directory") parser.add_argument("--repo", metavar="String", required=True, help=f"Repository name") parser.add_argument("--branch", metavar="String", default='master', help='Branch name, by default "master" branch') parser.add_argument("--revision", metavar="String", default='HEAD', help='SHA of commit') parser.add_argument("--target-branch", metavar="String", help='Target branch name') parser.add_argument( "--build-event", default=Build_event.PRE_COMMIT.value, choices=[build_event.value for build_event in Build_event], help='Event of build') parser.add_argument("--commit-time", metavar="String", help='Will switch to the commit before specified time') args = parser.parse_args() configure_logger() log = logging.getLogger('manifest_runner') log.info('Manifest preparing started') try: manifest_runner = ManifestRunner(**vars(args)) manifest_runner.run() log.info('Manifest preparing completed') except Exception: log.exception('Exception occurred:') log.info('Manifest preparing failed') exit(ErrorCode.CRITICAL.value)
def _vs_component(self, name, solution_path, msbuild_args=None, vs_version="vs2017", dependencies=None, env=None, verbose=False): """ Handler for VS components :param name: Name of action :type name: String :param solution_path: Path to solution file :type solution_path: pathlib.Path :param msbuild_args: Arguments of 'msbuild' :type msbuild_args: Dictionary :param vs_version: Version of Visual Studio :type vs_version: String :param dependencies: Dependency of other actions :type dependencies: List :param env: Environment variables for script :type env: None | Dict :return: None | Exception """ ms_arguments = deepcopy( self._config_variables.get('MSBUILD_ARGUMENTS', {})) if msbuild_args: for key, value in msbuild_args.items(): if isinstance(value, dict): ms_arguments[key] = { **ms_arguments.get(key, {}), **msbuild_args[key] } else: ms_arguments[key] = msbuild_args[key] if self._current_stage == Stage.BUILD.value: configure_logger( name, self._options['LOGS_DIR'] / 'build' / f'{name}.log') self._actions[Stage.BUILD.value].append( VsComponent(name, solution_path, ms_arguments, vs_version, dependencies, env, verbose))
def main(): """ Command line API for extracting repo """ parser = argparse.ArgumentParser( prog="git_worker.py", formatter_class=argparse.RawTextHelpFormatter) parser.add_argument( "--root-dir", metavar="PATH", default='.', help= f"Path where repository will be stored, by default it is current directory" ) parser.add_argument( "--repo-name", metavar="PATH", required=True, help= f"Path where repository will be stored, by default it is current directory" ) parser.add_argument("--url", metavar="String", default=None, help='Url link to repository') parser.add_argument("--branch", metavar="String", default='master', help='Branch name, by default "master" branch') parser.add_argument("--commit-id", metavar="String", default='HEAD', help='SHA of commit') parser.add_argument("--commit-time", metavar="String", default=None, help='Will switch to the commit before specified time') parser.add_argument("--proxy", metavar="String", default=False, help='Will switch to the commit before specified time') args = parser.parse_args() configure_logger() extract_repo(**vars(args))
def main(): parser = argparse.ArgumentParser(prog="component_checker.py", description='Checks existence of component on share', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('-p', '--path-to-manifest', help='Path to manifest file', required=True) parser.add_argument('-c', '--component-name', help='Component name to check', required=True) args = parser.parse_args() configure_logger() log = logging.getLogger('component_checker.main') try: check_component_existence(args.path_to_manifest, args.component_name) except Exception: log.exception('Exception occurred') exit(ErrorCode.CRITICAL.value) exit(0)
def main(): parser = argparse.ArgumentParser() parser.add_argument("-p", "--repo-path", metavar="String", help="Path to repository") parser.add_argument("-r", "--revision", metavar="String", help="Revision to check") args = parser.parse_args() logger_conf.configure_logger() checker = Checker(repo_path=args.repo_path, revision=args.revision) if checker.check(): exit(0) exit(ErrorCode.CRITICAL.value)
def _action(self, name, stage=None, cmd=None, work_dir=None, env=None, callfunc=None, verbose=False): """ Handler for 'action' from build config file :param name: Name of action :type name: String :param stage: Stage type :type stage: Stage :param cmd: command line script :type cmd: None | String :param work_dir: Path where script will execute :type work_dir: None | pathlib.Path :param env: Environment variables for script :type env: None | Dict :param callfunc: python function, which need to execute :type callfunc: tuple (function_name, args, kwargs) | None :return: None | Exception """ if not stage: stage = Stage.BUILD.value else: stage = stage.value if not work_dir: work_dir = self._options["ROOT_DIR"] if stage in [Stage.BUILD.value, Stage.INSTALL.value]: work_dir = self._options["BUILD_DIR"] if stage == Stage.BUILD.value and self._current_stage == Stage.BUILD.value: configure_logger( name, self._options['LOGS_DIR'] / 'build' / f'{name}.log') self._actions[stage].append( Action(name, stage, cmd, work_dir, env, callfunc, verbose))
def main(): """ Run stages of test product :return: None """ parser = argparse.ArgumentParser(prog="tests_runner.py", formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('-ar', "--artifacts", metavar="PATH", required=True, help="Path to a manifest file or directory with the file") parser.add_argument('-d', "--root-dir", metavar="PATH", required=True, help="Path to worker directory") parser.add_argument("--stage", default=TestStage.TEST.value, choices=[stage.value for stage in TestStage], help="Current executable stage") args = parser.parse_args() configure_logger() if args.stage != TestStage.CLEAN.value: configure_logger(logs_path=pathlib.Path(args.root_dir) / 'logs' / f'{args.stage}.log') log = logging.getLogger('tests_runner.main') try: tests_runner = TestRunner(artifacts=pathlib.Path(args.artifacts), root_dir=pathlib.Path(args.root_dir), current_stage=args.stage) if tests_runner.generate_config(): no_errors = tests_runner.run_stage(args.stage) else: log.critical('Failed to process the product configuration') no_errors = False except Exception: no_errors = False log.exception('Exception occurred:') if no_errors: log.info('-' * 50) log.info("%s STAGE COMPLETED", args.stage.upper()) else: log.error('-' * 50) log.error("%s STAGE FAILED", args.stage.upper()) exit(ErrorCode.CRITICAL.value)
def install_pkg(pkg_path): """ :param pkg_path: path to pkg to install :type: pathlib.Path :return: Flag whether pkg installed :rtype: bool """ configure_logger('package_manager.install_pkg') log = logging.getLogger() cmd = _CMD_PATTERN["INSTALL"].get(get_os_name()).format(pkg_path=pkg_path) err, out = cmd_exec(cmd) if err == 0: log.debug(out) return True log.info(out) return False
def main(): parser = argparse.ArgumentParser(prog="update_version.py", formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('-b', '--branch', metavar='String', default='master', help='Branch name that will be replaced in the manifest file') parser.add_argument('-r', '--revision', metavar='String', required=True, help='SHA sum of commit that will be replaced in the manifest file') parser.add_argument('-t', "--commit-time", metavar="String", help='Will switch to the commit before specified time') parser.add_argument('-n', '--component-name', required=True, help='Component name which will be updated information') args = parser.parse_args() configure_logger() log = logging.getLogger('update_version') log.info(f'Started version update') tmp_dir = (pathlib.Path(__file__).parent / 'tmp').resolve() repo_name = 'product-configs' manifest_file = 'manifest.yml' component_updater = ComponentUpdater( tmp_dir=tmp_dir, repo_name=repo_name, component_name=args.component_name, manifest_file=manifest_file, branch=args.branch, revision=args.revision, commit_time=args.commit_time, log=log ) if not component_updater.update(): log.error('Update failed') exit(ErrorCode.CRITICAL.value) log.info(f'Version updated')
def main(): parser = argparse.ArgumentParser( prog="component_checker.py", description='Checks existence of component on share', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('-m', '--path-to-manifest', help='Path to manifest file', required=True) parser.add_argument('-c', '--component-name', help='Component name to check', required=True) parser.add_argument( '-b', "--build-type", default=Build_type.RELEASE.value, choices=[build_type.value for build_type in Build_type], help='Type of build') parser.add_argument( '-p', '--product-type', choices=[product_type.value for product_type in Product_type], help='Type of product') args = parser.parse_args() configure_logger() log = logging.getLogger('component_checker.main') try: check_component_existence(args.path_to_manifest, args.component_name, args.product_type, args.build_type) except Exception: log.exception('Exception occurred') exit(ErrorCode.CRITICAL.value) exit(0)
def is_pkg_installed(pkg_name): """ Check whether pkg is installed :param pkg_name: pkg name :type: String :return: Flag whether pkg is installed :rtype: bool """ configure_logger('package_manager.is_pkg_installed') log = logging.getLogger() cmd = _CMD_PATTERN["CHECK_INSTALLED"].get( get_os_name()).format(pkg_name=pkg_name) err, out = cmd_exec(cmd) if err == 0: log.debug(out) return True log.info(out) return False
def main(): """Extract whole infrastructure package or specified repository""" parser = argparse.ArgumentParser( prog="extract_repo.py", epilog= f"If you sepcify --repo-name={PRIVATE_KEY} you must specify commit-time and an optional commit-id. " f"For all other cases if you specify commit-time and commit-id, only commit-id will be used.", formatter_class=argparse.RawTextHelpFormatter) parser.add_argument( "--repo-name", metavar="String", required=True, help= f"""Repository name or "{OPEN_SOURCE_KEY}"/"{CLOSED_SOURCE_KEY}"/"{PRIVATE_KEY}" {OPEN_SOURCE_KEY} key uses for extracting open source infrastructure package {CLOSED_SOURCE_KEY} key uses for extracting closed source infrastructure package {PRIVATE_KEY} key uses for extracting private infrastructure package""") parser.add_argument( "--root-dir", metavar="PATH", default='.', help= f"Path where repository will be stored, by default it is current directory" ) parser.add_argument("--branch", metavar="String", default='master', help='Branch name, by default "master" branch') parser.add_argument("--commit-id", metavar="String", help='SHA of commit') parser.add_argument("--commit-time", metavar="String", help='Will switch to the commit before specified time') args = parser.parse_args() configure_logger() log = logging.getLogger('extract_repo.main') root_dir = pathlib.Path(args.root_dir).absolute() if args.repo_name == OPEN_SOURCE_KEY: log.info("EXTRACTING OPEN SOURCE INFRASTRUCTURE") extract_open_source_infrastructure(root_dir=root_dir, branch=args.branch, commit_id=args.commit_id, commit_time=args.commit_time) elif args.repo_name == CLOSED_SOURCE_KEY: log.info("EXTRACTING CLOSED SOURCE INFRASTRUCTURE") extract_closed_source_infrastructure(root_dir=root_dir, branch=args.branch, commit_id=args.commit_id, commit_time=args.commit_time) elif args.repo_name == PRIVATE_KEY: log.info("EXTRACTING PRIVATE INFRASTRUCTURE") extract_private_infrastructure(root_dir=root_dir, branch=args.branch, commit_id=args.commit_id, commit_time=args.commit_time) else: log.info("EXTRACTING") extract_repo(root_repo_dir=root_dir, repo_name=args.repo_name, branch=args.branch, commit_id=args.commit_id, commit_time=args.commit_time) exit_script()
def main(): """ Run stages of product build :return: None """ parser = argparse.ArgumentParser( prog="build_runner.py", formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('-bc', "--build-config", metavar="PATH", required=True, help="Path to a build configuration") parser.add_argument('-d', "--root-dir", metavar="PATH", required=True, help="Path to worker directory") parser.add_argument('-m', "--manifest", metavar="PATH", required=True, help="Path to manifest.yml file") parser.add_argument('-c', "--component", metavar="String", required=True, help="Component name that will be built from manifest") parser.add_argument( '-b', "--build-type", default=Build_type.RELEASE.value, choices=[build_type.value for build_type in Build_type], help='Type of build') parser.add_argument( '-p', "--product-type", default=Product_type.CLOSED_LINUX.value, choices=[product_type.value for product_type in Product_type], help='Type of product') parser.add_argument("--stage", default=Stage.BUILD.value, choices=[stage.value for stage in Stage], help="Current executable stage") parser.add_argument( '-ta', "--target-arch", nargs='*', default=[target_arch.value for target_arch in TargetArch], choices=[target_arch.value for target_arch in TargetArch], help='Architecture of target platform') parsed_args, unknown_args = parser.parse_known_args() configure_logger() if parsed_args.stage != Stage.CLEAN.value: configure_logger(logs_path=pathlib.Path(parsed_args.root_dir) / 'logs' / f'{parsed_args.stage}.log') log = logging.getLogger('build_runner.main') # remove duplicated values target_arch = list(set(parsed_args.target_arch)) custom_cli_args = {} if unknown_args: for arg in unknown_args: try: arg = arg.split('=') custom_cli_args[arg[0]] = arg[1] except Exception: log.exception(f'Wrong argument layout: {arg}') exit(ErrorCode.CRITICAL) try: build_config = BuildGenerator( build_config_path=pathlib.Path( parsed_args.build_config).absolute(), root_dir=pathlib.Path(parsed_args.root_dir).absolute(), manifest=pathlib.Path(parsed_args.manifest), component=parsed_args.component, build_type=parsed_args.build_type, product_type=parsed_args.product_type, custom_cli_args=custom_cli_args, stage=parsed_args.stage, target_arch=target_arch) # prepare build configuration if build_config.generate_config(): no_errors = build_config.run_stage(parsed_args.stage) else: no_errors = False except Exception: no_errors = False log.exception('Exception occurred') build_state_file = pathlib.Path(parsed_args.root_dir) / 'build_state' if no_errors: if not build_state_file.exists(): build_state_file.write_text(json.dumps({'status': "PASS"})) log.info('-' * 50) log.info("%s STAGE COMPLETED", parsed_args.stage.upper()) else: build_state_file.write_text(json.dumps({'status': "FAIL"})) log.error('-' * 50) log.error("%s STAGE FAILED", parsed_args.stage.upper()) exit(ErrorCode.CRITICAL.value)
def main(): """ Run stages of product build :return: None """ parser = argparse.ArgumentParser( prog="build_runner.py", formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("--version", action="version", version="%(prog)s 1.0") parser.add_argument('-bc', "--build-config", metavar="PATH", required=True, help="Path to a build configuration") parser.add_argument('-d', "--root-dir", metavar="PATH", required=True, help="Path to worker directory") parser.add_argument('-r', "--changed-repo", metavar="String", help='''Changed repository information in format: <repo_name>:<branch>:<commit_id> (ex: MediaSDK:master:52199a19d7809a77e3a474b195592cc427226c61)''') parser.add_argument('-s', "--repo-states", metavar="PATH", help="Path to repo_states.json file") parser.add_argument('-m', "--manifest", metavar="PATH", help="Path to manifest.yml file") parser.add_argument('-c', "--component", metavar="String", help="Component name that will be built from manifest") parser.add_argument( '-b', "--build-type", default=Build_type.RELEASE.value, choices=[build_type.value for build_type in Build_type], help='Type of build') parser.add_argument( '-p', "--product-type", default=Product_type.CLOSED_LINUX.value, choices=[product_type.value for product_type in Product_type], help='Type of product') parser.add_argument( '-e', "--build-event", default=Build_event.PRE_COMMIT.value, choices=[build_event.value for build_event in Build_event], help='Event of build') parser.add_argument("--stage", default=Stage.BUILD.value, choices=[stage.value for stage in Stage], help="Current executable stage") parser.add_argument('-t', "--commit-time", metavar='datetime', help="Time of commits (ex. 2017-11-02 07:36:40)") parser.add_argument( '-ta', "--target-arch", nargs='*', default=[target_arch.value for target_arch in TargetArch], choices=[target_arch.value for target_arch in TargetArch], help='Architecture of target platform') parser.add_argument( '-tb', "--target-branch", help=f'All not triggered repos will be checkout to this branch.') parsed_args, unknown_args = parser.parse_known_args() configure_logger() if parsed_args.stage != Stage.CLEAN.value: configure_logger(logs_path=pathlib.Path(parsed_args.root_dir) / 'logs' / f'{parsed_args.stage}.log') log = logging.getLogger('build_runner.main') # remove duplicated values target_arch = list(set(parsed_args.target_arch)) custom_cli_args = {} if unknown_args: for arg in unknown_args: try: arg = arg.split('=') custom_cli_args[arg[0]] = arg[1] except: log.exception(f'Wrong argument layout: {arg}') exit(ErrorCode.CRITICAL) if parsed_args.commit_time: commit_time = datetime.strptime(parsed_args.commit_time, '%Y-%m-%d %H:%M:%S') else: commit_time = None try: build_config = BuildGenerator( build_config_path=pathlib.Path( parsed_args.build_config).absolute(), root_dir=pathlib.Path(parsed_args.root_dir).absolute(), build_type=parsed_args.build_type, product_type=parsed_args.product_type, build_event=parsed_args.build_event, commit_time=commit_time, changed_repo=parsed_args.changed_repo, repo_states_file_path=parsed_args.repo_states, custom_cli_args=custom_cli_args, stage=parsed_args.stage, target_arch=target_arch, target_branch=parsed_args.target_branch, manifest_file=parsed_args.manifest, component_name=parsed_args.component) if not parsed_args.changed_repo \ and not parsed_args.repo_states \ and (not parsed_args.manifest or not parsed_args.component): log.warning( '"--changed-repo" or "--repo-states" or "--manifest" and "--component" ' 'arguments are not set, "HEAD" revision and "master" branch will be used' ) elif parsed_args.changed_repo and parsed_args.repo_states: log.warning( 'The --repo-states argument is ignored because the --changed-repo is set' ) # prepare build configuration if build_config.generate_config(): # run stage of build no_errors = build_config.run_stage(parsed_args.stage) else: log.critical('Failed to process the product configuration') no_errors = False except Exception: no_errors = False log.exception('Exception occurred') build_state_file = pathlib.Path(parsed_args.root_dir) / 'build_state' if no_errors: if not build_state_file.exists(): build_state_file.write_text(json.dumps({'status': "PASS"})) log.info('-' * 50) log.info("%s STAGE COMPLETED", parsed_args.stage.upper()) else: build_state_file.write_text(json.dumps({'status': "FAIL"})) log.error('-' * 50) log.error("%s STAGE FAILED", parsed_args.stage.upper()) exit(ErrorCode.CRITICAL.value)
def main(): """ Run stages of test product :return: None """ parser = argparse.ArgumentParser( prog="tests_runner.py", formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('-d', "--root-dir", metavar="PATH", required=True, help="Path to worker directory") parser.add_argument('-tc', "--test-config", metavar="PATH", required=True, help="Path to a test configuration") parser.add_argument( '-ar', "--manifest", metavar="PATH", required=True, help="Path to a manifest file or directory with the file") parser.add_argument('-c', "--component", metavar="String", required=True, help="Component name that will be built from manifest") parser.add_argument( '-p', "--product-type", choices=[product_type.value for product_type in Product_type], help='Type of product') parser.add_argument( '-b', "--build-type", default=Build_type.RELEASE.value, choices=[build_type.value for build_type in Build_type], help='Type of build') parser.add_argument("--custom-types", nargs='*', help="Set custom product types for components\n" "(ex. component_name:product_type)") parser.add_argument("--stage", default=TestStage.TEST.value, choices=[stage.value for stage in TestStage], help="Current executable stage") args = parser.parse_args() configure_logger() if args.stage != TestStage.CLEAN.value: configure_logger(logs_path=pathlib.Path(args.root_dir) / 'logs' / f'{args.stage}.log') log = logging.getLogger('tests_runner.main') custom_types = None if args.custom_types: custom_types = dict(custom.split(':') for custom in args.custom_types) try: tests_runner = TestRunner(root_dir=pathlib.Path(args.root_dir), test_config=pathlib.Path(args.test_config), manifest=pathlib.Path(args.manifest), component=args.component, current_stage=args.stage, product_type=args.product_type, build_type=args.build_type, custom_types=custom_types) if tests_runner.generate_config(): no_errors = tests_runner.run_stage(args.stage) else: no_errors = False except Exception: no_errors = False log.exception('Exception occurred:') if no_errors: log.info('-' * 50) log.info("%s STAGE COMPLETED", args.stage.upper()) else: log.error('-' * 50) log.error("%s STAGE FAILED", args.stage.upper()) exit(ErrorCode.CRITICAL.value)