def main() -> None: """Asserts the new PR comprises at least one news file and it adheres to the required standard.""" parser = argparse.ArgumentParser(description="Check correctly formatted news files exist on feature branch.") parser.add_argument("-b", "--current-branch", help="Name of the current branch", nargs="?") parser.add_argument("-l", "--local", action="store_true", help="perform checks directly on local repository") parser.add_argument( "-v", "--verbose", action="count", default=0, help="Verbosity, by default errors are reported.", ) args = parser.parse_args() set_log_level(args.verbose) with ( LocalProjectRepository() # type: ignore if args.local else ProjectTempClone(desired_branch_name=args.current_branch) ) as git: if git.is_current_branch_feature(): root_dir = configuration.get_value(ConfigurationVariable.PROJECT_ROOT) absolute_news_dir = configuration.get_value(ConfigurationVariable.NEWS_DIR) news_dir = str(pathlib.Path(absolute_news_dir).relative_to(root_dir)) try: validate_news_files( git=git, news_dir=news_dir, root_dir=root_dir, ) except Exception as e: log_exception(logger, e) sys.exit(1)
def _get_documentation_config() -> Tuple[Path, str]: docs_dir = Path( configuration.get_value( ConfigurationVariable.DOCUMENTATION_PRODUCTION_OUTPUT_PATH)) module_to_document = configuration.get_value( ConfigurationVariable.MODULE_TO_DOCUMENT) return docs_dir, module_to_document
def _get_aws_config() -> Tuple[str, dict]: s3_region = configuration.get_value("AWS_DEFAULT_REGION") s3_config = { "aws_access_key_id": configuration.get_value("AWS_ACCESS_KEY_ID"), "aws_secret_access_key": configuration.get_value("AWS_SECRET_ACCESS_KEY"), } return s3_region, s3_config
def main() -> None: """Parses command line arguments and generates docs.""" parser = argparse.ArgumentParser() parser.add_argument( "--output_directory", help="Output directory for docs html files.", default=configuration.get_value( ConfigurationVariable.DOCUMENTATION_DEFAULT_OUTPUT_PATH), ) args = parser.parse_args() output_directory = Path(args.output_directory) module = configuration.get_value(ConfigurationVariable.MODULE_TO_DOCUMENT) sys.exit(generate_docs(output_directory=output_directory, module=module))
def get_tool_config(template_file: Path) -> dict: """Gets the configuration for licenseheaders.""" copyright_dates = _determines_copyright_dates() return { "owner": configuration.get_value(ConfigurationVariable.ORGANISATION), "dir": configuration.get_value(ConfigurationVariable.PROJECT_ROOT), "projname": configuration.get_value(ConfigurationVariable.PROJECT_NAME), "tmpl": str(template_file), "years": copyright_dates, "additional-extensions": "python=.toml", "exclude": FILES_TO_IGNORE, }
def _update_licensing_summary() -> SpdxProject: project = get_current_spdx_project() project.generate_licensing_summary( Path( configuration.get_value( ConfigurationVariable.DOCUMENTATION_PRODUCTION_OUTPUT_PATH))) return project
def reviewer_email(self) -> str: """Gets the document reviewer's email. Returns: document reviewer's email """ return str(configuration.get_value(ConfigurationVariable.BOT_EMAIL))
def reviewer(self) -> str: """Gets the document's reviewer. Returns: document's reviewer """ return str(configuration.get_value(ConfigurationVariable.BOT_USERNAME))
def organisation(self) -> str: """Gets the organisation. Returns: the organisation in charge """ return str(configuration.get_value(ConfigurationVariable.ORGANISATION))
def upload_file(file: Path, bucket_dir: Optional[str], bucket_name: str) -> None: """Uploads a file onto AWS S3. Args: file: path to the file to upload bucket_dir: name of the folder where to put the file in S3 bucket bucket_name: name of the bucket to target """ if not bucket_name: bucket_name = str( configuration.get_value(ConfigurationVariable.AWS_BUCKET)) logger.info(f"Uploading {file} to AWS") if not file.exists(): raise FileNotFoundError(file) s3_region, s3_config = _get_aws_config() client = boto3.client("s3", **s3_config) dest_filename = file.name key = f"{bucket_dir}/{dest_filename}" extension = "".join(file.suffixes) bucket = bucket_name client.upload_file( str(file), bucket, key, ExtraArgs={ "ContentType": mimetypes.types_map.get(extension, "application/octet-stream") } if extension else {}, ) # Ensures the file is publicly available and reachable # by anyone having access to the bucket. client.put_object_acl(ACL="public-read", Bucket=bucket, Key=key)
def upload_directory(dir: Path, bucket_dir: str, bucket_name: str) -> None: """Uploads the contents of a directory (recursively) onto AWS S3. Args: dir: folder to upload bucket_dir: name of the folder where to put the directory contents in S3 bucket bucket_name: name of the bucket to target """ if not bucket_name: bucket_name = str( configuration.get_value(ConfigurationVariable.AWS_BUCKET)) logger.info(f"Uploading {dir} to AWS") if not dir.exists(): raise FileNotFoundError(dir) if dir.is_file(): upload_file(dir, bucket_dir, bucket_name) return def onerror(exception: Exception) -> None: logger.error(exception) real_dir_path = dir.resolve() for root, dirs, files in os.walk(str(real_dir_path), followlinks=True, onerror=onerror): new_bucket_dir = _determine_destination(bucket_dir, real_dir_path, Path(root)) _upload_directory_directories(bucket_name, dirs, new_bucket_dir, root) _upload_directory_file_contents(bucket_name, files, new_bucket_dir, root)
def main() -> None: """Parses command line arguments and retrieves project configuration values.""" parser = argparse.ArgumentParser( description="Retrieves project configuration values.") group = parser.add_mutually_exclusive_group(required=True) group.add_argument("-c", "--config-variable", help="variable key string", type=str) group.add_argument("-k", "--key", help="configuration variable", type=str, choices=ConfigurationVariable.choices()) parser.add_argument("-v", "--verbose", action="count", default=0, help="Verbosity, by default errors are reported.") args = parser.parse_args() set_log_level(args.verbose) try: print( configuration.get_value( ConfigurationVariable.parse(args.key) if args.key else args. config_variable)) except Exception as e: log_exception(logger, e) sys.exit(1)
def generate_validation_report_and_publish(): """Calls for the creation of the validation report and uploads to final location.""" with TemporaryDirectory() as tmp_directory: platform_validator = PlatformValidator(tmp_directory, True) platform_validator.render_results() if platform_validator.processing_error: raise Exception("Report generation failed") upload_directory(tmp_directory, "validation", configuration.get_value(ConfigurationVariable.AWS_BUCKET))
def organisation_email(self) -> str: """Gets the organisation's email. Returns: organisation's email """ return str( configuration.get_value(ConfigurationVariable.ORGANISATION_EMAIL))
def test_git_clone(self): """Ensures a fully fledged clone is created.""" with ProjectTempClone(desired_branch_name="master") as clone: self.assertTrue(isinstance(clone, GitWrapper)) self.assertEqual("master", str(clone.get_current_branch())) self.assertNotEqual( configuration.get_value(ConfigurationVariable.PROJECT_ROOT), str(clone.root))
def _generate_header_template() -> str: """Generates the header template which is put at the top of source files.""" return LICENCE_HEADER_TEMPLATE.format( licence_identifier=configuration.get_value( ConfigurationVariable.FILE_LICENCE_IDENTIFIER), author="${owner}", date="${years}", )
def _commit_release_changes(git: GitWrapper, version: str, commit_message: str) -> None: logger.info(f"Committing release [{version}]...") git.add( configuration.get_value( ConfigurationVariable.DOCUMENTATION_PRODUCTION_OUTPUT_PATH)) _add_version_changes(git) _commit_changes(commit_message, git)
def test_project_metadata_generation_and_parsing(self): generate_package_info() current_package = configuration.get_value(ConfigurationVariable.PACKAGE_NAME) metadata = get_all_packages_metadata_lines(current_package) self.assertIsNotNone(metadata) self.assertGreaterEqual(len(metadata), 1) self.assertIn( current_package, [parse_package_metadata_lines(metadata_lines).name for metadata_lines in metadata] )
def test_package_metadata_parser(self): generate_package_info() current_package = configuration.get_value(ConfigurationVariable.PACKAGE_NAME) parser = ProjectMetadataParser(package_name=current_package) metadata = parser.project_metadata self.assertIsNotNone(metadata) self.assertEqual(metadata.package_name, current_package) self.assertIsNotNone(metadata.project_metadata) self.assertEqual(metadata.project_metadata.name, current_package) self.assertEqual(metadata.package_name, current_package) self.assertIsNotNone(metadata.dependencies_metadata) self.assertGreaterEqual(len(metadata.dependencies_metadata), 1)
def __init__( self, package_metadata: PackageMetadata, other_document_refs: List[DependencySpdxDocumentRef] = list(), is_dependency: bool = False, document_namespace: str = None, ): """Constructor.""" self._project_root = Path( configuration.get_value(ConfigurationVariable.PROJECT_ROOT)) self._project_uuid = str( configuration.get_value(ConfigurationVariable.PROJECT_UUID)) self._project_config = Path( configuration.get_value(ConfigurationVariable.PROJECT_CONFIG)) self._project_source = self._project_root.joinpath( configuration.get_value(ConfigurationVariable.SOURCE_DIR)) self._package_metadata: PackageMetadata = package_metadata self._is_dependency: bool = is_dependency self._other_document_references: List[ DependencySpdxDocumentRef] = other_document_refs self._document_namespace = document_namespace
def test_basic_properties(self): """Checks basic properties are set as expected.""" git = ProjectGitWrapper() self.assertEqual( configuration.get_value(ConfigurationVariable.PROJECT_ROOT), str(git.root)) version = git.git_version() self.assertIsNotNone(version) self.assertTrue("." in version) self.assertTrue(git.get_commit_count() > 0) self.assertIsNotNone(git.uncommitted_changes) self.assertIsNotNone(git.get_remote_url())
def _check_credentials() -> None: # Checks the GitHub token is defined configuration.get_value(ConfigurationVariable.GIT_TOKEN) # Checks that twine username is defined configuration.get_value(ENVVAR_TWINE_USERNAME) # Checks that twine password is defined configuration.get_value(ENVVAR_TWINE_PASSWORD)
def test_finds_first_available_file_path_in_news_dir(self): news_dir = configuration.get_value(ConfigurationVariable.NEWS_DIR) news_file_name_today = datetime.now().strftime("%Y%m%d") news_file_path_today = str(pathlib.Path(news_dir, news_file_name_today)) for news_type in NewsType: with self.subTest(f"It determines available file path for {news_type}."): with Patcher() as patcher: patcher.fs.create_file(f"{news_file_path_today}.{news_type.name}") patcher.fs.create_file(f"{news_file_path_today}01.{news_type.name}") file_path = determine_news_file_path(news_type) self.assertEqual(file_path, pathlib.Path(news_dir, f"{news_file_name_today}02.{news_type.name}"))
def _generate_changelog(version: Optional[str], use_news_files: bool) -> None: """Creates a towncrier log of the release. Will only create a log entry if we are using news files. Args: version: the semver version of the release use_news_files: are we generating the release from news files """ if use_news_files: logger.info(":: Generating a new changelog") project_config_path = configuration.get_value( ConfigurationVariable.PROJECT_CONFIG) with cd(os.path.dirname(project_config_path)): subprocess.check_call( ["towncrier", "--yes", '--name=""', f'--version="{version}"'])
def raise_github_pr(pr_info: PullRequestInfo) -> None: """Raise a PR on github using the GIT_TOKEN environment variable to authenticate. Args: pr_info: data structure containing information about the PR to raise. """ logger.info(f"Raising PR {pr_info!r}") git_token = configuration.get_value(ConfigurationVariable.GIT_TOKEN) github_instance = Github(git_token) repo = github_instance.get_repo(pr_info.repo) try: repo.create_pull(title=pr_info.subject, body=pr_info.body, head=pr_info.head_branch, base=pr_info.base_branch) except GithubException as err: logging.info(err.data["errors"][0]["message"])
def _release_to_pypi() -> None: logger.info("Releasing to PyPI") logger.info("Generating a release package") root = configuration.get_value(ConfigurationVariable.PROJECT_ROOT) with cd(root): subprocess.check_call([ sys.executable, "setup.py", "clean", "--all", "sdist", "-d", OUTPUT_DIRECTORY, "--formats=gztar", "bdist_wheel", "-d", OUTPUT_DIRECTORY, ]) _upload_to_test_pypi() _upload_to_pypi()
def _calculate_version(commit_type: CommitType, use_news_files: bool) -> Tuple[bool, Optional[str]]: """Calculates the version for the release. eg. "0.1.2" Args: commit_type: use_news_files: Should the version be dependant on changes recorded in news files Returns: Tuple containing a flag stating whether it is a new version or not A semver-style version for the latest release """ BUMP_TYPES = { CommitType.DEVELOPMENT: "build", CommitType.BETA: "prerelease" } is_release = commit_type == CommitType.RELEASE enable_file_triggers = True if use_news_files else None bump = BUMP_TYPES.get(commit_type) project_config_path = configuration.get_value( ConfigurationVariable.PROJECT_CONFIG) new_version: Optional[str] = None is_new_version: bool = False with cd(os.path.dirname(project_config_path)): old, _, updates = auto_version_tool.main( release=is_release, enable_file_triggers=enable_file_triggers, commit_count_as=bump, config_path=project_config_path, ) # Autoversion second returned value is not actually the new version # There seem to be a bug in autoversion. # This is why the following needs to be done to determine the version new_version = updates["__version__"] is_new_version = old != new_version logger.info(":: Determining the new version") logger.info(f"Version: {new_version}") return is_new_version, new_version
def test_finds_first_available_file_path_in_news_dir(self): news_dir = configuration.get_value(ConfigurationVariable.NEWS_DIR) news_file_name_today = datetime.now().strftime("%Y%m%d%H%M%S") news_file_path_today = str(pathlib.Path(news_dir, news_file_name_today)) for news_type in NewsType: with self.subTest( f"It determines available file path for {news_type}."): with TemporaryDirectory() as tmp_dir: pathlib.Path( tmp_dir, f"{news_file_path_today}.{news_type.name}").touch() pathlib.Path( tmp_dir, f"{news_file_path_today}01.{news_type.name}").touch() file_path = determine_news_file_path(news_type) self.assertEqual( file_path, pathlib.Path( news_dir, f"{news_file_name_today}02.{news_type.name}"))
from github import Github, GithubException from mbed_tools_ci_scripts.create_news_file import create_news_file, NewsType from mbed_tools_ci_scripts.utils import git_helpers from mbed_tools_ci_scripts.utils.configuration import configuration, ConfigurationVariable from mbed_tools_lib.exceptions import ToolsError from mbed_tools_lib.logging import log_exception, set_log_level from mbed_targets._internal.board_database import SNAPSHOT_FILENAME from mbed_targets.boards import Boards logger = logging.getLogger() BOARD_DATABASE_PATH = Path( configuration.get_value(ConfigurationVariable.PROJECT_ROOT), "mbed_targets", "_internal", "data", SNAPSHOT_FILENAME, ) class PullRequestInfo(NamedTuple): """Data structure containing info required to raise a Github PR.""" repo: str head_branch: str base_branch: str subject: str body: str
create-news-file "Fixed a bug" --type bugfix """ import argparse import enum import logging import pathlib import sys from datetime import datetime from mbed_tools_ci_scripts.utils.configuration import configuration, ConfigurationVariable from mbed_tools_ci_scripts.assert_news import validate_news_file from mbed_tools_ci_scripts.utils.logging import log_exception logger = logging.getLogger(__name__) NEWS_DIR = configuration.get_value(ConfigurationVariable.NEWS_DIR) class NewsType(enum.Enum): """Describes the type of news we're writing.""" bugfix = 0 doc = 1 feature = 2 major = 3 misc = 4 removal = 5 def create_news_file(news_text: str, news_type: NewsType) -> pathlib.Path: """Facilitates creating a news file, determining it's file name based on the type."""