def main() -> None:
"""Asserts the new PR comprises at least one news file and it adheres to the required standard."""
parser = argparse.ArgumentParser(description="Check correctly formatted news files exist on feature branch.")
parser.add_argument("-b", "--current-branch", help="Name of the current branch", nargs="?")
parser.add_argument("-l", "--local", action="store_true", help="perform checks directly on local repository")
parser.add_argument(
"-v", "--verbose", action="count", default=0, help="Verbosity, by default errors are reported.",
)
args = parser.parse_args()
set_log_level(args.verbose)
with (
LocalProjectRepository() # type: ignore
if args.local
else ProjectTempClone(desired_branch_name=args.current_branch)
) as git:
if git.is_current_branch_feature():
root_dir = configuration.get_value(ConfigurationVariable.PROJECT_ROOT)
absolute_news_dir = configuration.get_value(ConfigurationVariable.NEWS_DIR)
news_dir = str(pathlib.Path(absolute_news_dir).relative_to(root_dir))
try:
validate_news_files(
git=git, news_dir=news_dir, root_dir=root_dir,
)
except Exception as e:
log_exception(logger, e)
sys.exit(1)
def _get_documentation_config() -> Tuple[Path, str]:
docs_dir = Path(
configuration.get_value(
ConfigurationVariable.DOCUMENTATION_PRODUCTION_OUTPUT_PATH))
module_to_document = configuration.get_value(
ConfigurationVariable.MODULE_TO_DOCUMENT)
return docs_dir, module_to_document
def _get_aws_config() -> Tuple[str, dict]:
s3_region = configuration.get_value("AWS_DEFAULT_REGION")
s3_config = {
"aws_access_key_id": configuration.get_value("AWS_ACCESS_KEY_ID"),
"aws_secret_access_key":
configuration.get_value("AWS_SECRET_ACCESS_KEY"),
}
return s3_region, s3_config
def main() -> None:
"""Parses command line arguments and generates docs."""
parser = argparse.ArgumentParser()
parser.add_argument(
"--output_directory",
help="Output directory for docs html files.",
default=configuration.get_value(
ConfigurationVariable.DOCUMENTATION_DEFAULT_OUTPUT_PATH),
)
args = parser.parse_args()
output_directory = Path(args.output_directory)
module = configuration.get_value(ConfigurationVariable.MODULE_TO_DOCUMENT)
sys.exit(generate_docs(output_directory=output_directory, module=module))
def get_tool_config(template_file: Path) -> dict:
"""Gets the configuration for licenseheaders."""
copyright_dates = _determines_copyright_dates()
return {
"owner": configuration.get_value(ConfigurationVariable.ORGANISATION),
"dir": configuration.get_value(ConfigurationVariable.PROJECT_ROOT),
"projname":
configuration.get_value(ConfigurationVariable.PROJECT_NAME),
"tmpl": str(template_file),
"years": copyright_dates,
"additional-extensions": "python=.toml",
"exclude": FILES_TO_IGNORE,
}
def _update_licensing_summary() -> SpdxProject:
project = get_current_spdx_project()
project.generate_licensing_summary(
Path(
configuration.get_value(
ConfigurationVariable.DOCUMENTATION_PRODUCTION_OUTPUT_PATH)))
return project
def reviewer_email(self) -> str:
"""Gets the document reviewer's email.
Returns:
document reviewer's email
"""
return str(configuration.get_value(ConfigurationVariable.BOT_EMAIL))
def reviewer(self) -> str:
"""Gets the document's reviewer.
Returns:
document's reviewer
"""
return str(configuration.get_value(ConfigurationVariable.BOT_USERNAME))
def organisation(self) -> str:
"""Gets the organisation.
Returns:
the organisation in charge
"""
return str(configuration.get_value(ConfigurationVariable.ORGANISATION))
def upload_file(file: Path, bucket_dir: Optional[str],
bucket_name: str) -> None:
"""Uploads a file onto AWS S3.
Args:
file: path to the file to upload
bucket_dir: name of the folder where to put the file in S3 bucket
bucket_name: name of the bucket to target
"""
if not bucket_name:
bucket_name = str(
configuration.get_value(ConfigurationVariable.AWS_BUCKET))
logger.info(f"Uploading {file} to AWS")
if not file.exists():
raise FileNotFoundError(file)
s3_region, s3_config = _get_aws_config()
client = boto3.client("s3", **s3_config)
dest_filename = file.name
key = f"{bucket_dir}/{dest_filename}"
extension = "".join(file.suffixes)
bucket = bucket_name
client.upload_file(
str(file),
bucket,
key,
ExtraArgs={
"ContentType":
mimetypes.types_map.get(extension, "application/octet-stream")
} if extension else {},
)
# Ensures the file is publicly available and reachable
# by anyone having access to the bucket.
client.put_object_acl(ACL="public-read", Bucket=bucket, Key=key)
def upload_directory(dir: Path, bucket_dir: str, bucket_name: str) -> None:
"""Uploads the contents of a directory (recursively) onto AWS S3.
Args:
dir: folder to upload
bucket_dir: name of the folder where to put the directory contents in S3 bucket
bucket_name: name of the bucket to target
"""
if not bucket_name:
bucket_name = str(
configuration.get_value(ConfigurationVariable.AWS_BUCKET))
logger.info(f"Uploading {dir} to AWS")
if not dir.exists():
raise FileNotFoundError(dir)
if dir.is_file():
upload_file(dir, bucket_dir, bucket_name)
return
def onerror(exception: Exception) -> None:
logger.error(exception)
real_dir_path = dir.resolve()
for root, dirs, files in os.walk(str(real_dir_path),
followlinks=True,
onerror=onerror):
new_bucket_dir = _determine_destination(bucket_dir, real_dir_path,
Path(root))
_upload_directory_directories(bucket_name, dirs, new_bucket_dir, root)
_upload_directory_file_contents(bucket_name, files, new_bucket_dir,
root)
def main() -> None:
"""Parses command line arguments and retrieves project configuration values."""
parser = argparse.ArgumentParser(
description="Retrieves project configuration values.")
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("-c",
"--config-variable",
help="variable key string",
type=str)
group.add_argument("-k",
"--key",
help="configuration variable",
type=str,
choices=ConfigurationVariable.choices())
parser.add_argument("-v",
"--verbose",
action="count",
default=0,
help="Verbosity, by default errors are reported.")
args = parser.parse_args()
set_log_level(args.verbose)
try:
print(
configuration.get_value(
ConfigurationVariable.parse(args.key) if args.key else args.
config_variable))
except Exception as e:
log_exception(logger, e)
sys.exit(1)
def generate_validation_report_and_publish():
"""Calls for the creation of the validation report and uploads to final location."""
with TemporaryDirectory() as tmp_directory:
platform_validator = PlatformValidator(tmp_directory, True)
platform_validator.render_results()
if platform_validator.processing_error:
raise Exception("Report generation failed")
upload_directory(tmp_directory, "validation", configuration.get_value(ConfigurationVariable.AWS_BUCKET))
def organisation_email(self) -> str:
"""Gets the organisation's email.
Returns:
organisation's email
"""
return str(
configuration.get_value(ConfigurationVariable.ORGANISATION_EMAIL))
def test_git_clone(self):
"""Ensures a fully fledged clone is created."""
with ProjectTempClone(desired_branch_name="master") as clone:
self.assertTrue(isinstance(clone, GitWrapper))
self.assertEqual("master", str(clone.get_current_branch()))
self.assertNotEqual(
configuration.get_value(ConfigurationVariable.PROJECT_ROOT),
str(clone.root))
def _generate_header_template() -> str:
"""Generates the header template which is put at the top of source files."""
return LICENCE_HEADER_TEMPLATE.format(
licence_identifier=configuration.get_value(
ConfigurationVariable.FILE_LICENCE_IDENTIFIER),
author="${owner}",
date="${years}",
)
def _commit_release_changes(git: GitWrapper, version: str,
commit_message: str) -> None:
logger.info(f"Committing release [{version}]...")
git.add(
configuration.get_value(
ConfigurationVariable.DOCUMENTATION_PRODUCTION_OUTPUT_PATH))
_add_version_changes(git)
_commit_changes(commit_message, git)
def test_project_metadata_generation_and_parsing(self):
generate_package_info()
current_package = configuration.get_value(ConfigurationVariable.PACKAGE_NAME)
metadata = get_all_packages_metadata_lines(current_package)
self.assertIsNotNone(metadata)
self.assertGreaterEqual(len(metadata), 1)
self.assertIn(
current_package, [parse_package_metadata_lines(metadata_lines).name for metadata_lines in metadata]
)
def test_package_metadata_parser(self):
generate_package_info()
current_package = configuration.get_value(ConfigurationVariable.PACKAGE_NAME)
parser = ProjectMetadataParser(package_name=current_package)
metadata = parser.project_metadata
self.assertIsNotNone(metadata)
self.assertEqual(metadata.package_name, current_package)
self.assertIsNotNone(metadata.project_metadata)
self.assertEqual(metadata.project_metadata.name, current_package)
self.assertEqual(metadata.package_name, current_package)
self.assertIsNotNone(metadata.dependencies_metadata)
self.assertGreaterEqual(len(metadata.dependencies_metadata), 1)
def __init__(
self,
package_metadata: PackageMetadata,
other_document_refs: List[DependencySpdxDocumentRef] = list(),
is_dependency: bool = False,
document_namespace: str = None,
):
"""Constructor."""
self._project_root = Path(
configuration.get_value(ConfigurationVariable.PROJECT_ROOT))
self._project_uuid = str(
configuration.get_value(ConfigurationVariable.PROJECT_UUID))
self._project_config = Path(
configuration.get_value(ConfigurationVariable.PROJECT_CONFIG))
self._project_source = self._project_root.joinpath(
configuration.get_value(ConfigurationVariable.SOURCE_DIR))
self._package_metadata: PackageMetadata = package_metadata
self._is_dependency: bool = is_dependency
self._other_document_references: List[
DependencySpdxDocumentRef] = other_document_refs
self._document_namespace = document_namespace
def test_basic_properties(self):
"""Checks basic properties are set as expected."""
git = ProjectGitWrapper()
self.assertEqual(
configuration.get_value(ConfigurationVariable.PROJECT_ROOT),
str(git.root))
version = git.git_version()
self.assertIsNotNone(version)
self.assertTrue("." in version)
self.assertTrue(git.get_commit_count() > 0)
self.assertIsNotNone(git.uncommitted_changes)
self.assertIsNotNone(git.get_remote_url())
def _check_credentials() -> None:
# Checks the GitHub token is defined
configuration.get_value(ConfigurationVariable.GIT_TOKEN)
# Checks that twine username is defined
configuration.get_value(ENVVAR_TWINE_USERNAME)
# Checks that twine password is defined
configuration.get_value(ENVVAR_TWINE_PASSWORD)
def test_finds_first_available_file_path_in_news_dir(self):
news_dir = configuration.get_value(ConfigurationVariable.NEWS_DIR)
news_file_name_today = datetime.now().strftime("%Y%m%d")
news_file_path_today = str(pathlib.Path(news_dir, news_file_name_today))
for news_type in NewsType:
with self.subTest(f"It determines available file path for {news_type}."):
with Patcher() as patcher:
patcher.fs.create_file(f"{news_file_path_today}.{news_type.name}")
patcher.fs.create_file(f"{news_file_path_today}01.{news_type.name}")
file_path = determine_news_file_path(news_type)
self.assertEqual(file_path, pathlib.Path(news_dir, f"{news_file_name_today}02.{news_type.name}"))
def _generate_changelog(version: Optional[str], use_news_files: bool) -> None:
"""Creates a towncrier log of the release.
Will only create a log entry if we are using news files.
Args:
version: the semver version of the release
use_news_files: are we generating the release from news files
"""
if use_news_files:
logger.info(":: Generating a new changelog")
project_config_path = configuration.get_value(
ConfigurationVariable.PROJECT_CONFIG)
with cd(os.path.dirname(project_config_path)):
subprocess.check_call(
["towncrier", "--yes", '--name=""', f'--version="{version}"'])
def raise_github_pr(pr_info: PullRequestInfo) -> None:
"""Raise a PR on github using the GIT_TOKEN environment variable to authenticate.
Args:
pr_info: data structure containing information about the PR to raise.
"""
logger.info(f"Raising PR {pr_info!r}")
git_token = configuration.get_value(ConfigurationVariable.GIT_TOKEN)
github_instance = Github(git_token)
repo = github_instance.get_repo(pr_info.repo)
try:
repo.create_pull(title=pr_info.subject,
body=pr_info.body,
head=pr_info.head_branch,
base=pr_info.base_branch)
except GithubException as err:
logging.info(err.data["errors"][0]["message"])
def _release_to_pypi() -> None:
logger.info("Releasing to PyPI")
logger.info("Generating a release package")
root = configuration.get_value(ConfigurationVariable.PROJECT_ROOT)
with cd(root):
subprocess.check_call([
sys.executable,
"setup.py",
"clean",
"--all",
"sdist",
"-d",
OUTPUT_DIRECTORY,
"--formats=gztar",
"bdist_wheel",
"-d",
OUTPUT_DIRECTORY,
])
_upload_to_test_pypi()
_upload_to_pypi()
def _calculate_version(commit_type: CommitType,
use_news_files: bool) -> Tuple[bool, Optional[str]]:
"""Calculates the version for the release.
eg. "0.1.2"
Args:
commit_type:
use_news_files: Should the version be dependant on changes recorded in news files
Returns:
Tuple containing
a flag stating whether it is a new version or not
A semver-style version for the latest release
"""
BUMP_TYPES = {
CommitType.DEVELOPMENT: "build",
CommitType.BETA: "prerelease"
}
is_release = commit_type == CommitType.RELEASE
enable_file_triggers = True if use_news_files else None
bump = BUMP_TYPES.get(commit_type)
project_config_path = configuration.get_value(
ConfigurationVariable.PROJECT_CONFIG)
new_version: Optional[str] = None
is_new_version: bool = False
with cd(os.path.dirname(project_config_path)):
old, _, updates = auto_version_tool.main(
release=is_release,
enable_file_triggers=enable_file_triggers,
commit_count_as=bump,
config_path=project_config_path,
)
# Autoversion second returned value is not actually the new version
# There seem to be a bug in autoversion.
# This is why the following needs to be done to determine the version
new_version = updates["__version__"]
is_new_version = old != new_version
logger.info(":: Determining the new version")
logger.info(f"Version: {new_version}")
return is_new_version, new_version
def test_finds_first_available_file_path_in_news_dir(self):
news_dir = configuration.get_value(ConfigurationVariable.NEWS_DIR)
news_file_name_today = datetime.now().strftime("%Y%m%d%H%M%S")
news_file_path_today = str(pathlib.Path(news_dir,
news_file_name_today))
for news_type in NewsType:
with self.subTest(
f"It determines available file path for {news_type}."):
with TemporaryDirectory() as tmp_dir:
pathlib.Path(
tmp_dir,
f"{news_file_path_today}.{news_type.name}").touch()
pathlib.Path(
tmp_dir,
f"{news_file_path_today}01.{news_type.name}").touch()
file_path = determine_news_file_path(news_type)
self.assertEqual(
file_path,
pathlib.Path(
news_dir,
f"{news_file_name_today}02.{news_type.name}"))
from github import Github, GithubException
from mbed_tools_ci_scripts.create_news_file import create_news_file, NewsType
from mbed_tools_ci_scripts.utils import git_helpers
from mbed_tools_ci_scripts.utils.configuration import configuration, ConfigurationVariable
from mbed_tools_lib.exceptions import ToolsError
from mbed_tools_lib.logging import log_exception, set_log_level
from mbed_targets._internal.board_database import SNAPSHOT_FILENAME
from mbed_targets.boards import Boards
logger = logging.getLogger()
BOARD_DATABASE_PATH = Path(
configuration.get_value(ConfigurationVariable.PROJECT_ROOT),
"mbed_targets",
"_internal",
"data",
SNAPSHOT_FILENAME,
)
class PullRequestInfo(NamedTuple):
"""Data structure containing info required to raise a Github PR."""
repo: str
head_branch: str
base_branch: str
subject: str
body: str
create-news-file "Fixed a bug" --type bugfix
"""
import argparse
import enum
import logging
import pathlib
import sys
from datetime import datetime
from mbed_tools_ci_scripts.utils.configuration import configuration, ConfigurationVariable
from mbed_tools_ci_scripts.assert_news import validate_news_file
from mbed_tools_ci_scripts.utils.logging import log_exception
logger = logging.getLogger(__name__)
NEWS_DIR = configuration.get_value(ConfigurationVariable.NEWS_DIR)
class NewsType(enum.Enum):
"""Describes the type of news we're writing."""
bugfix = 0
doc = 1
feature = 2
major = 3
misc = 4
removal = 5
def create_news_file(news_text: str, news_type: NewsType) -> pathlib.Path:
"""Facilitates creating a news file, determining it's file name based on the type."""
