def publish_pypi(target, args): """ Build and publish the target on a pypi repository """ config = Configuration(args) assert config.has_pypi_auth(), "Missing PyPi authentication" # Build the project setup = target.check_path("setup.py") logger.info(f"Building Python project using {setup}") sandbox.run_setup(setup, ["clean", "sdist", "bdist_wheel"]) # Check some files were produced dist = target.check_path("dist") build = glob.glob(f"{dist}/*") assert len(build) > 0, "No built files found" logger.info("Will upload {}".format(", ".join(map(os.path.basename, build)))) # Use default repository repository = args.repository or DEFAULT_REPOSITORY logger.info(f"Will upload on {repository}") # Upload it through twine upload_settings = Settings( username=config.pypi["username"], password=config.pypi["password"], repository_url=repository, verbose=True, disable_progress_bar=False, ) twine_upload(upload_settings, build) logger.info("PyPi publication finished.")
def push_artifacts(target, args): ''' Push all artifacts from dependant tasks ''' assert args.task_id is not None, 'Missing task id' # Load config from file/secret config = Configuration(args) assert config.has_docker_auth(), 'Missing Docker authentication' # Setup skopeo skopeo = Skopeo( config.docker['registry'], config.docker['username'], config.docker['password'], ) # Load queue service queue = taskcluster.Queue(config.get_taskcluster_options()) # Load current task description to list its dependencies logger.info('Loading task status {}'.format(args.task_id)) task = queue.task(args.task_id) nb_deps = len(task['dependencies']) assert nb_deps > 0, 'No task dependencies' # Load dependencies artifacts artifacts = load_artifacts(task, queue, args.artifact_filter, args.exclude_filter) for task_id, artifact_name in artifacts: push_artifact(queue, skopeo, task_id, artifact_name) logger.info('All found artifacts were pushed.')
def cargo_publish(target: Target, args: argparse.Namespace) -> None: """ Publish a crate on crates.io """ # Load config from file/secret config = Configuration(args) assert config.has_cargo_auth(), "Missing Cargo authentication" # Build the package to publish on crates.io subprocess.run(["cargo", "publish", "--no-verify", "--dry-run"], check=True) # Publish the crate on crates.io # stdout and stderr are captured to avoid leaking the token proc = subprocess.run( ["cargo", "publish", "--no-verify", "--token", config.cargo["token"]], capture_output=True, text=True, # Return stdout and stderr output as strings ) # If an error is occurred while publishing the crate # Do not fail when a `crate already uploaded` error is found and # the option to ignore that kind of error is enabled if proc.returncode != 0 and not (args.ignore_published and "is already uploaded" in proc.stderr): raise Exception("Failed to publish the crate on crates.io")
def push_artifacts(target, args): """ Push all artifacts from dependent tasks """ assert args.task_id is not None, "Missing task id" # Load config from file/secret config = Configuration(args) assert config.has_docker_auth(), "Missing Docker authentication" if args.push_tool == "skopeo": push_tool = Skopeo() elif args.push_tool == "docker": push_tool = Docker() else: raise ValueError("Not supported push tool: {}".format(args.push_tool)) push_tool.login(config.docker["registry"], config.docker["username"], config.docker["password"]) # Load queue service queue = taskcluster.Queue(config.get_taskcluster_options()) # Load dependencies artifacts artifacts = load_artifacts(args.task_id, queue, args.artifact_filter, args.exclude_filter) for task_id, artifact_name in artifacts: push_artifact(queue, push_tool, task_id, artifact_name) logger.info("All found artifacts were pushed.")
def build_image(target, args): """ Build a docker image and allow save/push """ if args.build_tool == "img": build_tool = Img(cache=args.cache) elif args.build_tool == "docker": build_tool = Docker() elif args.build_tool == "dind": build_tool = DinD() else: raise ValueError("Unsupported build tool: {}".format(args.build_tool)) # Load config from file/secret config = Configuration(args) # Check the dockerfile is available in target dockerfile = target.check_path(args.dockerfile) # Check the output is writable output = None if args.write: output = os.path.realpath(args.write) assert output.lower().endswith( ".tar"), "Destination path must ends in .tar" assert os.access(os.path.dirname(output), os.W_OK | os.W_OK), "Destination is not writable" # Build the tags base_image = args.image or "taskboot-{}".format(uuid.uuid4()) tags = gen_docker_images(base_image, args.tag, args.registry) if args.push: assert config.has_docker_auth(), "Missing Docker authentication" registry = config.docker["registry"] if registry != args.registry: msg = "The credentials are the ones for %r not %r" logger.warning(msg, registry, args.registry) # Login on docker build_tool.login(registry, config.docker["username"], config.docker["password"]) # Build the image build_tool.build(target.dir, dockerfile, tags, args.build_arg) # Write the produced image if output: build_tool.save(tags, output) zstd_compress(output) # Push the produced image if args.push: for tag in tags: build_tool.push(tag)
def push_s3(target, args): """ Push files from a remote task on an AWS S3 bucket """ assert args.task_id is not None, "Missing task id" assert not args.artifact_folder.endswith( "/"), "Artifact folder {} must not end in /".format( args.artifact_folder) # Load config from file/secret config = Configuration(args) assert config.has_aws_auth(), "Missing AWS authentication" # Configure boto3 client s3 = boto3.client( "s3", aws_access_key_id=config.aws["access_key_id"], aws_secret_access_key=config.aws["secret_access_key"], ) # Check the bucket is available try: s3.head_bucket(Bucket=args.bucket) logger.info("S3 Bucket {} is available".format(args.bucket)) except botocore.exceptions.ClientError as e: logger.error("Bucket {} unavailable: {}".format(args.bucket, e)) return # Load queue service queue = taskcluster.Queue(config.get_taskcluster_options()) # Download all files from the specified artifact folder # These files are then uploaded on the bucket, stripping the artifact folder # from their final path artifacts = load_artifacts(args.task_id, queue, "{}/*".format(args.artifact_folder)) for task_id, artifact_name in artifacts: # Download each artifact assert artifact_name.startswith(args.artifact_folder) local_path = download_artifact(queue, task_id, artifact_name) # Detect mime/type to set valid content-type for web requests content_type, _ = mimetypes.guess_type(local_path) # Push that artifact on the S3 bucket, without the artifact folder s3_path = artifact_name[len(args.artifact_folder) + 1:] s3.put_object( Bucket=args.bucket, Key=s3_path, Body=open(local_path, "rb"), ContentType=content_type, ) logger.info("Uploaded {} as {} on S3".format(s3_path, content_type))
def stage_deps(target, args): """Pull image dependencies into the `img` store. Arguments: target (taskboot.target.Target): Target args (argparse.Namespace): CLI arguments Returns: None """ create_cert(CA_KEY, CA_CRT, ca=True) create_cert(SRV_KEY, SRV_CRT, ca_key=CA_KEY, ca_cert=CA_CRT) img_tool = Img(cache=args.cache) # retrieve image archives from dependency tasks to /images image_path = Path(mkdtemp(prefix="image-deps-")) try: config = Configuration(Namespace(secret=None, config=None)) queue = taskcluster.Queue(config.get_taskcluster_options()) # load images into the img image store via Docker registry with Registry(): for task_id, artifact_name in load_artifacts( args.task_id, queue, "public/**.tar.zst"): img = download_artifact(queue, task_id, artifact_name, image_path) image_name = Path(artifact_name).stem check_call([ "skopeo", "copy", f"docker-archive:{img}", f"docker://localhost/mozillasecurity/{image_name}:latest", ]) img.unlink() img_tool.run( ["pull", f"localhost/mozillasecurity/{image_name}:latest"]) img_tool.run([ "tag", f"localhost/mozillasecurity/{image_name}:latest", f"{args.registry}/mozillasecurity/{image_name}:latest", ]) img_tool.run([ "tag", f"localhost/mozillasecurity/{image_name}:latest", (f"{args.registry}/mozillasecurity/" f"{image_name}:{args.git_revision}"), ]) finally: rmtree(image_path) # workaround https://github.com/genuinetools/img/issues/206 patch_dockerfile(target.check_path(args.dockerfile), img_tool.list_images())
def build_image(target, args): ''' Build a docker image and allow save/push ''' docker = Docker(cache=args.cache) # Load config from file/secret config = Configuration(args) # Check the dockerfile is available in target dockerfile = target.check_path(args.dockerfile) # Check the output is writable output = None if args.write: output = os.path.realpath(args.write) assert output.lower().endswith( '.tar'), 'Destination path must ends in .tar' assert os.access(os.path.dirname(output), os.W_OK | os.W_OK), \ 'Destination is not writable' # Build the tags base_image = args.image or 'taskboot-{}'.format(uuid.uuid4()) tags = gen_docker_images(base_image, args.tag, args.registry) if args.push: assert config.has_docker_auth(), 'Missing Docker authentication' registry = config.docker['registry'] if registry != args.registry: msg = "The credentials are the ones for %r not %r" logger.warning(msg, registry, args.registry) # Login on docker docker.login( registry, config.docker['username'], config.docker['password'], ) # Build the image docker.build(target.dir, dockerfile, tags, args.build_arg) # Write the produced image if output: for tag in tags: docker.save(tag, output) # Push the produced image if args.push: for tag in tags: docker.push(tag)
def heroku_release(target, args): """ Push all artifacts from dependent tasks """ assert args.task_id is not None, "Missing task id" # Load config from file/secret config = Configuration(args) assert ("username" in config.heroku and "password" in config.heroku), "Missing Heroku authentication" # Setup skopeo skopeo = Skopeo() skopeo.login(HEROKU_REGISTRY, config.heroku["username"], config.heroku["password"]) updates_payload = [] for heroku_dyno_name, _, artifact_path in load_named_artifacts( config, args.task_id, args.artifacts): # Push the Docker image custom_tag_name = f"{HEROKU_REGISTRY}/{args.heroku_app}/{heroku_dyno_name}" artifact_path, ext = os.path.splitext(artifact_path) assert ext == ".zst" zstd_decompress(artifact_path) skopeo.push_archive(artifact_path, custom_tag_name) # Get the Docker image id image_id = docker_id_archive(artifact_path) updates_payload.append({ "type": heroku_dyno_name, "docker_image": image_id }) # Trigger a release on Heroku logger.info( "Deploying update for dyno types: %r", list(sorted(x["type"] for x in updates_payload)), ) updates_payload = {"updates": updates_payload} logger.debug("Using payload: %r", updates_payload) r = requests.patch( f"https://api.heroku.com/apps/{args.heroku_app}/formation", json=updates_payload, headers={ "Accept": "application/vnd.heroku+json; version=3.docker-releases", "Authorization": f"Bearer {config.heroku['password']}", }, ) logger.debug("Heroku deployment answer: %s", r.text) r.raise_for_status() logger.info(f"The {args.heroku_app} application has been updated")
def git_push(target: Target, args: argparse.Namespace) -> None: """ Push commits on a repository """ # Load config from file/secret config = Configuration(args) assert config.has_git_auth(), "Missing Git authentication" # Set remote repository repo_link = "https://{}:{}@{}.git".format(args.user, config.git["token"], args.repository) subprocess.run(["git", "remote", "set-url", "origin", repo_link]) # Push on repository if args.force_push: command = ["git", "push", "-f", "origin", args.branch] else: command = ["git", "push", "origin", args.branch] subprocess.run(command, check=True)
def build_hook(target, args): """ Read a hook definition file and either create or update the hook """ hook_file_path = target.check_path(args.hook_file) hook_group_id = args.hook_group_id hook_id = args.hook_id with open(hook_file_path) as hook_file: payload = json.load(hook_file) # Load config from file/secret config = Configuration(args) hooks = taskcluster.Hooks(config.get_taskcluster_options()) hooks.ping() hook_name = "{}/{}".format(hook_group_id, hook_id) logger.info("Checking if hook %s exists", hook_name) try: hooks.hook(hook_group_id, hook_id) hook_exists = True logger.info("Hook %s exists", hook_name) except taskcluster.exceptions.TaskclusterRestFailure: hook_exists = False logger.info("Hook %s does not exists", hook_name) if hook_exists: hooks.updateHook(hook_group_id, hook_id, payload) logger.info("Hook %s was successfully updated", hook_name) else: hooks.createHook(hook_group_id, hook_id, payload) logger.info("Hook %s was successfully created", hook_name) hook_url = taskcluster_urls.ui( config.get_root_url(), "hooks/{}/{}".format(hook_group_id, hook_id)) logger.info("Hook URL for debugging: %r", hook_url)
def cargo_publish(target: Target, args: argparse.Namespace) -> None: """ Publish a crate on crates.io """ # Load config from file/secret config = Configuration(args) assert config.has_cargo_auth(), "Missing Cargo authentication" # Build the package to publish on crates.io subprocess.run(["cargo", "publish", "--no-verify", "--dry-run"], check=True) # Publish the crate on crates.io # stdout and stderr are captured to avoid leaking the token proc = subprocess.run( ["cargo", "publish", "--no-verify", "--token", config.cargo["token"]], capture_output=True, ) if proc.returncode != 0: raise Exception("Failed to publish the crate on crates.io")
def retrieve_artifacts(target: Target, args: argparse.Namespace) -> None: """ Retrieve all artifacts from a task """ assert args.task_id is not None, "Missing task id" # Load config from file/secret config = Configuration(args) # Replace the path to the artifact with the load_named_version format # worker-type:artifact path artifacts = [ str(pathlib.Path(artifact).stem) + ":" + artifact for artifact in args.artifacts ] # Load dependencies artifacts for _, artifact_name, artifact_path in load_named_artifacts( config, args.task_id, artifacts, args.output_path): logger.info(f"{artifact_name} has been downloaded to {artifact_path}") logger.info("All found artifacts were downloaded.")
def github_release(target: Target, args: argparse.Namespace) -> None: """ Push all artifacts from dependent tasks """ assert args.task_id is not None, "Missing task id" # Load config from file/secret config = Configuration(args) assert config.has_git_auth(), "Missing Github authentication" # Check if local or dependent task assets are used if args.local_asset is None: # Check the assets before any Github change is applied assets = list(load_named_artifacts(config, args.task_id, args.asset)) else: # Create a list of tuples structured in this way # (name, artifact_name, artifact_path) assets = [( str(pathlib.Path(artifact_path).stem), artifact_path, pathlib.Path(artifact_path), ) for artifact_path in args.local_asset] # Setup GitHub API client and load repository github = Github(config.git["token"]) try: repository = github.get_repo(args.repository) logger.info( f"Loaded Github repository {repository.full_name} #{repository.id}" ) except UnknownObjectException: raise Exception(f"Repository {args.repository} is not available") # Check that tag exists, it must be created by the user manually # Usually this task is triggered on a github tag event logger.debug(f"Checking git tag {args.version}") try: tag = repository.get_git_ref(f"tags/{args.version}") logger.info(f"Found existing tag {args.version}") except UnknownObjectException: raise Exception(f"Tag {args.version} does not exist on {repository}") # Check if requested release exists logger.debug(f"Checking requested release {args.version}") try: release = repository.get_release(args.version) logger.info(f"Found existing release {args.version}") except UnknownObjectException: # Create new release logger.info(f"Creating new release {args.version}") release = repository.create_git_release( tag=args.version, name=args.version, message=build_release_notes(repository, tag), target_commitish=tag.object.sha, ) # Upload every named asset for asset_name, _, artifact_path in assets: logger.info(f"Uploading asset {asset_name} using {artifact_path}") release.upload_asset(name=asset_name, path=artifact_path, label=asset_name) logger.info(f"Release available as {release.html_url}")
def heroku_release(target, args): ''' Push all artifacts from dependant tasks ''' assert args.task_id is not None, 'Missing task id' # Load config from file/secret config = Configuration(args) assert 'username' in config.heroku and 'password' in config.heroku, 'Missing Heroku authentication' # Setup skopeo skopeo = Skopeo( HEROKU_REGISTRY, config.heroku['username'], config.heroku['password'], ) # Load queue service queue = taskcluster.Queue(config.get_taskcluster_options()) # Load current task description to list its dependencies logger.info('Loading task status {}'.format(args.task_id)) task = queue.task(args.task_id) nb_deps = len(task['dependencies']) assert nb_deps > 0, 'No task dependencies' # Get the list of matching artifacts as we should get only one matching_artifacts = load_artifacts(task, queue, args.artifact_filter, args.exclude_filter) # Push the Docker image if len(matching_artifacts) == 0: raise ValueError(f"No artifact found for {args.artifact_filter}") elif len(matching_artifacts) > 1: raise ValueError( f"More than one artifact found for {args.artifact_filter}: {matching_artifacts!r}" ) else: task_id, artifact_name = matching_artifacts[0] custom_tag_name = f"{HEROKU_REGISTRY}/{args.heroku_app}/{args.heroku_dyno_type}" artifact_path = download_artifact(queue, task_id, artifact_name) skopeo.push_archive(artifact_path, custom_tag_name) # Get the Docker image id image_id = docker_id_archive(artifact_path) # Trigger a release on Heroku logger.info( f"Deploying image id {image_id!r} to Heroku app {args.heroku_app!r} dyno {args.heroku_dyno_type!r}" ) update = dict( type=args.heroku_dyno_type, docker_image=image_id, ) r = requests.patch( f'https://api.heroku.com/apps/{args.heroku_app}/formation', json=dict(updates=[update]), headers={ 'Accept': 'application/vnd.heroku+json; version=3.docker-releases', 'Authorization': f"Bearer {config.heroku['password']}", }, ) r.raise_for_status() logger.info( f'The {args.heroku_app}/{args.heroku_dyno_type} application has been updated' )
def push_s3(target: Target, args: argparse.Namespace) -> None: """ Push files from a remote task on an AWS S3 bucket """ assert args.task_id is not None, "Missing task id" assert not args.artifact_folder.endswith( "/"), "Artifact folder {} must not end in /".format( args.artifact_folder) # Load config from file/secret config = Configuration(args) assert config.has_aws_auth(), "Missing AWS authentication" # Configure boto3 client s3 = boto3.client( "s3", aws_access_key_id=config.aws["access_key_id"], aws_secret_access_key=config.aws["secret_access_key"], ) # Check the bucket is available try: s3.head_bucket(Bucket=args.bucket) logger.info("S3 Bucket {} is available".format(args.bucket)) except botocore.exceptions.ClientError as e: logger.error("Bucket {} unavailable: {}".format(args.bucket, e)) return # Load queue service queue = taskcluster.Queue(config.get_taskcluster_options()) # Download all files from the specified artifact folder # These files are then uploaded on the bucket, stripping the artifact folder # from their final path artifacts = load_artifacts(args.task_id, queue, "{}/*".format(args.artifact_folder)) for task_id, artifact_name in artifacts: # Download each artifact assert artifact_name.startswith(args.artifact_folder) local_path = download_artifact(queue, task_id, artifact_name) # Detect mime/type to set valid content-type for web requests content_type, _ = mimetypes.guess_type(local_path) if content_type is None: # Use a default content type to avoid crashes on upload # when a file's MIME type is not detected content_type = "text/plain" # Push that artifact on the S3 bucket, without the artifact folder s3_path = artifact_name[len(args.artifact_folder) + 1:] s3.put_object( Bucket=args.bucket, Key=s3_path, Body=open(local_path, "rb"), ContentType=content_type, ) logger.info("Uploaded {} as {} on S3".format(s3_path, content_type)) cloudfront_distribution_id = config.aws.get("cloudfront_distribution_id") if cloudfront_distribution_id is not None: cloudfront_client = boto3.client( "cloudfront", aws_access_key_id=config.aws["access_key_id"], aws_secret_access_key=config.aws["secret_access_key"], ) cloudfront_client.create_invalidation( DistributionId=cloudfront_distribution_id, InvalidationBatch={ "Paths": { "Quantity": 1, "Items": [ "/*", ], }, "CallerReference": str(int(datetime.utcnow().timestamp())), }, ) logger.info("Cloudfront invalidation created")