def list_releases(name, last, contains, bucket=None):
repo = None
contains_oid = None
if contains:
repo = utils.git_repo()
contains_oid = utils.revparse(repo, contains)
if contains_oid not in repo:
raise Exception(f"Commit {contains_oid} does not exist in repo")
releases = get_releases(utils.s3_client(), name, bucket=bucket)
release_data = []
now = datetime.now(tz=timezone.utc)
last = int(last) if last else None
for i, rel in enumerate(releases):
if i == last:
break
release_dict = {
"version": rel.version,
"commit": rel.commit,
"timestamp": rel.timestamp,
"age": now - rel.timestamp,
"author": rel.author,
"rollback": rel.rollback,
"action_type": rel.action_type,
}
if contains:
release_dict["contains"] = release_contains(repo, rel, contains_oid, name)
release_data.append(release_dict)
utils.printfmt(release_data, tabular=True)
def log(_, name, range, resolve=False, verbose=False, profile=None):
"""
Show git log between versions and/or commits.
This resolves catapult versions (eg `v123`) into git commits, in
addition to anything that git can map to a single commit, eg
`fc20299e`, `my_branch`, `some_tag`.
"""
repo = utils.git_repo()
client = utils.s3_client(profile)
lx, _, rx = range.partition("..")
def resolve_range(ref):
if ref.startswith("v") and ref[1:].isdigit():
release = get_release(client, name, int(ref[1:]))
ref = release.commit
return utils.revparse(repo, ref)
start = resolve_range(lx)
end = resolve_range(rx)
if resolve:
text = f"{start.hex}...{end.hex}"
else:
changelog = utils.changelog(repo, end, start)
text = changelog.text if verbose else changelog.short_text
print(text)
def find(_, name, commit=None):
"""
Search a release from the commit hash.
"""
repo = utils.git_repo()
commit = commit or next(utils.git_log(repo)).hex
client = utils.s3_client()
releases = {
release.commit: release
for release in get_releases(client, name)
}
release = None
for log in utils.git_log(repo):
release = releases.get(log.hex, release)
if commit == log.hex:
break
if release:
utils.printfmt(release)
else:
LOG.error("commit not released yet")
def log(_, name, git_range, resolve=False):
"""
Search a release from the commit hash.
"""
repo = utils.git_repo()
client = utils.s3_client()
lx, _, rx = git_range.partition("..")
def resolve_range(ref):
if ref.startswith("v") and ref[1:].isdigit():
release = get_release(client, name, int(ref[1:]))
return release.commit
return ref
start = resolve_range(lx)
end = resolve_range(rx)
if resolve:
text = f"{start}...{end}"
else:
text = utils.changelog(repo, end, start).text
print(text)
def find(_, name, commit=None, profile=None):
"""
Find the first release containing a specific commit.
"""
if commit is None:
commit = "HEAD"
repo = utils.git_repo()
oid = utils.revparse(repo, commit)
client = utils.s3_client(profile)
releases = {
release.commit: release
for release in get_releases(client, name)
}
release = None
for log in utils.git_log(repo):
if log.hex in releases:
release = releases[log.hex]
if oid.hex == log.hex:
break
if release:
utils.printfmt(release)
else:
LOG.error("Commit not released yet")
def ls(_, name, last=None):
"""
Show all the project's releases.
"""
releases = get_releases(utils.s3_client(), name)
if last is not None:
releases = list(releases)[:int(last)]
utils.printfmt(list(releases))
def get(_, name, version, profile=None):
"""
Show the release.
"""
release = get_release(utils.s3_client(profile), name, int(version))
if release:
utils.printfmt(release)
else:
utils.fatal("Release does not exist")
def current(_, name, profile=None):
"""
Show current release.
"""
release = next(get_releases(utils.s3_client(profile), name), None)
if release:
utils.printfmt(release)
else:
utils.fatal("Release does not exist")
def get(_, name, version):
"""
Show the release.
"""
release = get_release(utils.s3_client(), name, int(version))
if release:
utils.printfmt(release)
else:
LOG.critical("release does not exist")
sys.exit(1)
def current(_, name):
"""
Show current release.
"""
release = next(get_releases(utils.s3_client(), name), None)
if release:
utils.printfmt(release)
else:
LOG.critical("release does not exist")
sys.exit(1)
def current(_, name, env, bucket=None, profile=None):
"""
Show current running version.
"""
client = utils.s3_client(profile)
if bucket is None:
bucket = utils.get_config()["deploy"][env]["s3_bucket"]
last_deploy = next(get_releases(client, name, bucket=bucket), None)
if last_deploy:
utils.printfmt(last_deploy)
else:
utils.fatal("Release does not exist")
def ls(_):
"""
List all the projects managed with catapult.
"""
client = utils.s3_client()
projects = []
config = utils.get_config()
bucket = config["release"]["s3_bucket"]
deploys = config["deploy"]
resp = client.list_objects_v2(Bucket=bucket)
for data in resp.get("Contents", []):
name = data["Key"]
projects.append(name)
projects = sorted(projects)
_projects = []
for name in projects:
try:
release = get_release(client, bucket, name)
except InvalidRelease:
continue
data = {
"Name":
name,
"Latest Release":
f"v{release.version} {release.timestamp} ({release.commit})",
}
for env_name, cfg in deploys.items():
env_version, env_commit, env_timestamp = get_deployed_version(
client, cfg["s3_bucket"], name)
data[env_name.title(
)] = f"v{env_version} {env_timestamp} ({env_commit})"
_projects.append(data)
projects = _projects
utils.printfmt(projects)
def new(
ctx,
name,
commit=None,
version=None,
dry=False,
yes=False,
image_name=None,
rollback=False,
):
"""
Create a new release.
"""
repo = utils.git_repo()
client = utils.s3_client()
latest = next(get_releases(client, name), None)
if commit is None:
# get last commit
commit = next(utils.git_log(repo), None)
commit = commit and commit.hex
if version is None:
# crate next version
version = 1 if latest is None else latest.version + 1
else:
version = int(version)
image_id = _get_image_id(ctx, image_name or name, commit)
if image_id is None:
LOG.critical("image ID not found")
sys.exit(1)
changelog = utils.changelog(repo, commit, latest and latest.commit)
release = Release(
version=version,
commit=commit,
changelog=changelog.text,
version_id="",
image=image_id,
timestamp=datetime.now(),
author=utils.get_author(repo),
rollback=changelog.rollback,
)
utils.printfmt(release)
if dry:
return
if release.rollback:
utils.warning("this is a rollback! :warning:\n")
if not rollback:
utils.warning("missing flag --rollback\n")
utils.error("aborted!\n")
sys.exit(1)
if not yes:
if release.rollback:
ok = utils.confirm("sure you want to start a rollback?",
style=utils.TextStyle.warning)
if not ok:
utils.error("aborted!\n")
sys.exit(1)
ok = utils.confirm("sure you want to create this release?")
if not ok:
sys.exit(1)
put_release(client,
utils.get_config()["release"]["s3_bucket"], name, release)
utils.success("created new release :tada:\n")
def new(
ctx,
name,
commit=None,
version=None,
dry=False,
yes=False,
image_name=None,
image_id=None,
rollback=False,
filter_files_path=None,
profile=None,
):
"""
Create a new release.
"""
repo = utils.git_repo()
client = utils.s3_client(profile)
latest = next(get_releases(client, name), None)
latest_oid = git.Oid(hex=latest.commit) if latest else None
if commit is None:
commit = "HEAD"
commit_oid = utils.revparse(repo, commit)
if version is None:
# create next version
version = 1 if latest is None else latest.version + 1
else:
version = int(version)
if image_id is None:
image_id = _get_image_id(ctx,
commit_oid,
name=name,
image_name=image_name)
if image_id is None:
utils.fatal("Image not found")
keep_only_files = None
if filter_files_path:
with open(filter_files_path) as fp:
keep_only_files = [line.strip() for line in fp]
changelog = utils.changelog(repo,
commit_oid,
latest_oid,
keep_only_files=keep_only_files)
action_type = ActionType.automated if config.IS_CONCOURSE else ActionType.manual
release = Release(
version=version,
commit=commit_oid.hex,
changelog=changelog.short_text,
version_id="",
image=image_id,
timestamp=datetime.now(),
author=utils.get_author(repo, commit_oid),
rollback=changelog.rollback,
action_type=action_type,
commits=[commit.hex for commit in changelog.logs],
)
utils.printfmt(release)
if dry:
return
if release.rollback:
utils.warning("This is a rollback! :warning:\n")
if not rollback:
utils.warning("Missing flag --rollback\n")
utils.fatal("Aborted!")
if not yes:
if release.rollback:
ok = utils.confirm(
"Are you sure you want to create a rollback release?",
style=utils.TextStyle.yellow,
)
if not ok:
utils.fatal("Aborted!")
ok = utils.confirm("Are you sure you want to create this release?")
if not ok:
utils.fatal("Aborted!")
put_release(client, _get_bucket(), name, release)
utils.success("Created new release :tada:\n")
def list_projects(
contains, only, permissions, utc, env, releases_only, profile
) -> List[Project]:
contains_oid = None
repo = None
if contains:
repo = utils.git_repo()
contains_oid = utils.revparse(repo, contains)
if contains_oid not in repo:
raise Exception(f"Commit {contains_oid} does not exist in repo")
if only is not None:
only = re.compile(only)
if env is not None:
env = set(env.split(","))
client = utils.s3_client(profile)
config = utils.get_config()
release_bucket = config["release"]["s3_bucket"]
deploys = config["deploy"]
resp = client.list_objects_v2(Bucket=release_bucket)
project_names = sorted(data["Key"] for data in resp.get("Contents", []))
can_release = {}
can_deploy = {}
if permissions:
iam_client = utils.iam_client(profile)
can_release = check_perms(iam_client, release_bucket, project_names, profile)
can_deploy = {
env_name: check_perms(iam_client, cfg["s3_bucket"], project_names, profile)
for env_name, cfg in deploys.items()
}
_projects = []
now = datetime.now(tz=timezone.utc)
localzone = get_localzone()
for name in project_names:
if only and only.search(name) is None:
continue
try:
release = fetch_release(client, release_bucket, name)
except InvalidRelease:
continue
timestamp_utc = release.timestamp
timestamp = timestamp_utc if utc else timestamp_utc.astimezone(localzone)
if releases_only or env is None:
_projects.append(
Project(
name=name,
version=release.version,
behind=0,
commit=release.commit,
timestamp=timestamp,
age=now - timestamp_utc,
type=ProjectType.release,
contains=(
release_contains(repo, release, contains_oid, name)
if contains
else None
),
env_name="",
permission=can_release.get(name),
action_type=release.action_type,
author=release.author,
)
)
if releases_only:
continue
for env_name, cfg in deploys.items():
try:
deploy = fetch_release(client, cfg["s3_bucket"], name)
except InvalidRelease:
continue
timestamp_utc = deploy.timestamp
timestamp = timestamp_utc if utc else timestamp_utc.astimezone(localzone)
if not env or env_name in env:
_projects.append(
Project(
name=name,
version=deploy.version,
behind=release.version - deploy.version,
commit=deploy.commit,
timestamp=timestamp,
age=now - timestamp_utc,
type=ProjectType.deploy,
env_name=env_name,
contains=(
release_contains(repo, deploy, contains_oid, name)
if contains
else None
),
permission=can_deploy.get(env_name, {}).get(name),
action_type=deploy.action_type,
author=deploy.author,
)
)
return _projects
def start(_,
name,
env,
version=None,
bucket=None,
dry=False,
yes=False,
rollback=False):
"""
Deploy a release on an environment.
"""
client = utils.s3_client()
repo = utils.git_repo()
if version is None:
release = next(get_releases(client, name), None)
else:
release = get_release(client, name, int(version))
if release is None:
LOG.critical("Release not found")
sys.exit(1)
if bucket is None:
bucket = utils.get_config()["deploy"][env]["s3_bucket"]
last_deploy = next(get_releases(client, name, bucket=bucket), None)
if last_deploy is None:
# first deploy is always None
changelog_text = release.changelog
is_rollback = release.rollback
else:
# create a changelog from the latest deploy commit
changelog = utils.changelog(repo, release.commit, last_deploy.commit)
changelog_text = changelog.text
is_rollback = changelog.rollback
release = dataclasses.replace(
release,
changelog=changelog_text,
timestamp=datetime.now(),
author=utils.get_author(repo),
rollback=is_rollback,
)
utils.printfmt(release)
if dry:
return
if release.rollback:
utils.warning("This is a rollback! :warning:\n")
if not rollback:
utils.warning("Missing flag --rollback\n")
utils.error("Aborted!\n")
sys.exit(1)
if not yes:
if release.rollback:
ok = utils.confirm(
"Are you sure you want to start a rollback deployment?",
style=utils.TextStyle.warning,
)
if not ok:
utils.error("Aborted!\n")
sys.exit(1)
ok = utils.confirm("Are you sure you want to start this deployment?")
if not ok:
utils.error("Aborted!\n")
sys.exit(1)
put_release(client, bucket, name, release)
utils.success("Started new deployment :rocket:\n")
def ls(_, contains=None, sort=None, reverse=False, only=None):
"""
List all the projects managed with catapult.
Optionally pass a full SHA-1 hash of a commit in the current repo,
and each release/deploy will be marked with 'Y' if it contains that
commit, 'N' if it doesn't, or '?' if it can't be determined (eg
perhaps the App belongs to another repo).
"""
contains_oid = None
repo = None
if contains:
contains_oid = git.Oid(hex=contains)
repo = utils.git_repo()
if contains_oid not in repo:
raise Exception(f"Commit {contains_oid} does not exist in repo")
valid_sort_keys = list(Project._fields)
if not contains:
valid_sort_keys.remove("contains")
sort_keys = [] if sort is None else sort.split(",")
if any(sort_key not in valid_sort_keys for sort_key in sort_keys):
raise Exception(
f"Invalid sort key in {sort!r}. Valid sort keys: {valid_sort_keys}"
)
if only is not None:
only = only.split(",")
client = utils.s3_client()
config = utils.get_config()
bucket = config["release"]["s3_bucket"]
deploys = config["deploy"]
resp = client.list_objects_v2(Bucket=bucket)
project_names = sorted(data["Key"] for data in resp.get("Contents", []))
_projects = []
now = datetime.now(tz=timezone.utc)
for name in project_names:
if only and name not in only:
continue
try:
release = get_release(client, bucket, name)
except InvalidRelease:
continue
_projects.append(
Project(
name=name,
version=release.version,
commit=release.commit,
timestamp=release.timestamp,
age=now - release.timestamp,
type=ProjectType.release,
contains=release_contains(repo, release, contains_oid, name)
if contains else None,
env_name="",
))
for env_name, cfg in deploys.items():
try:
deploy = get_release(client, cfg["s3_bucket"], name)
except InvalidRelease:
continue
_projects.append(
Project(
name=name,
version=deploy.version,
commit=deploy.commit,
timestamp=deploy.timestamp,
age=now - deploy.timestamp,
type=ProjectType.deploy,
env_name=env_name,
contains=release_contains(repo, deploy, contains_oid, name)
if contains else None,
))
project_dicts = []
for project in _projects:
project_dict = project._asdict()
if not contains:
project_dict.pop("contains")
project_dict["type"] = project_dict["type"].name
project_dicts.append(project_dict)
if sort_keys:
project_dicts.sort(key=itemgetter(*sort_keys), reverse=reverse)
utils.printfmt(project_dicts, tabular=True)
def start(
_,
name,
env,
version=None,
bucket=None,
dry=False,
yes=False,
rollback=False,
profile=None,
):
"""
Deploy a release on an environment.
"""
client = utils.s3_client(profile)
repo = utils.git_repo()
if version is None:
release = next(get_releases(client, name), None)
else:
release = get_release(client, name, int(version))
if release is None:
utils.fatal("Release not found")
if bucket is None:
bucket = utils.get_config()["deploy"][env]["s3_bucket"]
last_deploy = next(get_releases(client, name, bucket=bucket), None)
last_deployed_version = int(last_deploy.version) if last_deploy else 0
if version is not None:
since = min(int(version), last_deployed_version)
else:
since = last_deployed_version
releases = list(get_releases(client, name, since=since))
# the field `commits` is not present in all documents as it was introduced
# in a later version. if any of the releases doesn't track them, we'll
# skip the commit filtering to avoid not showing commits in the changelog.
if any(rel.commits is None for rel in releases):
commits = None
else:
commits = [
commit for rel in releases if rel.commits for commit in rel.commits
]
if last_deploy is None:
# first deploy is always None
changelog = utils.changelog(repo,
release.commit,
None,
keep_only_commits=commits)
changelog_text = changelog.short_text
is_rollback = release.rollback
else:
# create a changelog from the latest deploy commit
changelog = utils.changelog(
repo,
git.Oid(hex=release.commit),
git.Oid(hex=last_deploy.commit),
keep_only_commits=commits,
)
changelog_text = changelog.short_text
is_rollback = changelog.rollback
action_type = ActionType.automated if config.IS_CONCOURSE else ActionType.manual
release = dataclasses.replace(
release,
changelog=changelog_text,
timestamp=datetime.now(),
author=utils.get_author(repo, git.Oid(hex=release.commit)),
rollback=is_rollback,
action_type=action_type,
commits=commits,
)
utils.printfmt(release)
if dry:
return
if release.rollback:
commit_count = len(changelog.logs)
utils.warning(":warning: This is a rollback! :warning:\n")
utils.warning(
f":warning: You are rolling back from {name} v{last_deployed_version} to v{version} :warning:\n"
)
utils.warning(
f":warning: This will remove the above {commit_count} commits from {env} :warning:\n"
)
if not rollback:
utils.error("Missing flag --rollback\n")
utils.fatal("Aborted!")
if not yes:
if release.rollback:
ok = utils.confirm(
"Are you sure you want to start a rollback deployment?",
style=utils.TextStyle.yellow,
)
if not ok:
utils.fatal("Aborted!")
ok = utils.confirm("Are you sure you want to start this deployment?")
if not ok:
utils.fatal("Aborted!")
put_release(client, bucket, name, release)
utils.success("Started new deployment :rocket:\n")
def ls(
_,
author=False,
contains=None,
sort=None,
reverse=False,
only=None,
permissions=False,
):
"""
List all the projects managed with catapult.
Optionally pass a full SHA-1 hash of a commit in the current repo,
and each release/deploy will be marked with 'Y' if it contains that
commit, 'N' if it doesn't, or '?' if it can't be determined (eg
perhaps the App belongs to another repo).
"""
contains_oid = None
repo = None
optional_columns = {
"author": bool(author),
"contains": bool(contains),
"permission": bool(permissions),
}
if contains:
repo = utils.git_repo()
contains_oid = utils.revparse(repo, contains)
if contains_oid not in repo:
raise Exception(f"Commit {contains_oid} does not exist in repo")
valid_sort_keys = list(Project._fields)
for column_name, show_column in optional_columns.items():
if not show_column:
valid_sort_keys.remove(column_name)
sort_keys = [] if sort is None else sort.split(",")
if any(sort_key not in valid_sort_keys for sort_key in sort_keys):
raise Exception(
f"Invalid sort key in {sort!r}. Valid sort keys: {valid_sort_keys}"
)
if only is not None:
only = only.split(",")
client = utils.s3_client()
config = utils.get_config()
release_bucket = config["release"]["s3_bucket"]
deploys = config["deploy"]
resp = client.list_objects_v2(Bucket=release_bucket)
project_names = sorted(data["Key"] for data in resp.get("Contents", []))
can_release = {}
can_deploy = {}
if permissions:
iam_client = utils.iam_client()
can_release = check_perms(iam_client, release_bucket, project_names)
can_deploy = {
env_name: check_perms(iam_client, cfg["s3_bucket"], project_names)
for env_name, cfg in deploys.items()
}
_projects = []
now = datetime.now(tz=timezone.utc)
for name in project_names:
if only and name not in only:
continue
try:
release = get_release(client, release_bucket, name)
except InvalidRelease:
continue
_projects.append(
Project(
name=name,
version=release.version,
behind=0,
commit=release.commit,
timestamp=release.timestamp,
age=now - release.timestamp,
type=ProjectType.release,
contains=(release_contains(repo, release, contains_oid, name)
if contains else None),
env_name="",
permission=can_release.get(name),
action_type=release.action_type,
author=release.author,
))
for env_name, cfg in deploys.items():
try:
deploy = get_release(client, cfg["s3_bucket"], name)
except InvalidRelease:
continue
_projects.append(
Project(
name=name,
version=deploy.version,
behind=release.version - deploy.version,
commit=deploy.commit,
timestamp=deploy.timestamp,
age=now - deploy.timestamp,
type=ProjectType.deploy,
env_name=env_name,
contains=(release_contains(repo, deploy, contains_oid,
name) if contains else None),
permission=can_deploy.get(env_name, {}).get(name),
action_type=deploy.action_type,
author=deploy.author,
))
project_dicts = []
for project in _projects:
project_dict = project._asdict()
for column_name, show_column in optional_columns.items():
if not show_column:
project_dict.pop(column_name)
style = (utils.TextStyle.yellow
if project_dict["type"] is ProjectType.release else
utils.TextStyle.blue)
project_dict["name"] = utils.Formatted(project_dict["name"], style)
project_dict["type"] = utils.Formatted(project_dict["type"].name,
style)
project_dicts.append(project_dict)
if sort_keys:
project_dicts.sort(key=itemgetter(*sort_keys), reverse=reverse)
utils.printfmt(project_dicts, tabular=True)
