def lint_milestone(ctx):
"""
Make sure PRs are assigned a milestone
"""
pr_url = os.environ.get("CIRCLE_PULL_REQUEST")
if pr_url:
import requests
pr_id = pr_url.rsplit('/')[-1]
res = requests.get(
"https://api.github.com/repos/DataDog/datadog-agent/issues/{}".
format(pr_id))
pr = res.json()
if pr.get("milestone"):
print("Milestone: %s" % pr["milestone"].get("title", "NO_TITLE"))
return
print("PR %s requires a milestone" % pr_url)
raise Exit(code=1)
# The PR has not been created yet
else:
print("PR not yet created, skipping check for milestone")
def image_build(ctx, arch='amd64', base_dir="omnibus", python_version="2", skip_tests=False):
"""
Build the docker image
"""
BOTH_VERSIONS = ["both", "2+3"]
VALID_VERSIONS = ["2", "3"] + BOTH_VERSIONS
if python_version not in VALID_VERSIONS:
raise ParseError("provided python_version is invalid")
build_context = "Dockerfiles/agent"
base_dir = base_dir or os.environ.get("OMNIBUS_BASE_DIR")
pkg_dir = os.path.join(base_dir, 'pkg')
deb_glob = 'datadog-agent*_{}.deb'.format(arch)
dockerfile_path = "{}/{}/Dockerfile".format(build_context, arch)
list_of_files = glob.glob(os.path.join(pkg_dir, deb_glob))
# get the last debian package built
if not list_of_files:
print("No debian package build found in {}".format(pkg_dir))
print("See agent.omnibus-build")
raise Exit(code=1)
latest_file = max(list_of_files, key=os.path.getctime)
shutil.copy2(latest_file, build_context)
# Pull base image with content trust enabled
pull_base_images(ctx, dockerfile_path, signed_pull=True)
common_build_opts = "-t {} -f {}".format(AGENT_TAG, dockerfile_path)
if python_version not in BOTH_VERSIONS:
common_build_opts = "{} --build-arg PYTHON_VERSION={}".format(common_build_opts, python_version)
# Build with the testing target
if not skip_tests:
ctx.run("docker build {} --target testing {}".format(common_build_opts, build_context))
# Build with the release target
ctx.run("docker build {} --target release {}".format(common_build_opts, build_context))
ctx.run("rm {}/{}".format(build_context, deb_glob))
def mirror_image(ctx,
src_image,
dst_image="datadog/docker-library",
dst_tag="auto"):
"""
Pull an upstream image and mirror it to our docker-library repository
for integration tests. Tag format should be A-Z_n_n_n
"""
if dst_tag == "auto":
# Autogenerate tag
match = re.search('([^:\/\s]+):[v]?(.*)$', src_image)
if not match:
print(
"Cannot guess destination tag for {}, please provide a --dst-tag option"
.format(src_image))
raise Exit(1)
dst_tag = "_".join(match.groups()).replace(".", "_")
dst = "{}:{}".format(dst_image, dst_tag)
print("Uploading {} to {}".format(src_image, dst))
# TODO: use docker python lib
ctx.run("docker pull {src} && docker tag {src} {dst} && docker push {dst}".
format(src=src_image, dst=dst))
def _get_release_version_from_release_json(release_json, major_version, version_re, release_json_key=None):
"""
If release_json_key is None, returns the highest version entry in release.json.
If release_json_key is set, returns the entry for release_json_key of the highest version entry in release.json.
"""
release_version = None
release_component_version = None
# Get the release entry for the given Agent major version
release_entry_name = release_entry_for(major_version)
release_json_entry = release_json.get(release_entry_name, None)
# Check that the release entry exists, otherwise fail
if release_json_entry:
release_version = release_entry_name
# Check that the component's version is defined in the release entry
if release_json_key is not None:
match = version_re.match(release_json_entry.get(release_json_key, ""))
if match:
release_component_version = _create_version_from_match(match)
else:
print(
"{} does not have a valid {} ({}), ignoring".format(
release_entry_name, release_json_key, release_json_entry.get(release_json_key, "")
)
)
if not release_version:
raise Exit("Couldn't find any matching {} version.".format(release_version), 1)
if release_json_key is not None:
return release_component_version
return release_version
def dev_env_cleanup(ctx, name="kind"):
"""Remove traces of the dev env."""
validate_kind_version()
clusters = run("kind get clusters", hide=True).stdout.strip().splitlines()
if name in clusters:
run("kind delete cluster --name={}".format(name), hide=True)
else:
raise Exit(message="Unable to find cluster named: {}".format(name))
run('for frr in $(docker ps -a -f name=frr --format {{.Names}}) ; do '
' docker rm -f $frr ; '
'done', hide=True)
# cleanup bgp configs
dev_env_dir = os.getcwd() + "/dev-env/bgp"
frr_volume_dir = dev_env_dir + "/frr-volume"
# sudo because past docker runs will have changed ownership of this dir
run('sudo rm -rf "%s"' % frr_volume_dir)
run('rm -f "%s"/config.yaml' % dev_env_dir)
# cleanup layer2 configs
dev_env_dir = os.getcwd() + "/dev-env/layer2"
run('rm -f "%s"/config.yaml' % dev_env_dir)
def upstream(c, collection='f5_modules', verbose=False):
"""Copy all module utils, to the local/ansible_collections/f5networks/collection_name directory.
"""
coll_dest = '{0}/local/ansible_collections/f5networks/{1}'.format(
BASE_DIR, collection)
module_utils_dst = '{0}/local/ansible_collections/f5networks/{1}/plugins/module_utils/'.format(
BASE_DIR, collection)
module_utils_src = '{0}/library/module_utils/network/f5/'.format(BASE_DIR)
purge_upstreamed_files(c, module_utils_dst, coll_dest, verbose)
create_directories(c, coll_dest, module_utils_dst, verbose)
files_upstream(c, module_utils_src, module_utils_dst)
retries = 0
while not cmp_dir(module_utils_src, module_utils_dst):
purge_upstreamed_files(c, module_utils_dst, coll_dest, verbose)
create_directories(c, coll_dest, module_utils_dst, verbose)
files_upstream(c, module_utils_src, module_utils_dst)
retries = retries + 1
if retries > 2:
raise Exit('Failed to upstream module utils, exiting.')
print("Module utils files upstreamed successfully.")
def lint_teamassignment(ctx):
"""
Make sure PRs are assigned a team label
"""
pr_url = os.environ.get("CIRCLE_PULL_REQUEST")
if pr_url:
import requests
pr_id = pr_url.rsplit('/')[-1]
res = requests.get("https://api.github.com/repos/DataDog/datadog-agent/issues/{}".format(pr_id))
issue = res.json()
for label in issue.get('labels', {}):
if re.match('team/', label['name']):
print("Team Assignment: {}".format(label['name']))
return
print("PR {} requires team assignment".format(pr_url))
raise Exit(code=1)
# The PR has not been created yet
else:
print("PR not yet created, skipping check for team assignment")
def misspell(ctx, targets):
"""
Run misspell on targets.
Example invokation:
inv misspell --targets=./pkg/collector/check,./pkg/aggregator
"""
if isinstance(targets, basestring):
# when this function is called from the command line, targets are passed
# as comma separated tokens in a string
targets = targets.split(',')
result = ctx.run("misspell " + " ".join(targets), hide=True)
legit_misspells = []
for found_misspell in result.stdout.split("\n"):
if len(found_misspell.strip()) > 0:
if not any([ignored_target in found_misspell for ignored_target in MISSPELL_IGNORED_TARGETS]):
legit_misspells.append(found_misspell)
if len(legit_misspells) > 0:
print("Misspell issues found:\n" + "\n".join(legit_misspells))
raise Exit(code=2)
else:
print("misspell found no issues")
def lint(ctx, targets):
"""
Run golint on targets. If targets are not specified,
the value from `invoke.yaml` will be used.
Example invokation:
inv lint --targets=./pkg/collector/check,./pkg/aggregator
"""
if isinstance(targets, str):
# when this function is called from the command line, targets are passed
# as comma separated tokens in a string
targets = targets.split(',')
# add the /... suffix to the targets
targets_list = ["{}/...".format(t) for t in targets]
result = ctx.run("go run golang.org/x/lint/golint {}".format(
' '.join(targets_list)))
if result.stdout:
files = []
skipped_files = set()
for line in (out for out in result.stdout.split('\n') if out):
fname = os.path.basename(line.split(":")[0])
if fname in MODULE_WHITELIST:
skipped_files.add(fname)
continue
files.append(fname)
if files:
print("Linting issues found in {} files.".format(len(files)))
raise Exit(code=1)
if skipped_files:
for skipped in skipped_files:
print("Allowed errors in whitelisted file {}".format(skipped))
print("golint found no issues")
def image_build(ctx, arch='amd64', tag=AGENT_TAG, push=False):
"""
Build the docker image
"""
dca_binary = glob.glob(os.path.join(BIN_PATH, "datadog-cluster-agent"))
# get the last debian package built
if not dca_binary:
print(f"No bin found in {BIN_PATH}")
print("See cluster-agent.build")
raise Exit(code=1)
latest_file = max(dca_binary, key=os.path.getctime)
ctx.run(f"chmod +x {latest_file}")
build_context = "Dockerfiles/cluster-agent"
exec_path = f"{build_context}/datadog-cluster-agent.{arch}"
dockerfile_path = f"{build_context}/{arch}/Dockerfile"
shutil.copy2(latest_file, exec_path)
ctx.run(f"docker build -t {tag} {build_context} -f {dockerfile_path}")
ctx.run(f"rm {exec_path}")
if push:
ctx.run(f"docker push {tag}")
def create_rc(
ctx,
major_versions="6,7",
integration_version=None,
omnibus_software_version=None,
jmxfetch_version=None,
omnibus_ruby_version=None,
security_agent_policies_version=None,
macos_build_version=None,
):
"""
Takes whatever version is the highest in release.json and adds a new RC to it.
If there was no RC, creates one and bump minor version. If there was an RC, create RC + 1.
Update internal module dependencies with the new RC.
"""
if sys.version_info[0] < 3:
print("Must use Python 3 for this task")
return Exit(code=1)
list_major_versions = major_versions.split(",")
print("Creating RC for agent version(s) {}".format(list_major_versions))
list_major_versions = [int(x) for x in list_major_versions]
highest_major = 0
for version in list_major_versions:
if int(version) > highest_major:
highest_major = version
github_token = os.environ.get('GITHUB_TOKEN')
if github_token is None:
print(
"Error: set the GITHUB_TOKEN environment variable.\nYou can create one by going to"
" https://github.com/settings/tokens. It should have at least the 'repo' permissions."
)
return Exit(code=1)
# We want to match:
# - X.Y.Z
# - X.Y.Z-rc.t
# - vX.Y(.Z) (security-agent-policies repo)
version_re = re.compile(r'(v)?(\d+)[.](\d+)([.](\d+))?(-rc\.(\d+))?')
with open("release.json", "r") as release_json_stream:
release_json = json.load(release_json_stream,
object_pairs_hook=OrderedDict)
highest_version = _get_highest_version_from_release_json(
release_json, highest_major, version_re)
# jmxfetch and security-agent-policies follow their own version scheme
highest_jmxfetch_version = _get_highest_version_from_release_json(
release_json,
highest_major,
version_re,
release_json_key="JMXFETCH_VERSION",
)
highest_security_agent_policies_version = _get_highest_version_from_release_json(
release_json,
highest_major,
version_re,
release_json_key="SECURITY_AGENT_POLICIES_VERSION",
)
if highest_version["rc"] is None:
# No RC exists, create one
highest_version["minor"] = highest_version["minor"] + 1
highest_version["rc"] = 1
else:
# An RC exists, create next RC
highest_version["rc"] = highest_version["rc"] + 1
new_rc = _stringify_version(highest_version)
print("Creating {}".format(new_rc))
if not integration_version:
integration_version = _get_highest_repo_version(
github_token, "integrations-core", highest_version, version_re)
print("integrations-core's tag is {}".format(
_stringify_version(integration_version)))
if not omnibus_software_version:
omnibus_software_version = _get_highest_repo_version(
github_token, "omnibus-software", highest_version, version_re)
print("omnibus-software's tag is {}".format(
_stringify_version(omnibus_software_version)))
if not jmxfetch_version:
jmxfetch_version = _get_highest_repo_version(github_token, "jmxfetch",
highest_jmxfetch_version,
version_re)
print("jmxfetch's tag is {}".format(_stringify_version(jmxfetch_version)))
if not omnibus_ruby_version:
omnibus_ruby_version = _get_highest_repo_version(
github_token, "omnibus-ruby", highest_version, version_re)
print("omnibus-ruby's tag is {}".format(
_stringify_version(omnibus_ruby_version)))
if not security_agent_policies_version:
security_agent_policies_version = _get_highest_repo_version(
github_token, "security-agent-policies",
highest_security_agent_policies_version, version_re)
print("security-agent-policies' tag is {}".format(
_stringify_version(security_agent_policies_version)))
if not macos_build_version:
macos_build_version = _get_highest_repo_version(
github_token, "datadog-agent-macos-build", highest_version,
version_re)
print("datadog-agent-macos-build's tag is {}".format(
_stringify_version(macos_build_version)))
_save_release_json(
release_json,
list_major_versions,
highest_version,
integration_version,
omnibus_software_version,
omnibus_ruby_version,
jmxfetch_version,
security_agent_policies_version,
macos_build_version,
)
# Update internal module dependencies
update_modules(ctx, _stringify_version(highest_version))
def finish(
ctx,
major_versions="6,7",
integration_version=None,
omnibus_software_version=None,
jmxfetch_version=None,
omnibus_ruby_version=None,
security_agent_policies_version=None,
macos_build_version=None,
ignore_rc_tag=False,
):
"""
Creates new entry in the release.json file for the new version. Removes all the RC entries.
Update internal module dependencies with the new version.
"""
if sys.version_info[0] < 3:
print("Must use Python 3 for this task")
return Exit(code=1)
list_major_versions = major_versions.split(",")
print("Finishing release for major version(s) {}".format(
list_major_versions))
list_major_versions = [int(x) for x in list_major_versions]
highest_major = 0
for version in list_major_versions:
if int(version) > highest_major:
highest_major = version
github_token = os.environ.get('GITHUB_TOKEN')
if github_token is None:
print(
"Error: set the GITHUB_TOKEN environment variable.\nYou can create one by going to"
" https://github.com/settings/tokens. It should have at least the 'repo' permissions."
)
return Exit(code=1)
# We want to match:
# - X.Y.Z
# - X.Y.Z-rc.t
# - vX.Y(.Z) (security-agent-policies repo)
version_re = re.compile(r'(v)?(\d+)[.](\d+)([.](\d+))?(-rc\.(\d+))?')
with open("release.json", "r") as release_json_stream:
release_json = json.load(release_json_stream,
object_pairs_hook=OrderedDict)
highest_version = _get_highest_version_from_release_json(
release_json, highest_major, version_re)
# jmxfetch and security-agent-policies follow their own version scheme
highest_jmxfetch_version = _get_highest_version_from_release_json(
release_json,
highest_major,
version_re,
release_json_key="JMXFETCH_VERSION",
)
highest_security_agent_policies_version = _get_highest_version_from_release_json(
release_json,
highest_major,
version_re,
release_json_key="SECURITY_AGENT_POLICIES_VERSION",
)
# Erase RCs
for major_version in list_major_versions:
highest_version["major"] = major_version
rc = highest_version["rc"]
while highest_version["rc"] not in [0, None]:
# In case we have skipped an RC in the file...
try:
release_json.pop(_stringify_version(highest_version))
finally:
highest_version["rc"] = highest_version["rc"] - 1
highest_version["rc"] = rc
# Tags in other repos are based on the highest major (e.g. for releasing version 6.X.Y and 7.X.Y they will tag only 7.X.Y)
highest_version["major"] = highest_major
# We don't want to fetch RC tags
highest_version["rc"] = None
if not integration_version:
integration_version = _get_highest_repo_version(
github_token, "integrations-core", highest_version, version_re)
if integration_version is None:
print(
"ERROR: No version found for integrations-core - did you create the tag?"
)
return Exit(code=1)
if integration_version["rc"] is not None:
print(
"ERROR: integrations-core tag is still an RC tag. That's probably NOT what you want in the final artifact."
)
if ignore_rc_tag:
print("Continuing with RC tag on integrations-core.")
else:
print("Aborting.")
return Exit(code=1)
print("integrations-core's tag is {}".format(
_stringify_version(integration_version)))
if not omnibus_software_version:
omnibus_software_version = _get_highest_repo_version(
github_token, "omnibus-software", highest_version, version_re)
if omnibus_software_version is None:
print(
"ERROR: No version found for omnibus-software - did you create the tag?"
)
return Exit(code=1)
if omnibus_software_version["rc"] is not None:
print(
"ERROR: omnibus-software tag is still an RC tag. That's probably NOT what you want in the final artifact."
)
if ignore_rc_tag:
print("Continuing with RC tag on omnibus-software.")
else:
print("Aborting.")
return Exit(code=1)
print("omnibus-software's tag is {}".format(
_stringify_version(omnibus_software_version)))
if not jmxfetch_version:
jmxfetch_version = _get_highest_repo_version(github_token, "jmxfetch",
highest_jmxfetch_version,
version_re)
print("jmxfetch's tag is {}".format(_stringify_version(jmxfetch_version)))
if not omnibus_ruby_version:
omnibus_ruby_version = _get_highest_repo_version(
github_token, "omnibus-ruby", highest_version, version_re)
if omnibus_ruby_version is None:
print(
"ERROR: No version found for omnibus-ruby - did you create the tag?"
)
return Exit(code=1)
if omnibus_ruby_version["rc"] is not None:
print(
"ERROR: omnibus-ruby tag is still an RC tag. That's probably NOT what you want in the final artifact."
)
if ignore_rc_tag:
print("Continuing with RC tag on omnibus-ruby.")
else:
print("Aborting.")
return Exit(code=1)
print("omnibus-ruby's tag is {}".format(
_stringify_version(omnibus_ruby_version)))
if not security_agent_policies_version:
security_agent_policies_version = _get_highest_repo_version(
github_token, "security-agent-policies",
highest_security_agent_policies_version, version_re)
print("security-agent-policies' tag is {}".format(
_stringify_version(security_agent_policies_version)))
if not macos_build_version:
macos_build_version = _get_highest_repo_version(
github_token, "datadog-agent-macos-build", highest_version,
version_re)
if macos_build_version is None:
print(
"ERROR: No version found for datadog-agent-macos-build - did you create the tag?"
)
return Exit(code=1)
if macos_build_version["rc"] is not None:
print(
"ERROR: datadog-agent-macos-build tag is still an RC tag. That's probably NOT what you want in the final artifact."
)
if ignore_rc_tag:
print("Continuing with RC tag on datadog-agent-macos-build.")
else:
print("Aborting.")
return Exit(code=1)
print("datadog-agent-macos-build' tag is {}".format(
_stringify_version(macos_build_version)))
_save_release_json(
release_json,
list_major_versions,
highest_version,
integration_version,
omnibus_software_version,
omnibus_ruby_version,
jmxfetch_version,
security_agent_policies_version,
macos_build_version,
)
# Update internal module dependencies
update_modules(ctx, _stringify_version(highest_version))
def test(
ctx,
module=None,
targets=None,
coverage=False,
build_include=None,
build_exclude=None,
verbose=False,
race=False,
profile=False,
fail_on_fmt=False,
rtloader_root=None,
python_home_2=None,
python_home_3=None,
cpus=0,
major_version='7',
python_runtimes='3',
timeout=120,
arch="x64",
cache=True,
skip_linters=False,
save_result_json=None,
go_mod="mod",
):
"""
Run all the tools and tests on the given module and targets.
A module should be provided as the path to one of the go modules in the repository.
Targets should be provided as a comma-separated list of relative paths within the given module.
If targets are provided but no module is set, the main module (".") is used.
If no module or target is set the tests are run against all modules and targets.
Example invokation:
inv test --targets=./pkg/collector/check,./pkg/aggregator --race
inv test --module=. --race
"""
if isinstance(module, str):
# when this function is called from the command line, targets are passed
# as comma separated tokens in a string
if isinstance(targets, str):
modules = [GoModule(module, targets=targets.split(','))]
else:
modules = [m for m in DEFAULT_MODULES.values() if m.path == module]
elif isinstance(targets, str):
modules = [GoModule(".", targets=targets.split(','))]
else:
print("Using default modules and targets")
modules = DEFAULT_MODULES.values()
build_include = (get_default_build_tags(build="test-with-process-tags",
arch=arch) if build_include is None
else filter_incompatible_tags(build_include.split(","),
arch=arch))
build_exclude = [] if build_exclude is None else build_exclude.split(",")
build_tags = get_build_tags(build_include, build_exclude)
timeout = int(timeout)
# explicitly run these tasks instead of using pre-tasks so we can
# pass the `target` param (pre-tasks are invoked without parameters)
print("--- go generating:")
generate(ctx)
if skip_linters:
print("--- [skipping linters]")
else:
print("--- Linting licenses:")
lint_licenses(ctx)
print("--- Linting filenames:")
lint_filenames(ctx)
# Until all packages whitelisted in .golangci.yml are fixed and removed
# from the 'skip-dirs' list we need to keep using the old functions that
# lint without build flags (linting some file is better than no linting).
print("--- Vetting and linting (legacy):")
for module in modules:
print("----- Module '{}'".format(module.full_path()))
if not module.condition():
print("----- Skipped")
continue
with ctx.cd(module.full_path()):
vet(ctx,
targets=module.targets,
rtloader_root=rtloader_root,
build_tags=build_tags,
arch=arch)
fmt(ctx, targets=module.targets, fail_on_fmt=fail_on_fmt)
lint(ctx, targets=module.targets)
misspell(ctx, targets=module.targets)
ineffassign(ctx, targets=module.targets)
staticcheck(ctx,
targets=module.targets,
build_tags=build_tags,
arch=arch)
# for now we only run golangci_lint on Unix as the Windows env need more work
if sys.platform != 'win32':
print("--- golangci_lint:")
for module in modules:
print("----- Module '{}'".format(module.full_path()))
if not module.condition():
print("----- Skipped")
continue
with ctx.cd(module.full_path()):
golangci_lint(ctx,
targets=module.targets,
rtloader_root=rtloader_root,
build_tags=build_tags,
arch=arch)
with open(PROFILE_COV, "w") as f_cov:
f_cov.write("mode: count")
ldflags, gcflags, env = get_build_flags(
ctx,
rtloader_root=rtloader_root,
python_home_2=python_home_2,
python_home_3=python_home_3,
major_version=major_version,
python_runtimes=python_runtimes,
)
if sys.platform == 'win32':
env['CGO_LDFLAGS'] += ' -Wl,--allow-multiple-definition'
if profile:
test_profiler = TestProfiler()
else:
test_profiler = None # Use stdout
race_opt = ""
covermode_opt = ""
build_cpus_opt = ""
if cpus:
build_cpus_opt = "-p {}".format(cpus)
if race:
# race doesn't appear to be supported on non-x64 platforms
if arch == "x86":
print(
"\n -- Warning... disabling race test, not supported on this platform --\n"
)
else:
race_opt = "-race"
# Needed to fix an issue when using -race + gcc 10.x on Windows
# https://github.com/bazelbuild/rules_go/issues/2614
if sys.platform == 'win32':
ldflags += " -linkmode=external"
if coverage:
if race:
# atomic is quite expensive but it's the only way to run
# both the coverage and the race detector at the same time
# without getting false positives from the cover counter
covermode_opt = "-covermode=atomic"
else:
covermode_opt = "-covermode=count"
print("\n--- Running unit tests:")
coverprofile = ""
if coverage:
coverprofile = "-coverprofile={}".format(PROFILE_COV)
nocache = '-count=1' if not cache else ''
build_tags.append("test")
TMP_JSON = 'tmp.json'
if save_result_json and os.path.isfile(save_result_json):
# Remove existing file since we append to it.
# We don't need to do that for TMP_JSON since gotestsum overwrites the output.
print("Removing existing '{}' file".format(save_result_json))
os.remove(save_result_json)
cmd = 'go run gotest.tools/gotestsum {json_flag} --format pkgname -- {verbose} -mod={go_mod} -vet=off -timeout {timeout}s -tags "{go_build_tags}" -gcflags="{gcflags}" '
cmd += '-ldflags="{ldflags}" {build_cpus} {race_opt} -short {covermode_opt} {coverprofile} {nocache} {pkg_folder}'
args = {
"go_mod":
go_mod,
"go_build_tags":
" ".join(build_tags),
"gcflags":
gcflags,
"ldflags":
ldflags,
"race_opt":
race_opt,
"build_cpus":
build_cpus_opt,
"covermode_opt":
covermode_opt,
"coverprofile":
coverprofile,
"timeout":
timeout,
"verbose":
'-v' if verbose else '',
"nocache":
nocache,
"json_flag":
'--jsonfile "{}" '.format(TMP_JSON) if save_result_json else "",
}
failed_modules = []
for module in modules:
print("----- Module '{}'".format(module.full_path()))
if not module.condition():
print("----- Skipped")
continue
with ctx.cd(module.full_path()):
res = ctx.run(
cmd.format(pkg_folder=' '.join(
"{}/...".format(t) if not t.endswith("/...") else t
for t in module.targets),
**args),
env=env,
out_stream=test_profiler,
warn=True,
)
if res.exited is None or res.exited > 0:
failed_modules.append(module.full_path())
if save_result_json:
with open(save_result_json, 'ab') as json_file, open(
os.path.join(module.full_path(), TMP_JSON),
'rb') as module_file:
json_file.write(module_file.read())
if failed_modules:
# Exit if any of the modules failed
raise Exit(
code=1,
message="Unit tests failed in the following modules: {}".format(
', '.join(failed_modules)))
if coverage:
print("\n--- Test coverage:")
ctx.run("go tool cover -func {}".format(PROFILE_COV))
if profile:
print("\n--- Top 15 packages sorted by run time:")
test_profiler.print_sorted(15)
def deps(ctx, no_checks=False, core_dir=None, verbose=False, android=False, dep_vendor_only=False, no_dep_ensure=False):
"""
Setup Go dependencies
"""
deps = get_deps('deps')
order = deps.get("order", deps.keys())
for dependency in order:
tool = deps.get(dependency)
if not tool:
print("Malformed bootstrap JSON, dependency {} not found". format(dependency))
raise Exit(code=1)
print("processing checkout tool {}".format(dependency))
process_deps(ctx, dependency, tool.get('version'), tool.get('type'), 'checkout', verbose=verbose)
order = deps.get("order", deps.keys())
for dependency in order:
tool = deps.get(dependency)
print("processing get tool {}".format(dependency))
process_deps(ctx, dependency, tool.get('version'), tool.get('type'), 'install', verbose=verbose)
if android:
ndkhome = os.environ.get('ANDROID_NDK_HOME')
if not ndkhome:
print("set ANDROID_NDK_HOME to build android")
raise Exit(code=1)
cmd = "gomobile init -ndk {}". format(ndkhome)
print("gomobile command {}". format(cmd))
ctx.run(cmd)
if not no_dep_ensure:
# source level deps
print("calling dep ensure")
start = datetime.datetime.now()
verbosity = ' -v' if verbose else ''
vendor_only = ' --vendor-only' if dep_vendor_only else ''
ctx.run("dep ensure{}{}".format(verbosity, vendor_only))
dep_done = datetime.datetime.now()
# If github.com/DataDog/datadog-agent gets vendored too - nuke it
#
# This may happen as a result of having to introduce DEPPROJECTROOT
# in our builders to get around a known-issue with go dep, and the
# strange GOPATH situation in our builders.
#
# This is only a workaround, we should eliminate the need to resort
# to DEPPROJECTROOT.
if os.path.exists('vendor/github.com/DataDog/datadog-agent'):
print("Removing vendored github.com/DataDog/datadog-agent")
shutil.rmtree('vendor/github.com/DataDog/datadog-agent')
# make sure PSUTIL is gone on windows; the dep ensure above will vendor it
# in because it's necessary on other platforms
if not android and sys.platform == 'win32':
print("Removing PSUTIL on Windows")
ctx.run("rd /s/q vendor\\github.com\\shirou\\gopsutil")
# Make sure that golang.org/x/mobile is deleted. It will get vendored in
# because we use it, and there's no way to exclude; however, we must use
# the version from $GOPATH
if os.path.exists('vendor/golang.org/x/mobile'):
print("Removing vendored golang.org/x/mobile")
shutil.rmtree('vendor/golang.org/x/mobile')
checks_start = datetime.datetime.now()
if not no_checks:
verbosity = 'v' if verbose else 'q'
core_dir = core_dir or os.getenv('DD_CORE_DIR')
if core_dir:
checks_base = os.path.join(os.path.abspath(core_dir), 'datadog_checks_base')
ctx.run('pip install -{} -e "{}[deps]"'.format(verbosity, checks_base))
else:
core_dir = os.path.join(os.getcwd(), 'vendor', 'integrations-core')
checks_base = os.path.join(core_dir, 'datadog_checks_base')
if not os.path.isdir(core_dir):
ctx.run('git clone -{} https://github.com/DataDog/integrations-core {}'.format(verbosity, core_dir))
ctx.run('pip install -{} "{}[deps]"'.format(verbosity, checks_base))
checks_done = datetime.datetime.now()
if not no_dep_ensure:
print("dep ensure, elapsed: {}".format(dep_done - start))
print("checks install elapsed: {}".format(checks_done - checks_start))
def update_changelog(ctx, new_version):
"""
Quick task to generate the new CHANGELOG using reno when releasing a minor
version (linux/macOS only).
"""
new_version_int = list(map(int, new_version.split(".")))
if len(new_version_int) != 3:
print("Error: invalid version: {}".format(new_version_int))
raise Exit(1)
# let's avoid losing uncommitted change with 'git reset --hard'
try:
ctx.run("git diff --exit-code HEAD", hide="both")
except Failure as e:
print(
"Error: You have uncommitted change, please commit or stash before using update_changelog"
)
return
# make sure we are up to date
ctx.run("git fetch")
# let's check that the tag for the new version is present (needed by reno)
try:
ctx.run("git tag --list | grep {}".format(new_version))
except Failure as e:
print("Missing '{}' git tag: mandatory to use 'reno'".format(
new_version))
raise
# removing releasenotes from bugfix on the old minor.
branching_point = "{}.{}.0-devel".format(new_version_int[0],
new_version_int[1])
previous_minor = "{}.{}".format(new_version_int[0], new_version_int[1] - 1)
if previous_minor == "7.15":
previous_minor = "6.15" # 7.15 is the first release in the 7.x series
log_result = ctx.run(
"git log {}...remotes/origin/{}.x --name-only --oneline | \
grep releasenotes/notes/ || true".format(branching_point,
previous_minor))
log_result = log_result.stdout.replace('\n', ' ').strip()
if len(log_result) > 0:
ctx.run("git rm --ignore-unmatch {}".format(log_result))
# generate the new changelog
ctx.run("reno report \
--ignore-cache \
--earliest-version {} \
--version {} \
--no-show-source > /tmp/new_changelog.rst".format(
branching_point, new_version))
# reseting git
ctx.run("git reset --hard HEAD")
# mac's `sed` has a different syntax for the "-i" paramter
sed_i_arg = "-i"
if sys.platform == 'darwin':
sed_i_arg = "-i ''"
# check whether there is a v6 tag on the same v7 tag, if so add the v6 tag to the release title
v6_tag = ""
if new_version_int[0] == 7:
v6_tag = _find_v6_tag(ctx, new_version)
if v6_tag:
ctx.run(
"sed {0} -E 's#^{1}#{1} / {2}#' /tmp/new_changelog.rst".format(
sed_i_arg, new_version, v6_tag))
# remove the old header from the existing changelog
ctx.run("sed {0} -e '1,4d' CHANGELOG.rst".format(sed_i_arg))
# merging to CHANGELOG.rst
ctx.run(
"cat CHANGELOG.rst >> /tmp/new_changelog.rst && mv /tmp/new_changelog.rst CHANGELOG.rst"
)
# commit new CHANGELOG
ctx.run("git add CHANGELOG.rst \
&& git commit -m \"Update CHANGELOG for {}\"".format(new_version))
def dockerize_test(ctx, binary, skip_cleanup=False):
"""
Run a go test in a remote docker environment and pipe its output to stdout.
Host and target systems must be identical (test is built on the host).
"""
import docker
client = docker.from_env()
temp_folder = tempfile.mkdtemp(prefix="ddtest-")
ctx.run("cp %s %s/test.bin" % (binary, temp_folder))
with open("%s/Dockerfile" % temp_folder, 'w') as stream:
stream.write("""FROM debian:stretch-slim
ENV DOCKER_DD_AGENT=yes
WORKDIR /
ADD https://github.com/docker/compose/releases/download/1.16.1/docker-compose-Linux-x86_64 /bin/docker-compose
RUN echo "1804b0ce6596efe707b9cab05d74b161833ed503f0535a937dd5d17bea8fc50a /bin/docker-compose" > sum && \
sha256sum -c sum && \
chmod +x /bin/docker-compose
CMD /test.bin
COPY test.bin /test.bin
""")
# Handle optional testdata folder
if os.path.isdir("./testdata"):
ctx.run("cp -R testdata %s" % temp_folder)
stream.write("COPY testdata /testdata")
test_image, _ = client.images.build(path=temp_folder, rm=True)
scratch_volume = client.volumes.create()
test_container = client.containers.run(
test_image.id,
detach=True,
pid_mode="host", # For origin detection
environment=[
"SCRATCH_VOLUME_NAME=" + scratch_volume.name,
"SCRATCH_VOLUME_PATH=/tmp/scratch",
],
volumes={
'/var/run/docker.sock': {
'bind': '/var/run/docker.sock',
'mode': 'ro'
},
'/proc': {
'bind': '/host/proc',
'mode': 'ro'
},
'/sys/fs/cgroup': {
'bind': '/host/sys/fs/cgroup',
'mode': 'ro'
},
scratch_volume.name: {
'bind': '/tmp/scratch',
'mode': 'rw'
}
})
exit_code = test_container.wait()['StatusCode']
print(test_container.logs(stdout=True, stderr=False, stream=False))
sys.stderr.write(
test_container.logs(stdout=False, stderr=True, stream=False))
if not skip_cleanup:
shutil.rmtree(temp_folder)
test_container.remove(v=True, force=True)
scratch_volume.remove(force=True)
client.images.remove(test_image.id)
if exit_code != 0:
raise Exit(code=exit_code)
def get_integrations_from_cache(ctx, python, bucket, branch, integrations_dir, target_dir, integrations, awscli="aws"):
"""
Get cached integration wheels for given integrations.
python: Python version to retrieve integrations for
bucket: S3 bucket to retrieve integration wheels from
branch: namespace in the bucket to get the integration wheels from
integrations_dir: directory with Git repository of integrations
target_dir: local directory to put integration wheels to
integrations: comma-separated names of the integrations to try to retrieve from cache
awscli: AWS CLI executable to call
"""
integrations_hashes = {}
for integration in integrations.strip().split(","):
integration_path = os.path.join(integrations_dir, integration)
if not os.path.exists(integration_path):
raise Exit(f"Integration {integration} given, but doesn't exist in {integrations_dir}", code=2)
last_commit = ctx.run(
LAST_DIRECTORY_COMMIT_PATTERN.format(integrations_dir=integrations_dir, integration=integration),
hide="both",
echo=False,
)
integrations_hashes[integration] = last_commit.stdout.strip()
print(f"Trying to retrieve {len(integrations_hashes)} integration wheels from cache")
# On windows, maximum length of a command line call is 8191 characters, therefore
# we do multiple syncs that fit within that limit (we use 8100 as a nice round number
# and just to make sure we don't do any of-by-one errors that would break this).
# WINDOWS NOTES: on Windows, the awscli is usually in program files, so we have to wrap the
# executable in quotes; also we have to not put the * in quotes, as there's no
# expansion on it, unlike on Linux
exclude_wildcard = "*" if platform.system().lower() == "windows" else "'*'"
sync_command_prefix = (
f"\"{awscli}\" s3 sync s3://{bucket} {target_dir} --no-sign-request --exclude {exclude_wildcard}"
)
sync_commands = [[[sync_command_prefix], len(sync_command_prefix)]]
for integration, hash in integrations_hashes.items():
include_arg = " --include " + CACHED_WHEEL_FULL_PATH_PATTERN.format(
hash=hash,
integration=integration,
python_version=python,
branch=branch,
)
if len(include_arg) + sync_commands[-1][1] > 8100:
sync_commands.append([[sync_command_prefix], len(sync_command_prefix)])
sync_commands[-1][0].append(include_arg)
sync_commands[-1][1] += len(include_arg)
for sync_command in sync_commands:
ctx.run("".join(sync_command[0]))
found = []
# move all wheel files directly to the target_dir, so they're easy to find/work with in Omnibus
for integration in sorted(integrations_hashes):
hash = integrations_hashes[integration]
original_path_glob = os.path.join(
target_dir,
CACHED_WHEEL_FULL_PATH_PATTERN.format(
hash=hash,
integration=integration,
python_version=python,
branch=branch,
),
)
files_matched = glob.glob(original_path_glob)
if len(files_matched) == 0:
continue
elif len(files_matched) > 1:
raise Exit(
f"More than 1 wheel for integration {integration} matched by {original_path_glob}: {files_matched}"
)
wheel_path = files_matched[0]
print(f"Found cached wheel for integration {integration}")
shutil.move(wheel_path, target_dir)
found.append(f"datadog_{integration}")
print(f"Found {len(found)} cached integration wheels")
with open(os.path.join(target_dir, "found.txt"), "w") as f:
f.write('\n'.join(found))
def update_dca_changelog(ctx, new_version, agent_version):
"""
Quick task to generate the new CHANGELOG-DCA using reno when releasing a minor
version (linux/macOS only).
"""
new_version_int = list(map(int, new_version.split(".")))
if len(new_version_int) != 3:
print("Error: invalid version: {}".format(new_version_int))
raise Exit(1)
agent_version_int = list(map(int, agent_version.split(".")))
if len(agent_version_int) != 3:
print("Error: invalid version: {}".format(agent_version_int))
raise Exit(1)
# let's avoid losing uncommitted change with 'git reset --hard'
try:
ctx.run("git diff --exit-code HEAD", hide="both")
except Failure:
print(
"Error: You have uncommitted changes, please commit or stash before using update-dca-changelog"
)
return
# make sure we are up to date
ctx.run("git fetch")
# let's check that the tag for the new version is present (needed by reno)
try:
ctx.run("git tag --list | grep dca-{}".format(new_version))
except Failure:
print("Missing 'dca-{}' git tag: mandatory to use 'reno'".format(
new_version))
raise
# Cluster agent minor releases are in sync with the agent's, bugfixes are not necessarily.
# We rely on the agent's devel tag to enforce the sync between both releases.
branching_point_agent = "{}.{}.0-devel".format(agent_version_int[0],
agent_version_int[1])
previous_minor_branchoff = "dca-{}.{}.X".format(new_version_int[0],
new_version_int[1] - 1)
log_result = ctx.run(
"git log {}...remotes/origin/{} --name-only --oneline | \
grep releasenotes-dca/notes/ || true".format(
branching_point_agent, previous_minor_branchoff))
log_result = log_result.stdout.replace('\n', ' ').strip()
# Do not include release notes that were added in the previous minor release branch (previous_minor_branchoff)
# and the branch-off points for the current release (pined by the agent's devel tag)
if len(log_result) > 0:
ctx.run("git rm --ignore-unmatch {}".format(log_result))
current_branchoff = "dca-{}.{}.X".format(new_version_int[0],
new_version_int[1])
# generate the new changelog. Specifying branch in case this is run outside the release branch that contains the tag.
ctx.run("reno --rel-notes-dir releasenotes-dca report \
--ignore-cache \
--branch {} \
--version dca-{} \
--no-show-source > /tmp/new_changelog-dca.rst".format(
current_branchoff, new_version))
# reseting git
ctx.run("git reset --hard HEAD")
# mac's `sed` has a different syntax for the "-i" paramter
sed_i_arg = "-i"
if sys.platform == 'darwin':
sed_i_arg = "-i ''"
# remove the old header from the existing changelog
ctx.run("sed {0} -e '1,4d' CHANGELOG-DCA.rst".format(sed_i_arg))
if sys.platform != 'darwin':
# sed on darwin doesn't support `-z`. On mac, you will need to manually update the following.
ctx.run(
"sed -z {0} -e 's/dca-{1}\\n===={2}/{1}\\n{2}/' /tmp/new_changelog-dca.rst"
.format(sed_i_arg, new_version, '=' * len(new_version)))
# merging to CHANGELOG.rst
ctx.run(
"cat CHANGELOG-DCA.rst >> /tmp/new_changelog-dca.rst && mv /tmp/new_changelog-dca.rst CHANGELOG-DCA.rst"
)
# commit new CHANGELOG
ctx.run("git add CHANGELOG-DCA.rst")
print("\nCommit this with:")
print(
"git commit -m \"[DCA] Update CHANGELOG for {}\"".format(new_version))
def check_version(agent_version):
"""Check Agent version to see if it is valid."""
version_re = re.compile(r'7[.](\d+)[.](\d+)(-rc\.(\d+))?')
if not version_re.match(agent_version):
raise Exit(message="Version should be of the form 7.Y.Z or 7.Y.Z-rc.t")
def abort():
raise Exit(code=-1)
def deps(
ctx,
verbose=False,
android=False,
no_bootstrap=False,
no_dep_ensure=False,
):
"""
Setup Go dependencies
"""
if not no_bootstrap:
deps = get_deps('deps')
order = deps.get("order", deps.keys())
for dependency in order:
tool = deps.get(dependency)
if not tool:
print(
"Malformed bootstrap JSON, dependency {} not found".format(
dependency))
raise Exit(code=1)
print("processing checkout tool {}".format(dependency))
process_deps(ctx,
dependency,
tool.get('version'),
tool.get('type'),
'checkout',
verbose=verbose)
order = deps.get("order", deps.keys())
for dependency in order:
tool = deps.get(dependency)
if tool.get('install', True):
print("processing get tool {}".format(dependency))
process_deps(
ctx,
dependency,
tool.get('version'),
tool.get('type'),
'install',
cmd=tool.get('cmd'),
verbose=verbose,
)
if android:
ndkhome = os.environ.get('ANDROID_NDK_HOME')
if not ndkhome:
print("set ANDROID_NDK_HOME to build android")
raise Exit(code=1)
cmd = "gomobile init -ndk {}".format(ndkhome)
print("gomobile command {}".format(cmd))
ctx.run(cmd)
if not no_dep_ensure:
# source level deps
print("calling go mod vendor")
start = datetime.datetime.now()
verbosity = ' -v' if verbose else ''
ctx.run("go mod vendor{}".format(verbosity))
# use modvendor to copy missing files dependencies
ctx.run('{}/bin/modvendor -copy="**/*.c **/*.h **/*.proto"{}'.format(
get_gopath(ctx), verbosity))
dep_done = datetime.datetime.now()
# If github.com/DataDog/datadog-agent gets vendored too - nuke it
#
# This may happen as a result of having to introduce DEPPROJECTROOT
# in our builders to get around a known-issue with go dep, and the
# strange GOPATH situation in our builders.
#
# This is only a workaround, we should eliminate the need to resort
# to DEPPROJECTROOT.
if os.path.exists('vendor/github.com/DataDog/datadog-agent'):
print("Removing vendored github.com/DataDog/datadog-agent")
shutil.rmtree('vendor/github.com/DataDog/datadog-agent')
# make sure PSUTIL is gone on windows; the go mod above will vendor it
# in because it's necessary on other platforms
if not android and sys.platform == 'win32':
print("Removing PSUTIL on Windows")
ctx.run("rd /s/q vendor\\github.com\\shirou\\gopsutil")
# Make sure that golang.org/x/mobile is deleted. It will get vendored in
# because we use it, and there's no way to exclude; however, we must use
# the version from $GOPATH
if os.path.exists('vendor/golang.org/x/mobile'):
print("Removing vendored golang.org/x/mobile")
shutil.rmtree('vendor/golang.org/x/mobile')
if not no_dep_ensure:
print("go mod vendor, elapsed: {}".format(dep_done - start))
def deps(ctx, no_checks=False, core_dir=None, verbose=False, android=False):
"""
Setup Go dependencies
"""
verbosity = ' -v' if verbose else ''
deps = get_deps('deps')
order = deps.get("order", deps.keys())
env = load_release_versions(ctx)
for dependency in order:
tool = deps.get(dependency)
if not tool:
print("Malformed bootstrap JSON, dependency {} not found".format(
dependency))
raise Exit(code=1)
print("processing checkout tool {}".format(dependency))
process_deps(ctx,
dependency,
tool.get('version'),
tool.get('type'),
'checkout',
verbose=verbose)
order = deps.get("order", deps.keys())
for dependency in order:
tool = deps.get(dependency)
if tool.get('install', True):
print("processing get tool {}".format(dependency))
process_deps(ctx,
dependency,
tool.get('version'),
tool.get('type'),
'install',
verbose=verbose)
if android:
ndkhome = os.environ.get('ANDROID_NDK_HOME')
if not ndkhome:
print("set ANDROID_NDK_HOME to build android")
raise Exit(code=1)
cmd = "gomobile init -ndk {}".format(ndkhome)
print("gomobile command {}".format(cmd))
ctx.run(cmd)
# source level deps
print("calling dep ensure")
start = datetime.datetime.now()
ctx.run("dep ensure{}".format(verbosity))
dep_done = datetime.datetime.now()
# If github.com/DataDog/datadog-agent gets vendored too - nuke it
#
# This may happen as a result of having to introduce DEPPROJECTROOT
# in our builders to get around a known-issue with go dep, and the
# strange GOPATH situation in our builders.
#
# This is only a workaround, we should eliminate the need to resort
# to DEPPROJECTROOT.
if os.path.exists('vendor/github.com/DataDog/datadog-agent'):
print("Removing vendored github.com/DataDog/datadog-agent")
shutil.rmtree('vendor/github.com/DataDog/datadog-agent')
# make sure PSUTIL is gone on windows; the dep ensure above will vendor it
# in because it's necessary on other platforms
if not android and sys.platform == 'win32':
print("Removing PSUTIL on Windows")
ctx.run("rd /s/q vendor\\github.com\\shirou\\gopsutil")
# Make sure that golang.org/x/mobile is deleted. It will get vendored in
# because we use it, and there's no way to exclude; however, we must use
# the version from $GOPATH
if os.path.exists('vendor/golang.org/x/mobile'):
print("Removing vendored golang.org/x/mobile")
shutil.rmtree('vendor/golang.org/x/mobile')
checks_start = datetime.datetime.now()
if not no_checks:
verbosity = 'v' if verbose else 'q'
core_dir = core_dir or os.getenv('STACKSTATE_INTEGRATIONS_DIR')
if core_dir:
checks_base = os.path.join(os.path.abspath(core_dir),
'stackstate_checks_base')
ctx.run('pip install -{} -e {}'.format(verbosity, checks_base))
ctx.run('pip install -{} -r {}'.format(
verbosity, os.path.join(checks_base, 'requirements.in')))
else:
core_dir = os.path.join(os.getcwd(), 'vendor',
'stackstate-agent-integrations')
checks_base = os.path.join(core_dir, 'stackstate_checks_base')
if not os.path.isdir(core_dir):
ctx.run(
'git clone -{} https://github.com/StackVista/stackstate-agent-integrations {}'
.format(verbosity, core_dir))
integrations_core_version = os.getenv(
'STACKSTATE_INTEGRATIONS_VERSION'
) or env['STACKSTATE_INTEGRATIONS_VERSION'] or "master"
ctx.run('git -C {} checkout {}'.format(core_dir,
integrations_core_version))
tags_output = ctx.run("git -C " + core_dir +
" ls-remote --tags | awk -F/ '{ print $3 }'")
tags = tags_output.stdout
if integrations_core_version in tags:
ctx.run('git -C {} pull origin master'.format(core_dir))
else:
ctx.run('git -C {} pull'.format(core_dir))
ctx.run('pip install -{} {}'.format(verbosity, checks_base))
ctx.run('pip install -{} -r {}'.format(
verbosity, os.path.join(checks_base, 'requirements.in')))
checks_done = datetime.datetime.now()
print("dep ensure, elapsed: {}".format(dep_done - start))
print("checks install elapsed: {}".format(checks_done - checks_start))
def finish(ctx,
major_versions="6,7",
integration_version=None,
omnibus_software_version=None,
jmxfetch_version=None,
omnibus_ruby_version=None,
ignore_rc_tag=False):
"""
Creates new entry in the release.json file for the new version. Removes all the RC entries.
"""
list_major_versions = major_versions.split(",")
print("Creating new agent version(s) {}".format(list_major_versions))
list_major_versions = map(lambda x: int(x), list_major_versions)
highest_major = 0
for version in list_major_versions:
if int(version) > highest_major:
highest_major = version
github_token = os.environ.get('GITHUB_TOKEN')
if github_token is None:
print(
"Error: set the GITHUB_TOKEN environment variable.\nYou can create one by going to"
" https://github.com/settings/tokens. It should have at least the 'repo' permissions."
)
return Exit(code=1)
version_re = re.compile('(\\d+)[.](\\d+)[.](\\d+)(-rc\\.(\\d+))?')
auth = "https://{}:x-oauth-basic@".format(github_token)
with open("release.json", "r") as release_json_stream:
release_json = json.load(release_json_stream,
object_pairs_hook=OrderedDict)
highest_version, highest_jmxfetch_version = _get_highest_version_from_release_json(
release_json, highest_major, version_re)
# Erase RCs
for major_version in list_major_versions:
highest_version["major"] = major_version
rc = highest_version["rc"]
while highest_version["rc"] not in [0, None]:
# In case we have skipped an RC in the file...
try:
release_json.pop(_stringify_version(highest_version))
finally:
highest_version["rc"] = highest_version["rc"] - 1
highest_version["rc"] = rc
# Tags in other repos are based on the highest major (e.g. for releasing version 6.X.Y and 7.X.Y they will tag only 7.X.Y)
highest_version["major"] = highest_major
# We don't want to fetch RC tags
highest_version["rc"] = None
if not integration_version:
integration_version = _get_highest_repo_version(
auth, "integrations-core", highest_version, version_re)
if integration_version is None:
print(
"EREROR: No version found for integrationscore - did you create the tag ?"
)
return Exit(code=1)
if integration_version["rc"] != None:
print(
"ERROR: Integration-Core tag is still an RC tag. That's probably NOT what you want in the final artifact."
)
if ignore_rc_tag:
print("Continuing with RC tag on Integration-Core.")
else:
print("Aborting.")
return Exit(code=1)
integration_version = _stringify_version(integration_version)
print("Integration-Core's tag is {}".format(integration_version))
if not omnibus_software_version:
omnibus_software_version = _get_highest_repo_version(
auth, "omnibus-software", highest_version, version_re)
if omnibus_software_version is None:
print(
"EREROR: No version found for omnibus-software - did you create the tag ?"
)
return Exit(code=1)
if omnibus_software_version["rc"] != None:
print(
"ERROR: Omnibus-Software tag is still an RC tag. That's probably NOT what you want in the final artifact."
)
if ignore_rc_tag:
print("Continuing with RC tag on Omnibus-Software.")
else:
print("Aborting.")
return Exit(code=1)
omnibus_software_version = _stringify_version(omnibus_software_version)
print("Omnibus-Software's tag is {}".format(omnibus_software_version))
if not jmxfetch_version:
jmxfetch_version = _get_highest_repo_version(auth, "jmxfetch",
highest_jmxfetch_version,
version_re)
jmxfetch_version = _stringify_version(jmxfetch_version)
print("Jmxfetch's tag is {}".format(jmxfetch_version))
if not omnibus_ruby_version:
print(
"ERROR: No omnibus_ruby_version found. Please specify it manually via '--omnibus-ruby-version' until we start tagging omnibus-ruby builds."
)
return Exit(code=1)
_save_release_json(release_json, list_major_versions, highest_version,
integration_version, omnibus_software_version,
omnibus_ruby_version, jmxfetch_version)
def test(c):
with c.prefix("source env/bin/activate"):
result = c.run("nosetests -v", warn=True)
if not result.ok and not confirm("Tests failed. Continue?"):
raise Exit()
def create_rc(ctx,
major_versions="6,7",
integration_version=None,
omnibus_software_version=None,
jmxfetch_version=None,
omnibus_ruby_version=None):
"""
Takes whatever version is the highest in release.json and adds a new RC to it.
If there was no RC, creates one and bump minor version. If there was an RC, create RC + 1.
"""
list_major_versions = major_versions.split(",")
if list_major_versions.count < 1:
print("Specify at least one major version to release")
return Exit(code=1)
print("Creating RC for agent version(s) {}".format(list_major_versions))
list_major_versions = map(lambda x: int(x), list_major_versions)
highest_major = 0
for version in list_major_versions:
if int(version) > highest_major:
highest_major = version
github_token = os.environ.get('GITHUB_TOKEN')
if github_token is None:
print(
"Error: set the GITHUB_TOKEN environment variable.\nYou can create one by going to"
" https://github.com/settings/tokens. It should have at least the 'repo' permissions."
)
return Exit(code=1)
version_re = re.compile('(\\d+)[.](\\d+)[.](\\d+)(-rc\\.(\\d+))?')
auth = "https://{}:x-oauth-basic@".format(github_token)
with open("release.json", "r") as release_json_stream:
release_json = json.load(release_json_stream,
object_pairs_hook=OrderedDict)
highest_version, highest_jmxfetch_version = _get_highest_version_from_release_json(
release_json, highest_major, version_re)
if highest_version["rc"] is None:
# No RC exists, create one
highest_version["minor"] = highest_version["minor"] + 1
highest_version["rc"] = 1
else:
# An RC exists, create next RC
highest_version["rc"] = highest_version["rc"] + 1
new_rc = _stringify_version(highest_version)
print("Creating {}".format(new_rc))
if not integration_version:
integration_version = _get_highest_repo_version(
auth, "integrations-core", highest_version, version_re)
integration_version = _stringify_version(integration_version)
print("Integration-Core's tag is {}".format(integration_version))
if not omnibus_software_version:
omnibus_software_version = _get_highest_repo_version(
auth, "omnibus-software", highest_version, version_re)
omnibus_software_version = _stringify_version(omnibus_software_version)
print("Omnibus-Software's tag is {}".format(omnibus_software_version))
if not jmxfetch_version:
jmxfetch_version = _get_highest_repo_version(auth, "jmxfetch",
highest_jmxfetch_version,
version_re)
jmxfetch_version = _stringify_version(jmxfetch_version)
print("Jmxfetch's tag is {}".format(jmxfetch_version))
if not omnibus_ruby_version:
print(
"ERROR: No omnibus_ruby_version found. Please specify it manually via '--omnibus-ruby-version' until we start tagging omnibus-ruby builds."
)
return Exit(code=1)
_save_release_json(release_json, list_major_versions, highest_version,
integration_version, omnibus_software_version,
omnibus_ruby_version, jmxfetch_version)
def bgp_dev_env():
dev_env_dir = os.getcwd() + "/dev-env/bgp"
frr_volume_dir = dev_env_dir + "/frr-volume"
# TODO -- The IP address handling will need updates to add support for IPv6
# We need the IPs for each Node in the cluster to place them in the BGP
# router configuration file (bgpd.conf). Each Node will peer with this
# router.
node_ips = run(
"kubectl get nodes -o jsonpath='{.items[*].status.addresses"
"[?(@.type==\"InternalIP\")].address}{\"\\n\"}'",
echo=True)
node_ips = node_ips.stdout.strip().split()
if len(node_ips) != 3:
raise Exit(message='Expected 3 nodes, got %d' % len(node_ips))
# Create a new directory that will be used as the config volume for frr.
try:
# sudo because past docker runs will have changed ownership of this dir
run('sudo rm -rf "%s"' % frr_volume_dir)
os.mkdir(frr_volume_dir)
except FileExistsError:
pass
except Exception as e:
raise Exit(message='Failed to create frr-volume directory: %s' %
str(e))
# These two config files are static, so we just copy them straight in.
shutil.copyfile("%s/frr/zebra.conf" % dev_env_dir,
"%s/zebra.conf" % frr_volume_dir)
shutil.copyfile("%s/frr/daemons" % dev_env_dir,
"%s/daemons" % frr_volume_dir)
# bgpd.conf is created from a template so that we can include the current
# Node IPs.
with open("%s/frr/bgpd.conf.tmpl" % dev_env_dir, 'r') as f:
bgpd_config = "! THIS FILE IS AUTOGENERATED\n" + f.read()
for n in range(0, len(node_ips)):
bgpd_config = bgpd_config.replace("NODE%d_IP" % n, node_ips[n])
with open("%s/bgpd.conf" % frr_volume_dir, 'w') as f:
f.write(bgpd_config)
# Run a BGP router in a container for all of the speakers to peer with.
run(
'for frr in $(docker ps -a -f name=frr --format {{.Names}}) ; do '
' docker rm -f $frr ; '
'done',
echo=True)
run("docker run -d --privileged --network kind --rm --name frr --volume %s:/etc/frr "
"frrouting/frr:latest" % frr_volume_dir,
echo=True)
peer_address = run(
'docker inspect -f "{{ '
'range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" frr',
echo=True)
with open("%s/config.yaml.tmpl" % dev_env_dir, 'r') as f:
mlb_config = "# THIS FILE IS AUTOGENERATED\n" + f.read()
mlb_config = mlb_config.replace("PEER_ADDRESS",
peer_address.stdout.strip())
with open("%s/config.yaml" % dev_env_dir, 'w') as f:
f.write(mlb_config)
# Apply the MetalLB ConfigMap
run("kubectl apply -f %s/config.yaml" % dev_env_dir)
def check_if_logged_in_to_heroku(c):
if not local("heroku auth:whoami", warn=True):
raise Exit(
'Log-in with the "heroku login -i" command before running this '
"command.")
def release(ctx, version, skip_release_notes=False):
"""Tag a new release."""
status = run("git status --porcelain", hide=True).stdout.strip()
if status != "":
raise Exit(message="git checkout not clean, cannot release")
version = semver.parse_version_info(version)
is_patch_release = version.patch != 0
# Check that we have release notes for the desired version.
run("git checkout main", echo=True)
if not skip_release_notes:
with open("website/content/release-notes/_index.md") as release_notes:
if "## Version {}".format(version) not in release_notes.read():
raise Exit(message="no release notes for v{}".format(version))
# Move HEAD to the correct release branch - either a new one, or
# an existing one.
if is_patch_release:
run("git checkout v{}.{}".format(version.major, version.minor),
echo=True)
else:
run("git checkout -b v{}.{}".format(version.major, version.minor),
echo=True)
# Copy over release notes from main.
if not skip_release_notes:
run("git checkout main -- website/content/release-notes/_index.md",
echo=True)
# Update links on the website to point to files at the version
# we're creating.
if is_patch_release:
previous_version = "v{}.{}.{}".format(version.major, version.minor,
version.patch - 1)
else:
previous_version = "main"
def _replace(pattern):
oldpat = pattern.format(previous_version)
newpat = pattern.format("v{}").format(version)
run("perl -pi -e 's#{}#{}#g' website/content/*.md website/content/*/*.md"
.format(oldpat, newpat),
echo=True)
_replace("/metallb/metallb/{}")
_replace("/metallb/metallb/tree/{}")
_replace("/metallb/metallb/blob/{}")
# Update the version listed on the website sidebar
run("perl -pi -e 's/MetalLB .*/MetalLB v{}/g' website/content/_header.md".
format(version),
echo=True)
# Update the manifests with the new version
run("perl -pi -e 's,image: metallb/speaker:.*,image: metallb/speaker:v{},g' manifests/metallb.yaml"
.format(version),
echo=True)
run("perl -pi -e 's,image: metallb/controller:.*,image: metallb/controller:v{},g' manifests/metallb.yaml"
.format(version),
echo=True)
# Update the version in kustomize instructions
#
# TODO: Check if kustomize instructions really need the version in the
# website or if there is a simpler way. For now, though, we just replace the
# only page that mentions the version on release.
run("perl -pi -e 's,github.com/metallb/metallb//manifests\?ref=.*,github.com/metallb/metallb//manifests\?ref=v{},g' website/content/installation/_index.md"
.format(version),
echo=True)
# Update the version embedded in the binary
run("perl -pi -e 's/version\s+=.*/version = \"{}\"/g' internal/version/version.go"
.format(version),
echo=True)
run("gofmt -w internal/version/version.go", echo=True)
run("git commit -a -m 'Automated update for release v{}'".format(version),
echo=True)
run("git tag v{} -m 'See the release notes for details:\n\nhttps://metallb.universe.tf/release-notes/#version-{}-{}-{}'"
.format(version, version.major, version.minor, version.patch),
echo=True)
run("git checkout main", echo=True)
def lint_releasenote(ctx):
"""
Lint release notes with Reno
"""
# checking if a releasenote has been added/changed
pr_url = os.environ.get("CIRCLE_PULL_REQUEST")
if pr_url:
import requests
pr_id = pr_url.rsplit('/')[-1]
# first check 'changelog/no-changelog' label
res = requests.get(
"https://api.github.com/repos/DataDog/datadog-agent/issues/{}".
format(pr_id))
issue = res.json()
if any([
l['name'] == 'changelog/no-changelog'
for l in issue.get('labels', {})
]):
print(
"'changelog/no-changelog' label found on the PR: skipping linting"
)
return
# Then check that at least one note was touched by the PR
url = "https://api.github.com/repos/DataDog/datadog-agent/pulls/{}/files".format(
pr_id)
# traverse paginated github response
while True:
res = requests.get(url)
files = res.json()
if any([f['filename'].startswith("releasenotes/notes/") or \
f['filename'].startswith("releasenotes-dca/notes/") for f in files]):
break
if 'next' in res.links:
url = res.links['next']['url']
else:
print("Error: No releasenote was found for this PR. Please add one using 'reno'"\
", or apply the label 'changelog/no-changelog' to the PR.")
raise Exit(code=1)
# The PR has not been created yet, let's compare with master (the usual base branch of the future PR)
else:
branch = os.environ.get("CIRCLE_BRANCH")
if branch is None:
print("No branch found, skipping reno linting")
else:
if re.match(r".*/.*", branch) is None:
print(
"{} is not a feature branch, skipping reno linting".format(
branch))
else:
import requests
# Then check that in the diff with master, at least one note was touched
url = "https://api.github.com/repos/DataDog/datadog-agent/compare/master...{}".format(
branch)
# traverse paginated github response
while True:
res = requests.get(url)
files = res.json().get("files", {})
if any([f['filename'].startswith("releasenotes/notes/") or \
f['filename'].startswith("releasenotes-dca/notes/") for f in files]):
break
if 'next' in res.links:
url = res.links['next']['url']
else:
print("Error: No releasenote was found for this PR. Please add one using 'reno'"\
", or apply the label 'changelog/no-changelog' to the PR.")
raise Exit(code=1)
ctx.run("reno lint")
def update_installscript_changelog(ctx, new_version):
"""
Quick task to generate the new CHANGELOG-INSTALLSCRIPT using reno when releasing a minor
version (linux/macOS only).
"""
new_version_int = list(map(int, new_version.split(".")))
if len(new_version_int) != 3:
print("Error: invalid version: {}".format(new_version_int))
raise Exit(1)
# let's avoid losing uncommitted change with 'git reset --hard'
try:
ctx.run("git diff --exit-code HEAD", hide="both")
except Failure:
print(
"Error: You have uncommitted changes, please commit or stash before using update-installscript-changelog"
)
return
# make sure we are up to date
ctx.run("git fetch")
# let's check that the tag for the new version is present (needed by reno)
try:
ctx.run("git tag --list | grep installscript-{}".format(new_version))
except Failure:
print("Missing 'installscript-{}' git tag: mandatory to use 'reno'".
format(new_version))
raise
# generate the new changelog
ctx.run("reno --rel-notes-dir releasenotes-installscript report \
--ignore-cache \
--version installscript-{} \
--no-show-source > /tmp/new_changelog-installscript.rst".format(
new_version))
# reseting git
ctx.run("git reset --hard HEAD")
# mac's `sed` has a different syntax for the "-i" paramter
sed_i_arg = "-i"
if sys.platform == 'darwin':
sed_i_arg = "-i ''"
# remove the old header from the existing changelog
ctx.run("sed {0} -e '1,4d' CHANGELOG-INSTALLSCRIPT.rst".format(sed_i_arg))
if sys.platform != 'darwin':
# sed on darwin doesn't support `-z`. On mac, you will need to manually update the following.
ctx.run(
"sed -z {0} -e 's/installscript-{1}\\n===={2}/{1}\\n{2}/' /tmp/new_changelog-installscript.rst"
.format(sed_i_arg, new_version, '=' * len(new_version)))
# merging to CHANGELOG-INSTALLSCRIPT.rst
ctx.run(
"cat CHANGELOG-INSTALLSCRIPT.rst >> /tmp/new_changelog-installscript.rst && mv /tmp/new_changelog-installscript.rst CHANGELOG-INSTALLSCRIPT.rst"
)
# commit new CHANGELOG-INSTALLSCRIPT
ctx.run("git add CHANGELOG-INSTALLSCRIPT.rst")
print("\nCommit this with:")
print(
"git commit -m \"[INSTALLSCRIPT] Update CHANGELOG-INSTALLSCRIPT for {}\""
.format(new_version))
