def test_migrate_versionfilter(self):
m = migrater.Migrater("src", "dst")
m.gather("openexr", "2019")
current_version = list(index.Index().iter_versions(
constants.ImageType.PACKAGE, "openexr"))[1]
self.assertEqual(len(m.migration_list), 1)
minfo = m.migration_list[0]
self.assertEqual(minfo.image, "ci-package-openexr")
oexr_version = list(index.Index().iter_versions(
constants.ImageType.PACKAGE, "openexr"))[1]
self.assertEqual(minfo.version, oexr_version)
self.assertEqual(
minfo.source,
f"{constants.DOCKER_REGISTRY}/src/ci-package-openexr:{current_version}",
)
self.assertEqual(
minfo.destination,
f"{constants.DOCKER_REGISTRY}/dst/ci-package-openexr:{current_version}",
)
m.migrate(dry_run=True)
self.assertEqual(
m.cmds,
[
f"docker pull {constants.DOCKER_REGISTRY}/src/ci-package-openexr:{current_version}",
f"docker tag {constants.DOCKER_REGISTRY}/src/ci-package-openexr:{current_version} {constants.DOCKER_REGISTRY}/dst/ci-package-openexr:{current_version}",
f"docker push {constants.DOCKER_REGISTRY}/dst/ci-package-openexr:{current_version}",
f"docker tag {constants.DOCKER_REGISTRY}/dst/ci-package-openexr:{current_version} {constants.DOCKER_REGISTRY}/dst/ci-package-openexr:2019",
f"docker push {constants.DOCKER_REGISTRY}/dst/ci-package-openexr:2019",
f"docker tag {constants.DOCKER_REGISTRY}/dst/ci-package-openexr:{current_version} {constants.DOCKER_REGISTRY}/dst/ci-package-openexr:latest",
f"docker push {constants.DOCKER_REGISTRY}/dst/ci-package-openexr:latest",
],
)
def images():
"""Lists all known CI images in this format: IMAGEGROUP/ci-IMAGE:VERSION
"""
for group, images in index.Index().groups[
constants.ImageType.IMAGE].items():
for image in images:
image_name = utils.get_image_name(constants.ImageType.IMAGE, image)
for version in index.Index().iter_versions(
constants.ImageType.IMAGE, image):
click.echo(f"{group}/{image_name}:{version}")
def __init__(
self,
type_: constants.ImageType,
names: typing.List[str],
versions: typing.List[str],
targets: typing.List[str],
):
self.index = index.Index()
self.type = type_
self.names = names
self.versions = [utils.get_major_version(v) for v in versions]
for name in self.names:
if name not in constants.GROUPS[self.type]:
raise TypeError(f"Group {name} is not valid!")
self.images = []
for images in [constants.GROUPS[self.type][n] for n in self.names]:
self.images.extend(images)
self.targets = targets
logger.debug(
"GroupInfo: type=%s names=%s versions=%s images=%s targets=%s",
self.type,
self.names,
self.versions,
self.images,
self.targets,
)
def test_migrate_cli(self):
runner = CliRunner()
result = runner.invoke(
aswfdocker.cli,
[
"migrate",
"--from",
"src",
"--to",
"dst",
"--package",
"openexr",
"--version",
"2019",
"--dry-run",
],
input="y\n",
)
self.assertEqual(result.exit_code, 0)
current_version = list(
index.Index().iter_versions(constants.ImageType.PACKAGE, "openexr")
)[1]
reg = constants.DOCKER_REGISTRY
self.assertEqual(
result.output,
f"Are you sure you want to migrate the following 1 packages?\n"
f"{reg}/src/ci-package-openexr:{current_version} -> "
f"{reg}/dst/ci-package-openexr:{current_version}\n"
f" [y/N]: y\n"
f"INFO:aswfdocker.migrater:Migrating "
f"{reg}/src/ci-package-openexr:{current_version} -> "
f"{reg}/dst/ci-package-openexr:{current_version}\n"
f"Migration done.\n",
)
def dockergen(context, image_name, check):
"""Generates a Docker file and readme from image data and template
"""
if image_name == "all":
images = []
for gimages in index.Index().groups[
constants.ImageType.IMAGE].values():
images.extend(gimages)
else:
images = [image_name]
if check:
for image in images:
path, ok = aswf_dockergen.DockerGen(image).check_dockerfile()
if not ok:
click.secho(f"{path} is not up to date!", fg="red")
context.exit(1)
else:
click.secho(f"{path} is up to date", fg="green")
path, ok = aswf_dockergen.DockerGen(image).check_readme()
if not ok:
click.secho(f"{path} is not up to date!", fg="red")
context.exit(1)
else:
click.secho(f"{path} is up to date", fg="green")
else:
for image in images:
path = aswf_dockergen.DockerGen(image).generate_dockerfile()
click.echo(f"Generated {path}")
path = aswf_dockergen.DockerGen(image).generate_readme()
click.echo(f"Generated {path}")
def test_package_baseqt_2019_dict(self):
r = releaser.Releaser(
self.build_info,
groupinfo.GroupInfo(
names=["base1"],
versions=["2019"],
type_=constants.ImageType.PACKAGE,
targets=["boost"],
),
sha=utils.get_current_sha(),
)
r.gh.repo.create_git_tag_and_release = mock.MagicMock()
class U:
name = "testuser"
email = "*****@*****.**"
r.gh.github.get_user = mock.MagicMock(return_value=U())
r.gather()
r.release(dry_run=False)
boost_version = list(index.Index().iter_versions(
constants.ImageType.PACKAGE, "boost"))[1]
r.gh.repo.create_git_tag_and_release.assert_called_once_with(
tag=f"aswflocaltesting/ci-package-boost/{boost_version}",
draft=False,
object=utils.get_current_sha(),
prerelease=False,
release_message=
f"Inspect released Docker image here: https://hub.docker.com/r/aswflocaltesting/ci-package-boost/tags?name={boost_version}",
release_name=f"aswflocaltesting/ci-package-boost:{boost_version}",
tag_message=f"aswflocaltesting/ci-package-boost:{boost_version}",
type="commit",
tagger=mock.ANY,
)
def test_migrate_cli(self):
current_version = list(index.Index().iter_versions(
constants.ImageType.PACKAGE, "boost"))[1]
runner = CliRunner()
result = runner.invoke(
aswfdocker.cli,
[
"release",
"-n",
f"aswf/ci-package-boost:{current_version}",
"--sha",
utils.get_current_sha(),
"--dry-run",
],
input="y\n",
)
self.assertFalse(result.exception)
self.assertEqual(result.exit_code, 0)
self.assertEqual(
result.output,
f"""Are you sure you want to create the following 1 release on sha={utils.get_current_sha()}?
aswf/ci-package-boost:{current_version}
[y/N]: y
INFO:aswfdocker.releaser:Would create this GitHub release on current commit: aswf/ci-package-boost:{current_version}
Release done.
""",
)
def test_image_base_2019_dict(self):
b = builder.Builder(
self.build_info,
groupinfo.GroupInfo(
names=["base"],
versions=["2019"],
type_=constants.ImageType.IMAGE,
targets=[],
),
)
base_version = list(
index.Index().iter_versions(constants.ImageType.IMAGE, "base")
)[1]
baked = b.make_bake_dict()
self.assertEqual(
baked["target"]["ci-base-2019"]["tags"],
[
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:2019",
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:{base_version}",
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:latest",
],
)
self.assertEqual(
baked["target"]["ci-base-2019"]["args"]["ASWF_VERSION"], base_version
)
def test_package_baseqt_2019_dict(self):
b = builder.Builder(
self.build_info,
groupinfo.GroupInfo(
names=["base2"],
versions=["2019"],
type_=constants.ImageType.PACKAGE,
targets=[],
),
)
qt_version = list(
index.Index().iter_versions(constants.ImageType.PACKAGE, "qt")
)[1]
baked = b.make_bake_dict()
self.assertEqual(
baked["target"]["ci-package-qt-2019"]["tags"],
[
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-package-qt:2019",
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-package-qt:{qt_version}",
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-package-qt:latest",
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-package-qt:2019-5.12.6",
],
)
self.assertEqual(
baked["target"]["ci-package-qt-2019"]["args"]["ASWF_VERSION"], qt_version
)
self.assertEqual(
baked["target"]["ci-package-qt-2019"]["dockerfile"],
"packages/base2/Dockerfile",
)
def __init__(
self,
type_: constants.ImageType,
names: typing.List[str],
versions: typing.List[str],
targets: typing.List[str],
):
self.index = index.Index()
self.type = type_
self.names = names
for name in self.names:
if name not in self.index.groups[self.type]:
raise TypeError(f"Group {name} is not valid!")
self.images = []
for images in [self.index.groups[self.type][n] for n in self.names]:
self.images.extend(images)
if not versions or (len(versions) == 1
and versions[0] == constants.ALL):
version_set = set()
for image in self.images:
for v in self.index.iter_versions(self.type, image):
version_set.add(utils.get_major_version(v))
versions = sorted(version_set)
self.versions = [utils.get_major_version(v) for v in sorted(versions)]
self.targets = targets
logger.debug(
"GroupInfo: type=%s names=%s versions=%s images=%s targets=%s",
self.type,
self.names,
self.versions,
self.images,
self.targets,
)
def get_group_info(build_info, ci_image_type, groups, versions, full_name,
targets):
idx = index.Index()
if full_name:
org, image_type, target, version = full_name
versions = [version]
targets = [target]
try:
groups = [idx.get_group_from_image(image_type, target)]
except RuntimeError as e:
raise click.BadOptionUsage(option_name="--full-name",
message=e.args[0])
build_info.set_org(org)
else:
if ci_image_type is None:
image_type = constants.ImageType.IMAGE
else:
image_type = constants.ImageType[ci_image_type]
if not groups and targets:
groups = [idx.get_group_from_image(image_type, targets[0])]
group_info = groupinfo.GroupInfo(
type_=image_type,
names=groups,
versions=versions,
targets=targets,
)
return group_info
def _get_image_data(self):
image_data_path = os.path.join(utils.get_git_top_level(),
f"ci-{self.image_name}/image.yaml")
with open(image_data_path) as f:
image_data = yaml.load(f, Loader=yaml.FullLoader)
image_data["index"] = index.Index()
image_data["constants"] = constants
return image_data
def __init__(
self,
build_info: aswfinfo.ASWFInfo,
group_info: groupinfo.GroupInfo,
push: bool = False,
):
self.push = push
self.build_info = build_info
self.group_info = group_info
self.index = index.Index()
def packages():
"""Lists all known ci packages in this format: PACKAGEGROUP/ci-package-PACKAGE:VERSION
"""
for group, packages in constants.GROUPS[
constants.ImageType.PACKAGE].items():
for package in packages:
image_name = utils.get_image_name(constants.ImageType.PACKAGE,
package)
for version in index.Index().iter_versions(
constants.ImageType.PACKAGE, package):
click.echo(f"{group}/{image_name}:{version}")
def test_cli_images(self):
runner = CliRunner()
result = runner.invoke(aswfdocker.cli, ["images"], catch_exceptions=False)
self.assertEqual(result.exit_code, 0)
imgs = result.output.split("\n")
self.assertGreater(len(imgs), 15)
common_version = list(
index.Index().iter_versions(constants.ImageType.IMAGE, "common")
)[0]
self.assertEqual(
imgs[0], f"common/ci-common:{common_version}",
)
def test_package_baseqt_2019_dict(self):
b = builder.Builder(
self.build_info,
groupinfo.GroupInfo(
names=["base2"],
versions=["2019"],
type_=constants.ImageType.PACKAGE,
targets=[],
),
)
qt_version = list(index.Index().iter_versions(
constants.ImageType.PACKAGE, "qt"))[1]
self.assertEqual(
b.make_bake_dict(),
{
"group": {
"default": {
"targets": ["ci-package-qt-2019"]
}
},
"target": {
"ci-package-qt-2019": {
"context":
".",
"dockerfile":
"packages/Dockerfile",
"args": {
"ASWF_ORG": "aswflocaltesting",
"ASWF_PKG_ORG": "aswftesting",
"ASWF_VERSION": qt_version,
"CI_COMMON_VERSION": "1",
"DTS_VERSION": "6",
"PYTHON_VERSION": "2.7",
"VFXPLATFORM_VERSION": "2019",
},
"labels": {
"org.opencontainers.image.created":
constants.DEV_BUILD_DATE,
"org.opencontainers.image.revision":
constants.DEV_BUILD_DATE,
},
"tags": [
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-package-qt:2019",
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-package-qt:{qt_version}",
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-package-qt:latest",
],
"target":
"ci-package-qt",
"output": ["type=docker"],
}
},
},
)
def test_cli_packages(self):
runner = CliRunner()
result = runner.invoke(aswfdocker.cli, ["packages"], catch_exceptions=False)
self.assertEqual(result.exit_code, 0)
pkgs = result.output.split("\n")
self.assertGreater(len(pkgs), 20)
clang_version = list(
index.Index().iter_versions(constants.ImageType.PACKAGE, "clang")
)[0]
self.assertEqual(
pkgs[0], f"common/ci-package-clang:{clang_version}",
)
def test_image_base_2019clang_dict(self):
b = builder.Builder(
self.build_info,
groupinfo.GroupInfo(
names=["vfx3"],
versions=["2019-clang9"],
type_=constants.ImageType.IMAGE,
targets=["openvdb"],
),
)
openvdb_version = list(
index.Index().iter_versions(constants.ImageType.IMAGE, "openvdb")
)[4]
self.assertEqual(
b.make_bake_dict(),
{
"group": {"default": {"targets": ["ci-openvdb-2019-clang9"]}},
"target": {
"ci-openvdb-2019-clang9": {
"context": ".",
"dockerfile": "ci-openvdb/Dockerfile",
"args": {
"ASWF_ALEMBIC_VERSION": "1.7.11",
"ASWF_ORG": "aswflocaltesting",
"ASWF_PKG_ORG": "aswftesting",
"ASWF_VERSION": openvdb_version,
"ASWF_BLOSC_VERSION": "1.5.0",
"ASWF_BOOST_VERSION": "1.66.0",
"ASWF_CCACHE_VERSION": "4.0",
"CI_COMMON_VERSION": "1",
"ASWF_CLANG_MAJOR_VERSION": "9",
"ASWF_CLANG_VERSION": "9.0.1",
"ASWF_CMAKE_VERSION": "3.12.4",
"ASWF_CPPUNIT_VERSION": "1.14.0",
"ASWF_CUDA_VERSION": "10.2",
"ASWF_DTS_VERSION": "6",
"ASWF_GLEW_VERSION": "2.1.0",
"ASWF_GLFW_VERSION": "3.1.2",
"ASWF_HDF5_VERSION": "1.8.21",
"ASWF_LOG4CPLUS_VERSION": "1.1.2",
"ASWF_NINJA_VERSION": "1.10.1",
"ASWF_NUMPY_VERSION": "1.14",
"ASWF_OCIO_CONFIGS_VERSION": "1.0_r2",
"ASWF_OCIO_VERSION": "1.1.0",
"ASWF_OIIO_VERSION": "2.0.8",
"ASWF_OPENEXR_VERSION": "2.3.0",
"ASWF_OPENSUBDIV_VERSION": "3_3_3",
"ASWF_OPENVDB_VERSION": "6.2.1",
"ASWF_OSL_VERSION": "1.10.9",
"ASWF_OTIO_VERSION": "0.12.1",
"ASWF_PARTIO_VERSION": "1.10.1",
"ASWF_PTEX_VERSION": "2.1.33",
"ASWF_PYBIND11_VERSION": "2.6.2",
"ASWF_PYSIDE_VERSION": "5.12.6",
"ASWF_PYTHON_VERSION": "2.7.15",
"ASWF_PYTHON_MAJOR_MINOR_VERSION": "2.7",
"ASWF_QT_VERSION": "5.12.6",
"ASWF_SONAR_VERSION": "4.5.0.2216",
"ASWF_TBB_VERSION": "2018",
"ASWF_USD_VERSION": "19.11",
"ASWF_VFXPLATFORM_VERSION": "2019",
},
"labels": {
"org.opencontainers.image.created": constants.DEV_BUILD_DATE,
"org.opencontainers.image.revision": constants.DEV_BUILD_DATE,
},
"tags": [
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-openvdb:2019-clang9",
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-openvdb:{openvdb_version}",
],
"output": ["type=docker"],
}
},
},
)
def setUp(self):
self.index = index.Index()
def __init__(self, from_org: str, to_org: str):
self.from_org = from_org
self.to_org = to_org
self.migration_list: typing.List[MigrateInfo] = []
self.cmds: typing.List[str] = []
self.index = index.Index()
def test_image_base_2019_2020_dict(self):
b = builder.Builder(
self.build_info,
groupinfo.GroupInfo(
names=["base"],
versions=["2019", "2020"],
type_=constants.ImageType.IMAGE,
targets=[],
),
)
base_versions = list(index.Index().iter_versions(
constants.ImageType.IMAGE, "base"))
self.assertEqual(
b.make_bake_dict(),
{
"group": {
"default": {
"targets": ["ci-base-2019", "ci-base-2020"]
}
},
"target": {
"ci-base-2020": {
"context":
".",
"dockerfile":
"ci-base/Dockerfile",
"args": {
"ASWF_ORG": "aswflocaltesting",
"ASWF_PKG_ORG": "aswftesting",
"ASWF_VERSION": base_versions[2],
"CI_COMMON_VERSION": "1",
"DTS_VERSION": "6",
"PYTHON_VERSION": "3.7",
"VFXPLATFORM_VERSION": "2020",
},
"labels": {
"org.opencontainers.image.created":
constants.DEV_BUILD_DATE,
"org.opencontainers.image.revision":
constants.DEV_BUILD_DATE,
},
"tags": [
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:2020",
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:{base_versions[2]}",
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:preview",
],
"output": ["type=docker"],
},
"ci-base-2019": {
"context":
".",
"dockerfile":
"ci-base/Dockerfile",
"args": {
"ASWF_ORG": "aswflocaltesting",
"ASWF_PKG_ORG": "aswftesting",
"ASWF_VERSION": base_versions[1],
"CI_COMMON_VERSION": "1",
"DTS_VERSION": "6",
"PYTHON_VERSION": "2.7",
"VFXPLATFORM_VERSION": "2019",
},
"labels": {
"org.opencontainers.image.created":
constants.DEV_BUILD_DATE,
"org.opencontainers.image.revision":
constants.DEV_BUILD_DATE,
},
"tags": [
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:2019",
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:{base_versions[1]}",
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:latest",
],
"output": ["type=docker"],
},
},
},
)
