def test_migrate_versionfilter(self): m = migrater.Migrater("src", "dst") m.gather("openexr", "2019") current_version = list(index.Index().iter_versions( constants.ImageType.PACKAGE, "openexr"))[1] self.assertEqual(len(m.migration_list), 1) minfo = m.migration_list[0] self.assertEqual(minfo.image, "ci-package-openexr") oexr_version = list(index.Index().iter_versions( constants.ImageType.PACKAGE, "openexr"))[1] self.assertEqual(minfo.version, oexr_version) self.assertEqual( minfo.source, f"{constants.DOCKER_REGISTRY}/src/ci-package-openexr:{current_version}", ) self.assertEqual( minfo.destination, f"{constants.DOCKER_REGISTRY}/dst/ci-package-openexr:{current_version}", ) m.migrate(dry_run=True) self.assertEqual( m.cmds, [ f"docker pull {constants.DOCKER_REGISTRY}/src/ci-package-openexr:{current_version}", f"docker tag {constants.DOCKER_REGISTRY}/src/ci-package-openexr:{current_version} {constants.DOCKER_REGISTRY}/dst/ci-package-openexr:{current_version}", f"docker push {constants.DOCKER_REGISTRY}/dst/ci-package-openexr:{current_version}", f"docker tag {constants.DOCKER_REGISTRY}/dst/ci-package-openexr:{current_version} {constants.DOCKER_REGISTRY}/dst/ci-package-openexr:2019", f"docker push {constants.DOCKER_REGISTRY}/dst/ci-package-openexr:2019", f"docker tag {constants.DOCKER_REGISTRY}/dst/ci-package-openexr:{current_version} {constants.DOCKER_REGISTRY}/dst/ci-package-openexr:latest", f"docker push {constants.DOCKER_REGISTRY}/dst/ci-package-openexr:latest", ], )
def images(): """Lists all known CI images in this format: IMAGEGROUP/ci-IMAGE:VERSION """ for group, images in index.Index().groups[ constants.ImageType.IMAGE].items(): for image in images: image_name = utils.get_image_name(constants.ImageType.IMAGE, image) for version in index.Index().iter_versions( constants.ImageType.IMAGE, image): click.echo(f"{group}/{image_name}:{version}")
def __init__( self, type_: constants.ImageType, names: typing.List[str], versions: typing.List[str], targets: typing.List[str], ): self.index = index.Index() self.type = type_ self.names = names self.versions = [utils.get_major_version(v) for v in versions] for name in self.names: if name not in constants.GROUPS[self.type]: raise TypeError(f"Group {name} is not valid!") self.images = [] for images in [constants.GROUPS[self.type][n] for n in self.names]: self.images.extend(images) self.targets = targets logger.debug( "GroupInfo: type=%s names=%s versions=%s images=%s targets=%s", self.type, self.names, self.versions, self.images, self.targets, )
def test_migrate_cli(self): runner = CliRunner() result = runner.invoke( aswfdocker.cli, [ "migrate", "--from", "src", "--to", "dst", "--package", "openexr", "--version", "2019", "--dry-run", ], input="y\n", ) self.assertEqual(result.exit_code, 0) current_version = list( index.Index().iter_versions(constants.ImageType.PACKAGE, "openexr") )[1] reg = constants.DOCKER_REGISTRY self.assertEqual( result.output, f"Are you sure you want to migrate the following 1 packages?\n" f"{reg}/src/ci-package-openexr:{current_version} -> " f"{reg}/dst/ci-package-openexr:{current_version}\n" f" [y/N]: y\n" f"INFO:aswfdocker.migrater:Migrating " f"{reg}/src/ci-package-openexr:{current_version} -> " f"{reg}/dst/ci-package-openexr:{current_version}\n" f"Migration done.\n", )
def dockergen(context, image_name, check): """Generates a Docker file and readme from image data and template """ if image_name == "all": images = [] for gimages in index.Index().groups[ constants.ImageType.IMAGE].values(): images.extend(gimages) else: images = [image_name] if check: for image in images: path, ok = aswf_dockergen.DockerGen(image).check_dockerfile() if not ok: click.secho(f"{path} is not up to date!", fg="red") context.exit(1) else: click.secho(f"{path} is up to date", fg="green") path, ok = aswf_dockergen.DockerGen(image).check_readme() if not ok: click.secho(f"{path} is not up to date!", fg="red") context.exit(1) else: click.secho(f"{path} is up to date", fg="green") else: for image in images: path = aswf_dockergen.DockerGen(image).generate_dockerfile() click.echo(f"Generated {path}") path = aswf_dockergen.DockerGen(image).generate_readme() click.echo(f"Generated {path}")
def test_package_baseqt_2019_dict(self): r = releaser.Releaser( self.build_info, groupinfo.GroupInfo( names=["base1"], versions=["2019"], type_=constants.ImageType.PACKAGE, targets=["boost"], ), sha=utils.get_current_sha(), ) r.gh.repo.create_git_tag_and_release = mock.MagicMock() class U: name = "testuser" email = "*****@*****.**" r.gh.github.get_user = mock.MagicMock(return_value=U()) r.gather() r.release(dry_run=False) boost_version = list(index.Index().iter_versions( constants.ImageType.PACKAGE, "boost"))[1] r.gh.repo.create_git_tag_and_release.assert_called_once_with( tag=f"aswflocaltesting/ci-package-boost/{boost_version}", draft=False, object=utils.get_current_sha(), prerelease=False, release_message= f"Inspect released Docker image here: https://hub.docker.com/r/aswflocaltesting/ci-package-boost/tags?name={boost_version}", release_name=f"aswflocaltesting/ci-package-boost:{boost_version}", tag_message=f"aswflocaltesting/ci-package-boost:{boost_version}", type="commit", tagger=mock.ANY, )
def test_migrate_cli(self): current_version = list(index.Index().iter_versions( constants.ImageType.PACKAGE, "boost"))[1] runner = CliRunner() result = runner.invoke( aswfdocker.cli, [ "release", "-n", f"aswf/ci-package-boost:{current_version}", "--sha", utils.get_current_sha(), "--dry-run", ], input="y\n", ) self.assertFalse(result.exception) self.assertEqual(result.exit_code, 0) self.assertEqual( result.output, f"""Are you sure you want to create the following 1 release on sha={utils.get_current_sha()}? aswf/ci-package-boost:{current_version} [y/N]: y INFO:aswfdocker.releaser:Would create this GitHub release on current commit: aswf/ci-package-boost:{current_version} Release done. """, )
def test_image_base_2019_dict(self): b = builder.Builder( self.build_info, groupinfo.GroupInfo( names=["base"], versions=["2019"], type_=constants.ImageType.IMAGE, targets=[], ), ) base_version = list( index.Index().iter_versions(constants.ImageType.IMAGE, "base") )[1] baked = b.make_bake_dict() self.assertEqual( baked["target"]["ci-base-2019"]["tags"], [ f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:2019", f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:{base_version}", f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:latest", ], ) self.assertEqual( baked["target"]["ci-base-2019"]["args"]["ASWF_VERSION"], base_version )
def test_package_baseqt_2019_dict(self): b = builder.Builder( self.build_info, groupinfo.GroupInfo( names=["base2"], versions=["2019"], type_=constants.ImageType.PACKAGE, targets=[], ), ) qt_version = list( index.Index().iter_versions(constants.ImageType.PACKAGE, "qt") )[1] baked = b.make_bake_dict() self.assertEqual( baked["target"]["ci-package-qt-2019"]["tags"], [ f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-package-qt:2019", f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-package-qt:{qt_version}", f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-package-qt:latest", f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-package-qt:2019-5.12.6", ], ) self.assertEqual( baked["target"]["ci-package-qt-2019"]["args"]["ASWF_VERSION"], qt_version ) self.assertEqual( baked["target"]["ci-package-qt-2019"]["dockerfile"], "packages/base2/Dockerfile", )
def __init__( self, type_: constants.ImageType, names: typing.List[str], versions: typing.List[str], targets: typing.List[str], ): self.index = index.Index() self.type = type_ self.names = names for name in self.names: if name not in self.index.groups[self.type]: raise TypeError(f"Group {name} is not valid!") self.images = [] for images in [self.index.groups[self.type][n] for n in self.names]: self.images.extend(images) if not versions or (len(versions) == 1 and versions[0] == constants.ALL): version_set = set() for image in self.images: for v in self.index.iter_versions(self.type, image): version_set.add(utils.get_major_version(v)) versions = sorted(version_set) self.versions = [utils.get_major_version(v) for v in sorted(versions)] self.targets = targets logger.debug( "GroupInfo: type=%s names=%s versions=%s images=%s targets=%s", self.type, self.names, self.versions, self.images, self.targets, )
def get_group_info(build_info, ci_image_type, groups, versions, full_name, targets): idx = index.Index() if full_name: org, image_type, target, version = full_name versions = [version] targets = [target] try: groups = [idx.get_group_from_image(image_type, target)] except RuntimeError as e: raise click.BadOptionUsage(option_name="--full-name", message=e.args[0]) build_info.set_org(org) else: if ci_image_type is None: image_type = constants.ImageType.IMAGE else: image_type = constants.ImageType[ci_image_type] if not groups and targets: groups = [idx.get_group_from_image(image_type, targets[0])] group_info = groupinfo.GroupInfo( type_=image_type, names=groups, versions=versions, targets=targets, ) return group_info
def _get_image_data(self): image_data_path = os.path.join(utils.get_git_top_level(), f"ci-{self.image_name}/image.yaml") with open(image_data_path) as f: image_data = yaml.load(f, Loader=yaml.FullLoader) image_data["index"] = index.Index() image_data["constants"] = constants return image_data
def __init__( self, build_info: aswfinfo.ASWFInfo, group_info: groupinfo.GroupInfo, push: bool = False, ): self.push = push self.build_info = build_info self.group_info = group_info self.index = index.Index()
def packages(): """Lists all known ci packages in this format: PACKAGEGROUP/ci-package-PACKAGE:VERSION """ for group, packages in constants.GROUPS[ constants.ImageType.PACKAGE].items(): for package in packages: image_name = utils.get_image_name(constants.ImageType.PACKAGE, package) for version in index.Index().iter_versions( constants.ImageType.PACKAGE, package): click.echo(f"{group}/{image_name}:{version}")
def test_cli_images(self): runner = CliRunner() result = runner.invoke(aswfdocker.cli, ["images"], catch_exceptions=False) self.assertEqual(result.exit_code, 0) imgs = result.output.split("\n") self.assertGreater(len(imgs), 15) common_version = list( index.Index().iter_versions(constants.ImageType.IMAGE, "common") )[0] self.assertEqual( imgs[0], f"common/ci-common:{common_version}", )
def test_package_baseqt_2019_dict(self): b = builder.Builder( self.build_info, groupinfo.GroupInfo( names=["base2"], versions=["2019"], type_=constants.ImageType.PACKAGE, targets=[], ), ) qt_version = list(index.Index().iter_versions( constants.ImageType.PACKAGE, "qt"))[1] self.assertEqual( b.make_bake_dict(), { "group": { "default": { "targets": ["ci-package-qt-2019"] } }, "target": { "ci-package-qt-2019": { "context": ".", "dockerfile": "packages/Dockerfile", "args": { "ASWF_ORG": "aswflocaltesting", "ASWF_PKG_ORG": "aswftesting", "ASWF_VERSION": qt_version, "CI_COMMON_VERSION": "1", "DTS_VERSION": "6", "PYTHON_VERSION": "2.7", "VFXPLATFORM_VERSION": "2019", }, "labels": { "org.opencontainers.image.created": constants.DEV_BUILD_DATE, "org.opencontainers.image.revision": constants.DEV_BUILD_DATE, }, "tags": [ f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-package-qt:2019", f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-package-qt:{qt_version}", f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-package-qt:latest", ], "target": "ci-package-qt", "output": ["type=docker"], } }, }, )
def test_cli_packages(self): runner = CliRunner() result = runner.invoke(aswfdocker.cli, ["packages"], catch_exceptions=False) self.assertEqual(result.exit_code, 0) pkgs = result.output.split("\n") self.assertGreater(len(pkgs), 20) clang_version = list( index.Index().iter_versions(constants.ImageType.PACKAGE, "clang") )[0] self.assertEqual( pkgs[0], f"common/ci-package-clang:{clang_version}", )
def test_image_base_2019clang_dict(self): b = builder.Builder( self.build_info, groupinfo.GroupInfo( names=["vfx3"], versions=["2019-clang9"], type_=constants.ImageType.IMAGE, targets=["openvdb"], ), ) openvdb_version = list( index.Index().iter_versions(constants.ImageType.IMAGE, "openvdb") )[4] self.assertEqual( b.make_bake_dict(), { "group": {"default": {"targets": ["ci-openvdb-2019-clang9"]}}, "target": { "ci-openvdb-2019-clang9": { "context": ".", "dockerfile": "ci-openvdb/Dockerfile", "args": { "ASWF_ALEMBIC_VERSION": "1.7.11", "ASWF_ORG": "aswflocaltesting", "ASWF_PKG_ORG": "aswftesting", "ASWF_VERSION": openvdb_version, "ASWF_BLOSC_VERSION": "1.5.0", "ASWF_BOOST_VERSION": "1.66.0", "ASWF_CCACHE_VERSION": "4.0", "CI_COMMON_VERSION": "1", "ASWF_CLANG_MAJOR_VERSION": "9", "ASWF_CLANG_VERSION": "9.0.1", "ASWF_CMAKE_VERSION": "3.12.4", "ASWF_CPPUNIT_VERSION": "1.14.0", "ASWF_CUDA_VERSION": "10.2", "ASWF_DTS_VERSION": "6", "ASWF_GLEW_VERSION": "2.1.0", "ASWF_GLFW_VERSION": "3.1.2", "ASWF_HDF5_VERSION": "1.8.21", "ASWF_LOG4CPLUS_VERSION": "1.1.2", "ASWF_NINJA_VERSION": "1.10.1", "ASWF_NUMPY_VERSION": "1.14", "ASWF_OCIO_CONFIGS_VERSION": "1.0_r2", "ASWF_OCIO_VERSION": "1.1.0", "ASWF_OIIO_VERSION": "2.0.8", "ASWF_OPENEXR_VERSION": "2.3.0", "ASWF_OPENSUBDIV_VERSION": "3_3_3", "ASWF_OPENVDB_VERSION": "6.2.1", "ASWF_OSL_VERSION": "1.10.9", "ASWF_OTIO_VERSION": "0.12.1", "ASWF_PARTIO_VERSION": "1.10.1", "ASWF_PTEX_VERSION": "2.1.33", "ASWF_PYBIND11_VERSION": "2.6.2", "ASWF_PYSIDE_VERSION": "5.12.6", "ASWF_PYTHON_VERSION": "2.7.15", "ASWF_PYTHON_MAJOR_MINOR_VERSION": "2.7", "ASWF_QT_VERSION": "5.12.6", "ASWF_SONAR_VERSION": "4.5.0.2216", "ASWF_TBB_VERSION": "2018", "ASWF_USD_VERSION": "19.11", "ASWF_VFXPLATFORM_VERSION": "2019", }, "labels": { "org.opencontainers.image.created": constants.DEV_BUILD_DATE, "org.opencontainers.image.revision": constants.DEV_BUILD_DATE, }, "tags": [ f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-openvdb:2019-clang9", f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-openvdb:{openvdb_version}", ], "output": ["type=docker"], } }, }, )
def setUp(self): self.index = index.Index()
def __init__(self, from_org: str, to_org: str): self.from_org = from_org self.to_org = to_org self.migration_list: typing.List[MigrateInfo] = [] self.cmds: typing.List[str] = [] self.index = index.Index()
def test_image_base_2019_2020_dict(self): b = builder.Builder( self.build_info, groupinfo.GroupInfo( names=["base"], versions=["2019", "2020"], type_=constants.ImageType.IMAGE, targets=[], ), ) base_versions = list(index.Index().iter_versions( constants.ImageType.IMAGE, "base")) self.assertEqual( b.make_bake_dict(), { "group": { "default": { "targets": ["ci-base-2019", "ci-base-2020"] } }, "target": { "ci-base-2020": { "context": ".", "dockerfile": "ci-base/Dockerfile", "args": { "ASWF_ORG": "aswflocaltesting", "ASWF_PKG_ORG": "aswftesting", "ASWF_VERSION": base_versions[2], "CI_COMMON_VERSION": "1", "DTS_VERSION": "6", "PYTHON_VERSION": "3.7", "VFXPLATFORM_VERSION": "2020", }, "labels": { "org.opencontainers.image.created": constants.DEV_BUILD_DATE, "org.opencontainers.image.revision": constants.DEV_BUILD_DATE, }, "tags": [ f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:2020", f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:{base_versions[2]}", f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:preview", ], "output": ["type=docker"], }, "ci-base-2019": { "context": ".", "dockerfile": "ci-base/Dockerfile", "args": { "ASWF_ORG": "aswflocaltesting", "ASWF_PKG_ORG": "aswftesting", "ASWF_VERSION": base_versions[1], "CI_COMMON_VERSION": "1", "DTS_VERSION": "6", "PYTHON_VERSION": "2.7", "VFXPLATFORM_VERSION": "2019", }, "labels": { "org.opencontainers.image.created": constants.DEV_BUILD_DATE, "org.opencontainers.image.revision": constants.DEV_BUILD_DATE, }, "tags": [ f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:2019", f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:{base_versions[1]}", f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:latest", ], "output": ["type=docker"], }, }, }, )