def _generate_all_framework_version_fixtures(metafunc):
for fw in FRAMEWORKS_FOR_GENERATED_VERSION_FIXTURES:
config = image_uris.config_for_framework(fw.replace("_", "-"))
if "scope" in config:
_parametrize_framework_version_fixtures(metafunc, fw, config)
else:
for image_scope in config.keys():
fixture_prefix = f"{fw}_{image_scope}" if image_scope not in fw else fw
_parametrize_framework_version_fixtures(
metafunc, fixture_prefix, config[image_scope])
def _generate_all_framework_version_fixtures(metafunc):
for fw in FRAMEWORKS_FOR_GENERATED_VERSION_FIXTURES:
config = image_uris.config_for_framework(fw.replace("_", "-"))
if "scope" in config:
_parametrize_framework_version_fixtures(metafunc, fw, config)
else:
for image_scope in config.keys():
_parametrize_framework_version_fixtures(
metafunc, "{}_{}".format(fw, image_scope),
config[image_scope])
def _huggingface_base_fm_version(huggingface_vesion, base_fw):
config = image_uris.config_for_framework("huggingface")
training_config = config.get("training")
original_version = huggingface_vesion
if "version_aliases" in training_config:
huggingface_vesion = training_config.get("version_aliases").get(
huggingface_vesion, huggingface_vesion)
version_config = training_config.get("versions").get(huggingface_vesion)
for key in list(version_config.keys()):
if key.startswith(base_fw):
base_fw_version = key[len(base_fw):]
if len(original_version.split(".")) == 2:
base_fw_version = ".".join(base_fw_version.split(".")[:-1])
return base_fw_version
def main():
"""Parse command line arguments, call corresponding methods."""
parser = argparse.ArgumentParser(description="Framework upgrade tool.")
parser.add_argument(
"--framework",
required=True,
help="Name of the framework (e.g. tensorflow, mxnet, etc.)")
parser.add_argument("--short-version",
help="Abbreviated framework version (e.g. 2.0)")
parser.add_argument("--full-version",
help="Full framework version (e.g. 2.0.1)")
parser.add_argument("--processors",
help="Suppoted processors (e.g. cpu, gpu)")
parser.add_argument("--py-versions",
help="Supported Python versions (e.g. py3,py37)")
parser.add_argument(
"--scope",
help="Scope for the Algorithm image (e.g. inference, training)")
parser.add_argument(
"--tag-prefix",
help="Tag prefix of the Algorithm image (e.g. ray-0.8.5-torch)")
parser.add_argument("--region",
help="New region to be added (e.g. us-west-2)")
parser.add_argument("--account", help="Registry account of new region")
args = parser.parse_args()
content = config_for_framework(args.framework)
if args.region or args.account:
if args.region and not args.account or args.account and not args.region:
raise ValueError(
"--region and --account must be used together to expand region."
)
add_region(content, args.region, args.account)
else:
add_version(
content,
args.short_version,
args.full_version,
args.scope,
args.processors,
args.py_versions,
args.tag_prefix,
)
file = os.path.join(IMAGE_URI_CONFIG_DIR, "{}.json".format(args.framework))
_write_dict_to_json(file, content)
def _huggingface_base_fm_version(huggingface_version, base_fw, fixture_prefix):
config = image_uris.config_for_framework("huggingface")
if fixture_prefix == "huggingface_training":
hf_config = config.get("training")
else:
hf_config = config.get("inference")
original_version = huggingface_version
if "version_aliases" in hf_config:
huggingface_version = hf_config.get("version_aliases").get(
huggingface_version, huggingface_version)
version_config = hf_config.get("versions").get(huggingface_version)
versions = list()
for key in list(version_config.keys()):
if key.startswith(base_fw):
base_fw_version = key[len(base_fw):]
if len(original_version.split(".")) == 2:
base_fw_version = ".".join(base_fw_version.split(".")[:-1])
versions.append(base_fw_version)
return versions
def _huggingface_base_fm_version(huggingface_version, base_fw, fixture_prefix):
config_name = ("huggingface-training-compiler"
if "training_compiler" in fixture_prefix else "huggingface")
config = image_uris.config_for_framework(config_name)
if "training" in fixture_prefix:
hf_config = config.get("training")
else:
hf_config = config.get("inference")
original_version = huggingface_version
if "version_aliases" in hf_config:
huggingface_version = hf_config.get("version_aliases").get(
huggingface_version, huggingface_version)
version_config = hf_config.get("versions").get(huggingface_version)
versions = list()
for key in list(version_config.keys()):
if key.startswith(base_fw):
base_fw_version = key[len(base_fw):]
if len(original_version.split(".")) == 2:
base_fw_version = ".".join(base_fw_version.split(".")[:-1])
versions.append(base_fw_version)
return sorted(versions, reverse=True)
