def print_info(self, buf=None, format_=FileFormat.yaml,
skip_attributes=None, include_release=False):
"""Print the contents of the package.
Args:
buf (file-like object): Stream to write to.
format_ (`FileFormat`): Format to write in.
skip_attributes (list of str): List of attributes to not print.
include_release (bool): If True, include release-related attributes,
such as 'timestamp' and 'changelog'
"""
data = self.validated_data().copy()
# config is a special case. We only really want to show any config settings
# that were in the package.py, not the entire Config contents that get
# grafted onto the Package/Variant instance. However Variant has an empy
# 'data' dict property, since it forwards data from its parent package.
data.pop("config", None)
if self.config:
if isinstance(self, Package):
config_dict = self.data.get("config")
else:
config_dict = self.parent.data.get("config")
data["config"] = config_dict
if not include_release:
skip_attributes = list(skip_attributes or []) + list(package_release_keys)
buf = buf or sys.stdout
dump_package_data(data, buf=buf, format_=format_,
skip_attributes=skip_attributes)
def _create_variant(self, variant, dry_run=False, overrides=None):
# special case overrides
variant_name = overrides.get("name") or variant.name
variant_version = overrides.get("version") or variant.version
overrides = (overrides or {}).copy()
overrides.pop("name", None)
overrides.pop("version", None)
# find or create the package family
family = self.get_package_family(variant_name)
if not family:
family = self._create_family(variant_name)
if isinstance(family, FileSystemCombinedPackageFamilyResource):
raise NotImplementedError(
"Cannot install variant into combined-style package file %r."
% family.filepath)
# find the package if it already exists
existing_package = None
for package in self.iter_packages(family):
if package.version == variant_version:
# during a build, the family/version dirs get created ahead of
# time, which causes a 'missing package definition file' error.
# This is fine, we can just ignore it and write the new file.
try:
package.validate_data()
except PackageDefinitionFileMissing:
break
uuids = set([variant.uuid, package.uuid])
if len(uuids) > 1 and None not in uuids:
raise ResourceError(
"Cannot install variant %r into package %r - the "
"packages are not the same (UUID mismatch)"
% (variant, package))
existing_package = package
if variant.index is None:
if package.variants:
raise ResourceError(
"Attempting to install a package without variants "
"(%r) into an existing package with variants (%r)"
% (variant, package))
elif not package.variants:
raise ResourceError(
"Attempting to install a variant (%r) into an existing "
"package without variants (%r)" % (variant, package))
existing_package_data = None
release_data = {}
# Need to treat 'config' as special case. In validated data, this is
# converted to a Config object. We need it as the raw dict that you'd
# see in a package.py.
#
def _get_package_data(pkg):
data = pkg.validated_data()
if hasattr(pkg, "_data"):
raw_data = pkg._data
else:
raw_data = pkg.resource._data
raw_config_data = raw_data.get('config')
data.pop("config", None)
if raw_config_data:
data["config"] = raw_config_data
return data
def _remove_build_keys(obj):
for key in package_build_only_keys:
obj.pop(key, None)
new_package_data = _get_package_data(variant.parent)
new_package_data.pop("variants", None)
new_package_data["name"] = variant_name
if variant_version:
new_package_data["version"] = variant_version
package_changed = False
_remove_build_keys(new_package_data)
if existing_package:
debug_print(
"Found existing package for installation of variant %s: %s",
variant.uri, existing_package.uri
)
existing_package_data = _get_package_data(existing_package)
_remove_build_keys(existing_package_data)
# detect case where new variant introduces package changes outside of variant
data_1 = existing_package_data.copy()
data_2 = new_package_data.copy()
for key in package_release_keys:
data_2.pop(key, None)
value = data_1.pop(key, None)
if value is not None:
release_data[key] = value
for key in ("format_version", "base", "variants"):
data_1.pop(key, None)
data_2.pop(key, None)
package_changed = (data_1 != data_2)
if debug_print:
if package_changed:
from rez.utils.data_utils import get_dict_diff_str
debug_print("Variant %s package data differs from package %s",
variant.uri, existing_package.uri)
txt = get_dict_diff_str(data_1, data_2, "Changes:")
debug_print(txt)
else:
debug_print("Variant %s package data matches package %s",
variant.uri, existing_package.uri)
# check for existing installed variant
existing_installed_variant = None
installed_variant_index = None
if existing_package:
if variant.index is None:
existing_installed_variant = \
self.iter_variants(existing_package).next()
else:
variant_requires = variant.variant_requires
for variant_ in self.iter_variants(existing_package):
variant_requires_ = existing_package.variants[variant_.index]
if variant_requires_ == variant_requires:
installed_variant_index = variant_.index
existing_installed_variant = variant_
if existing_installed_variant:
debug_print(
"Variant %s already has installed equivalent: %s",
variant.uri, existing_installed_variant.uri
)
if dry_run:
if not package_changed:
return existing_installed_variant
else:
return None
# construct package data for new installed package definition
if existing_package:
_, file_ = os.path.split(existing_package.filepath)
package_filename, package_extension = os.path.splitext(file_)
package_extension = package_extension[1:]
package_format = FileFormat[package_extension]
if package_changed:
# graft together new package data, with existing package variants,
# and other data that needs to stay unchanged (eg timestamp)
package_data = new_package_data
if variant.index is not None:
package_data["variants"] = existing_package_data.get("variants", [])
else:
package_data = existing_package_data
else:
package_data = new_package_data
package_filename = _settings.package_filenames[0]
package_extension = "py"
package_format = FileFormat.py
# merge existing release data (if any) into the package. Note that when
# this data becomes variant-specific, this step will no longer be needed
package_data.update(release_data)
# merge the new variant into the package
if installed_variant_index is None and variant.index is not None:
variant_requires = variant.variant_requires
if not package_data.get("variants"):
package_data["variants"] = []
package_data["variants"].append(variant_requires)
installed_variant_index = len(package_data["variants"]) - 1
# a little data massaging is needed
package_data.pop("base", None)
# create version dir if it doesn't already exist
family_path = os.path.join(self.location, variant_name)
if variant_version:
pkg_base_path = os.path.join(family_path, str(variant_version))
else:
pkg_base_path = family_path
if not os.path.exists(pkg_base_path):
os.makedirs(pkg_base_path)
# Apply overrides.
#
# If we're installing into an existing package, then existing attributes
# in that package take precedence over `overrides`. If we're installing
# to a new package, then `overrides` takes precedence always.
#
# This is done so that variants added to an existing package don't change
# attributes such as 'timestamp' or release-related fields like 'revision'.
#
for key, value in overrides.iteritems():
if existing_package:
if key not in package_data:
package_data[key] = value
else:
if value is self.remove:
package_data.pop(key, None)
else:
package_data[key] = value
# timestamp defaults to now if not specified
if not package_data.get("timestamp"):
package_data["timestamp"] = int(time.time())
# format version is always set
package_data["format_version"] = format_version
# write out new package definition file
package_file = ".".join([package_filename, package_extension])
filepath = os.path.join(pkg_base_path, package_file)
with make_path_writable(pkg_base_path):
with open_file_for_write(filepath, mode=self.package_file_mode) as f:
dump_package_data(package_data, buf=f, format_=package_format)
# delete the tmp 'building' file.
if variant_version:
filename = self.building_prefix + str(variant_version)
filepath = os.path.join(family_path, filename)
if os.path.exists(filepath):
try:
os.remove(filepath)
except:
pass
# delete other stale building files; previous failed releases may have
# left some around
try:
self._delete_stale_build_tagfiles(family_path)
except:
pass
# touch the family dir, this keeps memcached resolves updated properly
os.utime(family_path, None)
# load new variant
new_variant = None
self.clear_caches()
family = self.get_package_family(variant_name)
if family:
for package in self.iter_packages(family):
if package.version == variant_version:
for variant_ in self.iter_variants(package):
if variant_.index == installed_variant_index:
new_variant = variant_
break
elif new_variant:
break
if not new_variant:
raise RezSystemError("Internal failure - expected installed variant")
return new_variant
def _write_package(self):
with open(self.packagefile, 'w') as f:
dump_package_data(self.package_data, f, format_=FileFormat.yaml)
def _write_package(self):
with open(self.packagefile, 'w') as f:
dump_package_data(self.package_data, f, format_=FileFormat.yaml)
def _create_variant(self, variant, dry_run=False, overrides=None):
# find or create the package family
family = self.get_package_family(variant.name)
if not family:
family = self._create_family(variant.name)
if isinstance(family, FileSystemCombinedPackageFamilyResource):
raise NotImplementedError(
"Cannot install variant into combined-style package file %r."
% family.filepath)
# find the package if it already exists
existing_package = None
for package in self.iter_packages(family):
if package.version == variant.version:
# during a build, the family/version dirs get created ahead of
# time, which causes a 'missing package definition file' error.
# This is fine, we can just ignore it and write the new file.
try:
package.validate_data()
except PackageDefinitionFileMissing:
break
uuids = set([variant.uuid, package.uuid])
if len(uuids) > 1 and None not in uuids:
raise ResourceError(
"Cannot install variant %r into package %r - the "
"packages are not the same (UUID mismatch)"
% (variant, package))
existing_package = package
if variant.index is None:
if package.variants:
raise ResourceError(
"Attempting to install a package without variants "
"(%r) into an existing package with variants (%r)"
% (variant, package))
elif not package.variants:
raise ResourceError(
"Attempting to install a variant (%r) into an existing "
"package without variants (%r)" % (variant, package))
installed_variant_index = None
existing_package_data = None
existing_variants_data = None
release_data = {}
new_package_data = variant.parent.validated_data()
new_package_data.pop("variants", None)
package_changed = False
if existing_package:
existing_package_data = existing_package.validated_data()
# detect case where new variant introduces package changes outside of variant
data_1 = existing_package_data.copy()
data_2 = new_package_data.copy()
for key in package_release_keys:
data_2.pop(key, None)
value = data_1.pop(key, None)
if value is not None:
release_data[key] = value
for key in ("base", "variants"):
data_1.pop(key, None)
data_2.pop(key, None)
package_changed = (data_1 != data_2)
# special case - installing a no-variant pkg into a no-variant pkg
if existing_package and variant.index is None:
if dry_run and not package_changed:
variant_ = self.iter_variants(existing_package).next()
return variant_
else:
# just replace the package
existing_package = None
if existing_package:
# see if variant already exists in package
variant_requires = variant.variant_requires
for variant_ in self.iter_variants(existing_package):
variant_requires_ = existing_package.variants[variant_.index]
if variant_requires_ == variant_requires:
installed_variant_index = variant_.index
if dry_run and not package_changed:
return variant_
break
parent_package = existing_package
_, file_ = os.path.split(existing_package.filepath)
package_filename, package_extension = os.path.splitext(file_)
package_extension = package_extension[1:]
package_format = FileFormat[package_extension]
if package_changed:
# graft together new package data, with existing package variants,
# and other data that needs to stay unchanged (eg timestamp)
package_data = new_package_data
package_data["variants"] = existing_package_data.get("variants", [])
else:
package_data = existing_package_data
else:
parent_package = variant.parent
package_data = new_package_data
package_filename = _settings.package_filenames[0]
package_extension = "py"
package_format = FileFormat.py
if dry_run:
return None
# merge existing release data (if any) into the package. Note that when
# this data becomes variant-specific, this step will no longer be needed
package_data.update(release_data)
# merge the new variant into the package
if installed_variant_index is None and variant.index is not None:
variant_requires = variant.variant_requires
if not package_data.get("variants"):
package_data["variants"] = []
package_data["variants"].append(variant_requires)
installed_variant_index = len(package_data["variants"]) - 1
# a little data massaging is needed
package_data["config"] = parent_package._data.get("config")
package_data.pop("base", None)
# create version dir and write out the new package definition file
family_path = os.path.join(self.location, variant.name)
if variant.version:
path = os.path.join(family_path, str(variant.version))
else:
path = family_path
if not os.path.exists(path):
os.makedirs(path)
# add the timestamp
overrides = overrides or {}
overrides["timestamp"] = int(time.time())
# apply attribute overrides
for key, value in overrides.iteritems():
if package_data.get(key) is None:
package_data[key] = value
package_file = ".".join([package_filename, package_extension])
filepath = os.path.join(path, package_file)
with open_file_for_write(filepath) as f:
dump_package_data(package_data, buf=f, format_=package_format)
# touch the family dir, this keeps memcached resolves updated properly
os.utime(family_path, None)
# load new variant
new_variant = None
self.clear_caches()
family = self.get_package_family(variant.name)
if family:
for package in self.iter_packages(family):
if package.version == variant.version:
for variant_ in self.iter_variants(package):
if variant_.index == installed_variant_index:
new_variant = variant_
break
elif new_variant:
break
if not new_variant:
raise RezSystemError("Internal failure - expected installed variant")
return new_variant
def _create_variant(self, variant, dry_run=False, overrides=None):
# find or create the package family
family = self.get_package_family(variant.name)
if not family:
family = self._create_family(variant.name)
if isinstance(family, FileSystemCombinedPackageFamilyResource):
raise NotImplementedError(
"Cannot install variant into combined-style package file %r." %
family.filepath)
# find the package if it already exists
existing_package = None
for package in self.iter_packages(family):
if package.version == variant.version:
# during a build, the family/version dirs get created ahead of
# time, which causes a 'missing package definition file' error.
# This is fine, we can just ignore it and write the new file.
try:
package.validate_data()
except PackageDefinitionFileMissing:
break
uuids = set([variant.uuid, package.uuid])
if len(uuids) > 1 and None not in uuids:
raise ResourceError(
"Cannot install variant %r into package %r - the "
"packages are not the same (UUID mismatch)" %
(variant, package))
existing_package = package
if variant.index is None:
if package.variants:
raise ResourceError(
"Attempting to install a package without variants "
"(%r) into an existing package with variants (%r)"
% (variant, package))
elif not package.variants:
raise ResourceError(
"Attempting to install a variant (%r) into an existing "
"package without variants (%r)" % (variant, package))
installed_variant_index = None
existing_package_data = None
existing_variants_data = None
release_data = {}
new_package_data = variant.parent.validated_data()
new_package_data.pop("variants", None)
package_changed = False
if existing_package:
existing_package_data = existing_package.validated_data()
# detect case where new variant introduces package changes outside of variant
data_1 = existing_package_data.copy()
data_2 = new_package_data.copy()
for key in package_release_keys:
data_2.pop(key, None)
value = data_1.pop(key, None)
if value is not None:
release_data[key] = value
for key in ("base", "variants"):
data_1.pop(key, None)
data_2.pop(key, None)
package_changed = (data_1 != data_2)
# special case - installing a no-variant pkg into a no-variant pkg
if existing_package and variant.index is None:
if dry_run and not package_changed:
variant_ = self.iter_variants(existing_package).next()
return variant_
else:
# just replace the package
existing_package = None
if existing_package:
# see if variant already exists in package
variant_requires = variant.variant_requires
for variant_ in self.iter_variants(existing_package):
variant_requires_ = existing_package.variants[variant_.index]
if variant_requires_ == variant_requires:
installed_variant_index = variant_.index
if dry_run and not package_changed:
return variant_
break
parent_package = existing_package
_, file_ = os.path.split(existing_package.filepath)
package_filename, package_extension = os.path.splitext(file_)
package_extension = package_extension[1:]
package_format = FileFormat[package_extension]
if package_changed:
# graft together new package data, with existing package variants,
# and other data that needs to stay unchanged (eg timestamp)
package_data = new_package_data
package_data["variants"] = existing_package_data.get(
"variants", [])
else:
package_data = existing_package_data
else:
parent_package = variant.parent
package_data = new_package_data
package_filename = _settings.package_filenames[0]
package_extension = "py"
package_format = FileFormat.py
if dry_run:
return None
# merge existing release data (if any) into the package. Note that when
# this data becomes variant-specific, this step will no longer be needed
package_data.update(release_data)
# merge the new variant into the package
if installed_variant_index is None and variant.index is not None:
variant_requires = variant.variant_requires
if not package_data.get("variants"):
package_data["variants"] = []
package_data["variants"].append(variant_requires)
installed_variant_index = len(package_data["variants"]) - 1
# a little data massaging is needed
package_data["config"] = parent_package._data.get("config")
package_data.pop("base", None)
# create version dir and write out the new package definition file
family_path = os.path.join(self.location, variant.name)
if variant.version:
path = os.path.join(family_path, str(variant.version))
else:
path = family_path
if not os.path.exists(path):
os.makedirs(path)
# add the timestamp
overrides = overrides or {}
overrides["timestamp"] = int(time.time())
# apply attribute overrides
for key, value in overrides.iteritems():
if package_data.get(key) is None:
package_data[key] = value
package_file = ".".join([package_filename, package_extension])
filepath = os.path.join(path, package_file)
with open_file_for_write(filepath) as f:
dump_package_data(package_data, buf=f, format_=package_format)
# touch the family dir, this keeps memcached resolves updated properly
os.utime(family_path, None)
# load new variant
new_variant = None
self.clear_caches()
family = self.get_package_family(variant.name)
if family:
for package in self.iter_packages(family):
if package.version == variant.version:
for variant_ in self.iter_variants(package):
if variant_.index == installed_variant_index:
new_variant = variant_
break
elif new_variant:
break
if not new_variant:
raise RezSystemError(
"Internal failure - expected installed variant")
return new_variant
def _create_variant(self, variant, dry_run=False, overrides=None):
# special case overrides
variant_name = overrides.get("name") or variant.name
variant_version = overrides.get("version") or variant.version
overrides = (overrides or {}).copy()
overrides.pop("name", None)
overrides.pop("version", None)
# find or create the package family
family = self.get_package_family(variant_name)
if not family:
family = self._create_family(variant_name)
if isinstance(family, FileSystemCombinedPackageFamilyResource):
raise NotImplementedError(
"Cannot install variant into combined-style package file %r."
% family.filepath)
# find the package if it already exists
existing_package = None
for package in self.iter_packages(family):
if package.version == variant_version:
# during a build, the family/version dirs get created ahead of
# time, which causes a 'missing package definition file' error.
# This is fine, we can just ignore it and write the new file.
try:
package.validate_data()
except PackageDefinitionFileMissing:
break
uuids = set([variant.uuid, package.uuid])
if len(uuids) > 1 and None not in uuids:
raise ResourceError(
"Cannot install variant %r into package %r - the "
"packages are not the same (UUID mismatch)"
% (variant, package))
existing_package = package
if variant.index is None:
if package.variants:
raise ResourceError(
"Attempting to install a package without variants "
"(%r) into an existing package with variants (%r)"
% (variant, package))
elif not package.variants:
raise ResourceError(
"Attempting to install a variant (%r) into an existing "
"package without variants (%r)" % (variant, package))
existing_package_data = None
release_data = {}
# Need to treat 'config' as special case. In validated data, this is
# converted to a Config object. We need it as the raw dict that you'd
# see in a package.py.
#
def _get_package_data(pkg):
data = pkg.validated_data()
if hasattr(pkg, "_data"):
raw_data = pkg._data
else:
raw_data = pkg.resource._data
raw_config_data = raw_data.get('config')
data.pop("config", None)
if raw_config_data:
data["config"] = raw_config_data
return data
def _remove_build_keys(obj):
for key in package_build_only_keys:
obj.pop(key, None)
new_package_data = _get_package_data(variant.parent)
new_package_data.pop("variants", None)
new_package_data["name"] = variant_name
if variant_version:
new_package_data["version"] = variant_version
package_changed = False
_remove_build_keys(new_package_data)
if existing_package:
debug_print(
"Found existing package for installation of variant %s: %s",
variant.uri, existing_package.uri
)
existing_package_data = _get_package_data(existing_package)
_remove_build_keys(existing_package_data)
# detect case where new variant introduces package changes outside of variant
data_1 = existing_package_data.copy()
data_2 = new_package_data.copy()
for key in package_release_keys:
data_2.pop(key, None)
value = data_1.pop(key, None)
if value is not None:
release_data[key] = value
for key in ("format_version", "base", "variants"):
data_1.pop(key, None)
data_2.pop(key, None)
package_changed = (data_1 != data_2)
if debug_print:
if package_changed:
from rez.utils.data_utils import get_dict_diff_str
debug_print("Variant %s package data differs from package %s",
variant.uri, existing_package.uri)
txt = get_dict_diff_str(data_1, data_2, "Changes:")
debug_print(txt)
else:
debug_print("Variant %s package data matches package %s",
variant.uri, existing_package.uri)
# check for existing installed variant
existing_installed_variant = None
installed_variant_index = None
if existing_package:
if variant.index is None:
existing_installed_variant = \
self.iter_variants(existing_package).next()
else:
variant_requires = variant.variant_requires
for variant_ in self.iter_variants(existing_package):
variant_requires_ = existing_package.variants[variant_.index]
if variant_requires_ == variant_requires:
installed_variant_index = variant_.index
existing_installed_variant = variant_
if existing_installed_variant:
debug_print(
"Variant %s already has installed equivalent: %s",
variant.uri, existing_installed_variant.uri
)
if dry_run:
if not package_changed:
return existing_installed_variant
else:
return None
# construct package data for new installed package definition
if existing_package:
_, file_ = os.path.split(existing_package.filepath)
package_filename, package_extension = os.path.splitext(file_)
package_extension = package_extension[1:]
package_format = FileFormat[package_extension]
if package_changed:
# graft together new package data, with existing package variants,
# and other data that needs to stay unchanged (eg timestamp)
package_data = new_package_data
if variant.index is not None:
package_data["variants"] = existing_package_data.get("variants", [])
else:
package_data = existing_package_data
else:
package_data = new_package_data
package_filename = _settings.package_filenames[0]
package_extension = "py"
package_format = FileFormat.py
# merge existing release data (if any) into the package. Note that when
# this data becomes variant-specific, this step will no longer be needed
package_data.update(release_data)
# merge the new variant into the package
if installed_variant_index is None and variant.index is not None:
variant_requires = variant.variant_requires
if not package_data.get("variants"):
package_data["variants"] = []
package_data["variants"].append(variant_requires)
installed_variant_index = len(package_data["variants"]) - 1
# a little data massaging is needed
package_data.pop("base", None)
# create version dir if it doesn't already exist
family_path = os.path.join(self.location, variant_name)
if variant_version:
pkg_base_path = os.path.join(family_path, str(variant_version))
else:
pkg_base_path = family_path
if not os.path.exists(pkg_base_path):
os.makedirs(pkg_base_path)
# Apply overrides.
#
# If we're installing into an existing package, then existing attributes
# in that package take precedence over `overrides`. If we're installing
# to a new package, then `overrides` takes precedence always.
#
# This is done so that variants added to an existing package don't change
# attributes such as 'timestamp' or release-related fields like 'revision'.
#
for key, value in overrides.iteritems():
if existing_package:
if key not in package_data:
package_data[key] = value
else:
if value is self.remove:
package_data.pop(key, None)
else:
package_data[key] = value
# timestamp defaults to now if not specified
if not package_data.get("timestamp"):
package_data["timestamp"] = int(time.time())
# format version is always set
package_data["format_version"] = format_version
# write out new package definition file
package_file = ".".join([package_filename, package_extension])
filepath = os.path.join(pkg_base_path, package_file)
with make_path_writable(pkg_base_path):
with open_file_for_write(filepath, mode=self.package_file_mode) as f:
dump_package_data(package_data, buf=f, format_=package_format)
# delete the tmp 'building' file.
if variant_version:
filename = self.building_prefix + str(variant_version)
filepath = os.path.join(family_path, filename)
if os.path.exists(filepath):
try:
os.remove(filepath)
except:
pass
# delete other stale building files; previous failed releases may have
# left some around
try:
self._delete_stale_build_tagfiles(family_path)
except:
pass
# touch the family dir, this keeps memcached resolves updated properly
os.utime(family_path, None)
# load new variant
new_variant = None
self.clear_caches()
family = self.get_package_family(variant_name)
if family:
for package in self.iter_packages(family):
if package.version == variant_version:
for variant_ in self.iter_variants(package):
if variant_.index == installed_variant_index:
new_variant = variant_
break
elif new_variant:
break
if not new_variant:
raise RezSystemError("Internal failure - expected installed variant")
return new_variant
def _write_package():
with open(packagefile, "w") as f:
dump_package_data(package_data, f, format_=FileFormat.yaml)
