def load_py(stream, filepath=None):
"""Load python-formatted data from a stream.
Args:
stream (file-like object).
Returns:
dict.
"""
g = __builtins__.copy()
scopes = ScopeContext()
g['scope'] = scopes
try:
exec stream in g
except Exception as e:
import traceback
frames = traceback.extract_tb(sys.exc_info()[2])
while filepath and frames and frames[0][0] != filepath:
frames = frames[1:]
msg = str(e)
stack = ''.join(traceback.format_list(frames)).strip()
if stack:
msg += ":\n" + stack
raise ResourceError(msg)
result = {}
excludes = set(('scope', '__builtins__'))
for k, v in g.iteritems():
if k not in excludes and \
(k not in __builtins__ or __builtins__[k] != v):
result[k] = v
def _process_objects(data):
for k, v in data.iteritems():
if isfunction(v):
data[k] = SourceCode.from_function(v)
elif isinstance(v, dict):
_process_objects(v)
return data
result.update(scopes.to_dict())
result = _process_objects(result)
return result
def load_py(stream, filepath=None):
"""Load python-formatted data from a stream.
Args:
stream (file-like object).
Returns:
dict.
"""
scopes = ScopeContext()
g = dict(scope=scopes,
early=early,
late=late,
include=include,
ModifyList=ModifyList,
InvalidPackageError=InvalidPackageError)
try:
exec stream in g
except Exception as e:
import traceback
frames = traceback.extract_tb(sys.exc_info()[2])
while filepath and frames and frames[0][0] != filepath:
frames = frames[1:]
msg = "Problem loading %s: %s" % (filepath, str(e))
stack = ''.join(traceback.format_list(frames)).strip()
if stack:
msg += ":\n" + stack
raise ResourceError(msg)
result = {}
excludes = set(('scope', 'InvalidPackageError', '__builtins__', 'early',
'late', 'include', 'ModifyList'))
for k, v in g.iteritems():
if k not in excludes and \
(k not in __builtins__ or __builtins__[k] != v):
result[k] = v
result.update(scopes.to_dict())
result = process_python_objects(result, filepath=filepath)
return result
def _check_class(resource, cls):
if not isinstance(resource, cls):
raise ResourceError("Expected %s, got %s"
% (cls.__name__, resource.__class__.__name__))
def get_resource_class(self, resource_key):
resource_class = self.resource_classes.get(resource_key)
if resource_class is None:
raise ResourceError("Error getting resource from pool: Unknown "
"resource type %r" % resource_key)
return resource_class
def _process(value):
if isinstance(value, dict):
for k, v in value.items():
value[k] = _process(v)
return value
elif isfunction(value):
func = value
if hasattr(func, "_early"):
# run the function now, and replace with return value
#
# make a copy of the func with its own globals, and add 'this'
import types
fn = types.FunctionType(func.__code__,
func.__globals__.copy(),
name=func.__name__,
argdefs=func.__defaults__,
closure=func.__closure__)
# apply globals
fn.__globals__["this"] = EarlyThis(data)
fn.__globals__.update(get_objects())
# execute the function
args = py23.get_function_arg_names(func)
if len(args) not in (0, 1):
raise ResourceError("@early decorated function must "
"take zero or one args only")
if args:
# this 'data' arg support isn't needed anymore, but I'm
# supporting it til I know nobody is using it...
#
value_ = fn(data)
else:
value_ = fn()
# process again in case this is a function returning a function
return _process(value_)
elif hasattr(func, "_late"):
return SourceCode(func=func,
filepath=filepath,
eval_as_function=True)
elif func.__name__ in package_rex_keys:
# if a rex function, the code has to be eval'd NOT as a function,
# otherwise the globals dict doesn't get updated with any vars
# defined in the code, and that means rex code like this:
#
# rr = 'test'
# env.RR = '{rr}'
#
# ..won't work. It was never intentional that the above work, but
# it does, so now we have to keep it so.
#
return SourceCode(func=func,
filepath=filepath,
eval_as_function=False)
else:
# a normal function. Leave unchanged, it will be stripped after
return func
else:
return value
def _create_variant(self, variant, dry_run=False, overrides=None):
# special case overrides
variant_name = overrides.get("name") or variant.name
variant_version = overrides.get("version") or variant.version
overrides = (overrides or {}).copy()
overrides.pop("name", None)
overrides.pop("version", None)
# find or create the package family
family = self.get_package_family(variant_name)
if not family:
family = self._create_family(variant_name)
if isinstance(family, FileSystemCombinedPackageFamilyResource):
raise NotImplementedError(
"Cannot install variant into combined-style package file %r."
% family.filepath)
# find the package if it already exists
existing_package = None
for package in self.iter_packages(family):
if package.version == variant_version:
# during a build, the family/version dirs get created ahead of
# time, which causes a 'missing package definition file' error.
# This is fine, we can just ignore it and write the new file.
try:
package.validate_data()
except PackageDefinitionFileMissing:
break
uuids = set([variant.uuid, package.uuid])
if len(uuids) > 1 and None not in uuids:
raise ResourceError(
"Cannot install variant %r into package %r - the "
"packages are not the same (UUID mismatch)"
% (variant, package))
existing_package = package
if variant.index is None:
if package.variants:
raise ResourceError(
"Attempting to install a package without variants "
"(%r) into an existing package with variants (%r)"
% (variant, package))
elif not package.variants:
raise ResourceError(
"Attempting to install a variant (%r) into an existing "
"package without variants (%r)" % (variant, package))
existing_package_data = None
release_data = {}
# Need to treat 'config' as special case. In validated data, this is
# converted to a Config object. We need it as the raw dict that you'd
# see in a package.py.
#
def _get_package_data(pkg):
data = pkg.validated_data()
if hasattr(pkg, "_data"):
raw_data = pkg._data
else:
raw_data = pkg.resource._data
raw_config_data = raw_data.get('config')
data.pop("config", None)
if raw_config_data:
data["config"] = raw_config_data
return data
def _remove_build_keys(obj):
for key in package_build_only_keys:
obj.pop(key, None)
new_package_data = _get_package_data(variant.parent)
new_package_data.pop("variants", None)
new_package_data["name"] = variant_name
if variant_version:
new_package_data["version"] = variant_version
package_changed = False
_remove_build_keys(new_package_data)
if existing_package:
debug_print(
"Found existing package for installation of variant %s: %s",
variant.uri, existing_package.uri
)
existing_package_data = _get_package_data(existing_package)
_remove_build_keys(existing_package_data)
# detect case where new variant introduces package changes outside of variant
data_1 = existing_package_data.copy()
data_2 = new_package_data.copy()
for key in package_release_keys:
data_2.pop(key, None)
value = data_1.pop(key, None)
if value is not None:
release_data[key] = value
for key in ("format_version", "base", "variants"):
data_1.pop(key, None)
data_2.pop(key, None)
package_changed = (data_1 != data_2)
if debug_print:
if package_changed:
from rez.utils.data_utils import get_dict_diff_str
debug_print("Variant %s package data differs from package %s",
variant.uri, existing_package.uri)
txt = get_dict_diff_str(data_1, data_2, "Changes:")
debug_print(txt)
else:
debug_print("Variant %s package data matches package %s",
variant.uri, existing_package.uri)
# check for existing installed variant
existing_installed_variant = None
installed_variant_index = None
if existing_package:
if variant.index is None:
existing_installed_variant = \
self.iter_variants(existing_package).next()
else:
variant_requires = variant.variant_requires
for variant_ in self.iter_variants(existing_package):
variant_requires_ = existing_package.variants[variant_.index]
if variant_requires_ == variant_requires:
installed_variant_index = variant_.index
existing_installed_variant = variant_
if existing_installed_variant:
debug_print(
"Variant %s already has installed equivalent: %s",
variant.uri, existing_installed_variant.uri
)
if dry_run:
if not package_changed:
return existing_installed_variant
else:
return None
# construct package data for new installed package definition
if existing_package:
_, file_ = os.path.split(existing_package.filepath)
package_filename, package_extension = os.path.splitext(file_)
package_extension = package_extension[1:]
package_format = FileFormat[package_extension]
if package_changed:
# graft together new package data, with existing package variants,
# and other data that needs to stay unchanged (eg timestamp)
package_data = new_package_data
if variant.index is not None:
package_data["variants"] = existing_package_data.get("variants", [])
else:
package_data = existing_package_data
else:
package_data = new_package_data
package_filename = _settings.package_filenames[0]
package_extension = "py"
package_format = FileFormat.py
# merge existing release data (if any) into the package. Note that when
# this data becomes variant-specific, this step will no longer be needed
package_data.update(release_data)
# merge the new variant into the package
if installed_variant_index is None and variant.index is not None:
variant_requires = variant.variant_requires
if not package_data.get("variants"):
package_data["variants"] = []
package_data["variants"].append(variant_requires)
installed_variant_index = len(package_data["variants"]) - 1
# a little data massaging is needed
package_data.pop("base", None)
# create version dir if it doesn't already exist
family_path = os.path.join(self.location, variant_name)
if variant_version:
pkg_base_path = os.path.join(family_path, str(variant_version))
else:
pkg_base_path = family_path
if not os.path.exists(pkg_base_path):
os.makedirs(pkg_base_path)
# Apply overrides.
#
# If we're installing into an existing package, then existing attributes
# in that package take precedence over `overrides`. If we're installing
# to a new package, then `overrides` takes precedence always.
#
# This is done so that variants added to an existing package don't change
# attributes such as 'timestamp' or release-related fields like 'revision'.
#
for key, value in overrides.iteritems():
if existing_package:
if key not in package_data:
package_data[key] = value
else:
if value is self.remove:
package_data.pop(key, None)
else:
package_data[key] = value
# timestamp defaults to now if not specified
if not package_data.get("timestamp"):
package_data["timestamp"] = int(time.time())
# format version is always set
package_data["format_version"] = format_version
# write out new package definition file
package_file = ".".join([package_filename, package_extension])
filepath = os.path.join(pkg_base_path, package_file)
with make_path_writable(pkg_base_path):
with open_file_for_write(filepath, mode=self.package_file_mode) as f:
dump_package_data(package_data, buf=f, format_=package_format)
# delete the tmp 'building' file.
if variant_version:
filename = self.building_prefix + str(variant_version)
filepath = os.path.join(family_path, filename)
if os.path.exists(filepath):
try:
os.remove(filepath)
except:
pass
# delete other stale building files; previous failed releases may have
# left some around
try:
self._delete_stale_build_tagfiles(family_path)
except:
pass
# touch the family dir, this keeps memcached resolves updated properly
os.utime(family_path, None)
# load new variant
new_variant = None
self.clear_caches()
family = self.get_package_family(variant_name)
if family:
for package in self.iter_packages(family):
if package.version == variant_version:
for variant_ in self.iter_variants(package):
if variant_.index == installed_variant_index:
new_variant = variant_
break
elif new_variant:
break
if not new_variant:
raise RezSystemError("Internal failure - expected installed variant")
return new_variant
def _create_variant(self, variant, dry_run=False, overrides=None):
# find or create the package family
family = self.get_package_family(variant.name)
if not family:
family = self._create_family(variant.name)
if isinstance(family, FileSystemCombinedPackageFamilyResource):
raise NotImplementedError(
"Cannot install variant into combined-style package file %r." %
family.filepath)
# find the package if it already exists
existing_package = None
for package in self.iter_packages(family):
if package.version == variant.version:
# during a build, the family/version dirs get created ahead of
# time, which causes a 'missing package definition file' error.
# This is fine, we can just ignore it and write the new file.
try:
package.validate_data()
except PackageDefinitionFileMissing:
break
uuids = set([variant.uuid, package.uuid])
if len(uuids) > 1 and None not in uuids:
raise ResourceError(
"Cannot install variant %r into package %r - the "
"packages are not the same (UUID mismatch)" %
(variant, package))
existing_package = package
if variant.index is None:
if package.variants:
raise ResourceError(
"Attempting to install a package without variants "
"(%r) into an existing package with variants (%r)"
% (variant, package))
elif not package.variants:
raise ResourceError(
"Attempting to install a variant (%r) into an existing "
"package without variants (%r)" % (variant, package))
installed_variant_index = None
existing_package_data = None
existing_variants_data = None
release_data = {}
new_package_data = variant.parent.validated_data()
new_package_data.pop("variants", None)
package_changed = False
if existing_package:
existing_package_data = existing_package.validated_data()
# detect case where new variant introduces package changes outside of variant
data_1 = existing_package_data.copy()
data_2 = new_package_data.copy()
for key in package_release_keys:
data_2.pop(key, None)
value = data_1.pop(key, None)
if value is not None:
release_data[key] = value
for key in ("base", "variants"):
data_1.pop(key, None)
data_2.pop(key, None)
package_changed = (data_1 != data_2)
# special case - installing a no-variant pkg into a no-variant pkg
if existing_package and variant.index is None:
if dry_run and not package_changed:
variant_ = self.iter_variants(existing_package).next()
return variant_
else:
# just replace the package
existing_package = None
if existing_package:
# see if variant already exists in package
variant_requires = variant.variant_requires
for variant_ in self.iter_variants(existing_package):
variant_requires_ = existing_package.variants[variant_.index]
if variant_requires_ == variant_requires:
installed_variant_index = variant_.index
if dry_run and not package_changed:
return variant_
break
parent_package = existing_package
_, file_ = os.path.split(existing_package.filepath)
package_filename, package_extension = os.path.splitext(file_)
package_extension = package_extension[1:]
package_format = FileFormat[package_extension]
if package_changed:
# graft together new package data, with existing package variants,
# and other data that needs to stay unchanged (eg timestamp)
package_data = new_package_data
package_data["variants"] = existing_package_data.get(
"variants", [])
else:
package_data = existing_package_data
else:
parent_package = variant.parent
package_data = new_package_data
package_filename = _settings.package_filenames[0]
package_extension = "py"
package_format = FileFormat.py
if dry_run:
return None
# merge existing release data (if any) into the package. Note that when
# this data becomes variant-specific, this step will no longer be needed
package_data.update(release_data)
# merge the new variant into the package
if installed_variant_index is None and variant.index is not None:
variant_requires = variant.variant_requires
if not package_data.get("variants"):
package_data["variants"] = []
package_data["variants"].append(variant_requires)
installed_variant_index = len(package_data["variants"]) - 1
# a little data massaging is needed
package_data["config"] = parent_package._data.get("config")
package_data.pop("base", None)
# create version dir and write out the new package definition file
family_path = os.path.join(self.location, variant.name)
if variant.version:
path = os.path.join(family_path, str(variant.version))
else:
path = family_path
if not os.path.exists(path):
os.makedirs(path)
# add the timestamp
overrides = overrides or {}
overrides["timestamp"] = int(time.time())
# apply attribute overrides
for key, value in overrides.iteritems():
if package_data.get(key) is None:
package_data[key] = value
package_file = ".".join([package_filename, package_extension])
filepath = os.path.join(path, package_file)
with open_file_for_write(filepath) as f:
dump_package_data(package_data, buf=f, format_=package_format)
# touch the family dir, this keeps memcached resolves updated properly
os.utime(family_path, None)
# load new variant
new_variant = None
self.clear_caches()
family = self.get_package_family(variant.name)
if family:
for package in self.iter_packages(family):
if package.version == variant.version:
for variant_ in self.iter_variants(package):
if variant_.index == installed_variant_index:
new_variant = variant_
break
elif new_variant:
break
if not new_variant:
raise RezSystemError(
"Internal failure - expected installed variant")
return new_variant