def from_representation(self, representation_id):
"""Generate job from representation
Args:
representation_id (str): Avalon representation Id
"""
representation_id = io.ObjectId(representation_id)
representation = io.find_one({
"type": "representation",
"_id": representation_id
})
context = cget_representation_context(representation)
project = context["project"]
asset = context["asset"]
subset = context["subset"]
version = context["version"]
# Use loader to get local representation path
for Loader in self.available_loaders:
if not pipeline.is_compatible_loader(Loader, context):
continue
loader = Loader(context)
if hasattr(loader, "fname"):
repr_path = loader.fname
break
else:
raise Exception("Counld not find Loader for '%s'"
"" % representation["name"])
# Compute remote representation path
template = project["config"]["template"]["publish"]
remote_repr_path = template.format(
**{
"root": self.remote_root,
"project": project["name"],
"asset": asset["name"],
"silo": asset["silo"],
"subset": subset["name"],
"version": version["name"],
"representation": representation["name"],
})
# Get dir
if os.path.isdir(repr_path):
repr_dir = repr_path
remote_dir = remote_repr_path
elif os.path.isfile(repr_path):
repr_dir = os.path.dirname(repr_path)
remote_dir = os.path.dirname(remote_repr_path)
else:
raise Exception("Representation not exists.")
# Collect all files
jobs = list()
repr_dir = os.path.normpath(repr_dir)
remote_dir = os.path.normpath(remote_dir)
for head, dir, files in os.walk(repr_dir):
remote_head = remote_dir + head[len(repr_dir):]
for fname in files:
if any(fname.endswith(ext) for ext in self.FILE_IGNORE):
continue
local_file = os.path.join(head, fname)
remote_path = os.path.join(remote_head, fname)
jobs.append((local_file, remote_path))
if not jobs:
return
# Add job
description = ("[{asset}] {subset}.v{ver:0>3} - {repr}"
"".format(asset=asset["name"],
subset=subset["name"],
ver=version["name"],
repr=representation["name"]))
self.add_job(files=jobs,
type="Representation",
description=description)
def register(self, instance):
# Required environment variables
anatomy_data = instance.data["anatomyData"]
io.install()
context = instance.context
project_entity = instance.data["projectEntity"]
context_asset_name = context.data["assetEntity"]["name"]
asset_name = instance.data["asset"]
asset_entity = instance.data.get("assetEntity")
if not asset_entity or asset_entity["name"] != context_asset_name:
asset_entity = io.find_one({
"type": "asset",
"name": asset_name,
"parent": project_entity["_id"]
})
assert asset_entity, (
"No asset found by the name \"{0}\" in project \"{1}\""
).format(asset_name, project_entity["name"])
instance.data["assetEntity"] = asset_entity
# update anatomy data with asset specific keys
# - name should already been set
hierarchy = ""
parents = asset_entity["data"]["parents"]
if parents:
hierarchy = "/".join(parents)
anatomy_data["hierarchy"] = hierarchy
task_name = instance.data.get("task")
if task_name:
anatomy_data["task"] = task_name
anatomy_data["family"] = instance.data.get("family")
stagingdir = instance.data.get("stagingDir")
if not stagingdir:
self.log.info(
("{0} is missing reference to staging directory."
" Will try to get it from representation.").format(instance))
else:
self.log.debug(
"Establishing staging directory @ {0}".format(stagingdir))
# Ensure at least one file is set up for transfer in staging dir.
repres = instance.data.get("representations")
assert repres, "Instance has no files to transfer"
assert isinstance(
repres,
(list,
tuple)), ("Instance 'files' must be a list, got: {0} {1}".format(
str(type(repres)), str(repres)))
subset = self.get_subset(asset_entity, instance)
instance.data["subsetEntity"] = subset
version_number = instance.data["version"]
self.log.debug("Next version: v{}".format(version_number))
version_data = self.create_version_data(context, instance)
version_data_instance = instance.data.get('versionData')
if version_data_instance:
version_data.update(version_data_instance)
# TODO rename method from `create_version` to
# `prepare_version` or similar...
version = self.create_version(subset=subset,
version_number=version_number,
data=version_data)
self.log.debug("Creating version ...")
new_repre_names_low = [_repre["name"].lower() for _repre in repres]
existing_version = io.find_one({
'type': 'version',
'parent': subset["_id"],
'name': version_number
})
if existing_version is None:
version_id = io.insert_one(version).inserted_id
else:
# Check if instance have set `append` mode which cause that
# only replicated representations are set to archive
append_repres = instance.data.get("append", False)
# Update version data
# TODO query by _id and
io.update_many(
{
'type': 'version',
'parent': subset["_id"],
'name': version_number
}, {'$set': version})
version_id = existing_version['_id']
# Find representations of existing version and archive them
current_repres = list(
io.find({
"type": "representation",
"parent": version_id
}))
bulk_writes = []
for repre in current_repres:
if append_repres:
# archive only duplicated representations
if repre["name"].lower() not in new_repre_names_low:
continue
# Representation must change type,
# `_id` must be stored to other key and replaced with new
# - that is because new representations should have same ID
repre_id = repre["_id"]
bulk_writes.append(DeleteOne({"_id": repre_id}))
repre["orig_id"] = repre_id
repre["_id"] = io.ObjectId()
repre["type"] = "archived_representation"
bulk_writes.append(InsertOne(repre))
# bulk updates
if bulk_writes:
io._database[io.Session["AVALON_PROJECT"]].bulk_write(
bulk_writes)
version = io.find_one({"_id": version_id})
instance.data["versionEntity"] = version
existing_repres = list(
io.find({
"parent": version_id,
"type": "archived_representation"
}))
instance.data['version'] = version['name']
intent_value = instance.context.data.get("intent")
if intent_value and isinstance(intent_value, dict):
intent_value = intent_value.get("value")
if intent_value:
anatomy_data["intent"] = intent_value
anatomy = instance.context.data['anatomy']
# Find the representations to transfer amongst the files
# Each should be a single representation (as such, a single extension)
representations = []
destination_list = []
orig_transfers = []
if 'transfers' not in instance.data:
instance.data['transfers'] = []
else:
orig_transfers = list(instance.data['transfers'])
template_name = self.template_name_from_instance(instance)
published_representations = {}
for idx, repre in enumerate(instance.data["representations"]):
# reset transfers for next representation
# instance.data['transfers'] is used as a global variable
# in current codebase
instance.data['transfers'] = list(orig_transfers)
if "delete" in repre.get("tags", []):
continue
published_files = []
# create template data for Anatomy
template_data = copy.deepcopy(anatomy_data)
if intent_value is not None:
template_data["intent"] = intent_value
resolution_width = repre.get("resolutionWidth")
resolution_height = repre.get("resolutionHeight")
fps = instance.data.get("fps")
if resolution_width:
template_data["resolution_width"] = resolution_width
if resolution_width:
template_data["resolution_height"] = resolution_height
if resolution_width:
template_data["fps"] = fps
files = repre['files']
if repre.get('stagingDir'):
stagingdir = repre['stagingDir']
if repre.get("outputName"):
template_data["output"] = repre['outputName']
template_data["representation"] = repre["name"]
ext = repre["ext"]
if ext.startswith("."):
self.log.warning(
("Implementaion warning: <\"{}\">"
" Representation's extension stored under \"ext\" key "
" started with dot (\"{}\").").format(repre["name"], ext))
ext = ext[1:]
repre["ext"] = ext
template_data["ext"] = ext
template = os.path.normpath(
anatomy.templates[template_name]["path"])
sequence_repre = isinstance(files, list)
repre_context = None
if sequence_repre:
self.log.debug("files: {}".format(files))
src_collections, remainder = clique.assemble(files)
self.log.debug("src_tail_collections: {}".format(
str(src_collections)))
src_collection = src_collections[0]
# Assert that each member has identical suffix
src_head = src_collection.format("{head}")
src_tail = src_collection.format("{tail}")
# fix dst_padding
valid_files = [x for x in files if src_collection.match(x)]
padd_len = len(valid_files[0].replace(src_head, "").replace(
src_tail, ""))
src_padding_exp = "%0{}d".format(padd_len)
test_dest_files = list()
for i in [1, 2]:
template_data["frame"] = src_padding_exp % i
anatomy_filled = anatomy.format(template_data)
template_filled = anatomy_filled[template_name]["path"]
if repre_context is None:
repre_context = template_filled.used_values
test_dest_files.append(os.path.normpath(template_filled))
template_data["frame"] = repre_context["frame"]
self.log.debug("test_dest_files: {}".format(
str(test_dest_files)))
dst_collections, remainder = clique.assemble(test_dest_files)
dst_collection = dst_collections[0]
dst_head = dst_collection.format("{head}")
dst_tail = dst_collection.format("{tail}")
index_frame_start = None
# TODO use frame padding from right template group
if repre.get("frameStart") is not None:
frame_start_padding = int(anatomy.templates["render"].get(
"frame_padding",
anatomy.templates["render"].get("padding")))
index_frame_start = int(repre.get("frameStart"))
# exception for slate workflow
if index_frame_start and "slate" in instance.data["families"]:
index_frame_start -= 1
dst_padding_exp = src_padding_exp
dst_start_frame = None
for i in src_collection.indexes:
# TODO 1.) do not count padding in each index iteration
# 2.) do not count dst_padding from src_padding before
# index_frame_start check
src_padding = src_padding_exp % i
src_file_name = "{0}{1}{2}".format(src_head, src_padding,
src_tail)
dst_padding = src_padding_exp % i
if index_frame_start is not None:
dst_padding_exp = "%0{}d".format(frame_start_padding)
dst_padding = dst_padding_exp % index_frame_start
index_frame_start += 1
dst = "{0}{1}{2}".format(dst_head, dst_padding, dst_tail)
self.log.debug("destination: `{}`".format(dst))
src = os.path.join(stagingdir, src_file_name)
self.log.debug("source: {}".format(src))
instance.data["transfers"].append([src, dst])
published_files.append(dst)
# for adding first frame into db
if not dst_start_frame:
dst_start_frame = dst_padding
# Store used frame value to template data
template_data["frame"] = dst_start_frame
dst = "{0}{1}{2}".format(dst_head, dst_start_frame, dst_tail)
repre['published_path'] = dst
else:
# Single file
# _______
# | |\
# | |
# | |
# | |
# |_______|
#
template_data.pop("frame", None)
fname = files
assert not os.path.isabs(fname), (
"Given file name is a full path")
src = os.path.join(stagingdir, fname)
anatomy_filled = anatomy.format(template_data)
template_filled = anatomy_filled[template_name]["path"]
repre_context = template_filled.used_values
dst = os.path.normpath(template_filled)
instance.data["transfers"].append([src, dst])
published_files.append(dst)
repre['published_path'] = dst
self.log.debug("__ dst: {}".format(dst))
repre["publishedFiles"] = published_files
for key in self.db_representation_context_keys:
value = template_data.get(key)
if not value:
continue
repre_context[key] = template_data[key]
# Use previous representation's id if there are any
repre_id = None
repre_name_low = repre["name"].lower()
for _repre in existing_repres:
# NOTE should we check lowered names?
if repre_name_low == _repre["name"]:
repre_id = _repre["orig_id"]
break
# Create new id if existing representations does not match
if repre_id is None:
repre_id = io.ObjectId()
data = repre.get("data") or {}
data.update({'path': dst, 'template': template})
representation = {
"_id": repre_id,
"schema": "pype:representation-2.0",
"type": "representation",
"parent": version_id,
"name": repre['name'],
"data": data,
"dependencies": instance.data.get("dependencies", "").split(),
# Imprint shortcut to context
# for performance reasons.
"context": repre_context
}
if repre.get("outputName"):
representation["context"]["output"] = repre['outputName']
if sequence_repre and repre.get("frameStart"):
representation['context']['frame'] = (
dst_padding_exp % int(repre.get("frameStart")))
# any file that should be physically copied is expected in
# 'transfers' or 'hardlinks'
if instance.data.get('transfers', False) or \
instance.data.get('hardlinks', False):
# could throw exception, will be caught in 'process'
# all integration to DB is being done together lower,
# so no rollback needed
self.log.debug("Integrating source files to destination ...")
self.integrated_file_sizes.update(self.integrate(instance))
self.log.debug("Integrated files {}".format(
self.integrated_file_sizes))
# get 'files' info for representation and all attached resources
self.log.debug("Preparing files information ...")
representation["files"] = self.get_files_info(
instance, self.integrated_file_sizes)
self.log.debug("__ representation: {}".format(representation))
destination_list.append(dst)
self.log.debug("__ destination_list: {}".format(destination_list))
instance.data['destination_list'] = destination_list
representations.append(representation)
published_representations[repre_id] = {
"representation": representation,
"anatomy_data": template_data,
"published_files": published_files
}
self.log.debug("__ representations: {}".format(representations))
# Remove old representations if there are any (before insertion of new)
if existing_repres:
repre_ids_to_remove = []
for repre in existing_repres:
repre_ids_to_remove.append(repre["_id"])
io.delete_many({"_id": {"$in": repre_ids_to_remove}})
for rep in instance.data["representations"]:
self.log.debug("__ rep: {}".format(rep))
io.insert_many(representations)
instance.data["published_representations"] = (
published_representations)
# self.log.debug("Representation: {}".format(representations))
self.log.info("Registered {} items".format(len(representations)))
def switch_item(container,
asset_name=None,
subset_name=None,
representation_name=None):
"""Switch container asset, subset or representation of a container by name.
It'll always switch to the latest version - of course a different
approach could be implemented.
Args:
container (dict): data of the item to switch with
asset_name (str): name of the asset
subset_name (str): name of the subset
representation_name (str): name of the representation
Returns:
dict
"""
if all(not x for x in [asset_name, subset_name, representation_name]):
raise ValueError("Must have at least one change provided to switch.")
# Collect any of current asset, subset and representation if not provided
# so we can use the original name from those.
if any(not x for x in [asset_name, subset_name, representation_name]):
_id = io.ObjectId(container["representation"])
representation = io.find_one({"type": "representation", "_id": _id})
version, subset, asset, project = io.parenthood(representation)
if asset_name is None:
asset_name = asset["name"]
if subset_name is None:
subset_name = subset["name"]
if representation_name is None:
representation_name = representation["name"]
# Find the new one
asset = io.find_one({"name": asset_name, "type": "asset"})
assert asset, ("Could not find asset in the database with the name "
"'%s'" % asset_name)
subset = io.find_one({
"name": subset_name,
"type": "subset",
"parent": asset["_id"]
})
assert subset, ("Could not find subset in the database with the name "
"'%s'" % subset_name)
version = io.find_one({
"type": "version",
"parent": subset["_id"]
},
sort=[('name', -1)])
assert version, "Could not find a version for {}.{}".format(
asset_name, subset_name)
representation = io.find_one({
"name": representation_name,
"type": "representation",
"parent": version["_id"]
})
assert representation, ("Could not find representation in the database "
"with the name '%s'" % representation_name)
avalon.api.switch(container, representation)
return representation
def update_scene(set_container, containers, current_data, new_data, new_file):
"""Updates the hierarchy, assets and their matrix
Updates the following withing the scene:
* Setdress hierarchy alembic
* Matrix
* Parenting
* Representations
It removes any assets which are not present in the new build data
Args:
set_container (dict): the setdress container of the scene
containers (list): the list of containers under the setdress container
current_data (dict): the current build data of the setdress
new_data (dict): the new build data of the setdres
Returns:
processed_containers (list): all new and updated containers
"""
from pype.hosts.maya.lib import DEFAULT_MATRIX, get_container_transforms
set_namespace = set_container['namespace']
# Update the setdress hierarchy alembic
set_root = get_container_transforms(set_container, root=True)
set_hierarchy_root = cmds.listRelatives(set_root, fullPath=True)[0]
set_hierarchy_reference = cmds.referenceQuery(set_hierarchy_root,
referenceNode=True)
new_alembic = new_file.replace(".json", ".abc")
assert os.path.exists(new_alembic), "%s does not exist." % new_alembic
with unlocked(cmds.listRelatives(set_root, ad=True, fullPath=True)):
cmds.file(new_alembic,
loadReference=set_hierarchy_reference,
type="Alembic")
identity = DEFAULT_MATRIX[:]
processed_namespaces = set()
processed_containers = list()
new_lookup = _instances_by_namespace(new_data)
old_lookup = _instances_by_namespace(current_data)
for container in containers:
container_ns = container['namespace']
# Consider it processed here, even it it fails we want to store that
# the namespace was already available.
processed_namespaces.add(container_ns)
processed_containers.append(container['objectName'])
if container_ns in new_lookup:
root = get_container_transforms(container, root=True)
if not root:
log.error("Can't find root for %s", container['objectName'])
continue
old_instance = old_lookup.get(container_ns, {})
new_instance = new_lookup[container_ns]
# Update the matrix
# check matrix against old_data matrix to find local overrides
current_matrix = cmds.xform(root,
query=True,
matrix=True,
objectSpace=True)
original_matrix = old_instance.get("matrix", identity)
has_matrix_override = not matrix_equals(current_matrix,
original_matrix)
if has_matrix_override:
log.warning("Matrix override preserved on %s", container_ns)
else:
new_matrix = new_instance.get("matrix", identity)
cmds.xform(root, matrix=new_matrix, objectSpace=True)
# Update the parenting
if old_instance.get("parent", None) != new_instance["parent"]:
parent = to_namespace(new_instance['parent'], set_namespace)
if not cmds.objExists(parent):
log.error("Can't find parent %s", parent)
continue
# Set the new parent
cmds.lockNode(root, lock=False)
root = cmds.parent(root, parent, relative=True)
cmds.lockNode(root, lock=True)
# Update the representation
representation_current = container['representation']
representation_old = old_instance['representation']
representation_new = new_instance['representation']
has_representation_override = (representation_current !=
representation_old)
if representation_new != representation_current:
if has_representation_override:
log.warning(
"Your scene had local representation "
"overrides within the set. New "
"representations not loaded for %s.", container_ns)
continue
# We check it against the current 'loader' in the scene instead
# of the original data of the package that was loaded because
# an Artist might have made scene local overrides
if new_instance['loader'] != container['loader']:
log.warning(
"Loader is switched - local edits will be "
"lost. Removing: %s", container_ns)
# Remove this from the "has been processed" list so it's
# considered as new element and added afterwards.
processed_containers.pop()
processed_namespaces.remove(container_ns)
api.remove(container)
continue
# Check whether the conversion can be done by the Loader.
# They *must* use the same asset, subset and Loader for
# `api.update` to make sense.
old = io.find_one({"_id": io.ObjectId(representation_current)})
new = io.find_one({"_id": io.ObjectId(representation_new)})
is_valid = compare_representations(old=old, new=new)
if not is_valid:
log.error("Skipping: %s. See log for details.",
container_ns)
continue
new_version = new["context"]["version"]
api.update(container, version=new_version)
else:
# Remove this container because it's not in the new data
log.warning("Removing content: %s", container_ns)
api.remove(container)
# Add new assets
all_loaders = api.discover(api.Loader)
for representation_id, instances in new_data.items():
# Find the compatible loaders
loaders = api.loaders_from_representation(all_loaders,
representation_id)
for instance in instances:
# Already processed in update functionality
if instance['namespace'] in processed_namespaces:
continue
container = _add(instance=instance,
representation_id=representation_id,
loaders=loaders,
namespace=set_container['namespace'],
root=set_root)
# Add to the setdress container
cmds.sets(container, addElement=set_container['objectName'])
processed_containers.append(container)
return processed_containers
def process(self, instance):
self.log.debug(
"--- Integration of Hero version for subset `{}` begins.".format(
instance.data.get("subset", str(instance))
)
)
published_repres = instance.data.get("published_representations")
if not published_repres:
self.log.debug(
"*** There are not published representations on the instance."
)
return
project_name = api.Session["AVALON_PROJECT"]
# TODO raise error if Hero not set?
anatomy = instance.context.data["anatomy"]
if "hero" not in anatomy.templates:
self.log.warning("!!! Anatomy does not have set `hero` key!")
return
if "path" not in anatomy.templates["hero"]:
self.log.warning((
"!!! There is not set `path` template in `hero` anatomy"
" for project \"{}\"."
).format(project_name))
return
hero_template = anatomy.templates["hero"]["path"]
self.log.debug("`hero` template check was successful. `{}`".format(
hero_template
))
hero_publish_dir = self.get_publish_dir(instance)
src_version_entity = instance.data.get("versionEntity")
filtered_repre_ids = []
for repre_id, repre_info in published_repres.items():
repre = repre_info["representation"]
if repre["name"].lower() in self.ignored_representation_names:
self.log.debug(
"Filtering representation with name: `{}`".format(
repre["name"].lower()
)
)
filtered_repre_ids.append(repre_id)
for repre_id in filtered_repre_ids:
published_repres.pop(repre_id, None)
if not published_repres:
self.log.debug(
"*** All published representations were filtered by name."
)
return
if src_version_entity is None:
self.log.debug((
"Published version entity was not sent in representation data."
" Querying entity from database."
))
src_version_entity = (
self.version_from_representations(published_repres)
)
if not src_version_entity:
self.log.warning((
"!!! Can't find origin version in database."
" Skipping hero version publish."
))
return
all_copied_files = []
transfers = instance.data.get("transfers", list())
for _src, dst in transfers:
dst = os.path.normpath(dst)
if dst not in all_copied_files:
all_copied_files.append(dst)
hardlinks = instance.data.get("hardlinks", list())
for _src, dst in hardlinks:
dst = os.path.normpath(dst)
if dst not in all_copied_files:
all_copied_files.append(dst)
all_repre_file_paths = []
for repre_info in published_repres.values():
published_files = repre_info.get("published_files") or []
for file_path in published_files:
file_path = os.path.normpath(file_path)
if file_path not in all_repre_file_paths:
all_repre_file_paths.append(file_path)
# TODO this is not best practice of getting resources for publish
# WARNING due to this we must remove all files from hero publish dir
instance_publish_dir = os.path.normpath(
instance.data["publishDir"]
)
other_file_paths_mapping = []
for file_path in all_copied_files:
# Check if it is from publishDir
if not file_path.startswith(instance_publish_dir):
continue
if file_path in all_repre_file_paths:
continue
dst_filepath = file_path.replace(
instance_publish_dir, hero_publish_dir
)
other_file_paths_mapping.append((file_path, dst_filepath))
# Current version
old_version, old_repres = (
self.current_hero_ents(src_version_entity)
)
old_repres_by_name = {
repre["name"].lower(): repre for repre in old_repres
}
if old_version:
new_version_id = old_version["_id"]
else:
new_version_id = io.ObjectId()
new_hero_version = {
"_id": new_version_id,
"version_id": src_version_entity["_id"],
"parent": src_version_entity["parent"],
"type": "hero_version",
"schema": "pype:hero_version-1.0"
}
schema.validate(new_hero_version)
# Don't make changes in database until everything is O.K.
bulk_writes = []
if old_version:
self.log.debug("Replacing old hero version.")
bulk_writes.append(
ReplaceOne(
{"_id": new_hero_version["_id"]},
new_hero_version
)
)
else:
self.log.debug("Creating first hero version.")
bulk_writes.append(
InsertOne(new_hero_version)
)
# Separate old representations into `to replace` and `to delete`
old_repres_to_replace = {}
old_repres_to_delete = {}
for repre_info in published_repres.values():
repre = repre_info["representation"]
repre_name_low = repre["name"].lower()
if repre_name_low in old_repres_by_name:
old_repres_to_replace[repre_name_low] = (
old_repres_by_name.pop(repre_name_low)
)
if old_repres_by_name:
old_repres_to_delete = old_repres_by_name
archived_repres = list(io.find({
# Check what is type of archived representation
"type": "archived_repsentation",
"parent": new_version_id
}))
archived_repres_by_name = {}
for repre in archived_repres:
repre_name_low = repre["name"].lower()
archived_repres_by_name[repre_name_low] = repre
backup_hero_publish_dir = None
if os.path.exists(hero_publish_dir):
backup_hero_publish_dir = hero_publish_dir + ".BACKUP"
max_idx = 10
idx = 0
_backup_hero_publish_dir = backup_hero_publish_dir
while os.path.exists(_backup_hero_publish_dir):
self.log.debug((
"Backup folder already exists."
" Trying to remove \"{}\""
).format(_backup_hero_publish_dir))
try:
shutil.rmtree(_backup_hero_publish_dir)
backup_hero_publish_dir = _backup_hero_publish_dir
break
except Exception:
self.log.info((
"Could not remove previous backup folder."
" Trying to add index to folder name"
))
_backup_hero_publish_dir = (
backup_hero_publish_dir + str(idx)
)
if not os.path.exists(_backup_hero_publish_dir):
backup_hero_publish_dir = _backup_hero_publish_dir
break
if idx > max_idx:
raise AssertionError((
"Backup folders are fully occupied to max index \"{}\""
).format(max_idx))
break
idx += 1
self.log.debug("Backup folder path is \"{}\"".format(
backup_hero_publish_dir
))
try:
os.rename(hero_publish_dir, backup_hero_publish_dir)
except PermissionError:
raise AssertionError((
"Could not create hero version because it is not"
" possible to replace current hero files."
))
try:
src_to_dst_file_paths = []
for repre_info in published_repres.values():
# Skip if new repre does not have published repre files
published_files = repre_info["published_files"]
if len(published_files) == 0:
continue
# Prepare anatomy data
anatomy_data = repre_info["anatomy_data"]
anatomy_data.pop("version", None)
# Get filled path to repre context
anatomy_filled = anatomy.format(anatomy_data)
template_filled = anatomy_filled["hero"]["path"]
repre_data = {
"path": str(template_filled),
"template": hero_template
}
repre_context = template_filled.used_values
for key in self.db_representation_context_keys:
if (
key in repre_context or
key not in anatomy_data
):
continue
repre_context[key] = anatomy_data[key]
# Prepare new repre
repre = copy.deepcopy(repre_info["representation"])
repre["parent"] = new_hero_version["_id"]
repre["context"] = repre_context
repre["data"] = repre_data
repre.pop("_id", None)
# Prepare paths of source and destination files
if len(published_files) == 1:
src_to_dst_file_paths.append(
(published_files[0], template_filled)
)
else:
collections, remainders = clique.assemble(published_files)
if remainders or not collections or len(collections) > 1:
raise Exception((
"Integrity error. Files of published representation "
"is combination of frame collections and single files."
"Collections: `{}` Single files: `{}`"
).format(str(collections),
str(remainders)))
src_col = collections[0]
# Get head and tail for collection
frame_splitter = "_-_FRAME_SPLIT_-_"
anatomy_data["frame"] = frame_splitter
_anatomy_filled = anatomy.format(anatomy_data)
_template_filled = _anatomy_filled["hero"]["path"]
head, tail = _template_filled.split(frame_splitter)
padding = int(
anatomy.templates["render"].get(
"frame_padding",
anatomy.templates["render"].get("padding")
)
)
dst_col = clique.Collection(
head=head, padding=padding, tail=tail
)
dst_col.indexes.clear()
dst_col.indexes.update(src_col.indexes)
for src_file, dst_file in zip(src_col, dst_col):
src_to_dst_file_paths.append(
(src_file, dst_file)
)
# replace original file name with hero name in repre doc
for index in range(len(repre.get("files"))):
file = repre.get("files")[index]
file_name = os.path.basename(file.get('path'))
for src_file, dst_file in src_to_dst_file_paths:
src_file_name = os.path.basename(src_file)
if src_file_name == file_name:
repre["files"][index]["path"] = self._update_path(
anatomy, repre["files"][index]["path"],
src_file, dst_file)
repre["files"][index]["hash"] = self._update_hash(
repre["files"][index]["hash"],
src_file_name, dst_file
)
schema.validate(repre)
repre_name_low = repre["name"].lower()
# Replace current representation
if repre_name_low in old_repres_to_replace:
old_repre = old_repres_to_replace.pop(repre_name_low)
repre["_id"] = old_repre["_id"]
bulk_writes.append(
ReplaceOne(
{"_id": old_repre["_id"]},
repre
)
)
# Unarchive representation
elif repre_name_low in archived_repres_by_name:
archived_repre = archived_repres_by_name.pop(
repre_name_low
)
old_id = archived_repre["old_id"]
repre["_id"] = old_id
bulk_writes.append(
ReplaceOne(
{"old_id": old_id},
repre
)
)
# Create representation
else:
repre["_id"] = io.ObjectId()
bulk_writes.append(
InsertOne(repre)
)
self.path_checks = []
# Copy(hardlink) paths of source and destination files
# TODO should we *only* create hardlinks?
# TODO should we keep files for deletion until this is successful?
for src_path, dst_path in src_to_dst_file_paths:
self.copy_file(src_path, dst_path)
for src_path, dst_path in other_file_paths_mapping:
self.copy_file(src_path, dst_path)
# Archive not replaced old representations
for repre_name_low, repre in old_repres_to_delete.items():
# Replace archived representation (This is backup)
# - should not happen to have both repre and archived repre
if repre_name_low in archived_repres_by_name:
archived_repre = archived_repres_by_name.pop(
repre_name_low
)
repre["old_id"] = repre["_id"]
repre["_id"] = archived_repre["_id"]
repre["type"] = archived_repre["type"]
bulk_writes.append(
ReplaceOne(
{"_id": archived_repre["_id"]},
repre
)
)
else:
repre["old_id"] = repre["_id"]
repre["_id"] = io.ObjectId()
repre["type"] = "archived_representation"
bulk_writes.append(
InsertOne(repre)
)
if bulk_writes:
io._database[io.Session["AVALON_PROJECT"]].bulk_write(
bulk_writes
)
# Remove backuped previous hero
if (
backup_hero_publish_dir is not None and
os.path.exists(backup_hero_publish_dir)
):
shutil.rmtree(backup_hero_publish_dir)
except Exception:
if (
backup_hero_publish_dir is not None and
os.path.exists(backup_hero_publish_dir)
):
os.rename(backup_hero_publish_dir, hero_publish_dir)
self.log.error((
"!!! Creating of hero version failed."
" Previous hero version maybe lost some data!"
))
raise
self.log.debug((
"--- hero version integration for subset `{}`"
" seems to be successful."
).format(
instance.data.get("subset", str(instance))
))
def remap_to_published_texture(nodes, representation_id, dry_run=False):
"""For fixing previous bad implementations on texture management
This is for remapping texture file path from arbitrarily work path
to previous published path in published looks.
For some reason, some texture path may not changed to published path
after extraction. This is a fixing helper for the issue.
(NOTE) The issue should be resolved in following commits. :')
"""
file_nodes = cmds.ls(nodes, type="file")
count, file_data = lib.profiling_file_nodes(file_nodes)
if not count:
return
node_by_fpattern = dict()
for data in file_data:
data["pathMap"] = {
fn: data["dir"] + "/" + fn for fn in data["fnames"]
}
fpattern = data["fpattern"]
if fpattern not in node_by_fpattern:
node_by_fpattern[fpattern] = list()
node_by_fpattern[fpattern].append(data)
repr = io.find_one({"_id": io.ObjectId(representation_id)})
file_inventory = repr["data"].get("fileInventory")
if not file_inventory:
return
resolved_by_fpattern = lib.resolve_file_profile(repr, file_inventory)
# MAPPING
for fpattern, file_datas in node_by_fpattern.items():
if fpattern not in resolved_by_fpattern:
fpattern_ = fpattern.rsplit(".", 1)[0]
for resolved_fpattern in resolved_by_fpattern:
if fpattern_ == resolved_fpattern.rsplit(".", 1)[0]:
versioned_data = resolved_by_fpattern[resolved_fpattern]
break
else:
continue
else:
versioned_data = resolved_by_fpattern[fpattern]
data = file_datas[0]
file_nodes = [dat["node"] for dat in file_datas]
versioned_data.sort(key=lambda elem: elem[0]["version"],
reverse=True) # elem: (data, tmp_data)
for ver_data, resolved in versioned_data:
previous_files = resolved["pathMap"]
for file, abs_path in data["pathMap"].items():
if file not in previous_files:
file = file.rsplit(".", 1)[0]
for pre_file in previous_files:
if file == pre_file.rsplit(".", 1)[0]:
file = pre_file
abs_previous = previous_files[pre_file]
ext = pre_file.rsplit(".", 1)[1]
abs_path = abs_path.rsplit(".", 1)[0] + "." + ext
if not os.path.isfile(abs_path):
continue
else:
break
else:
# Possible different file pattern
break # Try previous version
else:
abs_previous = previous_files[file]
if not os.path.isfile(abs_previous):
# Previous file not exists (should not happen)
break # Try previous version
# (NOTE) We don't need to check on file size and modification
# time here since we are trying to map file to latest
# version of published one.
else:
# Version matched, consider as same file
head_file = sorted(previous_files)[0]
resolved_path = abs_previous[:-len(file)] + head_file
embedded_path = env_embedded_path(resolved_path)
fix_texture_file_nodes(file_nodes, embedded_path, dry_run)
# Proceed to next pattern
break
else:
# Not match with any previous version, this should not happen
log.warning("No version matched.")
with open(dry_run, "a") as path_log:
path_log.write("\n * " + data["dir"] + "/" + fpattern + "\n\n")
def unpack_textures(self, container):
import os
import shutil
from maya import cmds, mel
from avalon import api, io
project = io.find_one({"type": "project"},
projection={"name": True,
"config.template.publish": True})
asset = io.find_one({"_id": io.ObjectId(container["assetId"])},
projection={"name": True, "silo": True})
subset = io.find_one({"_id": io.ObjectId(container["subsetId"])},
projection={"name": True})
version = io.find_one({"_id": io.ObjectId(container["versionId"])},
projection={"name": True,
"data.dependencies": True})
# Find TexturePack
id = next(iter(version["data"]["dependencies"]))
dep_version = io.find_one({"_id": io.ObjectId(id)})
dep_subset = io.find_one({"_id": dep_version["parent"]})
dep_representation = io.find_one({"parent": dep_version["_id"],
"name": "TexturePack"})
# List texture versions
published = dict()
template_publish = project["config"]["template"]["publish"]
for data in dep_representation["data"]["fileInventory"]:
path = template_publish.format(
root=api.registered_root(),
project=project["name"],
silo=asset["silo"],
asset=asset["name"],
subset=dep_subset["name"],
version=data["version"],
representation="TexturePack",
)
published[data["version"]] = path
# Collect path,
# filter out textures that is being used in this look
file_nodes = cmds.ls(cmds.sets(container["objectName"], query=True),
type="file")
files = dict()
for node in file_nodes:
path = cmds.getAttr(node + ".fileTextureName",
expandEnvironmentVariables=True)
if not os.path.isfile(path):
continue
for v, p in published.items():
if path.startswith(p):
key = (v, p)
if key not in files:
files[key] = list()
files[key].append(node)
break
# Copy textures and change path
root = cmds.workspace(query=True, rootDirectory=True)
root += mel.eval('workspace -query -fileRuleEntry "sourceImages"')
root += "/_unpacked"
pattern = "/{asset}/{subset}.v{version:0>3}/TexturePack.v{texture:0>3}"
for (texture_version, src), nodes in files.items():
dst = root + pattern.format(asset=asset["name"],
subset=subset["name"],
version=version["name"],
texture=texture_version)
for node in nodes:
attr = node + ".fileTextureName"
path = cmds.getAttr(attr, expandEnvironmentVariables=True)
tail = path.split("TexturePack")[-1]
cmds.setAttr(attr, dst + tail, type="string")
if os.path.isdir(dst):
continue
shutil.copytree(src, dst)
def __get_representation(namespace):
container_node = pipeline.get_container_from_namespace(namespace)
_id = io.ObjectId(cmds.getAttr(container_node + ".representation"))
return io.find_one({"_id": _id})