def get_subset(self, asset, instance): subset_name = instance.data["subset"] subset = io.find_one({ "type": "subset", "parent": asset["_id"], "name": subset_name }) if subset is None: self.log.info("Subset '%s' not found, creating ..." % subset_name) self.log.debug("families. %s" % instance.data.get('families')) self.log.debug("families. %s" % type(instance.data.get('families'))) family = instance.data.get("family") families = [] if family: families.append(family) for _family in (instance.data.get("families") or []): if _family not in families: families.append(_family) _id = io.insert_one({ "schema": "openpype:subset-3.0", "type": "subset", "name": subset_name, "data": { "families": families }, "parent": asset["_id"] }).inserted_id subset = io.find_one({"_id": _id}) # add group if available if instance.data.get("subsetGroup"): io.update_many( { 'type': 'subset', '_id': io.ObjectId(subset["_id"]) }, { '$set': { 'data.subsetGroup': instance.data.get('subsetGroup') } }) # Update families on subset. families = [instance.data["family"]] families.extend(instance.data.get("families", [])) io.update_many({ "type": "subset", "_id": io.ObjectId(subset["_id"]) }, {"$set": { "data.families": families }}) return subset
def update_dependent(self, instance, version_id): version_id = str(version_id) field = "data.dependents." + version_id for version_id_, data in instance.data["dependencies"].items(): filter_ = {"_id": io.ObjectId(version_id_)} update = {"$set": {field: {"count": data["count"]}}} io.update_many(filter_, update)
def import_to_avalon(self, input_data, parent=None): for name in input_data: self.log.info("input_data[name]: {}".format(input_data[name])) entity_data = input_data[name] entity_type = entity_data["entity_type"] data = {} data["inputs"] = entity_data.get("inputs", []) data["entityType"] = entity_type # Custom attributes. for k, val in entity_data.get("custom_attributes", {}).items(): data[k] = val # Tasks. tasks = entity_data.get("tasks", []) if tasks is not None or len(tasks) > 0: data["tasks"] = tasks parents = [] visualParent = None # do not store project"s id as visualParent (silo asset) if self.project is not None: if self.project["_id"] != parent["_id"]: visualParent = parent["_id"] parents.extend(parent.get("data", {}).get("parents", [])) parents.append(parent["name"]) data["visualParent"] = visualParent data["parents"] = parents # Process project if entity_type.lower() == "project": entity = io.find_one({"type": "project"}) # TODO: should be in validator? assert (entity is not None), "Did not find project in DB" # get data from already existing project for key, value in entity.get("data", {}).items(): data[key] = value self.project = entity # Raise error if project or parent are not set elif self.project is None or parent is None: raise AssertionError("Collected items are not in right order!") # Else process assset else: entity = io.find_one({"type": "asset", "name": name}) # Create entity if doesn"t exist if entity is None: entity = self.create_avalon_asset(name, data) # Update entity data with input data io.update_many({"_id": entity["_id"]}, {"$set": {"data": data}}) if "childs" in entity_data: self.import_to_avalon(entity_data["childs"], entity)
def save_range(self): asset = self.assets.currentText() update = { "data.edit_in": self.start.value(), "data.edit_out": self.end.value(), "data.handles": self.handles.value(), } io.update_many({ "type": "asset", "name": asset }, update={"$set": update})
def process(self, instance): context = instance.context if not all(result["success"] for result in context.data["results"]): self.log.warning("Atomicity not held, aborting.") return # Integrate representations' to database self.log.info("Integrating representations to database ...") asset = context.data["assetDoc"] subset, version, representations = instance.data["toDatabase"] # Write subset if not exists filter = {"parent": asset["_id"], "name": subset["name"]} if io.find_one(filter) is None: io.insert_one(subset) # Write version if not exists filter = {"parent": subset["_id"], "name": version["name"]} existed_version = io.find_one(filter) if existed_version is None: # Write version and representations to database version_id = self.write_database(instance, version, representations) instance.data["insertedVersionId"] = version_id # Update dependent self.update_dependent(instance, version_id) else: self.log.info("Version existed, representation file has been " "overwritten.") # Update version document "data.time" filter_ = {"_id": existed_version["_id"]} update = {"$set": {"data.time": context.data["time"]}} io.update_many(filter_, update) # Update representation documents "data" for representation in representations: filter_ = { "name": representation["name"], "parent": existed_version["_id"], } update = {"$set": {"data": representation["data"]}} io.update_many(filter_, update)
def get_subset(self, asset, instance): subset_name = instance.data["subset"] subset = io.find_one({ "type": "subset", "parent": asset["_id"], "name": subset_name }) if subset is None: self.log.info("Subset '%s' not found, creating.." % subset_name) self.log.debug("families. %s" % instance.data.get('families')) self.log.debug("families. %s" % type(instance.data.get('families'))) _id = io.insert_one({ "schema": "pype:subset-3.0", "type": "subset", "name": subset_name, "data": { "families": instance.data.get("families", []) }, "parent": asset["_id"] }).inserted_id subset = io.find_one({"_id": _id}) # add group if available if instance.data.get("subsetGroup"): io.update_many( { 'type': 'subset', '_id': io.ObjectId(subset["_id"]) }, { '$set': { 'data.subsetGroup': instance.data.get('subsetGroup') } }) return subset
def get_hierarchy(asset_name=None): """ Obtain asset hierarchy path string from mongo db Returns: string: asset hierarchy path """ if not asset_name: asset_name = io.Session.get("AVALON_ASSET", os.environ["AVALON_ASSET"]) asset_entity = io.find_one({"type": 'asset', "name": asset_name}) not_set = "PARENTS_NOT_SET" entity_parents = asset_entity.get("data", {}).get("parents", not_set) # If entity already have parents then just return joined if entity_parents != not_set: return "/".join(entity_parents) # Else query parents through visualParents and store result to entity hierarchy_items = [] entity = asset_entity while True: parent_id = entity.get("data", {}).get("visualParent") if not parent_id: break entity = io.find_one({"_id": parent_id}) hierarchy_items.append(entity["name"]) # Add parents to entity data for next query entity_data = asset_entity.get("data", {}) entity_data["parents"] = hierarchy_items io.update_many({"_id": asset_entity["_id"]}, {"$set": { "data": entity_data }}) return "/".join(hierarchy_items)
def write_database(self): # Check database if still being latest for item in self._root_item.children(): subset_id = item["subsetId"] latest_repr = item["representation"] latest = io.find_one({ "type": "version", "parent": subset_id }, sort=[("name", -1)], projection={"name": True}) if not latest_repr["parent"] == latest["_id"]: # Found new version, should do refresh return for item in self._root_item.children(): if not item["isLatest"]: continue latest_repr = item["representation"] protected = set() for node in item.children(): is_locked = node["isLocked"] set_locked = node["setLocked"] if set_locked is None: if is_locked: protected.add(node["avalonId"]) elif set_locked: # Set to Lock protected.add(node["avalonId"]) io.update_many({"_id": latest_repr["_id"]}, {"$set": { "data.modelProtected": list(protected) }})
def process(self, instance): self.transfers = dict(packages=list(), files=list(), hardlinks=list()) # Check Delegation # # Contractor completed long-run publish process delegated = instance.context.data.get("contractorAccepted") # Is delegating long-run publish process if instance.data.get("useContractor") and not delegated: return # Assemble data and create version, representations subset, version, representations = self.register(instance) # Integrate representations' files to shareable space self.log.info("Integrating representations to shareable space ...") self.integrate() existed = io.find_one({ "parent": subset["_id"], "name": version["name"] }) if existed is not None: self.log.info("Version existed, representation file has been " "overwritten.") filter_ = {"_id": existed["_id"]} update = {"$set": {"data.time": instance.context.data["time"]}} io.update_many(filter_, update) return # Write version and representations to database version_id = self.write_database(instance, version, representations) instance.data["insertedVersionId"] = version_id # Update dependent self.update_dependent(instance, version_id)
def register(self, instance): # Required environment variables anatomy_data = instance.data["anatomyData"] io.install() context = instance.context project_entity = instance.data["projectEntity"] context_asset_name = context.data["assetEntity"]["name"] asset_name = instance.data["asset"] asset_entity = instance.data.get("assetEntity") if not asset_entity or asset_entity["name"] != context_asset_name: asset_entity = io.find_one({ "type": "asset", "name": asset_name, "parent": project_entity["_id"] }) assert asset_entity, ( "No asset found by the name \"{0}\" in project \"{1}\"" ).format(asset_name, project_entity["name"]) instance.data["assetEntity"] = asset_entity # update anatomy data with asset specific keys # - name should already been set hierarchy = "" parents = asset_entity["data"]["parents"] if parents: hierarchy = "/".join(parents) anatomy_data["hierarchy"] = hierarchy task_name = instance.data.get("task") if task_name: anatomy_data["task"] = task_name anatomy_data["family"] = instance.data.get("family") stagingdir = instance.data.get("stagingDir") if not stagingdir: self.log.info( ("{0} is missing reference to staging directory." " Will try to get it from representation.").format(instance)) else: self.log.debug( "Establishing staging directory @ {0}".format(stagingdir)) # Ensure at least one file is set up for transfer in staging dir. repres = instance.data.get("representations") assert repres, "Instance has no files to transfer" assert isinstance( repres, (list, tuple)), ("Instance 'files' must be a list, got: {0} {1}".format( str(type(repres)), str(repres))) subset = self.get_subset(asset_entity, instance) instance.data["subsetEntity"] = subset version_number = instance.data["version"] self.log.debug("Next version: v{}".format(version_number)) version_data = self.create_version_data(context, instance) version_data_instance = instance.data.get('versionData') if version_data_instance: version_data.update(version_data_instance) # TODO rename method from `create_version` to # `prepare_version` or similar... version = self.create_version(subset=subset, version_number=version_number, data=version_data) self.log.debug("Creating version ...") new_repre_names_low = [_repre["name"].lower() for _repre in repres] existing_version = io.find_one({ 'type': 'version', 'parent': subset["_id"], 'name': version_number }) if existing_version is None: version_id = io.insert_one(version).inserted_id else: # Check if instance have set `append` mode which cause that # only replicated representations are set to archive append_repres = instance.data.get("append", False) # Update version data # TODO query by _id and io.update_many( { 'type': 'version', 'parent': subset["_id"], 'name': version_number }, {'$set': version}) version_id = existing_version['_id'] # Find representations of existing version and archive them current_repres = list( io.find({ "type": "representation", "parent": version_id })) bulk_writes = [] for repre in current_repres: if append_repres: # archive only duplicated representations if repre["name"].lower() not in new_repre_names_low: continue # Representation must change type, # `_id` must be stored to other key and replaced with new # - that is because new representations should have same ID repre_id = repre["_id"] bulk_writes.append(DeleteOne({"_id": repre_id})) repre["orig_id"] = repre_id repre["_id"] = io.ObjectId() repre["type"] = "archived_representation" bulk_writes.append(InsertOne(repre)) # bulk updates if bulk_writes: io._database[io.Session["AVALON_PROJECT"]].bulk_write( bulk_writes) version = io.find_one({"_id": version_id}) instance.data["versionEntity"] = version existing_repres = list( io.find({ "parent": version_id, "type": "archived_representation" })) instance.data['version'] = version['name'] intent_value = instance.context.data.get("intent") if intent_value and isinstance(intent_value, dict): intent_value = intent_value.get("value") if intent_value: anatomy_data["intent"] = intent_value anatomy = instance.context.data['anatomy'] # Find the representations to transfer amongst the files # Each should be a single representation (as such, a single extension) representations = [] destination_list = [] if 'transfers' not in instance.data: instance.data['transfers'] = [] template_name = self.template_name_from_instance(instance) published_representations = {} for idx, repre in enumerate(instance.data["representations"]): published_files = [] # create template data for Anatomy template_data = copy.deepcopy(anatomy_data) if intent_value is not None: template_data["intent"] = intent_value resolution_width = repre.get("resolutionWidth") resolution_height = repre.get("resolutionHeight") fps = instance.data.get("fps") if resolution_width: template_data["resolution_width"] = resolution_width if resolution_width: template_data["resolution_height"] = resolution_height if resolution_width: template_data["fps"] = fps files = repre['files'] if repre.get('stagingDir'): stagingdir = repre['stagingDir'] if repre.get("outputName"): template_data["output"] = repre['outputName'] template = os.path.normpath( anatomy.templates[template_name]["path"]) sequence_repre = isinstance(files, list) repre_context = None if sequence_repre: self.log.debug("files: {}".format(files)) src_collections, remainder = clique.assemble(files) self.log.debug("src_tail_collections: {}".format( str(src_collections))) src_collection = src_collections[0] # Assert that each member has identical suffix src_head = src_collection.format("{head}") src_tail = src_collection.format("{tail}") # fix dst_padding valid_files = [x for x in files if src_collection.match(x)] padd_len = len(valid_files[0].replace(src_head, "").replace( src_tail, "")) src_padding_exp = "%0{}d".format(padd_len) test_dest_files = list() for i in [1, 2]: template_data["representation"] = repre['ext'] template_data["frame"] = src_padding_exp % i anatomy_filled = anatomy.format(template_data) template_filled = anatomy_filled[template_name]["path"] if repre_context is None: repre_context = template_filled.used_values test_dest_files.append(os.path.normpath(template_filled)) template_data["frame"] = repre_context["frame"] self.log.debug("test_dest_files: {}".format( str(test_dest_files))) dst_collections, remainder = clique.assemble(test_dest_files) dst_collection = dst_collections[0] dst_head = dst_collection.format("{head}") dst_tail = dst_collection.format("{tail}") index_frame_start = None if repre.get("frameStart"): frame_start_padding = int(anatomy.templates["render"].get( "frame_padding", anatomy.templates["render"].get("padding"))) index_frame_start = int(repre.get("frameStart")) # exception for slate workflow if index_frame_start and "slate" in instance.data["families"]: index_frame_start -= 1 dst_padding_exp = src_padding_exp dst_start_frame = None for i in src_collection.indexes: # TODO 1.) do not count padding in each index iteration # 2.) do not count dst_padding from src_padding before # index_frame_start check src_padding = src_padding_exp % i src_file_name = "{0}{1}{2}".format(src_head, src_padding, src_tail) dst_padding = src_padding_exp % i if index_frame_start: dst_padding_exp = "%0{}d".format(frame_start_padding) dst_padding = dst_padding_exp % index_frame_start index_frame_start += 1 dst = "{0}{1}{2}".format(dst_head, dst_padding, dst_tail).replace("..", ".") self.log.debug("destination: `{}`".format(dst)) src = os.path.join(stagingdir, src_file_name) self.log.debug("source: {}".format(src)) instance.data["transfers"].append([src, dst]) published_files.append(dst) # for adding first frame into db if not dst_start_frame: dst_start_frame = dst_padding # Store used frame value to template data template_data["frame"] = dst_start_frame dst = "{0}{1}{2}".format(dst_head, dst_start_frame, dst_tail).replace("..", ".") repre['published_path'] = dst else: # Single file # _______ # | |\ # | | # | | # | | # |_______| # template_data.pop("frame", None) fname = files assert not os.path.isabs(fname), ( "Given file name is a full path") template_data["representation"] = repre['ext'] src = os.path.join(stagingdir, fname) anatomy_filled = anatomy.format(template_data) template_filled = anatomy_filled[template_name]["path"] repre_context = template_filled.used_values dst = os.path.normpath(template_filled).replace("..", ".") instance.data["transfers"].append([src, dst]) published_files.append(dst) repre['published_path'] = dst self.log.debug("__ dst: {}".format(dst)) repre["publishedFiles"] = published_files for key in self.db_representation_context_keys: value = template_data.get(key) if not value: continue repre_context[key] = template_data[key] # Use previous representation's id if there are any repre_id = None repre_name_low = repre["name"].lower() for _repre in existing_repres: # NOTE should we check lowered names? if repre_name_low == _repre["name"]: repre_id = _repre["orig_id"] break # Create new id if existing representations does not match if repre_id is None: repre_id = io.ObjectId() representation = { "_id": repre_id, "schema": "pype:representation-2.0", "type": "representation", "parent": version_id, "name": repre['name'], "data": { 'path': dst, 'template': template }, "dependencies": instance.data.get("dependencies", "").split(), # Imprint shortcut to context # for performance reasons. "context": repre_context } if repre.get("outputName"): representation["context"]["output"] = repre['outputName'] if sequence_repre and repre.get("frameStart"): representation['context']['frame'] = ( dst_padding_exp % int(repre.get("frameStart"))) self.log.debug("__ representation: {}".format(representation)) destination_list.append(dst) self.log.debug("__ destination_list: {}".format(destination_list)) instance.data['destination_list'] = destination_list representations.append(representation) published_representations[repre_id] = { "representation": representation, "anatomy_data": template_data, "published_files": published_files } self.log.debug("__ representations: {}".format(representations)) # Remove old representations if there are any (before insertion of new) if existing_repres: repre_ids_to_remove = [] for repre in existing_repres: repre_ids_to_remove.append(repre["_id"]) io.delete_many({"_id": {"$in": repre_ids_to_remove}}) self.log.debug("__ representations: {}".format(representations)) for rep in instance.data["representations"]: self.log.debug("__ represNAME: {}".format(rep['name'])) self.log.debug("__ represPATH: {}".format(rep['published_path'])) io.insert_many(representations) instance.data["published_representations"] = ( published_representations) # self.log.debug("Representation: {}".format(representations)) self.log.info("Registered {} items".format(len(representations)))
def process(self, instance): if not os.environ.get("AVALON_THUMBNAIL_ROOT"): self.log.warning("AVALON_THUMBNAIL_ROOT is not set." " Skipping thumbnail integration.") return published_repres = instance.data.get("published_representations") if not published_repres: self.log.debug( "There are no published representations on the instance.") return project_name = api.Session["AVALON_PROJECT"] anatomy = instance.context.data["anatomy"] if "publish" not in anatomy.templates: self.log.warning("Anatomy is missing the \"publish\" key!") return if "thumbnail" not in anatomy.templates["publish"]: self.log.warning(( "There is no \"thumbnail\" template set for the project \"{}\"" ).format(project_name)) return thumb_repre = None thumb_repre_anatomy_data = None for repre_info in published_repres.values(): repre = repre_info["representation"] if repre["name"].lower() == "thumbnail": thumb_repre = repre thumb_repre_anatomy_data = repre_info["anatomy_data"] break if not thumb_repre: self.log.debug( "There is not representation with name \"thumbnail\"") return io.install() thumbnail_template = anatomy.templates["publish"]["thumbnail"] version = io.find_one({"_id": thumb_repre["parent"]}) if not version: raise AssertionError( "There does not exist version with id {}".format( str(thumb_repre["parent"]))) # Get full path to thumbnail file from representation src_full_path = os.path.normpath(thumb_repre["data"]["path"]) if not os.path.exists(src_full_path): self.log.warning( "Thumbnail file was not found. Path: {}".format(src_full_path)) return filename, file_extension = os.path.splitext(src_full_path) # Create id for mongo entity now to fill anatomy template thumbnail_id = ObjectId() # Prepare anatomy template fill data template_data = copy.deepcopy(thumb_repre_anatomy_data) template_data.update({ "_id": str(thumbnail_id), "thumbnail_root": os.environ.get("AVALON_THUMBNAIL_ROOT"), "ext": file_extension[1:], "thumbnail_type": "thumbnail" }) anatomy_filled = anatomy.format(template_data) template_filled = anatomy_filled["publish"]["thumbnail"] dst_full_path = os.path.normpath(str(template_filled)) self.log.debug("Copying file .. {} -> {}".format( src_full_path, dst_full_path)) dirname = os.path.dirname(dst_full_path) try: os.makedirs(dirname) except OSError as e: if e.errno != errno.EEXIST: tp, value, tb = sys.exc_info() six.reraise(tp, value, tb) shutil.copy(src_full_path, dst_full_path) # Clean template data from keys that are dynamic template_data.pop("_id") template_data.pop("thumbnail_root") repre_context = template_filled.used_values for key in self.required_context_keys: value = template_data.get(key) if not value: continue repre_context[key] = template_data[key] thumbnail_entity = { "_id": thumbnail_id, "type": "thumbnail", "schema": "pype:thumbnail-1.0", "data": { "template": thumbnail_template, "template_data": repre_context } } # Create thumbnail entity io.insert_one(thumbnail_entity) self.log.debug("Creating entity in database {}".format( str(thumbnail_entity))) # Set thumbnail id for version io.update_many({"_id": version["_id"]}, {"$set": { "data.thumbnail_id": thumbnail_id }}) self.log.debug("Setting thumbnail for version \"{}\" <{}>".format( version["name"], str(version["_id"]))) asset_entity = instance.data["assetEntity"] io.update_many({"_id": asset_entity["_id"]}, {"$set": { "data.thumbnail_id": thumbnail_id }}) self.log.debug("Setting thumbnail for asset \"{}\" <{}>".format( asset_entity["name"], str(version["_id"])))
def import_to_avalon(self, input_data, parent=None): for name in input_data: self.log.info("input_data[name]: {}".format(input_data[name])) entity_data = input_data[name] entity_type = entity_data["entity_type"] data = {} data["entityType"] = entity_type # Custom attributes. for k, val in entity_data.get("custom_attributes", {}).items(): data[k] = val if entity_type.lower() != "project": data["inputs"] = entity_data.get("inputs", []) # Tasks. tasks = entity_data.get("tasks", []) if tasks is not None or len(tasks) > 0: data["tasks"] = tasks parents = [] visualParent = None # do not store project"s id as visualParent (silo asset) if self.project is not None: if self.project["_id"] != parent["_id"]: visualParent = parent["_id"] parents.extend( parent.get("data", {}).get("parents", [])) parents.append(parent["name"]) data["visualParent"] = visualParent data["parents"] = parents update_data = True # Process project if entity_type.lower() == "project": entity = io.find_one({"type": "project"}) # TODO: should be in validator? assert (entity is not None), "Did not find project in DB" # get data from already existing project cur_entity_data = entity.get("data") or {} cur_entity_data.update(data) data = cur_entity_data self.project = entity # Raise error if project or parent are not set elif self.project is None or parent is None: raise AssertionError("Collected items are not in right order!") # Else process assset else: entity = io.find_one({"type": "asset", "name": name}) if entity: # Do not override data, only update cur_entity_data = entity.get("data") or {} cur_entity_data.update(data) data = cur_entity_data else: # Skip updating data update_data = False archived_entities = io.find({ "type": "archived_asset", "name": name }) unarchive_entity = None for archived_entity in archived_entities: archived_parents = (archived_entity.get( "data", {}).get("parents")) if data["parents"] == archived_parents: unarchive_entity = archived_entity break if unarchive_entity is None: # Create entity if doesn"t exist entity = self.create_avalon_asset(name, data) else: # Unarchive if entity was archived entity = self.unarchive_entity(unarchive_entity, data) if update_data: # Update entity data with input data io.update_many({"_id": entity["_id"]}, {"$set": { "data": data }}) if "childs" in entity_data: self.import_to_avalon(entity_data["childs"], entity)
def process(self, instance): import os import re import sys import json import subprocess from avalon import io asset_doc = instance.data["assetDoc"] asset_name = asset_doc["name"] # Check asset's rigging task option value_path = "taskOptions.rigging.autoModelUpdate.value" value = asset_doc["data"] for entry in value_path.split("."): value = value.get(entry, {}) if not value: # Auto model update not enabled return # Get subset, version documents from instance which just been # integrated. model_subset, model_version, _ = instance.data["toDatabase"] if model_version["name"] == 1: # First version of model, must not have dependent rig. return # Find all previous versions of model, only document id is needed. previous = io.find({ "type": "version", "parent": model_subset["_id"] }, sort=[("name", -1)], projection={"_id": True}, skip=1) # Skip the latest previous = set([str(p["_id"]) for p in previous]) if not previous: self.log.warning("Model is now on version %d but has no previous, " "skip updating rig." % model_version["name"]) return # Any latest version of rig may not be using the latest model, so # we iterate through all rig subsets' latest version and compare # the dependency data with all previous model versions to find the # dependent. dependent_rigs = dict() for rig_subset in io.find( { "type": "subset", "parent": asset_doc["_id"], "name": re.compile("rig*") }, projection={ "_id": True, "name": True }): latest_rig = io.find_one( { "type": "version", "parent": rig_subset["_id"] }, sort=[("name", -1)], projection={"data.dependencies": True}) if latest_rig is None: # Not likely to happen, but just in case continue # Consider dependent if any dependency matched in model versions dependencies = set(latest_rig["data"]["dependencies"].keys()) if dependencies.intersection(previous): dependent_rigs[str(latest_rig["_id"])] = rig_subset["name"] if not dependent_rigs: self.log.info("No rig to update, skip auto process.") return # Submit subprocess mayapy_exe = os.path.join(os.path.dirname(sys.executable), "mayapy.exe") cmd = [ mayapy_exe, __file__, "asset_name={}".format(str(asset_name)), "model_subset={}".format(str(model_subset["name"])), "rig_versions={}".format(json.dumps(dependent_rigs)), ] print("auto rig cmd: {}".format(cmd)) try: out_bytes = subprocess.check_output(cmd, shell=True) except subprocess.CalledProcessError: # Mark failed for future debug. io.update_many({"_id": model_version["_id"]}, {"$set": { "data.rigAutoUpdateFailed": True }}) raise Exception("Model publish success but Rig auto update " "failed. Please inform rigger or TD.") else: print(out_bytes)
def process(self, instance): context = instance.context if not all(result["success"] for result in context.data["results"]): self.log.warning("Atomicity not held, aborting.") return # Integrate representations' to database self.log.info("Integrating representations to database ...") asset = context.data["assetDoc"] subset, version, representations = instance.data["toDatabase"] # Write subset if not exists filter = {"parent": asset["_id"], "name": subset["name"]} if io.find_one(filter) is None: io.insert_one(subset) # Write version if not exists filter = {"parent": subset["_id"], "name": version["name"]} existed_version = io.find_one(filter) if existed_version is None: # Write version and representations to database version_id = self.write_database(instance, version, representations) instance.data["insertedVersionId"] = version_id # Update dependent self.update_dependent(instance, version_id) else: if context.data.get("_progressivePublishing"): if instance.data.get("_progressiveOutput") is None: pass # Not given any output, no progress change else: self.log.info("Update version publish progress.") # Update version document "data.time" filter_ = {"_id": existed_version["_id"]} update = {"$set": {"data.time": context.data["time"]}} if "progress" in version["data"]: # Update version document "progress.current" progress = version["data"]["progress"]["current"] update["$inc"] = {"data.progress.current": progress} else: pass # progress == -1, no progress update needed. io.update_many(filter_, update) else: self.log.info("Version existed, representation file has been " "overwritten.") # Update version document "data.time" filter_ = {"_id": existed_version["_id"]} update = {"$set": {"data.time": context.data["time"]}} io.update_many(filter_, update) # Update representation documents "data" for representation in representations: filter_ = { "name": representation["name"], "parent": existed_version["_id"], } update = {"$set": {"data": representation["data"]}} io.update_many(filter_, update)
def register(self, instance): # Required environment variables PROJECT = api.Session["AVALON_PROJECT"] ASSET = instance.data.get("asset") or api.Session["AVALON_ASSET"] TASK = instance.data.get("task") or api.Session["AVALON_TASK"] LOCATION = api.Session["AVALON_LOCATION"] context = instance.context # Atomicity # # Guarantee atomic publishes - each asset contains # an identical set of members. # __ # / o # / \ # | o | # \ / # o __/ # # for result in context.data["results"]: # if not result["success"]: # self.log.debug(result) # exc_type, exc_value, exc_traceback = result["error_info"] # extracted_traceback = traceback.extract_tb(exc_traceback)[-1] # self.log.debug( # "Error at line {}: \"{}\"".format( # extracted_traceback[1], result["error"] # ) # ) # assert all(result["success"] for result in context.data["results"]),( # "Atomicity not held, aborting.") # Assemble # # | # v # ---> <---- # ^ # | # stagingdir = instance.data.get("stagingDir") if not stagingdir: self.log.info('''{} is missing reference to staging directory Will try to get it from representation'''.format(instance)) # extra check if stagingDir actually exists and is available self.log.debug("Establishing staging directory @ %s" % stagingdir) # Ensure at least one file is set up for transfer in staging dir. repres = instance.data.get("representations", None) assert repres, "Instance has no files to transfer" assert isinstance(repres, (list, tuple)), ( "Instance 'files' must be a list, got: {0}".format(repres)) # FIXME: io is not initialized at this point for shell host io.install() project = io.find_one({"type": "project"}) asset = io.find_one({ "type": "asset", "name": ASSET, "parent": project["_id"] }) assert all([project, asset]), ("Could not find current project or " "asset '%s'" % ASSET) subset = self.get_subset(asset, instance) # get next version latest_version = io.find_one( { "type": "version", "parent": subset["_id"] }, {"name": True}, sort=[("name", -1)]) next_version = 1 if latest_version is not None: next_version += latest_version["name"] if instance.data.get('version'): next_version = int(instance.data.get('version')) # self.log.info("Verifying version from assumed destination") # assumed_data = instance.data["assumedTemplateData"] # assumed_version = assumed_data["version"] # if assumed_version != next_version: # raise AttributeError("Assumed version 'v{0:03d}' does not match" # "next version in database " # "('v{1:03d}')".format(assumed_version, # next_version)) self.log.debug("Next version: v{0:03d}".format(next_version)) version_data = self.create_version_data(context, instance) version_data_instance = instance.data.get('versionData') if version_data_instance: version_data.update(version_data_instance) version = self.create_version(subset=subset, version_number=next_version, locations=[LOCATION], data=version_data) self.log.debug("Creating version ...") existing_version = io.find_one({ 'type': 'version', 'parent': subset["_id"], 'name': next_version }) if existing_version is None: version_id = io.insert_one(version).inserted_id else: io.update_many( { 'type': 'version', 'parent': subset["_id"], 'name': next_version }, {'$set': version}) version_id = existing_version['_id'] instance.data['version'] = version['name'] # Write to disk # _ # | | # _| |_ # ____\ / # |\ \ / \ # \ \ v \ # \ \________. # \|________| # root = api.registered_root() hierarchy = "" parents = io.find_one({ "type": 'asset', "name": ASSET })['data']['parents'] if parents and len(parents) > 0: # hierarchy = os.path.sep.join(hierarchy) hierarchy = os.path.join(*parents) anatomy = instance.context.data['anatomy'] # Find the representations to transfer amongst the files # Each should be a single representation (as such, a single extension) representations = [] destination_list = [] template_name = 'publish' if 'transfers' not in instance.data: instance.data['transfers'] = [] for idx, repre in enumerate(instance.data["representations"]): # Collection # _______ # |______|\ # | |\| # | || # | || # | || # |_______| # # create template data for Anatomy template_data = { "root": root, "project": { "name": PROJECT, "code": project['data']['code'] }, "silo": asset.get('silo'), "task": TASK, "asset": ASSET, "family": instance.data['family'], "subset": subset["name"], "version": int(version["name"]), "hierarchy": hierarchy } files = repre['files'] if repre.get('stagingDir'): stagingdir = repre['stagingDir'] if repre.get('anatomy_template'): template_name = repre['anatomy_template'] template = os.path.normpath( anatomy.templates[template_name]["path"]) sequence_repre = isinstance(files, list) if sequence_repre: src_collections, remainder = clique.assemble(files) self.log.debug("src_tail_collections: {}".format( str(src_collections))) src_collection = src_collections[0] # Assert that each member has identical suffix src_head = src_collection.format("{head}") src_tail = src_collection.format("{tail}") # fix dst_padding valid_files = [x for x in files if src_collection.match(x)] padd_len = len(valid_files[0].replace(src_head, "").replace( src_tail, "")) src_padding_exp = "%0{}d".format(padd_len) test_dest_files = list() for i in [1, 2]: template_data["representation"] = repre['ext'] template_data["frame"] = src_padding_exp % i anatomy_filled = anatomy.format(template_data) test_dest_files.append( os.path.normpath( anatomy_filled[template_name]["path"])) self.log.debug("test_dest_files: {}".format( str(test_dest_files))) dst_collections, remainder = clique.assemble(test_dest_files) dst_collection = dst_collections[0] dst_head = dst_collection.format("{head}") dst_tail = dst_collection.format("{tail}") index_frame_start = None if repre.get("frameStart"): frame_start_padding = len(str(repre.get("frameEnd"))) index_frame_start = int(repre.get("frameStart")) dst_padding_exp = src_padding_exp dst_start_frame = None for i in src_collection.indexes: src_padding = src_padding_exp % i # for adding first frame into db if not dst_start_frame: dst_start_frame = src_padding src_file_name = "{0}{1}{2}".format(src_head, src_padding, src_tail) dst_padding = src_padding_exp % i if index_frame_start: dst_padding_exp = "%0{}d".format(frame_start_padding) dst_padding = dst_padding_exp % index_frame_start index_frame_start += 1 dst = "{0}{1}{2}".format(dst_head, dst_padding, dst_tail).replace("..", ".") self.log.debug("destination: `{}`".format(dst)) src = os.path.join(stagingdir, src_file_name) self.log.debug("source: {}".format(src)) instance.data["transfers"].append([src, dst]) dst = "{0}{1}{2}".format(dst_head, dst_start_frame, dst_tail).replace("..", ".") repre['published_path'] = dst else: # Single file # _______ # | |\ # | | # | | # | | # |_______| # template_data.pop("frame", None) fname = files assert not os.path.isabs(fname), ( "Given file name is a full path") template_data["representation"] = repre['ext'] if repre.get("outputName"): template_data["output"] = repre['outputName'] src = os.path.join(stagingdir, fname) anatomy_filled = anatomy.format(template_data) dst = os.path.normpath( anatomy_filled[template_name]["path"]).replace("..", ".") instance.data["transfers"].append([src, dst]) repre['published_path'] = dst self.log.debug("__ dst: {}".format(dst)) representation = { "schema": "pype:representation-2.0", "type": "representation", "parent": version_id, "name": repre['name'], "data": { 'path': dst, 'template': template }, "dependencies": instance.data.get("dependencies", "").split(), # Imprint shortcut to context # for performance reasons. "context": { "root": root, "project": { "name": PROJECT, "code": project['data']['code'] }, 'task': TASK, "silo": asset.get('silo'), "asset": ASSET, "family": instance.data['family'], "subset": subset["name"], "version": version["name"], "hierarchy": hierarchy, "representation": repre['ext'] } } if sequence_repre and repre.get("frameStart"): representation['context']['frame'] = repre.get("frameStart") self.log.debug("__ representation: {}".format(representation)) destination_list.append(dst) self.log.debug("__ destination_list: {}".format(destination_list)) instance.data['destination_list'] = destination_list representations.append(representation) self.log.debug("__ representations: {}".format(representations)) self.log.debug("__ representations: {}".format(representations)) for rep in instance.data["representations"]: self.log.debug("__ represNAME: {}".format(rep['name'])) self.log.debug("__ represPATH: {}".format(rep['published_path'])) io.insert_many(representations) # self.log.debug("Representation: {}".format(representations)) self.log.info("Registered {} items".format(len(representations)))