def get_app_environments_for_context(project_name,
asset_name,
task_name,
app_name,
env=None):
"""Prepare environment variables by context.
Args:
project_name (str): Name of project.
asset_name (str): Name of asset.
task_name (str): Name of task.
app_name (str): Name of application that is launched and can be found
by ApplicationManager.
env (dict): Initial environment variables. `os.environ` is used when
not passed.
Returns:
dict: Environments for passed context and application.
"""
from avalon.api import AvalonMongoDB
# Avalon database connection
dbcon = AvalonMongoDB()
dbcon.Session["AVALON_PROJECT"] = project_name
dbcon.install()
# Project document
project_doc = dbcon.find_one({"type": "project"})
asset_doc = dbcon.find_one({"type": "asset", "name": asset_name})
# Prepare app object which can be obtained only from ApplciationManager
app_manager = ApplicationManager()
app = app_manager.applications[app_name]
# Project's anatomy
anatomy = Anatomy(project_name)
data = EnvironmentPrepData({
"project_name": project_name,
"asset_name": asset_name,
"task_name": task_name,
"app_name": app_name,
"app": app,
"dbcon": dbcon,
"project_doc": project_doc,
"asset_doc": asset_doc,
"anatomy": anatomy,
"env": env
})
prepare_host_environments(data)
prepare_context_environments(data)
# Discard avalon connection
dbcon.uninstall()
return data["env"]
def start_timer(self, project_name, asset_name, task_name, hierarchy):
"""
Start timer for 'project_name', 'asset_name' and 'task_name'
Called from REST api by hosts.
Args:
project_name (string)
asset_name (string)
task_name (string)
hierarchy (string)
"""
dbconn = AvalonMongoDB()
dbconn.install()
dbconn.Session["AVALON_PROJECT"] = project_name
asset_doc = dbconn.find_one({"type": "asset", "name": asset_name})
if not asset_doc:
raise ValueError("Uknown asset {}".format(asset_name))
task_type = ''
try:
task_type = asset_doc["data"]["tasks"][task_name]["type"]
except KeyError:
self.log.warning(
"Couldn't find task_type for {}".format(task_name))
hierarchy = hierarchy.split("\\")
hierarchy.append(asset_name)
data = {
"project_name": project_name,
"task_name": task_name,
"task_type": task_type,
"hierarchy": hierarchy
}
self.timer_started(None, data)
class Delivery(BaseAction):
identifier = "delivery.action"
label = "Delivery"
description = "Deliver data to client"
role_list = ["Pypeclub", "Administrator", "Project manager"]
icon = statics_icon("ftrack", "action_icons", "Delivery.svg")
settings_key = "delivery_action"
def __init__(self, *args, **kwargs):
self.db_con = AvalonMongoDB()
super(Delivery, self).__init__(*args, **kwargs)
def discover(self, session, entities, event):
is_valid = False
for entity in entities:
if entity.entity_type.lower() == "assetversion":
is_valid = True
break
if is_valid:
is_valid = self.valid_roles(session, entities, event)
return is_valid
def interface(self, session, entities, event):
if event["data"].get("values", {}):
return
title = "Delivery data to Client"
items = []
item_splitter = {"type": "label", "value": "---"}
project_entity = self.get_project_from_entity(entities[0])
project_name = project_entity["full_name"]
self.db_con.install()
self.db_con.Session["AVALON_PROJECT"] = project_name
project_doc = self.db_con.find_one({"type": "project"})
if not project_doc:
return {
"success":
False,
"message":
("Didn't found project \"{}\" in avalon.").format(project_name)
}
repre_names = self._get_repre_names(entities)
self.db_con.uninstall()
items.append({
"type": "hidden",
"name": "__project_name__",
"value": project_name
})
# Prpeare anatomy data
anatomy = Anatomy(project_name)
new_anatomies = []
first = None
for key, template in (anatomy.templates.get("delivery") or {}).items():
# Use only keys with `{root}` or `{root[*]}` in value
if isinstance(template, str) and "{root" in template:
new_anatomies.append({"label": key, "value": key})
if first is None:
first = key
skipped = False
# Add message if there are any common components
if not repre_names or not new_anatomies:
skipped = True
items.append({
"type": "label",
"value": "<h1>Something went wrong:</h1>"
})
items.append({
"type": "hidden",
"name": "__skipped__",
"value": skipped
})
if not repre_names:
if len(entities) == 1:
items.append({
"type":
"label",
"value":
("- Selected entity doesn't have components to deliver.")
})
else:
items.append({
"type":
"label",
"value":
("- Selected entities don't have common components.")
})
# Add message if delivery anatomies are not set
if not new_anatomies:
items.append({
"type":
"label",
"value": ("- `\"delivery\"` anatomy key is not set in config.")
})
# Skip if there are any data shortcomings
if skipped:
return {"items": items, "title": title}
items.append({
"value": "<h1>Choose Components to deliver</h1>",
"type": "label"
})
for repre_name in repre_names:
items.append({
"type": "boolean",
"value": False,
"label": repre_name,
"name": repre_name
})
items.append(item_splitter)
items.append({
"value": "<h2>Location for delivery</h2>",
"type": "label"
})
items.append({
"type":
"label",
"value":
("<i>NOTE: It is possible to replace `root` key in anatomy.</i>")
})
items.append({
"type": "text",
"name": "__location_path__",
"empty_text": "Type location path here...(Optional)"
})
items.append(item_splitter)
items.append({
"value": "<h2>Anatomy of delivery files</h2>",
"type": "label"
})
items.append({
"type":
"label",
"value": ("<p><i>NOTE: These can be set in Anatomy.yaml"
" within `delivery` key.</i></p>")
})
items.append({
"type": "enumerator",
"name": "__new_anatomies__",
"data": new_anatomies,
"value": first
})
return {"items": items, "title": title}
def _get_repre_names(self, entities):
version_ids = self._get_interest_version_ids(entities)
repre_docs = self.db_con.find({
"type": "representation",
"parent": {
"$in": version_ids
}
})
return list(sorted(repre_docs.distinct("name")))
def _get_interest_version_ids(self, entities):
parent_ent_by_id = {}
subset_names = set()
version_nums = set()
for entity in entities:
asset = entity["asset"]
parent = asset["parent"]
parent_ent_by_id[parent["id"]] = parent
subset_name = asset["name"]
subset_names.add(subset_name)
version = entity["version"]
version_nums.add(version)
asset_docs_by_ftrack_id = self._get_asset_docs(parent_ent_by_id)
subset_docs = self._get_subset_docs(asset_docs_by_ftrack_id,
subset_names, entities)
version_docs = self._get_version_docs(asset_docs_by_ftrack_id,
subset_docs, version_nums,
entities)
return [version_doc["_id"] for version_doc in version_docs]
def _get_version_docs(self, asset_docs_by_ftrack_id, subset_docs,
version_nums, entities):
subset_docs_by_id = {
subset_doc["_id"]: subset_doc
for subset_doc in subset_docs
}
version_docs = list(
self.db_con.find({
"type": "version",
"parent": {
"$in": list(subset_docs_by_id.keys())
},
"name": {
"$in": list(version_nums)
}
}))
version_docs_by_parent_id = collections.defaultdict(dict)
for version_doc in version_docs:
subset_doc = subset_docs_by_id[version_doc["parent"]]
asset_id = subset_doc["parent"]
subset_name = subset_doc["name"]
version = version_doc["name"]
if version_docs_by_parent_id[asset_id].get(subset_name) is None:
version_docs_by_parent_id[asset_id][subset_name] = {}
version_docs_by_parent_id[asset_id][subset_name][version] = (
version_doc)
filtered_versions = []
for entity in entities:
asset = entity["asset"]
parent = asset["parent"]
asset_doc = asset_docs_by_ftrack_id[parent["id"]]
subsets_by_name = version_docs_by_parent_id.get(asset_doc["_id"])
if not subsets_by_name:
continue
subset_name = asset["name"]
version_docs_by_version = subsets_by_name.get(subset_name)
if not version_docs_by_version:
continue
version = entity["version"]
version_doc = version_docs_by_version.get(version)
if version_doc:
filtered_versions.append(version_doc)
return filtered_versions
def _get_subset_docs(self, asset_docs_by_ftrack_id, subset_names,
entities):
asset_doc_ids = list()
for asset_doc in asset_docs_by_ftrack_id.values():
asset_doc_ids.append(asset_doc["_id"])
subset_docs = list(
self.db_con.find({
"type": "subset",
"parent": {
"$in": asset_doc_ids
},
"name": {
"$in": list(subset_names)
}
}))
subset_docs_by_parent_id = collections.defaultdict(dict)
for subset_doc in subset_docs:
asset_id = subset_doc["parent"]
subset_name = subset_doc["name"]
subset_docs_by_parent_id[asset_id][subset_name] = subset_doc
filtered_subsets = []
for entity in entities:
asset = entity["asset"]
parent = asset["parent"]
asset_doc = asset_docs_by_ftrack_id[parent["id"]]
subsets_by_name = subset_docs_by_parent_id.get(asset_doc["_id"])
if not subsets_by_name:
continue
subset_name = asset["name"]
subset_doc = subsets_by_name.get(subset_name)
if subset_doc:
filtered_subsets.append(subset_doc)
return filtered_subsets
def _get_asset_docs(self, parent_ent_by_id):
asset_docs = list(
self.db_con.find({
"type": "asset",
"data.ftrackId": {
"$in": list(parent_ent_by_id.keys())
}
}))
asset_docs_by_ftrack_id = {
asset_doc["data"]["ftrackId"]: asset_doc
for asset_doc in asset_docs
}
entities_by_mongo_id = {}
entities_by_names = {}
for ftrack_id, entity in parent_ent_by_id.items():
if ftrack_id not in asset_docs_by_ftrack_id:
parent_mongo_id = entity["custom_attributes"].get(
CUST_ATTR_ID_KEY)
if parent_mongo_id:
entities_by_mongo_id[ObjectId(parent_mongo_id)] = entity
else:
entities_by_names[entity["name"]] = entity
expressions = []
if entities_by_mongo_id:
expression = {
"type": "asset",
"_id": {
"$in": list(entities_by_mongo_id.keys())
}
}
expressions.append(expression)
if entities_by_names:
expression = {
"type": "asset",
"name": {
"$in": list(entities_by_names.keys())
}
}
expressions.append(expression)
if expressions:
if len(expressions) == 1:
filter = expressions[0]
else:
filter = {"$or": expressions}
asset_docs = self.db_con.find(filter)
for asset_doc in asset_docs:
if asset_doc["_id"] in entities_by_mongo_id:
entity = entities_by_mongo_id[asset_doc["_id"]]
asset_docs_by_ftrack_id[entity["id"]] = asset_doc
elif asset_doc["name"] in entities_by_names:
entity = entities_by_names[asset_doc["name"]]
asset_docs_by_ftrack_id[entity["id"]] = asset_doc
return asset_docs_by_ftrack_id
def launch(self, session, entities, event):
if "values" not in event["data"]:
return
values = event["data"]["values"]
skipped = values.pop("__skipped__")
if skipped:
return None
user_id = event["source"]["user"]["id"]
user_entity = session.query(
"User where id is {}".format(user_id)).one()
job = session.create(
"Job", {
"user": user_entity,
"status": "running",
"data": json.dumps({"description": "Delivery processing."})
})
session.commit()
try:
self.db_con.install()
self.real_launch(session, entities, event)
job["status"] = "done"
except Exception:
self.log.warning("Failed during processing delivery action.",
exc_info=True)
finally:
if job["status"] != "done":
job["status"] = "failed"
session.commit()
self.db_con.uninstall()
if job["status"] == "failed":
return {
"success": False,
"message": "Delivery failed. Check logs for more information."
}
return True
def real_launch(self, session, entities, event):
self.log.info("Delivery action just started.")
report_items = collections.defaultdict(list)
values = event["data"]["values"]
location_path = values.pop("__location_path__")
anatomy_name = values.pop("__new_anatomies__")
project_name = values.pop("__project_name__")
repre_names = []
for key, value in values.items():
if value is True:
repre_names.append(key)
if not repre_names:
return {
"success": True,
"message": "Not selected components to deliver."
}
location_path = location_path.strip()
if location_path:
location_path = os.path.normpath(location_path)
if not os.path.exists(location_path):
os.makedirs(location_path)
self.db_con.Session["AVALON_PROJECT"] = project_name
self.log.debug("Collecting representations to process.")
version_ids = self._get_interest_version_ids(entities)
repres_to_deliver = list(
self.db_con.find({
"type": "representation",
"parent": {
"$in": version_ids
},
"name": {
"$in": repre_names
}
}))
anatomy = Anatomy(project_name)
format_dict = {}
if location_path:
location_path = location_path.replace("\\", "/")
root_names = anatomy.root_names_from_templates(
anatomy.templates["delivery"])
if root_names is None:
format_dict["root"] = location_path
else:
format_dict["root"] = {}
for name in root_names:
format_dict["root"][name] = location_path
datetime_data = config.get_datetime_data()
for repre in repres_to_deliver:
source_path = repre.get("data", {}).get("path")
debug_msg = "Processing representation {}".format(repre["_id"])
if source_path:
debug_msg += " with published path {}.".format(source_path)
self.log.debug(debug_msg)
# Get destination repre path
anatomy_data = copy.deepcopy(repre["context"])
anatomy_data.update(datetime_data)
anatomy_filled = anatomy.format_all(anatomy_data)
test_path = anatomy_filled["delivery"][anatomy_name]
if not test_path.solved:
msg = ("Missing keys in Representation's context"
" for anatomy template \"{}\".").format(anatomy_name)
if test_path.missing_keys:
keys = ", ".join(test_path.missing_keys)
sub_msg = (
"Representation: {}<br>- Missing keys: \"{}\"<br>"
).format(str(repre["_id"]), keys)
if test_path.invalid_types:
items = []
for key, value in test_path.invalid_types.items():
items.append("\"{}\" {}".format(key, str(value)))
keys = ", ".join(items)
sub_msg = ("Representation: {}<br>"
"- Invalid value DataType: \"{}\"<br>").format(
str(repre["_id"]), keys)
report_items[msg].append(sub_msg)
self.log.warning(
"{} Representation: \"{}\" Filled: <{}>".format(
msg, str(repre["_id"]), str(test_path)))
continue
# Get source repre path
frame = repre['context'].get('frame')
if frame:
repre["context"]["frame"] = len(str(frame)) * "#"
repre_path = self.path_from_represenation(repre, anatomy)
# TODO add backup solution where root of path from component
# is repalced with root
args = (repre_path, anatomy, anatomy_name, anatomy_data,
format_dict, report_items)
if not frame:
self.process_single_file(*args)
else:
self.process_sequence(*args)
return self.report(report_items)
def process_single_file(self, repre_path, anatomy, anatomy_name,
anatomy_data, format_dict, report_items):
anatomy_filled = anatomy.format(anatomy_data)
if format_dict:
template_result = anatomy_filled["delivery"][anatomy_name]
delivery_path = template_result.rootless.format(**format_dict)
else:
delivery_path = anatomy_filled["delivery"][anatomy_name]
delivery_folder = os.path.dirname(delivery_path)
if not os.path.exists(delivery_folder):
os.makedirs(delivery_folder)
self.copy_file(repre_path, delivery_path)
def process_sequence(self, repre_path, anatomy, anatomy_name, anatomy_data,
format_dict, report_items):
dir_path, file_name = os.path.split(str(repre_path))
base_name, ext = os.path.splitext(file_name)
file_name_items = None
if "#" in base_name:
file_name_items = [part for part in base_name.split("#") if part]
elif "%" in base_name:
file_name_items = base_name.split("%")
if not file_name_items:
msg = "Source file was not found"
report_items[msg].append(repre_path)
self.log.warning("{} <{}>".format(msg, repre_path))
return
src_collections, remainder = clique.assemble(os.listdir(dir_path))
src_collection = None
for col in src_collections:
if col.tail != ext:
continue
# skip if collection don't have same basename
if not col.head.startswith(file_name_items[0]):
continue
src_collection = col
break
if src_collection is None:
# TODO log error!
msg = "Source collection of files was not found"
report_items[msg].append(repre_path)
self.log.warning("{} <{}>".format(msg, repre_path))
return
frame_indicator = "@####@"
anatomy_data["frame"] = frame_indicator
anatomy_filled = anatomy.format(anatomy_data)
if format_dict:
template_result = anatomy_filled["delivery"][anatomy_name]
delivery_path = template_result.rootless.format(**format_dict)
else:
delivery_path = anatomy_filled["delivery"][anatomy_name]
delivery_folder = os.path.dirname(delivery_path)
dst_head, dst_tail = delivery_path.split(frame_indicator)
dst_padding = src_collection.padding
dst_collection = clique.Collection(head=dst_head,
tail=dst_tail,
padding=dst_padding)
if not os.path.exists(delivery_folder):
os.makedirs(delivery_folder)
src_head = src_collection.head
src_tail = src_collection.tail
for index in src_collection.indexes:
src_padding = src_collection.format("{padding}") % index
src_file_name = "{}{}{}".format(src_head, src_padding, src_tail)
src = os.path.normpath(os.path.join(dir_path, src_file_name))
dst_padding = dst_collection.format("{padding}") % index
dst = "{}{}{}".format(dst_head, dst_padding, dst_tail)
self.copy_file(src, dst)
def path_from_represenation(self, representation, anatomy):
try:
template = representation["data"]["template"]
except KeyError:
return None
try:
context = representation["context"]
context["root"] = anatomy.roots
path = pipeline.format_template_with_optional_keys(
context, template)
except KeyError:
# Template references unavailable data
return None
return os.path.normpath(path)
def copy_file(self, src_path, dst_path):
if os.path.exists(dst_path):
return
try:
filelink.create(src_path, dst_path, filelink.HARDLINK)
except OSError:
shutil.copyfile(src_path, dst_path)
def report(self, report_items):
items = []
title = "Delivery report"
for msg, _items in report_items.items():
if not _items:
continue
if items:
items.append({"type": "label", "value": "---"})
items.append({"type": "label", "value": "# {}".format(msg)})
if not isinstance(_items, (list, tuple)):
_items = [_items]
__items = []
for item in _items:
__items.append(str(item))
items.append({
"type": "label",
"value": '<p>{}</p>'.format("<br>".join(__items))
})
if not items:
return {"success": True, "message": "Delivery Finished"}
return {
"items": items,
"title": title,
"success": False,
"message": "Delivery Finished"
}
class AppplicationsAction(BaseAction):
"""Application Action class.
Args:
session (ftrack_api.Session): Session where action will be registered.
label (str): A descriptive string identifing your action.
varaint (str, optional): To group actions together, give them the same
label and specify a unique variant per action.
identifier (str): An unique identifier for app.
description (str): A verbose descriptive text for you action.
icon (str): Url path to icon which will be shown in Ftrack web.
"""
type = "Application"
label = "Application action"
identifier = "pype_app.{}.".format(str(uuid4()))
icon_url = os.environ.get("OPENPYPE_STATICS_SERVER")
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.application_manager = ApplicationManager()
self.dbcon = AvalonMongoDB()
def construct_requirements_validations(self):
# Override validation as this action does not need them
return
def register(self):
"""Registers the action, subscribing the discover and launch topics."""
discovery_subscription = (
"topic=ftrack.action.discover and source.user.username={0}"
).format(self.session.api_user)
self.session.event_hub.subscribe(discovery_subscription,
self._discover,
priority=self.priority)
launch_subscription = ("topic=ftrack.action.launch"
" and data.actionIdentifier={0}"
" and source.user.username={1}").format(
self.identifier + "*",
self.session.api_user)
self.session.event_hub.subscribe(launch_subscription, self._launch)
def _discover(self, event):
entities = self._translate_event(event)
items = self.discover(self.session, entities, event)
if items:
return {"items": items}
def discover(self, session, entities, event):
"""Return true if we can handle the selected entities.
Args:
session (ftrack_api.Session): Helps to query necessary data.
entities (list): Object of selected entities.
event (ftrack_api.Event): Ftrack event causing discover callback.
"""
if (len(entities) != 1 or entities[0].entity_type.lower() != "task"):
return False
entity = entities[0]
if entity["parent"].entity_type.lower() == "project":
return False
avalon_project_apps = event["data"].get("avalon_project_apps", None)
avalon_project_doc = event["data"].get("avalon_project_doc", None)
if avalon_project_apps is None:
if avalon_project_doc is None:
ft_project = self.get_project_from_entity(entity)
project_name = ft_project["full_name"]
if not self.dbcon.is_installed():
self.dbcon.install()
self.dbcon.Session["AVALON_PROJECT"] = project_name
avalon_project_doc = self.dbcon.find_one({"type": "project"
}) or False
event["data"]["avalon_project_doc"] = avalon_project_doc
if not avalon_project_doc:
return False
project_apps_config = avalon_project_doc["config"].get("apps", [])
avalon_project_apps = [app["name"]
for app in project_apps_config] or False
event["data"]["avalon_project_apps"] = avalon_project_apps
if not avalon_project_apps:
return False
items = []
for app_name in avalon_project_apps:
app = self.application_manager.applications.get(app_name)
if not app or not app.enabled:
continue
app_icon = app.icon
if app_icon and self.icon_url:
try:
app_icon = app_icon.format(self.icon_url)
except Exception:
self.log.warning(
("Couldn't fill icon path. Icon template: \"{}\""
" --- Icon url: \"{}\"").format(
app_icon, self.icon_url))
app_icon = None
items.append({
"label": app.group.label,
"variant": app.label,
"description": None,
"actionIdentifier": self.identifier + app_name,
"icon": app_icon
})
return items
def launch(self, session, entities, event):
"""Callback method for the custom action.
return either a bool (True if successful or False if the action failed)
or a dictionary with they keys `message` and `success`, the message
should be a string and will be displayed as feedback to the user,
success should be a bool, True if successful or False if the action
failed.
*session* is a `ftrack_api.Session` instance
*entities* is a list of tuples each containing the entity type and
the entity id. If the entity is a hierarchical you will always get
the entity type TypedContext, once retrieved through a get operation
you will have the "real" entity type ie. example Shot, Sequence
or Asset Build.
*event* the unmodified original event
"""
identifier = event["data"]["actionIdentifier"]
app_name = identifier[len(self.identifier):]
entity = entities[0]
task_name = entity["name"]
asset_name = entity["parent"]["name"]
project_name = entity["project"]["full_name"]
self.log.info(
("Ftrack launch app: \"{}\" on Project/Asset/Task: {}/{}/{}"
).format(app_name, project_name, asset_name, task_name))
try:
self.application_manager.launch(app_name,
project_name=project_name,
asset_name=asset_name,
task_name=task_name)
except ApplictionExecutableNotFound as exc:
self.log.warning(exc.exc_msg)
return {"success": False, "message": exc.msg}
except ApplicationLaunchFailed as exc:
self.log.error(str(exc))
return {"success": False, "message": str(exc)}
except Exception:
msg = "Unexpected failure of application launch {}".format(
self.label)
self.log.error(msg, exc_info=True)
return {"success": False, "message": msg}
return {"success": True, "message": "Launching {0}".format(self.label)}