def _via_web_service(self, payload): response = requests.post(self._url, json=payload, auth=tuple(self._auth)) if not response.ok: raise Exception(response.text) else: jobid = eval(response.text)["_id"] self.log.info("Success. JobID: %s" % jobid) return jobid
def _requests_post(self, *args, **kwargs): """ Wrapper for requests, disabling SSL certificate validation if DONT_VERIFY_SSL environment variable is found. This is useful when Deadline or Muster server are running with self-signed certificates and their certificate is not added to trusted certificates on client machines. WARNING: disabling SSL certificate validation is defeating one line of defense SSL is providing and it is not recommended. """ if 'verify' not in kwargs: kwargs['verify'] = False if os.getenv("PYPE_DONT_VERIFY_SSL", True) else True # noqa return requests.post(*args, **kwargs)
def payload_submit(self, instance, script_path, render_path, exe_node_name, responce_data=None): render_dir = os.path.normpath(os.path.dirname(render_path)) script_name = os.path.basename(script_path) jobname = "%s - %s" % (script_name, instance.name) output_filename_0 = self.preview_fname(render_path) if not responce_data: responce_data = {} try: # Ensure render folder exists os.makedirs(render_dir) except OSError: pass # define chunk and priority chunk_size = instance.data.get("deadlineChunkSize") if chunk_size == 0: chunk_size = self.deadline_chunk_size priority = instance.data.get("deadlinePriority") if not priority: priority = self.deadline_priority payload = { "JobInfo": { # Top-level group name "BatchName": script_name, # Asset dependency to wait for at least the scene file to sync. "AssetDependency0": script_path, # Job name, as seen in Monitor "Name": jobname, # Arbitrary username, for visualisation in Monitor "UserName": self._deadline_user, "Priority": priority, "ChunkSize": chunk_size, "Pool": self.deadline_pool, "SecondaryPool": self.deadline_pool_secondary, "Plugin": "Nuke", "Frames": "{start}-{end}".format(start=self._frame_start, end=self._frame_end), "Comment": self._comment, # Optional, enable double-click to preview rendered # frames from Deadline Monitor "OutputFilename0": output_filename_0.replace("\\", "/") }, "PluginInfo": { # Input "SceneFile": script_path, # Output directory and filename "OutputFilePath": render_dir.replace("\\", "/"), # "OutputFilePrefix": render_variables["filename_prefix"], # Mandatory for Deadline "Version": self._ver.group(), # Resolve relative references "ProjectPath": script_path, "AWSAssetFile0": render_path, # Only the specific write node is rendered. "WriteNode": exe_node_name }, # Mandatory for Deadline, may be empty "AuxFiles": [] } if responce_data.get("_id"): payload["JobInfo"].update({ "JobType": "Normal", "BatchName": responce_data["Props"]["Batch"], "JobDependency0": responce_data["_id"], "ChunkSize": 99999999 }) # Include critical environment variables with submission keys = [ "PYTHONPATH", "PATH", "AVALON_SCHEMA", "FTRACK_API_KEY", "FTRACK_API_USER", "FTRACK_SERVER", "PYBLISHPLUGINPATH", "NUKE_PATH", "TOOL_ENV", "PYPE_DEV" ] environment = dict( {key: os.environ[key] for key in keys if key in os.environ}, **api.Session) # self.log.debug("enviro: {}".format(pprint(environment))) for path in os.environ: if path.lower().startswith('pype_'): environment[path] = os.environ[path] # environment["PATH"] = os.environ["PATH"] # self.log.debug("enviro: {}".format(environment['PYPE_SCRIPTS'])) clean_environment = {} for key, value in environment.items(): clean_path = "" self.log.debug("key: {}".format(key)) if "://" in value: clean_path = value else: valid_paths = [] for path in value.split(os.pathsep): if not path: continue try: path.decode('UTF-8', 'strict') valid_paths.append(os.path.normpath(path)) except UnicodeDecodeError: print('path contains non UTF characters') if valid_paths: clean_path = os.pathsep.join(valid_paths) if key == "PYTHONPATH": clean_path = clean_path.replace('python2', 'python3') self.log.debug("clean path: {}".format(clean_path)) clean_environment[key] = clean_path environment = clean_environment payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format(key=key, value=environment[key]) for index, key in enumerate(environment) }) plugin = payload["JobInfo"]["Plugin"] self.log.info("using render plugin : {}".format(plugin)) self.log.info("Submitting..") self.log.info(json.dumps(payload, indent=4, sort_keys=True)) # adding expectied files to instance.data self.expected_files(instance, render_path) self.log.debug("__ expectedFiles: `{}`".format( instance.data["expectedFiles"])) response = requests.post(self.deadline_url, json=payload, timeout=10) if not response.ok: raise Exception(response.text) return response
def process(self, instance): DEADLINE_REST_URL = api.Session.get("DEADLINE_REST_URL", "http://localhost:8082") assert DEADLINE_REST_URL, "Requires DEADLINE_REST_URL" context = instance.context deadline_url = "{}/api/jobs".format(DEADLINE_REST_URL) deadline_user = context.data.get("deadlineUser", getpass.getuser()) filepath = context.data["currentFile"] filename = os.path.basename(filepath) task_name = "{} - {}".format(filename, instance.name) batch_name = "{} - (vrscene)".format(filename) # Get the output template for vrscenes vrscene_output = instance.data["vrsceneOutput"] # This is also the input file for the render job first_file = self.format_output_filename(instance, filename, vrscene_output) start_frame = int(instance.data["startFrame"]) end_frame = int(instance.data["endFrame"]) # Primary job self.log.info("Submitting export job ..") payload = { "JobInfo": { # Top-level group name "BatchName": batch_name, # Job name, as seen in Monitor "Name": "Export {} [{}-{}]".format(task_name, start_frame, end_frame), # Arbitrary username, for visualisation in Monitor "UserName": deadline_user, "Plugin": "MayaBatch", "Frames": "{}-{}".format(start_frame, end_frame), "FramesPerTask": instance.data.get("framesPerTask", 1), "Comment": context.data.get("comment", ""), "OutputFilename0": os.path.dirname(first_file), }, "PluginInfo": { # Renderer "Renderer": "vray", # Mandatory for Deadline "Version": cmds.about(version=True), # Input "SceneFile": filepath, "SkipExistingFrames": True, "UsingRenderLayers": True, "UseLegacyRenderLayers": True }, # Mandatory for Deadline, may be empty "AuxFiles": [] } environment = dict(AVALON_TOOLS="global;python36;maya2018") environment.update(api.Session.copy()) jobinfo_environment = self.build_jobinfo_environment(environment) payload["JobInfo"].update(jobinfo_environment) self.log.info("Job Data:\n{}".format(json.dumps(payload))) response = requests.post(url=deadline_url, json=payload) if not response.ok: raise RuntimeError(response.text) # Secondary job # Store job to create dependency chain dependency = response.json() if instance.data["suspendRenderJob"]: self.log.info("Skipping render job and publish job") return self.log.info("Submitting render job ..") start_frame = int(instance.data["startFrame"]) end_frame = int(instance.data["endFrame"]) ext = instance.data.get("ext", "exr") # Create output directory for renders render_ouput = self.format_output_filename(instance, filename, instance.data["outputDir"], dir=True) self.log.info("Render output: %s" % render_ouput) # Update output dir instance.data["outputDir"] = render_ouput # Format output file name sequence_filename = ".".join([instance.name, ext]) output_filename = os.path.join(render_ouput, sequence_filename) # Ensure folder exists: if not os.path.exists(render_ouput): os.makedirs(render_ouput) payload_b = { "JobInfo": { "JobDependency0": dependency["_id"], "BatchName": batch_name, "Name": "Render {} [{}-{}]".format(task_name, start_frame, end_frame), "UserName": deadline_user, "Frames": "{}-{}".format(start_frame, end_frame), "Plugin": "Vray", "OverrideTaskExtraInfoNames": False, "OutputFilename0": render_ouput, }, "PluginInfo": { "InputFilename": first_file, "OutputFilename": output_filename, "SeparateFilesPerFrame": True, "VRayEngine": "V-Ray", "Width": instance.data["resolution"][0], "Height": instance.data["resolution"][1], }, "AuxFiles": [], } # Add vray renderslave to environment tools = environment["AVALON_TOOLS"] + ";vrayrenderslave" environment_b = deepcopy(environment) environment_b["AVALON_TOOLS"] = tools jobinfo_environment_b = self.build_jobinfo_environment(environment_b) payload_b["JobInfo"].update(jobinfo_environment_b) self.log.info(json.dumps(payload_b)) # Post job to deadline response_b = requests.post(url=deadline_url, json=payload_b) if not response_b.ok: raise RuntimeError(response_b.text) # Add job for publish job if not instance.data.get("suspendPublishJob", False): instance.data["deadlineSubmissionJob"] = response_b.json()
def _submit_deadline_post_job(self, instance, job): """Submit publish job to Deadline. Deadline specific code separated from :meth:`process` for sake of more universal code. Muster post job is sent directly by Muster submitter, so this type of code isn't necessary for it. """ data = instance.data.copy() subset = data["subset"] job_name = "{batch} - {subset} [publish image sequence]".format( batch=job["Props"]["Name"], subset=subset) output_dir = instance.data["outputDir"] # Convert output dir to `{root}/rest/of/path/...` with Anatomy success, rootless_path = ( self.anatomy.find_root_template_from_path(output_dir)) if not success: # `rootless_path` is not set to `output_dir` if none of roots match self.log.warning( ("Could not find root path for remapping \"{}\"." " This may cause issues on farm.").format(output_dir)) rootless_path = output_dir # Generate the payload for Deadline submission payload = { "JobInfo": { "Plugin": "Python", "BatchName": job["Props"]["Batch"], "Name": job_name, "JobDependency0": job["_id"], "UserName": job["Props"]["User"], "Comment": instance.context.data.get("comment", ""), "Priority": job["Props"]["Pri"], "Pool": self.deadline_pool, "OutputDirectory0": output_dir }, "PluginInfo": { "Version": "3.6", "ScriptFile": _get_script(), "Arguments": "", "SingleFrameOnly": "True", }, # Mandatory for Deadline, may be empty "AuxFiles": [], } # Transfer the environment from the original job to this dependent # job so they use the same environment metadata_filename = "{}_metadata.json".format(subset) metadata_path = os.path.join(rootless_path, metadata_filename) environment = job["Props"].get("Env", {}) environment["PYPE_METADATA_FILE"] = metadata_path environment["AVALON_PROJECT"] = io.Session["AVALON_PROJECT"] environment["PYPE_LOG_NO_COLORS"] = "1" try: environment["PYPE_PYTHON_EXE"] = os.environ["PYPE_PYTHON_EXE"] except KeyError: # PYPE_PYTHON_EXE not set pass i = 0 for index, key in enumerate(environment): if key.upper() in self.enviro_filter: payload["JobInfo"].update({ "EnvironmentKeyValue%d" % i: "{key}={value}".format(key=key, value=environment[key]) }) i += 1 # remove secondary pool payload["JobInfo"].pop("SecondaryPool", None) self.log.info("Submitting Deadline job ...") # self.log.info(json.dumps(payload, indent=4, sort_keys=True)) url = "{}/api/jobs".format(self.DEADLINE_REST_URL) response = requests.post(url, json=payload, timeout=10) if not response.ok: raise Exception(response.text)
def _submit_deadline_post_job(self, instance, job): """ Deadline specific code separated from :meth:`process` for sake of more universal code. Muster post job is sent directly by Muster submitter, so this type of code isn't necessary for it. """ data = instance.data.copy() subset = data["subset"] job_name = "{batch} - {subset} [publish image sequence]".format( batch=job["Props"]["Name"], subset=subset ) metadata_filename = "{}_metadata.json".format(subset) output_dir = instance.data["outputDir"] metadata_path = os.path.join(output_dir, metadata_filename) metadata_path = os.path.normpath(metadata_path) mount_root = os.path.normpath(os.environ["PYPE_STUDIO_PROJECTS_MOUNT"]) network_root = os.environ["PYPE_STUDIO_PROJECTS_PATH"] metadata_path = metadata_path.replace(mount_root, network_root) metadata_path = metadata_path.replace(network_root, "{root}") # Generate the payload for Deadline submission payload = { "JobInfo": { "Plugin": "Python", "BatchName": job["Props"]["Batch"], "Name": job_name, "JobDependency0": job["_id"], "UserName": job["Props"]["User"], "Comment": instance.context.data.get("comment", ""), "Priority": job["Props"]["Pri"], "Pool": self.deadline_pool, "OutputDirectory0": output_dir }, "PluginInfo": { "Version": "3.6", "ScriptFile": _get_script(), "Arguments": "", "SingleFrameOnly": "True", }, # Mandatory for Deadline, may be empty "AuxFiles": [], } # Transfer the environment from the original job to this dependent # job so they use the same environment environment = job["Props"].get("Env", {}) environment["PYPE_METADATA_FILE"] = metadata_path environment["AVALON_PROJECT"] = io.Session["AVALON_PROJECT"] i = 0 for index, key in enumerate(environment): if key.upper() in self.enviro_filter: payload["JobInfo"].update( { "EnvironmentKeyValue%d" % i: "{key}={value}".format( key=key, value=environment[key] ) } ) i += 1 # remove secondary pool payload["JobInfo"].pop("SecondaryPool", None) self.log.info("Submitting Deadline job ...") # self.log.info(json.dumps(payload, indent=4, sort_keys=True)) url = "{}/api/jobs".format(self.DEADLINE_REST_URL) response = requests.post(url, json=payload) if not response.ok: raise Exception(response.text)
def _submit_deadline_post_job(self, instance, job, instances): """Submit publish job to Deadline. Deadline specific code separated from :meth:`process` for sake of more universal code. Muster post job is sent directly by Muster submitter, so this type of code isn't necessary for it. """ data = instance.data.copy() subset = data["subset"] job_name = "Publish - {subset}".format(subset=subset) output_dir = instance.data["outputDir"] # Generate the payload for Deadline submission payload = { "JobInfo": { "Plugin": "Python", "BatchName": job["Props"]["Batch"], "Name": job_name, "UserName": job["Props"]["User"], "Comment": instance.context.data.get("comment", ""), "Department": self.deadline_department, "ChunkSize": self.deadline_chunk_size, "Priority": job["Props"]["Pri"], "Group": self.deadline_group, "Pool": self.deadline_pool, "SecondaryPool": self.deadline_pool_secondary, "OutputDirectory0": output_dir }, "PluginInfo": { "Version": self.plugin_python_version, "ScriptFile": _get_script(self.publishing_script), "Arguments": "", "SingleFrameOnly": "True", }, # Mandatory for Deadline, may be empty "AuxFiles": [], } # add assembly jobs as dependencies if instance.data.get("tileRendering"): self.log.info("Adding tile assembly jobs as dependencies...") job_index = 0 for assembly_id in instance.data.get("assemblySubmissionJobs"): payload["JobInfo"]["JobDependency{}".format(job_index)] = assembly_id # noqa: E501 job_index += 1 else: payload["JobInfo"]["JobDependency0"] = job["_id"] # Transfer the environment from the original job to this dependent # job so they use the same environment metadata_path, roothless_metadata_path = self._create_metadata_path( instance) environment = job["Props"].get("Env", {}) environment["PYPE_METADATA_FILE"] = roothless_metadata_path environment["AVALON_PROJECT"] = io.Session["AVALON_PROJECT"] environment["PYPE_LOG_NO_COLORS"] = "1" try: environment["PYPE_PYTHON_EXE"] = os.environ["PYPE_PYTHON_EXE"] except KeyError: # PYPE_PYTHON_EXE not set pass i = 0 for index, key in enumerate(environment): if key.upper() in self.enviro_filter: payload["JobInfo"].update( { "EnvironmentKeyValue%d" % i: "{key}={value}".format( key=key, value=environment[key] ) } ) i += 1 # remove secondary pool payload["JobInfo"].pop("SecondaryPool", None) self.log.info("Submitting Deadline job ...") # self.log.info(json.dumps(payload, indent=4, sort_keys=True)) url = "{}/api/jobs".format(self.DEADLINE_REST_URL) response = requests.post(url, json=payload, timeout=10) if not response.ok: raise Exception(response.text)
def process(self, instance): context = instance.context key = "__hasRun{}".format(self.__class__.__name__) if context.data.get(key, False): return else: context.data[key] = True from avalon.fusion.lib import get_frame_path AVALON_DEADLINE = api.Session.get("AVALON_DEADLINE", "http://localhost:8082") assert AVALON_DEADLINE, "Requires AVALON_DEADLINE" # Collect all saver instances in context that are to be rendered saver_instances = [] for instance in context[:]: if not self.families[0] in instance.data.get("families"): # Allow only saver family instances continue if not instance.data.get("publish", True): # Skip inactive instances continue self.log.debug(instance.data["name"]) saver_instances.append(instance) if not saver_instances: raise RuntimeError("No instances found for Deadline submittion") fusion_version = int(context.data["fusionVersion"]) filepath = context.data["currentFile"] filename = os.path.basename(filepath) comment = context.data.get("comment", "") deadline_user = context.data.get("deadlineUser", getpass.getuser()) # Documentation for keys available at: # https://docs.thinkboxsoftware.com # /products/deadline/8.0/1_User%20Manual/manual # /manual-submission.html#job-info-file-options payload = { "JobInfo": { # Top-level group name "BatchName": filename, # Job name, as seen in Monitor "Name": filename, # User, as seen in Monitor "UserName": deadline_user, # Use a default submission pool for Fusion "Pool": "fusion", "Plugin": "Fusion", "Frames": "{start}-{end}".format(start=int(context.data["startFrame"]), end=int(context.data["endFrame"])), "Comment": comment, }, "PluginInfo": { # Input "FlowFile": filepath, # Mandatory for Deadline "Version": str(fusion_version), # Render in high quality "HighQuality": True, # Whether saver output should be checked after rendering # is complete "CheckOutput": True, # Proxy: higher numbers smaller images for faster test renders # 1 = no proxy quality "Proxy": 1, }, # Mandatory for Deadline, may be empty "AuxFiles": [] } # Enable going to rendered frames from Deadline Monitor for index, instance in enumerate(saver_instances): head, padding, tail = get_frame_path(instance.data["path"]) path = "{}{}{}".format(head, "#" * padding, tail) folder, filename = os.path.split(path) payload["JobInfo"]["OutputDirectory%d" % index] = folder payload["JobInfo"]["OutputFilename%d" % index] = filename # Include critical variables with submission keys = [ # TODO: This won't work if the slaves don't have accesss to # these paths, such as if slaves are running Linux and the # submitter is on Windows. "PYTHONPATH", "OFX_PLUGIN_PATH", "FUSION9_MasterPrefs" ] environment = dict( {key: os.environ[key] for key in keys if key in os.environ}, **api.Session) payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format(key=key, value=environment[key]) for index, key in enumerate(environment) }) self.log.info("Submitting..") self.log.info(json.dumps(payload, indent=4, sort_keys=True)) # E.g. http://192.168.0.1:8082/api/jobs url = "{}/api/jobs".format(AVALON_DEADLINE) response = requests.post(url, json=payload) if not response.ok: raise Exception(response.text) # Store the response for dependent job submission plug-ins for instance in saver_instances: instance.data["deadlineSubmissionJob"] = response.json()
def process(self, instance): DEADLINE_REST_URL = os.environ.get("DEADLINE_REST_URL", "http://localhost:8082") assert DEADLINE_REST_URL, "Requires DEADLINE_REST_URL" context = instance.context workspace = context.data["workspaceDir"] filepath = None allInstances = [] for result in context.data["results"]: if (result["instance"] is not None and result["instance"] not in allInstances): allInstances.append(result["instance"]) for inst in allInstances: print(inst) if inst.data['family'] == 'scene': filepath = inst.data['destination_list'][0] if not filepath: filepath = context.data["currentFile"] self.log.debug(filepath) filename = os.path.basename(filepath) comment = context.data.get("comment", "") scene = os.path.splitext(filename)[0] dirname = os.path.join(workspace, "renders") renderlayer = instance.data['setMembers'] # rs_beauty renderlayer_name = instance.data['subset'] # beauty renderlayer_globals = instance.data["renderGlobals"] legacy_layers = renderlayer_globals["UseLegacyRenderLayers"] deadline_user = context.data.get("deadlineUser", getpass.getuser()) jobname = "%s - %s" % (filename, instance.name) # Get the variables depending on the renderer render_variables = get_renderer_variables(renderlayer) output_filename_0 = preview_fname(folder=dirname, scene=scene, layer=renderlayer_name, padding=render_variables["padding"], ext=render_variables["ext"]) try: # Ensure render folder exists os.makedirs(dirname) except OSError: pass # Documentation for keys available at: # https://docs.thinkboxsoftware.com # /products/deadline/8.0/1_User%20Manual/manual # /manual-submission.html#job-info-file-options payload = { "JobInfo": { # Top-level group name "BatchName": filename, # Job name, as seen in Monitor "Name": jobname, # Arbitrary username, for visualisation in Monitor "UserName": deadline_user, "Plugin": instance.data.get("mayaRenderPlugin", "MayaBatch"), "Frames": "{start}-{end}x{step}".format( start=int(instance.data["startFrame"]), end=int(instance.data["endFrame"]), step=int(instance.data["byFrameStep"]), ), "Comment": comment, # Optional, enable double-click to preview rendered # frames from Deadline Monitor "OutputFilename0": output_filename_0.replace("\\", "/"), }, "PluginInfo": { # Input "SceneFile": filepath, # Output directory and filename "OutputFilePath": dirname.replace("\\", "/"), "OutputFilePrefix": render_variables["filename_prefix"], # Mandatory for Deadline "Version": cmds.about(version=True), # Only render layers are considered renderable in this pipeline "UsingRenderLayers": True, # Use legacy Render Layer system "UseLegacyRenderLayers": legacy_layers, # Render only this layer "RenderLayer": renderlayer, # Determine which renderer to use from the file itself "Renderer": instance.data["renderer"], # Resolve relative references "ProjectPath": workspace, }, # Mandatory for Deadline, may be empty "AuxFiles": [] } # Include critical environment variables with submission keys = [ # This will trigger `userSetup.py` on the slave # such that proper initialisation happens the same # way as it does on a local machine. # TODO(marcus): This won't work if the slaves don't # have accesss to these paths, such as if slaves are # running Linux and the submitter is on Windows. "PYTHONPATH", "PATH", "MTOA_EXTENSIONS_PATH", "MTOA_EXTENSIONS", "DYLD_LIBRARY_PATH", "MAYA_RENDER_DESC_PATH", "MAYA_MODULE_PATH", "ARNOLD_PLUGIN_PATH", "AVALON_SCHEMA", "FTRACK_API_KEY", "FTRACK_API_USER", "FTRACK_SERVER", "PYBLISHPLUGINPATH", # todo: This is a temporary fix for yeti variables "PEREGRINEL_LICENSE", "SOLIDANGLE_LICENSE", "ARNOLD_LICENSE" "MAYA_MODULE_PATH", "TOOL_ENV" ] environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **api.Session) #self.log.debug("enviro: {}".format(pprint(environment))) for path in os.environ: if path.lower().startswith('pype_'): environment[path] = os.environ[path] environment["PATH"] = os.environ["PATH"] self.log.debug("enviro: {}".format(environment['PYPE_SCRIPTS'])) clean_environment = {} for key in environment: clean_path = "" self.log.debug("key: {}".format(key)) to_process = environment[key] if key == "PYPE_STUDIO_CORE_MOUNT": clean_path = environment[key] elif "://" in environment[key]: clean_path = environment[key] elif os.pathsep not in to_process: try: path = environment[key] path.decode('UTF-8', 'strict') clean_path = os.path.normpath(path) except UnicodeDecodeError: print('path contains non UTF characters') else: for path in environment[key].split(os.pathsep): try: path.decode('UTF-8', 'strict') clean_path += os.path.normpath(path) + os.pathsep except UnicodeDecodeError: print('path contains non UTF characters') if key == "PYTHONPATH": clean_path = clean_path.replace('python2', 'python3') clean_path = clean_path.replace( os.path.normpath(environment['PYPE_STUDIO_CORE_MOUNT']), os.path.normpath(environment['PYPE_STUDIO_CORE'])) clean_environment[key] = clean_path environment = clean_environment payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format( key=key, value=environment[key] ) for index, key in enumerate(environment) }) # Include optional render globals render_globals = instance.data.get("renderGlobals", {}) payload["JobInfo"].update(render_globals) plugin = payload["JobInfo"]["Plugin"] self.log.info("using render plugin : {}".format(plugin)) self.preflight_check(instance) self.log.info("Submitting..") self.log.info(json.dumps(payload, indent=4, sort_keys=True)) # E.g. http://192.168.0.1:8082/api/jobs url = "{}/api/jobs".format(DEADLINE_REST_URL) response = requests.post(url, json=payload) if not response.ok: raise Exception(response.text) # Store output dir for unified publisher (filesequence) instance.data["outputDir"] = os.path.dirname(output_filename_0) instance.data["deadlineSubmissionJob"] = response.json()
def _submit_deadline_post_job(self, instance, job, instances): """Submit publish job to Deadline. Deadline specific code separated from :meth:`process` for sake of more universal code. Muster post job is sent directly by Muster submitter, so this type of code isn't necessary for it. """ data = instance.data.copy() subset = data["subset"] job_name = "Publish - {subset}".format(subset=subset) # instance.data.get("subset") != instances[0]["subset"] # 'Main' vs 'renderMain' override_version = None instance_version = instance.data.get("version") # take this if exists if instance_version != 1: override_version = instance_version output_dir = self._get_publish_folder( instance.context.data['anatomy'], deepcopy(instance.data["anatomyData"]), instance.data.get("asset"), instances[0]["subset"], 'render', override_version) # Transfer the environment from the original job to this dependent # job so they use the same environment metadata_path, roothless_metadata_path = \ self._create_metadata_path(instance) environment = job["Props"].get("Env", {}) environment["AVALON_PROJECT"] = io.Session["AVALON_PROJECT"] environment["AVALON_ASSET"] = io.Session["AVALON_ASSET"] environment["AVALON_TASK"] = io.Session["AVALON_TASK"] environment["AVALON_APP_NAME"] = os.environ.get("AVALON_APP_NAME") environment["PYPE_LOG_NO_COLORS"] = "1" environment["PYPE_USERNAME"] = instance.context.data["user"] environment["PYPE_PUBLISH_JOB"] = "1" environment["PYPE_RENDER_JOB"] = "0" args = ['publish', roothless_metadata_path] # Generate the payload for Deadline submission payload = { "JobInfo": { "Plugin": self.deadline_plugin, "BatchName": job["Props"]["Batch"], "Name": job_name, "UserName": job["Props"]["User"], "Comment": instance.context.data.get("comment", ""), "Department": self.deadline_department, "ChunkSize": self.deadline_chunk_size, "Priority": job["Props"]["Pri"], "Group": self.deadline_group, "Pool": self.deadline_pool, "SecondaryPool": self.deadline_pool_secondary, "OutputDirectory0": output_dir }, "PluginInfo": { "Version": self.plugin_pype_version, "Arguments": " ".join(args), "SingleFrameOnly": "True", }, # Mandatory for Deadline, may be empty "AuxFiles": [], } # add assembly jobs as dependencies if instance.data.get("tileRendering"): self.log.info("Adding tile assembly jobs as dependencies...") job_index = 0 for assembly_id in instance.data.get("assemblySubmissionJobs"): payload["JobInfo"]["JobDependency{}".format( job_index)] = assembly_id # noqa: E501 job_index += 1 else: payload["JobInfo"]["JobDependency0"] = job["_id"] i = 0 for index, key in enumerate(environment): if key.upper() in self.enviro_filter: payload["JobInfo"].update({ "EnvironmentKeyValue%d" % i: "{key}={value}".format(key=key, value=environment[key]) }) i += 1 # remove secondary pool payload["JobInfo"].pop("SecondaryPool", None) self.log.info("Submitting Deadline job ...") url = "{}/api/jobs".format(self.deadline_url) response = requests.post(url, json=payload, timeout=10) if not response.ok: raise Exception(response.text)
def fulfill(self, context, instances): assert "AVALON_DEADLINE" in avalon.api.Session, ( "Environment variable missing: 'AVALON_DEADLINE'") AVALON_DEADLINE = avalon.api.Session["AVALON_DEADLINE"] workspace = context.data["workspaceDir"] fpath = context.data["currentMaking"] fname = os.path.basename(fpath) name, ext = os.path.splitext(fname) comment = context.data.get("comment", "") project = context.data["projectDoc"] project_id = str(project["_id"])[-4:].upper() project_code = project["data"].get("codename", project_id) asset = context.data["assetDoc"]["name"] batch_name = "({projcode}): [{asset}] {filename}" batch_name = batch_name.format(projcode=project_code, asset=asset, filename=fname) script_file = os.path.join(os.path.dirname(__file__), "scripts", "avalon_contractor_publish.py") # E.g. http://192.168.0.1:8082/api/jobs url = "{}/api/jobs".format(AVALON_DEADLINE) # # Documentation about RESTful api # https://docs.thinkboxsoftware.com/products/deadline/ # 10.0/1_User%20Manual/manual/rest-jobs.html#rest-jobs-ref-label # # Documentation for keys available at: # https://docs.thinkboxsoftware.com # /products/deadline/8.0/1_User%20Manual/manual # /manual-submission.html#job-info-file-options auth = os.environ["AVALON_DEADLINE_AUTH"].split(":") # Grouping instances instance_group = dict() for instance in instances: dl_pool = instance.data["deadlinePool"] dl_group = instance.data["deadlineGroup"] dl_priority = instance.data["deadlinePriority"] group_key = (dl_pool, dl_group, dl_priority) if group_key not in instance_group: instance_group[group_key] = list() instance_group[group_key].append(instance) for settings, group in instance_group.items(): dl_pool, dl_group, dl_priority = settings if len(group) == 1: instance = group[0] job_name = "{subset} v{version:0>3}".format( subset=instance.data["subset"], version=instance.data["versionNext"], ) else: job_name = "queued %d subsets" % len(group) environment = dict() for instance in group: self.log.info("Adding instance: %s" % instance.data["subset"]) environment.update(self.assemble_environment(instance)) payload = { "JobInfo": { "Plugin": "MayaBatch", "BatchName": batch_name, # Top-level group name "Name": job_name, "UserName": getpass.getuser(), "MachineName": platform.node(), "Comment": comment, "Pool": dl_pool, "Group": dl_group, "Priority": dl_priority, "ExtraInfo0": project["name"], }, "PluginInfo": { # Input "SceneFile": fpath, # Resolve relative references "ProjectPath": workspace, # Mandatory for Deadline "Version": context.data["mayaVersion"], "ScriptJob": True, "ScriptFilename": script_file, }, # Mandatory for Deadline, may be empty "AuxFiles": [], "IdOnly": True } payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format(key=key, value=environment[key]) for index, key in enumerate(environment) }) self.log.info("Submitting..") self.log.info(json.dumps(payload, indent=4, sort_keys=True)) response = requests.post(url, json=payload, auth=tuple(auth)) if response.ok: jobid = eval(response.text)["_id"] self.log.info("Success. JobID: %s" % jobid) else: msg = response.text self.log.error(msg) raise Exception(msg) self.log.info("Completed.")
def process(self, instance): AVALON_DEADLINE = api.Session.get("AVALON_DEADLINE", "http://localhost:8082") assert AVALON_DEADLINE, "Requires AVALON_DEADLINE" # Get a submission job job = instance.data.get("deadlineSubmissionJob") if not job: raise RuntimeError("Can't continue without valid deadline " "submission prior to this plug-in.") data = instance.data.copy() subset = data["subset"] state = data.get("publishJobState", "Suspended") job_name = "{batch} - {subset} [publish image sequence]".format( batch=job["Props"]["Name"], subset=subset) # Add in start/end frame context = instance.context start = instance.data.get("startFrame", context.data["startFrame"]) end = instance.data.get("endFrame", context.data["endFrame"]) resources = [] # Add in regex for sequence filename # This assumes the output files start with subset name and ends with # a file extension. if "ext" in instance.data: ext = re.escape(instance.data["ext"]) else: ext = "\.\D+" regex = "^{subset}.*\d+{ext}$".format(subset=re.escape(subset), ext=ext) # Write metadata for publish job render_job = data.pop("deadlineSubmissionJob") metadata = { "regex": regex, "startFrame": start, "endFrame": end, "families": ["colorbleed.imagesequence"], # Optional metadata (for debugging) "metadata": { "instance": data, "job": job, "session": api.Session.copy() } } # Ensure output dir exists output_dir = instance.data["outputDir"] if not os.path.isdir(output_dir): os.makedirs(output_dir) if data.get("extendFrames", False): family = "colorbleed.imagesequence" override = data["overrideExistingFrame"] # override = data.get("overrideExistingFrame", False) out_file = render_job.get("OutFile") if not out_file: raise RuntimeError("OutFile not found in render job!") extension = os.path.splitext(out_file[0])[1] _ext = extension[1:] # Frame comparison prev_start = None prev_end = None resource_range = range(int(start), int(end) + 1) # Gather all the subset files (one subset per render pass!) subset_names = [data["subset"]] subset_names.extend(data.get("renderPasses", [])) for subset_name in subset_names: version = get_latest_version(asset_name=data["asset"], subset_name=subset_name, family=family) # Set prev start / end frames for comparison if not prev_start and not prev_end: prev_start = version["data"]["startFrame"] prev_end = version["data"]["endFrame"] subset_resources = get_resources(version, _ext) resource_files = get_resource_files(subset_resources, resource_range, override) resources.extend(resource_files) updated_start = min(start, prev_start) updated_end = max(end, prev_end) # Update metadata and instance start / end frame self.log.info("Updating start / end frame : " "{} - {}".format(updated_start, updated_end)) # TODO : Improve logic to get new frame range for the # publish job (publish_filesequence.py) # The current approach is not following Pyblish logic which is based # on Collect / Validate / Extract. # ---- Collect Plugins --- # Collect Extend Frames - Only run if extendFrames is toggled # # # Store in instance: # # # Previous rendered files per subset based on frames # # # --> Add to instance.data[resources] # # # Update publish frame range # ---- Validate Plugins --- # Validate Extend Frames # # # Check if instance has the requirements to extend frames # There might have been some things which can be added to the list # Please do so when fixing this. # Start frame metadata["startFrame"] = updated_start metadata["metadata"]["instance"]["startFrame"] = updated_start # End frame metadata["endFrame"] = updated_end metadata["metadata"]["instance"]["endFrame"] = updated_end metadata_filename = "{}_metadata.json".format(subset) metadata_path = os.path.join(output_dir, metadata_filename) with open(metadata_path, "w") as f: json.dump(metadata, f, indent=4, sort_keys=True) # Generate the payload for Deadline submission payload = { "JobInfo": { "Plugin": "Python", "BatchName": job["Props"]["Batch"], "Name": job_name, "JobType": "Normal", "JobDependency0": job["_id"], "UserName": job["Props"]["User"], "Comment": instance.context.data.get("comment", ""), "InitialStatus": state }, "PluginInfo": { "Version": "3.6", "ScriptFile": _get_script(), "Arguments": '--path "{}"'.format(metadata_path), "SingleFrameOnly": "True" }, # Mandatory for Deadline, may be empty "AuxFiles": [] } # Transfer the environment from the original job to this dependent # job so they use the same environment environment = job["Props"].get("Env", {}) payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format(key=key, value=environment[key]) for index, key in enumerate(environment) }) self.log.info("Submitting..") self.log.info(json.dumps(payload, indent=4, sort_keys=True)) url = "{}/api/jobs".format(AVALON_DEADLINE) response = requests.post(url, json=payload) if not response.ok: raise Exception(response.text) # Copy files from previous render if extendFrame is True if data.get("extendFrames", False): self.log.info("Preparing to copy ..") import shutil dest_path = data["outputDir"] for source in resources: src_file = os.path.basename(source) dest = os.path.join(dest_path, src_file) shutil.copy(source, dest) self.log.info("Finished copying %i files" % len(resources))
def process(self, instance): AVALON_DEADLINE = api.Session.get("AVALON_DEADLINE", "http://localhost:8082") assert AVALON_DEADLINE, "Requires AVALON_DEADLINE" context = instance.context workspace = context.data["workspaceDir"] filepath = context.data["currentFile"] filename = os.path.basename(filepath) comment = context.data.get("comment", "") scene = os.path.splitext(filename)[0] dirname = os.path.join(workspace, "renders") renderlayer = instance.data['setMembers'] # rs_beauty renderlayer_name = instance.name # beauty renderlayer_globals = instance.data["renderGlobals"] legacy_layers = renderlayer_globals["UseLegacyRenderLayers"] deadline_user = context.data.get("deadlineUser", getpass.getuser()) jobname = "%s - %s" % (filename, instance.name) # Get the variables depending on the renderer render_variables = get_renderer_variables(renderlayer) output_filename_0 = preview_fname(folder=dirname, scene=scene, layer=renderlayer_name, padding=render_variables["padding"], ext=render_variables["ext"]) try: # Ensure render folder exists os.makedirs(dirname) except OSError: pass # Documentation for keys available at: # https://docs.thinkboxsoftware.com # /products/deadline/8.0/1_User%20Manual/manual # /manual-submission.html#job-info-file-options payload = { "JobInfo": { # Top-level group name "BatchName": filename, # Job name, as seen in Monitor "Name": jobname, # Arbitrary username, for visualisation in Monitor "UserName": deadline_user, "Plugin": instance.data.get("mayaRenderPlugin", "MayaBatch"), "Frames": "{start}-{end}x{step}".format( start=int(instance.data["startFrame"]), end=int(instance.data["endFrame"]), step=int(instance.data["byFrameStep"]), ), "Comment": comment, # Optional, enable double-click to preview rendered # frames from Deadline Monitor "OutputFilename0": output_filename_0.replace("\\", "/"), }, "PluginInfo": { # Input "SceneFile": filepath, # Output directory and filename "OutputFilePath": dirname.replace("\\", "/"), "OutputFilePrefix": render_variables["filename_prefix"], # Mandatory for Deadline "Version": cmds.about(version=True), # Only render layers are considered renderable in this pipeline "UsingRenderLayers": True, # Use legacy Render Layer system "UseLegacyRenderLayers": legacy_layers, # Render only this layer "RenderLayer": renderlayer, # Determine which renderer to use from the file itself "Renderer": instance.data["renderer"], # Resolve relative references "ProjectPath": workspace, }, # Mandatory for Deadline, may be empty "AuxFiles": [] } # Include critical environment variables with submission keys = [ # This will trigger `userSetup.py` on the slave # such that proper initialisation happens the same # way as it does on a local machine. # TODO(marcus): This won't work if the slaves don't # have accesss to these paths, such as if slaves are # running Linux and the submitter is on Windows. "PYTHONPATH", # todo: This is a temporary fix for yeti variables "PEREGRINEL_LICENSE", "REDSHIFT_MAYAEXTENSIONSPATH", "VRAY_FOR_MAYA2018_PLUGINS_X64", "VRAY_PLUGINS_X64", "VRAY_USE_THREAD_AFFINITY", "MAYA_MODULE_PATH" ] environment = dict( {key: os.environ[key] for key in keys if key in os.environ}, **api.Session) PATHS = os.environ["PATH"].split(";") environment["PATH"] = ";".join( [p for p in PATHS if p.startswith("P:")]) payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format(key=key, value=environment[key]) for index, key in enumerate(environment) }) # Include optional render globals render_globals = instance.data.get("renderGlobals", {}) payload["JobInfo"].update(render_globals) self.log.info("using render plugin : {}".format( payload["JobInfo"]["Plugin"])) self.preflight_check(instance) self.log.info("Submitting..") self.log.info(json.dumps(payload, indent=4, sort_keys=True)) # E.g. http://192.168.0.1:8082/api/jobs url = "{}/api/jobs".format(AVALON_DEADLINE) response = requests.post(url, json=payload) if not response.ok: raise Exception(response.text) # Store output dir for unified publisher (filesequence) instance.data["outputDir"] = os.path.dirname(output_filename_0) instance.data["deadlineSubmissionJob"] = response.json()
def _submit_deadline_post_job(self, instance, job): """ Deadline specific code separated from :meth:`process` for sake of more universal code. Muster post job is sent directly by Muster submitter, so this type of code isn't necessary for it. """ data = instance.data.copy() subset = data["subset"] state = data.get("publishJobState", "Suspended") job_name = "{batch} - {subset} [publish image sequence]".format( batch=job["Props"]["Name"], subset=subset) metadata_filename = "{}_metadata.json".format(subset) output_dir = instance.data["outputDir"] metadata_path = os.path.join(output_dir, metadata_filename) # Generate the payload for Deadline submission payload = { "JobInfo": { "Plugin": "Python", "BatchName": job["Props"]["Batch"], "Name": job_name, "JobType": "Normal", "JobDependency0": job["_id"], "UserName": job["Props"]["User"], "Comment": instance.context.data.get("comment", ""), "InitialStatus": state, "Priority": job["Props"]["Pri"] }, "PluginInfo": { "Version": "3.6", "ScriptFile": _get_script(), "Arguments": '--paths "{}"'.format(metadata_path), "SingleFrameOnly": "True" }, # Mandatory for Deadline, may be empty "AuxFiles": [] } # Transfer the environment from the original job to this dependent # job so they use the same environment environment = job["Props"].get("Env", {}) i = 0 for index, key in enumerate(environment): self.log.info("KEY: {}".format(key)) self.log.info("FILTER: {}".format(self.enviro_filter)) if key.upper() in self.enviro_filter: payload["JobInfo"].update({ "EnvironmentKeyValue%d" % i: "{key}={value}".format(key=key, value=environment[key]) }) i += 1 # Avoid copied pools and remove secondary pool payload["JobInfo"]["Pool"] = "none" payload["JobInfo"].pop("SecondaryPool", None) self.log.info("Submitting..") self.log.info(json.dumps(payload, indent=4, sort_keys=True)) url = "{}/api/jobs".format(self.DEADLINE_REST_URL) response = requests.post(url, json=payload) if not response.ok: raise Exception(response.text)
def process(self, instance): import os import json import shutil import getpass from maya import cmds from avalon import api from avalon.vendor import requests assert "AVALON_DEADLINE" in api.Session, ( "Environment variable missing: 'AVALON_DEADLINE") AVALON_DEADLINE = api.Session["AVALON_DEADLINE"] context = instance.context workspace = context.data["workspaceDir"] fpath = context.data["currentFile"] fname = os.path.basename(fpath) name, ext = os.path.splitext(fname) comment = context.data.get("comment", "") dirname = os.path.join(workspace, "renders", name) try: os.makedirs(dirname) except OSError: pass # E.g. http://192.168.0.1:8082/api/jobs url = "{}/api/jobs?JobID=%s".format(AVALON_DEADLINE) # Documentation for keys available at: # https://docs.thinkboxsoftware.com # /products/deadline/8.0/1_User%20Manual/manual # /manual-submission.html#job-info-file-options payload = { "JobInfo": { # Top-level group name "BatchName": fname, # Job name, as seen in Monitor "Name": "%s - %s" % (fname, instance.name), # Arbitrary username, for visualisation in Monitor "UserName": getpass.getuser(), "Plugin": "MayaBatch", "Frames": "{start}-{end}x{step}".format( start=int(instance.data["startFrame"]), end=int(instance.data["endFrame"]), step=int(instance.data["byFrameStep"]), ), "Comment": comment, # Optional, enable double-click to preview rendered # frames from Deadline Monitor "OutputFilename0": self.preview_fname(instance), }, "PluginInfo": { # Input "SceneFile": fpath, # Output directory and filename "OutputFilePath": dirname, "OutputFilePrefix": "<RenderLayer>/<RenderLayer>", # Mandatory for Deadline "Version": cmds.about(version=True), # Only render layers are considered renderable in this pipeline "UsingRenderLayers": True, # Render only this layer "RenderLayer": instance.name, # Determine which renderer to use from the file itself "Renderer": "file", # Resolve relative references "ProjectPath": workspace, }, # Mandatory for Deadline, may be empty "AuxFiles": [] } # Include critical variables with submission environment = dict( { # This will trigger `userSetup.py` on the slave # such that proper initialisation happens the same # way as it does on a local machine. # TODO(marcus): This won't work if the slaves don't # have accesss to these paths, such as if slaves are # running Linux and the submitter is on Windows. "PYTHONPATH": os.getenv("PYTHONPATH", ""), }, **api.Session) payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format(key=key, value=environment[key]) for index, key in enumerate(environment) }) # Include optional render globals payload["JobInfo"].update(instance.data.get("renderGlobals", {})) self.preflight_check(instance) self.log.info("Submitting..") self.log.info(json.dumps(payload, indent=4, sort_keys=True)) response = requests.post(url, json=payload) if response.ok: # Write metadata for publish fname = os.path.join(dirname, instance.name + ".json") data = { "submission": payload, "session": api.Session, "instance": instance.data, "jobs": [response.json()], } with open(fname, "w") as f: json.dump(data, f, indent=4, sort_keys=True) else: try: shutil.rmtree(dirname) except OSError: # This is nice-to-have, but not critical to the operation pass raise Exception(response.text)
def payload_submit(self, instance, script_path, render_path ): resolution_width = instance.data["resolutionWidth"] resolution_height = instance.data["resolutionHeight"] render_dir = os.path.normpath(os.path.dirname(render_path)) render_path = os.path.normpath(render_path) script_name = os.path.basename(script_path) jobname = "%s - %s" % (script_name, instance.name) output_filename_0 = self.preview_fname(render_path) try: # Ensure render folder exists os.makedirs(render_dir) except OSError: pass # define chunk and priority chunk_size = instance.context.data.get("chunk") if chunk_size == 0: chunk_size = self.deadline_chunk_size # search for %02d pattern in name, and padding number search_results = re.search(r"(.%0)(\d)(d)[._]", render_path).groups() split_patern = "".join(search_results) padding_number = int(search_results[1]) args = [ f"<QUOTE>{script_path}<QUOTE>", "-a", "-16", "-s <STARTFRAME>", "-e <ENDFRAME>", f"-d <QUOTE>{render_dir}<QUOTE>", f"-x {resolution_width}", f"-y {resolution_height}", f"-r <QUOTE>{render_path.replace(split_patern, '')}<QUOTE>", f"-= AbsoluteFrameNumber=on -= PadDigits={padding_number}", "-= ClearAttachment=on", ] payload = { "JobInfo": { # Job name, as seen in Monitor "Name": jobname, # plugin definition "Plugin": "CelAction", # Top-level group name "BatchName": script_name, # Arbitrary username, for visualisation in Monitor "UserName": self._deadline_user, "Department": self.deadline_department, "Priority": self.deadline_priority, "Group": self.deadline_group, "Pool": self.deadline_pool, "SecondaryPool": self.deadline_pool_secondary, "ChunkSize": chunk_size, "Frames": f"{self._frame_start}-{self._frame_end}", "Comment": self._comment, # Optional, enable double-click to preview rendered # frames from Deadline Monitor "OutputFilename0": output_filename_0.replace("\\", "/"), # # Asset dependency to wait for at least the scene file to sync. # "AssetDependency0": script_path "ScheduledType": "Once", "JobDelay": "00:00:08:00" }, "PluginInfo": { # Input "SceneFile": script_path, # Output directory "OutputFilePath": render_dir.replace("\\", "/"), # Plugin attributes "StartupDirectory": "", "Arguments": " ".join(args), # Resolve relative references "ProjectPath": script_path, "AWSAssetFile0": render_path, }, # Mandatory for Deadline, may be empty "AuxFiles": [] } plugin = payload["JobInfo"]["Plugin"] self.log.info("using render plugin : {}".format(plugin)) i = 0 for key, values in dict(os.environ).items(): if key.upper() in self.enviro_filter: payload["JobInfo"].update( { "EnvironmentKeyValue%d" % i: "{key}={value}".format( key=key, value=values ) } ) i += 1 self.log.info("Submitting..") self.log.info(json.dumps(payload, indent=4, sort_keys=True)) # adding expectied files to instance.data self.expected_files(instance, render_path) self.log.debug("__ expectedFiles: `{}`".format( instance.data["expectedFiles"])) response = requests.post(self.deadline_url, json=payload) if not response.ok: raise Exception(response.text) return response
def process(self, instance): node = None for x in instance: if x.Class() == "Write": node = x if node is None: return DEADLINE_REST_URL = os.environ.get("DEADLINE_REST_URL", "http://localhost:8082") assert DEADLINE_REST_URL, "Requires DEADLINE_REST_URL" context = instance.context workspace = os.path.dirname(context.data["currentFile"]) filepath = None # get path path = nuke.filename(node) output_dir = instance.data['outputDir'] filepath = context.data["currentFile"] self.log.debug(filepath) filename = os.path.basename(filepath) comment = context.data.get("comment", "") dirname = os.path.join(workspace, "renders") deadline_user = context.data.get("deadlineUser", getpass.getuser()) jobname = "%s - %s" % (filename, instance.name) ver = re.search(r"\d+\.\d+", context.data.get("hostVersion")) try: # Ensure render folder exists os.makedirs(dirname) except OSError: pass # Documentation for keys available at: # https://docs.thinkboxsoftware.com # /products/deadline/8.0/1_User%20Manual/manual # /manual-submission.html#job-info-file-options payload = { "JobInfo": { # Top-level group name "BatchName": filename, # Job name, as seen in Monitor "Name": jobname, # Arbitrary username, for visualisation in Monitor "UserName": deadline_user, "Plugin": "Nuke", "Frames": "{start}-{end}".format(start=int(instance.data["frameStart"]), end=int(instance.data["frameEnd"])), "ChunkSize": instance.data["deadlineChunkSize"], "Priority": instance.data["deadlinePriority"], "Comment": comment, # Optional, enable double-click to preview rendered # frames from Deadline Monitor # "OutputFilename0": output_filename_0.replace("\\", "/"), }, "PluginInfo": { # Input "SceneFile": filepath, # Output directory and filename "OutputFilePath": dirname.replace("\\", "/"), # "OutputFilePrefix": render_variables["filename_prefix"], # Mandatory for Deadline "Version": ver.group(), # Resolve relative references "ProjectPath": workspace, # Only the specific write node is rendered. "WriteNode": instance[0].name() }, # Mandatory for Deadline, may be empty "AuxFiles": [] } # Include critical environment variables with submission keys = [ # This will trigger `userSetup.py` on the slave # such that proper initialisation happens the same # way as it does on a local machine. # TODO(marcus): This won't work if the slaves don't # have accesss to these paths, such as if slaves are # running Linux and the submitter is on Windows. "PYTHONPATH", "PATH", "AVALON_SCHEMA", "FTRACK_API_KEY", "FTRACK_API_USER", "FTRACK_SERVER", "PYBLISHPLUGINPATH", "NUKE_PATH", "TOOL_ENV" ] environment = dict( {key: os.environ[key] for key in keys if key in os.environ}, **api.Session) # self.log.debug("enviro: {}".format(pprint(environment))) for path in os.environ: if path.lower().startswith('pype_'): environment[path] = os.environ[path] environment["PATH"] = os.environ["PATH"] # self.log.debug("enviro: {}".format(environment['PYPE_SCRIPTS'])) clean_environment = {} for key in environment: clean_path = "" self.log.debug("key: {}".format(key)) to_process = environment[key] if key == "PYPE_STUDIO_CORE_MOUNT": clean_path = environment[key] elif "://" in environment[key]: clean_path = environment[key] elif os.pathsep not in to_process: try: path = environment[key] path.decode('UTF-8', 'strict') clean_path = os.path.normpath(path) except UnicodeDecodeError: print('path contains non UTF characters') else: for path in environment[key].split(os.pathsep): try: path.decode('UTF-8', 'strict') clean_path += os.path.normpath(path) + os.pathsep except UnicodeDecodeError: print('path contains non UTF characters') if key == "PYTHONPATH": clean_path = clean_path.replace('python2', 'python3') clean_path = clean_path.replace( os.path.normpath( environment['PYPE_STUDIO_CORE_MOUNT']), # noqa os.path.normpath(environment['PYPE_STUDIO_CORE_PATH'])) # noqa clean_environment[key] = clean_path environment = clean_environment payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format(key=key, value=environment[key]) for index, key in enumerate(environment) }) plugin = payload["JobInfo"]["Plugin"] self.log.info("using render plugin : {}".format(plugin)) self.preflight_check(instance) self.log.info("Submitting..") self.log.info(json.dumps(payload, indent=4, sort_keys=True)) # E.g. http://192.168.0.1:8082/api/jobs url = "{}/api/jobs".format(DEADLINE_REST_URL) response = requests.post(url, json=payload) if not response.ok: raise Exception(response.text) # Store output dir for unified publisher (filesequence) instance.data["deadlineSubmissionJob"] = response.json() instance.data["publishJobState"] = "Active"