def job_run(self, job): """Create and queue tasks from job object""" # check job name if job["name"] in DrQueueJob.query_jobnames(): raise ValueError("Job name %s is already used!" % job["name"]) return False # run job only on matching os os_list = self.query_engines_of_os(job["limits"]["os"]) # run job only on matching minram minram_list = self.query_engines_with_minram(job["limits"]["minram"]) # run job only on matching mincores mincores_list = self.query_engines_with_mincores(job["limits"]["mincores"]) # check pool members pool_list = self.query_engines_of_pool(job["limits"]["pool"]) # check limits self.match_all_limits(os_list, minram_list, mincores_list, pool_list) # save job in database job_id = DrQueueJob.store_db(job) # job_id from db is be used as session name self.ip_client.session.session = str(job_id) # set owner of job self.ip_client.session.username = job["owner"] # set number of retries for each task self.lbview.retries = job["retries"] # depend on another job (it's tasks) if ("depend" in job["limits"]) and (job["limits"]["depend"] != None): depend_job = self.query_job_by_name(job["limits"]["depend"]) depend_tasks = self.query_task_list(depend_job["_id"]) task_ids = [] for task in depend_tasks: task_ids.append(task["msg_id"]) self.lbview.after = task_ids # check frame numbers if not (job["startframe"] >= 1): raise ValueError("Invalid value for startframe. Has to be equal or greater than 1.") return False if not (job["endframe"] >= 1): raise ValueError("Invalid value for endframe. Has to be equal or greater than 1.") return False if not (job["endframe"] >= job["startframe"]): raise ValueError("Invalid value for endframe. Has be to equal or greater than startframe.") return False if job["endframe"] > job["startframe"]: if not (job["endframe"] - job["startframe"] >= job["blocksize"]): raise ValueError("Invalid value for blocksize. Has to be equal or lower than endframe-startframe.") return False if job["endframe"] == job["startframe"]: if job["blocksize"] != 1: raise ValueError("Invalid value for blocksize. Has to be equal 1 if endframe equals startframe.") return False task_frames = range(job["startframe"], job["endframe"] + 1, job["blocksize"]) for x in task_frames: # prepare script input env_dict = { "DRQUEUE_FRAME": x, "DRQUEUE_BLOCKSIZE": job["blocksize"], "DRQUEUE_ENDFRAME": job["endframe"], "DRQUEUE_SCENEFILE": job["scenefile"], "DRQUEUE_LOGFILE": job["name"] + "-" + str(x) + "_" + str(x + job["blocksize"] - 1) + ".log", } # optional elements if "renderdir" in job: env_dict["DRQUEUE_RENDERDIR"] = job["renderdir"] if "projectdir" in job: env_dict["DRQUEUE_PROJECTDIR"] = job["projectdir"] if "configdir" in job: env_dict["DRQUEUE_CONFIGDIR"] = job["configdir"] if "imagefile" in job: env_dict["DRQUEUE_IMAGEFILE"] = job["imagefile"] if "precommand" in job: env_dict["DRQUEUE_PRECOMMAND"] = job["precommand"] if "renderer" in job: env_dict["DRQUEUE_RENDERER"] = job["renderer"] if "fileformat" in job: env_dict["DRQUEUE_FILEFORMAT"] = job["fileformat"] if "postcommand" in job: env_dict["DRQUEUE_POSTCOMMAND"] = job["postcommand"] if "viewcommand" in job: env_dict["DRQUEUE_VIEWCOMMAND"] = job["viewcommand"] if "worldfile" in job: env_dict["DRQUEUE_WORLDFILE"] = job["worldfile"] if "terrainfile" in job: env_dict["DRQUEUE_TERRAINFILE"] = job["terrainfile"] if "composition" in job: env_dict["DRQUEUE_COMPOSITION"] = job["composition"] if "camera" in job: env_dict["DRQUEUE_CAMERA"] = job["camera"] if "resx" in job: env_dict["DRQUEUE_RESX"] = job["resx"] if "resy" in job: env_dict["DRQUEUE_RESY"] = job["resy"] if "renderpass" in job: env_dict["DRQUEUE_RENDERPASS"] = job["renderpass"] if "rendertype" in job: env_dict["DRQUEUE_RENDERTYPE"] = job["rendertype"] if "fileextension" in job: env_dict["DRQUEUE_FILEEXTENSION"] = job["fileextension"] if "stepframe" in job: env_dict["DRQUEUE_STEPFRAME"] = job["stepframe"] if "custom_bucket" in job: env_dict["DRQUEUE_CUSTOM_BUCKET"] = job["custom_bucket"] if "bucketsize" in job: env_dict["DRQUEUE_BUCKETSIZE"] = job["bucketsize"] if "custom_lod" in job: env_dict["DRQUEUE_CUSTOM_LOD"] = job["custom_lod"] if "lod" in job: env_dict["DRQUEUE_LOD"] = job["lod"] if "custom_varyaa" in job: env_dict["DRQUEUE_CUSTOM_VARYAA"] = job["custom_varyaa"] if "varyaa" in job: env_dict["DRQUEUE_VARYAA"] = job["varyaa"] if "raytrace" in job: env_dict["DRQUEUE_RAYTRACE"] = job["raytrace"] if "antialias" in job: env_dict["DRQUEUE_ANTIALIAS"] = job["antialias"] if "custom_bdepth" in job: env_dict["DRQUEUE_CUSTOM_BDEPTH"] = job["custom_bdepth"] if "bdepth" in job: env_dict["DRQUEUE_BDEPTH"] = job["bdepth"] if "custom_zdepth" in job: env_dict["DRQUEUE_CUSTOM_ZDEPTH"] = job["custom_zdepth"] if "zdepth" in job: env_dict["DRQUEUE_ZDEPTH"] = job["zdepth"] if "custom_cracks" in job: env_dict["DRQUEUE_CUSTOM_CRACKS"] = job["custom_cracks"] if "cracks" in job: env_dict["DRQUEUE_CRACKS"] = job["cracks"] if "custom_quality" in job: env_dict["DRQUEUE_CUSTOM_QUALITY"] = job["custom_quality"] if "quality" in job: env_dict["DRQUEUE_QUALITY"] = job["quality"] if "custom_qfiner" in job: env_dict["DRQUEUE_CUSTOM_QFINER"] = job["custom_qfiner"] if "qfiner" in job: env_dict["DRQUEUE_QFINER"] = job["qfiner"] if "custom_smultiplier" in job: env_dict["DRQUEUE_CUSTOM_SMULTIPLIER"] = job["custom_smultiplier"] if "smultiplier" in job: env_dict["DRQUEUE_SMULTIPLIER"] = job["smultiplier"] if "custom_mpcache" in job: env_dict["DRQUEUE_CUSTOM_MPCACHE"] = job["custom_mpcache"] if "mpcache" in job: env_dict["DRQUEUE_MPCACHE"] = job["mpcache"] if "custom_smpolygon" in job: env_dict["DRQUEUE_CUSTOM_SMPOLYGON"] = job["custom_smpolygon"] if "smpolygon" in job: env_dict["DRQUEUE_SMPOLYGON"] = job["smpolygon"] if "custom_wh" in job: env_dict["DRQUEUE_CUSTOM_WH"] = job["custom_wh"] if "custom_type" in job: env_dict["DRQUEUE_CUSTOM_TYPE"] = job["custom_type"] if "ctype" in job: env_dict["DRQUEUE_CTYPE"] = job["ctype"] if "skipframes" in job: env_dict["DRQUEUE_SKIPFRAMES"] = job["skipframes"] # run task on cluster render_script = DrQueue.get_rendertemplate(job["renderer"]) ar = self.lbview.apply(DrQueue.run_script_with_env, render_script, env_dict) # wait for pyzmq send to complete communication (avoid race condition) ar.wait_for_send() return True