def url_by_submitdir(submit_dir, db_type, config_properties=None, top_dir=None): """ Get URL from the submit directory """ if not submit_dir: raise ConnectionError("A submit directory should be provided with the type parameter.") if not db_type: raise ConnectionError("A type should be provided with the property file.") # From the submit dir, we need the wf_uuid # Getting values from the submit_dir braindump file top_level_wf_params = utils.slurp_braindb(submit_dir) # Return if we cannot parse the braindump.txt file if not top_level_wf_params: raise ConnectionError("File 'braindump.txt' not found in %s" % (submit_dir)) # Load the top-level braindump now if top_dir is not None if top_dir is not None: # Getting values from the top_dir braindump file top_level_wf_params = utils.slurp_braindb(top_dir) # Return if we cannot parse the braindump.txt file if not top_level_wf_params: raise ConnectionError("File 'braindump.txt' not found in %s" % (top_dir)) # Get the location of the properties file from braindump top_level_prop_file = None # Get properties tag from braindump if "properties" in top_level_wf_params: top_level_prop_file = top_level_wf_params["properties"] # Create the full path by using the submit_dir key from braindump if "submit_dir" in top_level_wf_params: top_level_prop_file = os.path.join(top_level_wf_params["submit_dir"], top_level_prop_file) return url_by_properties(config_properties, db_type, submit_dir, rundir_properties=top_level_prop_file)
def _parse_top_level_wf_params(submit_dir, top_dir): """ Parse the top level workflow parameters. :param submit_dir: path of the directory :param top_dir: path to the workflow top directory :return: top level workflow parameters """ dir = submit_dir if top_dir: dir = top_dir top_level_wf_params = utils.slurp_braindb(dir) # Return if we cannot parse the braindump.txt file if not top_level_wf_params: raise ConnectionError("File 'braindump.txt' not found in %s." % dir) if top_level_wf_params["root_wf_uuid"] == top_level_wf_params["wf_uuid"]: return top_level_wf_params while True: dir = os.path.abspath(os.path.join(os.path.abspath(dir), '..')) if dir == os.path.realpath('/..'): top_level_wf_params = None break top_level_wf_params = utils.slurp_braindb(dir) if top_level_wf_params and top_level_wf_params["root_wf_uuid"] == top_level_wf_params["wf_uuid"]: break if not top_level_wf_params: raise ConnectionError("Unable to find file 'braindump.txt' in parent folders.") return top_level_wf_params
def _parse_top_level_wf_params(submit_dir, top_dir): """ Parse the top level workflow parameters. :param submit_dir: path of the directory :param top_dir: path to the workflow top directory :return: top level workflow parameters """ top_level_wf_params = None dir = submit_dir if top_dir: dir = top_dir top_level_wf_params = utils.slurp_braindb(dir) # Return if we cannot parse the braindump.txt file if not top_level_wf_params: raise ConnectionError("File 'braindump.txt' not found in %s." % dir) if top_level_wf_params["root_wf_uuid"] == top_level_wf_params["wf_uuid"]: return top_level_wf_params while True: dir = os.path.abspath(os.path.join(os.path.abspath(dir), '..')) if dir == os.path.realpath('/..'): top_level_wf_params = None break top_level_wf_params = utils.slurp_braindb(dir) if top_level_wf_params and top_level_wf_params["root_wf_uuid"] == top_level_wf_params["wf_uuid"]: break if not top_level_wf_params: raise ConnectionError("Unable to find file 'braindump.txt' in parent folders.") return top_level_wf_params
def setup(submit_dir , config_properties): """ Setup the populate module @submit_dir submit directory path of the workflow run @config_properties path to the propery file """ # global reference global global_base_submit_dir global global_braindb_submit_dir global global_db_url global global_top_wf_uuid global_base_submit_dir = submit_dir #Getting values from braindump file config = utils.slurp_braindb(submit_dir) if ('submit_dir' in config or 'run' in config): if 'submit_dir' in config: global_braindb_submit_dir = os.path.abspath(config['submit_dir']) else: global_braindb_submit_dir = os.path.abspath(config['run']) else: logger.error("Submit directory cannot be found in the braindump.txt . ") sys.exit(1) # Create the sqllite db url global_db_url = connection.url_by_submitdir(submit_dir, connection.DBType.WORKFLOW, config_properties) global_top_wf_uuid = connection.get_wf_uuid(submit_dir) if global_db_url is None: sys.exit(1)
def purge_wf_uuid_from_database(rundir, output_db): """ This function purges a workflow id from the output database. """ if output_db.lower().find('sqlite:///') == 0: # Ok, we have a SQLite database, let's get the filename and check if it exists filename = output_db[10:] # Check if SQLite database exists if not os.path.isfile(filename): # No, nothing to do return # Parse the braindump file wfparams = utils.slurp_braindb(rundir) if "wf_uuid" in wfparams: if wfparams["wf_uuid"] is not None: # Get wf_uuid wf_uuid = wfparams["wf_uuid"] e = StampedeExpunge(output_db, wf_uuid) e.expunge() # Done, make this connection go away e = None
def _get_workflow_uri(props=None, submit_dir=None): """ Get WORKFLOW URI """ if props: dburi = props.property("pegasus.catalog.workflow.url") if dburi: return dburi dburi = props.property("pegasus.monitord.output") if dburi: return dburi if submit_dir: # From the submit dir, we need the wf_uuid # Getting values from the submit_dir braindump file top_level_wf_params = utils.slurp_braindb(submit_dir) # The default case is a .stampede.db file with the dag name as base dag_file_name = "" if top_level_wf_params.has_key("dag"): dag_file_name = top_level_wf_params["dag"] else: raise ConnectionError("DAG file name cannot be found in the braindump.txt.") # Create the sqllite db url dag_file_name = os.path.basename(dag_file_name) output_db_file = (submit_dir) + "/" + dag_file_name[: dag_file_name.find(".dag")] + ".stampede.db" dburi = "sqlite:///" + output_db_file return dburi return None
def __init__(self, submitdir, raise_err=True): self.submitdir = os.path.abspath(submitdir) self.submitdir_exists = True if not os.path.isdir(submitdir): self.submitdir_exists = False if raise_err is False: return raise SubmitDirException("Invalid submit dir: %s" % submitdir) self.braindump_file = os.path.join(self.submitdir, "braindump.yml") if not os.path.isfile(self.braindump_file): self.braindump_file = os.path.join(self.submitdir, "braindump.txt") # Read the braindump file self.braindump = utils.slurp_braindb(os.path.join(self.submitdir)) # Read some attributes from braindump file self.wf_uuid = self.braindump["wf_uuid"] self.root_wf_uuid = self.braindump["root_wf_uuid"] self.user = self.braindump["user"] self.archname = os.path.join(self.submitdir, "archive.tar.gz")
def setup(submit_dir, config_properties): """ Setup the populate module @submit_dir submit directory path of the workflow run @config_properties path to the propery file """ # global reference global global_base_submit_dir global global_braindb_submit_dir global global_db_url global global_top_wf_uuid global_base_submit_dir = submit_dir #Getting values from braindump file config = utils.slurp_braindb(submit_dir) if (config.has_key('submit_dir') or config.has_key('run')): if config.has_key('submit_dir'): global_braindb_submit_dir = os.path.abspath(config['submit_dir']) else: global_braindb_submit_dir = os.path.abspath(config['run']) else: logger.error( "Submit directory cannot be found in the braindump.txt . ") sys.exit(1) # Create the sqllite db url global_db_url, global_top_wf_uuid = db_utils.get_db_url_wf_uuid( submit_dir, config_properties) if global_db_url is None: sys.exit(1)
def setup(submit_dir, config_properties): """ Setup the populate module @submit_dir submit directory path of the workflow run @config_properties path to the propery file """ # global reference global global_base_submit_dir global global_braindb_submit_dir global global_db_url global global_top_wf_uuid global_base_submit_dir = submit_dir #Getting values from braindump file config = utils.slurp_braindb(submit_dir) if ('submit_dir' in config or 'run' in config): if 'submit_dir' in config: global_braindb_submit_dir = os.path.abspath(config['submit_dir']) else: global_braindb_submit_dir = os.path.abspath(config['run']) else: logger.error( "Submit directory cannot be found in the braindump.txt . ") sys.exit(1) # Create the sqllite db url global_db_url = connection.url_by_submitdir(submit_dir, connection.DBType.WORKFLOW, config_properties) global_top_wf_uuid = connection.get_wf_uuid(submit_dir) if global_db_url is None: sys.exit(1)
def setup(submit_dir , config_properties): """ Setup the populate module @submit_dir submit directory path of the workflow run @config_properties path to the propery file """ # global reference global global_base_submit_dir global global_braindb_submit_dir global global_db_url global global_top_wf_uuid global_base_submit_dir = submit_dir #Getting values from braindump file config = utils.slurp_braindb(submit_dir) if (config.has_key('submit_dir') or config.has_key('run')): if config.has_key('submit_dir'): global_braindb_submit_dir = os.path.abspath(config['submit_dir']) else: global_braindb_submit_dir = os.path.abspath(config['run']) else: logger.error("Submit directory cannot be found in the braindump.txt . ") sys.exit(1) # Create the sqllite db url global_db_url, global_top_wf_uuid = db_utils.get_db_url_wf_uuid(submit_dir, config_properties) if global_db_url is None: sys.exit(1)
def pegasus_remove(ctx, dag_id=None, verbose=False, submit_dir=None): """pegasus-remove helps you remove an entire workflow.""" if not submit_dir and not dag_id: print( "You must provide either a dag_id or dagdirectory to remove a workflow." ) ctx.exit(1) if submit_dir: cwd = os.getcwd() submit_dir = str(Path(submit_dir).resolve()) try: os.chdir(submit_dir) except PermissionError: click.secho( click.style("Error: ", fg="red", bold=True) + "Cannot change to directory %s" % submit_dir) ctx.exit(1) config = slurp_braindb(submit_dir) if not config: click.secho( click.style("Error: ", fg="red", bold=True) + "%s is not a valid submit-dir" % submit_dir) ctx.exit(1) dag_log_file = config["dag"] + ".dagman.out" pattern = re.compile(r"\.([0-9\.]+) \(CONDOR_DAGMAN\) STARTING UP") with open(dag_log_file) as fp: for line in fp.readlines(): match = pattern.search(line) if match: dag_id = match.group(1) else: if not dag_id: click.secho( click.style("Error: ", fg="red", bold=True) + "You must provide either a dag-id or dag-directory to remove a workflow." ) ctx.exit(1) os.chdir(cwd) if dag_id: condor_rm = shutil.which("condor_rm") cmd = (condor_rm, dag_id) rv = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if rv.returncode == 0: click.echo(rv.stdout.decode().strip()) click.secho("✨ Success", fg="green") else: click.echo(rv.stderr.decode().strip()) click.secho("Error ", fg="red", bold=True) ctx.exit(42)
def get_workflow_uuid(submit_dir): braindump = os.path.join(submit_dir, 'braindump.txt') if not os.path.isfile(braindump): raise ValueError('Not a valid workflow submit directory: %r' % submit_dir) braindump = utils.slurp_braindb(submit_dir) return braindump['root_wf_uuid'], braindump['wf_uuid']
def get_workflow_uuid(submit_dir): bdump_yml = Path(submit_dir) / "braindump.yml" bdump_txt = Path(submit_dir) / "braindump.txt" if bdump_yml.exists() is False and bdump_txt.exists() is False: raise ValueError("Not a valid workflow submit directory: %r" % submit_dir) braindump = utils.slurp_braindb(submit_dir) return braindump["root_wf_uuid"], braindump["wf_uuid"]
def purge_wf_uuid_from_dashboard_database(rundir, output_db): """ This function purges a workflow id from the output database. """ # Parse the braindump file wfparams = utils.slurp_braindb(rundir) wf_uuid = wfparams.get("wf_uuid", None) if "wf_uuid" is None: return expunge.delete_dashboard_workflow(output_db, wf_uuid)
def _parse_top_level_wf_params(dir): """ Parse the top level workflow parameters. :param dir: path of the directory :return: top level workflow parameters """ top_level_wf_params = utils.slurp_braindb(dir) # Return if we cannot parse the braindump.txt file if not top_level_wf_params: raise ConnectionError("File 'braindump.txt' not found in %s" % (dir)) return top_level_wf_params
def purge_wf_uuid_from_dashboard_database(rundir, output_db): """ This function purges a workflow id from the output database. """ # Parse the braindump file wfparams = utils.slurp_braindb(rundir) wf_uuid = wfparams.get("wf_uuid", None) if "wf_uuid" is None: return e = DashboardExpunge(output_db, wf_uuid) e.expunge() # Done, make this connection go away e = None
def purge_wf_uuid_from_dashboard_database(rundir, output_db): """ This function purges a workflow id from the output database. """ # Parse the braindump file wfparams = utils.slurp_braindb(rundir) if "wf_uuid" in wfparams: if wfparams["wf_uuid"] is not None: # Get wf_uuid wf_uuid = wfparams["wf_uuid"] e = DashboardExpunge(output_db, wf_uuid) e.expunge() # Done, make this connection go away e = None
def purge_wf_uuid_from_database(rundir, output_db): """ This function purges a workflow id from the output database. """ # PM-652 do nothing for sqlite # DB is already rotated in pegasus-monitord if output_db.lower().startswith('sqlite'): return # Parse the braindump file wfparams = utils.slurp_braindb(rundir) wf_uuid = wfparams.get("wf_uuid", None) if "wf_uuid" is None: return expunge.delete_workflow(output_db, wf_uuid)
def get_wf_uuid(submit_dir): # From the submit dir, we need the wf_uuid # Getting values from the submit_dir braindump file top_level_wf_params = utils.slurp_braindb(submit_dir) # Return if we cannot parse the braindump.txt file if not top_level_wf_params: log.error("Unable to process braindump.txt in %s" % (submit_dir)) return None # Get wf_uuid for this workflow wf_uuid = None if (top_level_wf_params.has_key('wf_uuid')): wf_uuid = top_level_wf_params['wf_uuid'] else: log.error("workflow id cannot be found in the braindump.txt ") return None return wf_uuid
def get_wf_uuid(submit_dir): # From the submit dir, we need the wf_uuid # Getting values from the submit_dir braindump file top_level_wf_params = utils.slurp_braindb(submit_dir) # Return if we cannot parse the braindump.txt file if not top_level_wf_params: logger.error("Unable to process braindump.txt in %s" % (submit_dir)) return None # Get wf_uuid for this workflow wf_uuid = None if (top_level_wf_params.has_key('wf_uuid')): wf_uuid = top_level_wf_params['wf_uuid'] else: logger.error("workflow id cannot be found in the braindump.txt ") return None return wf_uuid
def populate_chart(wf_uuid , expand = False): """ Populates the workflow info object corresponding to the wf_uuid @param wf_uuid the workflow uuid @param expand expand workflow or not. """ workflow_stampede_stats = get_wf_stats(wf_uuid , expand) workflow_info = populate_workflow_details(workflow_stampede_stats) sub_wf_uuids = workflow_stampede_stats.get_sub_workflow_ids() workflow_info.sub_wf_id_uuids = sub_wf_uuids if len(sub_wf_uuids) > 0: workflow_info.job_instance_id_sub_wf_uuid_map = get_job_inst_sub_workflow_map(workflow_stampede_stats ) config = utils.slurp_braindb(rlb(workflow_info.submit_dir)) if (config.has_key('dag')): dag_file_name = config['dag'] workflow_info.dag_label = dag_file_name[:dag_file_name.find(".dag")] workflow_info.dag_file_path = os.path.join(rlb(workflow_info.submit_dir), dag_file_name) if (config.has_key('dax')): workflow_info.dax_file_path = config['dax'] return workflow_stampede_stats, workflow_info
def purge_wf_uuid_from_database(rundir, output_db): """ This function purges a workflow id from the output database. """ # PM-652 do nothing for sqlite # DB is already rotated in pegasus-monitord if output_db.lower().startswith("sqlite"): return # Parse the braindump file wfparams = utils.slurp_braindb(rundir) wf_uuid = wfparams.get("wf_uuid", None) if "wf_uuid" is None: return e = StampedeExpunge(output_db, wf_uuid) e.expunge() # Done, make this connection go away e = None
def populate_chart(wf_uuid, expand=False): """ Populates the workflow info object corresponding to the wf_uuid @param wf_uuid the workflow uuid @param expand expand workflow or not. """ workflow_stampede_stats = get_wf_stats(wf_uuid, expand) workflow_info = populate_workflow_details(workflow_stampede_stats) sub_wf_uuids = workflow_stampede_stats.get_sub_workflow_ids() workflow_info.sub_wf_id_uuids = sub_wf_uuids if len(sub_wf_uuids) > 0: workflow_info.job_instance_id_sub_wf_uuid_map = get_job_inst_sub_workflow_map( workflow_stampede_stats) config = utils.slurp_braindb(rlb(workflow_info.submit_dir)) if ('dag' in config): dag_file_name = config['dag'] workflow_info.dag_label = dag_file_name[:dag_file_name.find(".dag")] workflow_info.dag_file_path = os.path.join( rlb(workflow_info.submit_dir), dag_file_name) if ('dax' in config): workflow_info.dax_file_path = config['dax'] return workflow_stampede_stats, workflow_info
def pegasus_run(ctx, grid=False, json=False, verbose=0, submit_dir=None): """.""" logging.basicConfig(level=logging.ERROR - (min(verbose, 3) * 10)) os.umask(0o022) cwd = os.getcwd() config = slurp_braindb(submit_dir) submit_dir = str(Path(submit_dir).resolve()) if not config: click.secho( click.style("Error: ", fg="red", bold=True) + "%s is not a valid submit-dir" % submit_dir) ctx.exit(1) try: os.chdir(submit_dir) except PermissionError: click.secho( click.style("Error: ", fg="red", bold=True) + "Cannot change to directory %s" % submit_dir) ctx.exit(1) if grid: try: grid_check() except (FileNotFoundError, PermissionError, ValueError) as e: click.secho(click.style("Error: ", fg="red", bold=True) + str(e)) ctx.exit(1) if config["dag"]: try: # sanity check: Is the DAG file there? check_dag(config["dag"]) # PM-870 we have already changed to the directory, don't prepend $run again dag_sub_file = config["dag"] + ".condor.sub" # PM-702: clean up .halt files from pegasus-halt halt_released = False if Path(config["dag"] + ".halt").exists(): click.echo( "Found a previously halted workflow. Releasing it now.") os.system("find . -name '*.dag.halt' -exec rm {} \\;") halt_released = True # After the switch from condor_submit_dag, we lost the check to see if # a workflow is already running. This replaces those checks. if Path("monitord.pid").exists(): if halt_released: ctx.exit(0) else: click.secho( click.style("Error: ", fg="red", bold=True) + "It looks like the workflow is already running! If you are sure\n" + " that is not the case, please remove the monitord.pid file and try\n" + " again.", err=True, ) ctx.exit(1) # PM-797 do condor_submit on dagman.condor.sub file if it exists exec_dag(dag_sub_file, config["condor_log"]) log.debug("# dagman is running") if json: click.echo(dumps(config)) else: click.secho(""" Your workflow has been started and is running in the base directory: %(submit_dir)s *** To monitor the workflow you can run *** pegasus-status -l %(submit_dir)s *** To remove your workflow run *** pegasus-remove %(submit_dir)s""" % {"submit_dir": submit_dir}) except (FileNotFoundError, PermissionError, ValueError) as e: click.secho(click.style("Error: ", fg="red", bold=True) + str(e)) ctx.exit(1) except subprocess.CalledProcessError as e: rc = e.returncode if rc != 0: click.secho( click.style("Error: ", fg="red", bold=True) + "Running %s failed with %d" % (e.cmd, rc)) ctx.exit(rc) elif config["type"] == "shell": try: exec_script(config["script"]) click.secho("✨ Success", fg="green") except (FileNotFoundError, PermissionError) as e: click.secho(click.style("Error: ", fg="red", bold=True) + str(e)) ctx.exit(1) except subprocess.CalledProcessError as e: rc = e.returncode if rc != 0: click.secho( click.style("Error: ", fg="red", bold=True) + "Running %s failed with %d" % (config["script"], rc)) ctx.exit(rc) os.chdir(cwd)
def get_db_url_wf_uuid(submit_dir, config_properties, top_dir=None): """ Utility method for returning the db_url and wf_uuid given the submit_dir and pegasus properties file. @submit_dir submit directory path @config_properties config properties file path @top_dir directory of the top-level workflow (where the database is) """ # From the submit dir, we need the wf_uuid # Getting values from the submit_dir braindump file top_level_wf_params = utils.slurp_braindb(submit_dir) # Return if we cannot parse the braindump.txt file if not top_level_wf_params: logger.error("Unable to process braindump.txt in %s" % (submit_dir)) return None, None # Get wf_uuid for this workflow wf_uuid = None if (top_level_wf_params.has_key('wf_uuid')): wf_uuid = top_level_wf_params['wf_uuid'] else: logger.error("workflow id cannot be found in the braindump.txt ") return None, None # Load the top-level braindump now if top_dir is not None if top_dir is not None: # Getting values from the top_dir braindump file top_level_wf_params = utils.slurp_braindb(top_dir) # Return if we cannot parse the braindump.txt file if not top_level_wf_params: logger.error("Unable to process braindump.txt in %s" % (top_dir)) return None, None # Get the location of the properties file from braindump top_level_prop_file = None # Get properties tag from braindump if "properties" in top_level_wf_params: top_level_prop_file = top_level_wf_params["properties"] # Create the full path by using the submit_dir key from braindump if "submit_dir" in top_level_wf_params: top_level_prop_file = os.path.join(top_level_wf_params["submit_dir"], top_level_prop_file) # Parse, and process properties props = properties.Properties() props.new(config_file=config_properties, rundir_propfile=top_level_prop_file) # Ok, now figure out the database URL output_db_url = None if props.property('pegasus.monitord.output') is not None: output_db_url = props.property('pegasus.monitord.output') # Return, if not using sqlite or mysql if not (output_db_url.startswith("mysql:") or output_db_url.startswith("sqlite:")): logger.error("Unable to find database file from the properties file ") return None, None else: # Ok, the default case is a .stampede.db file with the dag name as base dag_file_name = '' if (top_level_wf_params.has_key('dag')): dag_file_name = top_level_wf_params['dag'] else: logger.error("dag file name cannot be found in the braindump.txt") return None, None # Create the sqllite db url output_db_file = (top_dir or submit_dir) + "/" + dag_file_name[:dag_file_name.find(".dag")] + ".stampede.db" output_db_url = "sqlite:///" + output_db_file if not os.path.isfile(output_db_file): logger.error("Unable to find database file in " + (top_dir or submit_dir)) return None, None # Ok, all done! return output_db_url, wf_uuid