def url_by_submitdir(submit_dir, db_type, config_properties=None, top_dir=None):
""" Get URL from the submit directory """
if not submit_dir:
raise ConnectionError("A submit directory should be provided with the type parameter.")
if not db_type:
raise ConnectionError("A type should be provided with the property file.")
# From the submit dir, we need the wf_uuid
# Getting values from the submit_dir braindump file
top_level_wf_params = utils.slurp_braindb(submit_dir)
# Return if we cannot parse the braindump.txt file
if not top_level_wf_params:
raise ConnectionError("File 'braindump.txt' not found in %s" % (submit_dir))
# Load the top-level braindump now if top_dir is not None
if top_dir is not None:
# Getting values from the top_dir braindump file
top_level_wf_params = utils.slurp_braindb(top_dir)
# Return if we cannot parse the braindump.txt file
if not top_level_wf_params:
raise ConnectionError("File 'braindump.txt' not found in %s" % (top_dir))
# Get the location of the properties file from braindump
top_level_prop_file = None
# Get properties tag from braindump
if "properties" in top_level_wf_params:
top_level_prop_file = top_level_wf_params["properties"]
# Create the full path by using the submit_dir key from braindump
if "submit_dir" in top_level_wf_params:
top_level_prop_file = os.path.join(top_level_wf_params["submit_dir"], top_level_prop_file)
return url_by_properties(config_properties, db_type, submit_dir, rundir_properties=top_level_prop_file)
def _parse_top_level_wf_params(submit_dir, top_dir):
"""
Parse the top level workflow parameters.
:param submit_dir: path of the directory
:param top_dir: path to the workflow top directory
:return: top level workflow parameters
"""
dir = submit_dir
if top_dir:
dir = top_dir
top_level_wf_params = utils.slurp_braindb(dir)
# Return if we cannot parse the braindump.txt file
if not top_level_wf_params:
raise ConnectionError("File 'braindump.txt' not found in %s." % dir)
if top_level_wf_params["root_wf_uuid"] == top_level_wf_params["wf_uuid"]:
return top_level_wf_params
while True:
dir = os.path.abspath(os.path.join(os.path.abspath(dir), '..'))
if dir == os.path.realpath('/..'):
top_level_wf_params = None
break
top_level_wf_params = utils.slurp_braindb(dir)
if top_level_wf_params and top_level_wf_params["root_wf_uuid"] == top_level_wf_params["wf_uuid"]:
break
if not top_level_wf_params:
raise ConnectionError("Unable to find file 'braindump.txt' in parent folders.")
return top_level_wf_params
def _parse_top_level_wf_params(submit_dir, top_dir):
"""
Parse the top level workflow parameters.
:param submit_dir: path of the directory
:param top_dir: path to the workflow top directory
:return: top level workflow parameters
"""
top_level_wf_params = None
dir = submit_dir
if top_dir:
dir = top_dir
top_level_wf_params = utils.slurp_braindb(dir)
# Return if we cannot parse the braindump.txt file
if not top_level_wf_params:
raise ConnectionError("File 'braindump.txt' not found in %s." % dir)
if top_level_wf_params["root_wf_uuid"] == top_level_wf_params["wf_uuid"]:
return top_level_wf_params
while True:
dir = os.path.abspath(os.path.join(os.path.abspath(dir), '..'))
if dir == os.path.realpath('/..'):
top_level_wf_params = None
break
top_level_wf_params = utils.slurp_braindb(dir)
if top_level_wf_params and top_level_wf_params["root_wf_uuid"] == top_level_wf_params["wf_uuid"]:
break
if not top_level_wf_params:
raise ConnectionError("Unable to find file 'braindump.txt' in parent folders.")
return top_level_wf_params
def setup(submit_dir , config_properties):
"""
Setup the populate module
@submit_dir submit directory path of the workflow run
@config_properties path to the propery file
"""
# global reference
global global_base_submit_dir
global global_braindb_submit_dir
global global_db_url
global global_top_wf_uuid
global_base_submit_dir = submit_dir
#Getting values from braindump file
config = utils.slurp_braindb(submit_dir)
if ('submit_dir' in config or 'run' in config):
if 'submit_dir' in config:
global_braindb_submit_dir = os.path.abspath(config['submit_dir'])
else:
global_braindb_submit_dir = os.path.abspath(config['run'])
else:
logger.error("Submit directory cannot be found in the braindump.txt . ")
sys.exit(1)
# Create the sqllite db url
global_db_url = connection.url_by_submitdir(submit_dir, connection.DBType.WORKFLOW, config_properties)
global_top_wf_uuid = connection.get_wf_uuid(submit_dir)
if global_db_url is None:
sys.exit(1)
def purge_wf_uuid_from_database(rundir, output_db):
"""
This function purges a workflow id from the output database.
"""
if output_db.lower().find('sqlite:///') == 0:
# Ok, we have a SQLite database, let's get the filename and check if it exists
filename = output_db[10:]
# Check if SQLite database exists
if not os.path.isfile(filename):
# No, nothing to do
return
# Parse the braindump file
wfparams = utils.slurp_braindb(rundir)
if "wf_uuid" in wfparams:
if wfparams["wf_uuid"] is not None:
# Get wf_uuid
wf_uuid = wfparams["wf_uuid"]
e = StampedeExpunge(output_db, wf_uuid)
e.expunge()
# Done, make this connection go away
e = None
def _get_workflow_uri(props=None, submit_dir=None):
""" Get WORKFLOW URI """
if props:
dburi = props.property("pegasus.catalog.workflow.url")
if dburi:
return dburi
dburi = props.property("pegasus.monitord.output")
if dburi:
return dburi
if submit_dir:
# From the submit dir, we need the wf_uuid
# Getting values from the submit_dir braindump file
top_level_wf_params = utils.slurp_braindb(submit_dir)
# The default case is a .stampede.db file with the dag name as base
dag_file_name = ""
if top_level_wf_params.has_key("dag"):
dag_file_name = top_level_wf_params["dag"]
else:
raise ConnectionError("DAG file name cannot be found in the braindump.txt.")
# Create the sqllite db url
dag_file_name = os.path.basename(dag_file_name)
output_db_file = (submit_dir) + "/" + dag_file_name[: dag_file_name.find(".dag")] + ".stampede.db"
dburi = "sqlite:///" + output_db_file
return dburi
return None
def __init__(self, submitdir, raise_err=True):
self.submitdir = os.path.abspath(submitdir)
self.submitdir_exists = True
if not os.path.isdir(submitdir):
self.submitdir_exists = False
if raise_err is False:
return
raise SubmitDirException("Invalid submit dir: %s" % submitdir)
self.braindump_file = os.path.join(self.submitdir, "braindump.yml")
if not os.path.isfile(self.braindump_file):
self.braindump_file = os.path.join(self.submitdir, "braindump.txt")
# Read the braindump file
self.braindump = utils.slurp_braindb(os.path.join(self.submitdir))
# Read some attributes from braindump file
self.wf_uuid = self.braindump["wf_uuid"]
self.root_wf_uuid = self.braindump["root_wf_uuid"]
self.user = self.braindump["user"]
self.archname = os.path.join(self.submitdir, "archive.tar.gz")
def purge_wf_uuid_from_database(rundir, output_db):
"""
This function purges a workflow id from the output database.
"""
if output_db.lower().find('sqlite:///') == 0:
# Ok, we have a SQLite database, let's get the filename and check if it exists
filename = output_db[10:]
# Check if SQLite database exists
if not os.path.isfile(filename):
# No, nothing to do
return
# Parse the braindump file
wfparams = utils.slurp_braindb(rundir)
if "wf_uuid" in wfparams:
if wfparams["wf_uuid"] is not None:
# Get wf_uuid
wf_uuid = wfparams["wf_uuid"]
e = StampedeExpunge(output_db, wf_uuid)
e.expunge()
# Done, make this connection go away
e = None
def setup(submit_dir, config_properties):
"""
Setup the populate module
@submit_dir submit directory path of the workflow run
@config_properties path to the propery file
"""
# global reference
global global_base_submit_dir
global global_braindb_submit_dir
global global_db_url
global global_top_wf_uuid
global_base_submit_dir = submit_dir
#Getting values from braindump file
config = utils.slurp_braindb(submit_dir)
if (config.has_key('submit_dir') or config.has_key('run')):
if config.has_key('submit_dir'):
global_braindb_submit_dir = os.path.abspath(config['submit_dir'])
else:
global_braindb_submit_dir = os.path.abspath(config['run'])
else:
logger.error(
"Submit directory cannot be found in the braindump.txt . ")
sys.exit(1)
# Create the sqllite db url
global_db_url, global_top_wf_uuid = db_utils.get_db_url_wf_uuid(
submit_dir, config_properties)
if global_db_url is None:
sys.exit(1)
def setup(submit_dir, config_properties):
"""
Setup the populate module
@submit_dir submit directory path of the workflow run
@config_properties path to the propery file
"""
# global reference
global global_base_submit_dir
global global_braindb_submit_dir
global global_db_url
global global_top_wf_uuid
global_base_submit_dir = submit_dir
#Getting values from braindump file
config = utils.slurp_braindb(submit_dir)
if ('submit_dir' in config or 'run' in config):
if 'submit_dir' in config:
global_braindb_submit_dir = os.path.abspath(config['submit_dir'])
else:
global_braindb_submit_dir = os.path.abspath(config['run'])
else:
logger.error(
"Submit directory cannot be found in the braindump.txt . ")
sys.exit(1)
# Create the sqllite db url
global_db_url = connection.url_by_submitdir(submit_dir,
connection.DBType.WORKFLOW,
config_properties)
global_top_wf_uuid = connection.get_wf_uuid(submit_dir)
if global_db_url is None:
sys.exit(1)
def setup(submit_dir , config_properties):
"""
Setup the populate module
@submit_dir submit directory path of the workflow run
@config_properties path to the propery file
"""
# global reference
global global_base_submit_dir
global global_braindb_submit_dir
global global_db_url
global global_top_wf_uuid
global_base_submit_dir = submit_dir
#Getting values from braindump file
config = utils.slurp_braindb(submit_dir)
if (config.has_key('submit_dir') or config.has_key('run')):
if config.has_key('submit_dir'):
global_braindb_submit_dir = os.path.abspath(config['submit_dir'])
else:
global_braindb_submit_dir = os.path.abspath(config['run'])
else:
logger.error("Submit directory cannot be found in the braindump.txt . ")
sys.exit(1)
# Create the sqllite db url
global_db_url, global_top_wf_uuid = db_utils.get_db_url_wf_uuid(submit_dir, config_properties)
if global_db_url is None:
sys.exit(1)
def pegasus_remove(ctx, dag_id=None, verbose=False, submit_dir=None):
"""pegasus-remove helps you remove an entire workflow."""
if not submit_dir and not dag_id:
print(
"You must provide either a dag_id or dagdirectory to remove a workflow."
)
ctx.exit(1)
if submit_dir:
cwd = os.getcwd()
submit_dir = str(Path(submit_dir).resolve())
try:
os.chdir(submit_dir)
except PermissionError:
click.secho(
click.style("Error: ", fg="red", bold=True) +
"Cannot change to directory %s" % submit_dir)
ctx.exit(1)
config = slurp_braindb(submit_dir)
if not config:
click.secho(
click.style("Error: ", fg="red", bold=True) +
"%s is not a valid submit-dir" % submit_dir)
ctx.exit(1)
dag_log_file = config["dag"] + ".dagman.out"
pattern = re.compile(r"\.([0-9\.]+) \(CONDOR_DAGMAN\) STARTING UP")
with open(dag_log_file) as fp:
for line in fp.readlines():
match = pattern.search(line)
if match:
dag_id = match.group(1)
else:
if not dag_id:
click.secho(
click.style("Error: ", fg="red", bold=True) +
"You must provide either a dag-id or dag-directory to remove a workflow."
)
ctx.exit(1)
os.chdir(cwd)
if dag_id:
condor_rm = shutil.which("condor_rm")
cmd = (condor_rm, dag_id)
rv = subprocess.run(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if rv.returncode == 0:
click.echo(rv.stdout.decode().strip())
click.secho("✨ Success", fg="green")
else:
click.echo(rv.stderr.decode().strip())
click.secho("Error ", fg="red", bold=True)
ctx.exit(42)
def get_workflow_uuid(submit_dir):
braindump = os.path.join(submit_dir, 'braindump.txt')
if not os.path.isfile(braindump):
raise ValueError('Not a valid workflow submit directory: %r' %
submit_dir)
braindump = utils.slurp_braindb(submit_dir)
return braindump['root_wf_uuid'], braindump['wf_uuid']
def get_workflow_uuid(submit_dir):
bdump_yml = Path(submit_dir) / "braindump.yml"
bdump_txt = Path(submit_dir) / "braindump.txt"
if bdump_yml.exists() is False and bdump_txt.exists() is False:
raise ValueError("Not a valid workflow submit directory: %r" %
submit_dir)
braindump = utils.slurp_braindb(submit_dir)
return braindump["root_wf_uuid"], braindump["wf_uuid"]
def purge_wf_uuid_from_dashboard_database(rundir, output_db):
"""
This function purges a workflow id from the output database.
"""
# Parse the braindump file
wfparams = utils.slurp_braindb(rundir)
wf_uuid = wfparams.get("wf_uuid", None)
if "wf_uuid" is None:
return
expunge.delete_dashboard_workflow(output_db, wf_uuid)
def _parse_top_level_wf_params(dir):
"""
Parse the top level workflow parameters.
:param dir: path of the directory
:return: top level workflow parameters
"""
top_level_wf_params = utils.slurp_braindb(dir)
# Return if we cannot parse the braindump.txt file
if not top_level_wf_params:
raise ConnectionError("File 'braindump.txt' not found in %s" % (dir))
return top_level_wf_params
def purge_wf_uuid_from_dashboard_database(rundir, output_db):
"""
This function purges a workflow id from the output database.
"""
# Parse the braindump file
wfparams = utils.slurp_braindb(rundir)
wf_uuid = wfparams.get("wf_uuid", None)
if "wf_uuid" is None:
return
expunge.delete_dashboard_workflow(output_db, wf_uuid)
def _parse_top_level_wf_params(dir):
"""
Parse the top level workflow parameters.
:param dir: path of the directory
:return: top level workflow parameters
"""
top_level_wf_params = utils.slurp_braindb(dir)
# Return if we cannot parse the braindump.txt file
if not top_level_wf_params:
raise ConnectionError("File 'braindump.txt' not found in %s" % (dir))
return top_level_wf_params
def purge_wf_uuid_from_dashboard_database(rundir, output_db):
"""
This function purges a workflow id from the output database.
"""
# Parse the braindump file
wfparams = utils.slurp_braindb(rundir)
wf_uuid = wfparams.get("wf_uuid", None)
if "wf_uuid" is None:
return
e = DashboardExpunge(output_db, wf_uuid)
e.expunge()
# Done, make this connection go away
e = None
def purge_wf_uuid_from_dashboard_database(rundir, output_db):
"""
This function purges a workflow id from the output database.
"""
# Parse the braindump file
wfparams = utils.slurp_braindb(rundir)
wf_uuid = wfparams.get("wf_uuid", None)
if "wf_uuid" is None:
return
e = DashboardExpunge(output_db, wf_uuid)
e.expunge()
# Done, make this connection go away
e = None
def purge_wf_uuid_from_dashboard_database(rundir, output_db):
"""
This function purges a workflow id from the output database.
"""
# Parse the braindump file
wfparams = utils.slurp_braindb(rundir)
if "wf_uuid" in wfparams:
if wfparams["wf_uuid"] is not None:
# Get wf_uuid
wf_uuid = wfparams["wf_uuid"]
e = DashboardExpunge(output_db, wf_uuid)
e.expunge()
# Done, make this connection go away
e = None
def purge_wf_uuid_from_database(rundir, output_db):
"""
This function purges a workflow id from the output database.
"""
# PM-652 do nothing for sqlite
# DB is already rotated in pegasus-monitord
if output_db.lower().startswith('sqlite'):
return
# Parse the braindump file
wfparams = utils.slurp_braindb(rundir)
wf_uuid = wfparams.get("wf_uuid", None)
if "wf_uuid" is None:
return
expunge.delete_workflow(output_db, wf_uuid)
def purge_wf_uuid_from_dashboard_database(rundir, output_db):
"""
This function purges a workflow id from the output database.
"""
# Parse the braindump file
wfparams = utils.slurp_braindb(rundir)
if "wf_uuid" in wfparams:
if wfparams["wf_uuid"] is not None:
# Get wf_uuid
wf_uuid = wfparams["wf_uuid"]
e = DashboardExpunge(output_db, wf_uuid)
e.expunge()
# Done, make this connection go away
e = None
def purge_wf_uuid_from_database(rundir, output_db):
"""
This function purges a workflow id from the output database.
"""
# PM-652 do nothing for sqlite
# DB is already rotated in pegasus-monitord
if output_db.lower().startswith('sqlite'):
return
# Parse the braindump file
wfparams = utils.slurp_braindb(rundir)
wf_uuid = wfparams.get("wf_uuid", None)
if "wf_uuid" is None:
return
expunge.delete_workflow(output_db, wf_uuid)
def get_wf_uuid(submit_dir):
# From the submit dir, we need the wf_uuid
# Getting values from the submit_dir braindump file
top_level_wf_params = utils.slurp_braindb(submit_dir)
# Return if we cannot parse the braindump.txt file
if not top_level_wf_params:
log.error("Unable to process braindump.txt in %s" % (submit_dir))
return None
# Get wf_uuid for this workflow
wf_uuid = None
if (top_level_wf_params.has_key('wf_uuid')):
wf_uuid = top_level_wf_params['wf_uuid']
else:
log.error("workflow id cannot be found in the braindump.txt ")
return None
return wf_uuid
def get_wf_uuid(submit_dir):
# From the submit dir, we need the wf_uuid
# Getting values from the submit_dir braindump file
top_level_wf_params = utils.slurp_braindb(submit_dir)
# Return if we cannot parse the braindump.txt file
if not top_level_wf_params:
logger.error("Unable to process braindump.txt in %s" % (submit_dir))
return None
# Get wf_uuid for this workflow
wf_uuid = None
if (top_level_wf_params.has_key('wf_uuid')):
wf_uuid = top_level_wf_params['wf_uuid']
else:
logger.error("workflow id cannot be found in the braindump.txt ")
return None
return wf_uuid
def populate_chart(wf_uuid , expand = False):
"""
Populates the workflow info object corresponding to the wf_uuid
@param wf_uuid the workflow uuid
@param expand expand workflow or not.
"""
workflow_stampede_stats = get_wf_stats(wf_uuid , expand)
workflow_info = populate_workflow_details(workflow_stampede_stats)
sub_wf_uuids = workflow_stampede_stats.get_sub_workflow_ids()
workflow_info.sub_wf_id_uuids = sub_wf_uuids
if len(sub_wf_uuids) > 0:
workflow_info.job_instance_id_sub_wf_uuid_map = get_job_inst_sub_workflow_map(workflow_stampede_stats )
config = utils.slurp_braindb(rlb(workflow_info.submit_dir))
if (config.has_key('dag')):
dag_file_name = config['dag']
workflow_info.dag_label = dag_file_name[:dag_file_name.find(".dag")]
workflow_info.dag_file_path = os.path.join(rlb(workflow_info.submit_dir), dag_file_name)
if (config.has_key('dax')):
workflow_info.dax_file_path = config['dax']
return workflow_stampede_stats, workflow_info
def purge_wf_uuid_from_database(rundir, output_db):
"""
This function purges a workflow id from the output database.
"""
# PM-652 do nothing for sqlite
# DB is already rotated in pegasus-monitord
if output_db.lower().startswith("sqlite"):
return
# Parse the braindump file
wfparams = utils.slurp_braindb(rundir)
wf_uuid = wfparams.get("wf_uuid", None)
if "wf_uuid" is None:
return
e = StampedeExpunge(output_db, wf_uuid)
e.expunge()
# Done, make this connection go away
e = None
def populate_chart(wf_uuid, expand=False):
"""
Populates the workflow info object corresponding to the wf_uuid
@param wf_uuid the workflow uuid
@param expand expand workflow or not.
"""
workflow_stampede_stats = get_wf_stats(wf_uuid, expand)
workflow_info = populate_workflow_details(workflow_stampede_stats)
sub_wf_uuids = workflow_stampede_stats.get_sub_workflow_ids()
workflow_info.sub_wf_id_uuids = sub_wf_uuids
if len(sub_wf_uuids) > 0:
workflow_info.job_instance_id_sub_wf_uuid_map = get_job_inst_sub_workflow_map(
workflow_stampede_stats)
config = utils.slurp_braindb(rlb(workflow_info.submit_dir))
if ('dag' in config):
dag_file_name = config['dag']
workflow_info.dag_label = dag_file_name[:dag_file_name.find(".dag")]
workflow_info.dag_file_path = os.path.join(
rlb(workflow_info.submit_dir), dag_file_name)
if ('dax' in config):
workflow_info.dax_file_path = config['dax']
return workflow_stampede_stats, workflow_info
def pegasus_run(ctx, grid=False, json=False, verbose=0, submit_dir=None):
"""."""
logging.basicConfig(level=logging.ERROR - (min(verbose, 3) * 10))
os.umask(0o022)
cwd = os.getcwd()
config = slurp_braindb(submit_dir)
submit_dir = str(Path(submit_dir).resolve())
if not config:
click.secho(
click.style("Error: ", fg="red", bold=True) +
"%s is not a valid submit-dir" % submit_dir)
ctx.exit(1)
try:
os.chdir(submit_dir)
except PermissionError:
click.secho(
click.style("Error: ", fg="red", bold=True) +
"Cannot change to directory %s" % submit_dir)
ctx.exit(1)
if grid:
try:
grid_check()
except (FileNotFoundError, PermissionError, ValueError) as e:
click.secho(click.style("Error: ", fg="red", bold=True) + str(e))
ctx.exit(1)
if config["dag"]:
try:
# sanity check: Is the DAG file there?
check_dag(config["dag"])
# PM-870 we have already changed to the directory, don't prepend $run again
dag_sub_file = config["dag"] + ".condor.sub"
# PM-702: clean up .halt files from pegasus-halt
halt_released = False
if Path(config["dag"] + ".halt").exists():
click.echo(
"Found a previously halted workflow. Releasing it now.")
os.system("find . -name '*.dag.halt' -exec rm {} \\;")
halt_released = True
# After the switch from condor_submit_dag, we lost the check to see if
# a workflow is already running. This replaces those checks.
if Path("monitord.pid").exists():
if halt_released:
ctx.exit(0)
else:
click.secho(
click.style("Error: ", fg="red", bold=True) +
"It looks like the workflow is already running! If you are sure\n"
+
" that is not the case, please remove the monitord.pid file and try\n"
+ " again.",
err=True,
)
ctx.exit(1)
# PM-797 do condor_submit on dagman.condor.sub file if it exists
exec_dag(dag_sub_file, config["condor_log"])
log.debug("# dagman is running")
if json:
click.echo(dumps(config))
else:
click.secho("""
Your workflow has been started and is running in the base directory:
%(submit_dir)s
*** To monitor the workflow you can run ***
pegasus-status -l %(submit_dir)s
*** To remove your workflow run ***
pegasus-remove %(submit_dir)s""" % {"submit_dir": submit_dir})
except (FileNotFoundError, PermissionError, ValueError) as e:
click.secho(click.style("Error: ", fg="red", bold=True) + str(e))
ctx.exit(1)
except subprocess.CalledProcessError as e:
rc = e.returncode
if rc != 0:
click.secho(
click.style("Error: ", fg="red", bold=True) +
"Running %s failed with %d" % (e.cmd, rc))
ctx.exit(rc)
elif config["type"] == "shell":
try:
exec_script(config["script"])
click.secho("✨ Success", fg="green")
except (FileNotFoundError, PermissionError) as e:
click.secho(click.style("Error: ", fg="red", bold=True) + str(e))
ctx.exit(1)
except subprocess.CalledProcessError as e:
rc = e.returncode
if rc != 0:
click.secho(
click.style("Error: ", fg="red", bold=True) +
"Running %s failed with %d" % (config["script"], rc))
ctx.exit(rc)
os.chdir(cwd)
def get_db_url_wf_uuid(submit_dir, config_properties, top_dir=None):
"""
Utility method for returning the db_url and wf_uuid given the submit_dir and pegasus properties file.
@submit_dir submit directory path
@config_properties config properties file path
@top_dir directory of the top-level workflow (where the database is)
"""
# From the submit dir, we need the wf_uuid
# Getting values from the submit_dir braindump file
top_level_wf_params = utils.slurp_braindb(submit_dir)
# Return if we cannot parse the braindump.txt file
if not top_level_wf_params:
logger.error("Unable to process braindump.txt in %s" % (submit_dir))
return None, None
# Get wf_uuid for this workflow
wf_uuid = None
if (top_level_wf_params.has_key('wf_uuid')):
wf_uuid = top_level_wf_params['wf_uuid']
else:
logger.error("workflow id cannot be found in the braindump.txt ")
return None, None
# Load the top-level braindump now if top_dir is not None
if top_dir is not None:
# Getting values from the top_dir braindump file
top_level_wf_params = utils.slurp_braindb(top_dir)
# Return if we cannot parse the braindump.txt file
if not top_level_wf_params:
logger.error("Unable to process braindump.txt in %s" % (top_dir))
return None, None
# Get the location of the properties file from braindump
top_level_prop_file = None
# Get properties tag from braindump
if "properties" in top_level_wf_params:
top_level_prop_file = top_level_wf_params["properties"]
# Create the full path by using the submit_dir key from braindump
if "submit_dir" in top_level_wf_params:
top_level_prop_file = os.path.join(top_level_wf_params["submit_dir"], top_level_prop_file)
# Parse, and process properties
props = properties.Properties()
props.new(config_file=config_properties, rundir_propfile=top_level_prop_file)
# Ok, now figure out the database URL
output_db_url = None
if props.property('pegasus.monitord.output') is not None:
output_db_url = props.property('pegasus.monitord.output')
# Return, if not using sqlite or mysql
if not (output_db_url.startswith("mysql:") or output_db_url.startswith("sqlite:")):
logger.error("Unable to find database file from the properties file ")
return None, None
else:
# Ok, the default case is a .stampede.db file with the dag name as base
dag_file_name = ''
if (top_level_wf_params.has_key('dag')):
dag_file_name = top_level_wf_params['dag']
else:
logger.error("dag file name cannot be found in the braindump.txt")
return None, None
# Create the sqllite db url
output_db_file = (top_dir or submit_dir) + "/" + dag_file_name[:dag_file_name.find(".dag")] + ".stampede.db"
output_db_url = "sqlite:///" + output_db_file
if not os.path.isfile(output_db_file):
logger.error("Unable to find database file in " + (top_dir or submit_dir))
return None, None
# Ok, all done!
return output_db_url, wf_uuid
def get_db_url_wf_uuid(submit_dir, config_properties, top_dir=None):
"""
Utility method for returning the db_url and wf_uuid given the submit_dir and pegasus properties file.
@submit_dir submit directory path
@config_properties config properties file path
@top_dir directory of the top-level workflow (where the database is)
"""
# From the submit dir, we need the wf_uuid
# Getting values from the submit_dir braindump file
top_level_wf_params = utils.slurp_braindb(submit_dir)
# Return if we cannot parse the braindump.txt file
if not top_level_wf_params:
logger.error("Unable to process braindump.txt in %s" % (submit_dir))
return None, None
# Get wf_uuid for this workflow
wf_uuid = None
if (top_level_wf_params.has_key('wf_uuid')):
wf_uuid = top_level_wf_params['wf_uuid']
else:
logger.error("workflow id cannot be found in the braindump.txt ")
return None, None
# Load the top-level braindump now if top_dir is not None
if top_dir is not None:
# Getting values from the top_dir braindump file
top_level_wf_params = utils.slurp_braindb(top_dir)
# Return if we cannot parse the braindump.txt file
if not top_level_wf_params:
logger.error("Unable to process braindump.txt in %s" % (top_dir))
return None, None
# Get the location of the properties file from braindump
top_level_prop_file = None
# Get properties tag from braindump
if "properties" in top_level_wf_params:
top_level_prop_file = top_level_wf_params["properties"]
# Create the full path by using the submit_dir key from braindump
if "submit_dir" in top_level_wf_params:
top_level_prop_file = os.path.join(top_level_wf_params["submit_dir"], top_level_prop_file)
# Parse, and process properties
props = properties.Properties()
props.new(config_file=config_properties, rundir_propfile=top_level_prop_file)
# Ok, now figure out the database URL
output_db_url = None
if props.property('pegasus.monitord.output') is not None:
output_db_url = props.property('pegasus.monitord.output')
# Return, if not using sqlite or mysql
if not (output_db_url.startswith("mysql:") or output_db_url.startswith("sqlite:")):
logger.error("Unable to find database file from the properties file ")
return None, None
else:
# Ok, the default case is a .stampede.db file with the dag name as base
dag_file_name = ''
if (top_level_wf_params.has_key('dag')):
dag_file_name = top_level_wf_params['dag']
else:
logger.error("dag file name cannot be found in the braindump.txt")
return None, None
# Create the sqllite db url
output_db_file = (top_dir or submit_dir) + "/" + dag_file_name[:dag_file_name.find(".dag")] + ".stampede.db"
output_db_url = "sqlite:///" + output_db_file
if not os.path.isfile(output_db_file):
logger.error("Unable to find database file in " + (top_dir or submit_dir))
return None, None
# Ok, all done!
return output_db_url, wf_uuid