def connect_by_properties(
config_properties,
db_type,
cl_properties=None,
echo=False,
schema_check=True,
create=False,
pegasus_version=None,
force=False,
verbose=True,
):
""" Connect to the database from properties file and database type """
props = properties.Properties()
props.new(config_file=config_properties)
_merge_properties(props, cl_properties)
dburi = url_by_properties(config_properties, db_type, props=props)
return connect(
dburi,
echo,
schema_check,
create=create,
pegasus_version=pegasus_version,
force=force,
db_type=db_type,
props=props,
verbose=verbose,
)
def url_by_properties(config_properties, db_type, submit_dir=None, top_dir=None, rundir_properties=None,
cl_properties=None, props=None):
""" Get URL from the property file """
# Validate parameters
if not db_type:
raise ConnectionError("A type should be provided with the property file.", db_type=db_type)
# Parse, and process properties
if not props:
props = properties.Properties()
props.new(config_file=config_properties, rundir_propfile=rundir_properties)
_merge_properties(props, cl_properties)
if db_type.upper() == DBType.JDBCRC:
dburi = _get_jdbcrc_uri(props)
elif db_type.upper() == DBType.MASTER:
dburi = _get_master_uri(props)
elif db_type.upper() == DBType.WORKFLOW:
dburi = _get_workflow_uri(props, submit_dir, top_dir)
else:
raise ConnectionError("Invalid database type '%s'." % db_type, db_type=db_type)
if dburi:
log.debug("Using database: %s" % dburi)
return dburi
raise ConnectionError("Unable to find a database URI to connect.", db_type=db_type)
def get_workflow_connect_props(props, db_type):
"""
Returns connection properties for workflow database
:param props:
:return:
"""
if props is None:
return None
# first get the default one's with the star notation
connect_props = properties.Properties(
props.propertyset("pegasus.catalog.*.", True))
prefix = "pegasus.catalog.workflow"
if db_type == connection.DBType.MASTER:
prefix = "pegasus.catalog.dashboard"
# over ride these with workflow specific or dashboard specific props
addons = props.propertyset(prefix + ".", True)
for key in addons:
connect_props.property(key, addons[key])
return connect_props
def __init__(self, dest, enc, prefix=STAMPEDE_NS, props=None, **kw):
super(MultiplexEventSink, self).__init__()
self._endpoints = {}
self._log.info("Multiplexed Event Sink Connection Properties %s",
props)
for key in props.keyset():
if key.endswith(".url"):
sink_name = key[0:key.rfind(".url")]
# remove from our copy sink_name properties if they exist
endpoint_props = properties.Properties(
props.propertyset(sink_name + ".", remove=True))
try:
self._endpoints[sink_name] = create_wf_event_sink(
props.property(key),
db_type=connection.DBType.WORKFLOW,
enc=enc,
prefix=prefix,
props=endpoint_props,
multiplexed=True,
**kw)
except:
self._log.error(
"[multiplex event sender] Unable to connect to endpoint %s with props %s . Disabling"
% (sink_name, endpoint_props))
self._log.error(traceback.format_exc())
def __init__(self, dest, enc, prefix=STAMPEDE_NS, props=None, **kw):
super(MultiplexEventSink, self).__init__()
self._endpoints = {}
additional_sink_props = props.propertyset(
"pegasus.catalog.workflow" + ".", True)
# we delete from our copy pegasus.catalog.workflow.url as we want with default prefix
if "url" in additional_sink_props.keys():
del additional_sink_props["url"]
additional_sink_props["default.url"] = dest
for key in additional_sink_props:
if key.endswith(".url"):
# remove from our copy pegasus.catalog.workflow.url if exists
endpoint_props = properties.Properties(
props.propertyset("pegasus.catalog.workflow" + ".", False))
endpoint_props.remove("pegasus.catalog.workflow.url")
self._endpoints[
key[0:key.rfind(".url")]] = create_wf_event_sink(
additional_sink_props[key],
enc=enc,
prefix=prefix,
props=endpoint_props,
multiplexed=True,
**kw)
def get_db_url(config_properties):
# Parse, and process properties
props = properties.Properties()
props.new(config_file=config_properties)
# Ok, now figure out the database URL
output_db_url = None
if props.property('pegasus.monitord.output') is not None:
output_db_url = props.property('pegasus.monitord.output')
# Return, if not using sqlite or mysql
if not (output_db_url.startswith("mysql:") or output_db_url.startswith("sqlite:")):
logger.error("Unable to find database file from the properties file ")
return None
return output_db_url
return None
def get_db_url_wf_uuid(submit_dir, config_properties, top_dir=None):
"""
Utility method for returning the db_url and wf_uuid given the submit_dir and pegasus properties file.
@submit_dir submit directory path
@config_properties config properties file path
@top_dir directory of the top-level workflow (where the database is)
"""
# From the submit dir, we need the wf_uuid
# Getting values from the submit_dir braindump file
top_level_wf_params = utils.slurp_braindb(submit_dir)
# Return if we cannot parse the braindump.txt file
if not top_level_wf_params:
logger.error("Unable to process braindump.txt in %s" % (submit_dir))
return None, None
# Get wf_uuid for this workflow
wf_uuid = None
if (top_level_wf_params.has_key('wf_uuid')):
wf_uuid = top_level_wf_params['wf_uuid']
else:
logger.error("workflow id cannot be found in the braindump.txt ")
return None, None
# Load the top-level braindump now if top_dir is not None
if top_dir is not None:
# Getting values from the top_dir braindump file
top_level_wf_params = utils.slurp_braindb(top_dir)
# Return if we cannot parse the braindump.txt file
if not top_level_wf_params:
logger.error("Unable to process braindump.txt in %s" % (top_dir))
return None, None
# Get the location of the properties file from braindump
top_level_prop_file = None
# Get properties tag from braindump
if "properties" in top_level_wf_params:
top_level_prop_file = top_level_wf_params["properties"]
# Create the full path by using the submit_dir key from braindump
if "submit_dir" in top_level_wf_params:
top_level_prop_file = os.path.join(top_level_wf_params["submit_dir"], top_level_prop_file)
# Parse, and process properties
props = properties.Properties()
props.new(config_file=config_properties, rundir_propfile=top_level_prop_file)
# Ok, now figure out the database URL
output_db_url = None
if props.property('pegasus.monitord.output') is not None:
output_db_url = props.property('pegasus.monitord.output')
# Return, if not using sqlite or mysql
if not (output_db_url.startswith("mysql:") or output_db_url.startswith("sqlite:")):
logger.error("Unable to find database file from the properties file ")
return None, None
else:
# Ok, the default case is a .stampede.db file with the dag name as base
dag_file_name = ''
if (top_level_wf_params.has_key('dag')):
dag_file_name = top_level_wf_params['dag']
else:
logger.error("dag file name cannot be found in the braindump.txt")
return None, None
# Create the sqllite db url
output_db_file = (top_dir or submit_dir) + "/" + dag_file_name[:dag_file_name.find(".dag")] + ".stampede.db"
output_db_url = "sqlite:///" + output_db_file
if not os.path.isfile(output_db_file):
logger.error("Unable to find database file in " + (top_dir or submit_dir))
return None, None
# Ok, all done!
return output_db_url, wf_uuid
