def connect_by_properties( config_properties, db_type, cl_properties=None, echo=False, schema_check=True, create=False, pegasus_version=None, force=False, verbose=True, ): """ Connect to the database from properties file and database type """ props = properties.Properties() props.new(config_file=config_properties) _merge_properties(props, cl_properties) dburi = url_by_properties(config_properties, db_type, props=props) return connect( dburi, echo, schema_check, create=create, pegasus_version=pegasus_version, force=force, db_type=db_type, props=props, verbose=verbose, )
def url_by_properties(config_properties, db_type, submit_dir=None, top_dir=None, rundir_properties=None, cl_properties=None, props=None): """ Get URL from the property file """ # Validate parameters if not db_type: raise ConnectionError("A type should be provided with the property file.", db_type=db_type) # Parse, and process properties if not props: props = properties.Properties() props.new(config_file=config_properties, rundir_propfile=rundir_properties) _merge_properties(props, cl_properties) if db_type.upper() == DBType.JDBCRC: dburi = _get_jdbcrc_uri(props) elif db_type.upper() == DBType.MASTER: dburi = _get_master_uri(props) elif db_type.upper() == DBType.WORKFLOW: dburi = _get_workflow_uri(props, submit_dir, top_dir) else: raise ConnectionError("Invalid database type '%s'." % db_type, db_type=db_type) if dburi: log.debug("Using database: %s" % dburi) return dburi raise ConnectionError("Unable to find a database URI to connect.", db_type=db_type)
def get_workflow_connect_props(props, db_type): """ Returns connection properties for workflow database :param props: :return: """ if props is None: return None # first get the default one's with the star notation connect_props = properties.Properties( props.propertyset("pegasus.catalog.*.", True)) prefix = "pegasus.catalog.workflow" if db_type == connection.DBType.MASTER: prefix = "pegasus.catalog.dashboard" # over ride these with workflow specific or dashboard specific props addons = props.propertyset(prefix + ".", True) for key in addons: connect_props.property(key, addons[key]) return connect_props
def __init__(self, dest, enc, prefix=STAMPEDE_NS, props=None, **kw): super(MultiplexEventSink, self).__init__() self._endpoints = {} self._log.info("Multiplexed Event Sink Connection Properties %s", props) for key in props.keyset(): if key.endswith(".url"): sink_name = key[0:key.rfind(".url")] # remove from our copy sink_name properties if they exist endpoint_props = properties.Properties( props.propertyset(sink_name + ".", remove=True)) try: self._endpoints[sink_name] = create_wf_event_sink( props.property(key), db_type=connection.DBType.WORKFLOW, enc=enc, prefix=prefix, props=endpoint_props, multiplexed=True, **kw) except: self._log.error( "[multiplex event sender] Unable to connect to endpoint %s with props %s . Disabling" % (sink_name, endpoint_props)) self._log.error(traceback.format_exc())
def __init__(self, dest, enc, prefix=STAMPEDE_NS, props=None, **kw): super(MultiplexEventSink, self).__init__() self._endpoints = {} additional_sink_props = props.propertyset( "pegasus.catalog.workflow" + ".", True) # we delete from our copy pegasus.catalog.workflow.url as we want with default prefix if "url" in additional_sink_props.keys(): del additional_sink_props["url"] additional_sink_props["default.url"] = dest for key in additional_sink_props: if key.endswith(".url"): # remove from our copy pegasus.catalog.workflow.url if exists endpoint_props = properties.Properties( props.propertyset("pegasus.catalog.workflow" + ".", False)) endpoint_props.remove("pegasus.catalog.workflow.url") self._endpoints[ key[0:key.rfind(".url")]] = create_wf_event_sink( additional_sink_props[key], enc=enc, prefix=prefix, props=endpoint_props, multiplexed=True, **kw)
def get_db_url(config_properties): # Parse, and process properties props = properties.Properties() props.new(config_file=config_properties) # Ok, now figure out the database URL output_db_url = None if props.property('pegasus.monitord.output') is not None: output_db_url = props.property('pegasus.monitord.output') # Return, if not using sqlite or mysql if not (output_db_url.startswith("mysql:") or output_db_url.startswith("sqlite:")): logger.error("Unable to find database file from the properties file ") return None return output_db_url return None
def get_db_url_wf_uuid(submit_dir, config_properties, top_dir=None): """ Utility method for returning the db_url and wf_uuid given the submit_dir and pegasus properties file. @submit_dir submit directory path @config_properties config properties file path @top_dir directory of the top-level workflow (where the database is) """ # From the submit dir, we need the wf_uuid # Getting values from the submit_dir braindump file top_level_wf_params = utils.slurp_braindb(submit_dir) # Return if we cannot parse the braindump.txt file if not top_level_wf_params: logger.error("Unable to process braindump.txt in %s" % (submit_dir)) return None, None # Get wf_uuid for this workflow wf_uuid = None if (top_level_wf_params.has_key('wf_uuid')): wf_uuid = top_level_wf_params['wf_uuid'] else: logger.error("workflow id cannot be found in the braindump.txt ") return None, None # Load the top-level braindump now if top_dir is not None if top_dir is not None: # Getting values from the top_dir braindump file top_level_wf_params = utils.slurp_braindb(top_dir) # Return if we cannot parse the braindump.txt file if not top_level_wf_params: logger.error("Unable to process braindump.txt in %s" % (top_dir)) return None, None # Get the location of the properties file from braindump top_level_prop_file = None # Get properties tag from braindump if "properties" in top_level_wf_params: top_level_prop_file = top_level_wf_params["properties"] # Create the full path by using the submit_dir key from braindump if "submit_dir" in top_level_wf_params: top_level_prop_file = os.path.join(top_level_wf_params["submit_dir"], top_level_prop_file) # Parse, and process properties props = properties.Properties() props.new(config_file=config_properties, rundir_propfile=top_level_prop_file) # Ok, now figure out the database URL output_db_url = None if props.property('pegasus.monitord.output') is not None: output_db_url = props.property('pegasus.monitord.output') # Return, if not using sqlite or mysql if not (output_db_url.startswith("mysql:") or output_db_url.startswith("sqlite:")): logger.error("Unable to find database file from the properties file ") return None, None else: # Ok, the default case is a .stampede.db file with the dag name as base dag_file_name = '' if (top_level_wf_params.has_key('dag')): dag_file_name = top_level_wf_params['dag'] else: logger.error("dag file name cannot be found in the braindump.txt") return None, None # Create the sqllite db url output_db_file = (top_dir or submit_dir) + "/" + dag_file_name[:dag_file_name.find(".dag")] + ".stampede.db" output_db_url = "sqlite:///" + output_db_file if not os.path.isfile(output_db_file): logger.error("Unable to find database file in " + (top_dir or submit_dir)) return None, None # Ok, all done! return output_db_url, wf_uuid