def _getDomFromFile(self, session, fileName, parser=''): """Read, parse and return configuration from a file. Read in an XML file from disk to get the configuration for this object. """ # We need to be able to read in configurations from disk if not os.path.exists(fileName): raise FileDoesNotExistException(fileName) f = open(fileName, 'r') doc = BootstrapDocument(f) # Look on self for instantiated parser, otherwise use bootstrap p = self.get_path(session, 'parser', None) try: if (p is not None): record = p.process_document(session, doc) elif parser == 'minidom': record = BSParser.process_document(session, doc) else: record = BSLxmlParser.process_document(session, doc) except Exception as e: raise ConfigFileException("Cannot parse %s: %s" % (fileName, e)) dom = record.get_dom(session) f.close() return dom
def _getDomFromUrl(self, session, url, parser=''): """Read, parse and return configuration from a file. Read in an XML file to get the configuration for this object. """ # We need to be able to read in configurations urlparts = urlsplit(url) if urlparts.scheme in ('http', 'https'): f = urllib2.urlopen(url) elif urlparts.scheme == "irods": try: from cheshire3.grid.irods_utils import open_irodsUrl except ImportError: self.log_error(session, "Unable to include file at {0}, " "Missing Dependency: irods (PyRods)" ) return else: f = open_irodsUrl(url) if not f: self.log_error(session, "Unable to include file at {0}, " "File not found".format(url) ) return else: if not os.path.isfile(urlparts.path): raise FileDoesNotExistException(urlparts.path) f = open(urlparts.path, 'r') doc = BootstrapDocument(f) # Look on self for instantiated parser, otherwise use bootstrap p = self.get_path(session, 'parser', None) try: if (p is not None): record = p.process_document(session, doc) elif parser == 'minidom': record = BSParser.process_document(session, doc) else: record = BSLxmlParser.process_document(session, doc) except Exception as e: raise ConfigFileException("Cannot parse %s: %s" % (url, e)) dom = record.get_dom(session) f.close() return dom
def identify_database(session, cwd): """Identify and return identifier of current database. Walk up directories looking for a .cheshire3 directory or config.xml file. Raise an error if current working directory is not part of a Cheshire3 database. """ # Establish upper boundary directory beyond which we shouldn't look boundary = os.path.abspath(os.path.expanduser('~')) out_of_bounds_msg = textwrap.fill( "Current working directory is not part of a Cheshire3 database. " "Refusing to look any further up the directory tree than: {0} " "Please provide a database identifier using the --database option." "".format(boundary) ) while True: if ( cwd == os.path.split(boundary)[0] or len(cwd) < len(boundary) ): raise EnvironmentError(out_of_bounds_msg) c3_dir = os.path.join(cwd, '.cheshire3') if os.path.exists(c3_dir) and os.path.isdir(c3_dir): conf_file_path = os.path.join(c3_dir, 'config.xml') else: # No .cheshire3 directory - maybe old-style database conf_file_path = os.path.join(cwd, 'config.xml') try: with open(conf_file_path, 'r') as cfh: doc = BootstrapDocument(cfh) record = BSLxmlParser.process_document(session, doc) except IOError: try: cwd, foo = os.path.split(cwd) except: raise EnvironmentError(out_of_bounds_msg) else: del foo else: dom = record.get_dom(session) dbid = dom.attrib['id'] return dbid
def identify_database(session, cwd): """Identify and return identifier of current database. Walk up directories looking for a .cheshire3 directory or config.xml file. Raise an error if current working directory is not part of a Cheshire3 database. """ # Establish upper boundary directory beyond which we shouldn't look boundary = os.path.abspath(os.path.expanduser('~')) out_of_bounds_msg = textwrap.fill( "Current working directory is not part of a Cheshire3 database. " "Refusing to look any further up the directory tree than: {0} " "Please provide a database identifier using the --database option." "".format(boundary)) while True: if (cwd == os.path.split(boundary)[0] or len(cwd) < len(boundary)): raise EnvironmentError(out_of_bounds_msg) c3_dir = os.path.join(cwd, '.cheshire3') if os.path.exists(c3_dir) and os.path.isdir(c3_dir): conf_file_path = os.path.join(c3_dir, 'config.xml') else: # No .cheshire3 directory - maybe old-style database conf_file_path = os.path.join(cwd, 'config.xml') try: with open(conf_file_path, 'r') as cfh: doc = BootstrapDocument(cfh) record = BSLxmlParser.process_document(session, doc) except IOError: try: cwd, foo = os.path.split(cwd) except: raise EnvironmentError(out_of_bounds_msg) else: del foo else: dom = record.get_dom(session) dbid = dom.attrib['id'] return dbid
def _process_data(self, session, id, data, parser=None): # Split from fetch record for Iterators doc = StringDocument(data) if (parser is not None): rec = parser.process_document(session, doc) elif (self.outParser is not None): rec = self.outParser.process_document(session, doc) elif (self.outWorkflow is not None): rec = self.outWorkflow.process(session, doc) else: # Assume raw XML into LXML # try and set self.parser to an LxmlParser try: p = session.server.get_object(session, 'LxmlParser') self.parser = p rec = p.process_document(session, doc) except: rec = BSLxmlParser.process_document(session, doc) # Ensure basic required info rec.id = id rec.recordStore = self.id return rec
def _process_data(self, session, id, data, parser=None): # Split from fetch record for Iterators doc = StringDocument(data) if (parser != None): rec = parser.process_document(session, doc) elif (self.outParser != None): rec = self.outParser.process_document(session, doc) elif (self.outWorkflow != None): rec = self.outWorkflow.process(session, doc) else: # Assume raw XML into LXML # try and set self.parser to an LxmlParser try: p = session.server.get_object(session, 'LxmlParser') self.parser = p rec = p.process_document(session, doc) except: rec = BSLxmlParser.process_document(session, doc) # Ensure basic required info rec.id = id rec.recordStore = self.id return rec
def register_databaseConfigFile(self, session, file_path): """Register a Cheshire3 Database config file. Register a configuration file for a Cheshire3 :py:class:`~cheshire3.baseObjects.Database` with the :py:class:`~cheshire3.baseObjects.Server`. This process simply tells the :py:class:`~cheshire3.baseObjects.Server` that it should include the configuration(s) in your file (it does not ingest your file) so you don't need to re-register when you make changes to the file. """ # Read in proposed config file docFac = self.get_object(session, 'defaultDocumentFactory') docFac.load(session, file_path) confdoc = docFac.get_document(session) try: confrec = BSLxmlParser.process_document(session, confdoc) except XMLSyntaxError as e: msg = ("Config file {0} is not well-formed and valid XML: " "{1}".format(file_path, e.message)) self.log_critical(session, msg) raise ConfigFileException(msg) # Extract the database identifier confdom = confrec.get_dom(session) dbid = self._get_newDatabaseId(session, confdom) if dbid is None: msg = ("Config file {0} must have an 'id' attribute at the " "top-level".format(file_path)) self.log_critical(session, msg) raise ConfigFileException(msg) # Generate plugin XML plugin = E.config( E.subConfigs( E.path( {'type': "database", 'id': dbid}, file_path ) ) ) # Try to do this by writing config plugin file if possible serverDefaultPath = self.get_path(session, 'defaultPath', cheshire3Root) userSpecificPath = os.path.join(os.path.expanduser('~'), '.cheshire3-server') pluginPath = os.path.join('configs', 'databases', '{0}.xml'.format(dbid)) try: pluginfh = open(os.path.join(serverDefaultPath, pluginPath), 'w') except IOError: try: pluginfh = open(os.path.join(userSpecificPath, pluginPath), 'w') except IOError: msg = ("Database plugin directory {0} unavailable for writing" "".format(os.path.join(userSpecificPath, pluginPath))) self.log_critical(session, msg) raise FileSystemException(msg) pluginfh.write( etree.tostring( plugin, pretty_print=True, encoding="utf-8" ) ) pluginfh.close() self.log_info(session, "Database configured in {0} registered with Cheshire3 " "Server {1}".format(file_path, self.id))
def register_databaseConfigFile(self, session, file_path): """Register a Cheshire3 Database config file. Register a configuration file for a Cheshire3 Database with the server. This process simply tells the server that it should include the configuration(s) in your file (it does not ingest your file) so you don't need to re-register when you make changes to the file. """ # Read in proposed config file with open(file_path, 'r') as fh: confdoc = BootstrapDocument(fh) # Check it's parsable try: confrec = BSLxmlParser.process_document(session, confdoc) except XMLSyntaxError as e: msg = ("Config file {0} is not well-formed and valid XML: " "{1}".format(file_path, e.message)) self.log_critical(session, msg) raise ConfigFileException(msg) # Extract the database identifier confdom = confrec.get_dom(session) dbid = confdom.attrib.get('id', None) if dbid is None: msg = ("Config file {0} must have an 'id' attribute at the " "top-level".format(file_path)) self.log_critical(session, msg) raise ConfigFileException(msg) # Check that the identifier is not already in use by existing database try: self.get_object(session, dbid) except ObjectDoesNotExistException: # Doesn't exists, so OK to register it pass else: msg = ("Database with id '{0}' is already registered. " "Please specify a different id in your configurations " "file.".format(dbid)) self.log_critical(session, msg) raise ConfigFileException(msg) # Generate plugin XML plugin = E.config( E.subConfigs(E.path({ 'type': "database", 'id': dbid }, file_path))) # Try to do this by writing config plugin file if possible serverDefaultPath = self.get_path(session, 'defaultPath', cheshire3Root) userSpecificPath = os.path.join(os.path.expanduser('~'), '.cheshire3-server') pluginPath = os.path.join('configs', 'databases', '{0}.xml'.format(dbid)) try: pluginfh = open(os.path.join(serverDefaultPath, pluginPath), 'w') except IOError: try: pluginfh = open(os.path.join(userSpecificPath, pluginPath), 'w') except IOError: msg = ("Database plugin directory {0} unavailable for writing" "".format(os.path.join(userSpecificPath, pluginPath))) self.log_critical(session, msg) raise FileSystemException(msg) pluginfh.write( etree.tostring(plugin, pretty_print=True, encoding="utf-8")) pluginfh.close() self.log_info( session, "Database configured in {0} registered with Cheshire3 " "Server {1}".format(file_path, self.id))
def register_databaseConfigFile(self, session, file_path): """Register a Cheshire3 Database config file. Register a configuration file for a Cheshire3 Database with the server. This process simply tells the server that it should include the configuration(s) in your file (it does not ingest your file) so you don't need to re-register when you make changes to the file. """ # Read in proposed config file with open(file_path, 'r') as fh: confdoc = BootstrapDocument(fh) # Check it's parsable try: confrec = BSLxmlParser.process_document(session, confdoc) except XMLSyntaxError as e: msg = ("Config file {0} is not well-formed and valid XML: " "{1}".format(file_path, e.message)) self.log_critical(session, msg) raise ConfigFileException(msg) # Extract the database identifier confdom = confrec.get_dom(session) dbid = confdom.attrib.get('id', None) if dbid is None: msg = ("Config file {0} must have an 'id' attribute at the " "top-level".format(file_path)) self.log_critical(session, msg) raise ConfigFileException(msg) # Check that the identifier is not already in use by existing database try: self.get_object(session, dbid) except ObjectDoesNotExistException: # Doesn't exists, so OK to register it pass else: msg = ("Database with id '{0}' is already registered. " "Please specify a different id in your configurations " "file.".format(dbid)) self.log_critical(session, msg) raise ConfigFileException(msg) # Generate plugin XML plugin = E.config( E.subConfigs( E.path({'type': "database", 'id': dbid}, file_path ) ) ) # Try to do this by writing config plugin file if possible serverDefaultPath = self.get_path(session, 'defaultPath', cheshire3Root) userSpecificPath = os.path.join(os.path.expanduser('~'), '.cheshire3-server') pluginPath = os.path.join('configs', 'databases', '{0}.xml'.format(dbid)) try: pluginfh = open(os.path.join(serverDefaultPath, pluginPath), 'w') except IOError: try: pluginfh = open(os.path.join(userSpecificPath, pluginPath), 'w') except IOError: msg = ("Database plugin directory {0} unavailable for writing" "".format(os.path.join(userSpecificPath, pluginPath))) self.log_critical(session, msg) raise FileSystemException(msg) pluginfh.write(etree.tostring(plugin, pretty_print=True, encoding="utf-8")) pluginfh.close() self.log_info(session, "Database configured in {0} registered with Cheshire3 " "Server {1}".format(file_path, self.id))