def _getDomFromFile(self, session, fileName, parser=''):
"""Read, parse and return configuration from a file.
Read in an XML file from disk to get the configuration for this
object.
"""
# We need to be able to read in configurations from disk
if not os.path.exists(fileName):
raise FileDoesNotExistException(fileName)
f = open(fileName, 'r')
doc = BootstrapDocument(f)
# Look on self for instantiated parser, otherwise use bootstrap
p = self.get_path(session, 'parser', None)
try:
if (p is not None):
record = p.process_document(session, doc)
elif parser == 'minidom':
record = BSParser.process_document(session, doc)
else:
record = BSLxmlParser.process_document(session, doc)
except Exception as e:
raise ConfigFileException("Cannot parse %s: %s" % (fileName, e))
dom = record.get_dom(session)
f.close()
return dom
def _getDomFromFile(self, session, fileName, parser=''):
"""Read, parse and return configuration from a file.
Read in an XML file from disk to get the configuration for this
object.
"""
# We need to be able to read in configurations from disk
if not os.path.exists(fileName):
raise FileDoesNotExistException(fileName)
f = open(fileName, 'r')
doc = BootstrapDocument(f)
# Look on self for instantiated parser, otherwise use bootstrap
p = self.get_path(session, 'parser', None)
try:
if (p is not None):
record = p.process_document(session, doc)
elif parser == 'minidom':
record = BSParser.process_document(session, doc)
else:
record = BSLxmlParser.process_document(session, doc)
except Exception as e:
raise ConfigFileException("Cannot parse %s: %s" % (fileName, e))
dom = record.get_dom(session)
f.close()
return dom
def _getDomFromUrl(self, session, url, parser=''):
"""Read, parse and return configuration from a file.
Read in an XML file to get the configuration for this object.
"""
# We need to be able to read in configurations
urlparts = urlsplit(url)
if urlparts.scheme in ('http', 'https'):
f = urllib2.urlopen(url)
elif urlparts.scheme == "irods":
try:
from cheshire3.grid.irods_utils import open_irodsUrl
except ImportError:
self.log_error(session,
"Unable to include file at {0}, "
"Missing Dependency: irods (PyRods)"
)
return
else:
f = open_irodsUrl(url)
if not f:
self.log_error(session,
"Unable to include file at {0}, "
"File not found".format(url)
)
return
else:
if not os.path.isfile(urlparts.path):
raise FileDoesNotExistException(urlparts.path)
f = open(urlparts.path, 'r')
doc = BootstrapDocument(f)
# Look on self for instantiated parser, otherwise use bootstrap
p = self.get_path(session, 'parser', None)
try:
if (p is not None):
record = p.process_document(session, doc)
elif parser == 'minidom':
record = BSParser.process_document(session, doc)
else:
record = BSLxmlParser.process_document(session, doc)
except Exception as e:
raise ConfigFileException("Cannot parse %s: %s" % (url, e))
dom = record.get_dom(session)
f.close()
return dom
def identify_database(session, cwd):
"""Identify and return identifier of current database.
Walk up directories looking for a .cheshire3 directory or config.xml file.
Raise an error if current working directory is not part of a Cheshire3
database.
"""
# Establish upper boundary directory beyond which we shouldn't look
boundary = os.path.abspath(os.path.expanduser('~'))
out_of_bounds_msg = textwrap.fill(
"Current working directory is not part of a Cheshire3 database. "
"Refusing to look any further up the directory tree than: {0} "
"Please provide a database identifier using the --database option."
"".format(boundary)
)
while True:
if (
cwd == os.path.split(boundary)[0] or
len(cwd) < len(boundary)
):
raise EnvironmentError(out_of_bounds_msg)
c3_dir = os.path.join(cwd, '.cheshire3')
if os.path.exists(c3_dir) and os.path.isdir(c3_dir):
conf_file_path = os.path.join(c3_dir, 'config.xml')
else:
# No .cheshire3 directory - maybe old-style database
conf_file_path = os.path.join(cwd, 'config.xml')
try:
with open(conf_file_path, 'r') as cfh:
doc = BootstrapDocument(cfh)
record = BSLxmlParser.process_document(session, doc)
except IOError:
try:
cwd, foo = os.path.split(cwd)
except:
raise EnvironmentError(out_of_bounds_msg)
else:
del foo
else:
dom = record.get_dom(session)
dbid = dom.attrib['id']
return dbid
def identify_database(session, cwd):
"""Identify and return identifier of current database.
Walk up directories looking for a .cheshire3 directory or config.xml file.
Raise an error if current working directory is not part of a Cheshire3
database.
"""
# Establish upper boundary directory beyond which we shouldn't look
boundary = os.path.abspath(os.path.expanduser('~'))
out_of_bounds_msg = textwrap.fill(
"Current working directory is not part of a Cheshire3 database. "
"Refusing to look any further up the directory tree than: {0} "
"Please provide a database identifier using the --database option."
"".format(boundary))
while True:
if (cwd == os.path.split(boundary)[0] or len(cwd) < len(boundary)):
raise EnvironmentError(out_of_bounds_msg)
c3_dir = os.path.join(cwd, '.cheshire3')
if os.path.exists(c3_dir) and os.path.isdir(c3_dir):
conf_file_path = os.path.join(c3_dir, 'config.xml')
else:
# No .cheshire3 directory - maybe old-style database
conf_file_path = os.path.join(cwd, 'config.xml')
try:
with open(conf_file_path, 'r') as cfh:
doc = BootstrapDocument(cfh)
record = BSLxmlParser.process_document(session, doc)
except IOError:
try:
cwd, foo = os.path.split(cwd)
except:
raise EnvironmentError(out_of_bounds_msg)
else:
del foo
else:
dom = record.get_dom(session)
dbid = dom.attrib['id']
return dbid
def _process_data(self, session, id, data, parser=None):
# Split from fetch record for Iterators
doc = StringDocument(data)
if (parser is not None):
rec = parser.process_document(session, doc)
elif (self.outParser is not None):
rec = self.outParser.process_document(session, doc)
elif (self.outWorkflow is not None):
rec = self.outWorkflow.process(session, doc)
else:
# Assume raw XML into LXML
# try and set self.parser to an LxmlParser
try:
p = session.server.get_object(session, 'LxmlParser')
self.parser = p
rec = p.process_document(session, doc)
except:
rec = BSLxmlParser.process_document(session, doc)
# Ensure basic required info
rec.id = id
rec.recordStore = self.id
return rec
def _process_data(self, session, id, data, parser=None):
# Split from fetch record for Iterators
doc = StringDocument(data)
if (parser != None):
rec = parser.process_document(session, doc)
elif (self.outParser != None):
rec = self.outParser.process_document(session, doc)
elif (self.outWorkflow != None):
rec = self.outWorkflow.process(session, doc)
else:
# Assume raw XML into LXML
# try and set self.parser to an LxmlParser
try:
p = session.server.get_object(session, 'LxmlParser')
self.parser = p
rec = p.process_document(session, doc)
except:
rec = BSLxmlParser.process_document(session, doc)
# Ensure basic required info
rec.id = id
rec.recordStore = self.id
return rec
def register_databaseConfigFile(self, session, file_path):
"""Register a Cheshire3 Database config file.
Register a configuration file for a Cheshire3
:py:class:`~cheshire3.baseObjects.Database` with the
:py:class:`~cheshire3.baseObjects.Server`.
This process simply tells the :py:class:`~cheshire3.baseObjects.Server`
that it should include the configuration(s) in your file (it does not
ingest your file) so you don't need to re-register when you make
changes to the file.
"""
# Read in proposed config file
docFac = self.get_object(session, 'defaultDocumentFactory')
docFac.load(session, file_path)
confdoc = docFac.get_document(session)
try:
confrec = BSLxmlParser.process_document(session, confdoc)
except XMLSyntaxError as e:
msg = ("Config file {0} is not well-formed and valid XML: "
"{1}".format(file_path, e.message))
self.log_critical(session, msg)
raise ConfigFileException(msg)
# Extract the database identifier
confdom = confrec.get_dom(session)
dbid = self._get_newDatabaseId(session, confdom)
if dbid is None:
msg = ("Config file {0} must have an 'id' attribute at the "
"top-level".format(file_path))
self.log_critical(session, msg)
raise ConfigFileException(msg)
# Generate plugin XML
plugin = E.config(
E.subConfigs(
E.path(
{'type': "database", 'id': dbid},
file_path
)
)
)
# Try to do this by writing config plugin file if possible
serverDefaultPath = self.get_path(session,
'defaultPath',
cheshire3Root)
userSpecificPath = os.path.join(os.path.expanduser('~'),
'.cheshire3-server')
pluginPath = os.path.join('configs',
'databases',
'{0}.xml'.format(dbid))
try:
pluginfh = open(os.path.join(serverDefaultPath, pluginPath), 'w')
except IOError:
try:
pluginfh = open(os.path.join(userSpecificPath, pluginPath),
'w')
except IOError:
msg = ("Database plugin directory {0} unavailable for writing"
"".format(os.path.join(userSpecificPath, pluginPath)))
self.log_critical(session, msg)
raise FileSystemException(msg)
pluginfh.write(
etree.tostring(
plugin,
pretty_print=True,
encoding="utf-8"
)
)
pluginfh.close()
self.log_info(session,
"Database configured in {0} registered with Cheshire3 "
"Server {1}".format(file_path, self.id))
def register_databaseConfigFile(self, session, file_path):
"""Register a Cheshire3 Database config file.
Register a configuration file for a Cheshire3 Database with
the server.
This process simply tells the server that it should include the
configuration(s) in your file (it does not ingest your file) so
you don't need to re-register when you make changes to the file.
"""
# Read in proposed config file
with open(file_path, 'r') as fh:
confdoc = BootstrapDocument(fh)
# Check it's parsable
try:
confrec = BSLxmlParser.process_document(session, confdoc)
except XMLSyntaxError as e:
msg = ("Config file {0} is not well-formed and valid XML: "
"{1}".format(file_path, e.message))
self.log_critical(session, msg)
raise ConfigFileException(msg)
# Extract the database identifier
confdom = confrec.get_dom(session)
dbid = confdom.attrib.get('id', None)
if dbid is None:
msg = ("Config file {0} must have an 'id' attribute at the "
"top-level".format(file_path))
self.log_critical(session, msg)
raise ConfigFileException(msg)
# Check that the identifier is not already in use by existing database
try:
self.get_object(session, dbid)
except ObjectDoesNotExistException:
# Doesn't exists, so OK to register it
pass
else:
msg = ("Database with id '{0}' is already registered. "
"Please specify a different id in your configurations "
"file.".format(dbid))
self.log_critical(session, msg)
raise ConfigFileException(msg)
# Generate plugin XML
plugin = E.config(
E.subConfigs(E.path({
'type': "database",
'id': dbid
}, file_path)))
# Try to do this by writing config plugin file if possible
serverDefaultPath = self.get_path(session, 'defaultPath',
cheshire3Root)
userSpecificPath = os.path.join(os.path.expanduser('~'),
'.cheshire3-server')
pluginPath = os.path.join('configs', 'databases',
'{0}.xml'.format(dbid))
try:
pluginfh = open(os.path.join(serverDefaultPath, pluginPath), 'w')
except IOError:
try:
pluginfh = open(os.path.join(userSpecificPath, pluginPath),
'w')
except IOError:
msg = ("Database plugin directory {0} unavailable for writing"
"".format(os.path.join(userSpecificPath, pluginPath)))
self.log_critical(session, msg)
raise FileSystemException(msg)
pluginfh.write(
etree.tostring(plugin, pretty_print=True, encoding="utf-8"))
pluginfh.close()
self.log_info(
session, "Database configured in {0} registered with Cheshire3 "
"Server {1}".format(file_path, self.id))
def register_databaseConfigFile(self, session, file_path):
"""Register a Cheshire3 Database config file.
Register a configuration file for a Cheshire3 Database with
the server.
This process simply tells the server that it should include the
configuration(s) in your file (it does not ingest your file) so
you don't need to re-register when you make changes to the file.
"""
# Read in proposed config file
with open(file_path, 'r') as fh:
confdoc = BootstrapDocument(fh)
# Check it's parsable
try:
confrec = BSLxmlParser.process_document(session, confdoc)
except XMLSyntaxError as e:
msg = ("Config file {0} is not well-formed and valid XML: "
"{1}".format(file_path, e.message))
self.log_critical(session, msg)
raise ConfigFileException(msg)
# Extract the database identifier
confdom = confrec.get_dom(session)
dbid = confdom.attrib.get('id', None)
if dbid is None:
msg = ("Config file {0} must have an 'id' attribute at the "
"top-level".format(file_path))
self.log_critical(session, msg)
raise ConfigFileException(msg)
# Check that the identifier is not already in use by existing database
try:
self.get_object(session, dbid)
except ObjectDoesNotExistException:
# Doesn't exists, so OK to register it
pass
else:
msg = ("Database with id '{0}' is already registered. "
"Please specify a different id in your configurations "
"file.".format(dbid))
self.log_critical(session, msg)
raise ConfigFileException(msg)
# Generate plugin XML
plugin = E.config(
E.subConfigs(
E.path({'type': "database", 'id': dbid},
file_path
)
)
)
# Try to do this by writing config plugin file if possible
serverDefaultPath = self.get_path(session,
'defaultPath',
cheshire3Root)
userSpecificPath = os.path.join(os.path.expanduser('~'),
'.cheshire3-server')
pluginPath = os.path.join('configs',
'databases',
'{0}.xml'.format(dbid))
try:
pluginfh = open(os.path.join(serverDefaultPath, pluginPath), 'w')
except IOError:
try:
pluginfh = open(os.path.join(userSpecificPath, pluginPath),
'w')
except IOError:
msg = ("Database plugin directory {0} unavailable for writing"
"".format(os.path.join(userSpecificPath, pluginPath)))
self.log_critical(session, msg)
raise FileSystemException(msg)
pluginfh.write(etree.tostring(plugin,
pretty_print=True,
encoding="utf-8"))
pluginfh.close()
self.log_info(session,
"Database configured in {0} registered with Cheshire3 "
"Server {1}".format(file_path, self.id))