def _override_settings(config_file, settings, arguments):
''' Parse optional config file and change arguments accordingly '''
config = ConfigParser.ConfigParser()
try:
config.read(config_file)
except ConfigParser.ParsingError:
raise ConfigParser.ParsingError(
"Unable to parse the given configuration file ({})".format(
config_file))
# Replace input files with those given in the config file
#if config.get('General', 'FILE1') != '' and config.get('General', 'FILE2') != '':
# arguments = [config.get('General', 'FILE1'), config.get('General', 'FILE2')]
# config.remove_option('General', 'FILE1')
# config.remove_option('General', 'FILE2')
# Replace all other settings set in the config file
sections = config.sections()
for section in sections:
section_settings = [name for name, setting in config.items(section)]
for setting in section_settings:
if config.get(section, setting):
settings._update_careful(
{setting: config.get(section, setting)})
return (settings, arguments)
def _read(self, fp, fpname):
cursect = None
optname = None
lineno = 0
e = None
while True:
line = fp.readline()
if not line:
break
lineno = lineno + 1
if line.strip() == '' or line[0] in '#;':
continue
if line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR":
continue
if line[0].isspace() and cursect is not None and optname:
value = line.strip()
if value:
cursect[optname] = "%s\n%s" % (cursect[optname], value)
else:
try:
value = line[:line.index(';')].strip()
except ValueError:
value = line.strip()
if value[0] == '[' and value[-1] == ']' and len(value) > 2:
sectname = value[1:-1]
if sectname in self._sections:
cursect = self._sections[sectname]
elif sectname == "DEFAULT":
cursect = self._defaults
else:
cursect = self._dict()
cursect['__name__'] = sectname
self._sections[sectname] = cursect
optname = None
elif cursect is None:
raise ConfigParser.MissingSectionHeaderError(
fpname, lineno, line)
else:
mo = self.OPTCRE.match(line)
if mo:
optname, vi, optval = mo.group('option', 'vi', 'value')
if vi in ('=', ':') and ';' in optval:
pos = optval.find(';')
if pos != -1 and optval[pos - 1].isspace():
optval = optval[:pos]
optval = optval.strip()
if optval == '""':
optval = ''
optname = self.optionxform(optname.rstrip())
cursect[optname] = optval
else:
if not e:
e = ConfigParser.ParsingError(fpname)
e.append(lineno, repr(line))
if e:
raise e
def validate_config_option(config, option_name, default_value=None):
if not config.has_option("default",option_name):
if default_value:
gc3libs.log.warning("No option '%s' defined. " \
"Using default: '%s'" % (option_name, str(default_value)))
return default_value
else:
raise ConfigParser.ParsingError("No option '%s' defined." % option_name)
else:
return config.get('default',option_name)
def typexform(optname, optval, fpname, lineno, line):
if not ':' in optname:
return optname, optval, None
optname, xformername = optname.split(':', 1)
xformer = xforms.get(xformername)
if xformer is None:
e = ConfigParser.ParsingError(fpname)
e.append(lineno, repr(line))
raise e
return optname, xformer(optval), xformername
def parse(self, cip, section, config):
host = cip.get(section, "host")
port = cip.getint(section, "port")
key = cip.get(section, "key")
iv = cip.get(section, "iv")
if (len(key) != 16 or len(iv) != 8):
raise ConfigParser.ParsingError(
"Key length must be 16 and iv length must be 8.")
info = GameStateServerInfo(host, port, key, iv)
config.setGameStateServerInfo(info)
def test_parsingerror(self):
import pickle
e1 = ConfigParser.ParsingError('source')
e1.append(1, 'line1')
e1.append(2, 'line2')
e1.append(3, 'line3')
pickled = pickle.dumps(e1)
e2 = pickle.loads(pickled)
self.assertEqual(e1.message, e2.message)
self.assertEqual(e1.args, e2.args)
self.assertEqual(e1.filename, e2.filename)
self.assertEqual(e1.errors, e2.errors)
self.assertEqual(repr(e1), repr(e2))
def __parseLogging(self,conf):
log_to_console = self.getboolean("Logging","log_to_console")
log_to_file = self.getboolean("Logging","log_to_file")
log_dir = self.get("Logging","log_dir")
file_log_level = self.get("Logging","file_log_level")
console_log_level = self.get("Logging","console_log_level")
conf.use_console_logging = log_to_console
conf.use_file_logging = log_to_file
conf.log_dir = log_dir
if(file_log_level.upper() == "DEBUG"):
conf.file_log_level = logging.DEBUG
elif(file_log_level.upper() == "INFO"):
conf.file_log_level = logging.INFO
elif(file_log_level.upper() == "WARNING"):
conf.file_log_level = logging.WARNING
elif(file_log_level.upper() == "ERROR"):
conf.file_log_level = logging.ERROR
elif(file_log_level.upper() == "CRITICAL"):
conf.file_log_level = logging.CRITICAL
else:
msg = "Unknown file log level (%s), should be one of: debug, info, warning, error, critical" % file_log_level
raise ConfigParser.ParsingError(msg)
if(console_log_level.upper() == "DEBUG"):
conf.console_log_level = logging.DEBUG
elif(console_log_level.upper() == "INFO"):
conf.console_log_level = logging.INFO
elif(console_log_level.upper() == "WARNING"):
conf.console_log_level = logging.WARNING
elif(console_log_level.upper() == "ERROR"):
conf.console_log_level = logging.ERROR
elif(console_log_level.upper() == "CRITICAL"):
conf.console_log_level = logging.CRITICAL
else:
msg = "Unknown console log level (%s), should be one of: debug, info, warning, error, critical" % log_level
raise ConfigParser.ParsingError(msg)
def parse(self, cip, section, config):
name = section[8:]
script = cip.get(section, "script")
timeout = int(cip.get(section, "timeout"))
offscore = int(cip.get(section, "offscore"))
defscore = int(cip.get(section, "defscore"))
if (cip.debug == False):
if (os.access(script, os.X_OK) == False):
raise ConfigParser.ParsingError(
"Script %s does not appear to be executable!" % (script))
servicebot_conf = _getServiceBotConfig(config)
servicebot_conf.addServiceInfo(name, script, timeout, offscore,
defscore)
def __init__(self, config_file):
self.config_file = config_file
logging.basicConfig(format='%(asctime)-15s %(levelname)s: %(message)s')
self.logger = logging.getLogger('preseed_output')
self.logger.setLevel(logging.INFO)
self.cfparser = ConfigParser.ConfigParser()
try:
if self.cfparser.read(self.config_file) == []:
raise ConfigParser.ParsingError('No config file detected.')
except ConfigParser.ParsingError:
self.logger.error('Unable to parse file %s' % self.config_file)
sys.exit(1)
self.loaded_template = OrderedDict()
# special functions are handled by handle_X methods
self.special_sections = ['network', 'mirrors', 'partitioning']
def load(self,config_path):
try:
self.config_path = config_path
self.read(config_path)
conf = Config()
assert(os.path.isfile(config_path)),"%s does not exsist!" % config_path
assert("base_path" in self.defaults()),"Config file %s has no base_path set!" % config_path
msg = "No such directory: %s, be sure to edit the base_path in the config file!" % self.defaults()["base_path"]
assert(os.path.isdir(self.defaults()["base_path"])),msg
conf.setBasePath(self.defaults()["base_path"])
#Set up logging first
self.__parseLogging(conf)
sections = self.sections()
sections.remove("Logging")
for section in sections:
processed = False
for handler in self.handlers:
if(handler.canHandle(section)):
processed = True
handler.parse(self,section,conf)
break
if(processed == False):
msg = "Unknown Section: %s" % section
raise ConfigParser.ParsingError(msg)
return conf
except ConfigParser.ParsingError as e:
err = "Error parsing %s: %s" %(self.config_path,e)
sys.exit(err)
except ConfigParser.NoSectionError as e:
err = "Error parsing %s: A required section is missing: %s" %(self.config_path,e.section)
sys.exit(err)
except ConfigParser.NoOptionError as e:
err = "Error parsing %s: A required option in section %s is missing: %s" % (
self.config_path,e.section,e.option)
sys.exit(err)
"""
def load(self, path):
parser = ConfigParser.SafeConfigParser()
try:
cfg_str = '[root]\n' + open(path, 'r').read()
cfg_fp = StringIO.StringIO(cfg_str)
parser = ConfigParser.RawConfigParser(allow_no_value=False)
parser.readfp(cfg_fp)
self.__dict__.update(parser.items("root"))
except (ConfigParser.ParsingError) as e:
error = str(e)
line = error[error.find("[line") + 5:error.find("]")].strip()
raise ConfigParser.ParsingError(
"Failed to parse config file. Error on line: " + line)
self.checkConfig()
def _read_config_file(self, config_file):
config = ConfigParser.RawConfigParser()
config.read(self.params.config_file)
if not config.has_section("default"):
raise ConfigParser.ParsingError("Section 'default' not found")
self.extra['s3cfg'] = os.path.expandvars(validate_config_option(config,
"S3ConfigFileLocation",
DEFAULT_S3CFG))
self.extra['input'] = os.path.expandvars(validate_config_option(config,
"InputLocation"))
self.extra['output'] = os.path.expandvars(validate_config_option(config,
"OutputLocation"))
self.extra['preserve-output'] = validate_config_option(config,
"PreserveOutputWhenInputRemoved")
self.extra['computing_resources'] = validate_config_option(config,
"computing_resources",
DEFAULT_CLOUD_ALL)
self.extra['max_running'] = validate_config_option(config,
"max_running",
DEFAULT_MAX_RUNNING)
self.extra['max_retries'] = validate_config_option(config,
"max_retries",
DEFAULT_MAX_RETRIES)
self.extra['log_level'] = validate_config_option(config,
"log_level",
DEFAULT_LOG_LEVEL)
self.extra['save_simulation_log'] = validate_config_option(config,
"save_simulation_log",
False)
self.extra['simulation_log_location'] = os.path.expandvars(validate_config_option(config,
"simulation_log_location",
False))
def _read_config_file(self, config_file):
import ConfigParser
config = ConfigParser.RawConfigParser()
config.read(self.params.config_file)
if not config.has_section("default"):
raise ConfigParser.ParsingError("Section 'default' not found")
self.extra['s3cmd_conf_location'] = validate_config_option(
config, "s3cmd_conf_location", DEFAULT_S3CFG)
self.extra['fs_endpoint'] = validate_config_option(
config, "fs_endpoint")
self.extra['model_input'] = validate_config_option(
config, "model_input")
self.extra['model_output'] = validate_config_option(
config, "model_output")
self.extra['computing_resources'] = validate_config_option(
config, "computing_resources", DEFAULT_CLOUD_ALL)
self.extra['max_running'] = validate_config_option(
config, "max_running", DEFAULT_MAX_RUNNING)
self.extra['max_retries'] = validate_config_option(
config, "max_retries", DEFAULT_MAX_RETRIES)
self.extra['log_level'] = validate_config_option(
config, "log_level", DEFAULT_LOG_LEVEL)
self.extra['save_simulation_log'] = validate_config_option(
config, "save_simulation_log", False)
self.extra['simulation_log_location'] = validate_config_option(
config, "simulation_log_location", False)
import ConfigParser
import os
import sys
CONFIG_FILE = os.path.expanduser('~/.config/btn-api')
config = ConfigParser.SafeConfigParser()
if os.path.exists(CONFIG_FILE):
config.read(CONFIG_FILE)
else:
config.add_section('config')
config.set('config', 'api-key', "YOUR API KEY")
config.set('config', 'api-host', "http://api.btnapps.net/")
config.set('config', 'api-port', '80')
with open(CONFIG_FILE, 'wb') as configfile:
config.write(configfile)
if not config.has_option('config', 'api-key'):
raise ConfigParser.ParsingError("Invalid config file: %s" % CONFIG_FILE)
api_key = config.get('config', 'api-key')
api_host = config.get('config', 'api-host')
api_port = config.getint('config', 'api-port')
if api_key is None:
raise ConfigParser.Error("Invalid API key in config file: %s" %
CONFIG_FILE)
from btnapi import BtnApi
sys.modules[__name__] = BtnApi(api_key, api_host, api_port)
def __read(self, fp, fpname):
"""Parse a sectioned setup file.
The sections in setup file contains a title line at the top,
indicated by a name in square brackets (`[]'), plus key/value
options lines, indicated by `name: value' format lines.
Continuation are represented by an embedded newline then
leading whitespace. Blank lines, lines beginning with a '#',
and just about everything else is ignored.
"""
cursect = None # None, or a dictionary
optname = None
lineno = 0
e = None # None, or an exception
while 1:
line = fp.readline()
if not line:
break
lineno = lineno + 1
# comment or blank line?
line = line.strip()
if not line:
continue
discard = False
for c in self.__comments:
if line.startswith(c):
discard = True
if discard:
continue
else:
# is it a section header?
mo = self.SECTCRE.match(line)
if mo:
sectname = self.sectionxform(mo.group('header'))
if self.__sections.has_key(sectname):
cursect = self.__sections[sectname]
else:
cursect = {'__name__': [(mo.group('header'), mo.group('header'))]}
self.__sections[sectname] = cursect
# So sections can't start with a continuation line
optname = None
# no section header in the file?
elif cursect is None:
raise ConfigParser.MissingSectionHeaderError(fpname, lineno, `line`)
# an option line?
else:
# Check if this is a key = value line
# option, spaces/tabs, : or =, spaces/tabs, value [ ;comments]
OPTWITHCOMMENTSCRE = re.compile(
r'(?P<option>[]\-[\w_.*,(){}$#]+)'
r'(?P<s1>[ \t]*)(?P<vi>[:=])(?P<s2>[ \t]*)'
'(?P<value>.*)(?P<comments>\s+[%s].*)$' % '|'.join(self.__comments)
)
# First attempt: ignore comments
mo = OPTWITHCOMMENTSCRE.match(line)
if not mo:
mo = self.OPTCRE.match(line)
if mo:
optname, vi, optval = mo.group('option', 'vi', 'value')
optval = optval.strip()
# allow empty values
if optval == '""':
optval = ''
keyname = self.optionxform(optname)
if not cursect.has_key(keyname):
cursect[keyname] = [] # list of (value, realoptionname)
cursect[keyname].append((optval, optname)) # we backup the real optname, with case information
else:
# a non-fatal parsing error occurred. set up the
# exception but keep going. the exception will be
# raised at the end of the file and will contain a
# list of all bogus lines
if not e:
e = ConfigParser.ParsingError(fpname)
e.append(lineno, `line`)
# if any parsing errors occurred, raise an exception
if e:
raise e
def _read(self, fp, fpname):
cursect = None
optname = None
lineno = 0
e = None
while True:
line = fp.readline()
if not line:
break
lineno += 1
if line.strip() == '' or line[0] in '#;':
continue
if line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR":
continue
if line[0].isspace() and cursect is not None and optname:
value = line.strip()
if value:
if isinstance(cursect[optname], set):
_temp_item = list(cursect[optname])
del cursect[optname]
cursect[optname] = _temp_item
elif isinstance(cursect[optname], (str, unicode)):
_temp_item = [cursect[optname]]
del cursect[optname]
cursect[optname] = _temp_item
cursect[optname].append(value)
else:
mo = self.SECTCRE.match(line)
if mo:
sectname = mo.group('header')
if sectname in self._sections:
cursect = self._sections[sectname]
elif sectname == 'DEFAULT':
cursect = self._defaults
else:
cursect = self._dict()
self._sections[sectname] = cursect
optname = None
elif cursect is None:
raise ConfigParser.MissingSectionHeaderError(
fpname, lineno, line)
else:
mo = self._optcre.match(line)
if mo:
optname, vi, optval = mo.group('option', 'vi', 'value')
optname = self.optionxform(optname.rstrip())
if optval is not None:
if vi in ('=', ':') and ';' in optval:
pos = optval.find(';')
if pos != -1 and optval[pos - 1].isspace():
optval = optval[:pos]
optval = optval.strip()
if optval == '""':
optval = ''
cursect[optname] = optval
else:
if not e:
e = ConfigParser.ParsingError(fpname)
e.append(lineno, repr(line))
if e:
raise e
all_sections = [self._defaults]
all_sections.extend(self._sections.values())
for options in all_sections:
for name, val in options.items():
if isinstance(val, list):
_temp_item = '\n'.join(val)
del options[name]
options[name] = _temp_item
def _read(self, fp, fpname):
cursect = None
optname = None
lineno = 0
is_multi_line = False
e = None
def string_decode(v):
if v[-1] == '\\':
v = v[:-1]
# end cut trailing escapes to prevent decode error
if PY3:
return v.encode(defenc).decode('unicode_escape')
else:
return v.decode('string_escape')
# end
# end
while True:
# we assume to read binary !
line = fp.readline().decode(defenc)
if not line:
break
lineno = lineno + 1
# comment or blank line?
if line.strip() == '' or self.re_comment.match(line):
continue
if line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR":
# no leading whitespace
continue
# is it a section header?
mo = self.SECTCRE.match(line.strip())
if not is_multi_line and mo:
sectname = mo.group('header').strip()
if sectname in self._sections:
cursect = self._sections[sectname]
elif sectname == cp.DEFAULTSECT:
cursect = self._defaults
else:
cursect = self._dict((('__name__', sectname), ))
self._sections[sectname] = cursect
self._proxies[sectname] = None
# So sections can't start with a continuation line
optname = None
# no section header in the file?
elif cursect is None:
raise cp.MissingSectionHeaderError(fpname, lineno, line)
# an option line?
elif not is_multi_line:
mo = self.OPTCRE.match(line)
if mo:
# We might just have handled the last line, which could
# contain a quotation we want to remove
optname, vi, optval = mo.group('option', 'vi', 'value')
if vi in ('=', ':') and ';' in optval and not optval.strip(
).startswith('"'):
pos = optval.find(';')
if pos != -1 and optval[pos - 1].isspace():
optval = optval[:pos]
optval = optval.strip()
if optval == '""':
optval = ''
# end handle empty string
optname = self.optionxform(optname.rstrip())
if len(optval
) > 1 and optval[0] == '"' and optval[-1] != '"':
is_multi_line = True
optval = string_decode(optval[1:])
# end handle multi-line
cursect[optname] = optval
else:
if not self.OPTVALUEONLY.match(line):
if not e:
e = cp.ParsingError(fpname)
e.append(lineno, repr(line))
continue
else:
line = line.rstrip()
if line.endswith('"'):
is_multi_line = False
line = line[:-1]
# end handle quotations
cursect[optname] += string_decode(line)
# END parse section or option
# END while reading
# if any parsing errors occurred, raise an exception
if e:
raise e
def _read(self, fp, fpname):
"""This is a copy of Python 2.6's _read() method, with support for
continuation lines removed."""
cursect = None # None, or a dictionary
optname = None
comment = 0
lineno = 0
e = None # None, or an exception
while True:
line = fp.readline()
if not line:
break
lineno = lineno + 1
# comment or blank line?
if ((line.strip() == '' or line[0] in '#;') or
(line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR")):
self._sections["comment-%d" % comment] = line
comment += 1
# a section header or option header?
else:
# is it a section header?
mo = self.SECTCRE.match(line)
if mo:
sectname = mo.group('header')
if sectname in self._sections:
cursect = self._sections[sectname]
elif sectname == cp.DEFAULTSECT:
cursect = self._defaults
else:
cursect = self._dict()
cursect['__name__'] = sectname
self._sections[sectname] = cursect
# So sections can't start with a continuation line
optname = None
# no section header in the file?
elif cursect is None:
raise cp.MissingSectionHeaderError(fpname, lineno, line)
# an option line?
else:
mo = self.OPTCRE.match(line)
if mo:
optname, vi, optval = mo.group('option', 'vi', 'value')
if vi in ('=', ':') and ';' in optval:
# ';' is a comment delimiter only if it follows
# a spacing character
pos = optval.find(';')
if pos != -1 and optval[pos-1].isspace():
optval = optval[:pos]
optval = optval.strip()
# allow empty values
if optval == '""':
optval = ''
optname = self.optionxform(optname.rstrip())
cursect[optname] = optval
else:
# a non-fatal parsing error occurred. set up the
# exception but keep going. the exception will be
# raised at the end of the file and will contain a
# list of all bogus lines
if not e:
e = cp.ParsingError(fpname)
e.append(lineno, repr(line))
# if any parsing errors occurred, raise an exception
if e:
raise e
def _read(self, fp, fpname):
"""Parse a sectioned setup file, retaining comments"""
cursect = None # None, or a dictionary
optname = None # None, or a string
lineno = 0 # Integer
e = None # None, or an exception
# initialize the toplevel section and make it current
self._sections = collections.OrderedDict()
cursect = collections.OrderedDict()
cursect['__name__'] = self.TOPLEVEL
self._sections[self.TOPLEVEL] = cursect
for line in fp.readlines():
lineno = lineno + 1
# we only process non-blank lines
line = line.strip()
if line:
# comment line?
if (line.split(None, 1)[0].lower() == 'rem'
and line[0] in "rR") or line[0] in "#;":
cursect['_comment_' + str(lineno)] = line
else:
# continuation line?
if line[0].isspace() and cursect is not None and optname:
if value:
cursect[optname].append(value)
# a section header or option header?
else:
# is it a section header?
mo = self.SECTCRE.match(line)
if mo:
sectname = mo.group('header')
if sectname in self._sections:
cursect = self._sections[sectname]
else:
cursect = collections.OrderedDict()
cursect['__name__'] = sectname
self._sections[sectname] = cursect
# So sections can't start with a continuation line
optname = None
# an option line?
else:
mo = self._optcre.match(line)
if mo:
optname, vi, optval = mo.group(
'option', 'vi', 'value')
optname = self.optionxform(optname.rstrip())
# This check is fine because the OPTCRE cannot
# match if it would set optval to None
if optval is not None:
if vi in ('=', ':') and ';' in optval:
# ';' is a comment delimiter only if it follows
# a spacing character
pos = optval.find(';')
if pos != -1 and optval[pos -
1].isspace():
optval = optval[:pos]
optval = optval.strip()
# allow empty values
if optval == '""':
optval = ''
cursect[optname] = [optval]
else:
# valueless option handling
cursect[optname] = optval
else:
# a non-fatal parsing error occurred. set up the
# exception but keep going. the exception will be
# raised at the end of the file and will contain a
# list of all bogus lines
if not e:
e = ConfigParser.ParsingError(fpname)
e.append(lineno, repr(line))
# if any parsing errors occurred, raise an exception
if e:
raise e
# join the multi-line values collected while reading
all_sections = [self._defaults]
all_sections.extend(self._sections.values())
for options in all_sections:
for name, val in options.items():
if isinstance(val, list):
options[name] = '\n'.join(val)
def _read(self, fp, fpname):
"""A direct copy of the py2.4 version of the super class's _read method
to assure it uses ordered dicts. Had to change one line to make it work.
Future versions have this fixed, but in fact its quite embarrassing for the
guys not to have done it right in the first place !
Removed big comments to make it more compact.
Made sure it ignores initial whitespace as git uses tabs"""
cursect = None # None, or a dictionary
optname = None
lineno = 0
is_multi_line = False
e = None # None, or an exception
def string_decode(v):
if v[-1] == '\\':
v = v[:-1]
# end cut trailing escapes to prevent decode error
if PY3:
return v.encode(defenc).decode('unicode_escape')
else:
return v.decode('string_escape')
# end
# end
while True:
# we assume to read binary !
line = fp.readline().decode(defenc)
if not line:
break
lineno = lineno + 1
# comment or blank line?
if line.strip() == '' or self.re_comment.match(line):
continue
if line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR":
# no leading whitespace
continue
# is it a section header?
mo = self.SECTCRE.match(line.strip())
if not is_multi_line and mo:
sectname = mo.group('header').strip()
if sectname in self._sections:
cursect = self._sections[sectname]
elif sectname == cp.DEFAULTSECT:
cursect = self._defaults
else:
cursect = self._dict((('__name__', sectname), ))
self._sections[sectname] = cursect
self._proxies[sectname] = None
# So sections can't start with a continuation line
optname = None
# no section header in the file?
elif cursect is None:
raise cp.MissingSectionHeaderError(fpname, lineno, line)
# an option line?
elif not is_multi_line:
mo = self.OPTCRE.match(line)
if mo:
# We might just have handled the last line, which could contain a quotation we want to remove
optname, vi, optval = mo.group('option', 'vi', 'value')
if vi in ('=', ':') and ';' in optval and not optval.strip(
).startswith('"'):
pos = optval.find(';')
if pos != -1 and optval[pos - 1].isspace():
optval = optval[:pos]
optval = optval.strip()
if optval == '""':
optval = ''
# end handle empty string
optname = self.optionxform(optname.rstrip())
if len(optval
) > 1 and optval[0] == '"' and optval[-1] != '"':
is_multi_line = True
optval = string_decode(optval[1:])
# end handle multi-line
cursect[optname] = optval
else:
# check if it's an option with no value - it's just ignored by git
if not self.OPTVALUEONLY.match(line):
if not e:
e = cp.ParsingError(fpname)
e.append(lineno, repr(line))
continue
else:
line = line.rstrip()
if line.endswith('"'):
is_multi_line = False
line = line[:-1]
# end handle quotations
cursect[optname] += string_decode(line)
# END parse section or option
# END while reading
# if any parsing errors occurred, raise an exception
if e:
raise e
def _read(self, fp, fpname):
"""Parse a sectioned setup file.
The sections in setup file contains a title line at the top,
indicated by a name in square brackets (`[]'), plus key/value
options lines, indicated by `name: value' format lines.
Continuations are represented by an embedded newline then
leading whitespace. Blank lines, lines beginning with a '#',
and just about everything else are ignored.
"""
cursect = None # None, or a dictionary
optname = None
lineno = 0
e = None # None, or an exception
while True:
line = fp.readline()
if not line:
break
lineno = lineno + 1
# comment or blank line?
if line.strip() == '' or line[0] in '#;':
continue
if line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR":
# no leading whitespace
continue
# continuation line?
if line[0].isspace() and cursect is not None and optname:
value = line.strip()
if value:
cursect[optname].append(value)
# a section header or option header?
else:
#JAZ add environment variable expansion
if not getattr(self, 'no_expand_vars', False):
line = os.path.expandvars(line)
# is it a section header?
mo = self.SECTCRE.match(line)
if mo:
sectname = mo.group('header')
if sectname in self._sections:
cursect = self._sections[sectname]
elif sectname == ConfigParser.DEFAULTSECT:
cursect = self._defaults
else:
cursect = self._dict()
cursect['__name__'] = sectname
self._sections[sectname] = cursect
# So sections can't start with a continuation line
optname = None
# no section header in the file?
elif line.lower().startswith('%include'):
if not getattr(self, 'no_expand_includes', False):
include_statement, filename = line.split()
filename = filename.strip('"').strip("'")
sys.stdout.write("Reading included ini file: `" +
filename + "'\n")
if not os.path.exists(filename):
# TODO: remove direct sys.stderr writes
sys.stderr.write("Tried to include non-existent "
"ini file: `" + filename + "'\n")
raise IOError("Tried to include non-existent "
"ini file: `" + filename + "'\n")
self.read(filename)
cursect = None
elif cursect is None:
raise ConfigParser.MissingSectionHeaderError(
fpname, lineno, line)
# an option line?
else:
mo = self._optcre.match(line)
if mo:
optname, vi, optval = mo.group('option', 'vi', 'value')
optname = self.optionxform(optname.rstrip())
# This check is fine because the OPTCRE cannot
# match if it would set optval to None
if optval is not None:
if vi in ('=', ':') and ';' in optval:
# ';' is a comment delimiter only if it
# follows a spacing character
pos = optval.find(';')
if pos != -1 and optval[pos - 1].isspace():
optval = optval[:pos]
optval = optval.strip()
# allow empty values
if optval == '""':
optval = ''
cursect[optname] = [optval]
else:
# valueless option handling
cursect[optname] = optval
else:
# a non-fatal parsing error occurred. set up the
# exception but keep going. the exception will be
# raised at the end of the file and will contain
# a list of all bogus lines
if not e:
e = ConfigParser.ParsingError(fpname)
e.append(lineno, repr(line))
# if any parsing errors occurred, raise an exception
if e:
raise e
# join the multi-line values collected while reading
all_sections = [self._defaults]
all_sections.extend(self._sections.values())
for options in all_sections:
for name, val in options.items():
if isinstance(val, list):
options[name] = '\n'.join(val)
def _read(self, fp, fpname):
self._sections[''] = self._dict()
cursect = self._sections[''] # None, or a dictionary
optname = None
lineno = 0
e = None # None, or an exception
while True:
line = fp.readline()
if not line:
break
lineno += 1
# comment or blank line?
if line.strip() == '' or line[0] in '#;':
continue
if line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR":
# no leading whitespace
continue
# continuation line?
if line[0].isspace() and cursect is not None and optname:
value = line.strip()
if value:
cursect[optname].append(value)
# a section header or option header?
else:
# is it a section header?
mo = self.SECTCRE.match(line)
if mo:
sectname = mo.group('header')
if sectname in self._sections:
cursect = self._sections[sectname]
elif sectname == ConfigParser.DEFAULTSECT:
cursect = self._defaults
else:
cursect = self._dict()
cursect['__name__'] = sectname
self._sections[sectname] = cursect
# So sections can't start with a continuation line
optname = None
# no section header in the file?
elif cursect is None:
raise ConfigParser.MissingSectionHeaderError(
fpname, lineno, line)
# an option line?
else:
mo = self._optcre.match(line)
if mo:
optname, vi, optval = mo.group('option', 'vi', 'value')
optname = self.optionxform(optname.rstrip())
# This check is fine because the OPTCRE cannot
# match if it would set optval to None
if optval is not None:
if vi in ('=', ':') and ';' in optval:
# ';' is a comment delimiter only if it follows
# a spacing character
pos = optval.find(';')
if pos != -1 and optval[pos - 1].isspace():
optval = optval[:pos]
optval = optval.strip()
# allow empty values
if optval == '""':
optval = ''
cursect[optname] = [optval]
else:
# valueless option handling
cursect[optname] = optval
else:
# a non-fatal parsing error occurred. set up the
# exception but keep going. the exception will be
# raised at the end of the file and will contain a
# list of all bogus lines
if not e:
e = ConfigParser.ParsingError(fpname)
e.append(lineno, repr(line))
# if any parsing errors occurred, raise an exception
if e:
raise e
# join the multi-line values collected while reading
all_sections = [self._defaults]
all_sections.extend(self._sections.values())
for options in all_sections:
for name, val in options.items():
if isinstance(val, list):
options[name] = '\n'.join(val)
def _read(self, fp, fpname):
"""Parse a sectioned setup file.
The sections in setup file contains a title line at the top,
indicated by a name in square brackets (`[]'), plus rel_path/value
options lines, indicated by `name: value' format lines.
Continuations are represented by an embedded newline then
leading whitespace. Blank lines, lines beginning with a '#',
and just about everything else are ignored.
"""
cursect = None # None, or a dictionary
optname = None
lineno = 0
e = None # None, or an exception
while True:
line = fp.readline()
if not line:
break
lineno = lineno + 1
# comment or blank line?
if line.strip() == '' or line[0] in '#;':
continue
if line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR":
# no leading whitespace
continue
# continuation line?
# REPLACED FOR SPLUNK CONF LINE CONTINUATION (e.g. \\\n)
# if line[0].isspace() and cursect is not None and optname:
# value = line.strip()
# if value:
# cursect[optname].append(value)
# a section header or option header?
else:
# is it a section header?
mo = self.SECTCRE.match(line)
if mo:
sectname = mo.group('header')
if sectname in self._sections:
cursect = self._sections[sectname]
elif sectname == ConfigParser.DEFAULTSECT:
cursect = self._defaults
else:
cursect = self._dict()
cursect['__name__'] = sectname
self._sections[sectname] = cursect
# So sections can't start with a continuation line
optname = None
# no section header in the file?
elif cursect is None:
raise ConfigParser.MissingSectionHeaderError(
fpname, lineno, line)
# an option line?
else:
mo = self._optcre.match(line)
if mo:
optname, vi, optval, continued = mo.group(
'option', 'vi', 'value', 'continued')
optname = self.optionxform(optname.rstrip())
# This check is fine because the OPTCRE cannot
# match if it would set optval to None
if optval is not None:
optval = optval.strip()
if continued:
optval = [optval]
# loop until we reach end of multi-line value
while continued:
line = fp.readline()
if not line:
break
lineno = lineno + 1
match = self.OPTCRE_CONTINUED.match(line)
if match:
value, continued = match.group(
'value', 'continued')
optval.append(value)
elif optval == '""':
optval = ['']
else:
optval = [optval]
# allow empty values
cursect[optname] = optval
else:
# valueless option handling
cursect[optname] = optval
else:
# a non-fatal parsing error occurred. set up the
# exception but keep going. the exception will be
# raised at the end of the file and will contain a
# list of all bogus lines
if not e:
e = ConfigParser.ParsingError(fpname)
e.append(lineno, repr(line))
# if any parsing errors occurred, raise an exception
if e:
raise e
# join the multi-line values collected while reading
all_sections = [self._defaults]
all_sections.extend(self._sections.values())
for options in all_sections:
for name, val in options.items():
if isinstance(val, list):
options[name] = '\n'.join(val)
def _read(self, fp, fpname):
"""Parse a sectioned setup file.
The sections in setup file contains a title line at the top,
indicated by a name in square brackets (`[]'), plus key/value
options lines, indicated by `name: value' format lines.
Continuations are represented by an embedded newline then
leading whitespace. Blank lines, lines beginning with a '#',
and just about everything else are ignored.
"""
cursect = None # None, or a dictionary
optname = None
lineno = 0
e = None # None, or an exception
while True:
line = fp.readline()
if not line:
break
lineno = lineno + 1
# comment or blank line?
if line.strip() == '' or line[0] in '#;':
continue
if line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR":
# no leading whitespace
continue
# continuation line?
if line[0].isspace() and cursect is not None and optname:
value = line.strip()
if value:
cursect[optname].append(value)
# a section header or option header?
else:
# is it a section header?
mo = self.SECTCRE.match(line)
if mo:
sectname = mo.group('header')
if sectname in self._sections:
cursect = self._sections[sectname]
elif sectname == ConfigParser.DEFAULTSECT:
cursect = self._defaults
else:
cursect = self._dict()
cursect['__name__'] = sectname
self._sections[sectname] = cursect
# So sections can't start with a continuation line
optname = None
# no section header in the file?
elif cursect is None:
raise ConfigParser.MissingSectionHeaderError(
fpname, lineno, line)
# an option line?
else:
mo = self.OPTCRE.match(line)
if mo:
optname, vi, optval = mo.group('option', 'vi', 'value')
optname = self.optionxform(optname.rstrip())
# This check is fine because the OPTCRE cannot
# match if it would set optval to None
if optval is not None:
if vi in ('=', ':') and ';' in optval:
# ';' is a comment delimiter only if it follows
# a spacing character
pos = optval.find(';')
if pos != -1 and optval[pos - 1].isspace():
optval = optval[:pos]
optval = optval.strip()
# ADD THESE LINES
splittedComments = optval.split('#')
s = _expandSystemVariables(optname,
splittedComments[0])
optval = s.strip().strip("'").strip('"')
#if len(splittedComments) > 1:
# optval += " #" + " ".join(splittedComments[1:])
# END OF ADD
# allow empty values
if optval == '""':
optval = ''
# REPLACE following line (original):
#cursect[optname] = [optval]
# BY THESE LINES:
# Check if this optname already exists
if (optname in cursect) and (cursect[optname]
is not None):
cursect[optname][0] += ',' + optval
else:
cursect[optname] = [optval]
# END OF SUBSTITUTION
else:
# valueless option handling
cursect[optname] = optval
else:
# a non-fatal parsing error occurred. set up the
# exception but keep going. the exception will be
# raised at the end of the file and will contain a
# list of all bogus lines
if not e:
e = ConfigParser.ParsingError(fpname)
e.append(lineno, repr(line))
# if any parsing errors occurred, raise an exception
if e:
raise e
# join the multi-line values collected while reading
all_sections = [self._defaults]
all_sections.extend(self._sections.values())
for options in all_sections:
for name, val in options.items():
if isinstance(val, list):
options[name] = '\n'.join(val)
def _read(self, fp, fpname):
"""Parse a sectioned setup file.
The sections in setup file contains a title line at the top,
indicated by a name in square brackets (`[]'), plus key/value
options lines, indicated by `name: value' format lines.
Continuations are represented by an embedded newline then
leading whitespace. Blank lines, lines beginning with a '#',
and just about everything else are ignored.
"""
cursect = None # None, or a dictionary
curxformer = None
optname = None
lineno = 0
e = None # None, or an exception
while True:
line = fp.readline()
if not line:
break
lineno = lineno + 1
# comment or blank line?
if line.strip() == '' or line[0] in '#;':
continue
# continuation line?
if line[0].isspace() and cursect is not None and optname:
value = line.strip()
if value:
old = cursect[optname]
if curxformer:
value = extenders[curxformer](old, value)
else:
cursect[optname] = "%s\n%s" % (old, value)
# a section header or option header?
else:
# is it a section header?
mo = self.SECTCRE.match(line)
if mo:
sectname = mo.group('header')
if sectname in self._sections:
cursect = self._sections[sectname]
elif sectname == ConfigParser.DEFAULTSECT:
cursect = self._defaults
else:
cursect = {'__name__': sectname}
self._sections[sectname] = cursect
# So sections can't start with a continuation line
optname = None
# no section header in the file?
elif cursect is None:
raise ConfigParser.MissingSectionHeaderError(
fpname, lineno, line)
# an option line?
else:
mo = self.OPTCRE.match(line)
if mo:
optname, vi, optval = mo.group('option', 'vi', 'value')
if vi == '=' and ';' in optval:
# ';' is a comment delimiter only if it follows
# a spacing character
pos = optval.find(';')
if pos != -1 and optval[pos - 1].isspace():
optval = optval[:pos]
optval = optval.strip()
# allow empty values
if optval == '""':
optval = ''
optname = optname.rstrip()
optname, optval, curxformer = typexform(
optname, optval, fpname, lineno, line)
cursect[optname] = optval
else:
# a non-fatal parsing error occurred. set up the
# exception but keep going. the exception will be
# raised at the end of the file and will contain a
# list of all bogus lines
if not e:
e = ConfigParser.ParsingError(fpname)
e.append(lineno, repr(line))
# if any parsing errors occurred, raise an exception
if e:
raise e
def load_configuration(opts):
"""
Loads the configuration file, if it exists. Otherwise, creates a default
configuration and saves it. It then applies any command-line argument
overrides that exist.
Args:
opts: Command-line options
Returns:
A dictionary of key/value pairs that represent the base configuration
"""
# Set up a configuration parser with default values
config = ConfigParser.SafeConfigParser()
config.add_section('Simulator')
config.set('Simulator', 'url', 'http://127.0.0.1:8800')
config.set('Simulator',
'workload',
'txnintegration.integer_key_workload.IntegerKeyWorkload')
config.set('Simulator', 'rate', '12')
config.set('Simulator', 'discover', '15')
config.set('Simulator', 'log_file_enable', 'False')
config.set('Simulator', 'log_file', 'simulator.log')
config.set(
'Simulator',
'log_dir',
os.path.join(os.path.expanduser('~'), '.sawtooth/logs'))
config.set('Simulator', 'log_max_size', '0')
config.set('Simulator', 'log_backup_count', '10')
config.set('Simulator', 'verbose', '1')
# If there was a configuration file command-line option, then we
# will read it to override any of the defaults
if opts.config is not None:
if os.path.exists(opts.config):
config.read(opts.config)
else:
raise Exception('Config file does not exist: {}'.format(
opts.config))
# Otherwise, we are going to look in the default area
# <user home>/.sawtooth/simulator.cfg) and if it is there, we are going
# to read it. If the file does not exist, go ahead and create a default
# one for the user.
else:
config_file = \
os.path.join(
os.path.expanduser('~'),
'.sawtooth',
'simulator.cfg')
if os.path.exists(config_file):
config.read(config_file)
else:
if not os.path.exists(os.path.dirname(config_file)):
os.makedirs(os.path.dirname(config_file))
with open('{}.new'.format(config_file), 'w') as fd:
config.write(fd)
os.rename('{}.new'.format(config_file), config_file)
if config.getint('Simulator', 'rate') <= 0:
raise ConfigParser.ParsingError(
'Transaction rate in config file must be greater than zero')
if config.getint('Simulator', 'discover') <= 0:
raise ConfigParser.ParsingError(
'Discovery rate in config file must be greater than zero')
if config.getint('Simulator', 'log_max_size') < 0:
raise ConfigParser.ParsingError(
'Maximum log file size in config file must be greater than or '
'equal to zero')
if config.getint('Simulator', 'log_backup_count') <= 0:
raise ConfigParser.ParsingError(
'Log file backup count in config file must be greater than zero')
if config.getint('Simulator', 'verbose') < 0:
raise ConfigParser.ParsingError(
'Log file verbosity in config file must be greater than or '
'equal to zero')
# Now override any configuration values with corresponding command-line
# options that exist.
if opts.url is not None:
config.set('Simulator', 'url', opts.url)
if opts.workload is not None:
config.set('Simulator', 'workload', opts.workload)
if opts.rate is not None:
config.set('Simulator', 'rate', str(opts.rate))
if opts.discover is not None:
config.set('Simulator', 'discover', str(opts.discover))
if opts.verbose is not None:
config.set('Simulator', 'verbose', str(opts.verbose))
if opts.log_file_enable is not None:
config.set('Simulator', 'log_file_enable', str(opts.log_file_enable))
if opts.log_file is not None:
config.set('Simulator', 'log_file', opts.log_file)
if opts.log_dir is not None:
config.set('Simulator', 'log_dir', opts.log_dir)
if opts.log_max_size is not None:
config.set('Simulator', 'log_max_size', str(opts.log_max_size))
if opts.log_backup_count is not None:
config.set('Simulator', 'log_backup_count', str(opts.log_backup_count))
return config
def parse_cli(config_file):
'''
parseCLI()
This function parses the command line using the optparse module from the python standard library.
Though deprecated since python 2.7, optparse still is used in stead of argparse because the python
version available at the development systems was 2.6.
The options and arguments are stored in the global variables settings and arguments, respectively.
'''
# Read defaults
config = ConfigParser.ConfigParser()
try:
config.read(config_file)
except ConfigParser.ParsingError:
raise ConfigParser.ParsingError(
"Unable to parse the defaults file ({})".format(config_file))
parser = optparse.OptionParser()
parser.description = (
'This program performs a Smith-Waterman alignment of all sequences in FILE_1'
' against all sequences in FILE_2.\nBoth files should be in the fasta format.'
)
usage = '%prog [options] FILE_1 FILE_2'
parser.usage = usage
# general options
# TODO: Get final naming (convention) for all parameters!!
general_options = optparse.OptionGroup(
parser, 'Options that affect the general operation of the program')
general_options.add_option('-L',
'--logfile',
help='log events to FILE',
metavar="FILE",
dest='logfile')
general_options.add_option(
'--loglevel',
help='log level. Valid options are DEBUG, INFO, WARNING, ERROR'
' and CRITICAL',
dest='loglevel',
default=config.get('General', 'loglevel'))
general_options.add_option(
'-o',
'--out',
help='The file in which the program stores the generated output.'
'\nDefaults to ./output',
dest='out_file',
default=config.get('General', 'out_file'))
general_options.add_option(
'--outputformat',
help='The format of the file in which the program stores the '
'generated output.\nAvailable options are TXT and SAM.\nDefaults to txt',
dest='out_format',
default=config.get('General', 'out_format'))
general_options.add_option(
'-p',
'--program',
help='The program to be executed. Valid options are "aligner"'
' and "trimmer"',
dest='program',
default=config.get('General', 'program'))
general_options.add_option(
'-1',
'--filetype1',
help='File type of the first file. See bioPython IO for'
' available options',
dest='filetype1',
default=config.get('General', 'filetype1'))
general_options.add_option(
'-2',
'--filetype2',
help='File type of the second file. See bioPython IO for'
' available options',
dest='filetype2',
default=config.get('General', 'filetype2'))
general_options.add_option(
'-O',
'--override_output',
help='When output file exists, override it (T/F)',
dest='override_output',
default=config.get('General', 'override_output'))
general_options.add_option('-c',
'--configfile',
help='Give settings using configuration file',
dest='config_file',
default=False)
parser.add_option_group(general_options)
input_options = optparse.OptionGroup(
parser,
'start & stop indices for processing files. Handy for cluster processing. Leave all to zero to process all.'
)
input_options.add_option('--start_query',
help='start index in the query file (1)',
dest='start_query',
default=config.get("Input", "start_query"))
input_options.add_option('--end_query',
help='end index in the query file (1)',
dest='end_query',
default=config.get("Input", "end_query"))
input_options.add_option('--start_target',
help='start index in the target file (2)',
dest='start_target',
default=config.get("Input", "start_target"))
input_options.add_option('--end_target',
help='end index in the target file (2)',
dest='end_target',
default=config.get("Input", "end_target"))
input_options.add_option(
'--sequence_step',
help=
'Number of sequences read from file 2 before processing. Handy when processing NGS files.',
dest='sequence_step',
default=config.get('Input', 'sequence_step'))
input_options.add_option(
'--query_step',
help=
'Number of sequences read from file 1 before processing. Handy when processing NGS files.',
dest='query_step',
default=config.get('Input', 'query_step'))
parser.add_option_group(input_options)
aligner_options = optparse.OptionGroup(
parser, 'Options that affect the alignment.\nAligners include aligner'
' and mapper')
aligner_options.add_option('--customMatrix',
help='the custom matrix that should be used',
dest='custom_matrix')
aligner_options.add_option('-G',
'--gap_score',
help='Float. Penalty for a gap',
dest='gap_score',
default=config.get('Aligner', 'gap_score'))
aligner_options.add_option(
'-g',
'--gap_extension',
help=
'Float. Penalty for a gap extension. Set to zero to ignore this (faster)',
dest='gap_extension',
default=config.get('Aligner', 'gap_extension'))
aligner_options.add_option(
'-M',
'--matrixname',
help='The scoring to be used. Valid options are '
'"DNA-RNA", "BASIC", "Blosum62", "Blosum80" and "CUSTOM"',
dest='matrix_name',
default=config.get('Aligner', 'matrix_name'))
aligner_options.add_option('-q',
'--mismatch_score',
help='Float. Penalty for mismatch',
dest='mismatch_score',
default=config.get('Aligner', 'mismatch_score'))
aligner_options.add_option('-r',
'--match_score',
help='Float. Reward for match',
dest='match_score',
default=config.get('Aligner', 'match_score'))
aligner_options.add_option(
'--any',
help='Float. Score for a character which is neither in the nucleotide'
' list ("ACGTU"), nor equal to the anyNucleotide character ("N").\nOnly relevant'
' for use with the DNA-RNA scoring type.',
dest='any_score',
default=config.get('Aligner', 'any_score'))
aligner_options.add_option(
'--other',
help='Float. Score if the anyNucleotide character ("N") is present in'
' either query or subject.\nOnly relevant for use with the DNA-RNA scoring type.',
dest='other_score',
default=config.get('Aligner', 'other_score'))
aligner_options.add_option(
'--minimum',
help='Float. Sets the minimal score that initiates a back trace.'
' Do not set this very low: output may be flooded by hits.',
dest='minimum_score',
default=config.get('Aligner', 'minimum_score'))
aligner_options.add_option(
'--llimit',
help='Float. Sets the lower limit for the maximum score '
'which will be used to report a hit. pyPaSWAS will then also report hits with '
'a score lowerLimitScore * highest hit score. Set to <= 1.0. ',
dest='lower_limit_score',
default=config.get('Aligner', 'lower_limit_score'))
parser.add_option_group(aligner_options)
filter_options = optparse.OptionGroup(parser,
'Options for filtering the output')
filter_options.add_option(
'--filter_factor',
help='The filter factor to be used. Reports only hits within'
' filterFactor * highest possible score * length shortest sequence (or: defines'
' lowest value of the reported relative score). Set to <= 1.0',
dest='filter_factor',
default=config.get('Filter', 'filter_factor'))
filter_options.add_option('--query_coverage',
help='Minimum query coverage. Set to <= 1.0',
dest='query_coverage',
default=config.get('Filter', 'query_coverage'))
filter_options.add_option('--query_identity',
help='Minimum query identity. Set to <= 1.0',
dest='query_identity',
default=config.get('Filter', 'query_identity'))
filter_options.add_option(
'--relative_score',
help='Minimum relative score, defined by the alignment score'
' divided by the length of the shortest of the two sequences. Set to <= highest possible score'
', for example 5.0 in case of DNA',
dest='relative_score',
default=config.get('Filter', 'relative_score'))
filter_options.add_option(
'--base_score',
help='Minimum base score, defined by the alignment score'
' divided by the length of the alignment (including gaps). Set to <= highest possible score'
', for example 5.0 in case of DNA',
dest='base_score',
default=config.get('Filter', 'base_score'))
parser.add_option_group(filter_options)
graph_options = optparse.OptionGroup(
parser,
'Options to connect to a neo4j graph database and store mappings in a graph'
)
graph_options.add_option('--hostname',
help='Neo4j database host',
default=config.get("GraphDatabase", "hostname"),
dest="hostname")
graph_options.add_option('--username',
help='Neo4j user name',
default=config.get("GraphDatabase", "username"),
dest="username")
graph_options.add_option('--password',
help='Neo4j password',
default=config.get("GraphDatabase", "password"),
dest="password")
graph_options.add_option('--target_node',
help='Target node name',
default=config.get("GraphDatabase",
"target_node"),
dest="target_node")
graph_options.add_option('--sequence_node',
help='Sequence node name',
default=config.get("GraphDatabase",
"sequence_node"),
dest="sequence_node")
parser.add_option_group(graph_options)
device_options = optparse.OptionGroup(
parser, 'Options that affect the usage and settings of the '
'parallel devices')
device_options.add_option(
'--device',
help='the device on which the computations will be performed. '
'This should be an integer.',
dest='device_number',
default=config.get('Device', 'device_number'))
device_options.add_option(
'--number_of_compute_units',
help=
'Number of compute units to use (openCL only). Will not work on every device, recommended for CPU only. Set this 1 to use a single core on the device for example.'
'This should be an integer, using 0 for full device.',
dest='number_of_compute_units',
default=config.get('Device', 'number_of_compute_units'))
device_options.add_option(
'--sub_device',
help=
'the sub device on which the computations will be performed. Only used when number_of_compute_units > 0. '
'This should be an integer.',
dest='sub_device',
default=config.get('Device', 'sub_device'))
device_options.add_option(
'--limit_length',
help='Length of the longest sequence in characters to be read'
' from file. Lower this when memory of GPU is low.',
dest='limit_length',
default=config.get('Device', 'limit_length'))
device_options.add_option(
'--maximum_memory_usage',
help=
'Fraction (<= 1.0) of available device memory to use. Useful when several pyPaSWAS applications are running.',
dest="maximum_memory_usage",
default=config.get('Device', 'maximum_memory_usage'))
device_options.add_option(
'--njobs',
help='Sets the number of jobs run simultaneously on the grid. Will read'
' only part of the sequence file. (not implemented yet)',
dest='number_of_jobs')
device_options.add_option(
'--process_id',
help='Sets the processID of this job in the grid. ',
dest='process_id')
device_options.add_option('--max_genome_length',
help='Deprecated.\nDefaults to 200000',
dest='max_genome_length',
default=config.get('Device',
'max_genome_length'))
device_options.add_option(
'--recompile',
help=
'Recompile CUDA code? Set to F(alse) when sequences are of similar length: much faster.',
dest='recompile',
default=config.get('Device', 'recompile'))
device_options.add_option(
'--short_sequences',
help=
'Set to T(true) when aligning short sequences (trimming?) to maximize memory usage.',
dest='short_sequences',
default=config.get('Device', 'short_sequences'))
parser.add_option_group(device_options)
framework_options = optparse.OptionGroup(
parser,
'Determines which parallel computing framework to use for this program '
)
framework_options.add_option(
'--framework',
help=
'Choose which parallel computing framework to use, can be either CUDA or OpenCL ',
dest='framework',
default=config.get('Framework', 'language'))
parser.add_option_group(framework_options)
ocl_options = optparse.OptionGroup(
parser, 'Options for the usage of the OpenCL framework ')
ocl_options.add_option(
'--device_type',
help=
'Type of device to perform computations on (either CPU, GPU or ACCELARATOR)',
dest='device_type',
default=config.get('OpenCL', 'device_type'))
ocl_options.add_option(
'--platform_name',
help='Platform to run computations on (either Intel, NVIDIA or AMD)',
dest='platform_name',
default=config.get('OpenCL', 'platform_name'))
parser.add_option_group(ocl_options)
(settings, arguments) = parser.parse_args()
# If an extra configuration file is given, override settings as given by this file
if settings.config_file:
(settings, arguments) = _override_settings(settings.config_file,
settings, arguments)
if len(arguments) < 2:
raise InvalidOptionException('Missing input files')
return (settings, arguments)
def __init__(self, manifest, verbose=False):
"""
Parses the manifest and stores the information about the process tree
in a format usable by the class.
Raises IOError if :
- The path does not exist
- The file cannot be read
Raises ConfigParser.*Error if:
- Files does not contain section headers
- File cannot be parsed because of incorrect specification
:param manifest: Path to the manifest file that contains the
configuration for the process tree to be launched
:verbose: Print the process start and end information.
Genrates a lot of output. Disabled by default.
"""
self.verbose = verbose
# Children is a dictionary used to store information from the,
# Configuration file in a more usable format.
# Key : string contain the name of child process
# Value : A Named tuple of the form (max_time, (list of child processes of Key))
# Where each child process is a list of type: [count to run, name of child]
self.children = {}
cfgparser = ConfigParser.ConfigParser()
if not cfgparser.read(manifest):
raise IOError('The manifest %s could not be found/opened',
manifest)
sections = cfgparser.sections()
for section in sections:
# Maxtime is a mandatory option
# ConfigParser.NoOptionError is raised if maxtime does not exist
if '*' in section or ',' in section:
raise ConfigParser.ParsingError(
"%s is not a valid section name. "
"Section names cannot contain a '*' or ','." % section)
m_time = cfgparser.get(section, 'maxtime')
try:
m_time = int(m_time)
except ValueError:
raise ValueError(
'Expected maxtime to be an integer, specified %s' % m_time)
# No children option implies there are no further children
# Leaving the children option blank is an error.
try:
c = cfgparser.get(section, 'children')
if not c:
# If children is an empty field, assume no children
children = None
else:
# Tokenize chilren field, ignore empty strings
children = [[y.strip() for y in x.strip().split('*', 1)]
for x in c.split(',') if x]
try:
for i, child in enumerate(children):
# No multiplicate factor infront of a process implies 1
if len(child) == 1:
children[i] = [1, child[0]]
else:
children[i][0] = int(child[0])
if children[i][1] not in sections:
raise ConfigParser.ParsingError(
'No section corresponding to child %s' %
child[1])
except ValueError:
raise ValueError(
'Expected process count to be an integer, specified %s'
% child[0])
except ConfigParser.NoOptionError:
children = None
pn = ProcessNode(maxtime=m_time, children=children)
self.children[section] = pn
