def load_config(candidates):
"""Returns first valid configuration (parsed using Config)"""
for filename in candidates:
try:
config = Config(filename)
lo.info('loaded config ' + os.path.abspath(filename))
return config
except ConfigException as e:
lo.warning('failed to load "%s" : %s' % (filename, str(e)))
lo.error('could not find any config file')
return None
def load_config(candidates):
"""Returns first valid configuration (parsed using Config)"""
for filename in candidates:
try:
config = Config(filename)
lo.info('loaded config ' + os.path.abspath(filename))
return config
except ConfigException as e:
lo.warning('failed to load "%s" : %s' % (filename, str(e)))
lo.error('could not find any config file')
return None
def load_testsuites(testsuite_paths, selection, raise_exception=False):
''' loads testsuites specified by their full name, group name,
or directly specify testsuite by path of python file '''
testsuites = {}
groups = {}
for name, path in testsuite_paths.items():
parts = name.split('/')
groups.setdefault(parts[0], {})[name] = path
for name_or_path in selection:
try:
if (os.path.isfile(name_or_path)
and not name_or_path in testsuite_paths
and not name_or_path in groups):
# load from path
parts = name_or_path.split(os.path.sep)
name = os.path.splitext(parts[-1])[0]
if len(parts) > 1:
name = parts[-2] + '/' + name
path = name_or_path
testsuites[name] = load_testsuite(path)
elif name_or_path in groups:
# load group
for name, path in groups[name_or_path].items():
testsuites[name] = load_testsuite(path)
else:
# load by full name
name = name_or_path
if name in testsuite_paths:
testsuites[name] = load_testsuite(testsuite_paths[name])
else:
lo.warning('could not find testsuite "%s"' % name)
except genes.TestsuiteLoadingException as e:
if raise_exception:
raise e
lo.error('could not load testsuite from "%s" : %s' % (name, e))
return testsuites
def discover_testsuites(paths=[]):
''' returns dictionary mapping name to python file for all
testsuites discovered in the usual places: kvarq root path,
user home directory, current working directory,
KVARQ_TESTSUITES environment variable, and any more paths
specified paths as arguments -- later occurrences of the same
testsuite override previous '''
testsuite_paths = {}
# 1) discover in root path
root_base = os.path.abspath(os.path.join(get_root_path(), 'testsuites'))
lo.debug('discovering testsuites in root path')
add_testsuites_dir(testsuite_paths, root_base)
# 2) discover from $HOME
base = os.path.join(expanduser('~'), 'kvarq_testsuites')
lo.debug('discovering testsuites in home directory')
add_testsuites_dir(testsuite_paths, base)
# 3) discover from CWD if not in root path
cwd_base = os.path.abspath('testsuites')
if cwd_base != root_base:
lo.debug('discovering testsuites in current working directory')
add_testsuites_dir(testsuite_paths, cwd_base)
# 4) discover from KVARQ_TESTSUITES
from_env = os.environ.get('KVARQ_TESTSUITES')
if from_env:
lo.debug('discovering testsuites in $KVARQ_TESTSUITES')
for base in from_env.split(os.path.pathsep):
add_testsuites_dir(testsuite_paths, base)
# 5) explicitely specified paths
for base in paths:
if os.path.isdir(base):
lo.debug('discovering testsuites in "%s"' % base)
add_testsuites_dir(testsuite_paths, base)
else:
lo.warning('could not find directory "%s"' % base)
return testsuite_paths
def update(self, initial=False):
todo = set()
# get list of files
done_n = 0
for fname in os.listdir(self.xray_dir):
if os.path.isdir(self.path(fname)):
continue
if self.ignore(fname):
continue
if fname in self.ignored:
continue
if not self.image(fname):
lo.warning('ignoring image "%s" : %s' %
(fname, self.invalid(fname)))
self.ignored.add(fname)
continue
if self.invalid(fname):
lo.info('ignoring file "%s" (unknown extension)' % fname)
self.ignored.add(fname)
continue
if fname in self.done:
done_n += 1
else:
todo.add(fname)
if len(todo) != len(self.todo):
if initial:
lo.info('scanned "%s" : %d files to upload (%d already done)' %
(os.path.abspath(
self.xray_dir), len(todo), len(self.done)))
else:
lo.info('rescanned "%s" : %d new images' % (os.path.abspath(
self.xray_dir), len(todo) - len(self.todo)))
self.todo = todo
def update(self, initial=False):
todo = set()
# get list of files
done_n = 0
for fname in os.listdir(self.xray_dir):
if os.path.isdir(self.path(fname)):
continue
if self.ignore(fname):
continue
if fname in self.ignored:
continue
if not self.image(fname):
lo.warning('ignoring image "%s" : %s' % (fname, self.invalid(fname)))
self.ignored.add(fname)
continue
if self.invalid(fname):
lo.info('ignoring file "%s" (unknown extension)' % fname)
self.ignored.add(fname)
continue
if fname in self.done:
done_n += 1
else:
todo.add(fname)
if len(todo) != len(self.todo):
if initial:
lo.info('scanned "%s" : %d files to upload (%d already done)' % (
os.path.abspath(self.xray_dir), len(todo), len(self.done)))
else:
lo.info('rescanned "%s" : %d new images' % (
os.path.abspath(self.xray_dir), len(todo) - len(self.todo)))
self.todo = todo
def __init__(self, filename):
"""Read JSON configuration from specified file
Rises ConfigException if errors are encountered
"""
class ConfigNode:
pass
try:
with io.open(filename) as fd:
data = json.load(fd)
except IOError as e:
raise ConfigException('could not read from file : ' + filename)
except ValueError as e:
raise ConfigException('not valid JSON file : ' + str(e))
# uploader settings
self.title = extract_remove(data, 'title')
self.interval = extract_remove(data, 'interval')
self.dryrun = extract_remove(data, 'dryrun')
# odk settings
server = extract_remove(data, ['odk', 'server'])
url = urllib.parse.urlparse(server)
if not url.scheme or not url.hostname or not url.path:
raise ConfigException('incomplete server address "%s"' % server +
' : you must specify scheme (http/https), server, URI')
self.odk = ConfigNode()
self.odk.scheme = url.scheme
self.odk.hostname = url.hostname
self.odk.port = url.port or (url.scheme == 'https' and 443 or 80)
self.odk.path = url.path
self.odk.username = extract_remove(data, ['odk', 'username'])
self.odk.password = extract_remove(data, ['odk', 'password'])
# MS-SQL settings
self.mssql = ConfigNode()
self.mssql.database = extract_remove(data, ['mssql', 'database'])
self.mssql.server = extract_remove(data, ['mssql', 'server'])
if len(self.mssql.server.split('\\')) != 2:
raise ConfigException('server address should have form ' +
'"SERVER\\SQLINSTANCE"')
self.mssql.username = extract_remove(data, ['mssql', 'username'])
self.mssql.password = extract_remove(data, ['mssql', 'password'])
self.sqlitedb = extract_remove(data, 'sqlitedb')
if not os.path.isfile(self.sqlitedb):
raise ConfigException('cannot access database file "%s"' %
self.sqlitedb)
# tables
if not 'tables' in data:
raise ConfigException('missing key : tables')
self.sqls = {}
self.xforms = {}
self.rownames = {}
self.rowids = {}
names = [name for name in data['tables'].keys()]
for name in names:
xform = extract_remove(data, ['tables', name, 'xform'])
sql = extract_remove(data, ['tables', name, 'sql'])
self.rownames[name] = extract_remove(data, ['tables', name, 'rowname'])
self.rowids[name] = extract_remove(data, ['tables', name, 'rowid'])
try:
with io.open(xform) as fd:
self.xforms[name] = fd.read()
XForm(self.xforms[name])
except (XFormException, IOError) as e:
raise ConfigException('could not load XForm "%s" : %s' % (
xform, str(e)))
try:
with io.open(sql) as fd:
self.sqls[name] = fd.read()
except IOError as e:
raise ConfigException('could not load sql file "%s" : %s' % (
sql, str(e)))
for key in data:
lo.warning('ignoring config key : ' + key)
def __init__(self, filename):
"""Read JSON configuration from specified file
Rises ConfigException if errors are encountered
"""
class ConfigNode:
pass
try:
with io.open(filename) as fd:
data = json.load(fd)
except IOError as e:
raise ConfigException('could not read from file : ' + filename)
except ValueError as e:
raise ConfigException('not valid JSON file : ' + str(e))
# uploader settings
self.title = extract_remove(data, 'title')
self.interval = extract_remove(data, 'interval')
self.dryrun = extract_remove(data, 'dryrun')
# odk settings
server = extract_remove(data, ['odk', 'server'])
url = urllib.parse.urlparse(server)
if not url.scheme or not url.hostname or not url.path:
raise ConfigException(
'incomplete server address "%s"' % server +
' : you must specify scheme (http/https), server, URI')
self.odk = ConfigNode()
self.odk.scheme = url.scheme
self.odk.hostname = url.hostname
self.odk.port = url.port or (url.scheme == 'https' and 443 or 80)
self.odk.path = url.path
self.odk.username = extract_remove(data, ['odk', 'username'])
self.odk.password = extract_remove(data, ['odk', 'password'])
# MS-SQL settings
self.mssql = ConfigNode()
self.mssql.database = extract_remove(data, ['mssql', 'database'])
self.mssql.server = extract_remove(data, ['mssql', 'server'])
if len(self.mssql.server.split('\\')) != 2:
raise ConfigException('server address should have form ' +
'"SERVER\\SQLINSTANCE"')
self.mssql.username = extract_remove(data, ['mssql', 'username'])
self.mssql.password = extract_remove(data, ['mssql', 'password'])
self.sqlitedb = extract_remove(data, 'sqlitedb')
if not os.path.isfile(self.sqlitedb):
raise ConfigException('cannot access database file "%s"' %
self.sqlitedb)
# tables
if not 'tables' in data:
raise ConfigException('missing key : tables')
self.sqls = {}
self.xforms = {}
self.rownames = {}
self.rowids = {}
names = [name for name in data['tables'].keys()]
for name in names:
xform = extract_remove(data, ['tables', name, 'xform'])
sql = extract_remove(data, ['tables', name, 'sql'])
self.rownames[name] = extract_remove(data,
['tables', name, 'rowname'])
self.rowids[name] = extract_remove(data, ['tables', name, 'rowid'])
try:
with io.open(xform) as fd:
self.xforms[name] = fd.read()
XForm(self.xforms[name])
except (XFormException, IOError) as e:
raise ConfigException('could not load XForm "%s" : %s' %
(xform, str(e)))
try:
with io.open(sql) as fd:
self.sqls[name] = fd.read()
except IOError as e:
raise ConfigException('could not load sql file "%s" : %s' %
(sql, str(e)))
for key in data:
lo.warning('ignoring config key : ' + key)
def __init__(self, filename):
"""Read JSON configuration from specified file
Rises ConfigException if errors are encountered
"""
try:
data = json.load(io.open(filename))
except ValueError as e:
raise ConfigException('not valid JSON file : ' + str(e))
def extract_key(key):
if not key in data:
raise ConfigException('missing key : ' + key)
value = data[key]
del data[key]
return value
server = extract_key('server')
url = urllib.parse.urlparse(server)
if not url.scheme or not url.hostname or not url.path:
raise ConfigException('incomplete server address "%s"' % server +
' : you must specify scheme (http/https), server, URI')
self.scheme = url.scheme
self.hostname = url.hostname
self.port = url.port or (url.scheme == 'https' and 443 or 80)
self.path = url.path
self.username = extract_key('username')
self.password = extract_key('password')
self.xray_dir = extract_key('xray_dir')
self.convert_executable = extract_key('convert_executable')
try:
subprocess.check_call([
self.convert_executable,
'--version'
])
except subprocess.CalledProcessError as e:
raise ConfigException('convert executable "%s" raised error : %s' % (
self.convert_executable, str(e)))
except OSError as e:
if '[Errno 2]' in str(e):
raise ConfigException('could not find executable "%s"' %
self.convert_executable)
try:
self.id_re = re.compile(extract_key('id_re'))
self.manual_fields = {
key: re.compile(value)
for key, value in extract_key('manual_fields').items()
}
except RegularExpressionException as e:
raise ConfigException('invalid regular expression : ' + str(e))
self.xform = extract_key('xform')
try:
XForm(io.open(self.xform).read())
except XFormException as e:
raise ConfigException('could not load XForm "%s" : %s' % (
self.xform, str(e)))
self.pixels = extract_key('pixels')
self.auto = bool(extract_key('auto'))
if self.auto and self.manual_fields:
raise ConfigException('cannot perform automatic uploads ' +
'with non-empty manual_fields (%s)' %
', '.join(self.manual_fields))
try:
interval = extract_key('interval')
self.interval = float(interval)
except ValueError:
raise ConfigException('cannot parse interval "%s"' % interval)
for key in data:
lo.warning('ignoring config key : ' + key)
def __init__(self, filename):
"""Read JSON configuration from specified file
Rises ConfigException if errors are encountered
"""
try:
data = json.load(io.open(filename))
except ValueError as e:
raise ConfigException('not valid JSON file : ' + str(e))
def extract_key(key):
if not key in data:
raise ConfigException('missing key : ' + key)
value = data[key]
del data[key]
return value
server = extract_key('server')
url = urllib.parse.urlparse(server)
if not url.scheme or not url.hostname or not url.path:
raise ConfigException(
'incomplete server address "%s"' % server +
' : you must specify scheme (http/https), server, URI')
self.scheme = url.scheme
self.hostname = url.hostname
self.port = url.port or (url.scheme == 'https' and 443 or 80)
self.path = url.path
self.username = extract_key('username')
self.password = extract_key('password')
self.xray_dir = extract_key('xray_dir')
self.convert_executable = extract_key('convert_executable')
try:
subprocess.check_call([self.convert_executable, '--version'])
except subprocess.CalledProcessError as e:
raise ConfigException('convert executable "%s" raised error : %s' %
(self.convert_executable, str(e)))
except OSError as e:
if '[Errno 2]' in str(e):
raise ConfigException('could not find executable "%s"' %
self.convert_executable)
try:
self.id_re = re.compile(extract_key('id_re'))
self.manual_fields = {
key: re.compile(value)
for key, value in extract_key('manual_fields').items()
}
except RegularExpressionException as e:
raise ConfigException('invalid regular expression : ' + str(e))
self.xform = extract_key('xform')
try:
XForm(io.open(self.xform).read())
except XFormException as e:
raise ConfigException('could not load XForm "%s" : %s' %
(self.xform, str(e)))
self.pixels = extract_key('pixels')
self.auto = bool(extract_key('auto'))
if self.auto and self.manual_fields:
raise ConfigException('cannot perform automatic uploads ' +
'with non-empty manual_fields (%s)' %
', '.join(self.manual_fields))
try:
interval = extract_key('interval')
self.interval = float(interval)
except ValueError:
raise ConfigException('cannot parse interval "%s"' % interval)
for key in data:
lo.warning('ignoring config key : ' + key)
