def boot(self):
"""Boot up the experiment, e.g. load config etc."""
# Load Config/Args for experiment
if self.config_file:
try:
with open(os.path.join(self._path, self.config_file),
'r') as cfg:
self.config.set(json.load(cfg))
except Exception as exc:
print_failure(exc, 2)
# Load Experiment and testcase repos
try:
self.repos['experiment'] = self.app.repositories.get(
'ExperimentRepository')
self.repos['testcase'] = self.app.repositories.get(
'TestCaseRepository')
self.repos['experiment'] = self.repos['experiment'].from_dict({
'name':
self.__class__.__name__.replace('Experiment', ''),
'start':
datetime.now(),
'config_file':
self.config_file,
'config_content':
json.dumps(self.config.all()),
'tests': []
})
self.repos['experiment'].create()
except Exception as exc:
print_failure(exc, 2)
def up(self, migration=None):
"""Upgrade to a new migration revision.
Args:
migration (Migration, optional): Defaults to None. The Migration Class to upgrade to.
Raises:
TypeError: if migration is not a valid migration
"""
# Select first migration to run
if migration is None:
migrated = self._get_migrated_revisions()
migrations = self.get_migration_files(self.path)
migration = [mig for mig in migrations if mig[:14] not in migrated]
if len(migration) == 0:
print(colored('› Migrations are all up to date.', 'green'))
return
migration = self.migrations.get(migration[0])
# update migration
try:
migration.up()
self._update_revision(migration.revision)
print(colored('› Migrated', 'green'), colored(migration, 'cyan'))
except Exception as exc:
print_failure(
'Error while ugrading migration {}: {}'.format(migration, exc),
1)
def status(app):
"""List experiment status.
Args:
app (App): App Service Container.
"""
data = []
headers = [
colored('Experiment', 'yellow'),
colored('Config File', 'yellow'),
colored('Times Executed', 'yellow')
]
try:
exps = Experiment.get_status(app)
for experiment in exps.values():
data.append([
colored(experiment['name'], 'cyan'),
colored(experiment.get('config_file'), 'cyan'),
colored(experiment['count'], 'cyan')
])
except Exception as exc:
print_failure(exc, 2)
print(tabulate(data, headers=headers, tablefmt='psql'))
def add_command(self, name, cmd):
"""Add a new command to the parser.
Args:
name (str): Name of the command
cmd (function, AbstractCommand): Command Handler
"""
if (not isinstance(cmd, types.FunctionType)
and not issubclass(cmd, AbstractCommand)):
print_failure(
"{}-Command must inherit from AbstractCommand!".format(name),
1)
# setup command
cmd = cmd() # type: AbstractCommand
command = self._subparsers.add_parser(
name,
help=cmd.help,
description=colored(cmd.description, 'yellow'),
formatter_class=ColoredHelpFormatter,
add_help=False)
command.add_argument('-h',
'--help',
action='help',
default=argparse.SUPPRESS,
help='Show this help message and exit.')
command.titles('Arguments', 'Options', color='cyan')
# Add arguments and bind command
for arg, opt in cmd.arguments.items():
command.add_argument(arg, **opt)
command.set_defaults(func=cmd.handle)
self.commands[name] = command
def create(self, name):
"""Create a new plot class instance.
Args:
name (str): Name of the plot.
Returns:
AbstractPlot: Plot Class
"""
# Get plot data from config and validate it
data = self.parse(name)
config = self.validate(data)
# Load plot class
path = self.app.config.get('app.plots.path', 'plots')
plots = find_files(self.app.root, path, name, '(plot|chart|graph).py')
if not plots:
print_failure(
'Could not find plot named "{}" under path "{}"'.format(
name, path),
exit_code=1)
plot = load_class(plots[0], path, AbstractPlot)
# Init plot class
return plot(self.app.repositories.get('ExperimentRepository'), config,
config.get('type', 'plot'))
def load_class(src, module, subclass=None):
"""Try to load a class from a module.
Args:
src (str): Source file
module (str): Module name
subclass (object, optional): Defaults to None. Object the class needs to be derived from
Returns:
object: loaded class
"""
with open(src, 'rb') as filehandler:
# try to load class from module
try:
mod = imp.load_source(module, src, filehandler)
klass = getattr(mod, os.path.basename(src)[:-3])
except Exception as exc:
print_failure('Could not load file: %s' % str(exc), exit_code=2)
# Check if class is derived from subclass
if subclass and issubclass(klass, subclass) is False:
msg = '{} must be derived from the {}.{} class'.format(
klass.__name__, subclass.__module__, subclass.__name__)
print_failure(msg, exit_code=3)
return klass
def down(self, migration=None):
"""Downgrade to an old migration revision.
Args:
migration (Migration, optional): Defaults to None. The Migration Class to upgrade to.
Raises:
TypeError: if migration is not a valid migration
"""
# Select first migration to run
if migration is None:
# get finished migrations
migrated = self._get_migrated_revisions()
migrations = self.get_migration_files(self.path)[::-1]
migration = [mig for mig in migrations if mig[:14] in migrated]
if len(migration) == 0:
print(colored('× There are no migrations to downgrade.',
'red'))
return
migration = self.migrations.get(migration[0])
# downgrade migration
try:
migration.down()
self._update_revision(migration.revision, delete=True)
print(colored('› Migrated', 'green'), colored(migration, 'cyan'))
except Exception as exc:
print_failure(
'Error while downgrading migration {}: {}'.format(
migration, exc), 1)
def dispatch(self):
"""Use dispatch pattern to invoke class and let it handle the command."""
try:
# dispatch
options = self._parser.parse_args()
# only dispatch args if available
if len(vars(options)) > 1:
options.func(self.app, options)
else:
options.func(self.app)
except AttributeError:
# no command selected
print_failure('Please specify a command')
self._parser.print_help(sys.stderr)
sys.exit(2)
def bootstrap(self):
"""Bootstrap the app, i.e. setup config and logger."""
# Load Config
loader = Loader(_path_join(self.root, self.config_path), self.config)
loader.load_config_files()
# Add experiments and repo folders to path
paths = [
self.config.get('app.experiments.path', 'experiments'),
self.config.get('storage.repositories.path', 'repositories')
]
for path in paths:
sys.path.insert(0, os.path.abspath(_path_join(self.root, path)))
# Setup logger
self._set_logger()
self.log.info('Bootstrap App')
# Setup Datastore
self.log.info('Setup Data Store')
default_db = {'drivername': 'sqlite', 'database': 'experimentum.db'}
database = self.config.get('storage.datastore', default_db)
# Absolute file path based on root for sqlite databases (except in-memory)
if database['drivername'] == 'sqlite' and database['database'] != '':
database['database'] = _path_join(self.root, database['database'])
self.setup_datastore(database)
if not isinstance(self.store, AbstractStore):
msg = 'The "{}" Store implementation must implement the AbstractStore interface'
print_failure(msg.format(self.store.__class__), 1)
# Load and map Repositories
self.repositories = RepositoryLoader(self, self.base_repository,
self.store)
self.repositories.load()
# Aliases
self.register_aliases()
# Register commands
self.log.info('Register Commands')
self.cmd_manager = CommandManager(
self, self.config.get('app.prog', 'app'),
self.config.get('app.description', ''))
self._add_commands()
def table(self, name):
"""Create a blueprint for an existing table.
Args:
name (str): Name of the table
Yields:
Blueprint: New Instance of a table blueprint
"""
try:
blueprint = self.app.make('blueprint', name)
yield blueprint
except Exception as exc:
print_failure('Error while creating blueprint: ' + str(exc), 1)
self._build(blueprint)
def parse(self, plot):
"""Parse the config file for the current plot.
Args:
plot (str): Name of the plot.
Returns:
object: Config data for the plot
"""
config = self.app.config.get('plots.{}'.format(plot), None)
if config is None:
print_failure('Could not parse config for plot "%s". '
'Make sure it exists in the config file!' % plot,
exit_code=1)
return config
def get(self, repository):
"""Return class of a loaded repository if it exists.
Args:
repository (str): Name of repository class.
Returns:
AbstractRepository: Repository class
"""
repo = self._repos.get(repository, None)
if repo is None:
print_failure(
'Could not load repository {}. Are you sure it exists?'.format(
repository),
exit_code=1)
return repo
def fib(n, seq):
"""Calculate the nth fibonacci number.
Args:
n (int)
seq (list): List of fibonacci numbers
Returns:
int
"""
if n < 0:
print_failure('Incorrect Input', exit_code=1)
elif n <= len(seq):
return seq[n - 1]
tmp = fib(n - 1, seq) + fib(n - 2, seq)
seq.append(tmp)
return tmp
def make(self, alias, *args, **kwargs):
"""Create an instance of an aliased class.
Args:
alias (str): Name of class alias.
Raises:
Exception: if the alias does not exists.
Returns:
object: Instance of the aliased class
"""
klass = self.aliases.get(alias, None)
if klass is None:
print_failure(
"Class with alias '{}' does not exist.".format(alias), 1)
return klass(*args, **kwargs)
def save(self, result, iteration):
"""Save the test results in the data store.
Args:
result (dict): Result of experiment test run.
iteration (int): Number of test run iteration.
"""
data = {
'experiment_id': self.repos['experiment'].id,
'iteration': iteration,
'performances': []
}
data.update(result)
data['performances'].extend(self.performance.export())
try:
self.repos['testcase'].from_dict(data).create()
except Exception as exc:
for msg in str(exc).split('\n'):
print_failure(msg)
raise SystemExit(-1)
def load(self):
"""Load all repositories from the repo folder and try to map them to the store."""
# Get Repository files
files = find_files(
self.app.root,
self.app.config.get('storage.repositories.path', 'repositories'))
for filename in files:
name = os.path.basename(filename)[:-3]
# Try to load and map the repos
try:
module = self._import_file(
name, os.path.abspath(os.path.dirname(filename)))
repo = getattr(module, name)
repo.mapping(repo, self.store)
self._repos[name] = repo
except Exception as exc:
print_failure('Could not load repository. ' + str(exc),
exit_code=1)
def load(app, path, name):
"""Load and initialize an experiment class.
Args:
app (App): Main app calss
path (str): Path to experiments folder.
name (str): Name of experiment.
Returns:
Experiment: Loaded experiment.
"""
# Find Experiment Files
files = find_files(app.root, path, name, remove='experiment.py')
if not files:
print_failure(
'Could not find experiment named "{}" under path "{}"'.format(
name, path),
exit_code=1)
# Load Experiment class if possible
experiment = load_class(files[0], 'experiments', Experiment)
return experiment(app, path)
def from_dict(cls, data):
"""Create a new Repository instance based on a dictionary entry.
Args:
data (dict): Repository Data
Returns:
AbstractRepository: Repository instance filled with data.
"""
init = {}
relations = {}
relationships = cls.__relationships__
for key, val in data.items():
if key in relationships:
if isinstance(val, list):
relations[key] = [
relationships[key][0].from_dict(v) for v in val
]
else:
relations[key] = [relationships[key][0].from_dict(val)]
else:
init[key] = val
try:
repo = cls(**init)
except TypeError as err:
# Unexpected Argument
if 'unexpected keyword argument' in str(err):
print_failure(
"{}. Either fix the typo or add it the repository constructor."
.format(str(err).replace('__init__()', cls.__name__), ), 1)
# Determine missing parameters
elif 'required positional argument' in str(
err) or 'takes exactly' in str(err):
import inspect
if hasattr(inspect, 'getfullargspec'):
getargspec = inspect.getfullargspec
else:
getargspec = inspect.getargspec
available = set(getargspec(cls.__init__).args[1:])
passed = set(init.keys())
print_failure(
"The {} class is missing the following parameters: {}".
format(cls.__name__, available - passed), 2)
else:
print_failure(err, -1)
for key, val in relations.items():
repo[key] = val
return repo
def test_print_failure_with_exit(capsys):
with pytest.raises(SystemExit) as pytest_wrapped_e:
print_failure('My failure message', 2)
assert 'My failure message' in capsys.readouterr().err
assert pytest_wrapped_e.type == SystemExit
assert pytest_wrapped_e.value.code == 2
def test_print_failure(capsys):
print_failure('My failure message')
assert 'My failure message' in capsys.readouterr().err