def create_workload(name,
kind='basic',
where='local',
check_name=True,
**kwargs):
if check_name:
if name in [wl.name for wl in pluginloader.list_plugins('workload')]:
raise CommandError(
'Workload with name "{}" already exists.'.format(name))
class_name = get_class_name(name)
if where == 'local':
workload_dir = _d(os.path.join(settings.plugins_directory, name))
else:
workload_dir = _d(os.path.join(where, name))
try:
# Note: `create_funcs` mapping is listed below
create_funcs[kind](workload_dir, name, kind, class_name, **kwargs)
except KeyError:
raise CommandError('Unknown workload type: {}'.format(kind))
# pylint: disable=superfluous-parens
print('Workload created in {}'.format(workload_dir))
def execute(self, state, args): # pylint: disable=too-many-branches
if not psycopg2:
raise CommandError('The module psycopg2 is required for the wa ' +
'create database command.')
self.get_schema(self.schemafilepath)
# Display the version if needed and exit
if args.schema_version:
self.logger.info('The current schema version is {}'.format(
self.schemaversion))
return
if args.dbname == 'postgres':
raise ValueError('Databasename to create cannot be postgres.')
# Open user configuration
with open(args.config_file, 'r') as config_file:
config = yaml.load(config_file)
if 'postgres' in config and not args.force_update_config:
raise CommandError(
"The entry 'postgres' already exists in the config file. "
+ "Please specify the -F flag to force an update.")
possible_connection_errors = [
(re.compile('FATAL: role ".*" does not exist'),
'Username does not exist or password is incorrect'),
(re.compile('FATAL: password authentication failed for user'),
'Password was incorrect'),
(re.compile('fe_sendauth: no password supplied'),
'Passwordless connection is not enabled. '
'Please enable trust in pg_hba for this host '
'or use a password'),
(re.compile('FATAL: no pg_hba.conf entry for'),
'Host is not allowed to connect to the specified database '
'using this user according to pg_hba.conf. Please change the '
'rules in pg_hba or your connection method'),
(re.compile('FATAL: pg_hba.conf rejects connection'),
'Connection was rejected by pg_hba.conf'),
]
def predicate(error, handle):
if handle[0].match(str(error)):
raise CommandError(handle[1] + ': \n' + str(error))
# Attempt to create database
try:
self.create_database(args)
except OperationalError as e:
for handle in possible_connection_errors:
predicate(e, handle)
raise e
# Update the configuration file
_update_configuration_file(args, config)
def execute(self, state, args):
for subcmd in self.subcommands:
if subcmd.name == args.what:
subcmd.execute(state, args)
break
else:
raise CommandError('Not a valid create parameter: {}'.format(args.name))
def _validate_version(self):
conn = connect(user=self.username,
password=self.password,
host=self.postgres_host,
port=self.postgres_port)
if conn.server_version < 90400:
msg = 'Postgres version too low. Please ensure that you are using atleast v9.4'
raise CommandError(msg)
def _check_database_existence(self):
try:
connect(dbname=self.dbname,
user=self.username,
password=self.password,
host=self.postgres_host,
port=self.postgres_port)
except OperationalError as e:
# Expect an operational error (database's non-existence)
if not re.compile('FATAL: database ".*" does not exist').match(
str(e)):
raise e
else:
if not self.force:
raise CommandError(
"Database {} already exists. ".format(self.dbname) +
"Please specify the -f flag to create it from afresh.")
def create_extensions_package(self,
location,
name,
setup_template_path,
overwrite=False):
package_path = os.path.join(location, name)
if os.path.exists(package_path):
if overwrite:
self.logger.info(
'overwriting existing "{}"'.format(package_path))
shutil.rmtree(package_path)
else:
raise CommandError(
'Location "{}" already exists.'.format(package_path))
actual_package_path = os.path.join(package_path, name)
os.makedirs(actual_package_path)
setup_text = render_template(setup_template_path, {
'package_name': name,
'user': getpass.getuser()
})
with open(os.path.join(package_path, 'setup.py'), 'w') as wfh:
wfh.write(setup_text)
touch(os.path.join(actual_package_path, '__init__.py'))
def execute(self, config, args): # pylint: disable=arguments-differ,too-many-branches,too-many-statements
process_directory = os.path.expandvars(args.directory)
self.logger.debug(
'Using process directory: {}'.format(process_directory))
if not os.path.exists(process_directory):
msg = 'Path `{}` does not exist, please specify a valid path.'
raise CommandError(msg.format(process_directory))
if not args.recursive:
output_list = [RunOutput(process_directory)]
else:
output_list = [
output for output in discover_wa_outputs(process_directory)
]
pc = ProcessContext()
for run_output in output_list:
if not args.recursive:
self.logger.info('Installing output processors')
else:
self.logger.info(
'Install output processors for run in path `{}`'.format(
run_output.basepath))
logfile = os.path.join(run_output.basepath, 'process.log')
i = 0
while os.path.exists(logfile):
i += 1
logfile = os.path.join(run_output.basepath,
'process-{}.log'.format(i))
log.add_file(logfile)
pm = ProcessorManager(loader=config.plugin_cache)
for proc in config.get_processors():
pm.install(proc, pc)
if args.additional_processors:
for proc in args.additional_processors:
# Do not add any processors that are already present since
# duplicate entries do not get disabled.
try:
pm.get_output_processor(proc)
except ValueError:
pm.install(proc, pc)
pm.validate()
pm.initialize(pc)
pc.run_output = run_output
pc.target_info = run_output.target_info
for job_output in run_output.jobs:
pc.job_output = job_output
pm.enable_all()
if not args.force:
for augmentation in job_output.spec.augmentations:
try:
pm.disable(augmentation)
except ValueError:
pass
msg = 'Processing job {} {} iteration {}'
self.logger.info(
msg.format(job_output.id, job_output.label,
job_output.iteration))
pm.process_job_output(pc)
pm.export_job_output(pc)
job_output.write_result()
pm.enable_all()
if not args.force:
for augmentation in run_output.augmentations:
try:
pm.disable(augmentation)
except ValueError:
pass
self.logger.info('Processing run')
pm.process_run_output(pc)
pm.export_run_output(pc)
pm.finalize(pc)
run_output.write_result()
self.logger.info('Done.')
def predicate(error, handle):
if handle[0].match(str(error)):
raise CommandError(handle[1] + ': \n' + str(error))
