def get_sweep_workload_specs(self, old_specs, sweep_spec, context):
new_specs = []
for old_spec in old_specs:
for freq in sweep_spec['frequencies']:
spec = old_spec.copy()
if 'runtime_params' in sweep_spec:
spec.runtime_parameters = merge_dicts(
spec.runtime_parameters,
sweep_spec['runtime_params'],
dict_type=OrderedDict)
if 'workload_params' in sweep_spec:
spec.workload_parameters = merge_dicts(
spec.workload_parameters,
sweep_spec['workload_params'],
dict_type=OrderedDict)
spec.runtime_parameters['{}_governor'.format(
sweep_spec['cluster'])] = "userspace"
spec.runtime_parameters['{}_frequency'.format(
sweep_spec['cluster'])] = freq
spec.id = '{}_{}_{}'.format(spec.id, sweep_spec['label'], freq)
spec.classifiers['core'] = sweep_spec['cluster']
spec.classifiers['freq'] = freq
spec.load(self.device, context.config.ext_loader)
spec.workload.init_resources(context)
spec.workload.validate()
new_specs.append(spec)
return new_specs
def get_sweep_workload_specs(self, old_specs, sweep_spec, context):
new_specs = []
for old_spec in old_specs:
for freq in sweep_spec['frequencies']:
spec = old_spec.copy()
if 'runtime_params' in sweep_spec:
spec.runtime_parameters = merge_dicts(spec.runtime_parameters,
sweep_spec['runtime_params'],
dict_type=OrderedDict)
if 'workload_params' in sweep_spec:
spec.workload_parameters = merge_dicts(spec.workload_parameters,
sweep_spec['workload_params'],
dict_type=OrderedDict)
spec.runtime_parameters['{}_governor'.format(sweep_spec['cluster'])] = "userspace"
spec.runtime_parameters['{}_frequency'.format(sweep_spec['cluster'])] = freq
spec.id = '{}_{}_{}'.format(spec.id, sweep_spec['label'], freq)
spec.classifiers['core'] = sweep_spec['cluster']
spec.classifiers['freq'] = freq
for k, v in sweep_spec.get('classifiers', {}).iteritems():
spec.classifiers[k] = v
spec.load(self.device, context.config.ext_loader)
spec.workload.init_resources(context)
spec.workload.validate()
new_specs.append(spec)
return new_specs
def test_merge_dict_lists(self):
base = {'a': [1, 3, 2]}
other = {'a': [3, 4, 5]}
result = merge_dicts(base, other)
assert_equal(result['a'], [1, 3, 2, 3, 4, 5])
result = merge_dicts(base, other, list_duplicates='first')
assert_equal(result['a'], [1, 3, 2, 4, 5])
result = merge_dicts(base, other, list_duplicates='last')
assert_equal(result['a'], [1, 2, 3, 4, 5])
def set(self, param, value):
if param in [
'id', 'section_id', 'number_of_iterations', 'workload_name',
'label'
]:
if value is not None:
setattr(self, param, value)
elif param in [
'boot_parameters', 'runtime_parameters', 'workload_parameters',
'flash'
]:
setattr(
self, param,
merge_dicts(getattr(self, param),
value,
list_duplicates='last',
dict_type=OrderedDict,
should_normalize=False))
elif param in ['instrumentation']:
setattr(
self, param,
merge_lists(getattr(self, param), value, duplicates='last'))
else:
raise ValueError(
'Unexpected workload spec parameter: {}'.format(param))
def combine(self, *args):
"""
Combine the provided values according to the method for this
configuration item. Order matters -- values are assumed to be
in the order they were specified by the user. The resulting value
is also checked to patch the specified type.
"""
args = [a for a in args if a is not None]
if not args:
return self.valid_categories[self.category]
if self.method == 'keep' or len(args) == 1:
value = args[0]
elif self.method == 'replace':
value = args[-1]
elif self.method == 'merge':
if self.category == 'list':
value = merge_lists(*args, duplicates='last', dict_type=OrderedDict)
elif self.category == 'dict':
value = merge_dicts(*args,
should_merge_lists=True,
should_normalize=False,
list_duplicates='last',
dict_type=OrderedDict)
else:
raise ValueError('Unexpected category for merge : "{}"'.format(self.category))
elif callable(self.method):
value = self.method(*args)
else:
raise ValueError('Unexpected method: "{}"'.format(self.method))
return value
def _merge_config_dicts(*args, **kwargs):
"""Provides a different set of default settings for ```merge_dicts`` """
return merge_dicts(*args,
should_merge_lists=kwargs.get('should_merge_lists', False),
should_normalize=kwargs.get('should_normalize', False),
list_duplicates=kwargs.get('list_duplicates', 'last'),
dict_type=kwargs.get('dict_type', OrderedDict))
def combine(self, *args):
"""
Combine the provided values according to the method for this
configuration item. Order matters -- values are assumed to be
in the order they were specified by the user. The resulting value
is also checked to patch the specified type.
"""
args = [a for a in args if a is not None]
if not args:
return self.valid_categories[self.category]
if self.method == 'keep' or len(args) == 1:
value = args[0]
elif self.method == 'replace':
value = args[-1]
elif self.method == 'merge':
if self.category == 'list':
value = merge_lists(*args,
duplicates='last',
dict_type=OrderedDict)
elif self.category == 'dict':
value = merge_dicts(*args,
should_merge_lists=True,
should_normalize=False,
list_duplicates='last',
dict_type=OrderedDict)
else:
raise ValueError('Unexpected category for merge : "{}"'.format(
self.category))
elif callable(self.method):
value = self.method(*args)
else:
raise ValueError('Unexpected method: "{}"'.format(self.method))
return value
def _merge_config_dicts(*args, **kwargs):
"""Provides a different set of default settings for ```merge_dicts`` """
return merge_dicts(*args,
should_merge_lists=kwargs.get('should_merge_lists',
False),
should_normalize=kwargs.get('should_normalize', False),
list_duplicates=kwargs.get('list_duplicates', 'last'),
dict_type=kwargs.get('dict_type', OrderedDict))
def get_default_config(self, ext_name):
"""
Returns the default configuration for the specified extension name. The name may be an alias,
in which case, the returned config will be augmented with appropriate alias overrides.
"""
real_name, alias_config = self.resolve_alias(ext_name)
base_default_config = self.get_extension_class(real_name).get_default_config()
return merge_dicts(base_default_config, alias_config, list_duplicates='last', dict_type=OrderedDict)
def test_dict_merge(self):
base = {'a': 1, 'b': {'x': 9, 'z': 10}}
other = {'b': {'x': 7, 'y': 8}, 'c': [1, 2, 3]}
result = merge_dicts(base, other)
assert_equal(result['a'], 1)
assert_equal(result['b']['x'], 7)
assert_equal(result['b']['y'], 8)
assert_equal(result['b']['z'], 10)
assert_equal(result['c'], [1, 2, 3])
def get_default_config(self, ext_name):
"""
Returns the default configuration for the specified extension name. The name may be an alias,
in which case, the returned config will be augmented with appropriate alias overrides.
"""
real_name, alias_config = self.resolve_alias(ext_name)
base_default_config = self.get_extension_class(real_name).get_default_config()
return merge_dicts(base_default_config, alias_config, list_duplicates='last', dict_type=OrderedDict)
def update_from_dict(self, source):
normalized_source = dict(
(identifier(k), v) for k, v in source.iteritems())
self._config = merge_dicts(self._config,
normalized_source,
list_duplicates='first',
match_types=False,
dict_type=OrderedDict)
self._loaded = True
def flash(self, image_bundle=None, images=None):
self.prelude_done = False
to_flash = {}
if image_bundle: # pylint: disable=access-member-before-definition
image_bundle = expand_path(image_bundle)
to_flash = self._bundle_to_images(image_bundle)
to_flash = merge_dicts(to_flash, images or {}, should_normalize=False)
for partition, image_path in to_flash.iteritems():
self.logger.debug('flashing {}'.format(partition))
self._flash_image(self.owner, partition, expand_path(image_path))
fastboot_command('reboot')
def set(self, param, value):
if param in ['id', 'section_id', 'number_of_iterations', 'workload_name', 'label']:
if value is not None:
setattr(self, param, value)
elif param in ['boot_parameters', 'runtime_parameters', 'workload_parameters', 'flash', 'classifiers']:
setattr(self, param, merge_dicts(getattr(self, param), value, list_duplicates='last',
dict_type=OrderedDict, should_normalize=False))
elif param in ['instrumentation']:
setattr(self, param, merge_lists(getattr(self, param), value, duplicates='last'))
else:
raise ValueError('Unexpected workload spec parameter: {}'.format(param))
def flash(self, image_bundle=None, images=None):
self.prelude_done = False
to_flash = {}
if image_bundle: # pylint: disable=access-member-before-definition
image_bundle = expand_path(image_bundle)
to_flash = self._bundle_to_images(image_bundle)
to_flash = merge_dicts(to_flash, images or {}, should_normalize=False)
for partition, image_path in to_flash.iteritems():
self.logger.debug('flashing {}'.format(partition))
self._flash_image(self.owner, partition, expand_path(image_path))
fastboot_command('reboot')
def add_metric(self,
name,
value,
units=None,
lower_is_better=False,
classifiers=None):
classifiers = merge_dicts(self.classifiers,
classifiers or {},
list_duplicates='last',
should_normalize=False)
self.metrics.append(
Metric(name, value, units, lower_is_better, classifiers))
def get_extension(self, name, *args, **kwargs):
"""
Return extension of the specified kind with the specified name. Any additional
parameters will be passed to the extension's __init__.
"""
name, base_kwargs = self.resolve_alias(name)
kind = kwargs.pop('kind', None)
kwargs = merge_dicts(base_kwargs, kwargs, list_duplicates='last', dict_type=OrderedDict)
cls = self.get_extension_class(name, kind)
extension = _instantiate(cls, args, kwargs)
extension.load_modules(self)
return extension
def get_extension(self, name, *args, **kwargs):
"""
Return extension of the specified kind with the specified name. Any additional
parameters will be passed to the extension's __init__.
"""
name, base_kwargs = self.resolve_alias(name)
kind = kwargs.pop('kind', None)
kwargs = merge_dicts(base_kwargs, kwargs, list_duplicates='last', dict_type=OrderedDict)
cls = self.get_extension_class(name, kind)
extension = _instantiate(cls, args, kwargs)
extension.load_modules(self)
return extension
def update_from_file(self, source):
ext = os.path.splitext(source)[1].lower() # pylint: disable=redefined-outer-name
try:
if ext in ['.py', '.pyo', '.pyc']:
new_config = load_struct_from_python(source)
elif ext == '.yaml':
new_config = load_struct_from_yaml(source)
else:
raise ConfigError('Unknown config format: {}'.format(source))
except (LoadSyntaxError, ValueError) as e:
raise ConfigError('Invalid config "{}":\n\t{}'.format(source, e))
self._config = merge_dicts(self._config, new_config,
list_duplicates='first',
match_types=False,
dict_type=OrderedDict)
self.loaded_files.append(source)
self._loaded = True
def update_from_file(self, source):
ext = os.path.splitext(source)[1].lower() # pylint: disable=redefined-outer-name
try:
if ext in ['.py', '.pyo', '.pyc']:
new_config = load_struct_from_python(source)
elif ext == '.yaml':
new_config = load_struct_from_yaml(source)
else:
raise ConfigError('Unknown config format: {}'.format(source))
except LoadSyntaxError as e:
raise ConfigError(e)
self._config = merge_dicts(self._config, new_config,
list_duplicates='first',
match_types=False,
dict_type=OrderedDict)
self.loaded_files.append(source)
self._loaded = True
def __init__(self, # pylint: disable=R0914,W0613
root_mount='/media/VEMSD',
disable_boot_configuration=False,
boot_firmware=None,
mode=None,
fs_medium='usb',
device_working_directory='/data/local/usecase',
bm_image='bm_v519r.axf',
serial_device='/dev/ttyS0',
serial_baud=38400,
serial_max_timeout=600,
serial_log=sys.stdout,
init_timeout=120,
always_delete_uefi_entry=True,
psci_enable=True,
host_working_directory=None,
a7_governor_tunables=None,
a15_governor_tunables=None,
adb_name=None,
# Compatibility with other android devices.
enable_screen_check=None, # pylint: disable=W0613
**kwargs
):
self.root_mount = root_mount
self.disable_boot_configuration = disable_boot_configuration
if not disable_boot_configuration:
self.boot_firmware = boot_firmware or 'uefi'
self.default_mode = mode or 'mp_a7_bootcluster'
elif boot_firmware or mode:
raise ConfigError('boot_firmware and/or mode cannot be specified when disable_boot_configuration is enabled.')
self.mode = self.default_mode
self.working_directory = device_working_directory
self.serial_device = serial_device
self.serial_baud = serial_baud
self.serial_max_timeout = serial_max_timeout
self.serial_log = serial_log
self.bootmon_prompt = re.compile('^([KLM]:\\\)?>', re.MULTILINE)
self.fs_medium = fs_medium.lower()
self.bm_image = bm_image
self.init_timeout = init_timeout
self.always_delete_uefi_entry = always_delete_uefi_entry
self.psci_enable = psci_enable
self.resource_dir = os.path.join(os.path.dirname(__file__), 'resources')
self.board_dir = os.path.join(self.root_mount, 'SITE1', 'HBI0249A')
self.board_file = 'board.txt'
self.board_file_bak = 'board.bak'
self.images_file = 'images.txt'
self.host_working_directory = host_working_directory or settings.meta_directory
if not a7_governor_tunables:
self.a7_governor_tunables = DEFAULT_A7_GOVERNOR_TUNABLES
else:
self.a7_governor_tunables = merge_dicts(DEFAULT_A7_GOVERNOR_TUNABLES, a7_governor_tunables)
if not a15_governor_tunables:
self.a15_governor_tunables = DEFAULT_A15_GOVERNOR_TUNABLES
else:
self.a15_governor_tunables = merge_dicts(DEFAULT_A15_GOVERNOR_TUNABLES, a15_governor_tunables)
self.adb_name = adb_name
def add_metric(self, name, value, units=None, lower_is_better=False, classifiers=None):
classifiers = merge_dicts(self.classifiers, classifiers or {},
list_duplicates='last', should_normalize=False)
self.metrics.append(Metric(name, value, units, lower_is_better, classifiers))
def update_from_dict(self, source):
normalized_source = dict((identifier(k), v) for k, v in source.iteritems())
self._config = merge_dicts(self._config, normalized_source, list_duplicates='first',
match_types=False, dict_type=OrderedDict)
self._loaded = True
def test_type_mismatch(self):
base = {'a': [1, 2, 3]}
other = {'a': 'test'}
merge_dicts(base, other, match_types=True)
def to_dict(self):
d = copy(self.__dict__)
d['uuid'] = str(self.uuid)
del d['config']
d = merge_dicts(d, self.config.to_dict())
return d
def to_dict(self):
d = copy(self.__dict__)
d["uuid"] = str(self.uuid)
del d["config"]
d = merge_dicts(d, self.config.to_dict())
return d
def __init__(
self, # pylint: disable=R0914,W0613
root_mount='/media/VEMSD',
disable_boot_configuration=False,
boot_firmware=None,
mode=None,
fs_medium='usb',
device_working_directory='/data/local/usecase',
bm_image='bm_v519r.axf',
serial_device='/dev/ttyS0',
serial_baud=38400,
serial_max_timeout=600,
serial_log=sys.stdout,
init_timeout=120,
always_delete_uefi_entry=True,
psci_enable=True,
host_working_directory=None,
a7_governor_tunables=None,
a15_governor_tunables=None,
adb_name=None,
# Compatibility with other android devices.
enable_screen_check=None, # pylint: disable=W0613
**kwargs):
self.root_mount = root_mount
self.disable_boot_configuration = disable_boot_configuration
if not disable_boot_configuration:
self.boot_firmware = boot_firmware or 'uefi'
self.default_mode = mode or 'mp_a7_bootcluster'
elif boot_firmware or mode:
raise ConfigError(
'boot_firmware and/or mode cannot be specified when disable_boot_configuration is enabled.'
)
self.mode = self.default_mode
self.working_directory = device_working_directory
self.serial_device = serial_device
self.serial_baud = serial_baud
self.serial_max_timeout = serial_max_timeout
self.serial_log = serial_log
self.bootmon_prompt = re.compile('^([KLM]:\\\)?>', re.MULTILINE)
self.fs_medium = fs_medium.lower()
self.bm_image = bm_image
self.init_timeout = init_timeout
self.always_delete_uefi_entry = always_delete_uefi_entry
self.psci_enable = psci_enable
self.resource_dir = os.path.join(os.path.dirname(__file__),
'resources')
self.board_dir = os.path.join(self.root_mount, 'SITE1', 'HBI0249A')
self.board_file = 'board.txt'
self.board_file_bak = 'board.bak'
self.images_file = 'images.txt'
self.host_working_directory = host_working_directory or settings.meta_directory
if not a7_governor_tunables:
self.a7_governor_tunables = DEFAULT_A7_GOVERNOR_TUNABLES
else:
self.a7_governor_tunables = merge_dicts(
DEFAULT_A7_GOVERNOR_TUNABLES, a7_governor_tunables)
if not a15_governor_tunables:
self.a15_governor_tunables = DEFAULT_A15_GOVERNOR_TUNABLES
else:
self.a15_governor_tunables = merge_dicts(
DEFAULT_A15_GOVERNOR_TUNABLES, a15_governor_tunables)
self.adb_name = adb_name
def to_dict(self):
d = copy(self.__dict__)
d['uuid'] = str(self.uuid)
del d['config']
d = merge_dicts(d, self.config.to_dict())
return d