def check_if_substitution_key(self, section_name, property_names):
result = [(property_name, False) for property_name in property_names]
if not self.is_substitution_allowed():
return result
section_name = JSONSchema.format_section_name(section_name).lower()
property_names = [
JSONSchema.format_property_name(property_name)
for property_name in property_names
]
section_property_name = self.get_property_default_value(
section_name, JSONSchema.NAME)
section_property_name = self.get_section_property(
section_name, JSONSchema.NAME, section_property_name)
for key in self._substitutions.keys():
key_items = key.split('.')
if len(key_items) == 3 and \
key_items[0] == section_name and \
key_items[1] == section_property_name and \
key_items[2] in property_names:
result[property_names.index(key_items[2])] = (key_items[2],
True)
continue
return result
def delete_section_property(self, section_name, property_name):
"""
Args:
section_name (str): the name of the section, case insensitive
property_name (str): the property name in the section
"""
section_name = JSONSchema.format_section_name(section_name).lower()
property_name = JSONSchema.format_property_name(property_name)
rebuild_data = False
if section_name in self._sections and \
'properties' in self._sections[section_name] and \
property_name in self._sections[section_name]['properties']:
del self._sections[section_name]['properties'][property_name]
rebuild_data = True
if rebuild_data:
contents = ''
properties = self._sections[section_name]['properties']
lastIndex = len(properties) - 1
for i, (key, value) in enumerate(properties.items()):
contents += '{}{}{}'.format(key,
InputParser._PROPVALUE_SEPARATOR,
value)
if i < lastIndex:
contents += '\n'
self._sections[section_name]['data'] = contents
def set_section_property(self, section_name, property_name, value):
section_name = JSONSchema.format_section_name(section_name).lower()
property_name = JSONSchema.format_property_name(property_name)
value = self._json_schema.check_property_value(section_name,
property_name, value)
types = self.get_property_types(section_name, property_name)
parser_temp = copy.deepcopy(self)
InputParser._set_section_property(parser_temp._sections, section_name,
property_name, value, types)
msg = self._json_schema.validate_property(parser_temp.to_JSON(),
section_name, property_name)
if msg is not None:
raise AquaChemistryError("{}.{}: Value '{}': '{}'".format(
section_name, property_name, value, msg))
InputParser._set_section_property(self._sections, section_name,
property_name, value, types)
if property_name == JSONSchema.NAME:
if InputParser.OPERATOR == section_name:
self._update_operator_input_schema()
# remove properties that are not valid for this section
default_properties = self.get_section_default_properties(
section_name)
if isinstance(default_properties, dict):
properties = self.get_section_properties(section_name)
for property_name in list(properties.keys()):
if property_name != JSONSchema.NAME and property_name not in default_properties:
self.delete_section_property(
section_name, property_name)
elif JSONSchema.PROBLEM == section_name:
self._update_algorithm_problem()
self._update_operator_problem()
elif InputParser.is_pluggable_section(section_name):
self._json_schema.update_pluggable_input_schemas(self)
# remove properties that are not valid for this section
default_properties = self.get_section_default_properties(
section_name)
if isinstance(default_properties, dict):
properties = self.get_section_properties(section_name)
for property_name in list(properties.keys()):
if property_name != JSONSchema.NAME and property_name not in default_properties:
self.delete_section_property(
section_name, property_name)
if section_name == JSONSchema.ALGORITHM:
self._update_dependency_sections()
elif value is not None:
value = str(value).lower().strip()
if len(value) > 0 and self.section_is_driver(value):
self._update_driver_input_schemas()
self._update_driver_sections()
self._sections = self._order_sections(self._sections)
def validate(self, args_dict):
schema_dict = self.CONFIGURATION.get('input_schema', None)
if schema_dict is None:
return
jsonSchema = JSONSchema(schema_dict)
schema_property_names = jsonSchema.get_default_section_names()
json_dict = {}
for property_name in schema_property_names:
if property_name in args_dict:
json_dict[property_name] = args_dict[property_name]
jsonSchema.validate(json_dict)
def is_pluggable_section(section_name):
section_name = JSONSchema.format_section_name(section_name)
for pluggable_type in local_pluggables_types():
if section_name == pluggable_type.value:
return True
return False
def section_is_text(self, section_name):
section_name = JSONSchema.format_section_name(section_name).lower()
types = self.get_section_types(section_name)
if len(types) > 0:
return 'string' in types
return False
def delete_section_properties(self, section_name):
"""
Args:
section_name (str): the name of the section, case insensitive
"""
section_name = JSONSchema.format_section_name(section_name).lower()
if section_name in self._sections:
del self._sections[section_name]
def get_section_property(self, section_name, property_name, default_value=None):
"""Return a property by name.
Args:
section_name (str): the name of the section, case insensitive
property_name (str): the property name in the section
default_value : default value in case it is not found
Returns:
Value: The property value
"""
section_name = JSONSchema.format_section_name(section_name).lower()
property_name = JSONSchema.format_property_name(property_name)
if section_name in self._sections:
section = self._sections[section_name]
if 'properties' in section and property_name in section['properties']:
return section['properties'][property_name]
return default_value
def __init__(self, input=None):
"""Create InputParser object."""
self._sections = OrderedDict()
self._original_sections = OrderedDict()
self._filename = None
self._inputdict = None
if input is not None:
if isinstance(input, dict):
self._inputdict = input
elif isinstance(input, str):
self._filename = input
else:
raise AquaChemistryError("Invalid parser input type.")
self._section_order = [
JSONSchema.NAME, JSONSchema.PROBLEM, InputParser.DRIVER,
InputParser._UNKNOWN, InputParser.OPERATOR, JSONSchema.ALGORITHM
]
for pluggable_type in local_pluggables_types():
if pluggable_type != JSONSchema.ALGORITHM:
self._section_order.append(pluggable_type)
self._section_order.append(JSONSchema.BACKEND)
jsonfile = os.path.join(os.path.dirname(__file__),
'substitutions.json')
with open(jsonfile) as json_file:
self._substitutions = json.load(json_file)
self._json_schema = JSONSchema(
os.path.join(os.path.dirname(__file__), 'input_schema.json'))
# get some properties from algorithms schema
self._json_schema.copy_section_from_aqua_schema(JSONSchema.ALGORITHM)
self._json_schema.copy_section_from_aqua_schema(JSONSchema.BACKEND)
self._json_schema.copy_section_from_aqua_schema(JSONSchema.PROBLEM)
self._json_schema.schema['properties'][JSONSchema.PROBLEM][
'properties'][InputParser.AUTO_SUBSTITUTIONS] = {
"type": "boolean",
"default": "true"
}
self._json_schema.populate_problem_names()
self._json_schema.commit_changes()
def delete_section_data(self, section_name):
"""
Deletes a section data.
Args:
section_name (str): the name of the section, case insensitive
"""
section_name = JSONSchema.format_section_name(section_name).lower()
if section_name in self._sections:
self._sections[section_name]['data'] = ''
self._sections[section_name]['properties'] = OrderedDict()
def set_section(self, section_name):
"""
Args:
section_name (str): the name of the section, case insensitive
"""
section_name = JSONSchema.format_section_name(section_name).lower()
if section_name not in self._sections:
self._sections[section_name] = OrderedDict([(JSONSchema.NAME,
section_name)])
self._sections[section_name]['properties'] = OrderedDict()
self._sections[section_name]['data'] = ''
self._sections = self._order_sections(self._sections)
def _set_section_property(sections, section_name, property_name, value,
types):
"""
Args:
section_name (str): the name of the section, case insensitive
property_name (str): the property name in the section
value : property value
types : schema valid types
"""
section_name = JSONSchema.format_section_name(section_name).lower()
property_name = JSONSchema.format_property_name(property_name)
value = JSONSchema.get_value(value, types)
if section_name not in sections:
sections[section_name] = OrderedDict([(JSONSchema.NAME,
section_name)])
if 'properties' not in sections[section_name]:
sections[section_name]['properties'] = OrderedDict()
# name should come first
if JSONSchema.NAME == property_name and property_name not in sections[
section_name]['properties']:
new_dict = OrderedDict([(property_name, value)])
new_dict.update(sections[section_name]['properties'])
sections[section_name]['properties'] = new_dict
else:
sections[section_name]['properties'][property_name] = value
# rebuild data
contents = ''
properties = sections[section_name]['properties']
lastIndex = len(properties) - 1
for i, (key, value) in enumerate(properties.items()):
contents += '{}{}{}'.format(key, InputParser._PROPVALUE_SEPARATOR,
value)
if i < lastIndex:
contents += '\n'
sections[section_name]['data'] = contents
def get_section(self, section_name):
"""Return a Section by name.
Args:
section_name (str): the name of the section, case insensitive
Returns:
Section: The section with this name
Raises:
AquaChemistryError: if the section does not exist.
"""
section_name = JSONSchema.format_section_name(section_name).lower()
try:
return self._sections[section_name]
except KeyError:
raise AquaChemistryError('No section "{0}"'.format(section_name))
def delete_section(self, section_name):
"""
Args:
section_name (str): the name of the section, case insensitive
"""
section_name = JSONSchema.format_section_name(section_name).lower()
if section_name not in self._sections:
return
del self._sections[section_name]
# update schema
self._json_schema.rollback_changes()
self._json_schema.update_pluggable_input_schemas(self)
self._update_driver_input_schemas()
self._update_operator_input_schema()
def get_section_data(self, section_name, default_value=None):
"""
Return a section data.
Args:
section_name (str): the name of the section, case insensitive
default_value : default value in case it is not found
Returns:
Value: data value
"""
section_name = JSONSchema.format_section_name(section_name).lower()
if section_name in self._sections:
section = self._sections[section_name]
if 'data' in section:
return section['data']
return default_value
def set_section_data(self, section_name, value):
"""
Sets a section data.
Args:
section_name (str): the name of the section, case insensitive
value : value to set
"""
section_name = JSONSchema.format_section_name(section_name).lower()
value = self._json_schema.check_section_value(section_name, value)
self._sections[section_name] = OrderedDict([(JSONSchema.NAME,
section_name)])
self._sections[section_name]['data'] = value
properties = OrderedDict()
if value is not None:
lines = str(value).splitlines()
for line in lines:
k, v = self._get_key_value(line)
if k is not None and v is not None:
properties[k] = v
self._sections[section_name]['properties'] = properties
def _get_key_value(line):
stripLine = line.strip()
pos = -1
for start_comment in InputParser._START_COMMENTS:
pos = stripLine.find(start_comment)
if pos >= 0:
break
if pos == 0:
return (None, None)
if pos > 0:
stripLine = stripLine[:pos].strip()
pos = stripLine.find(InputParser._PROPVALUE_SEPARATOR)
if pos > 0:
key = stripLine[0:pos].strip()
value = stripLine[pos + 1:].strip()
return (key, JSONSchema.get_value(value))
return (None, None)
def _load_parser_from_dict(self):
self._sections = OrderedDict()
for section_name, value in self._inputdict.items():
section_name = JSONSchema.format_section_name(section_name).lower()
self._sections[section_name] = OrderedDict()
self._sections[section_name]['properties'] = OrderedDict()
self._sections[section_name]['data'] = ''
if isinstance(value, dict):
for k, v in value.items():
self._sections[section_name]['properties'][k] = v
contents = ''
properties = self._sections[section_name]['properties']
lastIndex = len(properties) - 1
for i, (k, v) in enumerate(properties.items()):
contents += '{}{}{}'.format(
k, InputParser._PROPVALUE_SEPARATOR, v)
if i < lastIndex:
contents += '\n'
self._sections[section_name]['data'] = contents
elif isinstance(value, list) or isinstance(value, str):
lines = []
if isinstance(value, list):
lines = value
self._sections[section_name]['data'] = '\n'.join(
str(e) for e in value)
else:
lines = value.splitlines()
self._sections[section_name]['data'] = value
for line in lines:
k, v = self._get_key_value(line)
if k is not None and v is not None:
self._sections[section_name]['properties'][k] = v
else:
raise AquaChemistryError(
"Invalid parser input type for section {}".format(
section_name))
def section_is_driver(self, section_name):
section_name = JSONSchema.format_section_name(section_name).lower()
InputParser._load_driver_names()
return section_name in InputParser._DRIVER_NAMES
def is_pluggable_section(section_name):
return JSONSchema.format_section_name(
section_name).lower() in local_pluggables_types()
class InputParser(object):
"""Common input file parser."""
OPERATOR = 'operator'
DRIVER = 'driver'
AUTO_SUBSTITUTIONS = 'auto_substitutions'
_OLD_ENABLE_SUBSTITUTIONS = 'enable_substitutions'
_START_COMMENTS = ['#', '%']
_START_SECTION = '&'
_END_SECTION = '&end'
_PROPVALUE_SEPARATOR = '='
_OPTIMIZER = 'optimizer'
_VARIATIONAL_FORM = 'variational_form'
_UNKNOWN = 'unknown'
_HDF5_INPUT = 'hdf5_input'
_DRIVER_NAMES = None
_PROPERTY_ORDER = [JSONSchema.NAME, _UNKNOWN]
def __init__(self, input=None):
"""Create InputParser object."""
self._sections = OrderedDict()
self._original_sections = OrderedDict()
self._filename = None
self._inputdict = None
if input is not None:
if isinstance(input, dict):
self._inputdict = input
elif isinstance(input, str):
self._filename = input
else:
raise AquaChemistryError("Invalid parser input type.")
self._section_order = [
JSONSchema.NAME, JSONSchema.PROBLEM, InputParser.DRIVER,
InputParser._UNKNOWN, InputParser.OPERATOR, JSONSchema.ALGORITHM
]
for pluggable_type in local_pluggables_types():
if pluggable_type != JSONSchema.ALGORITHM:
self._section_order.append(pluggable_type)
self._section_order.append(JSONSchema.BACKEND)
jsonfile = os.path.join(os.path.dirname(__file__),
'substitutions.json')
with open(jsonfile) as json_file:
self._substitutions = json.load(json_file)
self._json_schema = JSONSchema(
os.path.join(os.path.dirname(__file__), 'input_schema.json'))
# get some properties from algorithms schema
self._json_schema.copy_section_from_aqua_schema(JSONSchema.ALGORITHM)
self._json_schema.copy_section_from_aqua_schema(JSONSchema.BACKEND)
self._json_schema.copy_section_from_aqua_schema(JSONSchema.PROBLEM)
self._json_schema.schema['properties'][JSONSchema.PROBLEM][
'properties'][InputParser.AUTO_SUBSTITUTIONS] = {
"type": "boolean",
"default": "true"
}
self._json_schema.populate_problem_names()
self._json_schema.commit_changes()
#logger.debug('Resolved Schema Input: {}'.format(json.dumps(self._json_schema.schema, sort_keys=True, indent=4)))
def _order_sections(self, sections):
sections_sorted = OrderedDict(
sorted(list(sections.items()),
key=lambda x: self._section_order.index(x[0])
if x[0] in self._section_order else self._section_order.
index(InputParser._UNKNOWN)))
for section, values in sections_sorted.items():
if not self.section_is_driver(
section) and 'properties' in values and isinstance(
values['properties'], dict):
sections_sorted[section]['properties'] = OrderedDict(
sorted(
list(values['properties'].items()),
key=lambda x: InputParser._PROPERTY_ORDER.index(x[0])
if x[0] in InputParser._PROPERTY_ORDER else InputParser
._PROPERTY_ORDER.index(InputParser._UNKNOWN)))
return sections_sorted
def parse(self):
"""Parse the data."""
if self._inputdict is None:
if self._filename is None:
raise AquaChemistryError("Missing input file")
section = None
self._sections = OrderedDict()
contents = ''
with open(self._filename, 'rt', encoding="utf8",
errors='ignore') as f:
for line in f:
contents += line
section = self._process_line(section, line)
contents = contents.strip().replace('\n', '').replace('\r', '')
if not (self._sections) and len(contents) > 0:
# check if input file was dictionary
try:
v = ast.literal_eval(contents)
if isinstance(v, dict):
self._inputdict = json.loads(json.dumps(v))
self._load_parser_from_dict()
except:
pass
else:
self._load_parser_from_dict()
# check for old enable_substitutions name
old_enable_substitutions = self.get_section_property(
JSONSchema.PROBLEM, InputParser._OLD_ENABLE_SUBSTITUTIONS)
if old_enable_substitutions is not None:
self.delete_section_property(JSONSchema.PROBLEM,
InputParser._OLD_ENABLE_SUBSTITUTIONS)
self.set_section_property(JSONSchema.PROBLEM,
InputParser.AUTO_SUBSTITUTIONS,
old_enable_substitutions)
self._json_schema.update_pluggable_input_schemas(self)
self._update_driver_input_schemas()
self._update_operator_input_schema()
self._sections = self._order_sections(self._sections)
self._original_sections = copy.deepcopy(self._sections)
def _load_parser_from_dict(self):
self._sections = OrderedDict()
for section_name, value in self._inputdict.items():
section_name = JSONSchema.format_section_name(section_name).lower()
self._sections[section_name] = OrderedDict()
self._sections[section_name]['properties'] = OrderedDict()
self._sections[section_name]['data'] = ''
if isinstance(value, dict):
for k, v in value.items():
self._sections[section_name]['properties'][k] = v
contents = ''
properties = self._sections[section_name]['properties']
lastIndex = len(properties) - 1
for i, (k, v) in enumerate(properties.items()):
contents += '{}{}{}'.format(
k, InputParser._PROPVALUE_SEPARATOR, v)
if i < lastIndex:
contents += '\n'
self._sections[section_name]['data'] = contents
elif isinstance(value, list) or isinstance(value, str):
lines = []
if isinstance(value, list):
lines = value
self._sections[section_name]['data'] = '\n'.join(
str(e) for e in value)
else:
lines = value.splitlines()
self._sections[section_name]['data'] = value
for line in lines:
k, v = self._get_key_value(line)
if k is not None and v is not None:
self._sections[section_name]['properties'][k] = v
else:
raise AquaChemistryError(
"Invalid parser input type for section {}".format(
section_name))
def is_modified(self):
"""
Returns true if data has been changed
"""
original_section_names = set(self._original_sections.keys())
section_names = set(self._sections.keys())
if original_section_names != section_names:
return True
for section_name in section_names:
original_section = self._original_sections[section_name]
section = self._sections[section_name]
if self.section_is_text(section_name):
original_data = original_section[
'data'] if 'data' in original_section else None
data = section['data'] if 'data' in section else None
if original_data != data:
return True
else:
original_properties = original_section[
'properties'] if 'properties' in original_section else None
properties = section[
'properties'] if 'properties' in section else None
if original_properties != properties:
return True
return False
@staticmethod
def is_pluggable_section(section_name):
return JSONSchema.format_section_name(
section_name).lower() in local_pluggables_types()
def get_section_types(self, section_name):
return self._json_schema.get_section_types(section_name)
def get_property_types(self, section_name, property_name):
return self._json_schema.get_property_types(section_name,
property_name)
def get_default_sections(self):
properties = self._json_schema.get_default_sections()
driver_name = self.get_section_property(InputParser.DRIVER,
JSONSchema.NAME)
if driver_name is not None:
properties[driver_name.lower()] = {"type": "object"}
return properties
def get_default_section_names(self):
sections = self.get_default_sections()
return list(sections.keys()) if sections is not None else []
def get_section_default_properties(self, section_name):
return self._json_schema.get_section_default_properties(section_name)
def allows_additional_properties(self, section_name):
return self._json_schema.allows_additional_properties(section_name)
def get_property_default_values(self, section_name, property_name):
return self._json_schema.get_property_default_values(
section_name, property_name)
def get_property_default_value(self, section_name, property_name):
return self._json_schema.get_property_default_value(
section_name, property_name)
def get_filename(self):
"""Return the filename."""
return self._filename
@staticmethod
def get_operator_problems(input_name):
config = get_chemistry_operator_configuration(input_name)
if 'problems' in config:
return config['problems']
return []
@staticmethod
def get_algorithm_problems(algo_name):
return JSONSchema.get_algorithm_problems(algo_name)
def _update_operator_input_schema(self):
# find operator
default_name = self.get_property_default_value(InputParser.OPERATOR,
JSONSchema.NAME)
operator_name = self.get_section_property(InputParser.OPERATOR,
JSONSchema.NAME,
default_name)
if operator_name is None:
# find the first valid input for the problem
problem_name = self.get_section_property(JSONSchema.PROBLEM,
JSONSchema.NAME)
if problem_name is None:
problem_name = self.get_property_default_value(
JSONSchema.PROBLEM, JSONSchema.NAME)
if problem_name is None:
raise AquaChemistryError(
"No algorithm 'problem' section found on input.")
for name in local_chemistry_operators():
if problem_name in self.get_operator_problems(name):
# set to the first input to solve the problem
operator_name = name
break
if operator_name is None:
# just remove fromm schema if none solves the problem
if InputParser.OPERATOR in self._json_schema.schema['properties']:
del self._json_schema.schema['properties'][
InputParser.OPERATOR]
return
if default_name is None:
default_name = operator_name
config = {}
try:
config = get_chemistry_operator_configuration(operator_name)
except:
pass
input_schema = config[
'input_schema'] if 'input_schema' in config else {}
properties = input_schema[
'properties'] if 'properties' in input_schema else {}
properties[JSONSchema.NAME] = {'type': 'string'}
required = input_schema[
'required'] if 'required' in input_schema else []
additionalProperties = input_schema[
'additionalProperties'] if 'additionalProperties' in input_schema else True
if default_name is not None:
properties[JSONSchema.NAME]['default'] = default_name
required.append(JSONSchema.NAME)
if InputParser.OPERATOR not in self._json_schema.schema['properties']:
self._json_schema.schema['properties'][InputParser.OPERATOR] = {
'type': 'object'
}
self._json_schema.schema['properties'][
InputParser.OPERATOR]['properties'] = properties
self._json_schema.schema['properties'][
InputParser.OPERATOR]['required'] = required
self._json_schema.schema['properties'][InputParser.OPERATOR][
'additionalProperties'] = additionalProperties
def _merge_dependencies(self):
algo_name = self.get_section_property(JSONSchema.ALGORITHM,
JSONSchema.NAME)
if algo_name is None:
return
config = get_algorithm_configuration(algo_name)
pluggable_dependencies = [] if 'depends' not in config else config[
'depends']
pluggable_defaults = {} if 'defaults' not in config else config[
'defaults']
for pluggable_type in local_pluggables_types():
if pluggable_type != JSONSchema.ALGORITHM and pluggable_type not in pluggable_dependencies:
# remove pluggables from input that are not in the dependencies
if pluggable_type in self._sections:
del self._sections[pluggable_type]
section_names = self.get_section_names()
for pluggable_type in pluggable_dependencies:
pluggable_name = None
new_properties = {}
if pluggable_type in pluggable_defaults:
for key, value in pluggable_defaults[pluggable_type].items():
if key == JSONSchema.NAME:
pluggable_name = pluggable_defaults[pluggable_type][
key]
else:
new_properties[key] = value
if pluggable_name is None:
continue
if pluggable_type not in section_names:
self.set_section(pluggable_type)
if self.get_section_property(pluggable_type,
JSONSchema.NAME) is None:
self.set_section_property(pluggable_type, JSONSchema.NAME,
pluggable_name)
if pluggable_name == self.get_section_property(
pluggable_type, JSONSchema.NAME):
properties = self.get_section_properties(pluggable_type)
if new_properties:
new_properties.update(properties)
else:
new_properties = properties
self.set_section_properties(pluggable_type, new_properties)
def _update_driver_input_schemas(self):
driver_name = self.get_section_property(InputParser.DRIVER,
JSONSchema.NAME)
if driver_name is not None:
driver_name = driver_name.strip().lower()
mgr = ConfigurationManager()
configs = mgr.configurations
for (name, config) in configs.items():
name = name.lower()
if driver_name is not None and driver_name == name:
input_schema = copy.deepcopy(
config['input_schema']) if 'input_schema' in config else {
'type': 'object'
}
if '$schema' in input_schema:
del input_schema['$schema']
if 'id' in input_schema:
del input_schema['id']
self._json_schema.schema['properties'][
driver_name] = input_schema
else:
if name in self._json_schema.schema['properties']:
del self._json_schema.schema['properties'][name]
@staticmethod
def _load_driver_names():
if InputParser._DRIVER_NAMES is None:
mgr = ConfigurationManager()
InputParser._DRIVER_NAMES = [
name.lower() for name in mgr.module_names
]
def _merge_default_values(self):
section_names = self.get_section_names()
if JSONSchema.NAME not in section_names:
self.set_section(JSONSchema.NAME)
if JSONSchema.ALGORITHM in section_names:
if JSONSchema.PROBLEM not in section_names:
self.set_section(JSONSchema.PROBLEM)
self._json_schema.update_pluggable_input_schemas(self)
self._merge_dependencies()
self._update_driver_sections()
self._update_driver_input_schemas()
self._update_operator_input_schema()
# do not merge any pluggable that doesn't have name default in schema
default_section_names = []
pluggable_types = local_pluggables_types()
for section_name in self.get_default_section_names():
if section_name in pluggable_types:
if self.get_property_default_value(
section_name, JSONSchema.NAME) is not None:
default_section_names.append(section_name)
else:
default_section_names.append(section_name)
section_names = set(
self.get_section_names()) | set(default_section_names)
for section_name in section_names:
if section_name not in self._sections:
self.set_section(section_name)
new_properties = self.get_section_default_properties(section_name)
if new_properties is not None:
if self.section_is_text(section_name):
text = self.get_section_text(section_name)
if (text is None or len(text) == 0) and \
isinstance(new_properties, str) and \
len(new_properties) > 0 and \
text != new_properties:
self.set_section_data(section_name, new_properties)
else:
properties = self.get_section_properties(section_name)
new_properties.update(properties)
self.set_section_properties(section_name, new_properties)
self._sections = self._order_sections(self._sections)
def validate_merge_defaults(self):
self._merge_default_values()
self._json_schema.validate(self.to_JSON())
self._validate_algorithm_problem()
self._validate_operator_problem()
def _validate_algorithm_problem(self):
algo_name = self.get_section_property(JSONSchema.ALGORITHM,
JSONSchema.NAME)
if algo_name is None:
return
problem_name = self.get_section_property(JSONSchema.PROBLEM,
JSONSchema.NAME)
if problem_name is None:
problem_name = self.get_property_default_value(
JSONSchema.PROBLEM, JSONSchema.NAME)
if problem_name is None:
raise AquaChemistryError(
"No algorithm 'problem' section found on input.")
problems = InputParser.get_algorithm_problems(algo_name)
if problem_name not in problems:
raise AquaChemistryError(
"Problem: {} not in the list of problems: {} for algorithm: {}."
.format(problem_name, problems, algo_name))
def _validate_operator_problem(self):
operator_name = self.get_section_property(InputParser.OPERATOR,
JSONSchema.NAME)
if operator_name is None:
return
problem_name = self.get_section_property(JSONSchema.PROBLEM,
JSONSchema.NAME)
if problem_name is None:
problem_name = self.get_property_default_value(
JSONSchema.PROBLEM, JSONSchema.NAME)
if problem_name is None:
raise AquaChemistryError(
"No algorithm 'problem' section found on input.")
problems = InputParser.get_operator_problems(operator_name)
if problem_name not in problems:
raise AquaChemistryError(
"Problem: {} not in the list of problems: {} for operator: {}."
.format(problem_name, problems, operator_name))
def to_JSON(self):
json_dict = OrderedDict()
for section_name in self.get_section_names():
if self.section_is_text(section_name):
json_dict[section_name] = self.get_section_text(section_name)
else:
json_dict[section_name] = self.get_section_properties(
section_name)
return json_dict
def to_dictionary(self):
dict = OrderedDict()
for section_name in self.get_section_names():
if self.section_is_text(section_name):
dict[section_name] = self.get_section_text(
section_name).splitlines()
else:
dict[section_name] = self.get_section_properties(section_name)
return dict
def commit_changes(self):
self._original_sections = copy.deepcopy(self._sections)
def save_to_file(self, file_name):
if file_name is None:
raise AquaChemistryError('Missing file path')
file_name = file_name.strip()
if len(file_name) == 0:
raise AquaChemistryError('Missing file path')
prev_filename = self.get_filename()
sections = copy.deepcopy(self.get_sections())
if prev_filename is not None:
prev_dirname = os.path.dirname(os.path.realpath(prev_filename))
dirname = os.path.dirname(os.path.realpath(file_name))
if prev_dirname != dirname:
InputParser._from_relative_to_abs_paths(
sections, prev_filename)
contents = ''
lastIndex = len(sections) - 1
for i, (section_name, section) in enumerate(sections.items()):
contents += '{}{}'.format(InputParser._START_SECTION, section_name)
if self.section_is_text(section_name):
value = section['data']
if value is not None:
contents += '\n{}'.format(str(value))
else:
if 'properties' in section:
for k, v in section['properties'].items():
contents += '\n {}{}{}'.format(
k, InputParser._PROPVALUE_SEPARATOR, str(v))
contents += '\n{}'.format(InputParser._END_SECTION)
if i < lastIndex:
contents += '\n\n'
with open(file_name, 'w') as f:
print(contents, file=f)
def export_dictionary(self, file_name):
if file_name is None:
raise AquaChemistryError('Missing file path')
file_name = file_name.strip()
if len(file_name) == 0:
raise AquaChemistryError('Missing file path')
value = json.loads(json.dumps(self.to_dictionary()))
value = pprint.pformat(value, indent=4)
with open(file_name, 'w') as f:
print(value, file=f)
@staticmethod
def _from_relative_to_abs_paths(sections, filename):
directory = os.path.dirname(filename)
for _, section in sections.items():
if 'properties' in section:
for key, value in section['properties'].items():
if key == InputParser._HDF5_INPUT:
if value is not None and not os.path.isabs(value):
value = os.path.abspath(
os.path.join(directory, value))
InputParser._set_section_property(
sections, section[JSONSchema.NAME], key, value,
['string'])
def section_is_driver(self, section_name):
section_name = JSONSchema.format_section_name(section_name).lower()
InputParser._load_driver_names()
return section_name in InputParser._DRIVER_NAMES
def section_is_text(self, section_name):
section_name = JSONSchema.format_section_name(section_name).lower()
types = self.get_section_types(section_name)
if len(types) > 0:
return 'string' in types
return False
def get_sections(self):
return self._sections
def get_section(self, section_name):
"""Return a Section by name.
Args:
section_name (str): the name of the section, case insensitive
Returns:
Section: The section with this name
Raises:
AquaChemistryError: if the section does not exist.
"""
section_name = JSONSchema.format_section_name(section_name).lower()
try:
return self._sections[section_name]
except KeyError:
raise AquaChemistryError('No section "{0}"'.format(section_name))
def get_section_text(self, section_name):
section = self.get_section(section_name)
if section is None:
return ''
if 'data' in section:
return section['data']
return ''
def get_section_properties(self, section_name):
section = self.get_section(section_name)
if section is None:
return {}
if 'properties' in section:
return section['properties']
return {}
def get_section_property(self,
section_name,
property_name,
default_value=None):
"""Return a property by name.
Args:
section_name (str): the name of the section, case insensitive
property_name (str): the property name in the section
default_value : default value in case it is not found
Returns:
Value: The property value
"""
section_name = JSONSchema.format_section_name(section_name).lower()
property_name = JSONSchema.format_property_name(property_name)
if section_name in self._sections:
section = self._sections[section_name]
if 'properties' in section and property_name in section[
'properties']:
return section['properties'][property_name]
return default_value
def get_section_data(self, section_name, default_value=None):
"""
Return a section data.
Args:
section_name (str): the name of the section, case insensitive
default_value : default value in case it is not found
Returns:
Value: data value
"""
section_name = JSONSchema.format_section_name(section_name).lower()
if section_name in self._sections:
section = self._sections[section_name]
if 'data' in section:
return section['data']
return default_value
def set_section(self, section_name):
"""
Args:
section_name (str): the name of the section, case insensitive
"""
section_name = JSONSchema.format_section_name(section_name).lower()
if section_name not in self._sections:
self._sections[section_name] = OrderedDict([(JSONSchema.NAME,
section_name)])
self._sections[section_name]['properties'] = OrderedDict()
self._sections[section_name]['data'] = ''
self._sections = self._order_sections(self._sections)
def delete_section(self, section_name):
"""
Args:
section_name (str): the name of the section, case insensitive
"""
section_name = JSONSchema.format_section_name(section_name).lower()
if section_name not in self._sections:
return
del self._sections[section_name]
# update schema
self._json_schema.rollback_changes()
self._json_schema.update_pluggable_input_schemas(self)
self._update_driver_input_schemas()
self._update_operator_input_schema()
def set_section_properties(self, section_name, properties):
self.delete_section_properties(section_name)
for property_name, value in properties.items():
self.set_section_property(section_name, property_name, value)
def set_section_property(self, section_name, property_name, value):
section_name = JSONSchema.format_section_name(section_name).lower()
property_name = JSONSchema.format_property_name(property_name)
value = self._json_schema.check_property_value(section_name,
property_name, value)
types = self.get_property_types(section_name, property_name)
parser_temp = copy.deepcopy(self)
InputParser._set_section_property(parser_temp._sections, section_name,
property_name, value, types)
msg = self._json_schema.validate_property(parser_temp.to_JSON(),
section_name, property_name)
if msg is not None:
raise AquaChemistryError("{}.{}: Value '{}': '{}'".format(
section_name, property_name, value, msg))
InputParser._set_section_property(self._sections, section_name,
property_name, value, types)
if property_name == JSONSchema.NAME:
if InputParser.OPERATOR == section_name:
self._update_operator_input_schema()
# remove properties that are not valid for this section
default_properties = self.get_section_default_properties(
section_name)
if isinstance(default_properties, dict):
properties = self.get_section_properties(section_name)
for property_name in list(properties.keys()):
if property_name != JSONSchema.NAME and property_name not in default_properties:
self.delete_section_property(
section_name, property_name)
elif JSONSchema.PROBLEM == section_name:
self._update_algorithm_problem()
self._update_operator_problem()
elif InputParser.is_pluggable_section(section_name):
self._json_schema.update_pluggable_input_schemas(self)
# remove properties that are not valid for this section
default_properties = self.get_section_default_properties(
section_name)
if isinstance(default_properties, dict):
properties = self.get_section_properties(section_name)
for property_name in list(properties.keys()):
if property_name != JSONSchema.NAME and property_name not in default_properties:
self.delete_section_property(
section_name, property_name)
if section_name == JSONSchema.ALGORITHM:
self._update_dependency_sections()
elif value is not None:
value = str(value).lower().strip()
if len(value) > 0 and self.section_is_driver(value):
self._update_driver_input_schemas()
self._update_driver_sections()
self._sections = self._order_sections(self._sections)
def _update_algorithm_problem(self):
problem_name = self.get_section_property(JSONSchema.PROBLEM,
JSONSchema.NAME)
if problem_name is None:
problem_name = self.get_property_default_value(
JSONSchema.PROBLEM, JSONSchema.NAME)
if problem_name is None:
raise AquaChemistryError(
"No algorithm 'problem' section found on input.")
algo_name = self.get_section_property(JSONSchema.ALGORITHM,
JSONSchema.NAME)
if algo_name is not None and problem_name in InputParser.get_algorithm_problems(
algo_name):
return
for algo_name in local_algorithms():
if problem_name in self.get_algorithm_problems(algo_name):
# set to the first algorithm to solve the problem
self.set_section_property(JSONSchema.ALGORITHM,
JSONSchema.NAME, algo_name)
return
# no algorithm solve this problem, remove section
self.delete_section(JSONSchema.ALGORITHM)
def _update_operator_problem(self):
problem_name = self.get_section_property(JSONSchema.PROBLEM,
JSONSchema.NAME)
if problem_name is None:
problem_name = self.get_property_default_value(
JSONSchema.PROBLEM, JSONSchema.NAME)
if problem_name is None:
raise AquaChemistryError(
"No algorithm 'problem' section found on input.")
operator_name = self.get_section_property(InputParser.OPERATOR,
JSONSchema.NAME)
if operator_name is not None and problem_name in InputParser.get_operator_problems(
operator_name):
return
for operator_name in local_chemistry_operators():
if problem_name in self.get_operator_problems(operator_name):
# set to the first input to solve the problem
self.set_section_property(InputParser.OPERATOR,
JSONSchema.NAME, operator_name)
return
# no input solve this problem, remove section
self.delete_section(InputParser.OPERATOR)
def _update_dependency_sections(self):
algo_name = self.get_section_property(JSONSchema.ALGORITHM,
JSONSchema.NAME)
config = {} if algo_name is None else get_algorithm_configuration(
algo_name)
classical = config['classical'] if 'classical' in config else False
pluggable_dependencies = [] if 'depends' not in config else config[
'depends']
pluggable_defaults = {} if 'defaults' not in config else config[
'defaults']
pluggable_types = local_pluggables_types()
for pluggable_type in pluggable_types:
# remove pluggables from input that are not in the dependencies
if pluggable_type != JSONSchema.ALGORITHM and pluggable_type not in pluggable_dependencies and pluggable_type in self._sections:
del self._sections[pluggable_type]
for pluggable_type in pluggable_dependencies:
pluggable_name = None
if pluggable_type in pluggable_defaults:
if JSONSchema.NAME in pluggable_defaults[pluggable_type]:
pluggable_name = pluggable_defaults[pluggable_type][
JSONSchema.NAME]
if pluggable_name is not None and pluggable_type not in self._sections:
self.set_section_property(pluggable_type, JSONSchema.NAME,
pluggable_name)
# update default values for new dependency pluggable types
self.set_section_properties(
pluggable_type,
self.get_section_default_properties(pluggable_type))
# update backend based on classical
if classical:
if JSONSchema.BACKEND in self._sections:
del self._sections[JSONSchema.BACKEND]
else:
if JSONSchema.BACKEND not in self._sections:
self.set_section_properties(
JSONSchema.BACKEND,
self.get_section_default_properties(JSONSchema.BACKEND))
# reorder sections
self._sections = self._order_sections(self._sections)
def _update_driver_sections(self):
driver_name = self.get_section_property(InputParser.DRIVER,
JSONSchema.NAME)
if driver_name is not None:
driver_name = driver_name.strip().lower()
mgr = ConfigurationManager()
configs = mgr.configurations
for (name, config) in configs.items():
name = name.lower()
if driver_name is not None and driver_name == name:
continue
if name in self._sections:
del self._sections[name]
if driver_name is not None and driver_name not in self._sections:
self.set_section(driver_name)
value = self.get_section_default_properties(driver_name)
if isinstance(value, dict):
for property_name, property_value in value.items():
self.set_section_property(driver_name, property_name,
property_value)
else:
if value is None:
types = self.get_section_types(driver_name)
if 'null' not in types:
if 'string' in types:
value = ''
elif 'object' in types:
value = {}
elif 'array' in types:
value = []
self.set_section_data(driver_name, value)
@staticmethod
def _set_section_property(sections, section_name, property_name, value,
types):
"""
Args:
section_name (str): the name of the section, case insensitive
property_name (str): the property name in the section
value : property value
types : schema valid types
"""
section_name = JSONSchema.format_section_name(section_name).lower()
property_name = JSONSchema.format_property_name(property_name)
value = JSONSchema.get_value(value, types)
if section_name not in sections:
sections[section_name] = OrderedDict([(JSONSchema.NAME,
section_name)])
if 'properties' not in sections[section_name]:
sections[section_name]['properties'] = OrderedDict()
# name should come first
if JSONSchema.NAME == property_name and property_name not in sections[
section_name]['properties']:
new_dict = OrderedDict([(property_name, value)])
new_dict.update(sections[section_name]['properties'])
sections[section_name]['properties'] = new_dict
else:
sections[section_name]['properties'][property_name] = value
# rebuild data
contents = ''
properties = sections[section_name]['properties']
lastIndex = len(properties) - 1
for i, (key, value) in enumerate(properties.items()):
contents += '{}{}{}'.format(key, InputParser._PROPVALUE_SEPARATOR,
value)
if i < lastIndex:
contents += '\n'
sections[section_name]['data'] = contents
def delete_section_property(self, section_name, property_name):
"""
Args:
section_name (str): the name of the section, case insensitive
property_name (str): the property name in the section
"""
section_name = JSONSchema.format_section_name(section_name).lower()
property_name = JSONSchema.format_property_name(property_name)
rebuild_data = False
if section_name in self._sections and \
'properties' in self._sections[section_name] and \
property_name in self._sections[section_name]['properties']:
del self._sections[section_name]['properties'][property_name]
rebuild_data = True
if rebuild_data:
contents = ''
properties = self._sections[section_name]['properties']
lastIndex = len(properties) - 1
for i, (key, value) in enumerate(properties.items()):
contents += '{}{}{}'.format(key,
InputParser._PROPVALUE_SEPARATOR,
value)
if i < lastIndex:
contents += '\n'
self._sections[section_name]['data'] = contents
def delete_section_properties(self, section_name):
"""
Args:
section_name (str): the name of the section, case insensitive
"""
section_name = JSONSchema.format_section_name(section_name).lower()
if section_name in self._sections:
del self._sections[section_name]
def set_section_data(self, section_name, value):
"""
Sets a section data.
Args:
section_name (str): the name of the section, case insensitive
value : value to set
"""
section_name = JSONSchema.format_section_name(section_name).lower()
value = self._json_schema.check_section_value(section_name, value)
self._sections[section_name] = OrderedDict([(JSONSchema.NAME,
section_name)])
self._sections[section_name]['data'] = value
properties = OrderedDict()
if value is not None:
lines = str(value).splitlines()
for line in lines:
k, v = self._get_key_value(line)
if k is not None and v is not None:
properties[k] = v
self._sections[section_name]['properties'] = properties
def delete_section_data(self, section_name):
"""
Deletes a section data.
Args:
section_name (str): the name of the section, case insensitive
"""
section_name = JSONSchema.format_section_name(section_name).lower()
if section_name in self._sections:
self._sections[section_name]['data'] = ''
self._sections[section_name]['properties'] = OrderedDict()
def get_section_names(self):
"""Return all the names of the sections."""
return list(self._sections.keys())
def is_substitution_allowed(self):
auto_substitutions = self.get_property_default_value(
JSONSchema.PROBLEM, InputParser.AUTO_SUBSTITUTIONS)
auto_substitutions = self.get_section_property(
JSONSchema.PROBLEM, InputParser.AUTO_SUBSTITUTIONS,
auto_substitutions)
if auto_substitutions is None:
auto_substitutions = True
return auto_substitutions
def check_if_substitution_key(self, section_name, property_names):
result = [(property_name, False) for property_name in property_names]
if not self.is_substitution_allowed():
return result
section_name = JSONSchema.format_section_name(section_name).lower()
property_names = [
JSONSchema.format_property_name(property_name)
for property_name in property_names
]
section_property_name = self.get_property_default_value(
section_name, JSONSchema.NAME)
section_property_name = self.get_section_property(
section_name, JSONSchema.NAME, section_property_name)
for key in self._substitutions.keys():
key_items = key.split('.')
if len(key_items) == 3 and \
key_items[0] == section_name and \
key_items[1] == section_property_name and \
key_items[2] in property_names:
result[property_names.index(key_items[2])] = (key_items[2],
True)
continue
return result
def process_substitutions(self, substitutions=None):
if substitutions is not None and not isinstance(substitutions, dict):
raise AquaChemistryError(
'Invalid substitution parameter: {}'.format(substitutions))
if not self.is_substitution_allowed():
return {}
result = {}
for key, value in self._substitutions.items():
key_items = key.split('.')
if len(key_items) != 3:
raise AquaChemistryError(
'Invalid substitution key: {}'.format(key))
name = self.get_property_default_value(key_items[0],
JSONSchema.NAME)
name = self.get_section_property(key_items[0], JSONSchema.NAME,
name)
if name != key_items[1]:
continue
value_set = False
value_items = value.split('.')
if len(value_items) == 3:
name = self.get_section_property(value_items[0],
JSONSchema.NAME)
if name == value_items[1]:
v = self.get_property_default_value(
value_items[0], value_items[2])
v = self.get_section_property(value_items[0],
value_items[2], v)
if v is not None:
self.set_section_property(key_items[0], key_items[2],
v)
result[key] = v
value_set = True
if value_set or substitutions is None:
continue
if value in substitutions:
self.set_section_property(key_items[0], key_items[2],
substitutions[value])
result[key] = substitutions[value]
return result
def _process_line(self, section, line):
stripLine = line.strip()
if len(stripLine) == 0:
if section is not None:
section['data'].append(line)
return section
if stripLine.lower().startswith(InputParser._END_SECTION):
if section is not None:
self._sections[section[
JSONSchema.NAME]] = self._process_section(section)
return None
if stripLine.startswith(InputParser._START_SECTION):
if section is not None:
raise AquaChemistryError(
'New section "{0}" starting before the end of previuos section "{1}"'
.format(line, section[JSONSchema.NAME]))
return OrderedDict([(JSONSchema.NAME, stripLine[1:].lower()),
('data', [])])
if section is None:
return section
section['data'].append(line)
return section
def _process_section(self, section):
contents = ''
sep_pos = -len(os.linesep)
lastIndex = len(section['data']) - 1
for i, line in enumerate(section['data']):
key, value = self._get_key_value(line)
if key is not None and value is not None:
if 'properties' not in section:
section['properties'] = OrderedDict()
section['properties'][key] = value
if i == lastIndex:
if len(line) >= len(
os.linesep) and line[sep_pos:] == os.linesep:
line = line[:sep_pos]
contents += line
section['data'] = contents
return section
@staticmethod
def _get_key_value(line):
stripLine = line.strip()
pos = -1
for start_comment in InputParser._START_COMMENTS:
pos = stripLine.find(start_comment)
if pos >= 0:
break
if pos == 0:
return (None, None)
if pos > 0:
stripLine = stripLine[:pos].strip()
pos = stripLine.find(InputParser._PROPVALUE_SEPARATOR)
if pos > 0:
key = stripLine[0:pos].strip()
value = stripLine[pos + 1:].strip()
return (key, JSONSchema.get_value(value))
return (None, None)
def get_algorithm_problems(algo_name):
return JSONSchema.get_algorithm_problems(algo_name)
