def get_command_fields_from_class(execution_class: Type[IExecutableCommand]) -> List[CommandField]:
execution_class_name: str = class_full_name(execution_class)
cmd = first(commands, condition=lambda c: c.execution_class_name == execution_class_name)
if cmd:
return command_fields.get(cmd.name, [])
return []
def _constrains_interface(self, scale: str, invoking_interface_name: str,
requested_interface_name: str,
parent_processor: Processor,
child_processor: Processor):
origin_factor = first(
parent_processor.factors,
lambda i: strcmp(i.name, invoking_interface_name))
if not origin_factor:
raise Exception("Invoking interface name '" +
invoking_interface_name +
"' not found for processor '" +
parent_processor.name + "'")
destination_factor = first(
child_processor.factors,
lambda i: strcmp(i.name, requested_interface_name))
if not destination_factor:
raise Exception("Requested interface name '" +
invoking_interface_name +
"' not found for processor '" +
parent_processor.name + "'")
if origin_factor.taxon != destination_factor.taxon:
# Search for an Interface Type Conversion defined in the ScaleChangeMap command
interface_types_transform = self._get_interface_types_transform(
origin_factor.taxon, parent_processor,
destination_factor.taxon, child_processor)
scale = FloatOrString.multiply(
scale, interface_types_transform.scaled_weight)
relationship = FactorsRelationScaleObservation.create_and_append(
origin=origin_factor,
destination=destination_factor,
observer=None,
quantity=scale)
# relationship = ProcessorsRelationUpscaleObservation.create_and_append(parent=parent_processor,
# child=child_processor,
# observer=None,
# factor_name=interface_name,
# quantity=scale)
self._glb_idx.put(relationship.key(), relationship)
def create_command(cmd_type,
name,
json_input,
source_block=None) -> ExecutableCommandIssuesPairType:
"""
Factory creating and initializing a command from its type, optional name and parameters
:param cmd_type: String describing the type of command, as found in the interchange JSON format
:param name: An optional name (label) for the command
:param json_input: Parameters specific of the command type (each command knows how to interpret,
validate and integrate them)
:param source_block: String defining the name of the origin block (in a spreadsheet is the worksheet name, but other block types could appear)
:return: The instance of the command and the issues creating it
:raise
"""
cmd: Optional[Command] = first(commands,
condition=lambda c: c.name == cmd_type)
if cmd:
if cmd.execution_class:
exec_cmd: "IExecutableCommand" = cmd.execution_class(
name) # Reflective call
exec_cmd._serialization_type = cmd_type # Injected attribute. Used later for serialization
exec_cmd._serialization_label = name # Injected attribute. Used later for serialization
exec_cmd._source_block_name = source_block
if isinstance(json_input, (str, dict, list)):
if json_input != {}:
issues = exec_cmd.json_deserialize(json_input)
else:
issues = []
else:
# NO SPECIFICATION
raise Exception("The command '" + cmd_type + " " +
name if name else "<unnamed>" +
" does not have a specification.")
return exec_cmd, issues # Return the command and the issues found during the deserialization
else:
return None, [
] # No execution class. Currently "import_commands" and "list_of_commands"
else:
# UNKNOWN COMMAND
raise Exception("Unknown command type: " + cmd_type)
def _get_factor_type_from_field(
self, hierarchy_field_name: str,
interface_type_field_name: str) -> FactorType:
interface_type_name = self._fields[interface_type_field_name]
if not interface_type_name:
raise CommandExecutionError(
f"The field '{interface_type_field_name}' has not been specified"
)
# Check if FactorType exists
interface_types = self._glb_idx.get(
FactorType.partial_key(interface_type_name))
if len(interface_types) == 1:
return interface_types[0]
elif len(interface_types) == 0:
raise CommandExecutionError(
f"The interface type '{interface_type_name}' has not been found"
)
else:
hierarchy_name = self._fields[hierarchy_field_name]
if not hierarchy_name:
raise CommandExecutionError(
f"The field '{hierarchy_field_name}' has not been specified and "
f"the interface type '{interface_type_name}' is not unique"
)
interface_type = first(
interface_types,
lambda t: strcmp(t.hierarchy.name, hierarchy_name))
if not interface_type:
raise CommandExecutionError(
f"The interface type '{interface_type_name}' has not been found in "
f"hierarchy '{hierarchy_name}'")
return interface_type
def _process_row(self, field_values: Dict[str, Any], subrow=None) -> None:
"""
Process a dictionary representing a row of the Interfaces command. The dictionary can come directly from
the worksheet or from a dataset.
:param field_values: dictionary
"""
# f_processor_name -> p
# f_interface_type_name -> it
# f_interface_name -> i
#
# IF NOT i AND it AND p => i_name = it.name => get or create "i"
# IF i AND it AND p => get or create "i", IF "i" exists, i.it MUST BE equal to "it" (IF NOT, error)
# IF i AND p AND NOT it => get "i" (MUST EXIST)
f_interface_type_name = field_values.get("interface_type")
f_interface_name = field_values.get("interface")
if not f_interface_name:
if not f_interface_type_name:
raise CommandExecutionError(
"At least one of InterfaceType or Interface must be defined"
+ subrow_issue_message(subrow))
f_interface_name = f_interface_type_name
processor = self.find_processor(field_values.get("processor"), subrow)
# Try to find Interface
f_orientation = field_values.get("orientation")
interface_type: Optional[FactorType] = None
interface: Optional[Factor] = None
interfaces: Sequence[Factor] = self._glb_idx.get(
Factor.partial_key(processor=processor, name=f_interface_name))
if len(interfaces) == 1:
interface = interfaces[0]
print(f"DEBUG - Interface '{interface.name}' found")
interface_type = interface.taxon
if f_interface_type_name and not strcmp(interface_type.name,
f_interface_type_name):
self._add_issue(
IType.WARNING,
f"The existing Interface '{interface.name}' has the InterfaceType "
f"'{interface_type.name}' which is different from the specified "
f"InterfaceType '{f_interface_type_name}'. Record skipped."
+ subrow_issue_message(subrow))
return
elif len(interfaces) > 1:
raise CommandExecutionError(
f"Interface '{f_interface_name}' found {str(len(interfaces))} times. "
f"It must be uniquely identified." +
subrow_issue_message(subrow))
elif len(interfaces) == 0:
# The interface does not exist, create it below
if not f_orientation:
raise CommandExecutionError(
f"Orientation must be defined for new Interfaces." +
subrow_issue_message(subrow))
# InterfaceType still not found
if not interface_type:
interface_type_name = ifnull(f_interface_type_name,
f_interface_name)
# Find FactorType
# TODO Allow creating a basic FactorType if it is not found?
interface_types: Sequence[FactorType] = self._glb_idx.get(
FactorType.partial_key(interface_type_name))
if len(interface_types) == 0:
raise CommandExecutionError(
f"InterfaceType '{interface_type_name}' not declared previously"
+ subrow_issue_message(subrow))
elif len(interface_types) > 1:
raise CommandExecutionError(
f"InterfaceType '{interface_type_name}' found {str(len(interface_types))} times. "
f"It must be uniquely identified." +
subrow_issue_message(subrow))
else:
interface_type = interface_types[0]
# Get attributes default values taken from Interface Type or Processor attributes
# Rows : value of (source) "processor.subsystem_type"
# Columns: value of (target) "interface_type.opposite_processor_type"
# Cells : CORRECTED value of "opposite_processor_type"
# +--------+-------+--------+-------+---------+
# | | Local | Env | Ext | ExtEnv |
# +--------+-------+--------+-------+---------+
# | Local | Local | Env | Ext | ExtEnv |
# | Env | Local | Env | Ext | ExtEnv? |
# | Ext | Ext | ExtEnv | Local | Env |
# | ExtEnv | Ext | ExtEnv | Local | Env? |
# +--------+-------+--------+-------+---------+
if interface_type.opposite_processor_type:
tmp = interface_type.opposite_processor_type.lower()
if processor.subsystem_type.lower() in ["local", "environment"
]: # First two rows
opposite_processor_type = tmp
else:
opposite_processor_type = InterfacesAndQualifiedQuantitiesCommand.invert[
tmp]
# TODO in doubt. Maybe these are undefined (values with question mark in the table)
# if tmp == "externalenvironment" and processor.subsystem_type.lower() in ["environment", "externalenvironment"]:
# pass
else:
opposite_processor_type = None
interface_type_values = {
"sphere": interface_type.sphere,
"roegen_type": interface_type.roegen_type,
"opposite_processor_type": opposite_processor_type
}
# Get internal and user-defined attributes in one dictionary
# Use: value specified in Interfaces ELSE value specified in InterfaceTypes ELSE first value of allowed values
attributes = {
c.name: ifnull(
field_values[c.name],
ifnull(interface_type_values.get(c.name),
head(c.allowed_values)))
for c in self._command_fields if c.attribute_of == Factor
}
if not interface:
# f_list: Sequence[Factor] = self._glb_idx.get(
# Factor.partial_key(processor=p, factor_type=ft, orientation=f_orientation))
#
# if len(f_list) > 0:
# raise CommandExecutionError(f"An interface called '{f_list[0].name}' for Processor '{f_processor_name}'"
# f" with InterfaceType '{f_interface_type_name}' and orientation "
# f"'{f_orientation}' already exists"+subrow_issue_message(subrow))
# Transform text of "interface_attributes" into a dictionary
interface_attributes = self.transform_text_attributes_into_dictionary(
field_values.get("interface_attributes"), subrow)
attributes.update(interface_attributes)
location = self.get_location(field_values.get("location"), subrow)
interface = Factor.create_and_append(
f_interface_name,
processor,
in_processor_type=FactorInProcessorType(external=False,
incoming=False),
taxon=interface_type,
geolocation=location,
tags=None,
attributes=attributes)
self._glb_idx.put(interface.key(), interface)
print(f"DEBUG - Interface '{interface.name}' created")
elif not interface.compare_attributes(attributes):
initial = ', '.join(
[f"{k}: {interface.get_attribute(k)}" for k in attributes])
new = ', '.join([f"{k}: {attributes[k]}" for k in attributes])
name = interface.processor.full_hierarchy_names(
self._glb_idx)[0] + ":" + interface.name
raise CommandExecutionError(
f"The same interface '{name}', is being redeclared with different properties. "
f"INITIAL: {initial}; NEW: {new}." +
subrow_issue_message(subrow))
f_unit = field_values.get("unit")
if not f_unit:
f_unit = interface_type.unit
# Unify unit (it must be done before considering RelativeTo -below-, because it adds a transformation to "f_unit")
f_value = field_values.get("value")
if f_value is not None and f_unit != interface_type.unit:
try:
f_value = UnitConversion.convert(f_value, f_unit,
interface_type.unit)
except DimensionalityError:
raise CommandExecutionError(
f"Dimensions of units in InterfaceType ({interface_type.unit}) and specified ({f_unit}) are not convertible"
+ subrow_issue_message(subrow))
f_unit = interface_type.unit
# Search for a relative_to interface
f_relative_to = field_values.get("relative_to")
relative_to_interface: Optional[Factor] = None
if f_relative_to:
try:
ast = parser_field_parsers.string_to_ast(
parser_field_parsers.factor_unit, f_relative_to)
except:
raise CommandExecutionError(
f"Could not parse the RelativeTo column, value {str(f_relative_to)}. "
+ subrow_issue_message(subrow))
relative_to_interface_name = ast_to_string(ast["factor"])
# rel_unit_name = ast["unparsed_unit"]
# try:
# f_unit = str((ureg(f_unit) / ureg(rel_unit_name)).units)
# except (UndefinedUnitError, AttributeError) as ex:
# raise CommandExecutionError(f"The final unit could not be computed, interface '{f_unit}' / "
# f"relative_to '{rel_unit_name}': {str(ex)}"+subrow_issue_message(subrow))
relative_to_interface = first(
interface.processor.factors,
lambda ifc: strcmp(ifc.name, relative_to_interface_name))
if not relative_to_interface:
raise CommandExecutionError(
f"Interface specified in 'relative_to' column "
f"'{relative_to_interface_name}' has not been found." +
subrow_issue_message(subrow))
if f_value is None and relative_to_interface is not None:
# Search for a Interface Type Conversion defined in the ScaleChangeMap command
interface_types_transforms: List[FactorTypesRelationUnidirectionalLinearTransformObservation] = \
find_factor_types_transform_relation(self._glb_idx, relative_to_interface.taxon, interface.taxon, processor, processor)
# Overwrite any specified unit, it doesn't make sense without a value, i.e. it cannot be used for conversion
f_unit = interface.taxon.unit
if len(interface_types_transforms) == 1:
f_value = interface_types_transforms[0].scaled_weight
else:
interface_types_transforms_message = "an interface type conversion doesn't exist" \
if (len(interface_types_transforms) == 0) \
else f"{len(interface_types_transforms)} interface type conversions exist"
f_value = "0"
self._add_issue(
IType.WARNING,
f"Field 'value' should be defined for interfaces having a "
f"'RelativeTo' interface, and {interface_types_transforms_message}. "
f"Using value '0'." + subrow_issue_message(subrow))
# Create quantitative observation
if f_value is not None:
f_uncertainty = field_values.get("uncertainty")
f_assessment = field_values.get("assessment")
f_pedigree_matrix = field_values.get("pedigree_matrix")
f_pedigree = field_values.get("pedigree")
f_time = field_values.get("time")
f_comments = field_values.get("comments")
f_source = field_values.get("qq_source")
# TODO: source is not being used
source = self.get_source(f_source, subrow)
# Find Observer
observer: Optional[Observer] = None
if f_source:
observer = self._glb_idx.get_one(
Observer.partial_key(f_source))
if not observer:
self._add_issue(
IType.WARNING,
f"Observer '{f_source}' has not been found." +
subrow_issue_message(subrow))
# If an observation exists then "time" is mandatory
if not f_time:
raise CommandExecutionError(
f"Field 'time' needs to be specified for the given observation."
+ subrow_issue_message(subrow))
# An interface can have multiple observations if each of them have a different [time, observer] combination
for observation in interface.quantitative_observations:
observer_name = observation.observer.name if observation.observer else None
if strcmp(observation.attributes["time"], f_time) and strcmp(
observer_name, f_source):
raise CommandExecutionError(
f"The interface '{interface.name}' in processor '{interface.processor.name}' already has an "
f"observation with time '{f_time}' and source '{f_source}'."
)
self.check_existence_of_pedigree_matrix(f_pedigree_matrix,
f_pedigree, subrow)
# Transform text of "number_attributes" into a dictionary
number_attributes = self.transform_text_attributes_into_dictionary(
field_values.get("number_attributes"), subrow)
o = _create_or_append_quantitative_observation(
interface, f_value, f_unit, f_uncertainty, f_assessment,
f_pedigree, f_pedigree_matrix, observer, relative_to_interface,
f_time, None, f_comments, None, number_attributes)
def commands_generator_from_ooxml_file(input, state, sublist, stack) -> nexinfosys.ExecutableCommandIssuesPairType:
"""
It reads an Office Open XML input
Yields a sequence of command_executors
:param input: A bytes input
:param state: State used to check variables
:param sublist: List of worksheets to consider. If not defined, ALL worksheets
:param stack: Stack of nested files. Just pass it...
:return:
"""
# Start the Excel reader
workbook = openpyxl.load_workbook(io.BytesIO(input), data_only=True)
# Command names (for the "list of commands" command)
command_names = create_dictionary(data={cmd_name: None for cmd_name in valid_v2_command_names})
worksheet_to_command = create_dictionary() # A dictionary to translate a worksheet to an equivalent command
if sublist:
# Force reading "ListOfCommands" commands
for sheet_name in workbook.sheetnames:
if first(commands, condition=lambda c: c.name == "list_of_commands" and c.regex.search(sheet_name)):
sublist.append(sheet_name)
# For each worksheet, get the command type, convert into primitive JSON
for sheet_number, sheet_name in enumerate(workbook.sheetnames):
if sublist:
if sheet_name not in sublist:
continue
issues = []
total_issues: List[Issue] = []
sheet = workbook[sheet_name]
c_label: str = None
c_content = None
name = sheet.title.strip()
# Use an equivalent command name
if name in worksheet_to_command:
name = worksheet_to_command[name]
# Extract worksheet matrices
m = binary_mask_from_worksheet(sheet, False)
t = obtain_rectangular_submatrices(m, only_remove_empty_bottom=True)
if len(t) == 0: # No data
continue
t = t[0] # Take just the first element, a tuple (top, bottom, left, right) representing a rectangular region
t = (t[0] + 1, t[1] + 1, t[2] + 1, t[3] + 1) # Indices start at 1
# v = worksheet_to_numpy_array(sheet)
# Find which COMMAND to parse, then parse it
cmd: Optional[nexinfosys.Command] = first(commands, condition=lambda c: c.regex.search(name))
c_type: str = cmd.name if cmd else None
if not c_type:
total_issues.append(Issue(sheet_number, sheet_name, None, IType.WARNING,
f"The worksheet name '{sheet_name}' does not have a supported command associated. Skipped."))
elif c_type == "etl_dataset":
if sheet.cell(row=t[0], column=t[2]).value:
t = (1, m.shape[0] + 1, 1, m.shape[1] + 1)
# Parse to read parameters
dataset_name = cmd.regex.search(name).group(2)
issues, c_label, c_content = cmd.parse_function(sheet, t, dataset_name, state)
else:
total_issues.append(
Issue(sheet_number, sheet_name, c_type, IType.ERROR,
f"It seems there are no parameters for the dataset import command at worksheet '{sheet_name}'"))
elif c_type == "list_of_commands":
issues, c_label, c_content = parse_command_in_worksheet(sheet, t, None, cmd.name)
c_type = None
if IType.ERROR not in [issue.itype for issue in issues]:
for r in c_content["items"]:
worksheet = r.get("worksheet", None)
command = r.get("command", None)
# Check if valid command
if command not in command_names:
total_issues.append(
Issue(sheet_number, sheet_name, None, IType.ERROR,
"Command '" + command + "' not recognized in List of Commands."))
else:
worksheet_to_command[worksheet] = command
elif c_type == "import_commands":
issues, c_label, c_content = parse_command_in_worksheet(sheet, t, None, cmd.name)
if IType.ERROR not in [issue.itype for issue in issues]:
# Declared at this point to avoid circular reference ("parsers_factory" imports "parsers_spreadsheet")
from nexinfosys.command_generators.parsers_factory import commands_container_parser_factory
# For each line, repeat the import
for r in c_content["items"]:
generator_type, file2, sublist2 = handle_import_commands(r)
yield from commands_container_parser_factory(generator_type, None, file2, state, sublist=sublist2, stack=stack)
print("Done")
elif c_type == "mapping":
groups = cmd.regex.search(name).groups()
if groups[2] and groups[8]:
origin = groups[2]
destination = groups[8]
elif not groups[2] and not groups[8]:
origin = None
destination = None
else:
total_issues.append(
Issue(sheet_number, sheet_name, c_type, IType.ERROR,
f"Either origin or destination are not correctly specified in the sheet name '{sheet_name}'"))
issues, c_label, c_content = cmd.parse_function(sheet, t, origin, destination)
elif c_type in ["datasetqry", "datasetdata"]:
issues, c_label, c_content = cmd.parse_function(sheet, t, sheet_name, state)
elif c_type == "hierarchy":
res = cmd.regex.search(name)
h_type = res.group(2)
c_label = res.group(3)
issues, _, c_content = cmd.parse_function(sheet, t, c_label, h_type)
elif c_type == "data_input":
group2_name = cmd.regex.search(name).group(2)
issues, c_label, c_content = cmd.parse_function(sheet, t, group2_name)
else:
# GENERIC command parser
if cmd.parse_function:
issues, c_label, c_content = cmd.parse_function(sheet, t, sheet_name)
else:
issues, c_label, c_content = parse_command_in_worksheet(sheet, t, sheet_name, cmd.name)
# -------------------------------------------------------------------------------------------------------------
# Command parsed, now append "issues"
errors = 0
if len(issues) > 0:
for i in issues:
if isinstance(i, nexinfosys.command_generators.Issue):
if i.itype == IType.ERROR:
errors += 1
issue = i # Issue(sheet_number, sheet_name, c_type, i.itype, i.description)
else:
if i[0] == 3:
errors += 1
issue = Issue(sheet_number, sheet_name, c_type, IType(i[0]), i[1])
total_issues.append(issue)
if errors == 0:
try:
if c_type:
cmd, issues = create_command(c_type, c_label, c_content, sheet_name)
else:
cmd = None
issues = []
except Exception as e:
cmd = None
issues = [(3, f"Could not create command of type '{c_type}': {e}")]
if issues:
for i in issues:
if isinstance(i, nexinfosys.command_generators.Issue):
issue = i # Issue(sheet_number, sheet_name, c_type, i.itype, i.description)
else:
issue = Issue(sheet_number, sheet_name, c_type, IType(i[0]), i[1])
total_issues.append(issue)
else:
print(issues) # Convenient for debugging purposes
cmd = None # cmd, _ = create_command(c_type, c_label, {}, sh_name)
yield cmd, total_issues
def _get_factor_types_from_field(
self, hierarchy_field_name: str,
interface_type_field_name: str) -> List[FactorType]:
""" Possibly obtain not only one but many InterfaceTypes """
hierarchy_name = self._fields[hierarchy_field_name]
interface_type_name = self._fields[interface_type_field_name]
if not interface_type_name and not hierarchy_name:
raise CommandExecutionError(
f"No hierarchy nor interface type have been specified. At least specify one of them."
)
elif interface_type_name and hierarchy_name:
interface_types = self._glb_idx.get(
FactorType.partial_key(interface_type_name))
if len(interface_types) == 1:
return [interface_types[0]]
elif len(interface_types) == 0:
raise CommandExecutionError(
f"The interface type '{interface_type_name}' has not been found"
)
else:
hierarchy_name = self._fields[hierarchy_field_name]
if not hierarchy_name:
raise CommandExecutionError(
f"The field '{hierarchy_field_name}' has not been specified and "
f"the interface type '{interface_type_name}' is not unique"
)
interface_type = first(
interface_types,
lambda t: strcmp(t.hierarchy.name, hierarchy_name))
if not interface_type:
raise CommandExecutionError(
f"The interface type '{interface_type_name}' has not been found in "
f"hierarchy '{hierarchy_name}'")
return [interface_type]
elif interface_type_name and not hierarchy_name:
interface_types = self._glb_idx.get(
FactorType.partial_key(interface_type_name))
if len(interface_types) == 1:
return [interface_types[0]]
elif len(interface_types) == 0:
raise CommandExecutionError(
f"The interface type '{interface_type_name}' has not been found"
)
else:
raise CommandExecutionError(
f"The field '{hierarchy_field_name}' has not been specified and "
f"the interface type '{interface_type_name}' is not unique"
)
elif not interface_type_name and hierarchy_name:
hie = self._glb_idx.get(Hierarchy.partial_key(hierarchy_name))
if len(hie) == 1:
# All children of "hierarchy_name"
return [v for v in hie[0].codes.values()]
elif len(hie) == 0:
raise CommandExecutionError(
f"The InterfaceTypes hierarchy '{hierarchy_name}' has not been found"
)
else:
raise CommandExecutionError(
f"The InterfaceTypes hierarchy '{hierarchy_name}' has been found multiple times!!"
)
# Check if FactorType exists
interface_types = self._glb_idx.get(
FactorType.partial_key(interface_type_name))
if len(interface_types) == 1:
return interface_types[0]
elif len(interface_types) == 0:
raise CommandExecutionError(
f"The interface type '{interface_type_name}' has not been found"
)
else:
hierarchy_name = self._fields[hierarchy_field_name]
if not hierarchy_name:
raise CommandExecutionError(
f"The field '{hierarchy_field_name}' has not been specified and "
f"the interface type '{interface_type_name}' is not unique"
)
interface_type = first(
interface_types,
lambda t: strcmp(t.hierarchy.name, hierarchy_name))
if not interface_type:
raise CommandExecutionError(
f"The interface type '{interface_type_name}' has not been found in "
f"hierarchy '{hierarchy_name}'")
return interface_type