def serialize_state(state: State):
"""
Serialization prepared for a given organization of the state
:return:
"""
def serialize_dataframe(df):
return df.to_json(orient="split", index=False), \
json.dumps({i[0]: str(i[1]) for i in df.dtypes.to_dict().items()})
# list(df.index.names), df.to_dict()
print(" serialize_state IN")
import copy
# "_datasets"
ns_ds = {}
# Save and nullify before deep copy
for ns in state.list_namespaces():
_, _, _, datasets, _ = get_case_study_registry_objects(state, ns)
ns_ds[ns] = datasets
state.set("_datasets", create_dictionary(), ns) # Nullify datasets
# !!! WARNING: It destroys "state", so a DEEP COPY is performed !!!
tmp = sys.getrecursionlimit()
sys.setrecursionlimit(10000)
state2 = copy.deepcopy(state)
sys.setrecursionlimit(tmp)
# Iterate all namespaces
for ns in state2.list_namespaces():
glb_idx, p_sets, hh, _, mappings = get_case_study_registry_objects(
state2, ns)
if glb_idx:
tmp = glb_idx.to_pickable()
state2.set("_glb_idx", tmp, ns)
datasets = ns_ds[ns]
# TODO Serialize other DataFrames.
# Process Datasets
for ds_name in datasets:
ds = datasets[ds_name]
if isinstance(ds.data, pd.DataFrame):
tmp = serialize_dataframe(ds.data)
else:
tmp = None
# ds.data = None
# DB serialize the datasets
lst2 = serialize(ds.get_objects_list())
lst2.append(tmp) # Append the serialized DataFrame
datasets[ds_name] = lst2
state2.set("_datasets", datasets, ns)
tmp = serialize_from_object(
state2) # <<<<<<<< SLOWEST !!!! (when debugging)
print(" serialize_state length: " + str(len(tmp)) + " OUT")
tmp = blosc.compress(bytearray(tmp, "utf-8"), cname="zlib", typesize=8)
print(" serialize_state compressed length: " + str(len(tmp)) + " OUT")
return tmp
def __init__(self, session_factory):
# Session factory with access to business logic database
self._session_factory = session_factory
# Interactive session ID
self._guid = str(uuid.uuid4())
# User identity, if given (can be an anonymous session)
self._identity = None # type: Identity
self._state = State() # To keep the state
self._reproducible_session = None # type: ReproducibleSession
def _get_scale_value(self, scale: str):
try:
value = float(scale)
except ValueError:
ast = string_to_ast(expression_with_parameters, scale)
evaluation_issues: List[Tuple[int, str]] = []
s = State()
value, unresolved_vars = ast_evaluator(exp=ast,
state=s,
obj=None,
issue_lst=evaluation_issues)
if len(evaluation_issues) > 0:
evaluation_issues_str = [i[1] for i in evaluation_issues]
raise CommandExecutionError(
f"Problems evaluating scale expression '{scale}': "
f"{', '.join(evaluation_issues_str)}")
elif len(unresolved_vars) > 0:
raise CommandExecutionError(
f"Unresolved variables evaluating the scale expression '{scale}':"
f" {', '.join(unresolved_vars)}")
elif not value:
raise CommandExecutionError(
f"The scale expression '{scale}' could not be evaluated.")
return value
def get_metadata(s: State) -> pd.DataFrame:
metadata_dictionary = s.get("_metadata")
lst = list()
lst.append(
("Case study code", metadata_dictionary.get("case_study_code", "")
if metadata_dictionary else ""))
lst.append(
("Case study name", metadata_dictionary.get("case_study_name", "")
if metadata_dictionary else ""))
lst.append(
("Title",
metadata_dictionary.get("title", "") if metadata_dictionary else ""))
lst.append(("Subject, topic and/or keywords",
metadata_dictionary.get("subject_topic_keywords", "")
if metadata_dictionary else ""))
lst.append(("Description", metadata_dictionary.get("description", "")
if metadata_dictionary else ""))
lst.append(("Geographical level",
metadata_dictionary.get("geographical_level", "")
if metadata_dictionary else "")) # A list
lst.append(("Dimensions", metadata_dictionary.get("dimensions", "")
if metadata_dictionary else "")) # A list
lst.append(("Reference documentation",
metadata_dictionary.get("reference_documentation", "")
if metadata_dictionary else ""))
lst.append(("Authors", metadata_dictionary.get("authors", "")
if metadata_dictionary else ""))
lst.append(("Date of elaboration",
metadata_dictionary.get("date_of_elaboration", "")
if metadata_dictionary else ""))
lst.append(
("Temporal situation", metadata_dictionary.get(
"temporal_situation", "") if metadata_dictionary else ""))
lst.append(("Geographical location",
metadata_dictionary.get("geographical_situation", "")
if metadata_dictionary else ""))
if metadata_dictionary:
lst.append(("DOI", metadata_dictionary.get("doi", "")[0]
if metadata_dictionary.get("doi") else ""))
else:
lst.append(("DOI", ""))
lst.append(("Language", metadata_dictionary.get("language", "")
if metadata_dictionary else ""))
lst.append(
("Restriction level", metadata_dictionary.get("restriction_level", "")
if metadata_dictionary else "")) # A list
if metadata_dictionary:
lst.append(("Version", metadata_dictionary.get("version", "")[0]
if metadata_dictionary.get("version") else ""))
else:
lst.append(("Version", ""))
# Expand lists
for i, t in enumerate(lst):
if isinstance(t[1], list):
tmp = [t[0]]
tmp.extend(t[1])
lst[i] = tuple(tmp)
return list_to_dataframe(lst)
def lcia_method(indicator: str, method: str=None, horizon: str=None,
state: State=None, lcia_methods: PartialRetrievalDictionary=None):
"""
:param indicator: Indicator name
:param method: LCIA method weighting
:param horizon: Time horizon
:param state: Current values of processor plus parameters
:param lcia_methods: Where LCIA data is collected
:return: A dictionary with the
"""
if indicator is None or indicator.strip() == "":
return None
k = dict(d=indicator)
if method:
k["m"] = method
if horizon:
k["h"] = horizon
ms = lcia_methods.get(key=k, key_and_value=True)
indices = create_dictionary()
for k, v in ms:
idx_name = f'{k["d"]}_{k["m"]}_{k["h"]}'
if idx_name in indices:
lst = indices[idx_name]
else:
lst = []
indices[idx_name] = lst
lst.append((k["i"], v[0], float(v[1])))
ifaces = create_dictionary()
for t in state.list_namespace_variables():
if not t[0].startswith("_"):
p = t[1] # * ureg(iface_unit)
ifaces[t[0]] = p
res = dict()
for name, lst in indices.items():
interfaces = []
weights = [] # From "
for t in lst:
if t[0] in ifaces:
v = ifaces[t[0]] # TODO .to(t[1])
interfaces.append(v)
weights.append(t[2])
# Calculate the value
ind = np.sum(np.multiply(interfaces, weights)) # * ureg(indicator_unit)
res[name] = ind
return res
def check_parameter_value(glb_idx, p, value, issues, sheet_name, row):
retval = True
if p.range:
try: # Try "numeric interval"
ast = string_to_ast(number_interval, p.range)
# try Convert value to float
ast2 = string_to_ast(expression_with_parameters, value)
evaluation_issues: List[Tuple[int, str]] = []
s = State()
value, unresolved_vars = ast_evaluator(exp=ast2, state=s, obj=None, issue_lst=evaluation_issues)
if value is not None:
try:
value = float(value)
left = ast["left"]
right = ast["right"]
left_number = ast["number_left"]
right_number = ast["number_right"]
if left == "[":
value_meets_left = value >= left_number
else:
value_meets_left = value > left_number
if right == "]":
value_meets_right = value <= right_number
else:
value_meets_right = value < right_number
if not value_meets_left or not value_meets_right:
issues.append(Issue(itype=IType.ERROR,
description=f"The value {value} specified for the parameter '{p.name}' is out of the range {p.range}",
location=IssueLocation(sheet_name=sheet_name, row=row, column=None)))
retval = False
except:
issues.append(Issue(itype=IType.ERROR,
description=f"The parameter '{p.name}' has a non numeric value '{value}', and has been constrained with a numeric range. Please, either change the Value or the Range",
location=IssueLocation(sheet_name=sheet_name, row=row, column=None)))
retval = False
else:
pass # The parameter depends on other parameters, a valid situation
except: # A hierarchy name
h = glb_idx.get(Hierarchy.partial_key(p.range))
h = h[0]
if value not in h.codes.keys():
issues.append(Issue(itype=IType.ERROR,
description=f"The value '{value}' specified for the parameter '{p.name}' is not in the codes of the hierarchy '{p.range}': {', '.join(h.codes.keys())}",
location=IssueLocation(sheet_name=sheet_name, row=row, column=None)))
retval = False
return retval
def export_model_to_json(state: State) -> str:
# Get Metadata dictionary
metadata = state.get("_metadata")
metadata_string = None
if metadata:
# Change lists of 1 element to a simple value
metadata = {k: v[0] if len(v) == 1 else v for k, v in metadata.items()}
metadata_string = '"Metadata": ' + json.dumps(metadata,
cls=CustomEncoder)
json_structure: JsonStructureType = OrderedDict({
"Parameters":
Parameter,
"CodeHierarchies":
Hierarchy,
"Observers":
Observer,
"InterfaceTypes":
FactorType,
"InterfaceTypeConverts":
FactorTypesRelationUnidirectionalLinearTransformObservation,
"Processors":
Processor,
"Relationships":
OrderedDict({
"PartOf": ProcessorsRelationPartOfObservation,
"Upscale": ProcessorsRelationUpscaleObservation,
"IsA": ProcessorsRelationIsAObservation,
"DirectedFlow": FactorsRelationDirectedFlowObservation
})
})
json_string = model_to_json(state, json_structure)
if metadata_string:
json_string = f'{metadata_string}, {json_string}'
return f'{{{json_string}}}'
def convert_generator_to_native(generator_type, file_type: str, file):
"""
Converts a generator
Creates a generator parser, then it feeds the file type and the file
The generator parser has to parse the file and to elaborate a native generator (JSON)
:param generator_type:
:param file_type:
:param file:
:return: Issues and output file
"""
output = []
if generator_type.lower() not in ["json", "native", "primitive"]:
# Create commands generator from factory (from generator_type and file_type)
state = State()
commands_generator = commands_container_parser_factory(
generator_type, file_type, file, state)
# Loop over the IExecutableCommand instances
for cmd, issues in commands_generator:
# If there are syntax ERRORS, STOP!!!
stop = False
if issues and len(issues) > 0:
for t in issues:
if t["type"] == 3: # Error
stop = True
break
output.append({
"command": cmd._serialization_type,
"label": cmd._serialization_label,
"content": cmd.json_serialize(),
"issues": issues
})
if stop:
break
return output
def persistable_to_executable_command(p_cmd: CommandsContainer, limit=1000):
"""
A persistable command can be either a single command or a sequence of commands (like a spreadsheet). In the future
it could even be a full script.
Because the executable command is DIRECTLY executable, it is not possible to convert from persistable to a single
executable command. But it is possible to obtain a list of executable commands, and this is the aim of this function
The size of the list can be limited by the parameter "limit". "0" is for unlimited
:param p_cmd:
:return: A list of IExecutableCommand
"""
# Create commands generator from factory (from generator_type and file_type)
state = State()
commands_generator = commands_container_parser_factory(
p_cmd.generator_type, p_cmd.file_type, p_cmd.file, state)
# Loop over the IExecutableCommand instances
issues_aggreg = []
outputs = []
count = 0
for cmd, issues in commands_generator:
# If there are syntax ERRORS, STOP!!!
stop = False
if issues and len(issues) > 0:
for t in issues:
if t[0] == 3: # Error
stop = True
if stop:
break
issues_aggreg.extend(issues)
count += 1
if count >= limit:
break
def get_interfaces(glb_idx: PartialRetrievalDictionary) -> pd.DataFrame:
# Used to examine "value" as expression, and find variables that are interface names vs parameter names
params = create_dictionary(
data={p.name: None
for p in glb_idx.get(Parameter.partial_key())})
s = State()
procs = glb_idx.get(Processor.partial_key())
d = {}
for p in procs:
parent_relations = glb_idx.get(
ProcessorsRelationPartOfObservation.partial_key(child=p))
d[p.ident] = set([p.parent_processor.ident for p in parent_relations])
lst = [[
"Processor", "InterfaceType", "Interface", "Sphere", "RoegenType",
"Orientation", "OppositeSubsystemType", "GeolocationRef",
"GeolocationCode", "InterfaceAttributes", "Value", "Unit",
"RelativeTo", "Uncertainty", "Assessment", "PedigreeMatrix",
"Pedigree", "Time", "Source", "NumberAttributes", "Comments"
]]
# Elaborate a DAG, then iterate over it
for ident in list(toposort.toposort_flatten(d)):
p = glb_idx.get(Processor.partial_key(ident=ident))[0]
ifaces = glb_idx.get((Factor.partial_key(processor=p)))
iface_names = create_dictionary(
data={iface.name: iface
for iface in ifaces})
# Elaborate DAG of Interfaces because of Observations
d = {}
for iface in ifaces:
if iface.ident not in d:
d[iface.ident] = set()
for obs in iface.quantitative_observations:
if obs.relative_factor:
d[iface.ident].add(obs.relative_factor.ident)
# Consider obs.value and non linear dependencies
if isinstance(obs.value, str):
ast = string_to_ast(expression_with_parameters, obs.value)
evaluation_issues = []
value, unresolved_vars = ast_evaluator(
exp=ast,
state=s,
obj=None,
issue_lst=evaluation_issues)
for unresolved in unresolved_vars:
if unresolved not in params:
d[iface.ident].add(iface_names[unresolved].ident)
for ident2 in list(toposort.toposort_flatten(d)):
iface = glb_idx.get(Factor.partial_key(ident=ident2))[0]
lst1 = [
iface.processor.name, iface.taxon.name, iface.name,
iface.sphere, iface.roegen_type.name, iface.orientation,
iface.opposite_processor_type, "", "", ""
]
observations = iface.quantitative_observations
if len(observations) > 0:
for obs in observations:
lst2 = [
obs.value,
obs.attributes.get("unit",
""), obs.relative_factor.name
if obs.relative_factor else "",
obs.attributes.get("spread", ""),
obs.attributes.get("assessment", ""),
obs.attributes.get("pedigree_template", ""),
obs.attributes.get("pedigree", ""),
obs.attributes.get("time", ""),
obs.observer.name if obs.observer else "", "",
obs.attributes.get("comments", "")
]
lst.append(lst1 + lst2)
else:
lst.append(lst1 + ["", "", "", "", "", "", "", "", "", ""])
return list_to_dataframe(lst)
def open(self,
session_factory,
uuid_: str = None,
recover_previous_state=True,
cr_new: CreateNew = CreateNew.NO,
allow_saving=True):
"""
Open a work session
+--------+--------+---------+-----------------------------------------------------------------------------------+
| UUID | cr_new | recover | Behavior |
+--------+--------+---------+-----------------------------------------------------------------------------------+
| !=None | True | True | Create new CS or version (branch) from "UUID", clone WS, recover State, append WS |
| !=None | True | False | Create new CS or version (branch) from "UUID", Zero State, first WS |
| !=None | False | True | Recover State, append WS |
| !=None | False | False | Zero State, append WS (overwrite type) |
| ==None | - | - | New CS and version, Zero State, first WS |
+--------+--------+---------+-----------------------------------------------------------------------------------+
Use cases:
* A new case study, from scratch. uuid_=None
* A new case study, copied from another case study. uuid_=<something>, cr_new=CreateNew.CASE_STUDY, recover_previous_state=True
* A new version of a case study
- Copying previous version
- Starting from scratch
* Continue a case study version
- But restart (from scratch)
* Can be a Transient session
:param uuid_: UUID of the case study or case study version. Can be None, for new case studies or for testing purposes.
:param recover_previous_state: If an existing version is specified, it will recover its state after execution of all command_executors
:param cr_new: If != CreateNew.NO, create either a case study or a new version. If == CreateNew.NO, append session to "uuid"
:param allow_saving: If True, it will allow saving at the end (it will be optional). If False, trying to save will generate an Exception
:return UUID of the case study version in use. If it is a new case study and it has not been saved, the value will be "None"
"""
# TODO Just register for now. But in the future it should control that there is no other "allow_saving" ReproducibleSession opened
# TODO for the same Case Study Version. So it implies modifying some state in CaseStudyVersion to have the UUID
# TODO of the active ReproducibleSession, even if it is not in the database. Register also the date of "lock", so the
# TODO lock can be removed in case of "hang" of the locker ReproducibleSession
self._allow_saving = allow_saving
self._sess_factory = session_factory
session = self._sess_factory()
if uuid_:
uuid_ = str(uuid_)
# Find UUID. Is it a Case Study or a Case Study version?
# If it is the former, look for the active version.
cs = session.query(CaseStudy).filter(
CaseStudy.uuid == uuid_).first()
if not cs:
vs = session.query(CaseStudyVersion).filter(
CaseStudyVersion.uuid == uuid_).first()
if not vs:
ss = session.query(CaseStudyVersionSession).filter(
CaseStudyVersionSession.uuid == uuid_).first()
if not ss:
raise Exception(
"Object '" + uuid_ +
"' not found, when opening a ReproducibleSession")
else:
vs = ss.version
cs = vs.case_study
else:
cs = vs.case_study
else: # A case study, find the latest version (the version modified latest -by activity, newest ReproducibleSession-)
max_date = None
max_version = None
for v in cs.versions:
for s in v.sessions:
if not max_date or s.open_instant > max_date:
max_date = s.open_instant
max_version = v
vs = max_version
cs = vs.case_study
# List of active sessions
# NOTE: instead of time ordering, the ID is used, assuming sessions with greater ID were created later
lst = session.query(CaseStudyVersionSession). \
filter(CaseStudyVersionSession.version_id == vs.id). \
order_by(CaseStudyVersionSession.id). \
all()
idx = 0
for i, ws in enumerate(lst):
if ws.restarts:
idx = i
lst = lst[idx:] # Cut the list, keep only active sessions
if cr_new != CreateNew.NO: # Create either a case study or a case study version
if cr_new == CreateNew.CASE_STUDY:
cs = copy.copy(cs) # New Case Study: COPY CaseStudy
else:
force_load(
cs
) # New Case Study Version: LOAD CaseStudy (then version it)
vs2 = copy.copy(vs) # COPY CaseStudyVersion
vs2.case_study = cs # Assign case study to the new version
if recover_previous_state: # If the new version keeps previous state, copy it also
vs2.state = vs.state # Copy state
vs2.state_version = vs.state_version
for ws in lst: # COPY active ReproducibleSessions
ws2 = copy.copy(ws)
ws2.version = vs2
for c in ws.commands: # COPY commands
c2 = copy.copy(c)
c2.session = ws2
vs = vs2
else:
# Load into memory
if len(lst) == 1:
ws = lst[0]
force_load(ws)
force_load(vs)
force_load(cs)
if recover_previous_state:
# Load state if it is persisted (if not EXECUTE, POTENTIALLY VERY SLOW)
if vs.state:
# Deserialize
self._isess._state = deserialize_state(
vs.state, vs.state_version)
else:
self._isess._state = State(
) # Zero State, execute all commands in sequence
for ws in lst:
for c in ws.commands:
execute_command_container(self._isess._state, c)
if cr_new == CreateNew.VERSION: # TODO Check if this works in all possible circumstances (combine the parameters of the function)
recover_previous_state = False
else:
self._isess._state = State()
else: # New Case Study AND new Case Study Version
cs = CaseStudy()
vs = CaseStudyVersion()
vs.creation_instant = datetime.datetime.utcnow()
vs.case_study = cs
# Detach Case Study and Case Study Version
if cs in session:
session.expunge(cs)
if vs in session:
session.expunge(vs)
# Create the Case Study Version Session
usr = session.query(User).filter(User.name == self._identity).first()
if usr:
force_load(usr)
else:
if allow_saving:
raise Exception(
"A user is required to register which user is authoring a case study"
)
# TODO !!!!NEW CODE, ADDED TO SUPPORT NEEDED FUNCTIONALITY. NEEDS BETTER CODING!!!!
restart = not recover_previous_state if uuid_ else True
if not restart:
self._session = ws
else:
self._session = CaseStudyVersionSession()
self._session.version = vs
self._session.who = usr
self._session.restarts = True
# If the Version existed, define "restarts" according to parameter "recover_previous_state"
# ElseIf it is the first Session -> RESTARTS=True
session.close()
# session.expunge_all()
self._sess_factory.remove()
def reset_state(self):
""" Restart state """
self._state = State()
class InteractiveSession:
"""
Main class for interaction with NIS
The first thing would be to identify the user and create a GUID for the session which can be used by the web server
to store and retrieve the interactive session state.
It receives command_executors, modifying state accordingly
If a reproducible session is opened,
"""
def __init__(self, session_factory):
# Session factory with access to business logic database
self._session_factory = session_factory
# Interactive session ID
self._guid = str(uuid.uuid4())
# User identity, if given (can be an anonymous session)
self._identity = None # type: Identity
self._state = State() # To keep the state
self._reproducible_session = None # type: ReproducibleSession
def reset_state(self):
""" Restart state """
self._state = State()
# TODO self._recordable_session = None ??
@property
def state(self):
return self._state
def get_sf(self):
return self._session_factory
def set_sf(self, session_factory):
self._session_factory = session_factory
if self._reproducible_session:
self._reproducible_session.set_sf(session_factory)
def open_db_session(self):
return self._session_factory()
def close_db_session(self):
self._session_factory.remove()
def quit(self):
"""
End interactive session
:return:
"""
self.close_reproducible_session()
self.close_db_session()
# --------------------------------------------------------------------------------------------
def identify(self, identity_information, testing=False):
"""
Given credentials of some type -identity_information-, link an interactive session to an identity.
The credentials can vary from an OAuth2 Token to user+password.
Depending on the type of credentials, invoke a type of "identificator" or other
An interactive session without identification is allowed to perform a subset of available operations
:param identity_information:
:return: True if the identification was successful, False if not
"""
# TODO Check the credentials
if isinstance(identity_information, dict):
if "user" in identity_information and testing:
# Check if the user is in the User's table
session = self._session_factory()
src = session.query(User).filter(
User.name == identity_information["user"]).first()
# Check if the dataset exists. "ETL" it if not
# ds = session.query(Dataset).\
# filter(Dataset.code == dataset).\
# join(Dataset.database).join(Database.data_source).\
# filter(DataSource.name == src_name).first()
force_load(src)
session.close()
self._session_factory.remove()
if src:
self._identity = src.name
if self._state:
self._state.set("_identity", self._identity)
return src is not None
elif "token" in identity_information:
# TODO Validate against some Authentication service
pass
def get_identity_id(self):
return self._identity
def unidentify(self):
# TODO The un-identification cannot be done in the following circumstances: any?
self._identity = None
if self._state:
self._state.set("_identity", self._identity)
# --------------------------------------------------------------------------------------------
# Reproducible sessions and commands INSIDE them
# --------------------------------------------------------------------------------------------
def open_reproducible_session(self,
case_study_version_uuid: str,
recover_previous_state=True,
cr_new: CreateNew = CreateNew.NO,
allow_saving=True):
self._reproducible_session = ReproducibleSession(self)
self._reproducible_session.open(self._session_factory,
case_study_version_uuid,
recover_previous_state, cr_new,
allow_saving)
def close_reproducible_session(self,
issues=None,
output=None,
save=False,
from_web_service=False,
cs_uuid=None,
cs_name=None):
if self._reproducible_session:
if save:
# TODO Save issues AND (maybe) output
self._reproducible_session.save(from_web_service,
cs_uuid=cs_uuid,
cs_name=cs_name)
uuid_, v_uuid, cs_uuid = self._reproducible_session.close()
self._reproducible_session = None
return uuid_, v_uuid, cs_uuid
else:
return None, None, None
def reproducible_session_opened(self):
return self._reproducible_session is not None
@property
def reproducible_session(self):
return self._reproducible_session
# --------------------------------------------------------------
def execute_executable_command(self, cmd: IExecutableCommand):
return execute_command(self._state, cmd)
def register_executable_command(self, cmd: IExecutableCommand):
self._reproducible_session.register_executable_command(cmd)
def register_andor_execute_command_generator1(self,
c: CommandsContainer,
register=True,
execute=False):
"""
Creates a generator parser, then it feeds the file type and the file
The generator parser has to parse the file and to generate command_executors as a Python generator
:param generator_type:
:param file_type:
:param file:
:param register: If True, register the command in the ReproducibleSession
:param execute: If True, execute the command in the ReproducibleSession
:return:
"""
if not self._reproducible_session:
raise Exception(
"In order to execute a command generator, a work session is needed"
)
if not register and not execute:
raise Exception(
"More than zero of the parameters 'register' and 'execute' must be True"
)
if register:
self._reproducible_session.register_persistable_command(c)
if execute:
c.execution_start = datetime.datetime.now()
pass_case_study = self._reproducible_session._session.version.case_study is not None
ret = self._reproducible_session.execute_command_generator(
c, pass_case_study)
c.execution_end = datetime.datetime.now()
return ret
# Or
# return execute_command_container(self._state, c)
else:
return None
def register_andor_execute_command_generator(self,
generator_type,
file_type: str,
file,
register=True,
execute=False):
"""
Creates a generator parser, then it feeds the file type and the file
The generator parser has to parse the file and to generate command_executors as a Python generator
:param generator_type:
:param file_type:
:param file:
:param register: If True, register the command in the ReproducibleSession
:param execute: If True, execute the command in the ReproducibleSession
:return:
"""
return self.register_andor_execute_command_generator1(
CommandsContainer.create(generator_type, file_type, file),
register, execute)
# --------------------------------------------------------------------------------------------
def get_case_studies(self):
""" Get a list of case studies READABLE by current identity (or public if anonymous) """
pass
def get_case_study_versions(self, case_study: str):
# TODO Check that the current user has READ access to the case study
pass
def get_case_study_version(self, case_study_version: str):
# TODO Check that the current user has READ access to the case study
pass
def get_case_study_version_variables(self, case_study_version: str):
""" A tree of variables, by type: processors, flows """
pass
def get_case_study_version_variable(self, case_study_version: str,
variable: str):
pass
def remove_case_study_version(self, case_study_version: str):
pass
def share_case_study(self, case_study: str, identities: List[str],
permission: str):
pass
def remove_case_study_share(self, case_study: str, identities: List[str],
permission: str):
pass
def get_case_study_permissions(self, case_study: str):
pass
def export_case_study_version(self, case_study_version: str):
pass
def import_case_study(self, file):
pass
"'Hola' + Param1"
]
for e in examples:
try:
ast = string_to_ast(arith_boolean_expression, e)
print(ast)
except:
traceback.print_exc()
print("Incorrect")
s = "c1 + c30 - c2 - 10"
res = string_to_ast(hierarchy_expression, s)
s = "ds.col"
res = string_to_ast(h_name, s)
s = State()
examples = [
"5-2017", "2017-5", "2017/5", "2017-05 - 2018-01", "2017",
"5-2017 - 2018-1", "2017-2018", "Year", "Month"
]
for example in examples:
print(example)
res = string_to_ast(time_expression, example)
print(res)
print(f'Is year = {is_year(example)}')
print(f'Is month = {is_month(example)}')
print("-------------------")
for list1 in [
"this, is, a,simple_ident , list",
"this,is,'NOT', a,simple_ident , list", " word ", "", " , ", None
def _init_and_process_row(self, row: Dict[str, Any]) -> None:
def obtain_dictionary_with_not_expandable_fields(d):
output = {}
for k, v in d.items():
if v is None or "{" not in v:
output[k] = v
return output
self._current_row_number = row["_row"]
self._fields = self._get_command_fields_values(row)
tmp_fields = self._fields
self._check_all_mandatory_fields_have_values()
# If expandable, do it now
expandable = row["_expandable"]
if expandable:
# Extract variables
state = State()
issues = []
asts = {}
referenced_variables = create_dictionary()
for e in expandable:
ast = parser_field_parsers.string_to_ast(
arith_boolean_expression, e)
c_name = f"{{{e}}}"
asts[c_name] = ast
res, vars = ast_evaluator(ast,
state,
None,
issues,
atomic_h_names=True)
for v in vars:
referenced_variables[v] = None
res = classify_variables2(referenced_variables.keys(),
self._datasets, self._hierarchies,
self._parameters)
ds_list = res["datasets"]
ds_concepts = res["ds_concepts"]
h_list = res["hierarchies"]
if len(ds_list) >= 1 and len(h_list) >= 1:
self._add_issue(
itype=IType.ERROR,
description="Dataset(s): " +
", ".join([d.name
for d in ds_list]) + ", and hierarchy(ies): " +
", ".join([h.name for h in h_list]) +
", have been specified. Only a single dataset is supported."
)
return
elif len(ds_list) > 1:
self._add_issue(
itype=IType.ERROR,
description="More than one dataset has been specified: " +
", ".join([d.name for d in ds_list]) +
", just one dataset is supported.")
return
elif len(h_list) > 0:
self._add_issue(
itype=IType.ERROR,
description="One or more hierarchies have been specified: "
+ ", ".join([h.name for h in h_list]))
return
if len(ds_list) == 1: # Expand dataset
ds = ds_list[0]
measure_requested = False
all_dimensions = set(
[c.code for c in ds.dimensions if not c.is_measure])
requested_dimensions = set()
requested_measures = set()
for con in ds_concepts:
found = False
for c in ds.dimensions:
if strcmp(c.code, con):
found = True
if c.is_measure:
measure_requested = True
requested_measures.add(c.code)
else: # Dimension
all_dimensions.remove(c.code)
requested_dimensions.add(c.code)
if not found:
self._add_issue(
itype=IType.ERROR,
description=
f"The concept '{{{ds.code}.{con}}}' is not in the dataset '{ds.code}'"
)
return
ds_concepts = list(requested_measures)
ds_concepts.extend(list(requested_dimensions))
all_dimensions_requested = len(all_dimensions) == 0
if measure_requested and not all_dimensions_requested:
self._add_issue(
IType.ERROR,
f"It is not possible to use a measure ({', '.join(requested_measures)}), if not all dimensions are used "
f"(cannot assume implicit aggregation). Dimensions not used: {', '.join(all_dimensions)}"
)
return
elif not measure_requested and not all_dimensions_requested:
# Reduce the Dataframe to unique tuples of the specified dimensions
# TODO Consider the current case -sensitive or not-sensitive-
data = ds.data[list(
requested_dimensions)].drop_duplicates()
else: # Take the dataset as-is
data = ds.data
# Remove Index, and do it NOT-INPLACE
data = data.reset_index()
# Drop rows with empty dimension value
import numpy as np
data = data.replace(r'^\s*$', np.NaN, regex=True)
data.dropna(subset=requested_dimensions, inplace=True)
const_dict = obtain_dictionary_with_not_expandable_fields(
self._fields) # row?
var_dict = set(
[f for f in self._fields.keys() if f not in const_dict])
re_concepts = {}
for c in ds_concepts:
c_name = f"{{{ds.code}.{c}}}"
if case_sensitive:
re_concepts[c_name] = re.compile(c_name)
else:
re_concepts[c_name] = re.compile(c_name, re.IGNORECASE)
location = IssueLocation(sheet_name=self._command_name,
row=self._current_row_number,
column=None)
already_parsed_fields = set(const_dict.keys())
for ds_row, row2 in enumerate(
data.iterrows()): # Each row in the dataset
# Initialize constant values (those with no "{..}" expressions)
row3 = const_dict.copy()
# Prepare state to evaluate functions
state = State()
for c in ds_concepts:
state.set(f"{ds.code}.{c}", str(row2[1][c]))
state.set(
"_glb_idx", self._glb_idx
) # Pass PartialRetrievalDictionary to the evaluator. For functions needing it
# Evaluate all functions
expressions = {}
for e, ast in asts.items():
res, vars = ast_evaluator(ast,
state,
None,
issues,
atomic_h_names=True)
expressions[e] = res
# Expansion into var_dict
for f in var_dict:
v = self._fields[f] # Initial value
for item in sorted(expressions.keys(),
key=len,
reverse=True):
v = v.replace(item, expressions[item])
row3[f] = v
# # Concepts change dictionary
# concepts = {}
# for c in ds_concepts:
# concepts[f"{{{ds.code}.{c}}}"] = str(row2[1][c])
# # Expansion into var_dict
# for f in var_dict:
# v = self._fields[f] # Initial value
# for item in sorted(concepts.keys(), key=len, reverse=True):
# v = re_concepts[item].sub(concepts[item], v)
# row3[f] = v
# Syntactic verification of the resulting expansion
processable, tmp_issues = parse_cmd_row_dict(
self._serialization_type, row3, already_parsed_fields,
location)
if len(tmp_issues) > 0:
self._issues.extend(tmp_issues)
# Process row
if processable:
self._fields = row3
self._process_row(row3, ds_row)
self._fields = tmp_fields
elif len(h_list) == 1: # Expand hierarchy
pass
else:
self._process_row(self._fields) # Process row
def ast_evaluator(exp: Dict, state: State, obj, issue_lst, evaluation_type="numeric", atomic_h_names=False, allowed_functions=global_functions) -> Union[Tuple[float, List[str]], Tuple[str, float, List[str]]]:
"""
Numerically evaluate the result of the parse of "expression" rule (not valid for the other "expression" rules)
:param exp: Dictionary representing the AST (output of "string_to_ast" function)
:param state: "State" used to obtain variables/objects
:param obj: An object used when evaluating hierarchical variables. simple names, functions and datasets are considered members of this object
:param issue_lst: List in which issues have to be annotated
:param evaluation_type: "numeric" for full evaluation, "static" to return True if the expression can be evaluated
(explicitly mentioned variables are defined previously)
:param atomic_h_names: If True, treat variable names as atomic (False processes them part by part, from left to right). Used in dataset expansion
:return: value (scalar EXCEPT for named parameters, which return a tuple "parameter name - parameter value"), list of unresolved variables
"""
val = None
unresolved_vars = set()
if "type" in exp:
t = exp["type"]
if t in ("int", "float", "str", "boolean"):
if evaluation_type == "numeric":
return exp["value"], unresolved_vars
elif evaluation_type == "static":
return unresolved_vars
elif t == "named_parameter":
# This one returns a tuple (parameter name, parameter value, unresolved variables)
v, tmp = ast_evaluator(exp["value"], state, obj, issue_lst, evaluation_type, atomic_h_names)
unresolved_vars.update(tmp)
return exp["param"], v, unresolved_vars
elif t == "key_value_list":
d = create_dictionary()
for k, v in exp["parts"].items():
d[k], tmp = ast_evaluator(v, state, obj, issue_lst, evaluation_type, atomic_h_names, allowed_functions)
unresolved_vars.update(tmp)
return d, unresolved_vars
elif t == "dataset":
# Function parameters and Slice parameters
func_params = [ast_evaluator(p, state, obj, issue_lst, evaluation_type, atomic_h_names, allowed_functions) for p in exp["func_params"]]
slice_params = [ast_evaluator(p, state, obj, issue_lst, evaluation_type, atomic_h_names, allowed_functions) for p in exp["slice_params"]]
if evaluation_type == "numeric":
# Find dataset named "exp["name"]"
if obj is None:
# Global dataset
ds = state.get(exp["name"], exp["ns"])
if not ds:
issue_lst.append((3, "Global dataset '" + exp["name"] + "' not found"))
else:
# Dataset inside "obj"
try:
ds = getattr(obj, exp["name"])
except:
ds = None
if not ds:
issue_lst.append((3, "Dataset '" + exp["name"] + "' local to "+str(obj)+" not found"))
if ds and isinstance(ds, ExternalDataset):
return ds.get_data(None, slice_params, None, None, func_params)
else:
return None
elif evaluation_type == "static":
# Find dataset named "exp["name"]"
if obj is None:
# Global dataset
ds = state.get(exp["name"], exp["ns"])
if not ds:
issue_lst.append((3, "Global dataset '" + exp["name"] + "' not found"))
else:
ds = True
else:
ds = True # We cannot be sure it will be found, but do not break the evaluation
# True if the Dataset is True, and the parameters are True
return ds and all(func_params) and all(slice_params)
elif t == "function": # Call function
# First, obtain the Parameters
args = []
kwargs = {}
can_resolve = True
for p in [ast_evaluator(p, state, obj, issue_lst, evaluation_type, atomic_h_names, allowed_functions) for p in exp["params"]]:
if len(p) == 3:
kwargs[p[0]] = p[1]
tmp = p[2]
else:
args.append(p[0])
tmp = p[1]
unresolved_vars.update(tmp)
if len(tmp) > 0:
can_resolve = False
if evaluation_type == "numeric":
if obj is None:
# Check if it can be resolved (all variables specified)
# Check if global function exists, then call it. There are no function namespaces (at least for now)
if can_resolve and exp["name"] in allowed_functions:
_f = allowed_functions[exp["name"]]
mod_name, func_name = _f["full_name"].rsplit('.', 1)
mod = importlib.import_module(mod_name)
func = getattr(mod, func_name)
if _f["kwargs"]:
kwargs.update(_f["kwargs"])
if _f["special_kwargs"]:
for sp_kwarg, name in _f["special_kwargs"].items():
if sp_kwarg == "PartialRetrievalDictionary":
kwargs[name] = state.get("_glb_idx")
elif sp_kwarg == "ProcessorsDOM":
kwargs[name] = state.get("_processors_dom")
elif sp_kwarg == "ProcessorsMap":
kwargs[name] = state.get("_processors_map")
elif sp_kwarg == "DataFrameGroup":
kwargs[name] = state.get("_df_group")
elif sp_kwarg == "IndicatorsDataFrameGroup":
kwargs[name] = state.get("_df_indicators_group")
elif sp_kwarg == "IndicatorState":
kwargs[name] = state
elif sp_kwarg == "LCIAMethods":
kwargs[name] = state.get("_lcia_methods")
# CALL FUNCTION!!
try:
obj = func(*args, **kwargs)
except Exception as e:
obj = None
issue_lst.append(str(e))
else:
# CALL FUNCTION LOCAL TO THE OBJECT (a "method")
try:
obj = getattr(obj, exp["name"])
obj = obj(*args, **kwargs)
except Exception as e:
obj = None
issue_lst.append(str(e))
return obj, unresolved_vars
elif evaluation_type == "static":
if obj is None:
# Check if global function exists, then call it. There are no function namespaces (at least for now)
if exp["name"] in allowed_functions:
_f = allowed_functions[exp["name"]]
mod_name, func_name = _f["full_name"].rsplit('.', 1)
mod = importlib.import_module(mod_name)
func = getattr(mod, func_name)
# True if everything is True: function defined and all parameters are True
obj = func and all(args) and all(kwargs.values())
else:
# Call local function (a "method")
obj = True
return obj
elif t == "h_var":
# Evaluate in sequence
obj = None
_namespace = exp.get("ns", None)
if atomic_h_names:
h_name = '.'.join(exp["parts"])
exp["parts"] = [h_name]
for o in exp["parts"]:
if isinstance(o, str):
# Simple name
if obj is None:
obj = state.get(o, _namespace)
if obj is None:
issue_lst.append((3, "'" + o + "' is not globally declared in namespace '" + (_namespace if _namespace else "default") + "'"))
if _namespace:
unresolved_vars.add(_namespace+"::"+o)
else:
unresolved_vars.add(o)
else:
if isinstance(obj, ExternalDataset):
# Check if "o" is column (measure) or dimension
if o in obj.get_columns() or o in obj.get_dimensions():
obj = obj.get_data(o, None)
else:
issue_lst.append((3, "'" + o + "' is not a measure or dimension of the dataset."))
else:
try:
obj = getattr(obj, o)
except:
issue_lst.append((3, "'" + o + "' is not a ."))
else:
# Dictionary: function call or dataset access
if obj is None:
o["ns"] = _namespace
obj = ast_evaluator(o, state, obj, issue_lst, evaluation_type, atomic_h_names, allowed_functions)
if obj is None or isinstance(obj, (str, int, float, bool)):
return obj, unresolved_vars
# TODO elif isinstance(obj, ...) depending on core object types, invoke a default method, or
# issue ERROR if it is not possible to cast to something simple
else:
return obj, unresolved_vars
elif t == "condition": # Evaluate IF part to a Boolean. If True, return the evaluation of the THEN part; if False, return None
if_result, tmp = ast_evaluator(exp["if"], state, obj, issue_lst, evaluation_type, atomic_h_names, allowed_functions)
unresolved_vars.update(tmp)
if len(tmp) == 0:
if if_result:
then_result, tmp = ast_evaluator(exp["then"], state, obj, issue_lst, evaluation_type, atomic_h_names, allowed_functions)
unresolved_vars.update(tmp)
if len(tmp) > 0:
then_result = None
return then_result, unresolved_vars
else:
return None, unresolved_vars
elif t == "conditions":
for c in exp["parts"]:
cond_result, tmp = ast_evaluator(c, state, obj, issue_lst, evaluation_type, atomic_h_names, allowed_functions)
unresolved_vars.update(tmp)
if len(tmp) == 0:
if cond_result:
return cond_result, unresolved_vars
return None, unresolved_vars
elif t == "reference":
return "[" + exp["ref_id"] + "]", unresolved_vars # TODO Return a special type
elif t in ("u+", "u-", "exponentials", "multipliers", "adders", "comparison", "not", "and", "or"): # Arithmetic and Boolean
# Evaluate recursively the left and right operands
if t in ("u+", "u-"):
if evaluation_type == "numeric":
current = 0
else:
current = True
tmp1 = [] # Unary operators do not have "left" side. So empty list for unresolved vars
else:
current, tmp1 = ast_evaluator(exp["terms"][0], state, obj, issue_lst, evaluation_type, atomic_h_names, allowed_functions)
unresolved_vars.update(tmp1)
for i, e in enumerate(exp["terms"][1:]):
following, tmp2 = ast_evaluator(e, state, obj, issue_lst, evaluation_type, atomic_h_names, allowed_functions)
unresolved_vars.update(tmp2)
if len(tmp1) == 0 and len(tmp2) == 0:
if evaluation_type == "numeric":
# Type casting for primitive types
# TODO For Object types, apply default conversion. If both sides are Object, assume number
if (isinstance(current, (int, float)) and isinstance(following, (int, float))) or \
(isinstance(current, bool) and isinstance(following, bool)) or \
(isinstance(current, str) and isinstance(following, str)):
pass # Do nothing
else: # In others cases, CAST to the operand of the left. This may result in an Exception
if current is not None:
following = type(current)(following)
op = exp["ops"][i].lower()
if op in ("+", "-", "u+", "u-"):
if current is None:
current = 0
if following is None:
following = 0
if op in ("-", "u-"):
following = -following
current += following
elif op in ("*", "/", "//", "%", "**", "^"):
if following is None:
following = 1
if current is None:
current = 1
if op == "*":
current *= following
elif op == "/":
current /= following
elif op == "//":
current //= following
elif op == "%":
current %= following
elif op in ("**", "^"):
current ^= following
elif op == "not":
current = not bool(following)
elif op == "and":
current = current and following
elif op == "or":
current = current or following
else: # Comparators
fn = opMap[op]
current = fn(current, following)
elif evaluation_type == "static":
current = current and following
else:
current = None # Could not evaluate because there are missing variables
if len(unresolved_vars) > 0:
current = None
return current, unresolved_vars
else:
issue_lst.append((3, "'type' = "+t+" not supported."))
else:
issue_lst.append((3, "'type' not present in "+str(exp)))
return val, unresolved_vars