def serialize_state(state: State):
"""
Serialization prepared for a given organization of the state
:return:
"""
def serialize_dataframe(df):
return df.to_json(orient="split") # list(df.index.names), df.to_dict()
print(" serialize_state IN")
import copy
# "_datasets"
ns_ds = {}
# Save and nullify before deep copy
for ns in state.list_namespaces():
_, _, _, datasets, _ = get_case_study_registry_objects(state, ns)
ns_ds[ns] = datasets
state.set("_datasets", create_dictionary(), ns) # Nullify datasets
# !!! WARNING: It destroys "state", so a DEEP COPY is performed !!!
tmp = sys.getrecursionlimit()
sys.setrecursionlimit(10000)
state2 = copy.deepcopy(state)
sys.setrecursionlimit(tmp)
# Iterate all namespaces
for ns in state2.list_namespaces():
glb_idx, p_sets, hh, _, mappings = get_case_study_registry_objects(
state2, ns)
if glb_idx:
tmp = glb_idx.to_pickable()
state2.set("_glb_idx", tmp, ns)
datasets = ns_ds[ns]
# TODO Serialize other DataFrames.
# Process Datasets
for ds_name in datasets:
ds = datasets[ds_name]
if isinstance(ds.data, pd.DataFrame):
tmp = serialize_dataframe(ds.data)
else:
tmp = None
# ds.data = None
# DB serialize the datasets
lst2 = serialize(ds.get_objects_list())
lst2.append(tmp) # Append the serialized DataFrame
datasets[ds_name] = lst2
state2.set("_datasets", datasets, ns)
tmp = serialize_from_object(
state2) # <<<<<<<< SLOWEST !!!! (when debugging)
print(" serialize_state length: " + str(len(tmp)) + " OUT")
return tmp
def evaluate_parameters_for_scenario(base_params: List[Parameter],
scenario_params: Dict[str, str]):
"""
Obtain a dictionary (parameter -> value), where parameter is a string and value is a literal: number, boolean,
category or string.
Start from the base parameters then overwrite with the values in the current scenario.
Parameters may depend on other parameters, so this has to be considered before evaluation.
No cycles are allowed in the dependencies, i.e., if P2 depends on P1, P1 cannot depend on P2.
To analyze this, first expressions are evaluated, extracting which parameters appear in each of them. Then a graph
is elaborated based on this information. Finally, an algorithm to find cycles is executed.
:param base_params:
:param scenario_params:
:return:
"""
# Create dictionary without evaluation
result_params = create_dictionary()
result_params.update(
{p.name: p.default_value
for p in base_params if p.default_value})
# Overwrite with scenario expressions or constants
result_params.update(scenario_params)
state = State()
known_params = create_dictionary()
unknown_params = create_dictionary()
# Now, evaluate ALL expressions
for param, expression in result_params.items():
value, ast, params, issues = evaluate_numeric_expression_with_parameters(
expression, state)
if not value: # It is not a constant, store the parameters on which this depends
if case_sensitive:
unknown_params[param] = (ast, set(params))
else:
unknown_params[param] = (ast, set([p.lower() for p in params]))
else: # It is a constant, store it
result_params[param] = value # Overwrite
known_params[param] = value
cycles = get_circular_dependencies(unknown_params)
if len(cycles) > 0:
raise Exception(
f"Parameters cannot have circular dependencies. {len(cycles)} cycles were detected: "
f"{':: '.join(cycles)}")
# Initialize state with known parameters
state.update(known_params)
# Loop until no new parameters can be evaluated
previous_len_unknown_params = len(unknown_params) + 1
while len(unknown_params) < previous_len_unknown_params:
previous_len_unknown_params = len(unknown_params)
for param in list(
unknown_params
): # A list(...) is used because the dictionary can be modified inside
ast, params = unknown_params[param]
if params.issubset(known_params):
value, _, _, issues = evaluate_numeric_expression_with_parameters(
ast, state)
if not value:
raise Exception(
f"It should be possible to evaluate the parameter '{param}'. "
f"Issues: {', '.join(issues)}")
else:
del unknown_params[param]
result_params[param] = value
state.set(param, value)
if len(unknown_params) > 0:
raise Exception(
f"Could not evaluate the following parameters: {', '.join(unknown_params)}"
)
return result_params
class InteractiveSession:
"""
Main class for interaction with NIS
The first thing would be to identify the user and create a GUID for the session which can be used by the web server
to store and retrieve the interactive session state.
It receives command_executors, modifying state accordingly
If a reproducible session is opened,
"""
def __init__(self, session_factory):
# Session factory with access to business logic database
self._session_factory = session_factory
# Interactive session ID
self._guid = str(uuid.uuid4())
# User identity, if given (can be an anonymous session)
self._identity = None # type: Identity
self._state = State() # To keep the state
self._reproducible_session = None # type: ReproducibleSession
def reset_state(self):
""" Restart state """
self._state = State()
# TODO self._recordable_session = None ??
@property
def state(self):
return self._state
def get_sf(self):
return self._session_factory
def set_sf(self, session_factory):
self._session_factory = session_factory
if self._reproducible_session:
self._reproducible_session.set_sf(session_factory)
def open_db_session(self):
return self._session_factory()
def close_db_session(self):
self._session_factory.remove()
def quit(self):
"""
End interactive session
:return:
"""
self.close_reproducible_session()
self.close_db_session()
# --------------------------------------------------------------------------------------------
def identify(self, identity_information, testing=False):
"""
Given credentials of some type -identity_information-, link an interactive session to an identity.
The credentials can vary from an OAuth2 Token to user+password.
Depending on the type of credentials, invoke a type of "identificator" or other
An interactive session without identification is allowed to perform a subset of available operations
:param identity_information:
:return: True if the identification was successful, False if not
"""
# TODO Check the credentials
if isinstance(identity_information, dict):
if "user" in identity_information and testing:
# Check if the user is in the User's table
session = self._session_factory()
src = session.query(User).filter(
User.name == identity_information["user"]).first()
# Check if the dataset exists. "ETL" it if not
# ds = session.query(Dataset).\
# filter(Dataset.code == dataset).\
# join(Dataset.database).join(Database.data_source).\
# filter(DataSource.name == src_name).first()
force_load(src)
session.close()
self._session_factory.remove()
if src:
self._identity = src.name
if self._state:
self._state.set("_identity", self._identity)
return src is not None
elif "token" in identity_information:
# TODO Validate against some Authentication service
pass
def get_identity_id(self):
return self._identity
def unidentify(self):
# TODO The un-identification cannot be done in the following circumstances: any?
self._identity = None
if self._state:
self._state.set("_identity", self._identity)
# --------------------------------------------------------------------------------------------
# Reproducible sessions and commands INSIDE them
# --------------------------------------------------------------------------------------------
def open_reproducible_session(self,
case_study_version_uuid: str,
recover_previous_state=True,
cr_new: CreateNew = CreateNew.NO,
allow_saving=True):
self._reproducible_session = ReproducibleSession(self)
self._reproducible_session.open(self._session_factory,
case_study_version_uuid,
recover_previous_state, cr_new,
allow_saving)
def close_reproducible_session(self,
issues=None,
output=None,
save=False,
from_web_service=False,
cs_uuid=None,
cs_name=None):
if self._reproducible_session:
if save:
# TODO Save issues AND (maybe) output
self._reproducible_session.save(from_web_service,
cs_uuid=cs_uuid,
cs_name=cs_name)
uuid_, v_uuid, cs_uuid = self._reproducible_session.close()
self._reproducible_session = None
return uuid_, v_uuid, cs_uuid
else:
return None, None, None
def reproducible_session_opened(self):
return self._reproducible_session is not None
@property
def reproducible_session(self):
return self._reproducible_session
# --------------------------------------------------------------
def execute_executable_command(self, cmd: IExecutableCommand):
return execute_command(self._state, cmd)
def register_executable_command(self, cmd: IExecutableCommand):
self._reproducible_session.register_executable_command(cmd)
def register_andor_execute_command_generator1(self,
c: CommandsContainer,
register=True,
execute=False):
"""
Creates a generator parser, then it feeds the file type and the file
The generator parser has to parse the file and to generate command_executors as a Python generator
:param generator_type:
:param file_type:
:param file:
:param register: If True, register the command in the ReproducibleSession
:param execute: If True, execute the command in the ReproducibleSession
:return:
"""
if not self._reproducible_session:
raise Exception(
"In order to execute a command generator, a work session is needed"
)
if not register and not execute:
raise Exception(
"More than zero of the parameters 'register' and 'execute' must be True"
)
if register:
self._reproducible_session.register_persistable_command(c)
if execute:
c.execution_start = datetime.datetime.now()
pass_case_study = self._reproducible_session._session.version.case_study is not None
ret = self._reproducible_session.execute_command_generator(
c, pass_case_study)
c.execution_end = datetime.datetime.now()
return ret
# Or
# return execute_command_container(self._state, c)
else:
return None
def register_andor_execute_command_generator(self,
generator_type,
file_type: str,
file,
register=True,
execute=False):
"""
Creates a generator parser, then it feeds the file type and the file
The generator parser has to parse the file and to generate command_executors as a Python generator
:param generator_type:
:param file_type:
:param file:
:param register: If True, register the command in the ReproducibleSession
:param execute: If True, execute the command in the ReproducibleSession
:return:
"""
return self.register_andor_execute_command_generator1(
CommandsContainer.create(generator_type, file_type, file),
register, execute)
# --------------------------------------------------------------------------------------------
def get_case_studies(self):
""" Get a list of case studies READABLE by current identity (or public if anonymous) """
pass
def get_case_study_versions(self, case_study: str):
# TODO Check that the current user has READ access to the case study
pass
def get_case_study_version(self, case_study_version: str):
# TODO Check that the current user has READ access to the case study
pass
def get_case_study_version_variables(self, case_study_version: str):
""" A tree of variables, by type: processors, flows """
pass
def get_case_study_version_variable(self, case_study_version: str,
variable: str):
pass
def remove_case_study_version(self, case_study_version: str):
pass
def share_case_study(self, case_study: str, identities: List[str],
permission: str):
pass
def remove_case_study_share(self, case_study: str, identities: List[str],
permission: str):
pass
def get_case_study_permissions(self, case_study: str):
pass
def export_case_study_version(self, case_study_version: str):
pass
def import_case_study(self, file):
pass
res = string_to_ast(h_name, s)
s = State()
examples = [
"5-2017", "2017-5", "2017/5", "2017-05 - 2018-01", "2017",
"5-2017 - 2018-1", "2017-2018", "Year", "Month"
]
for example in examples:
print(example)
res = string_to_ast(time_expression, example)
print(res)
print(f'Is year = {is_year(example)}')
print(f'Is month = {is_month(example)}')
print("-------------------")
s.set("HH", DottedDict({"Power": {"p": 34.5, "Price": 2.3}}))
s.set("EN", DottedDict({"Power": {"Price": 1.5}}))
s.set("HH", DottedDict({"Power": 25}), "ns2")
s.set("param1", 0.93)
s.set("param2", 0.9)
s.set("param3", 0.96)
examples = [
"EN(p1=1.5, p2=2.3)[d1='C11', d2='C21'].v2", # Simply sliced Variable Dataset (function call)
"a_function(p1=2, p2='cc', p3=1.3*param3)",
"-5+4*2", # Simple expression #1
"HH", # Simple name
"HH.Power.p", # Hierarchic name
"5", # Integer
"1.5", # Float
"1e-10", # Float scientific notation
"(5+4)*2", # Simple expression #2 (parenthesis)