def __init__(self):
try:
func_trace = FunctionalTrace(parent_trace=None, path_mask=None)
root_trace = func_trace.doing(
"Initializing KB_Session for Apodeixi CLI",
origination={'signaled_from': __file__})
# The initializer will set self.a6i_config. But in a sort of chicken-and egg situation, we find
# ourselves forced to load a "temporary" config object to figure out the class name of the initializer ot use.
# This "temporary" config might not be the "right class" if the initializer
# is not the default Apodeixi class (for example, if an Apodeixi extension is using a derived initializer
# class), but at least the "temporary" config will let us get the initializer class.
#
temporary_config = ApodeixiConfig(root_trace)
initializer_class_name = temporary_config.get_CLI_InitializerClassname(
root_trace)
try:
module_path, class_name = initializer_class_name.rsplit('.', 1)
module = import_module(module_path)
initializer_class = getattr(module, class_name)
initializer = initializer_class()
except (ImportError, AttributeError) as ex:
raise ApodeixiError(root_trace,
"Unable to construct class '" +
str(initializer_class_name) + "'",
data={"error": ex.msg})
initializer.initialize(root_trace, self)
# This will look like '210703.102746', meaning the 3rd of July of 2021 at 10:27 am (and 46 sec).
# Intention is this timestamp as an "identifier" of this KnowledgeBase session, by using as prefix
# to folders or files (e.g., sandboxes, logs) created during the existence of this KnowledgeBase session
dt = _datetime.datetime.today()
self.timestamp = dt.strftime("%y%m%d.%H%M%S")
self.error_count = 1 # Increments each time we log an error
except ApodeixiError as ex:
error_msg = CLI_ErrorReporting(None).report_a6i_error(
parent_trace=root_trace, a6i_error=ex)
# GOTCHA
# Use print, not click.echo or click exception because they don't correctly display styling
# (colors, underlines, etc.). So use vanilla Python print and then exit
print(error_msg)
_sys.exit()
except Exception as ex:
print("Unrecoverable error: " + str(ex))
_sys.exit()
def activateTestConfig(self):
'''
Modifies environment variables to ensure that Apodeixi uses a configuration specific for tests.
This change endures until the dual method self.deactivateTestConfig() is called.
'''
# Remember it before we change it to use a test configuration, and then restore it in the tearDown method
self.original_config_directory = _os.environ.get(self.CONFIG_DIRECTORY())
# Here we want the location of this class, not its concrete derived class,
# since the location of the Apodexei config to be used for tests is globally unique
# So we use __file__
#
# MODIFICATION ON APRIL, 2022: As part of adding tests in CI/CD, we made it possible for a CI/CD pipeline
# to specify a location for the Apodeixi config directory that should be used for integration tests.
# As a result, we only hard-code such config directory if we are not running in an environment where that
# has been already set. This enables:
# 1) Preservation of traditional Apodeixi feature for developers, whereby tests will "auto discover"
# the test_db to use even if the environment variable for the Apodeixi CONFIG_DIRECTORY is not set
# 2) Enablement of new use case for CI/CD: externally injected configuration of which config directory to use
if not self.INJECTED_CONFIG_DIRECTORY in _os.environ.keys():
# Use case: developer runs tests locally
_os.environ[self.CONFIG_DIRECTORY()] = _os.path.join(_os.path.dirname(__file__), '../../../../apodeixi-testdb')
else:
# Use case: CI/CD pipeline runs tests in a Docker container
_os.environ[self.CONFIG_DIRECTORY()] = _os.environ[self.INJECTED_CONFIG_DIRECTORY]
func_trace = FunctionalTrace( parent_trace = None,
path_mask = None) # path_mask has not been set yet as an attribute
root_trace = func_trace.doing("Loading Apodeixi configuration",
origination = {'signaled_from': __file__})
self.a6i_config = ApodeixiConfig(root_trace)
def test_a6i_config(self):
try:
root_trace = FunctionalTrace(
parent_trace=None, path_mask=self._path_mask).doing(
"Testing loading for Apodeixi Config")
config = ApodeixiConfig(root_trace)
# To ensure determistic output, mask parent part of any path that is mentioned in the configuration before
# displaying it in regression putput
#
clean_dict = DictionaryUtils().apply_lambda(
parent_trace=root_trace,
root_dict=config.config_dict,
root_dict_name="Apodeixi config",
lambda_function=self._path_mask)
config_txt = DictionaryFormatter().dict_2_nice(
parent_trace=root_trace, a_dict=clean_dict, flatten=True)
self._compare_to_expected_txt(parent_trace=root_trace,
output_txt=config_txt,
test_output_name='test_a6i_config',
save_output_txt=True)
except ApodeixiError as ex:
print(ex.trace_message())
self.assertTrue(1 == 2)
def initialize(self, parent_trace, kb_session):
'''
Sets these attributes of kb_session:
* kb_session.a6i_config
* kb_session.kb_rootdir
* kb_session.clientURL
* kb_session.store
* kb_session.kb
This method is intended to be called from within the KB_Session constructor.
@param kb_session A KB_Session instance that needs to be initialized
'''
my_trace = parent_trace.doing("Loading Apodeixi configuration",
origination={'signaled_from': __file__})
kb_session.a6i_config = ApodeixiConfig(my_trace)
my_trace = parent_trace.doing("Initializing file-based stack",
origination={'signaled_from': __file__})
kb_session.kb_rootdir = kb_session.a6i_config.get_KB_RootFolder(
my_trace)
kb_session.clientURL = kb_session.a6i_config.get_ExternalCollaborationFolder(
my_trace)
store_impl = Shutil_KBStore_Impl(parent_trace=my_trace,
kb_rootdir=kb_session.kb_rootdir,
clientURL=kb_session.clientURL)
kb_session.store = KnowledgeBaseStore(my_trace, store_impl)
my_trace = parent_trace.doing("Starting KnowledgeBase")
kb_session.kb = KnowledgeBase(my_trace,
kb_session.store,
a6i_config=kb_session.a6i_config)
def _impl_timebucket_standardization_test(self,
TEST_NAME,
HEADER,
INDEX_COL,
lower_level_key=None):
try:
root_trace = FunctionalTrace(
parent_trace=None, path_mask=self._path_mask).doing(
"Testing DataFrame timebucket tidying up")
a6i_config = ApodeixiConfig(root_trace)
INPUT_FULL_PATH = self.input_data + "/" + TEST_NAME + ".xlsx"
input_df = _pd.read_excel(io=INPUT_FULL_PATH,
header=HEADER,
index_col=INDEX_COL)
output_df, info = TimebucketStandardizer(
).standardizeAllTimebucketColumns(root_trace, a6i_config, input_df,
lower_level_key)
self._compare_to_expected_df(parent_trace=root_trace,
output_df=output_df,
test_output_name=TEST_NAME)
except ApodeixiError as ex:
print(ex.trace_message())
raise ex
def test_standardizeOneTimebucketColumn(self):
root_trace = FunctionalTrace(parent_trace=None, path_mask=self._path_mask)\
.doing("Testing parser for quarter time buckets")
a6i_config = ApodeixiConfig(root_trace)
tests = [
"Q3 FY23", ("Q3 FY 23"), (" Q3 ", "FY 23 "), ("Metrics", "FY23"),
("Perro", "casa"), "nata", ("FY 25", "Actuals"),
("Metrics", "Q2", "FY 2026", "Target"), ("Q 4 ", " FY 29"), 2023,
"Q3 FY23.1", (" Q3 ", "FY 23.2 "),
("Metrics", "Q2", "FY 2026.3", "Target.4"), (),
("Q1 FY24", "Actuals")
]
output = ""
for idx in range(len(tests)):
raw_col = tests[idx]
timebucket = None
try:
col_result, timebucket, timebucket_indices = \
TimebucketStandardizer().standardizeOneTimebucketColumn(root_trace, raw_col, a6i_config)
except ApodeixiError as ex:
col_result = str(ex)
timebucket = None
timebucket_indices = []
output += "\n\n'" + str(tests[idx]) + "' was parsed as:..........................." \
+ str(col_result) + " " + str(timebucket_indices)
if timebucket != None:
output += " (a FY_Quarter)"
if type(col_result) == tuple:
output += " (a tuple)"
self._compare_to_expected_txt(
parent_trace=root_trace,
output_txt=output,
test_output_name='test_standardizeOneTimebucketColumn',
save_output_txt=True)
def overwrite_test_context(self, parent_trace):
'''
This is a "trick" method needed so that CLI invocations run in the environment isolated for this test case (or
its children), as opposed to on the base environment.
It accomplishes this by "fooling" the CLI into thinking that "base environment" is actually the environment
isolated for this test case.
It does so by overwriting the value of the self.CONFIG_DIRECTORY() environment variable
but what is tricky is:
* By the time this method is called, this class no longer needs the self.CONFIG_DIRECTORY() environment
variable, since it was used in super().setUp() to initialize self.a6i_config and other properties, and
that is as it should be.
* Therefore, the modification in this method to self.CONFIG_DIRECTORY() is not going to impact this
test object. Instead, it will impact other objects that use it. There is no such object in Apodeixi itself,
but there is one in the CLI: the KB_Session class.
* The intent is then for the KB_Session class to initialize it's notion of self.a6i_config differently, so
that it is "fooled" into thinking that the "base environment" is this test cases's isolated environment.
* Each time the CLI is invoked, it constructs a KB_Session to initialiaze the KnowledgeBaseStore. Thus
the CLI will be using a store pointing to this test case's isolated environment. This is different than
for non-CLI tests, for whom the store points to the test knowledge base common to the Apodeixi test suite.
'''
# Before changing context, create the environment for this test, which will later become the
# "fake base environment" when we switch context. But this uses the "original" store, so must be done
# before we switch context, so we must select the stack here (and later we re-select it when
# switching context)
self.selectStack(parent_trace)
self.provisionIsolatedEnvironment(parent_trace)
# Remember original config before it is overwritten when we change context
original_a6i_config = self.a6i_config
# In case it is ever needed, remember this tests suite's value for the environment variable
self.config_directory_for_this_test_object = _os.environ.get(
self.CONFIG_DIRECTORY())
# OK, we start the context switch here.
# For this test case, we want the CLI to use a config file that is in the input folder
_os.environ[
self.CONFIG_DIRECTORY()] = self.input_data + "/" + self.scenario()
# Each CLI test has a dedicated folder containing the test environment for it, i.e., an entire
# test database (knowledge base folder, collaboration area folder) just for that test.
# These folders are referenced in a test-specific apodeixi_config.toml.
# In order to make the folders thus referenced not depend on the installation folder for the Apodeixi
# test database, we introduce the environment variable ${TEST_DB_DIR} that should be used in all these
# CLI-test-specific apodeixi_config.toml.
#
# Example: for CLI test for subproducts with id #1011, in the folder ...../test_db/input_data/1011/cli.subproducts,
# the apodeixi_config.toml should have a line like
# knowledge-base-root-folder = "${TEST_DB_DIR}/knowledge-base/envs/1011_ENV/kb"
# instead of
# knowledge-base-root-folder = "C:/Users/aleja/Documents/Code/chateauclaudia-labs/apodeixi/test_db/knowledge-base/envs/1011_ENV/kb"
#
# Such practice ensures that the test harness continues to work no matter where it is installed (for example,
# in a Docker container).
# To make this approach work, we hereby set that environment variable, whose value will be consulted by the
# ApodeixiConfig constructor when we do the context switch a few lines further below
_os.environ[self.TEST_DB_DIR] = self.test_db_dir
# Now overwrite parent's notion of self.a6i_config and of the self.test_config_dict
self.a6i_config = ApodeixiConfig(parent_trace)
self.selectStack(
parent_trace
) # Re-creates the store for this test with the "fake" base environment
# Set again the location of the test directory as per the original a6i config. We need it to mask non-deterministic
# paths
self.a6i_config.test_db_dir = original_a6i_config.test_db_dir
# Next time an environment is provisioned for this test, use this overwritten config for the name of the folder
self.test_config_dict = YAML_Utils().load(
parent_trace,
path=self.input_data + "/" + self.scenario() + '/test_config.yaml')
def _impl_ts_join_test(self, TEST_NAME, HEADER, LOWER_TAGS, UPPER_TAGS,
func, operation_type, ref_column):
'''
'''
try:
root_trace = FunctionalTrace(
parent_trace=None, path_mask=self._path_mask).doing(
"Testing TimebucketDataFrameJoiner")
a6i_config = ApodeixiConfig(root_trace)
REF_FULL_PATH = self.input_data + "/" + TEST_NAME + "_ref.xlsx"
reference_df = _pd.read_excel(io=REF_FULL_PATH)
DF_B_FULL_PATH = self.input_data + "/" + TEST_NAME + "_b.xlsx"
b_df = _pd.read_excel(io=DF_B_FULL_PATH, header=HEADER)
if operation_type != TimebucketDataFrameJoiner.CUMULATIVE_OPERATION:
DF_A_FULL_PATH = self.input_data + "/" + TEST_NAME + "_a.xlsx"
a_df = _pd.read_excel(io=DF_A_FULL_PATH, header=HEADER)
timebucket_df_list = [a_df, b_df]
else:
timebucket_df_list = [b_df]
joiner = TimebucketDataFrameJoiner(
root_trace,
reference_df=reference_df,
link_field='Country',
timebucket_df_list=timebucket_df_list,
timebucket_df_lower_tags=LOWER_TAGS,
timebucket_df_upper_tags=UPPER_TAGS,
a6i_config=a6i_config)
if func != None:
if operation_type == TimebucketDataFrameJoiner.BINARY_OPERATION:
joiner.enrich_with_tb_binary_operation(
root_trace,
a_ltag=LOWER_TAGS[0],
b_ltag=LOWER_TAGS[1],
c_ltag="% Target",
func=func)
elif operation_type == TimebucketDataFrameJoiner.UNARY_OPERATION:
joiner.enrich_with_tb_unary_operation(
root_trace,
ref_column=ref_column,
b_ltag=LOWER_TAGS[0],
c_ltag="% Target",
func=func)
elif operation_type == TimebucketDataFrameJoiner.CUMULATIVE_OPERATION:
joiner.enrich_with_tb_cumulative_operation(
root_trace,
b_ltag=LOWER_TAGS[0],
c_ltag="Cum Target",
func=func)
output_df = joiner.join_dataframes(root_trace)
self._compare_to_expected_df(parent_trace=root_trace,
output_df=output_df,
test_output_name=TEST_NAME)
except ApodeixiError as ex:
print(ex.trace_message())
raise ex