def retrieveManifest(self, parent_trace, manifest_handle):
'''
Returns a dict and a string.
The dict represents the unique manifest in the store that is identified by the `manifest handle`.
The string represents the full pathname for the manifest.
If none exists, it returns (None, None). That said, before giving up and returning (None, None),
this method will attempt to find the manifest in the parent environment if that is what is stipulated
in the current environment's configuration
@param manifest_handle A ManifestHandle instance that uniquely identifies the manifest we seek to retrieve.
'''
manifest, manifest_path = super().retrieveManifest(parent_trace, manifest_handle)
if manifest == None:
# Not found, so normally we should return None. But before giving up, look in parent environment
# if we have been configured to fail over the parent environment whenver we can't find something
if self._failover_manifest_reads_to_parent(parent_trace):
# Search in parent first, and copy anything found to the current environment
my_trace = parent_trace.doing("Searching in parent environment")
# Temporarily switch to the parent environment, and try again
original_env = self.current_environment(my_trace)
self.activate(my_trace, self.parent_environment(my_trace).name(my_trace))
manifest, manifest_path = self.retrieveManifest(my_trace, manifest_handle)
# Now that search in parent environment is done, reset back to original environment
self.activate(my_trace, original_env.name(my_trace))
# Populate current environment with anything found in the parent environment, but only if it is not
# already in current environment
if manifest != None:
my_trace = parent_trace.doing("Copying manifest from parent environment",
data = {"parent environment name":
self.parent_environment(my_trace).name(my_trace),
"current environment name":
self.current_environment(my_trace).name(my_trace)})
from_path = manifest_path
to_dir = self.current_environment(my_trace).postingsURL(parent_trace)
if not _os.path.exists(to_dir):
my_trace = parent_trace.doing("Copying a manifest file",
data = {"src_path": from_path,
"to_dir": to_dir})
PathUtils().create_path_if_needed(parent_trace=my_trace, path=to_dir)
PathUtils().copy_file(parent_trace, from_path, to_dir)
return manifest, manifest_path
def buildPostingHandle(self, parent_trace, excel_posting_path, sheet,
excel_range):
'''
Returns an PostingLabelHandle for the posting label embedded within the Excel spreadsheet that resides in
the path provided.
'''
kb_postings_url = self.getPostingsURL(parent_trace)
if PathUtils().is_parent(parent_trace=parent_trace,
parent_dir=kb_postings_url,
path=excel_posting_path):
# See Note below in the else clause. This case is rare, even if at first glance it would seem like the
# "normal" case.
relative_path, filename = PathUtils().relativize(
parent_trace=parent_trace,
root_dir=kb_postings_url,
full_path=excel_posting_path)
posting_api = self._filename_2_api(parent_trace, filename)
my_trace = parent_trace.doing(
"Building the filing coordinates",
data={"relative_path": str(relative_path)})
filing_coords = self._buildFilingCoords(
parent_trace=my_trace,
posting_api=posting_api,
relative_path=relative_path)
else: # Posting wasn't submitted from the "right" folder, so coordinates will have be inferred later when label
# is read.
# INTERESTING NOTE: This happens rather often if we are
# in a transaction (normal case) because kb_postings_url is the current environment's URL, which
# almost certainly is not a parent of excel_posting_path (the kb_postings_url would have tokens like
# 'store-transation.4' that couldn't possibly be part of the excel_posting_path).
filename = PathUtils().tokenizePath(parent_trace,
excel_posting_path)[-1]
posting_api = self._filename_2_api(parent_trace, filename)
env_config = self.current_environment(parent_trace).config(
parent_trace)
path_mask = env_config.path_mask
filing_coords = TBD_FilingCoordinates(fullpath=excel_posting_path,
posting_api=posting_api,
path_mask=path_mask)
# Now build the posting label handle
posting_handle = PostingLabelHandle(parent_trace=parent_trace,
posting_api=posting_api,
filing_coords=filing_coords,
excel_filename=filename,
excel_sheet=sheet,
excel_range=excel_range)
return posting_handle
def copy_posting_across_environments(self, parent_trace, handle, from_environment, to_environment):
'''
Copies the posting file denoted by the `handle` in the `from_environment` to the `to_environment`
'''
from_path = from_environment.postingsURL(parent_trace) + "/" + handle.getRelativePath(parent_trace)
to_path = to_environment.postingsURL(parent_trace) + "/" + handle.getRelativePath(parent_trace)
to_dir = _os.path.dirname(to_path)
my_trace = parent_trace.doing("Copying a posting file",
data = {"src_path": from_path,
"to_dir": to_dir})
if not _os.path.exists(to_dir):
PathUtils().create_path_if_needed(parent_trace=my_trace, path=to_dir)
PathUtils().copy_file(parent_trace, from_path, to_dir)
def loadForeignKeyConstraints(self, parent_trace):
'''
Loads this store's ForeignKeyConstraintsRegistry from the system area of the store
Returns two things:
* A ForeignKeyConstraintsRegistry object. If null, this signifies that there was none found in storage
* A string, for the path in the file system where the ForeignKeyConstraintsRegistry was retrieved from
'''
foreign_key_constraints, path = super().loadForeignKeyConstraints(parent_trace)
if foreign_key_constraints == None:
# Not found, so normally we should return None. But before giving up, look in parent environment
# if we have been configured to fail over the parent environment whenver we can't find something
if self._failover_manifest_reads_to_parent(parent_trace):
# Search in parent first, and copy anything found to the current environment
my_trace = parent_trace.doing("Searching in parent environment")
# Temporarily switch to the parent environment, and try again
original_env = self.current_environment(my_trace)
self.activate(my_trace, self.parent_environment(my_trace).name(my_trace))
foreign_key_constraints, path = self.loadForeignKeyConstraints(my_trace)
# Now that search in parent environment is done, reset back to original environment
self.activate(my_trace, original_env.name(my_trace))
# Populate current environment with anything found in the parent environment, but only if it is not
# already in current environment
if foreign_key_constraints != None:
my_trace = parent_trace.doing("Copying foreign key constraints from parent environment",
data = {"parent environment name":
self.parent_environment(my_trace).name(my_trace),
"current environment name":
self.current_environment(my_trace).name(my_trace)})
from_path = path
to_dir = self.current_environment(my_trace).postingsURL(parent_trace)
if not _os.path.exists(to_dir):
my_trace = parent_trace.doing("Copying a manifest file",
data = {"src_path": from_path,
"to_dir": to_dir})
PathUtils().create_path_if_needed(parent_trace=my_trace, path=to_dir)
PathUtils().copy_file(parent_trace, from_path, to_dir)
return foreign_key_constraints, path
def handle_warnings(self, parent_trace, warning_list):
'''
Helper method to catch warnings and turns them into ApodeixiErrors.
'''
if len(warning_list) > 0:
warning_dict = {}
for idx in range(len(warning_list)):
a_warning = warning_list[idx]
self.check_if_user_error(parent_trace, a_warning)
if self.check_if_should_ignore(parent_trace, a_warning):
# In this case, this particular warning should not trigger an ApodeixiError.
# So ignore it and move on to examining the next warning
continue
warning_dict['Warning ' + str(idx)] = str(a_warning.message)
warning_dict['... from'] = PathUtils().to_linux(
str(a_warning.filename))
warning_dict['... at line'] = str(a_warning.lineno)
trace_msg = "\n" + "-" * 60 + "\tWarnings Stack Trace\n\n"
trace_msg += str(a_warning.stacktrace)
trace_msg += "\n" + "-" * 60
warning_dict['Stack Trace'] = trace_msg
if len(warning_dict.keys()) > 0:
raise ApodeixiError(parent_trace,
"A dependency issued " +
str(len(warning_list)) + " warning(s)",
data={} | warning_dict)
def _buildFilingCoords(self, parent_trace, posting_api, relative_path):
'''
Helper method that concrete derived classes may choose to use as part of implementing `buildPostingHandle`,
to determine the FilingCoordinates to put into the posting handle.
'''
path_tokens = PathUtils().tokenizePath(parent_trace=parent_trace,
path=relative_path,
absolute=False)
my_trace = parent_trace.doing(
"Looking up filing class for given posting API",
data={'posting_api': posting_api})
filing_class = self.getFilingClass(parent_trace, posting_api)
if filing_class == None:
raise ApodeixiError(
my_trace,
"Can't build filing coordinates from a null filing class")
my_trace = parent_trace.doing(
"Validating that posting is in the right folder structure " +
"within the Knowledge Base")
filing_coords = filing_class().build(parent_trace=my_trace,
path_tokens=path_tokens)
if filing_coords == None:
raise ApodeixiError(
my_trace,
"Posting is not in the right folder within the Knowledge Base for this kind of API",
data={
'posting relative path tokens': path_tokens,
'posting api': posting_api,
'relative path expected by api': relative_path
})
return filing_coords
def setUp(self):
super().setUp()
# We can't rely on Python's built-in '__file__' property to find the location of the concrete class
# that is running, since we are in the parent class and we will get the parent class's filename, not the concrete class's.
# So instead we rely on the inspect package
me__file__ = inspect.getfile(self.__class__)
# self.input_data = _os.path.join(_os.path.dirname(__file__), 'input_data') # Doesn't work - use inpectt instead
self.input_data = _os.path.join(
_os.path.dirname(me__file__),
'input_data') # Works ! :-) Thanks inspect!
# self.output_data = _os.path.join(_os.path.dirname(__file__), 'output_data') # Doesn't work - use inpectt instead
self.output_data = _os.path.join(
_os.path.dirname(me__file__),
'output_data') # Works ! :-) Thanks inspect!
# self.output_data = _os.path.join(_os.path.dirname(__file__), 'output_data') # Doesn't work - use inpectt instead
self.expected_data = _os.path.join(
_os.path.dirname(me__file__),
'expected_data') # Works ! :-) Thanks inspect!
# Output data is not in source control, so if we are in a clean repo the folder might not exist, so created it
# if needed
root_trace = FunctionalTrace(None, path_mask=self._path_mask)
PathUtils().create_path_if_needed(root_trace, self.output_data)
# For unit tests, don't enforce referential integrity since we will test data in mock stores that may
# reference things that don't really exist
self.a6i_config.enforce_referential_integrity = False
def setUp(self):
super().setUp()
self.activateTestConfig()
# Used by derived classes to mask some paths that are logged out so that regression output is
# deterministic
root_trace = FunctionalTrace(parent_trace=None, path_mask=None).doing("Configuring test path mask",
origination = {'signaled_from': __file__})
self._path_mask = PathUtils().get_mask_lambda(parent_trace=root_trace, a6i_config=self.a6i_config)
def _report_error(self, parent_trace, error, report_header):
'''
Helper method to enrich the `report_header` that describes an error with tracing information
and links for the error
@param error An instance of ApodeixiError or Exception. The kind of tracing information differs based on the class.
@param report_header A string, with a high level description of the error.
@returns A string that extends `report_header` with additional tracing information
'''
high_level_msg = report_header
if self.kb_session != None:
log_folder = self.kb_session.kb_rootdir + "/" + File_KBEnv_Impl.LOGS_FOLDER
PathUtils().create_path_if_needed(parent_trace=parent_trace,
path=log_folder)
log_filename = self.kb_session.timestamp + "_errors.txt"
dt = _datetime.datetime.today()
tokens = _sys.argv
executable_path = tokens[0]
executable_name = _os.path.split(executable_path)[1]
command_typed_by_user = executable_name + " " + " ".join(
tokens[1:])
detailed_msg = dt.strftime(
"[%H:%M:%S %a %d %b %Y] => ") + command_typed_by_user
if issubclass(type(error), ApodeixiError):
trace_msg = self._a6i_error_trace(error)
elif issubclass(type(error), Exception):
trace_msg = self._generic_error_trace(error)
else:
raise ApodeixiError(
parent_trace,
"Can't report error because it is not an Exception",
data={
"type(error)": str(type(error)),
"error": str(error)
})
detailed_msg += trace_msg
with open(log_folder + "/" + log_filename, "a") as f:
f.write(detailed_msg)
high_level_msg += self.POINTERS("\n\nCheck error log at ")
high_level_msg += self.POINTERS(
self.UNDERLINE("file:///" + log_folder + "/" + log_filename))
return high_level_msg
def combined_mask(self, parent_trace, a6i_config):
MASK_SANDBOX = self.mask_sandbox_lambda(parent_trace)
MASK_PATH = PathUtils().get_mask_lambda(parent_trace, a6i_config)
MASK_TIMER = ApodeixiTimer().get_mask_lambda(parent_trace)
def MASK_COMBINED(txt1):
if txt1 == None:
return None
txt2 = MASK_PATH(txt1)
txt3 = MASK_SANDBOX(txt2)
txt4 = MASK_TIMER(txt3)
txt5 = _re.sub(pattern="[0-9]{6}", repl="<MASKED>", string=txt4)
return txt5
return MASK_COMBINED
def buildPostingHandle(self, parent_trace, excel_posting_path, sheet,
excel_range):
'''
Returns an Apodeixi Excel URL for the posting label embedded within the Excel spreadsheet that resides in the path provided.
'''
if PathUtils().is_leaf(parent_trace, excel_posting_path):
full_path = self.input_postings_dir + "/" + excel_posting_path
else:
full_path = excel_posting_path
return super().buildPostingHandle(parent_trace=parent_trace,
excel_posting_path=full_path,
sheet=sheet,
excel_range=excel_range)
def snapshot_generated_form(self, parent_trace, form_request_response):
'''
Helper method to remember a generated form at a given point in time, in case it is subsequently
modified and submitted as an update posting.
Basically, this will copy the generated form to self._regression_output_dir(parent_trace)
@param form_request_response A FormRequestResponse object with the information needed to locate
the generated form in question.
'''
form_path = form_request_response.clientURL(parent_trace) + "/" \
+ form_request_response.getRelativePath(parent_trace)
filename = _os.path.split(form_path)[1]
snapshot_name = self.next_form(filename)
dst_dir = self._regression_output_dir(parent_trace)
PathUtils().copy_file(parent_trace, form_path,
dst_dir + "/" + snapshot_name)
def _prepare_yaml_comparison(self, parent_trace, output_dict, test_output_name,
output_data_dir, expected_data_dir, save_output_dict=False):
'''
Helper method that does most of the heavy lifting when we seek to compare yaml-based
expected output.
On success, this returns a dictionary corresponding to the expected output, that the caller can then compare
with the parameter `output_dict`
The motivation for the existence of this method is that there are different ways of comparing yaml
files. For example:
* A "pure" comparison, done by method `_compare_to_expected_yaml`
* A "tolerance-based" comparison, done by method `_compare_yaml_within_tolerance`. This is used, for
example, to validate expected output where the contents of file systems is displayed. In such cases,
generated Excel files may display a size that differs by 1 or 2 bytes because of the non-determinism involved
in creating Excel files since "xlsx" files are really zip files with XML contents, and it is well
known that zip files are non-deterministcically created (for example, see
https://medium.com/@pat_wilson/building-deterministic-zip-files-with-built-in-commands-741275116a19).
Sometimes simply running Apodeixi in a different deployment or machine will cause generated Excel
files to change in size by 19 bytes or more.
So the tolerance level can be configured in the test_config.yaml, a file that is located under the
root folder for the testing database that you have configured to use in ApodeixiConfig (i.e.,
the folder above the knowledge base's root folder configured in ApodeixiConfig.)
'''
# Check not null, or else rest of actions will "gracefully do nothing" and give the false impression that test passes
# (at least it would erroneously pass when the expected output is set to an empty file)
self.assertIsNotNone(output_dict)
PathUtils().create_path_if_needed(parent_trace = parent_trace, path = output_data_dir)
# Persist output (based on save_output_dict flag)
if save_output_dict:
YAML_Utils().save( parent_trace,
data_dict = output_dict,
path = output_data_dir + '/' + test_output_name + '_OUTPUT.yaml')
# Retrieve expected output
expected_dict = YAML_Utils().load(parent_trace,
path = expected_data_dir + '/' + test_output_name + '_EXPECTED.yaml')
return expected_dict
def save_environment_metadata(self, parent_trace):
'''
Creates and saves a YAML file called "METATATA.yaml" in the root folder for self.
It is sufficient information from which to re-create the environment (for example, if it was
created in a different Python process, so this Python process wouldn't have an in-memory object
for it unless it loads it, leveraing the "METADATA.yaml" file).
This can happen when the CLI creates a sandbox that will later be used by subsequent commands.
Since each CLI invocation is its own Python process, different invocations can only share the
same sandbox environment if there is a way to persist and then load the state of an environment.
'''
ME = File_KBEnv_Impl
METADATA_FILENAME = "METADATA.yaml"
metadata_dict = {}
metadata_dict['name'] = self.name(parent_trace)
metadata_dict['parent'] = self.parent(parent_trace).name(parent_trace)
metadata_dict['postingsURL'] = self.postingsURL(parent_trace)
metadata_dict['manifestsURL'] = self.manifestsURL(parent_trace)
metadata_dict['clientURL'] = self.clientURL(parent_trace)
config = self.config(parent_trace)
config_dict = {}
config_dict['read_misses_policy'] = config.read_misses_policy
config_dict['use_timestamps'] = config.use_timestamps
metadata_dict['config'] = config_dict
if self == self._store.base_environment(parent_trace):
environment_dir = _os.path.dirname(self._store.base_environment(parent_trace). \
manifestsURL(parent_trace))
else:
root_dir = _os.path.dirname(self._store.base_environment(parent_trace). \
manifestsURL(parent_trace))
envs_dir = root_dir + "/" + ME.ENVS_FOLDER
environment_dir = envs_dir + "/" + self.name(parent_trace)
PathUtils().create_path_if_needed(parent_trace, environment_dir)
YAML_Utils().save(parent_trace,
data_dict=metadata_dict,
path=environment_dir + "/" + METADATA_FILENAME)
def _compare_to_expected_txt(self, parent_trace, output_txt, test_output_name,
output_data_dir, expected_data_dir, save_output_txt=False):
'''
Utility method for derived classes that create text files and need to check they match an expected output
previously saves as a text file as well.
It also saves the output as a yaml file, which can be copied to be the expected output when test case is created.
@param output_data_dir Directory to which to save any output.
@param expected_data_dir Directory from which to retrieve any previously saved expected output.
'''
# Check not null, or else rest of actions will "gracefully do nothing" and give the false impression that test passes
# (at least it would erroneously pass when the expected output is set to an empty file)
self.assertIsNotNone(output_txt)
PathUtils().create_path_if_needed(parent_trace = parent_trace, path = output_data_dir)
# Persist output (based on save_output_dict flag)
if save_output_txt:
# As documented in https://nbconvert.readthedocs.io/en/latest/execute_api.html
#
# May get an error like this unless we explicity use UTF8 encoding:
#
# File "C:\Alex\CodeImages\technos\anaconda3\envs\ea-journeys-env\lib\encodings\cp1252.py", line 19, in encode
# return codecs.charmap_encode(input,self.errors,encoding_table)[0]
# UnicodeEncodeError: 'charmap' codec can't encode character '\u2610' in position 61874: character maps to <undefined>
#
# Happens in particular when trying to save a string representing a Jupyter notebook's execution, since for the same
# reason above that string had to be written to a string using UTF8 encoding, so now if we save to a file we must use UTF8
with open(output_data_dir + '/' + test_output_name + '_OUTPUT.txt', 'w', encoding="utf8") as file:
file.write(str(output_txt))
# Retrieve expected output
with open(expected_data_dir + '/' + test_output_name + '_EXPECTED.txt', 'r', encoding="utf8") as file:
expected_txt = str(file.read())
self.assertEqual(str(output_txt), expected_txt)
def modify_form(self, parent_trace, form_request_response):
'''
Helper method to simulate an end-user's edits of a generated form, where the end-user's intention would
typically be to submit an update posting using the form.
This "simulation" is done by copying a manually created "edited form" from the self.inputs area
remember a generated form at a given point in time, in case it is subsequently
modified and submitted as an update posting.
Basically, this will copy the generated form to self.results_data/self.scenario()
@param form_request_response A FormRequestResponse object with the information needed to locate
the generated form in question.
'''
form_path = form_request_response.clientURL(
parent_trace) + "/" + form_request_response.getRelativePath(
parent_trace)
form_filename = _os.path.split(form_path)[1]
simulation_filename = self.getInputDataFolder(
parent_trace) + "/" + self.scenario() + "/" + self.currentTestName(
) + "." + form_filename
PathUtils().copy_file(parent_trace, simulation_filename, form_path)
def addSubEnvironment(self,
parent_trace,
parent_env,
name,
env_config,
isolate_collab_area=False):
'''
Creates and returns a new File_KBEnv_Impl with name `name` and using self as the parent environment.
@param parent_env A KB_Environment object, that contains this File_KBEnv_Impl as its implementation
(i.e., parent_env._impl == self)
@param name A string, used as the unique name of this environment among all environments in the store
@param env_config A KB_Environment_Config object that will be set as the configuration of the newly
created sub-environment
@param isolate_collab_area A boolean to determine how the `_clientURL`
attribute should be set for the sub-environment being created. This is an attribute that
points to folders external to the KnowledgeBase (such as SharePoint) from where users post
spreadsheets and into which users request generated forms and reports to be written to.
By default, the boolean is False, in which case the sub-environment shares the same
external collaboration folder as the parent.
If the caller sets it to True, then the sub-environment will have its own dedicated
local folder for the `_clientURL`.
This setting should be set to False in normal production usage. It is mainly in test situations
where the parent environment is a deterministic set of files that should not be mutated by
test cases. In such cases, test cases will need to create an environment to serve as
the test case's "root", and will need to have a notion of `_clientURL`
in that root. In those cases this flag should be set to True to ensure the required isolation.
'''
ME = File_KBEnv_Impl
root_dir = _os.path.dirname(
self._store.base_environment(parent_trace).manifestsURL(
parent_trace))
envs_dir = root_dir + "/" + ME.ENVS_FOLDER
PathUtils().create_path_if_needed(parent_trace, envs_dir)
self._store._validate_environment_name(parent_trace=parent_trace,
name=name)
sub_env_name = name.strip()
my_trace = parent_trace.doing(
"Checking sub environment's name is available")
if sub_env_name in list(_os.listdir(envs_dir)):
raise ApodeixiError(
my_trace,
"Can't create a environment with a name that is already used for another environment",
data={'sub_env_name': str(sub_env_name)})
if sub_env_name in self._children.keys():
raise ApodeixiError(
my_trace,
"Can't create a sub environment with a name that is already used for another environment",
data={'sub_env_name': str(sub_env_name)})
my_trace = parent_trace.doing("Creating sub environment's folders",
data={'sub_env_name': sub_env_name})
subenv_postings_rootdir = envs_dir + "/" + sub_env_name + "/" + ME.POSTINGS_ENV_DIR
subenv_manifests_rootdir = envs_dir + "/" + sub_env_name + "/" + ME.MANIFESTS_ENV_DIR
if isolate_collab_area:
subenv_collab_folder = envs_dir + "/" + sub_env_name + "/" + ME.COLLABORATION_DIR
else:
subenv_collab_folder = self._clientURL
PathUtils().create_path_if_needed(my_trace, subenv_postings_rootdir)
PathUtils().create_path_if_needed(my_trace, subenv_manifests_rootdir)
PathUtils().create_path_if_needed(my_trace, subenv_collab_folder)
my_trace = parent_trace.doing("Creating sub environment",
data={'sub_env_name': sub_env_name})
sub_env_impl = File_KBEnv_Impl(
parent_trace=my_trace,
name=sub_env_name,
store=self._store,
parent_environment=parent_env,
config=env_config,
postings_rootdir=subenv_postings_rootdir,
manifests_roodir=subenv_manifests_rootdir,
clientURL=subenv_collab_folder)
sub_env = KB_Environment(parent_trace=my_trace, impl=sub_env_impl)
self._children[sub_env_name] = sub_env
sub_env_impl.save_environment_metadata(my_trace)
return sub_env
def test_path_utils(self):
root_trace = FunctionalTrace(parent_trace=None, path_mask=self._path_mask).doing("Testing Path Utils")
try:
INPUT_FOLDER = self.input_data
OUTPUT_FOLDER = self.output_data
TEST_SCENARIO = 'test_path_utils'
OUTPUT_FILE = TEST_SCENARIO + '_OUTPUT.txt'
EXPECTED_FILE = TEST_SCENARIO + '_EXPECTED.txt'
list1 = [ 1000, 3000, 4000, 5000, 8000, 9000]
list2 = [ 1000, 2000, 3000, 4000, 5000, 6000, 7000, 9000]
root_dir, parent_dir = _os.path.split(self.input_data)
rel_real_file = parent_dir + '/' + TEST_SCENARIO + '_INPUT.txt'
real_file = root_dir + '/' + rel_real_file
rel_fake_file = parent_dir + '/' + TEST_SCENARIO + '_NOT_REAL.foo'
fake_file = root_dir + '/' + rel_fake_file
leaf = "my_file.txt"
non_leaf = "secrets/my_file.txt"
bad_path = "/august/marzo/time.txt"
# Test is_leaf
my_trace = root_trace.doing("Testing is_leaf")
output_txt = '============ Testing is_leaf ================\n'
val = PathUtils().is_leaf(my_trace, leaf)
output_txt += "\nis_leaf(" + leaf + ")\t\t= " + str(val)
val = PathUtils().is_leaf(my_trace, non_leaf)
output_txt += "\n\nis_leaf(" + non_leaf + ")\t\t= " + str(val)
# Test is_parent
my_trace = root_trace.doing("Testing is_parent")
output_txt += '\n\n============ Testing is_parent ================\n'
val = PathUtils().is_parent(my_trace, parent_dir = root_dir, path = real_file)
output_txt += "\nis_parent(" + "< ... >" \
+ ", < ... >/" + rel_real_file + ") = " + str(val)
val = PathUtils().is_parent(my_trace, parent_dir = root_dir, path = bad_path)
output_txt += "\n\nis_parent(" + "< ... >, " + bad_path + ") = " + str(val)
# Test relativize
my_trace = root_trace.doing("Testing relativize")
output_txt += '\n\n============ Testing is_parent ================\n'
val = PathUtils().relativize(my_trace, root_dir = root_dir, full_path = real_file)
output_txt += "\nrelativize(" + "< ... >" \
+ ", < ... >/" + rel_real_file + ") = \n\t\t" + str(val)
try:
val = PathUtils().relativize(my_trace, root_dir = root_dir, full_path = fake_file)
except ApodeixiError as ex:
val = str(ex)
output_txt += "\n\nrelativize(" + "< ... >" \
+ ", < ... >/" + rel_fake_file + ") = \n\t\t" + str(val)
# Test tokenize_path
my_trace = root_trace.doing("Testing tokenize_path")
output_txt += '\n\n============ Testing tokenize_path ================\n'
relative_path = "/visions\\ideas\\problems/corrections"
val = PathUtils().tokenizePath(my_trace, relative_path, absolute = False)
output_txt += "\ntokenizePath(" + relative_path + ") = \n\t\t" + str(val)
# To test tokenization of an absolute path, we must be sensitive to whether we are in Windows or Linux in order
# to produce deterministic regression output. This means:
# 1. The absolute path should start with "C:\" in windows and "/C/" in Linux.
# 2. Don't display the drive in the regression test output (i.e., don't print the first token)
if _os.name == "nt":
absolute_path = "C:\\visions\\ideas\\problems/corrections"
else:
absolute_path = "/C/visions\\ideas\\problems/corrections"
val = PathUtils().tokenizePath(my_trace, absolute_path)
output_txt += "\n\ntokenizePath(" + "C:\\visions\\ideas\\problems/corrections" + ")[1:] = \n\t\t" + str(val[1:])
self._compare_to_expected_txt( parent_trace = my_trace,
output_txt = output_txt,
test_output_name = TEST_SCENARIO,
save_output_txt = True)
except ApodeixiError as ex:
print(ex.trace_message())
self.assertTrue(1==2)
def _compare_to_expected_df(self, parent_trace, output_df, test_output_name,
output_data_dir, expected_data_dir, columns_to_ignore=[], id_column=None):
'''
Utility method for derived classes that creates DataFrames (saved as CSV files) and checks they match an expected output
previously saves as a CSV file as well.
It also saves the output as a CSV file, which can be copied to be the expected output when test case is created.
@param output_data_dir Directory to which to save any output.
@param expected_data_dir Directory from which to retrieve any previously saved expected output.
@param columns_to_ignore List of column names (possibly empty), for columns that should be excluded from the comparison
@param id_column A string representing the column that should be used to identify rows in comparison text produced.
If set to None, then the row index is used.
'''
OUTPUT_FOLDER = output_data_dir
EXPECTED_FOLDER = expected_data_dir
OUTPUT_FILE = test_output_name + '_OUTPUT.csv'
EXPECTED_FILE = test_output_name + '_EXPECTED.csv'
OUTPUT_COMPARISON_FILE = test_output_name + '_comparison_OUTPUT.txt'
EXPECTED_COMPARISON_FILE = test_output_name + '_comparison_EXPECTED.txt'
# Check not null, or else rest of actions will "gracefully do nothing" and give the false impression that test passes
# (at least it would erroneously pass when the expected output is set to an empty file)
self.assertIsNotNone(output_df)
PathUtils().create_path_if_needed(parent_trace = parent_trace, path = output_data_dir)
OUTPUT_COLUMNS = [col for col in output_df.columns if not col in columns_to_ignore]
output_df[OUTPUT_COLUMNS].to_csv(OUTPUT_FOLDER + '/' + OUTPUT_FILE)
if type(output_df.columns) == _pd.MultiIndex: # Will need headers to load properly
nb_levels = len(output_df.columns.levels)
header = list(range(nb_levels))
else:
header = 0
# Load the output we just saved, which we'll use for regression comparison since in Pandas the act of loading will
# slightly change formats (e.g., strings for numbers become Numpy numbers)
# and we want to apply the same such changes as were applied to the expected output,
# to avoid frivolous differences that don't deserve to cause this test to fail
loaded_output_df = self.load_csv(parent_trace,
path = OUTPUT_FOLDER + '/' + OUTPUT_FILE,
header = header)
# Retrieve expected output
expected_df = self.load_csv(parent_trace,
path = EXPECTED_FOLDER + '/' + EXPECTED_FILE,
header = header)
EXPECTED_COLUMNS = [col for col in expected_df.columns if not col in columns_to_ignore]
# GOTCHA:
#
# OUTPUT_COLUMNS may differ from LOADED_OUTPUT_COLUMNS in the case of MultiLevel indices because
# of padding introduced in the load. For example, a column like ('Comment', '') in OUTPUT_COLUMNS
# will become ('Comment', 'Unnamed: 1_leveL_1'). So to compare, we use the LOADED columns
LOADED_OUTPUT_COLUMNS = [col for col in loaded_output_df.columns if not col in columns_to_ignore]
my_trace = parent_trace.doing("Invoking the DataFrameComparator")
comparator = DataFrameComparator( df1 = loaded_output_df[LOADED_OUTPUT_COLUMNS],
df1_name = "output",
df2 = expected_df[EXPECTED_COLUMNS],
df2_name = "expected",
id_column = id_column)
check, comparison_dict = comparator.compare(my_trace)
df_comparison_nice = DictionaryFormatter().dict_2_nice( parent_trace = parent_trace,
a_dict = comparison_dict,
flatten = True)
with open(OUTPUT_FOLDER + '/' + OUTPUT_COMPARISON_FILE, 'w', encoding="utf8") as file:
file .write(df_comparison_nice)
try:
with open(EXPECTED_FOLDER + '/' + EXPECTED_COMPARISON_FILE, 'r', encoding="utf8") as file:
expected_df_comparison = file.read()
except FileNotFoundError as ex:
raise ApodeixiError(parent_trace, "Can't load comparison file because it doesn't exist",
data = {'file': EXPECTED_COMPARISON_FILE,
'path': EXPECTED_FOLDER + '/' + EXPECTED_COMPARISON_FILE,
'error': str(ex)})
self.assertEqual(df_comparison_nice, expected_df_comparison)
self.assertTrue(check)
def test_milestones_referenced_big_rock_version(self):
'''
Tests that integrity checks exist to prevent posting a milestones manifest if it references a
version of the big rocks that is not the latest.
'''
try:
self.setScenario("foreign_key.milestones_big_rock_version")
self.setCurrentTestName('fkey.ml_2_br')
self.selectTestDataLocation()
root_trace = FunctionalTrace(parent_trace=None, path_mask=self._path_mask) \
.doing("Running " + self.currentTestName())
PRODUCT_FILE = "products.static-data.admin.a6i.xlsx"
SCORING_CYCLE_FILE = "scoring-cycles.static-data.admin.a6i.xlsx"
BIG_ROCKS_v1_FILE = "opus.v1.big-rocks.journeys.a6i.xlsx"
BIG_ROCKS_v2_FILE = "opus.v2.big-rocks.journeys.a6i.xlsx"
MILESTONES_FILE = "opus.v1.milestone.journeys.a6i.xlsx"
BIG_ROCKS_API = "big-rocks.journeys.a6i"
MILESTONES_API = "milestone.journeys.a6i"
NAMESPACE = "my-corp.production"
SUB_NAMESPACE = "modernization"
REL_PATH_IN_EXT_COLLABORATION = "journeys/Dec 2020/FusionOpus/Default"
_path_of = self.fullpath_of
MASK_COMBINED = CLI_Utils().combined_mask(root_trace,
self.a6i_config)
# This will fool the CLI to treat our provisioned environment for this test as if it were the base environment
self.overwrite_test_context(
root_trace
) # Overwrites self.a6i_config , the store, the test_db, etc.
# For this test, we need to switch the working directory for click
my_trace = root_trace.doing(
"Running with working directory in the collaboration area")
store = self.stack().store()
clientURL = store.base_environment(my_trace).clientURL(my_trace)
working_area = clientURL + "/" + REL_PATH_IN_EXT_COLLABORATION
PathUtils().create_path_if_needed(my_trace, working_area)
_os.chdir(working_area)
COMMANDS = [
['post', '--timestamp', "_CLI__1",
_path_of(PRODUCT_FILE)],
[
'post', '--timestamp', "_CLI__2",
_path_of(SCORING_CYCLE_FILE)
],
[
'post', '--timestamp', "_CLI__3",
_path_of(BIG_ROCKS_v1_FILE)
], # v1 of big-rocks
[
'get', 'form', '--timestamp', "_CLI__4", MILESTONES_API,
NAMESPACE, SUB_NAMESPACE
], # milestones -> big-rocks v1
[
'get', 'form', '--timestamp', "_CLI__5", BIG_ROCKS_API,
NAMESPACE, SUB_NAMESPACE
],
[
'post', '--timestamp', "_CLI__6",
_path_of(BIG_ROCKS_v2_FILE)
], # v2 of big-rocks
['get', 'assertions'],
['post', '--timestamp', "_CLI__7",
_path_of(MILESTONES_FILE)], # Should trigger an error
]
self.skeleton_test(
parent_trace=my_trace,
cli_command_list=COMMANDS,
output_cleanining_lambda=MASK_COMBINED,
when_to_check_environment=CLI_Test_Skeleton.NEVER)
except ApodeixiError as ex:
print(ex.trace_message())
self.assertTrue(1 == 2)
def seedCurrentEnvironment(self, parent_trace, manifest_relative_folder,
postings_relative_folder):
'''
Populates the current environment's manifests or excel postings' area by copying the folder tree structures.
@param manifest_relative_folder A string. Should be a relative path that adheres to a valid path structure in the
KnowledgeBase store under the manifests folder.
It must also be the case that the input folder for this test has a subfolder called "manifests" which
contains the `manifest_relative_folder` as a sub-subfolder.
Behavior is to copy everything under the latter to the KnowledgeBase store's manifests area.
If set to None, nothing is copied.
Example: "my-corp.production/modernization.dec-2020.fusionopus.default"
@param postings_relative_folder A string. Should be a relative path that adheres to a valid path structure in the
KnowledgeBase store under the excel-postings folder.
It must also be the case that the input folder for this test has a subfolder called "excel-postings" which
contains the `postings_relative_folder` as a sub-subfolder.
Behavior is to copy everything under the latter to the KnowledgeBase store's manifests area.
If set to None, nothing is copied.
Example: "journeys/Dec 2020/FusionOpus/Default"
'''
INPUT_FOLDER = self.getInputDataFolder(
parent_trace) + "/" + self.scenario()
my_trace = self.trace_environment(
parent_trace,
"Seeding manifests under " + str(manifest_relative_folder))
if manifest_relative_folder != None:
src_folder = INPUT_FOLDER + "/manifests/" + manifest_relative_folder
PathUtils().checkPathExists(my_trace, src_folder)
manifestsURL = self.stack().store().current_environment(
my_trace).manifestsURL(my_trace)
dst_folder = manifestsURL + "/" + manifest_relative_folder
PathUtils().remove_folder_if_exists(my_trace, path=dst_folder)
try:
PathUtils().remove_folder_if_exists(my_trace, dst_folder)
_shutil.copytree(src=src_folder, dst=dst_folder, ignore=None)
except Exception as ex:
raise ApodeixiError(
my_trace,
"Found an error in seeding the manifests for test " +
self.scenario(),
data={
"URL to download from": src_folder,
"URL to copy to": manifestsURL,
"error": str(ex)
})
my_trace = self.trace_environment(
parent_trace,
"Seeding Excel postings under " + str(postings_relative_folder))
if postings_relative_folder != None:
src_folder = INPUT_FOLDER + "/excel-postings/" + postings_relative_folder
PathUtils().checkPathExists(my_trace, src_folder)
postingsURL = self.stack().store().current_environment(
my_trace).postingsURL(my_trace)
dst_folder = postingsURL + "/" + postings_relative_folder
PathUtils().remove_folder_if_exists(my_trace, path=dst_folder)
try:
PathUtils().remove_folder_if_exists(my_trace, dst_folder)
_shutil.copytree(src=src_folder, dst=dst_folder, ignore=None)
except Exception as ex:
raise ApodeixiError(
my_trace,
"Found an error in seeding the Excel postings for test " +
self.scenario(),
data={
"URL to download from": src_folder,
"URL to copy to": postingsURL,
"error": str(ex)
})
def test_notebook_run(self):
root_trace = FunctionalTrace(
parent_trace=None,
path_mask=self._path_mask).doing("Testing Notebook execution")
try:
INPUT_FOLDER = self.input_data
OUTPUT_FOLDER = self.output_data
EXPECTED_FOLDER = self.expected_data
TEST_SCENARIO = 'test_notebook_run'
PathUtils().create_path_if_needed(root_trace,
OUTPUT_FOLDER + "/notebooks/")
nb_utils = NotebookUtils(
src_folder=INPUT_FOLDER,
src_filename=TEST_SCENARIO + "_INPUT.ipynb",
destination_folder=OUTPUT_FOLDER + "/notebooks/",
destination_filename=TEST_SCENARIO +
"_executed_notebook.ipynb")
my_trace = root_trace.doing("Running notebook")
result_dict = nb_utils.run(my_trace)
# Remove a path with timestamps since it changes all the time
my_trace = root_trace.doing("Removing path with timestamps")
hide_timestamps = lambda x: '<Timestamps removed in test output>'
cleaned_dict = DictionaryUtils().replace_path(
parent_trace=my_trace,
root_dict=result_dict,
root_dict_name='nb_utils_run_result_dict',
path_list=['cells', '*', 'metadata', 'execution', '*'],
replacement_lambda=hide_timestamps)
my_trace = root_trace.doing(
"Hiding user_folders printed as output")
def _hide_root_folder(val):
'''
1) Hides root directory for paths displayed in output
2) Converts displayed paths to Linux format, so we get same output in Windows and Linux
@param val A string; normally the value of an entry in a dictionary
'''
folder_hints = [
'apodeixi\\util', 'apodeixi\\\\util', 'apodeixi/util'
]
result = val
for hint in folder_hints:
if hint in val: # val is a path, keep only what comes after 'src/apodeixi'.
result = '<Root directory hidden in test output>' + hint + val.split(
hint)[1]
if _os.name == "nt": # Display in Linux style
result = result.replace("\\\\", "/")
return result
return result
def _hide_version_nb(val):
'''
1) Hides root directory for paths displayed in output
2) Converts displayed paths to Linux format, so we get same output in Windows and Linux
3) Masks Python version numbers so that output does not depend on what version of Python is used to run tests
@param val A string; normally the value of an entry in a dictionary
'''
# First mask Python version numbers for vals like: " version: 3.9.7"
VERSION_NB_REGEX = _re.compile(r'[0-9]+.[0-9]+.[0-9]+')
result = _re.sub(VERSION_NB_REGEX, '<VERSION NB>', val)
return result
cleaned_dict = DictionaryUtils().replace_path(
parent_trace=my_trace,
root_dict=cleaned_dict,
root_dict_name='aha_configurer_result_dict',
path_list=['cells', '*', 'outputs', '*', 'data', 'text/plain'],
replacement_lambda=_hide_root_folder)
cleaned_dict = DictionaryUtils().replace_path(
parent_trace=my_trace,
root_dict=cleaned_dict,
root_dict_name='aha_configurer_result_dict',
path_list=['metadata', 'language_info', 'version'],
replacement_lambda=_hide_version_nb)
self._compare_to_expected_yaml(parent_trace=my_trace,
output_dict=cleaned_dict,
test_output_name=TEST_SCENARIO,
save_output_dict=True)
except ApodeixiError as ex:
print(ex.trace_message())
self.assertTrue(1 == 2)
def findLatestVersionManifest(self, parent_trace, manifest_api_name, namespace, name, kind):
'''
For a given manifest API, a manifest is logically identified by its name and kind properties within
a given namespace.
However, there might be multiple versions of a logical manifest (versions are integers starting
at 1, 2, 3, ..., with version increasing each time the manifest gets updated).
This method returns a manifest (as a dict) and a string.
The manifest is the most recent version of the manifest that is logically identified
by the parameters.
The 2nd returned value is the path to that manifest.
If no such manifest exists in the KnowledgeBase store then the first returned object is None.
Example: for file-based stores, a manifest may be stored in a filename like:
$KB_STORE/manifests/my-corp.production/modernization.default.dec-2020.fusionopus/big-rock.2.yaml
In this example,
* the namespace is "my-corp.production"
* the name is "modernization.default.dec-2020.fusionopus"
* the kind is "big-rock"
* the version is 2 (an int)
* the manifest api is embedded within the YAML file. The YAML file has a field called
"apiVersion" with a value like "delivery-planning.journeys.a6i.io/v1a", and the manifest api
is the substring without the suffix: "delivery-planning.journeys.a6i.io"
@param manifest_api_name A string representing the Apodeixi API defining the YAML schemas for the
manifest kinds subsumed under such API. The search for manifests is filtered to those
whose YAML representation declares itself as falling under this API.
Example: 'delivery-planning.journeys.a6i.io'
@param namespace A string. Represents the namespace in the KnowledgeBase store's manifests area
where to look for the manifest.
@param name A string representing the name of the manifest. Along with kind, this identifies a
unique logical manifest (other than version number)
@param kind A string representing the kind of the manifest. Along with kind, this identifies a unique
logical manifest (other than version number)
'''
manifest, manifest_path = super().findLatestVersionManifest(parent_trace, manifest_api_name,
namespace, name, kind)
if manifest == None:
# Not found, so normally we should return None. But before giving up, look in parent environment
# if we have been configured to fail over the parent environment whenver we can't find something
if self._failover_manifest_reads_to_parent(parent_trace):
# Search in parent first, and copy anything found to the current environment
my_trace = parent_trace.doing("Searching in parent environment")
# Temporarily switch to the parent environment, and try again
original_env = self.current_environment(my_trace)
self.activate(my_trace, self.parent_environment(my_trace).name(my_trace))
manifest, manifest_path = self.findLatestVersionManifest(my_trace, manifest_api_name,
namespace, name, kind)
# Now that search in parent environment is done, reset back to original environment
self.activate(my_trace, original_env.name(my_trace))
# Populate current environment with anything found in the parent environment, but only if it is not
# already in current environment
if manifest != None:
my_trace = parent_trace.doing("Copying manifest from parent environment",
data = {"parent environment name":
self.parent_environment(my_trace).name(my_trace),
"current environment name":
self.current_environment(my_trace).name(my_trace)})
from_path = manifest_path
to_dir = self.current_environment(my_trace).postingsURL(parent_trace)
if not _os.path.exists(to_dir):
my_trace = parent_trace.doing("Copying a manifest file",
data = {"src_path": from_path,
"to_dir": to_dir})
PathUtils().create_path_if_needed(parent_trace=my_trace, path=to_dir)
PathUtils().copy_file(parent_trace, from_path, to_dir)
return manifest, manifest_path
def _run_basic_flow(self, parent_trace, from_nothing, namespace,
subnamespace, posting_api, excel_relative_path,
excel_file, excel_sheet, nb_manifests_expected,
generated_form_worksheet, setup_dependencies):
'''
Main service offered by this script class.
It executes the script logic and creates and checks all regression output.
* Request a blind form (only if "from_nothing" = True) or import a previously created Excel posting file.
* Submit an initial posting
* Request update form
* Submit an update to initial posting
@param from_nothing A boolean to determine if we should request a form even before any manifest exists
(i.e., a "blank" form)
@param namespace A string. Only relevant if from_nothing=True. Used to delimit the search for manifests
when generating a form without explicit manifest handles being given.
@param subnamespace A string. Only relevant if from_nothing=True. Used to delimit search for manifests.
An optional string representing a slice of the namespace that further restricts
the manifest names to search. If set to None, not subspace is assumed.
Example: in the manifest name "modernization.default.dec-2020.fusionopus", the
token "modernization" is the subnamespace. The other tokens come from filing coordinates
for the posting from whence the manifest arose.
@param setup_dependencies A boolean. If True, dependencies like static data or referenced manifests will be created
before the test scenario's flows.
Tests a basic flow for a single posting API consisting of:
'''
try:
script_trace = parent_trace.doing(
"Test scenario",
data={
'excel_file': excel_file,
'scenario': self.myTest.scenario(),
'test name': self.myTest.currentTestName()
},
origination={
'signaled_from': __file__,
'concrete class': str(self.__class__.__name__)
})
my_trace = self.myTest.trace_environment(script_trace,
"Isolating test case")
if True:
self.myTest.provisionIsolatedEnvironment(my_trace)
if setup_dependencies:
my_trace = self.myTest.trace_environment(
script_trace, "Setting up dependency data")
self.myTest.setup_static_data(my_trace)
self.myTest.setup_reference_data(my_trace)
# Now that dependencies have been set up, let's check how the environment looks.
self.myTest.check_environment_contents(my_trace)
if from_nothing: # Make a blind call get `requestForm`, without knowing a priori the manifest handles
my_trace = self.myTest.trace_environment(
script_trace, "Blind call to 'requestForm' API")
if True:
store = self.myTest.stack().kb().store
blind_form_request = store.getBlindFormRequest(
parent_trace=my_trace,
relative_path=excel_relative_path,
posting_api=posting_api,
namespace=namespace,
subnamespace=subnamespace)
fr_response, fr_log_txt, \
fr_rep = self.myTest.stack().kb().requestForm( parent_trace = script_trace,
form_request = blind_form_request)
api_called = "initial requestForm"
self.check_log(my_trace, fr_log_txt, api_called=api_called)
self.check_kb_introspection(my_trace,
kb=self.myTest.stack().kb(),
api_called=api_called)
self.myTest.check_environment_contents(my_trace)
layout_info, pl_fmt_info, ws_fmt_info, label_ctx_nice \
= self._generated_form_test_output( my_trace,
blind_form_request,
fr_response,
fr_log_txt,
fr_rep,
generated_form_worksheet)
self.check_posting_label(my_trace,
label_ctx_nice,
api_called=api_called)
self.check_xl_layout(my_trace, layout_info,
generated_form_worksheet, api_called)
self.check_xl_format(
my_trace, pl_fmt_info,
ManifestRepresenter.POSTING_LABEL_SHEET, api_called)
self.check_xl_format(my_trace, ws_fmt_info,
generated_form_worksheet, api_called)
else: # Copy the input file into the collaboration area
my_trace = self.myTest.trace_environment(
script_trace,
"Copying input into the shared collaboration area")
input_path = self.myTest.getInputDataFolder(my_trace) \
+ "/" + self.myTest.scenario() \
+ "/" + self.myTest.currentTestName() \
+ ".original." + excel_file
clientURL = self.myTest.stack().store().current_environment(
my_trace).clientURL(my_trace)
collab_area_folder = clientURL + "/" + excel_relative_path
collab_area_path = collab_area_folder + "/" + excel_file
PathUtils().create_path_if_needed(my_trace, collab_area_folder)
PathUtils().copy_file(parent_trace, input_path,
collab_area_path)
self.myTest.check_environment_contents(my_trace)
my_trace = self.myTest.trace_environment(
script_trace, "Calling 'postByFile' API")
if True:
clientURL = self.myTest.stack().store().current_environment(
my_trace).clientURL(my_trace)
posting_path = clientURL + "/" + excel_relative_path + "/" + excel_file
response, log_txt = self.myTest.stack().kb().postByFile(
parent_trace=my_trace,
path_of_file_being_posted=posting_path,
excel_sheet=excel_sheet)
self.check_log(my_trace, log_txt, api_called="postByFile")
self.check_kb_introspection(my_trace,
kb=self.myTest.stack().kb(),
api_called="postByFile")
self.check_manifest_count(my_trace, response,
nb_manifests_expected)
self.check_manifests_contents(my_trace, response)
self.myTest.check_environment_contents(parent_trace=my_trace)
my_trace = self.myTest.trace_environment(
script_trace, "Calling 'requestForm' API")
form_request_responses = []
if True:
form_idx = 0
for form_request in response.optionalForms(
) + response.mandatoryForms():
fr_response, fr_log_txt, \
fr_rep = self.myTest.stack().kb().requestForm(parent_trace = script_trace,
form_request = form_request)
api_called = "requestForm #" + str(form_idx)
self.check_log(my_trace, fr_log_txt, api_called=api_called)
self.check_kb_introspection(my_trace,
kb=self.myTest.stack().kb(),
api_called=api_called)
self.myTest.check_environment_contents(my_trace)
layout_info, pl_fmt_info, ws_fmt_info, label_ctx_nice \
= self._generated_form_test_output( my_trace,
form_request,
fr_response,
fr_log_txt,
fr_rep,
generated_form_worksheet)
self.check_posting_label(my_trace,
label_ctx_nice,
api_called=api_called)
self.check_xl_layout(my_trace, layout_info,
generated_form_worksheet, api_called)
self.check_xl_format(
my_trace, pl_fmt_info,
ManifestRepresenter.POSTING_LABEL_SHEET, api_called)
self.check_xl_format(my_trace, ws_fmt_info,
generated_form_worksheet, api_called)
'''
Save the form before we change it
'''
self.myTest.snapshot_generated_form(my_trace, fr_response)
form_request_responses.append(fr_response)
form_idx += 1
my_trace = self.myTest.trace_environment(
script_trace, "Doing an update via 'postByFile' API")
if True:
for fr_response in form_request_responses:
# Copy the "modified form" that has some edits in it
self.myTest.modify_form(my_trace, fr_response)
form_path = fr_response.clientURL(
my_trace) + "/" + fr_response.getRelativePath(my_trace)
update_response, update_log_txt = self.myTest.stack().kb(
).postByFile(parent_trace=my_trace,
path_of_file_being_posted=form_path)
self.check_log(my_trace,
update_log_txt,
api_called="postByFile")
self.check_kb_introspection(my_trace,
kb=self.myTest.stack().kb(),
api_called="postByFile")
self.check_manifest_count(my_trace, update_response,
nb_manifests_expected)
self.check_manifests_contents(my_trace, update_response)
self.myTest.check_environment_contents(
parent_trace=my_trace)
# As a last step, generate the form resulting from the prior posting, just to make sure the
# end user can get something to work from
my_trace = self.myTest.trace_environment(
script_trace, "Calling 'requestForm' API again")
form_request_2nd_responses = []
if True:
form_idx = 0
for form_request in update_response.optionalForms(
) + update_response.mandatoryForms():
fr_response, fr_log_txt, \
fr_rep = self.myTest.stack().kb().requestForm(parent_trace = script_trace,
form_request = form_request)
api_called = "requestForm #" + str(form_idx)
self.check_log(my_trace, fr_log_txt, api_called=api_called)
self.check_kb_introspection(my_trace,
kb=self.myTest.stack().kb(),
api_called=api_called)
self.myTest.check_environment_contents(my_trace)
self.myTest.snapshot_generated_form(my_trace, fr_response)
form_request_2nd_responses.append(fr_response)
return form_request_2nd_responses
except ApodeixiError as ex:
print(ex.trace_message())
self.myTest.assertTrue(1 == 2)
# If we get this far, the tests failed since we should have returned within the try statement.
# So hardcode an informative failure.
self.myTest.assertTrue("Shouldn't have gotten to this line" == 0)
def run_script(self, parent_trace, SANDBOX_FUNC, cli_arguments_dict):
_path_of = self.myTest.fullpath_of
MASK_COMBINED = CLI_Utils().combined_mask(parent_trace,
self.myTest.a6i_config)
_s = CLI_BigRocks_and_Milestones_Script
_args = cli_arguments_dict
# This will fool the CLI to treat our provisioned environment for this test as if it were the base environment
self.myTest.overwrite_test_context(
parent_trace
) # Overwrites self.a6i_config , the store, the test_db, etc.
my_trace = parent_trace.doing(
"Running with working directory in the collaboration area")
store = self.myTest.stack().store()
if self.myTest.sandbox != None:
root_dir = _os.path.dirname(
store.base_environment(my_trace).manifestsURL(my_trace))
envs_dir = root_dir + "/" + File_KBEnv_Impl.ENVS_FOLDER
working_dir = envs_dir + "/" + self.myTest.sandbox + "/external-collaboration/" \
+ _args[_s.REL_PATH_IN_EXT_COLLABORATION]
else:
clientURL = store.base_environment(my_trace).clientURL(my_trace)
working_dir = clientURL + "/" + _args[
_s.REL_PATH_IN_EXT_COLLABORATION]
PathUtils().create_path_if_needed(parent_trace, working_dir)
_os.chdir(working_dir)
if SANDBOX_FUNC != None:
__dry_run = '--dry-run'
__environment = '--environment'
ENV_CHOICE = SANDBOX_FUNC
else: # Case for life runs
__dry_run = None
__environment = None
ENV_CHOICE = None
COMMANDS_1 = [
# Initialize static data
[
'post', __dry_run, '--timestamp', "_CLI__1",
_path_of(_args[_s.PRODUCT_FILE])
],
[
'post', __environment, ENV_CHOICE, '--timestamp', "_CLI__2",
_path_of(_args[_s.SCORING_CYCLE_FILE])
],
# Create big rocks v1
[
'get', 'form', __environment, ENV_CHOICE, '--timestamp',
"_CLI__3", _args[_s.BIG_ROCKS_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
],
[
'post', __environment, ENV_CHOICE, '--timestamp', "_CLI__4",
_path_of(_args[_s.BIG_ROCKS_v1_FILE])
],
# Create milestones v1
[
'get', 'form', __environment, ENV_CHOICE, '--timestamp',
"_CLI__5", _args[_s.BIG_MILESTONES_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
],
[
'post', __environment, ENV_CHOICE, '--timestamp', "_CLI__6",
_path_of(_args[_s.BIG_MILESTONES_v1_FILE])
],
]
self.myTest.skeleton_test(
parent_trace=my_trace,
cli_command_list=COMMANDS_1,
output_cleanining_lambda=MASK_COMBINED,
when_to_check_environment=CLI_Test_Skeleton.NEVER)
# Check that manifest is as expected
NAME = "experimental.march-2021.turbotax.iot-experiment"
NAMESPACE = "intuit.innovations"
self.myTest.check_manifest(my_trace,
'delivery-planning.journeys.a6i.io',
NAMESPACE, NAME, 'big-rock')
self.myTest.check_manifest(my_trace,
'delivery-planning.journeys.a6i.io',
NAMESPACE, NAME, 'big-rock-estimate')
self.myTest.check_manifest(my_trace,
'delivery-planning.journeys.a6i.io',
NAMESPACE, NAME, 'modernization-milestone')
COMMANDS_2 = [
# First try to update big rocks v2 - should fail due to foreign key constraints
[
'get', 'form', __environment, ENV_CHOICE, '--timestamp',
"_CLI__7", _args[_s.BIG_ROCKS_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
],
[
'post', __environment, ENV_CHOICE, '--timestamp', "_CLI__8",
_path_of(_args[_s.BIG_ROCKS_v2_FILE])
],
# Update milestones v2 - should remove the reference that caused big rocks v2 to fail
[
'get', 'form', __environment, ENV_CHOICE, '--timestamp',
"_CLI__9", _args[_s.BIG_MILESTONES_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
],
[
'post', __environment, ENV_CHOICE, '--timestamp', "_CLI__10",
_path_of(_args[_s.BIG_MILESTONES_v2_FILE])
],
# Second try to update big rocks v2 - should work now that user removed references in
# milestones v2 to the rocks that were removed in v2
[
'post', __environment, ENV_CHOICE, '--timestamp', "_CLI__11",
_path_of(_args[_s.BIG_ROCKS_v2_FILE])
],
]
self.myTest.skeleton_test(
parent_trace=my_trace,
cli_command_list=COMMANDS_2,
output_cleanining_lambda=MASK_COMBINED,
when_to_check_environment=CLI_Test_Skeleton.NEVER)
# Check that manifest is as expected
self.myTest.check_manifest(my_trace,
'delivery-planning.journeys.a6i.io',
NAMESPACE, NAME, 'big-rock')
self.myTest.check_manifest(my_trace,
'delivery-planning.journeys.a6i.io',
NAMESPACE, NAME, 'big-rock-estimate')
self.myTest.check_manifest(my_trace,
'delivery-planning.journeys.a6i.io',
NAMESPACE, NAME, 'modernization-milestone')
COMMANDS_3 = [
# Get final forms
[
'get', 'form', __environment, ENV_CHOICE, '--timestamp',
"_CLI__12", _args[_s.BIG_ROCKS_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
],
[
'get', 'form', __environment, ENV_CHOICE, '--timestamp',
"_CLI__13", _args[_s.BIG_MILESTONES_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
],
# Summarize assertions created
['get', 'assertions', __environment, ENV_CHOICE]
]
self.myTest.skeleton_test(
parent_trace=my_trace,
cli_command_list=COMMANDS_3,
output_cleanining_lambda=MASK_COMBINED,
when_to_check_environment=CLI_Test_Skeleton.ONLY_AT_END)
def describe_diff_response(self, parent_trace, kb_session, diff_result):
'''
Returns a string suitable for display in the Apodeixi CLI.
Also persists the same information as an Excel file in the reports area of the KnowledgeBase.
The string is formatted as a table that provides information on what Apodeixi did in response to a user
requesting a diff between two versions of a manifest
The table has a row for each noteworthy difference.
@param diff_result A ManifestDiffResult object encapsulating all the differences
'''
my_trace = parent_trace.doing("Constructing the diff report's data")
if True:
description_table = []
description_headers = ["Diff Type", "Entity", "Field", "Original Value", "New Value"]
headers_widths = [20, 50, 25, 40, 40]
# Important: order in list must match the order of the headers in `description_headers`. Required by
# the tabulate Python package.
for entity_desc in diff_result.added_entities_description(parent_trace):
description_table.append(["ENTITY ADDED", entity_desc, "", "", ""])
for entity_desc in diff_result.removed_entities_description(parent_trace):
description_table.append(["ENTITY REMOVED", entity_desc, "", "", ""])
changed_entities_dict = diff_result.changed_entities_description_dict(parent_trace)
for entity_desc in changed_entities_dict.keys():
entity_diff = changed_entities_dict[entity_desc]
for field in entity_diff.added_fields:
description_table.append(["FIELD ADDED", entity_desc, field, "", ""])
for field in entity_diff.removed_fields:
description_table.append(["FIELD REMOVED", entity_desc, field, "", ""])
for changed_value in entity_diff.changed_fields:
description_table.append(["FIELD CHANGED", entity_desc, changed_value.field,
changed_value.old_value,
changed_value.new_value])
diff_description = "\n" + diff_result.short_description + ":\n\n"
diff_description += tabulate(description_table, headers=description_headers)
diff_description += "\n"
# Save the report
my_trace = parent_trace.doing("Saving diff report")
if True:
reports_folder = kb_session.kb_rootdir + "/" + File_KBEnv_Impl.REPORTS_FOLDER
REPORT_FILENAME = kb_session.timestamp + "_" + diff_result.short_description.replace(" ", "_") \
+ "_diff.xlsx"
PathUtils().create_path_if_needed(parent_trace, reports_folder)
'''
report_df = _pd.DataFrame(data=description_table, columns=description_headers)
report_df.to_excel(reports_folder + "/" + REPORT_FILENAME)
'''
self._write_report( parent_trace = my_trace,
data = description_table,
columns = description_headers,
column_widths = headers_widths,
path = reports_folder + "/" + REPORT_FILENAME,
description = diff_result.long_description)
# Append pointer to report
#
my_trace = parent_trace.doing("Adding link to diff report")
if True:
cli_reporter = CLI_ErrorReporting(kb_session)
diff_description += cli_reporter.POINTERS("\n\nRetrieve Excel report at ")
diff_description += cli_reporter.POINTERS(cli_reporter.UNDERLINE("file:///" + reports_folder
+ "/" + REPORT_FILENAME))
return diff_description
def commitTransaction(self, parent_trace):
'''
Finalizes a transaction previously started by beginTransaction, by cascading any I/O previously done in
the transaction's isolation area to the store's persistent area.
'''
env, parent_env = self._validate_transaction_end_of_life(parent_trace)
src_postings_root = env.postingsURL(parent_trace)
dst_postings_root = parent_env.postingsURL(parent_trace)
src_manifests_root = env.manifestsURL(parent_trace)
dst_manifests_root = parent_env.manifestsURL(parent_trace)
src_clientURL_root = env.clientURL(parent_trace)
dst_clientURL_root = parent_env.clientURL(parent_trace)
# If the parent environment is also a transactional envinronment, we will have to record in it
# the events so that when the parent is committed, those events are cascaded to the parent's parent.
# But it may also be that the parent is not transactional, which is why the `parent_events`
# variable may be None and why we need to be checking for that all the time.
parent_name = parent_env.name(parent_trace)
if parent_name in self._transaction_events_dict.keys():
parent_events = self._transaction_events_dict[parent_name]
else:
parent_events = None
# **GOTCHA**
#
# Don't call pop()! We want to see the "last transaction's" environment, but not yet remove
# the last transaction (so peek, not pop). The reason is that if any of the subsequent code in this commit() method
# raises an exception, it will cause a subsequent problem for the abortTransaction method,
# since abortTransaction will look for the "last transaction" and will not find it (or will)
# find the wrong one) if we have poped. So use the [-1] notation to peek (not pop!) the last
# transaction. Later, just before exiting this method, do the pop()
ending_env = self._transactions_stack[-1]
events = self._transaction_events_dict[ending_env.name(parent_trace)]
for relative_path in events.posting_writes():
from_path = src_postings_root + "/" + relative_path
to_path = dst_postings_root + "/" + relative_path
to_dir = _os.path.dirname(to_path)
PathUtils().create_path_if_needed(parent_trace, to_dir)
PathUtils().copy_file(parent_trace, from_path, to_dir)
if parent_events != None:
parent_events.remember_posting_write(relative_path)
for relative_path in events.manifest_writes():
from_path = src_manifests_root + "/" + relative_path
to_path = dst_manifests_root + "/" + relative_path
to_dir = _os.path.dirname(to_path)
PathUtils().create_path_if_needed(parent_trace, to_dir)
PathUtils().copy_file(parent_trace, from_path, to_dir)
if parent_events != None:
parent_events.remember_manifest_write(relative_path)
for relative_path in events.clientURL_writes():
from_path = src_clientURL_root + "/" + relative_path
to_path = dst_clientURL_root + "/" + relative_path
# Normally clientURL is the same across environments (except mostly in test situations),
# so to prevent the copy operation from raising an exception make sure we only attempt to copy
# the file when the two paths are different
if not _os.path.samefile(from_path, to_path):
#if from_path != to_path:
to_dir = _os.path.dirname(to_path)
PathUtils().create_path_if_needed(parent_trace, to_dir)
PathUtils().copy_file(parent_trace, from_path, to_dir)
if parent_events != None:
parent_events.remember_clientURL_write(relative_path)
for relative_path in events.posting_deletes():
to_path = dst_postings_root + "/" + relative_path
if 0 == PathUtils().remove_file_if_exists(parent_trace, to_path):
if parent_events != None:
parent_events.remember_posting_delete(relative_path)
for relative_path in events.manifest_deletes():
to_path = dst_manifests_root + "/" + relative_path
if 0 == PathUtils().remove_file_if_exists(parent_trace, to_path):
if parent_events != None:
parent_events.remember_manifest_deletes(relative_path)
for relative_path in events.clientURL_deletes():
to_path = dst_clientURL_root + "/" + relative_path
if 0 == PathUtils().remove_file_if_exists(parent_trace, to_path):
if parent_events != None:
parent_events.remember_clientURL_deletes(relative_path)
# Last but not least: persist foreign key constraints and copy them to the
# parent environment
#
self.persistForeignKeyConstraints(parent_trace)
version = 1
FOREIGN_KEY_FILE = "foreign_key_contraints." + str(version) + ".yaml"
from_path = src_manifests_root + "/system/" + FOREIGN_KEY_FILE
to_dir = dst_manifests_root + "/system/"
PathUtils().create_path_if_needed(parent_trace, to_dir)
PathUtils().copy_file(parent_trace, from_path, to_dir)
# Now remove the environment of the transaction we just committed
self.removeEnvironment(parent_trace, env.name(parent_trace))
self.activate(parent_trace, parent_env.name(parent_trace))
# **GOTCHA**
#
# Now it is safe to pop() - it wasn't safe earlier because if any of the code in this method
# raised an exception after having popped the last transaction in the stack, the abortTransaction
#method woudld have failed since it wouldh't have found the last transaction to then abort it.
ending_env = self._transactions_stack.pop()
events = self._transaction_events_dict.pop(ending_env.name(parent_trace))
def run_script(self, parent_trace, SANDBOX_FUNC, cli_arguments_dict):
_path_of = self.myTest.fullpath_of
MASK_COMBINED = CLI_Utils().combined_mask(parent_trace,
self.myTest.a6i_config)
_s = CLI_Basic_Script
_args = cli_arguments_dict
# This will fool the CLI to treat our provisioned environment for this test as if it were the base environment
self.myTest.overwrite_test_context(
parent_trace
) # Overwrites self.a6i_config , the store, the test_db, etc.
if SANDBOX_FUNC != None:
__dry_run = '--dry-run'
__environment = '--environment'
ENV_CHOICE = SANDBOX_FUNC
else: # Case for life runs
__dry_run = None
__environment = None
ENV_CHOICE = None
COMMANDS = [
[
'post', __dry_run, '--timestamp', "_CLI__1",
_path_of(_args[_s.PRODUCT_FILE])
],
[
'post', __environment, ENV_CHOICE, '--timestamp', "_CLI__2",
_path_of(_args[_s.SCORING_CYCLE_FILE])
],
[
'post', __environment, ENV_CHOICE, '--timestamp', "_CLI__3",
_path_of(_args[_s.BIG_ROCKS_v1_FILE])
],
['get', 'products', __environment, ENV_CHOICE],
['get', 'scoring-cycles', __environment, ENV_CHOICE],
['get', 'namespaces'],
#['get', 'environments'], # Can't test- environment count non-deterministic
['get', 'apis'],
]
self.myTest.skeleton_test(parent_trace=parent_trace,
cli_command_list=COMMANDS,
output_cleanining_lambda=MASK_COMBINED)
# For the next test, we need to switch the working directory for click
my_trace = parent_trace.doing(
"Running with working directory in the collaboration area")
store = self.myTest.stack().store()
if self.myTest.sandbox != None:
root_dir = _os.path.dirname(
store.base_environment(my_trace).manifestsURL(my_trace))
envs_dir = root_dir + "/" + File_KBEnv_Impl.ENVS_FOLDER
working_dir = envs_dir + "/" + self.myTest.sandbox + "/external-collaboration/" \
+ _args[_s.REL_PATH_IN_EXT_COLLABORATION]
else:
clientURL = store.base_environment(my_trace).clientURL(my_trace)
working_dir = clientURL + "/" + _args[
_s.REL_PATH_IN_EXT_COLLABORATION]
PathUtils().create_path_if_needed(parent_trace, working_dir)
_os.chdir(working_dir)
COMMANDS_2 = [[
'get', 'form', __environment, ENV_CHOICE, '--timestamp', "_CLI__4",
_args[_s.BIG_ROCKS_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
],
[
'post', __environment, ENV_CHOICE, '--timestamp',
"_CLI__5",
_path_of(_args[_s.BIG_ROCKS_v2_FILE])
],
[
'get', 'form', __environment, ENV_CHOICE,
'--timestamp', "_CLI__6",
_args[_s.BIG_MILESTONES_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
],
[
'post', __environment, ENV_CHOICE, '--timestamp',
"_CLI__7",
_path_of(_args[_s.BIG_MILESTONES_v1_FILE])
],
[
'get', 'form', __environment, ENV_CHOICE,
'--timestamp', "_CLI__8",
_args[_s.BIG_MILESTONES_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
],
[
'post', __environment, ENV_CHOICE, '--timestamp',
"_CLI__9",
_path_of(_args[_s.BIG_MILESTONES_v2_FILE])
],
[
'get', 'form', __environment, ENV_CHOICE,
'--timestamp', "_CLI__10",
_args[_s.BIG_MILESTONES_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
], ['get', 'assertions', __environment, ENV_CHOICE]]
self.myTest.skeleton_test(parent_trace=parent_trace,
cli_command_list=COMMANDS_2,
output_cleanining_lambda=MASK_COMBINED)
def form(kb_session, posting_api, namespace, subnamespace, dry_run,
environment, timestamp):
'''
Requests a form (an Excel spreadsheet) which (after some edits, as appropriate) can be used as the
input to the post command.
'''
timer = ApodeixiTimer()
func_trace = FunctionalTrace(parent_trace=None, path_mask=None)
root_trace = func_trace.doing("CLI call to post",
origination={'signaled_from': __file__})
kb_operation_succeeded = False
try:
# Catch warnings and handle them so that we avoid spurious noise in the CLI due to noisy 3rd party libraries
with warnings.catch_warnings(record=True) as w:
WarningUtils().turn_traceback_on(root_trace, warnings_list=w)
if environment != None:
kb_session.store.activate(parent_trace=root_trace,
environment_name=environment)
click.echo(CLI_Utils().sandox_announcement(environment))
elif dry_run == True:
sandbox_name = kb_session.provisionSandbox(root_trace)
click.echo(CLI_Utils().sandox_announcement(sandbox_name))
'''
else:
raise ApodeixiError(root_trace, "Sorry, only sandbox-isolated runs are supported at this time. Aborting.")
'''
# Now that we have pinned down the environment (sandbox or not) in which to call the KnowledgeBase's services,
# set that environment's tag to use for KnoweldgeBase's posting logs, if the user set it.
if timestamp:
kb_session.store.current_environment(root_trace).config(
root_trace).use_timestamps = timestamp
my_trace = root_trace.doing(
"Invoking KnowledgeBase's requestForm service")
output_dir = _os.getcwd()
clientURL = kb_session.store.getClientURL(my_trace)
relative_path, void = PathUtils().relativize(parent_trace=my_trace,
root_dir=clientURL,
full_path=output_dir)
form_request = kb_session.store.getBlindFormRequest(
parent_trace=my_trace,
relative_path=relative_path,
posting_api=posting_api,
namespace=namespace,
subnamespace=subnamespace)
response, log_txt, rep = kb_session.kb.requestForm(
parent_trace=my_trace, form_request=form_request)
kb_operation_succeeded = True
manifests_description = CLI_Utils().describe_req_form_response(
my_trace,
form_request_response=response,
store=kb_session.store,
representer=rep)
click.echo(manifests_description)
output = "Success"
click.echo(output)
click.echo(timer.elapsed_time_message())
WarningUtils().handle_warnings(root_trace, warning_list=w)
except ApodeixiError as ex:
error_msg = CLI_ErrorReporting(kb_session).report_a6i_error(
parent_trace=root_trace, a6i_error=ex)
if kb_operation_succeeded:
error_msg = "KnowledgeBase operation completed, but run into a problem when preparing "\
+ "a description of the response:\n"\
+ error_msg
# GOTCHA
# Use print, not click.echo or click exception because they don't correctly display styling
# (colors, underlines, etc.). So use vanilla Python print and then exit
print(error_msg)
_sys.exit()
except Exception as ex:
try:
error_msg = CLI_ErrorReporting(kb_session).report_generic_error(
parent_trace=root_trace, generic_error=ex)
if kb_operation_succeeded:
error_msg = "KnowledgeBase operation completed, but run into a problem when preparing "\
+ "a description of the response:\n"\
+ error_msg
except Exception as ex2:
error_msg = "CLI run into trouble: found error:\n\n\t" + str(ex) + "\n\n" \
+ "To make things worse, when trying to produce an error log file with a "\
+ "stack trace, run into an additional error:\n\n\t" + str(ex2)
# GOTCHA
# Use print, not click.echo or click exception because they don't correctly display styling
# (colors, underlines, etc.). So use vanilla Python print and then exit
print(error_msg)
_sys.exit()
def run_script(self, parent_trace, SANDBOX_FUNC, cli_arguments_dict):
_path_of = self.myTest.fullpath_of
MASK_COMBINED = CLI_Utils().combined_mask(parent_trace, self.myTest.a6i_config)
_s = CLI_StaticData_Script
_args = cli_arguments_dict
# This will fool the CLI to treat our provisioned environment for this test as if it were the base environment
self.myTest.overwrite_test_context(parent_trace) # Overwrites self.a6i_config , the store, the test_db, etc.
if SANDBOX_FUNC != None:
__dry_run = '--dry-run'
__environment = '--environment'
ENV_CHOICE = SANDBOX_FUNC
else: # Case for life runs
__dry_run = None
__environment = None
ENV_CHOICE = None
# For the next test, we need to switch the working directory for click
my_trace = parent_trace.doing("Running with working directory in the collaboration area")
store = self.myTest.stack().store()
if self.myTest.sandbox != None:
root_dir = _os.path.dirname(store.base_environment(my_trace).manifestsURL(my_trace))
envs_dir = root_dir + "/" + File_KBEnv_Impl.ENVS_FOLDER
working_dir = envs_dir + "/" + self.myTest.sandbox + "/external-collaboration/" \
+ _args[_s.REL_PATH_IN_EXT_COLLABORATION]
else:
clientURL = store.base_environment(my_trace).clientURL(my_trace)
working_dir = clientURL + "/" + _args[_s.REL_PATH_IN_EXT_COLLABORATION]
PathUtils().create_path_if_needed(parent_trace, working_dir)
_os.chdir(working_dir)
COMMANDS = [
['get', 'form', __environment, ENV_CHOICE, '--timestamp', "_CLI__1",
_args[_s.STATIC_DATA_API], _args[_s.NAMESPACE]],
['post', __dry_run, '--timestamp', "_CLI__2",
_path_of(_args[_s.STATIC_DATA_v1_FILE])],
['get', 'form', __environment, ENV_CHOICE, '--timestamp', "_CLI__3",
_args[_s.STATIC_DATA_API], _args[_s.NAMESPACE]],
['post', __environment, ENV_CHOICE, '--timestamp', "_CLI__4",
_path_of(_args[_s.STATIC_DATA_v2_FILE])],
['get', 'form', __environment, ENV_CHOICE, '--timestamp', "_CLI__5",
_args[_s.STATIC_DATA_API], _args[_s.NAMESPACE]],
]
self.myTest.skeleton_test( parent_trace = parent_trace,
cli_command_list = COMMANDS,
output_cleanining_lambda = MASK_COMBINED,
when_to_check_environment = CLI_Test_Skeleton.ONLY_AT_END)