def _get_environment_filter(parent_trace, kb_session, all, environment):
'''
Helper method to get filter for which environment to look into when searching in the KnowledgeBase
store.
'''
if all == True:
environment_filter = CLI_Utils().get_environment_filter(
parent_trace,
kb_session,
filter_type=CLI_Utils.ANY_ENV_FILTER,
sandbox=None)
elif environment != None:
click.echo(CLI_Utils().sandox_announcement(environment))
environment_filter = CLI_Utils().get_environment_filter(
parent_trace,
kb_session,
filter_type=CLI_Utils.SPECIFIC_SANDBOX_ENV_FILTER,
sandbox=environment)
else:
environment_filter = CLI_Utils().get_environment_filter(
parent_trace,
kb_session,
filter_type=CLI_Utils.ONLY_BASE_ENV_FILTER,
sandbox=environment)
return environment_filter
def products(kb_session, all, environment):
'''
Gets the list of valid products for the system.
'''
timer = ApodeixiTimer()
func_trace = FunctionalTrace(parent_trace=None, path_mask=None)
root_trace = func_trace.doing("CLI call to get products",
origination={'signaled_from': __file__})
try:
environment_filter = _get_environment_filter(root_trace, kb_session,
all, environment)
products_description = CLI_Utils().get_products(
root_trace, kb_session, environment_filter)
click.echo(products_description)
output = "Success"
click.echo(output)
click.echo(timer.elapsed_time_message())
except ApodeixiError as ex:
error_msg = CLI_ErrorReporting(kb_session).report_a6i_error(
parent_trace=root_trace, a6i_error=ex)
# GOTCHA
# Use print, not click.echo or click exception because they don't correctly display styling
# (colors, underlines, etc.). So use vanilla Python print and then exit
print(error_msg)
_sys.exit()
except Exception as ex:
click.echo("Unrecoverable error: " + str(ex))
_sys.exit()
def apis(kb_session):
'''
Gets the list of posting and manifest APIs supported by the KnowledgeBase
'''
timer = ApodeixiTimer()
func_trace = FunctionalTrace(parent_trace=None, path_mask=None)
root_trace = func_trace.doing("CLI call to get posting APIs",
origination={'signaled_from': __file__})
try:
posting_apis_description = CLI_Utils().get_apodeixi_apis(
root_trace, kb_session)
click.echo(posting_apis_description)
output = "Success"
click.echo(output)
click.echo(timer.elapsed_time_message())
except ApodeixiError as ex:
error_msg = CLI_ErrorReporting(kb_session).report_a6i_error(
parent_trace=root_trace, a6i_error=ex)
# GOTCHA
# Use print, not click.echo or click exception because they don't correctly display styling
# (colors, underlines, etc.). So use vanilla Python print and then exit
print(error_msg)
_sys.exit()
except Exception as ex:
click.echo("Unrecoverable error: " + str(ex))
_sys.exit()
def test_cli_diff_basic(self):
'''
Tests diff functionality between the lastest version of a manifest and its prior version
'''
self.setScenario("cli.basic_diff")
self.setCurrentTestName('basic_diff')
self.selectTestDataLocation()
root_trace = FunctionalTrace(parent_trace=None, path_mask=self._path_mask) \
.doing("Running " + self.currentTestName())
PRODUCTS_FILE = 'products.static-data.admin.a6i.xlsx'
SCORING_CYCLES_FILE = 'scoring-cycles.static-data.admin.a6i.xlsx'
BIG_ROCKS_FILE_V1 = 'v1.big-rocks.journeys.a6i.xlsx'
BIG_ROCKS_FILE_V2 = 'v2.big-rocks.journeys.a6i.xlsx'
MANIFEST_API = 'delivery-planning.journeys.a6i.io'
KIND = 'big-rock'
NAMESPACE = 'cicloquimica.production'
NAME = 'modernization.fy-22.astrea.official'
#STATIC_DATA_WORKING_DIR = "admin/static-data"
#PRODUCT_WORKING_DIR = "journeys/FY 22/Astrea/Official"
_path_of = self.fullpath_of
MASK_COMBINED = CLI_Utils().combined_mask(root_trace, self.a6i_config)
# This will fool the CLI to treat our provisioned environment for this test as if it were the base environment
self.overwrite_test_context(
root_trace
) # Overwrites self.a6i_config , the store, the test_db, etc.
my_trace = root_trace.doing("Running commands")
COMMANDS = [
['post', '--timestamp', "_CLI__1",
_path_of(PRODUCTS_FILE)],
['post', '--timestamp', "_CLI__2",
_path_of(SCORING_CYCLES_FILE)],
['post', '--timestamp', "_CLI__3",
_path_of(BIG_ROCKS_FILE_V1)],
['post', '--timestamp', "_CLI__4",
_path_of(BIG_ROCKS_FILE_V2)],
['diff', MANIFEST_API, KIND, NAMESPACE, NAME]
]
self.skeleton_test(
parent_trace=my_trace,
cli_command_list=COMMANDS,
output_cleanining_lambda=MASK_COMBINED,
when_to_check_environment=CLI_Test_Skeleton.ONLY_AT_END)
def config(kb_session, environment):
'''
Displays the Apodeixi configuration.
'''
T0 = _datetime.datetime.now()
func_trace = FunctionalTrace(parent_trace=None, path_mask=None)
root_trace = func_trace.doing("CLI call to get products",
origination={'signaled_from': __file__})
try:
if environment != None:
kb_session.store.activate(parent_trace=root_trace,
environment_name=environment)
click.echo(CLI_Utils().sandox_announcement(environment))
client_url = kb_session.store.getClientURL(root_trace)
postings_url = kb_session.store.getPostingsURL(root_trace)
click.echo("\n----- Current environment -----")
click.echo("\nclient URL:\t\t" + str(client_url))
click.echo("postings URL:\t\t" + str(postings_url))
config_txt = DictionaryFormatter().dict_2_nice(
root_trace, kb_session.a6i_config.config_dict, flatten=True)
click.echo("\n\n----- Config Settings -----")
click.echo("\n" + config_txt)
output = "Success"
click.echo(output)
except ApodeixiError as ex:
error_msg = CLI_ErrorReporting(kb_session).report_a6i_error(
parent_trace=root_trace, a6i_error=ex)
# GOTCHA
# Use print, not click.echo or click exception because they don't correctly display styling
# (colors, underlines, etc.). So use vanilla Python print and then exit
print(error_msg)
_sys.exit()
except Exception as ex:
click.echo("Unrecoverable error: " + str(ex))
_sys.exit()
T1 = _datetime.datetime.now()
duration = T1 - T0
duration_msg = str(duration.seconds) + "." + str(
duration.microseconds) + " sec"
click.echo(duration_msg)
def diff(kb_session, manifest_api, kind, namespace, name):
'''
Makes a diff between two versions of a manifest.
For a list of valid MANIFEST_APIs and KINDs, try 'get apis'
For a list of valid NAMESPACEs and NAMEs, try 'get assertions'
MANIFEST_API must be a versionless manifest API.
Example: 'delivery-planning.journeys.a6i.io', (as opposed to 'delivery-planning.journeys.a6i.io/v1a').
'''
timer = ApodeixiTimer()
func_trace = FunctionalTrace(parent_trace=None, path_mask=None)
root_trace = func_trace.doing("CLI call to post",
origination={'signaled_from': __file__})
kb_operation_succeeded = False
try:
my_trace = root_trace.doing(
"Invoking ManifestUtils's postByFile service")
diff_result = ManifestUtils().diff_manifest(
parent_trace=my_trace,
store=kb_session.store,
manifest_api_name=manifest_api,
namespace=namespace,
name=name,
kind=kind,
version1=None,
version2=None)
kb_operation_succeeded = True
diff_description = CLI_Utils().describe_diff_response(
my_trace, kb_session, diff_result)
# GOTCHA:
# Make sure to remove non-ascii characters before passing the description to click.echo, since it
# will raise errors if there are characters like \uFFFFD in the description
#
diff_description = StringUtils().to_ascii(diff_description)
click.echo(diff_description)
output = "Success"
click.echo(output)
click.echo(timer.elapsed_time_message())
except ApodeixiError as ex:
error_msg = CLI_ErrorReporting(kb_session).report_a6i_error(
parent_trace=root_trace, a6i_error=ex)
if kb_operation_succeeded:
error_msg = "KnowledgeBase operation completed, but run into a problem when preparing "\
+ "a description of the response:\n"\
+ error_msg
# GOTCHA
# Use print, not click.echo or click exception because they don't correctly display styling
# (colors, underlines, etc.). So use vanilla Python print and then exit
print(error_msg)
_sys.exit()
except Exception as ex:
try:
error_msg = CLI_ErrorReporting(kb_session).report_generic_error(
parent_trace=root_trace, generic_error=ex)
if kb_operation_succeeded:
error_msg = "KnowledgeBase operation completed, but run into a problem when preparing "\
+ "a description of the response:\n"\
+ error_msg
except Exception as ex2:
error_msg = "CLI run into trouble: found error:\n\n\t" + str(ex) + "\n\n" \
+ "To make things worse, when trying to produce an error log file with a "\
+ "stack trace, run into an additional error:\n\n\t" + str(ex2)
# GOTCHA
# Use print, not click.echo or click exception because they don't correctly display styling
# (colors, underlines, etc.). So use vanilla Python print and then exit
print(error_msg)
_sys.exit()
def form(kb_session, posting_api, namespace, subnamespace, dry_run,
environment, timestamp):
'''
Requests a form (an Excel spreadsheet) which (after some edits, as appropriate) can be used as the
input to the post command.
'''
timer = ApodeixiTimer()
func_trace = FunctionalTrace(parent_trace=None, path_mask=None)
root_trace = func_trace.doing("CLI call to post",
origination={'signaled_from': __file__})
kb_operation_succeeded = False
try:
# Catch warnings and handle them so that we avoid spurious noise in the CLI due to noisy 3rd party libraries
with warnings.catch_warnings(record=True) as w:
WarningUtils().turn_traceback_on(root_trace, warnings_list=w)
if environment != None:
kb_session.store.activate(parent_trace=root_trace,
environment_name=environment)
click.echo(CLI_Utils().sandox_announcement(environment))
elif dry_run == True:
sandbox_name = kb_session.provisionSandbox(root_trace)
click.echo(CLI_Utils().sandox_announcement(sandbox_name))
'''
else:
raise ApodeixiError(root_trace, "Sorry, only sandbox-isolated runs are supported at this time. Aborting.")
'''
# Now that we have pinned down the environment (sandbox or not) in which to call the KnowledgeBase's services,
# set that environment's tag to use for KnoweldgeBase's posting logs, if the user set it.
if timestamp:
kb_session.store.current_environment(root_trace).config(
root_trace).use_timestamps = timestamp
my_trace = root_trace.doing(
"Invoking KnowledgeBase's requestForm service")
output_dir = _os.getcwd()
clientURL = kb_session.store.getClientURL(my_trace)
relative_path, void = PathUtils().relativize(parent_trace=my_trace,
root_dir=clientURL,
full_path=output_dir)
form_request = kb_session.store.getBlindFormRequest(
parent_trace=my_trace,
relative_path=relative_path,
posting_api=posting_api,
namespace=namespace,
subnamespace=subnamespace)
response, log_txt, rep = kb_session.kb.requestForm(
parent_trace=my_trace, form_request=form_request)
kb_operation_succeeded = True
manifests_description = CLI_Utils().describe_req_form_response(
my_trace,
form_request_response=response,
store=kb_session.store,
representer=rep)
click.echo(manifests_description)
output = "Success"
click.echo(output)
click.echo(timer.elapsed_time_message())
WarningUtils().handle_warnings(root_trace, warning_list=w)
except ApodeixiError as ex:
error_msg = CLI_ErrorReporting(kb_session).report_a6i_error(
parent_trace=root_trace, a6i_error=ex)
if kb_operation_succeeded:
error_msg = "KnowledgeBase operation completed, but run into a problem when preparing "\
+ "a description of the response:\n"\
+ error_msg
# GOTCHA
# Use print, not click.echo or click exception because they don't correctly display styling
# (colors, underlines, etc.). So use vanilla Python print and then exit
print(error_msg)
_sys.exit()
except Exception as ex:
try:
error_msg = CLI_ErrorReporting(kb_session).report_generic_error(
parent_trace=root_trace, generic_error=ex)
if kb_operation_succeeded:
error_msg = "KnowledgeBase operation completed, but run into a problem when preparing "\
+ "a description of the response:\n"\
+ error_msg
except Exception as ex2:
error_msg = "CLI run into trouble: found error:\n\n\t" + str(ex) + "\n\n" \
+ "To make things worse, when trying to produce an error log file with a "\
+ "stack trace, run into an additional error:\n\n\t" + str(ex2)
# GOTCHA
# Use print, not click.echo or click exception because they don't correctly display styling
# (colors, underlines, etc.). So use vanilla Python print and then exit
print(error_msg)
_sys.exit()
def post(kb_session, file, dry_run, environment, timestamp):
'''
Posts contents of an Excel file to the KnowledgeBase.
The filename must be of the form '<some string><posting API>.xlsx' for some supported KnowledgeBase posting API.
'''
timer = ApodeixiTimer()
func_trace = FunctionalTrace(parent_trace=None, path_mask=None)
root_trace = func_trace.doing("CLI call to post",
origination={'signaled_from': __file__})
kb_operation_succeeded = False
try:
if environment != None:
kb_session.store.activate(parent_trace=root_trace,
environment_name=environment)
click.echo(CLI_Utils().sandox_announcement(environment))
elif dry_run == True:
sandbox_name = kb_session.provisionSandbox(root_trace)
click.echo(CLI_Utils().sandox_announcement(sandbox_name))
'''
else:
raise ApodeixiError(root_trace, "Sorry, only sandbox-isolated runs are supported at this time. Aborting.")
'''
# Now that we have pinned down the environment (sandbox or not) in which to call the KnowledgeBase's services,
# set that environment's tag to use for KnoweldgeBase's posting logs, if the user set it.
if timestamp:
kb_session.store.current_environment(root_trace).config(
root_trace).use_timestamps = timestamp
if len(
_os.path.split(file)[0]
) == 0: # This must be a local file in our working directory, no folder was given
file = _os.getcwd() + "/" + file
my_trace = root_trace.doing(
"Invoking KnowledgeBase's postByFile service")
response, log_txt = kb_session.kb.postByFile(
parent_trace=my_trace,
path_of_file_being_posted=file,
excel_sheet="Posting Label")
kb_operation_succeeded = True
manifests_description = CLI_Utils().describe_post_response(
my_trace, response, kb_session.store)
click.echo(manifests_description)
output = "Success"
click.echo(output)
click.echo(timer.elapsed_time_message())
except ApodeixiError as ex:
error_msg = CLI_ErrorReporting(kb_session).report_a6i_error(
parent_trace=root_trace, a6i_error=ex)
if kb_operation_succeeded:
error_msg = "KnowledgeBase operation completed, but run into a problem when preparing "\
+ "a description of the response:\n"\
+ error_msg
# GOTCHA
# Use print, not click.echo or click exception because they don't correctly display styling
# (colors, underlines, etc.). So use vanilla Python print and then exit
print(error_msg)
_sys.exit()
except Exception as ex:
try:
error_msg = CLI_ErrorReporting(kb_session).report_generic_error(
parent_trace=root_trace, generic_error=ex)
if kb_operation_succeeded:
error_msg = "KnowledgeBase operation completed, but run into a problem when preparing "\
+ "a description of the response:\n"\
+ error_msg
except Exception as ex2:
error_msg = "CLI run into trouble: found error:\n\n\t" + str(ex) + "\n\n" \
+ "To make things worse, when trying to produce an error log file with a "\
+ "stack trace, run into an additional error:\n\n\t" + str(ex2)
# GOTCHA
# Use print, not click.echo or click exception because they don't correctly display styling
# (colors, underlines, etc.). So use vanilla Python print and then exit
print(error_msg)
_sys.exit()
def skeleton_test(self,
parent_trace,
cli_command_list,
output_cleanining_lambda,
when_to_check_environment=PER_COMMAND):
'''
@param when_to_check_environment A string enum, that determines how frequently to check the contents
of the environment as the CLI commands execulte. Possible values:
* CLI_Test_Skeleton.PER_COMMAND
* CLI_Test_Skeleton.ONLY_AT_END
* CLI_Test_Skeleton.NEVER
'''
ME = CLI_Test_Skeleton
try:
my_trace = self.trace_environment(parent_trace,
"Isolating test case")
if self.provisioned_env_name == None:
# This is the second time we provision the isolated environment, but now with a different context, i.e.,
# different self.a6i_config and different self.test_config_dict than the first time we provisioned
# an isolated environment, which was in self.setUp. See comments there. The environment provisioned
# here is a child of the one configured in self.setUp, and is fo
self.provisionIsolatedEnvironment(my_trace)
if when_to_check_environment == ME.PER_COMMAND:
self.check_environment_contents(my_trace)
self.provisioned_env_name = self.stack().store(
).current_environment(my_trace).name(my_trace)
else:
self.stack().store().activate(my_trace,
self.provisioned_env_name)
my_trace = self.trace_environment(
parent_trace,
"Invoking " + str(len(cli_command_list)) + " commands")
if True:
runner = CliRunner()
for raw_command_argv in cli_command_list:
# The raw_command_arv might include some lambdas that need to be evaluated not in order to
# determine the real argv to pass to the CLI. The reason there might be lambdas is that some
# parameters for some commands can only be determined after earlier commands are run, so they
# aren't known when the command list was defined, and only now that we have run prior commands
# can it be determined.
# Example:
# The sandbox to use, if flag "--sandbox" is on. That can only be known after
# self.sandbox is set, which happens when the first command runs.
def _unraw_param(param):
if callable(param):
return param()
else:
# As a precaution, make sure we return a string. Otherwise, if param is an int,
# click will later through some exception
return str(param)
# Note: two operations are being done here:
#
# 1) Replacing a "delayed parameter": a parameter that couldn't be given when the caller's code was
# written, but can at runtime, so the "delayed parameter" is a callable that, if called, would return
# the actual parameter to use. Example: the sandbox parameter, which is determined in the first
# post of the script and must be passed to all subsequent post commands so they continue the work
# in a common sandbox.
# 2) Filtering out nulls. That is a trick to enable the caller, for example, to use the same script
# for both dry runs and live runs. All the caller has to do is set the "--sandbox <sandbox>" to a
# value when using the script with a sandbox, and to None when doing it live.
command_argv = [
_unraw_param(param) for param in raw_command_argv
if param != None
]
loop_trace = self.trace_environment(
my_trace, "Executing '" +
" ".join([str(cmd) for cmd in command_argv]) + "'")
# Some Python libraries can be too noisy with warnings, and these get printed out to standard err/output
# where the CLI will regard as "part of output" and display them in regression test output. This makes
# regression output both ugly and sometimes non-deterministc.
# To remedy this, we change the warning context to catch all warnings and based on what we catch, either
# 1. raise an ApodeixiError so that the Apodeixi developer can change the code construct that led to the
# warning, possible as the ApodeixiError will include a stack trace to pin point where in the Apodeixi
# code the warning was triggered,
# 2. or ignore the warning if that is pure noise and no code change in Apodeixi could prevent it from being
# triggered
#
with warnings.catch_warnings(record=True) as w:
WarningUtils().turn_traceback_on(parent_trace,
warnings_list=w)
result = runner.invoke(self.cli, command_argv)
WarningUtils().handle_warnings(parent_trace,
warning_list=w)
if result.exit_code != 0:
raise ApodeixiError(loop_trace,
"CLI command failed",
data={
"CLI exit code":
str(result.exit_code),
"CLI exception":
str(result.exc_info),
"CLI output":
str(result.output),
"CLI traceback":
str(result.exc_info)
})
sandbox = CLI_Utils().infer_sandbox_name(
loop_trace, result.output)
if sandbox != None:
# We only overwrite self.sandbox if this particular command chose a sandbox. Otherwise
# we retain whatever self.sandbox was set by prior commands. This is important since some commands
# don't have a --sandbox option (Example: get namespaces), but that does not mean that
# our intention is to switch out of the sandbox and into the parent environment.
self.sandbox = sandbox
command_flags = [
token for token in command_argv
if token.startswith("--")
]
if command_argv[0] in ["post"]:
argv_without_arguments = command_argv[:1]
elif command_argv[0] in ["get"]:
argv_without_arguments = command_argv[:2]
elif command_argv[0] in ["import"]:
argv_without_arguments = command_argv[:2]
elif command_argv[0] in ["diff"]:
argv_without_arguments = command_argv[:1]
else:
raise ApodeixiError(
my_trace, "Command not recognized: '" +
str(command_argv[0]) + "'")
argv_without_arguments.extend(
command_flags) # Like post --dry-run
# Once we are done building it, command_without_flag_params will be something like
#
# => post --dry-run products.static-data.admin.a6i.xlsx
#
# or
#
# => post --sandbox products.static-data.admin.a6i.xlsx
#
# hence it will be suitable for inclusion in deterministic output. For example, we remove
# timestamp-sensitive sandbox names (if any) and also the full path for the posted file.
command_without_flag_params = " ".join(
argv_without_arguments)
if command_argv[0] in ["post"] or command_argv[:2] in [[
"get", "form"
]]:
# These are commands with a unique argument. Other commands lack it
path_posted = command_argv[-1]
unique_argument = _os.path.split(path_posted)[1]
command_without_flag_params += " " + unique_argument
elif command_argv[:2] in [["import", "aha"]]:
args = command_argv[-4:]
command_without_flag_params += " " + " ".join(args)
output_to_display = "=> " + command_without_flag_params + "\n\n"
if output_cleanining_lambda == None:
output_to_display += result.output
else:
output_to_display += output_cleanining_lambda(
result.output)
self.check_cli_output(
parent_trace=loop_trace,
cli_output=output_to_display,
cli_command=" ".join(
argv_without_arguments) # Like post --dry-run
)
if when_to_check_environment == ME.PER_COMMAND:
self._check_CLI_environment(loop_trace)
if when_to_check_environment == ME.ONLY_AT_END:
# We display the consolidated effect of all commands in the script onto the KnowledgeBase used by the CLI
self._check_CLI_environment(my_trace)
my_trace = self.trace_environment(parent_trace,
"Deactivating environment")
self.stack().store().deactivate(my_trace)
except ApodeixiError as ex:
click.echo(ex.trace_message())
self.assertTrue(1 == 2)
def run_script(self, parent_trace, SANDBOX_FUNC, cli_arguments_dict):
_path_of = self.myTest.fullpath_of
MASK_COMBINED = CLI_Utils().combined_mask(parent_trace,
self.myTest.a6i_config)
_s = CLI_Basic_Script
_args = cli_arguments_dict
# This will fool the CLI to treat our provisioned environment for this test as if it were the base environment
self.myTest.overwrite_test_context(
parent_trace
) # Overwrites self.a6i_config , the store, the test_db, etc.
if SANDBOX_FUNC != None:
__dry_run = '--dry-run'
__environment = '--environment'
ENV_CHOICE = SANDBOX_FUNC
else: # Case for life runs
__dry_run = None
__environment = None
ENV_CHOICE = None
COMMANDS = [
[
'post', __dry_run, '--timestamp', "_CLI__1",
_path_of(_args[_s.PRODUCT_FILE])
],
[
'post', __environment, ENV_CHOICE, '--timestamp', "_CLI__2",
_path_of(_args[_s.SCORING_CYCLE_FILE])
],
[
'post', __environment, ENV_CHOICE, '--timestamp', "_CLI__3",
_path_of(_args[_s.BIG_ROCKS_v1_FILE])
],
['get', 'products', __environment, ENV_CHOICE],
['get', 'scoring-cycles', __environment, ENV_CHOICE],
['get', 'namespaces'],
#['get', 'environments'], # Can't test- environment count non-deterministic
['get', 'apis'],
]
self.myTest.skeleton_test(parent_trace=parent_trace,
cli_command_list=COMMANDS,
output_cleanining_lambda=MASK_COMBINED)
# For the next test, we need to switch the working directory for click
my_trace = parent_trace.doing(
"Running with working directory in the collaboration area")
store = self.myTest.stack().store()
if self.myTest.sandbox != None:
root_dir = _os.path.dirname(
store.base_environment(my_trace).manifestsURL(my_trace))
envs_dir = root_dir + "/" + File_KBEnv_Impl.ENVS_FOLDER
working_dir = envs_dir + "/" + self.myTest.sandbox + "/external-collaboration/" \
+ _args[_s.REL_PATH_IN_EXT_COLLABORATION]
else:
clientURL = store.base_environment(my_trace).clientURL(my_trace)
working_dir = clientURL + "/" + _args[
_s.REL_PATH_IN_EXT_COLLABORATION]
PathUtils().create_path_if_needed(parent_trace, working_dir)
_os.chdir(working_dir)
COMMANDS_2 = [[
'get', 'form', __environment, ENV_CHOICE, '--timestamp', "_CLI__4",
_args[_s.BIG_ROCKS_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
],
[
'post', __environment, ENV_CHOICE, '--timestamp',
"_CLI__5",
_path_of(_args[_s.BIG_ROCKS_v2_FILE])
],
[
'get', 'form', __environment, ENV_CHOICE,
'--timestamp', "_CLI__6",
_args[_s.BIG_MILESTONES_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
],
[
'post', __environment, ENV_CHOICE, '--timestamp',
"_CLI__7",
_path_of(_args[_s.BIG_MILESTONES_v1_FILE])
],
[
'get', 'form', __environment, ENV_CHOICE,
'--timestamp', "_CLI__8",
_args[_s.BIG_MILESTONES_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
],
[
'post', __environment, ENV_CHOICE, '--timestamp',
"_CLI__9",
_path_of(_args[_s.BIG_MILESTONES_v2_FILE])
],
[
'get', 'form', __environment, ENV_CHOICE,
'--timestamp', "_CLI__10",
_args[_s.BIG_MILESTONES_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
], ['get', 'assertions', __environment, ENV_CHOICE]]
self.myTest.skeleton_test(parent_trace=parent_trace,
cli_command_list=COMMANDS_2,
output_cleanining_lambda=MASK_COMBINED)
def test_milestones_referenced_big_rock_version(self):
'''
Tests that integrity checks exist to prevent posting a milestones manifest if it references a
version of the big rocks that is not the latest.
'''
try:
self.setScenario("foreign_key.milestones_big_rock_version")
self.setCurrentTestName('fkey.ml_2_br')
self.selectTestDataLocation()
root_trace = FunctionalTrace(parent_trace=None, path_mask=self._path_mask) \
.doing("Running " + self.currentTestName())
PRODUCT_FILE = "products.static-data.admin.a6i.xlsx"
SCORING_CYCLE_FILE = "scoring-cycles.static-data.admin.a6i.xlsx"
BIG_ROCKS_v1_FILE = "opus.v1.big-rocks.journeys.a6i.xlsx"
BIG_ROCKS_v2_FILE = "opus.v2.big-rocks.journeys.a6i.xlsx"
MILESTONES_FILE = "opus.v1.milestone.journeys.a6i.xlsx"
BIG_ROCKS_API = "big-rocks.journeys.a6i"
MILESTONES_API = "milestone.journeys.a6i"
NAMESPACE = "my-corp.production"
SUB_NAMESPACE = "modernization"
REL_PATH_IN_EXT_COLLABORATION = "journeys/Dec 2020/FusionOpus/Default"
_path_of = self.fullpath_of
MASK_COMBINED = CLI_Utils().combined_mask(root_trace,
self.a6i_config)
# This will fool the CLI to treat our provisioned environment for this test as if it were the base environment
self.overwrite_test_context(
root_trace
) # Overwrites self.a6i_config , the store, the test_db, etc.
# For this test, we need to switch the working directory for click
my_trace = root_trace.doing(
"Running with working directory in the collaboration area")
store = self.stack().store()
clientURL = store.base_environment(my_trace).clientURL(my_trace)
working_area = clientURL + "/" + REL_PATH_IN_EXT_COLLABORATION
PathUtils().create_path_if_needed(my_trace, working_area)
_os.chdir(working_area)
COMMANDS = [
['post', '--timestamp', "_CLI__1",
_path_of(PRODUCT_FILE)],
[
'post', '--timestamp', "_CLI__2",
_path_of(SCORING_CYCLE_FILE)
],
[
'post', '--timestamp', "_CLI__3",
_path_of(BIG_ROCKS_v1_FILE)
], # v1 of big-rocks
[
'get', 'form', '--timestamp', "_CLI__4", MILESTONES_API,
NAMESPACE, SUB_NAMESPACE
], # milestones -> big-rocks v1
[
'get', 'form', '--timestamp', "_CLI__5", BIG_ROCKS_API,
NAMESPACE, SUB_NAMESPACE
],
[
'post', '--timestamp', "_CLI__6",
_path_of(BIG_ROCKS_v2_FILE)
], # v2 of big-rocks
['get', 'assertions'],
['post', '--timestamp', "_CLI__7",
_path_of(MILESTONES_FILE)], # Should trigger an error
]
self.skeleton_test(
parent_trace=my_trace,
cli_command_list=COMMANDS,
output_cleanining_lambda=MASK_COMBINED,
when_to_check_environment=CLI_Test_Skeleton.NEVER)
except ApodeixiError as ex:
print(ex.trace_message())
self.assertTrue(1 == 2)
def run_script(self, parent_trace, SANDBOX_FUNC, cli_arguments_dict):
_path_of = self.myTest.fullpath_of
MASK_COMBINED = CLI_Utils().combined_mask(parent_trace,
self.myTest.a6i_config)
_s = CLI_BigRocks_and_Milestones_Script
_args = cli_arguments_dict
# This will fool the CLI to treat our provisioned environment for this test as if it were the base environment
self.myTest.overwrite_test_context(
parent_trace
) # Overwrites self.a6i_config , the store, the test_db, etc.
my_trace = parent_trace.doing(
"Running with working directory in the collaboration area")
store = self.myTest.stack().store()
if self.myTest.sandbox != None:
root_dir = _os.path.dirname(
store.base_environment(my_trace).manifestsURL(my_trace))
envs_dir = root_dir + "/" + File_KBEnv_Impl.ENVS_FOLDER
working_dir = envs_dir + "/" + self.myTest.sandbox + "/external-collaboration/" \
+ _args[_s.REL_PATH_IN_EXT_COLLABORATION]
else:
clientURL = store.base_environment(my_trace).clientURL(my_trace)
working_dir = clientURL + "/" + _args[
_s.REL_PATH_IN_EXT_COLLABORATION]
PathUtils().create_path_if_needed(parent_trace, working_dir)
_os.chdir(working_dir)
if SANDBOX_FUNC != None:
__dry_run = '--dry-run'
__environment = '--environment'
ENV_CHOICE = SANDBOX_FUNC
else: # Case for life runs
__dry_run = None
__environment = None
ENV_CHOICE = None
COMMANDS_1 = [
# Initialize static data
[
'post', __dry_run, '--timestamp', "_CLI__1",
_path_of(_args[_s.PRODUCT_FILE])
],
[
'post', __environment, ENV_CHOICE, '--timestamp', "_CLI__2",
_path_of(_args[_s.SCORING_CYCLE_FILE])
],
# Create big rocks v1
[
'get', 'form', __environment, ENV_CHOICE, '--timestamp',
"_CLI__3", _args[_s.BIG_ROCKS_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
],
[
'post', __environment, ENV_CHOICE, '--timestamp', "_CLI__4",
_path_of(_args[_s.BIG_ROCKS_v1_FILE])
],
# Create milestones v1
[
'get', 'form', __environment, ENV_CHOICE, '--timestamp',
"_CLI__5", _args[_s.BIG_MILESTONES_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
],
[
'post', __environment, ENV_CHOICE, '--timestamp', "_CLI__6",
_path_of(_args[_s.BIG_MILESTONES_v1_FILE])
],
]
self.myTest.skeleton_test(
parent_trace=my_trace,
cli_command_list=COMMANDS_1,
output_cleanining_lambda=MASK_COMBINED,
when_to_check_environment=CLI_Test_Skeleton.NEVER)
# Check that manifest is as expected
NAME = "experimental.march-2021.turbotax.iot-experiment"
NAMESPACE = "intuit.innovations"
self.myTest.check_manifest(my_trace,
'delivery-planning.journeys.a6i.io',
NAMESPACE, NAME, 'big-rock')
self.myTest.check_manifest(my_trace,
'delivery-planning.journeys.a6i.io',
NAMESPACE, NAME, 'big-rock-estimate')
self.myTest.check_manifest(my_trace,
'delivery-planning.journeys.a6i.io',
NAMESPACE, NAME, 'modernization-milestone')
COMMANDS_2 = [
# First try to update big rocks v2 - should fail due to foreign key constraints
[
'get', 'form', __environment, ENV_CHOICE, '--timestamp',
"_CLI__7", _args[_s.BIG_ROCKS_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
],
[
'post', __environment, ENV_CHOICE, '--timestamp', "_CLI__8",
_path_of(_args[_s.BIG_ROCKS_v2_FILE])
],
# Update milestones v2 - should remove the reference that caused big rocks v2 to fail
[
'get', 'form', __environment, ENV_CHOICE, '--timestamp',
"_CLI__9", _args[_s.BIG_MILESTONES_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
],
[
'post', __environment, ENV_CHOICE, '--timestamp', "_CLI__10",
_path_of(_args[_s.BIG_MILESTONES_v2_FILE])
],
# Second try to update big rocks v2 - should work now that user removed references in
# milestones v2 to the rocks that were removed in v2
[
'post', __environment, ENV_CHOICE, '--timestamp', "_CLI__11",
_path_of(_args[_s.BIG_ROCKS_v2_FILE])
],
]
self.myTest.skeleton_test(
parent_trace=my_trace,
cli_command_list=COMMANDS_2,
output_cleanining_lambda=MASK_COMBINED,
when_to_check_environment=CLI_Test_Skeleton.NEVER)
# Check that manifest is as expected
self.myTest.check_manifest(my_trace,
'delivery-planning.journeys.a6i.io',
NAMESPACE, NAME, 'big-rock')
self.myTest.check_manifest(my_trace,
'delivery-planning.journeys.a6i.io',
NAMESPACE, NAME, 'big-rock-estimate')
self.myTest.check_manifest(my_trace,
'delivery-planning.journeys.a6i.io',
NAMESPACE, NAME, 'modernization-milestone')
COMMANDS_3 = [
# Get final forms
[
'get', 'form', __environment, ENV_CHOICE, '--timestamp',
"_CLI__12", _args[_s.BIG_ROCKS_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
],
[
'get', 'form', __environment, ENV_CHOICE, '--timestamp',
"_CLI__13", _args[_s.BIG_MILESTONES_API], _args[_s.NAMESPACE],
_args[_s.SUB_NAMESPACE]
],
# Summarize assertions created
['get', 'assertions', __environment, ENV_CHOICE]
]
self.myTest.skeleton_test(
parent_trace=my_trace,
cli_command_list=COMMANDS_3,
output_cleanining_lambda=MASK_COMBINED,
when_to_check_environment=CLI_Test_Skeleton.ONLY_AT_END)
def run_script(self, parent_trace, SANDBOX_FUNC, cli_arguments_dict):
_path_of = self.myTest.fullpath_of
MASK_COMBINED = CLI_Utils().combined_mask(parent_trace, self.myTest.a6i_config)
_s = CLI_StaticData_Script
_args = cli_arguments_dict
# This will fool the CLI to treat our provisioned environment for this test as if it were the base environment
self.myTest.overwrite_test_context(parent_trace) # Overwrites self.a6i_config , the store, the test_db, etc.
if SANDBOX_FUNC != None:
__dry_run = '--dry-run'
__environment = '--environment'
ENV_CHOICE = SANDBOX_FUNC
else: # Case for life runs
__dry_run = None
__environment = None
ENV_CHOICE = None
# For the next test, we need to switch the working directory for click
my_trace = parent_trace.doing("Running with working directory in the collaboration area")
store = self.myTest.stack().store()
if self.myTest.sandbox != None:
root_dir = _os.path.dirname(store.base_environment(my_trace).manifestsURL(my_trace))
envs_dir = root_dir + "/" + File_KBEnv_Impl.ENVS_FOLDER
working_dir = envs_dir + "/" + self.myTest.sandbox + "/external-collaboration/" \
+ _args[_s.REL_PATH_IN_EXT_COLLABORATION]
else:
clientURL = store.base_environment(my_trace).clientURL(my_trace)
working_dir = clientURL + "/" + _args[_s.REL_PATH_IN_EXT_COLLABORATION]
PathUtils().create_path_if_needed(parent_trace, working_dir)
_os.chdir(working_dir)
COMMANDS = [
['get', 'form', __environment, ENV_CHOICE, '--timestamp', "_CLI__1",
_args[_s.STATIC_DATA_API], _args[_s.NAMESPACE]],
['post', __dry_run, '--timestamp', "_CLI__2",
_path_of(_args[_s.STATIC_DATA_v1_FILE])],
['get', 'form', __environment, ENV_CHOICE, '--timestamp', "_CLI__3",
_args[_s.STATIC_DATA_API], _args[_s.NAMESPACE]],
['post', __environment, ENV_CHOICE, '--timestamp', "_CLI__4",
_path_of(_args[_s.STATIC_DATA_v2_FILE])],
['get', 'form', __environment, ENV_CHOICE, '--timestamp', "_CLI__5",
_args[_s.STATIC_DATA_API], _args[_s.NAMESPACE]],
]
self.myTest.skeleton_test( parent_trace = parent_trace,
cli_command_list = COMMANDS,
output_cleanining_lambda = MASK_COMBINED,
when_to_check_environment = CLI_Test_Skeleton.ONLY_AT_END)
def test_cli_rollover_basic(self):
'''
Tests rollover functionality when traversing fiscal years: if there is no previous manifest for FY 23 but one
exists from FY 22, then trying to retrieve the latest manifest in FY 23 will result in the latest one from FY 22,
as opposed to a template-based default.
'''
self.setScenario("cli.basic_rollover")
self.setCurrentTestName('basic_rollover')
self.selectTestDataLocation()
root_trace = FunctionalTrace(parent_trace=None, path_mask=self._path_mask) \
.doing("Running " + self.currentTestName())
PRODUCTS_FILE = 'products.static-data.admin.a6i.xlsx'
SCORING_CYCLES_FILE = 'scoring-cycles.static-data.admin.a6i.xlsx'
BIG_ROCKS_FILE_V1 = 'v1.big-rocks.journeys.a6i.xlsx'
BIG_ROCKS_FILE_V2 = 'v2.big-rocks.journeys.a6i.xlsx'
BIG_ROCKS_POSTING_API = 'big-rocks.journeys.a6i'
BIG_ROCKS_MANIFEST_API = "delivery-planning.journeys.a6i.io" #/v1a"
NAMESPACE = 'cicloquimica.production'
FY23_NAME = "modernization.fy-23.astrea.official"
SUB_NAMESPACE = 'modernization'
FY22_REL_PATH_IN_EXT_COLLAB = "journeys/FY 22/Astrea/Official"
FY23_REL_PATH_IN_EXT_COLLAB = "journeys/FY 23/Astrea/Official"
FY23_GENERATED_FORM_FILE = "Astrea.modernization.big-rocks.journeys.a6i.xlsx"
STATIC_DATA_WORKING_DIR = "admin/static-data"
_path_of = self.fullpath_of
MASK_COMBINED = CLI_Utils().combined_mask(root_trace, self.a6i_config)
# This will fool the CLI to treat our provisioned environment for this test as if it were the base environment
self.overwrite_test_context(
root_trace
) # Overwrites self.a6i_config , the store, the test_db, etc.
store = self.stack().store()
clientURL = store.base_environment(root_trace).clientURL(root_trace)
# Post the static data
my_trace = root_trace.doing("Running with working directory '" +
STATIC_DATA_WORKING_DIR + "'")
working_dir = clientURL + "/" + STATIC_DATA_WORKING_DIR
PathUtils().create_path_if_needed(my_trace, working_dir)
_os.chdir(working_dir)
COMMANDS_1 = [
['post', '--timestamp', "_CLI__1",
_path_of(PRODUCTS_FILE)],
['post', '--timestamp', "_CLI__2",
_path_of(SCORING_CYCLES_FILE)],
]
self.skeleton_test(parent_trace=my_trace,
cli_command_list=COMMANDS_1,
output_cleanining_lambda=MASK_COMBINED,
when_to_check_environment=CLI_Test_Skeleton.NEVER)
# Post the FY 22 big rocks
my_trace = root_trace.doing("Running with working directory '" +
FY22_REL_PATH_IN_EXT_COLLAB + "'")
fy22_working_dir = clientURL + "/" + FY22_REL_PATH_IN_EXT_COLLAB
PathUtils().create_path_if_needed(my_trace, fy22_working_dir)
_os.chdir(fy22_working_dir)
COMMANDS_2 = [[
'post', '--timestamp', "_CLI__3",
_path_of(BIG_ROCKS_FILE_V1)
], ['post', '--timestamp', "_CLI__4",
_path_of(BIG_ROCKS_FILE_V2)],
[
'get', 'form', '--timestamp', "_CLI__5",
BIG_ROCKS_POSTING_API, NAMESPACE, SUB_NAMESPACE
]]
self.skeleton_test(parent_trace=my_trace,
cli_command_list=COMMANDS_2,
output_cleanining_lambda=MASK_COMBINED,
when_to_check_environment=CLI_Test_Skeleton.NEVER)
# For the next test, we need to switch the working directory for click
my_trace = root_trace.doing("Running with working directory '" +
FY23_REL_PATH_IN_EXT_COLLAB + "'")
fy23_working_dir = clientURL + "/" + FY23_REL_PATH_IN_EXT_COLLAB
PathUtils().create_path_if_needed(my_trace, fy23_working_dir)
_os.chdir(fy23_working_dir)
COMMANDS_3 = [
[
'get', 'form', '--timestamp', "_CLI__6", BIG_ROCKS_POSTING_API,
NAMESPACE, SUB_NAMESPACE
],
[
'post', '--timestamp', "_CLI__7",
fy23_working_dir + "/" + FY23_GENERATED_FORM_FILE
],
]
self.skeleton_test(parent_trace=my_trace,
cli_command_list=COMMANDS_3,
output_cleanining_lambda=MASK_COMBINED,
when_to_check_environment=CLI_Test_Skeleton.NEVER)
# Check that manifest is as expected right after a first FY 23 manifest is created from a rollover
# For example, we expect its labels to include the rollFromName field to indicate its lineage as a continuation
# of FY 22
self.check_manifest(my_trace, BIG_ROCKS_MANIFEST_API, NAMESPACE,
FY23_NAME, 'big-rock')
# Now we do a full cycle get-form + post, to see that the new lineage in FY 23 can take off as expected
COMMANDS_4 = [
[
'get', 'form', '--timestamp', "_CLI__8", BIG_ROCKS_POSTING_API,
NAMESPACE, SUB_NAMESPACE
],
[
'post', '--timestamp', "_CLI__9",
fy23_working_dir + "/" + FY23_GENERATED_FORM_FILE
],
]
self.skeleton_test(
parent_trace=my_trace,
cli_command_list=COMMANDS_4,
output_cleanining_lambda=MASK_COMBINED,
when_to_check_environment=CLI_Test_Skeleton.ONLY_AT_END)
# Check that manifest is as expected right after we are no longer in a rollover situation, and are modifying
# previous manifests in FY 23.
#
# For example, it should no longer be the case that its labels include the rollFromName field, as it used to
# be the case when we created the first FY 23 manifest, since only that first manifested needed to
# to indicate its lineage as a continuation of FY 22
#
self.check_manifest(my_trace, BIG_ROCKS_MANIFEST_API, NAMESPACE,
FY23_NAME, 'big-rock')