def test_file_not_supported(self):
logger.set_level("error")
fs = LocalFileSystem()
with fs.open(from_root('/test/sample_data/unsupported_file_type.usf')) as f:
schema = from_file(f)
assert(isinstance(schema, InvalidSchema))
assert(schema.reason[0:32] == f"File type not supported for file")
def run_around_tests(self):
logger.set_level("fatal")
try:
yield
finally:
cc = from_root("/test/support/configs/CURRENT_CONFIG")
if path.exists(cc):
remove(cc)
def apply(file: str, overwrite: bool = False, log_level: Optional[str] = None, env: Optional[MasonEnvironment] = None):
environment: MasonEnvironment = env or MasonEnvironment().initialize()
logger.set_level(log_level)
response = Response()
all = Resources(environment).get_all(file)
for r in all:
response = r.save(environment.state_store, overwrite, response)
return response.with_status()
def test_basic_json_schema_invalid(self):
logger.set_level("fatal")
obj = object_from_json_schema({
"test": "test",
"type": "test"
}, BASIC_SCHEMA, BadTest)
if isinstance(obj, InvalidSchemaDict):
assert (obj.__class__.__name__ == "InvalidSchemaDict")
assert (obj.reason[0:27] == "Object creation failed for ")
def run_tests(namespace: str, command: str, do_mock: bool, log_level: str,
configs: List[str], callable, *args, **kwargs):
logger.set_level(log_level)
env = get_env()
load_dotenv(from_root("/../.env.example"))
workflow = kwargs.get("workflow") or False
if do_mock:
patches = get_patches()
with contextlib.ExitStack() as stack:
for p in patches:
stack.enter_context(p)
run_test(env, namespace, command, configs, workflow, callable)
else:
run_test(env, namespace, command, configs, workflow, callable)
def validate_workflow(
self, env: MasonEnvironment, command: str, config_id: str,
params: dict) -> Union[ValidWorkflow, InvalidWorkflow]:
logger.set_level("fatal")
res = base.Resources(env)
wf = res.get_workflow("testing_namespace", command)
config = res.get_config(config_id)
if wf and config and isinstance(wf, Workflow) and isinstance(
config, Config):
parameters = WorkflowParameters(parameter_dict=params)
validated = wf.validate(env, config, parameters)
return validated
else:
raise Exception("Invalid workflow or config")
def get(resource_type: Optional[str],
namespace: Optional[str] = None,
command: Optional[str] = None,
log_level: Optional[str] = "info",
env: Optional[MasonEnvironment] = None,
printer: Printer = ApiPrinter()):
environment: MasonEnvironment = env or MasonEnvironment().initialize()
logger.set_level(log_level)
resource_type = resource_type or "all"
res = base.Resources(environment)
all = res.get_resources(resource_type, namespace, command)
response = printer.print_resources(all,
resource_type,
namespace,
command,
environment=environment)
return response.with_status()
def run(resource_type: str,
namespace: str,
command: str,
parameter_string: Optional[str] = None,
param_file: Optional[str] = None,
config_id: Optional[str] = None,
log_level: Optional[str] = None,
env: Optional[MasonEnvironment] = None,
dry_run: bool = False,
parameters: Optional[dict] = None,
printer=ApiPrinter()):
response = Response()
environment: MasonEnvironment = env or MasonEnvironment().initialize()
logger.set_level(log_level)
res = base.Resources(environment)
resource: Union[Resource, MalformedResource] = res.get_resource(
resource_type, namespace, command)
config: Union[Config, MalformedResource] = res.get_best_config(config_id)
params: Union[Parameters, MalformedResource] = res.get_parameters(
resource_type, parameter_string, param_file, parameters)
if isinstance(resource, Resource) and isinstance(
config, Config) and isinstance(params, Parameters):
if dry_run:
response = validate_resource(resource, config, params,
environment).dry_run(
environment,
response).to_response(response)
else:
response = validate_resource(resource, config, params,
environment).run(
environment,
response).to_response(response)
else:
if isinstance(resource, MalformedResource):
response.add_error(f"Malformed Resource: {resource.get_message()}")
elif isinstance(config, MalformedResource):
response.add_error(f"Bad Config: {config.get_message()}")
elif isinstance(params, MalformedResource):
response.add_error(f"Bad Parameters: {params.get_message()}")
return printer.print_response(response)
def config(config_id: Optional[str],
set_current: bool = False,
log_level: Optional[str] = None,
env: Optional[MasonEnvironment] = None,
printer: Printer = ApiPrinter()):
environment = env or MasonEnvironment().initialize()
logger.set_level(log_level)
response = Response()
if set_current and config_id:
result = Resources(environment).set_session_config(config_id)
if isinstance(result, str):
response.add_error(result)
response.set_status(404)
else:
response.add_info(f"Set session config to {config_id}")
config_id = None
res = Resources(environment)
configs = res.get_resources("config", config_id)
response = printer.print_resources(configs,
"config",
config_id,
environment=environment)
return response.with_status()
def test_snappy_parquet_schema_support(self):
logger.set_level("info")
fs = LocalFileSystem()
with fs.open(from_root('/test/sample_data/sample.snappy.parquet')) as f:
schema = from_file(f)
assert(isinstance(schema, ParquetSchema))
def set_log_level(level: str = None):
logger.set_level(level or "fatal", False)