def print_configs(self,
configs: List[Config],
environment: Optional[MasonEnvironment] = None):
configs.sort(key=lambda o: o.id)
current_config: Optional[str] = None
if environment:
current_config = environment.state_store.get_session_config()
if len(configs) == 1:
str_resp = json.dumps(configs[0].to_dict(current_config),
indent=4,
sort_keys=False)
logger.info(highlight(str_resp, JsonLexer(), TerminalFormatter()))
elif len(configs) > 0:
to_values = list(
map(lambda c: c.extended_info(current_config), configs))
banner(f"Available Configs")
logger.info()
logger.info(
tabulate(to_values,
headers=[
"id", "execution", "metastore", "storage",
"scheduler"
]))
logger.info()
if current_config:
logger.info("* Current Session Configuration")
else:
logger.error(
"No configs. Register configs by running \"mason apply\"")
def print_response(self, response: Response):
def default(o):
if isinstance(o, (datetime.date, datetime.datetime)):
return o.isoformat()
resp, status = response.with_status()
logger.info(f"Response status: {status}")
str_resp = json.dumps(resp, indent=4, sort_keys=True, default=default)
logger.info(highlight(str_resp, JsonLexer(), TerminalFormatter()))
def print_workflows(self,
workflows: List[Workflow],
namespace: Optional[str] = None,
command: Optional[str] = None):
workflows.sort(key=lambda o: o.namespace)
if len(workflows) == 1:
str_resp = json.dumps(workflows[0].to_dict(),
indent=4,
sort_keys=False)
logger.info(highlight(str_resp, JsonLexer(), TerminalFormatter()))
elif len(workflows) > 0:
to_values = list(
map(lambda wf: [wf.namespace, wf.command, wf.description],
workflows))
namesp = f"Workflow "
if namespace:
namesp += f"{namespace}"
banner(f"Available {namesp} Methods")
logger.info()
logger.info(
tabulate(to_values,
headers=["namespace", "command", "description"]))
logger.info()
else:
logger.error(
"No workflows registered. Register operators by running \"mason apply\""
)
def infer_table(
self,
path: str,
name: Optional[str],
options: Optional[dict] = None,
resp: Optional[Response] = None
) -> Tuple[Union[Table, InvalidTables], Response]:
opt = options or {}
logger.info(f"Fetching keys at {path}")
response: Response = resp or Response()
path = self.get_path(path).full_path()
keys, response = self.list_keys(path, response)
logger.debug(f"{len(keys)} keys at {path}")
final: Union[Table, InvalidTables]
sample_size = opt.get("sample_size")
if sample_size:
import random
try:
ss = int(sample_size)
except TypeError:
logger.warning(f"Invalid sample size (int): {sample_size}")
ss = 3
logger.warning(
f"Sampling keys to determine schema. Sample size: {ss}.")
if ss < len(keys):
keys = random.sample(keys, ss)
if len(keys) > 0:
try:
valid, invalid_schemas = sequence(
list(
map(
lambda key: schemas.from_file(
self.client().open(key.full_path()), opt),
keys)), Schema, InvalidSchema)
non_empty = [
v for v in valid if not isinstance(v, EmptySchema)
]
validated, paths = CheckSchemas.find_conflicts(non_empty)
table = CheckSchemas.get_table(self.get_name(name, path),
validated, paths)
invalid_tables = list(
map(
lambda i: InvalidTable("Invalid Schema",
invalid_schema=i),
invalid_schemas))
if isinstance(table, Table):
final = table
else:
invalid_tables.append(table)
final = InvalidTables(invalid_tables)
except (ClientError, PermissionError) as e:
final = InvalidTables(
[InvalidTable(f"Not able to infer table: {message(e)}")])
else:
response.set_status(404)
final = InvalidTables([TableNotFound(f"No keys at {path}")])
return final, response
def add_info(self, info: str, log: bool = True):
info = self.add_timestamp(info)
if log:
logger.info(str(info))
self.info.append(info)