def print_configs(self, configs: List[Config], environment: Optional[MasonEnvironment] = None): configs.sort(key=lambda o: o.id) current_config: Optional[str] = None if environment: current_config = environment.state_store.get_session_config() if len(configs) == 1: str_resp = json.dumps(configs[0].to_dict(current_config), indent=4, sort_keys=False) logger.info(highlight(str_resp, JsonLexer(), TerminalFormatter())) elif len(configs) > 0: to_values = list( map(lambda c: c.extended_info(current_config), configs)) banner(f"Available Configs") logger.info() logger.info( tabulate(to_values, headers=[ "id", "execution", "metastore", "storage", "scheduler" ])) logger.info() if current_config: logger.info("* Current Session Configuration") else: logger.error( "No configs. Register configs by running \"mason apply\"")
def print_response(self, response: Response): def default(o): if isinstance(o, (datetime.date, datetime.datetime)): return o.isoformat() resp, status = response.with_status() logger.info(f"Response status: {status}") str_resp = json.dumps(resp, indent=4, sort_keys=True, default=default) logger.info(highlight(str_resp, JsonLexer(), TerminalFormatter()))
def print_workflows(self, workflows: List[Workflow], namespace: Optional[str] = None, command: Optional[str] = None): workflows.sort(key=lambda o: o.namespace) if len(workflows) == 1: str_resp = json.dumps(workflows[0].to_dict(), indent=4, sort_keys=False) logger.info(highlight(str_resp, JsonLexer(), TerminalFormatter())) elif len(workflows) > 0: to_values = list( map(lambda wf: [wf.namespace, wf.command, wf.description], workflows)) namesp = f"Workflow " if namespace: namesp += f"{namespace}" banner(f"Available {namesp} Methods") logger.info() logger.info( tabulate(to_values, headers=["namespace", "command", "description"])) logger.info() else: logger.error( "No workflows registered. Register operators by running \"mason apply\"" )
def infer_table( self, path: str, name: Optional[str], options: Optional[dict] = None, resp: Optional[Response] = None ) -> Tuple[Union[Table, InvalidTables], Response]: opt = options or {} logger.info(f"Fetching keys at {path}") response: Response = resp or Response() path = self.get_path(path).full_path() keys, response = self.list_keys(path, response) logger.debug(f"{len(keys)} keys at {path}") final: Union[Table, InvalidTables] sample_size = opt.get("sample_size") if sample_size: import random try: ss = int(sample_size) except TypeError: logger.warning(f"Invalid sample size (int): {sample_size}") ss = 3 logger.warning( f"Sampling keys to determine schema. Sample size: {ss}.") if ss < len(keys): keys = random.sample(keys, ss) if len(keys) > 0: try: valid, invalid_schemas = sequence( list( map( lambda key: schemas.from_file( self.client().open(key.full_path()), opt), keys)), Schema, InvalidSchema) non_empty = [ v for v in valid if not isinstance(v, EmptySchema) ] validated, paths = CheckSchemas.find_conflicts(non_empty) table = CheckSchemas.get_table(self.get_name(name, path), validated, paths) invalid_tables = list( map( lambda i: InvalidTable("Invalid Schema", invalid_schema=i), invalid_schemas)) if isinstance(table, Table): final = table else: invalid_tables.append(table) final = InvalidTables(invalid_tables) except (ClientError, PermissionError) as e: final = InvalidTables( [InvalidTable(f"Not able to infer table: {message(e)}")]) else: response.set_status(404) final = InvalidTables([TableNotFound(f"No keys at {path}")]) return final, response
def add_info(self, info: str, log: bool = True): info = self.add_timestamp(info) if log: logger.info(str(info)) self.info.append(info)