def echo_diff(actions: ActionList): for action in actions.actions: if action.is_deletion: echo_style(action.description, fg=Color.RED) elif action.is_creation: echo_style(action.description, fg=Color.GREEN) else: # Assumes update echo_style(action.description, fg=Color.YELLOW) current_yaml = dump_yaml( action.current.to_dict()) if action.current is not None else '' desired_yaml = dump_yaml( action.desired.to_dict()) if action.desired is not None else '' assert current_yaml is not None and desired_yaml is not None current_yaml_lines = current_yaml.splitlines(keepends=True) desired_yaml_lines = desired_yaml.splitlines(keepends=True) diff = difflib.unified_diff(current_yaml_lines, desired_yaml_lines, fromfile='current', tofile='desired') for line in diff: color = _LINE_START_TO_COLOR.get(line[0]) echo_style(line, fg=color, nl=False) echo_info('')
def validate() -> bool: """Check local files against schema.""" errors = [] try: validate_context() except ValidationError as e: errors.append(e) errors.extend(validate_local_state()) errors_by_severity = defaultdict(list) for error in errors: errors_by_severity[error.severity].append(error) if len(errors_by_severity[ValidationErrorSeverity.WARNING]) > 0: echo_warnings(errors_by_severity[ValidationErrorSeverity.WARNING]) echo_info('') if len(errors_by_severity[ValidationErrorSeverity.ERROR]) > 0: echo_errors(errors_by_severity[ValidationErrorSeverity.ERROR]) return False echo_info("Success: All files are valid.") return True
def create_command(): echo_info('Scaffolding a new transform...') name = click.prompt('name') connections = Connections.load() connection_names = connections.keys() if connections else [] connection_base_text = 'connection' if len(connection_names) == 0: connection_prompt_text = connection_base_text elif len(connection_names) > 3: connection_prompt_text = f'{connection_base_text} (Available - {{{",".join(list(connection_names)[:3])}}},...)' else: connection_prompt_text = f'{connection_base_text} (Available - {{{",".join(connection_names)}}})' # Assemble target based on input connection = click.prompt(connection_prompt_text) target_view_path = click.prompt(f'target: {connection}.', prompt_suffix="") target = f'{connection}.{target_view_path}' transform = PanoTransform(name=name, fields=[], target=target) writer = FileWriter() transform_path = Paths.transforms_dir( ) / f'{transform.name}{FileExtension.TRANSFORM_YAML.value}' if Path.exists(transform_path): echo_error(f'Transform {transform_path} already exists') else: writer.write_transform(transform)
def test_connection_command() -> None: """CLI command. Test connection by trying to connect to the database.""" connection = Connection.get() ok, error = Connection.test(connection) if ok: echo_info('{name}... OK') else: echo_info(f'FAIL: {error}')
def show_connection_command() -> None: """CLI command. List all connections.""" connection = Connection.get() if not connection: echo_info( 'No connection setup yet.\nUse "pano connection setup" to configure connection or edit pano.yaml file.' ) exit(0) echo_info(yaml.dump(connection))
def list_connections_command() -> None: """CLI command. List all connections.""" connections = Connections.load() if not connections: config_file = Paths.config_file() echo_info( f'No connections found.\n' f'Use "pano connection create" to create connection or edit "{config_file}" file.' ) exit(0) echo_info(yaml.dump(connections))
def exec_command( compile_only: bool = False, yes: bool = False, ): compiled_transforms: List[Tuple[CompiledTransform, Path]] = [] transforms_with_path = get_transforms() if len(transforms_with_path) == 0: echo_info('No transforms found...') return transform_compiler = TransformCompiler(get_company_id()) file_writer = FileWriter() echo_info('Compiling transforms...') with tqdm(transforms_with_path) as compiling_bar: for transform, transform_path in compiling_bar: try: compiled_transform = transform_compiler.compile( transform=transform) compiled_transforms.append( (compiled_transform, transform_path)) compiled_sql_path = file_writer.write_compiled_transform( compiled_transform) compiling_bar.write( f'[{transform.name}] writing compiled query to {compiled_sql_path}' ) except Exception as e: compiling_bar.write( f'\nError: Failed to compile transform {transform_path}:\n {str(e)}' ) if len(compiled_transforms) == 0: echo_info('No transforms to execute...') return if compile_only or not yes and not click.confirm( 'Do you want to execute transforms?'): return echo_info('Executing transforms...') with tqdm(compiled_transforms) as exec_bar: for (compiled_transform, transform_path) in exec_bar: try: exec_bar.write( f'Executing: {compiled_transform.transform.name} on {compiled_transform.transform.connection_name}' ) TransformExecutor.execute(compiled_transform) exec_bar.write( f'\u2713 [{compiled_transform.transform.connection_name}] {transform.name}' ) except Exception as e: exec_bar.write( f'\u2717 [{compiled_transform.transform.connection_name}] {transform.name} \nError: Failed to execute transform {transform_path}:\n {str(e)}' )
def create_command(): echo_info('Scaffolding a new transform...') name = click.prompt('name') target = click.prompt('target:', prompt_suffix="") transform = PanoTransform(name=name, fields=[], target=target) writer = FileWriter() transform_path = Paths.transforms_dir( ) / f'{transform.name}{FileExtension.TRANSFORM_YAML.value}' if Path.exists(transform_path): echo_error(f'Transform {transform_path} already exists') else: writer.write_transform(transform)
def test_connections_command(name: Optional[str] = None) -> None: """CLI command. Test connections by trying to connect to the database. Optionally you can specify name for specific connection to that only that.""" connections = Connections.load() if name is not None: if name not in connections: raise ConnectionNotFound(name) # Filter specified connection by name connections = {name: connections[name]} for name, connection in connections.items(): ok, error = Connections.test(connection) if ok: echo_info(f'{name}... OK') else: echo_info(f'{name}... FAIL: {error}')
def create_connection_command( name: str, connection_string: str, no_test: bool, ) -> None: """CLI command. Create new connection.""" connections = Connections.load() if name in connections: raise ConnectionAlreadyExistsException(name) new_connection = {'connection_string': connection_string} connections[name] = new_connection if not no_test: ok, error = Connections.test(new_connection) if not ok: raise ConnectionCreateException(error) Connections.save(connections) echo_info('Connection was successfully created!')
def invoke(self, ctx: Context): from panoramic.cli.validate import validate_local_state errors_by_severity: defaultdict = defaultdict(list) for error in validate_local_state(): errors_by_severity[error.severity].append(error) if len(errors_by_severity[ValidationErrorSeverity.WARNING]) > 0: echo_warnings(errors_by_severity[ValidationErrorSeverity.WARNING]) echo_info('') if len(errors_by_severity[ValidationErrorSeverity.ERROR]) > 0: echo_errors(errors_by_severity[ValidationErrorSeverity.ERROR]) sys.exit(1) # preload data for taxonomy and calculate TEL metadata from panoramic.cli.husky.core.taxonomy.getters import Taxonomy Taxonomy.preload_taxons_from_state() Taxonomy.precalculate_tel_metadata() return super().invoke(ctx)
def setup_connection_command( url: Optional[str], dialect: Optional[str], no_test: bool, ) -> None: """CLI command. Create new connection.""" if (url and dialect) or (not url and not dialect): raise ConnectionCreateException( 'Must specify either a URL or dialect, not both.') if url: connection = {'url': url} elif dialect: connection = {'dialect': dialect} if url and not no_test: ok, error = Connection.test(connection) if not ok: raise ConnectionCreateException(error) Connection.save(connection) echo_info('Connection was successfully created!')
def update_connection_command( name: str, connection_string: str, no_test: bool, ) -> None: """CLI command. Update specific connection.""" connections = Connections.load() if name not in connections: raise ConnectionNotFound(name) new_connection = {'connection_string': connection_string} connections[name] = new_connection if not no_test: ok, error = Connections.test(new_connection) if not ok: raise ConnectionUpdateException(error) for key, value in new_connection.items(): if value != '': connections[name][key] = value Connections.save(connections) echo_info('Connection was successfully updated!')
def scan(filter_reg_ex: Optional[str] = None): """Scan all metadata for given source and filter.""" connection_info = Connection.get() dialect_name = Connection.get_dialect_name(connection_info) query_runtime = EnumHelper.from_value_safe(HuskyQueryRuntime, dialect_name) if not query_runtime: raise UnsupportedDialectError(dialect_name) scanner_cls = Scanner.get_scanner(query_runtime) scanner = scanner_cls() echo_info('Started scanning the data source') scanner.scan(force_reset=True) echo_info('Finished scanning the data source') # apply regular expression as a filter on model names if filter_reg_ex: re_compiled = re.compile(filter_reg_ex) models = [ model for model in scanner.models.values() if re_compiled.match(model.model_name) ] else: models = list(scanner.models.values()) if len(scanner.models) == 0: echo_info('No tables have been found') return progress_bar = tqdm(total=len(scanner.models)) writer = FileWriter() for model in models: writer.write_scanned_model(model) progress_bar.write(f'Discovered model {model.model_name}') progress_bar.update() progress_bar.write(f'Scanned {progress_bar.total} tables')
def scaffold_missing_fields(target_dataset: Optional[str] = None, yes: bool = False, no_remote: bool = True): """Scaffold missing field files.""" echo_info('Loading local state...') state = get_local_state(target_dataset=target_dataset) errors = [] for dataset, (fields, models) in state.get_objects_by_package().items(): for idx, error in enumerate( validate_missing_files(fields, models, package_name=dataset)): if idx == 0: echo_info( f'\nFields referenced in models without definition in dataset {dataset}:' ) echo_info(f' {error.field_slug}') errors.append(error) if len(errors) == 0: echo_info('No issues found') return echo_info('') if not yes and not click.confirm( 'You will not be able to query these fields until you define them. Do you want to do that now?' ): # User decided not to fix issues return loaded_models: Dict[str, PanoModel] = {} if not no_remote: connection = Connection.get() dialect_name = Connection.get_dialect_name(connection) query_runtime = EnumHelper.from_value_safe(HuskyQueryRuntime, dialect_name) scanner_cls = Scanner.get_scanner(query_runtime) scanner = scanner_cls() echo_info('Scanning remote storage...') scanner.scan() echo_info('Finished scanning remote storage...') loaded_models = scanner.models echo_info('Scanning fields...') fields = scan_fields_for_errors(errors, loaded_models) action_list = ActionList( actions=[Action(desired=field) for field in fields]) echo_info('Updating local state...') executor = LocalExecutor() for action in action_list.actions: try: executor.execute(action) except Exception: echo_error(f'Error: Failed to execute action {action.description}') echo_info( f'Updated {executor.success_count}/{executor.total_count} fields')
def delete_orphaned_fields(target_dataset: Optional[str] = None, yes: bool = False): """Delete orphaned field files.""" echo_info('Loading local state...') state = get_local_state(target_dataset=target_dataset) action_list: ActionList[PanoField] = ActionList() for dataset, (fields, models) in state.get_objects_by_package().items(): fields_by_slug = {f.slug: f for f in fields} for idx, error in enumerate( validate_orphaned_files(fields, models, package_name=dataset)): if idx == 0: echo_info( f'\nFields without calculation or reference in a model in dataset {dataset}:' ) echo_info(f' {error.field_slug}') # Add deletion action action_list.add_action( Action(current=fields_by_slug[error.field_slug], desired=None)) if action_list.is_empty: echo_info('No issues found') return echo_info('') if not yes and not click.confirm( 'You will not be able to query these fields. Do you want to remove them?' ): # User decided not to fix issues return echo_info('Updating local state...') executor = LocalExecutor() for action in action_list.actions: try: executor.execute(action) except Exception: echo_error(f'Error: Failed to execute action {action.description}') echo_info( f'Updated {executor.success_count}/{executor.total_count} fields')
def detect_joins(target_dataset: Optional[str] = None, diff: bool = False, overwrite: bool = False, yes: bool = False): echo_info('Loading local state...') local_state = get_local_state(target_dataset=target_dataset) if local_state.is_empty: echo_info('No datasets to detect joins on') return models_by_virtual_data_source: Dict[Optional[str], Dict[str, PanoModel]] = defaultdict(dict) for model in local_state.models: # Prepare a mapping for a quick access when reconciling necessary changes later models_by_virtual_data_source[model.virtual_data_source][ model.model_name] = model action_list: ActionList[PanoModel] = ActionList() with tqdm(list(local_state.data_sources)) as bar: for dataset in bar: try: bar.write( f'Detecting joins for dataset {dataset.dataset_slug}') joins_by_model = detect_join_for_models([dataset.dataset_slug]) for model_name, joins in joins_by_model.items(): if not joins: bar.write( f'No joins detected for {model_name} under dataset {dataset.dataset_slug}' ) continue bar.write( f'Detected {len(joins)} joins for {model_name} under dataset {dataset.dataset_slug}' ) detected_join_objects = [ PanoModelJoin.from_dict(join_dict) for join_dict in joins ] current_model = models_by_virtual_data_source[ dataset.dataset_slug][model_name] desired_model = deepcopy(current_model) if overwrite: desired_model.joins = detected_join_objects else: for detected_join in detected_join_objects: # Only append joins that are not already defined if detected_join not in current_model.joins: desired_model.joins.append(detected_join) action_list.actions.append( Action(current=current_model, desired=desired_model)) except JoinException as join_exception: bar.write(f'Error: {str(join_exception)}') logger.debug(str(join_exception), exc_info=True) except Exception: error_msg = f'An unexpected error occured when detecting joins for {dataset.dataset_slug}' bar.write(f'Error: {error_msg}') logger.debug(error_msg, exc_info=True) finally: bar.update() if action_list.is_empty: echo_info('No joins detected') return echo_diff(action_list) if diff: # User decided to see the diff only return if not yes and not click.confirm('Do you want to proceed?'): # User decided not to update local models based on join suggestions return echo_info('Updating local state...') executor = LocalExecutor() for action in action_list.actions: try: executor.execute(action) except Exception: echo_error(f'Error: Failed to execute action {action.description}') echo_info( f'Updated {executor.success_count}/{executor.total_count} models')