def echo_diff(actions: ActionList):
for action in actions.actions:
if action.is_deletion:
echo_style(action.description, fg=Color.RED)
elif action.is_creation:
echo_style(action.description, fg=Color.GREEN)
else:
# Assumes update
echo_style(action.description, fg=Color.YELLOW)
current_yaml = dump_yaml(
action.current.to_dict()) if action.current is not None else ''
desired_yaml = dump_yaml(
action.desired.to_dict()) if action.desired is not None else ''
assert current_yaml is not None and desired_yaml is not None
current_yaml_lines = current_yaml.splitlines(keepends=True)
desired_yaml_lines = desired_yaml.splitlines(keepends=True)
diff = difflib.unified_diff(current_yaml_lines,
desired_yaml_lines,
fromfile='current',
tofile='desired')
for line in diff:
color = _LINE_START_TO_COLOR.get(line[0])
echo_style(line, fg=color, nl=False)
echo_info('')
def validate() -> bool:
"""Check local files against schema."""
errors = []
try:
validate_context()
except ValidationError as e:
errors.append(e)
errors.extend(validate_local_state())
errors_by_severity = defaultdict(list)
for error in errors:
errors_by_severity[error.severity].append(error)
if len(errors_by_severity[ValidationErrorSeverity.WARNING]) > 0:
echo_warnings(errors_by_severity[ValidationErrorSeverity.WARNING])
echo_info('')
if len(errors_by_severity[ValidationErrorSeverity.ERROR]) > 0:
echo_errors(errors_by_severity[ValidationErrorSeverity.ERROR])
return False
echo_info("Success: All files are valid.")
return True
def create_command():
echo_info('Scaffolding a new transform...')
name = click.prompt('name')
connections = Connections.load()
connection_names = connections.keys() if connections else []
connection_base_text = 'connection'
if len(connection_names) == 0:
connection_prompt_text = connection_base_text
elif len(connection_names) > 3:
connection_prompt_text = f'{connection_base_text} (Available - {{{",".join(list(connection_names)[:3])}}},...)'
else:
connection_prompt_text = f'{connection_base_text} (Available - {{{",".join(connection_names)}}})'
# Assemble target based on input
connection = click.prompt(connection_prompt_text)
target_view_path = click.prompt(f'target: {connection}.', prompt_suffix="")
target = f'{connection}.{target_view_path}'
transform = PanoTransform(name=name, fields=[], target=target)
writer = FileWriter()
transform_path = Paths.transforms_dir(
) / f'{transform.name}{FileExtension.TRANSFORM_YAML.value}'
if Path.exists(transform_path):
echo_error(f'Transform {transform_path} already exists')
else:
writer.write_transform(transform)
def test_connection_command() -> None:
"""CLI command. Test connection by trying to connect to the database."""
connection = Connection.get()
ok, error = Connection.test(connection)
if ok:
echo_info('{name}... OK')
else:
echo_info(f'FAIL: {error}')
def show_connection_command() -> None:
"""CLI command. List all connections."""
connection = Connection.get()
if not connection:
echo_info(
'No connection setup yet.\nUse "pano connection setup" to configure connection or edit pano.yaml file.'
)
exit(0)
echo_info(yaml.dump(connection))
def list_connections_command() -> None:
"""CLI command. List all connections."""
connections = Connections.load()
if not connections:
config_file = Paths.config_file()
echo_info(
f'No connections found.\n'
f'Use "pano connection create" to create connection or edit "{config_file}" file.'
)
exit(0)
echo_info(yaml.dump(connections))
def exec_command(
compile_only: bool = False,
yes: bool = False,
):
compiled_transforms: List[Tuple[CompiledTransform, Path]] = []
transforms_with_path = get_transforms()
if len(transforms_with_path) == 0:
echo_info('No transforms found...')
return
transform_compiler = TransformCompiler(get_company_id())
file_writer = FileWriter()
echo_info('Compiling transforms...')
with tqdm(transforms_with_path) as compiling_bar:
for transform, transform_path in compiling_bar:
try:
compiled_transform = transform_compiler.compile(
transform=transform)
compiled_transforms.append(
(compiled_transform, transform_path))
compiled_sql_path = file_writer.write_compiled_transform(
compiled_transform)
compiling_bar.write(
f'[{transform.name}] writing compiled query to {compiled_sql_path}'
)
except Exception as e:
compiling_bar.write(
f'\nError: Failed to compile transform {transform_path}:\n {str(e)}'
)
if len(compiled_transforms) == 0:
echo_info('No transforms to execute...')
return
if compile_only or not yes and not click.confirm(
'Do you want to execute transforms?'):
return
echo_info('Executing transforms...')
with tqdm(compiled_transforms) as exec_bar:
for (compiled_transform, transform_path) in exec_bar:
try:
exec_bar.write(
f'Executing: {compiled_transform.transform.name} on {compiled_transform.transform.connection_name}'
)
TransformExecutor.execute(compiled_transform)
exec_bar.write(
f'\u2713 [{compiled_transform.transform.connection_name}] {transform.name}'
)
except Exception as e:
exec_bar.write(
f'\u2717 [{compiled_transform.transform.connection_name}] {transform.name} \nError: Failed to execute transform {transform_path}:\n {str(e)}'
)
def create_command():
echo_info('Scaffolding a new transform...')
name = click.prompt('name')
target = click.prompt('target:', prompt_suffix="")
transform = PanoTransform(name=name, fields=[], target=target)
writer = FileWriter()
transform_path = Paths.transforms_dir(
) / f'{transform.name}{FileExtension.TRANSFORM_YAML.value}'
if Path.exists(transform_path):
echo_error(f'Transform {transform_path} already exists')
else:
writer.write_transform(transform)
def test_connections_command(name: Optional[str] = None) -> None:
"""CLI command. Test connections by trying to connect to the database.
Optionally you can specify name for specific connection to that only that."""
connections = Connections.load()
if name is not None:
if name not in connections:
raise ConnectionNotFound(name)
# Filter specified connection by name
connections = {name: connections[name]}
for name, connection in connections.items():
ok, error = Connections.test(connection)
if ok:
echo_info(f'{name}... OK')
else:
echo_info(f'{name}... FAIL: {error}')
def create_connection_command(
name: str,
connection_string: str,
no_test: bool,
) -> None:
"""CLI command. Create new connection."""
connections = Connections.load()
if name in connections:
raise ConnectionAlreadyExistsException(name)
new_connection = {'connection_string': connection_string}
connections[name] = new_connection
if not no_test:
ok, error = Connections.test(new_connection)
if not ok:
raise ConnectionCreateException(error)
Connections.save(connections)
echo_info('Connection was successfully created!')
def invoke(self, ctx: Context):
from panoramic.cli.validate import validate_local_state
errors_by_severity: defaultdict = defaultdict(list)
for error in validate_local_state():
errors_by_severity[error.severity].append(error)
if len(errors_by_severity[ValidationErrorSeverity.WARNING]) > 0:
echo_warnings(errors_by_severity[ValidationErrorSeverity.WARNING])
echo_info('')
if len(errors_by_severity[ValidationErrorSeverity.ERROR]) > 0:
echo_errors(errors_by_severity[ValidationErrorSeverity.ERROR])
sys.exit(1)
# preload data for taxonomy and calculate TEL metadata
from panoramic.cli.husky.core.taxonomy.getters import Taxonomy
Taxonomy.preload_taxons_from_state()
Taxonomy.precalculate_tel_metadata()
return super().invoke(ctx)
def setup_connection_command(
url: Optional[str],
dialect: Optional[str],
no_test: bool,
) -> None:
"""CLI command. Create new connection."""
if (url and dialect) or (not url and not dialect):
raise ConnectionCreateException(
'Must specify either a URL or dialect, not both.')
if url:
connection = {'url': url}
elif dialect:
connection = {'dialect': dialect}
if url and not no_test:
ok, error = Connection.test(connection)
if not ok:
raise ConnectionCreateException(error)
Connection.save(connection)
echo_info('Connection was successfully created!')
def update_connection_command(
name: str,
connection_string: str,
no_test: bool,
) -> None:
"""CLI command. Update specific connection."""
connections = Connections.load()
if name not in connections:
raise ConnectionNotFound(name)
new_connection = {'connection_string': connection_string}
connections[name] = new_connection
if not no_test:
ok, error = Connections.test(new_connection)
if not ok:
raise ConnectionUpdateException(error)
for key, value in new_connection.items():
if value != '':
connections[name][key] = value
Connections.save(connections)
echo_info('Connection was successfully updated!')
def scan(filter_reg_ex: Optional[str] = None):
"""Scan all metadata for given source and filter."""
connection_info = Connection.get()
dialect_name = Connection.get_dialect_name(connection_info)
query_runtime = EnumHelper.from_value_safe(HuskyQueryRuntime, dialect_name)
if not query_runtime:
raise UnsupportedDialectError(dialect_name)
scanner_cls = Scanner.get_scanner(query_runtime)
scanner = scanner_cls()
echo_info('Started scanning the data source')
scanner.scan(force_reset=True)
echo_info('Finished scanning the data source')
# apply regular expression as a filter on model names
if filter_reg_ex:
re_compiled = re.compile(filter_reg_ex)
models = [
model for model in scanner.models.values()
if re_compiled.match(model.model_name)
]
else:
models = list(scanner.models.values())
if len(scanner.models) == 0:
echo_info('No tables have been found')
return
progress_bar = tqdm(total=len(scanner.models))
writer = FileWriter()
for model in models:
writer.write_scanned_model(model)
progress_bar.write(f'Discovered model {model.model_name}')
progress_bar.update()
progress_bar.write(f'Scanned {progress_bar.total} tables')
def scaffold_missing_fields(target_dataset: Optional[str] = None,
yes: bool = False,
no_remote: bool = True):
"""Scaffold missing field files."""
echo_info('Loading local state...')
state = get_local_state(target_dataset=target_dataset)
errors = []
for dataset, (fields, models) in state.get_objects_by_package().items():
for idx, error in enumerate(
validate_missing_files(fields, models, package_name=dataset)):
if idx == 0:
echo_info(
f'\nFields referenced in models without definition in dataset {dataset}:'
)
echo_info(f' {error.field_slug}')
errors.append(error)
if len(errors) == 0:
echo_info('No issues found')
return
echo_info('')
if not yes and not click.confirm(
'You will not be able to query these fields until you define them. Do you want to do that now?'
):
# User decided not to fix issues
return
loaded_models: Dict[str, PanoModel] = {}
if not no_remote:
connection = Connection.get()
dialect_name = Connection.get_dialect_name(connection)
query_runtime = EnumHelper.from_value_safe(HuskyQueryRuntime,
dialect_name)
scanner_cls = Scanner.get_scanner(query_runtime)
scanner = scanner_cls()
echo_info('Scanning remote storage...')
scanner.scan()
echo_info('Finished scanning remote storage...')
loaded_models = scanner.models
echo_info('Scanning fields...')
fields = scan_fields_for_errors(errors, loaded_models)
action_list = ActionList(
actions=[Action(desired=field) for field in fields])
echo_info('Updating local state...')
executor = LocalExecutor()
for action in action_list.actions:
try:
executor.execute(action)
except Exception:
echo_error(f'Error: Failed to execute action {action.description}')
echo_info(
f'Updated {executor.success_count}/{executor.total_count} fields')
def delete_orphaned_fields(target_dataset: Optional[str] = None,
yes: bool = False):
"""Delete orphaned field files."""
echo_info('Loading local state...')
state = get_local_state(target_dataset=target_dataset)
action_list: ActionList[PanoField] = ActionList()
for dataset, (fields, models) in state.get_objects_by_package().items():
fields_by_slug = {f.slug: f for f in fields}
for idx, error in enumerate(
validate_orphaned_files(fields, models, package_name=dataset)):
if idx == 0:
echo_info(
f'\nFields without calculation or reference in a model in dataset {dataset}:'
)
echo_info(f' {error.field_slug}')
# Add deletion action
action_list.add_action(
Action(current=fields_by_slug[error.field_slug], desired=None))
if action_list.is_empty:
echo_info('No issues found')
return
echo_info('')
if not yes and not click.confirm(
'You will not be able to query these fields. Do you want to remove them?'
):
# User decided not to fix issues
return
echo_info('Updating local state...')
executor = LocalExecutor()
for action in action_list.actions:
try:
executor.execute(action)
except Exception:
echo_error(f'Error: Failed to execute action {action.description}')
echo_info(
f'Updated {executor.success_count}/{executor.total_count} fields')
def detect_joins(target_dataset: Optional[str] = None,
diff: bool = False,
overwrite: bool = False,
yes: bool = False):
echo_info('Loading local state...')
local_state = get_local_state(target_dataset=target_dataset)
if local_state.is_empty:
echo_info('No datasets to detect joins on')
return
models_by_virtual_data_source: Dict[Optional[str],
Dict[str,
PanoModel]] = defaultdict(dict)
for model in local_state.models:
# Prepare a mapping for a quick access when reconciling necessary changes later
models_by_virtual_data_source[model.virtual_data_source][
model.model_name] = model
action_list: ActionList[PanoModel] = ActionList()
with tqdm(list(local_state.data_sources)) as bar:
for dataset in bar:
try:
bar.write(
f'Detecting joins for dataset {dataset.dataset_slug}')
joins_by_model = detect_join_for_models([dataset.dataset_slug])
for model_name, joins in joins_by_model.items():
if not joins:
bar.write(
f'No joins detected for {model_name} under dataset {dataset.dataset_slug}'
)
continue
bar.write(
f'Detected {len(joins)} joins for {model_name} under dataset {dataset.dataset_slug}'
)
detected_join_objects = [
PanoModelJoin.from_dict(join_dict)
for join_dict in joins
]
current_model = models_by_virtual_data_source[
dataset.dataset_slug][model_name]
desired_model = deepcopy(current_model)
if overwrite:
desired_model.joins = detected_join_objects
else:
for detected_join in detected_join_objects:
# Only append joins that are not already defined
if detected_join not in current_model.joins:
desired_model.joins.append(detected_join)
action_list.actions.append(
Action(current=current_model, desired=desired_model))
except JoinException as join_exception:
bar.write(f'Error: {str(join_exception)}')
logger.debug(str(join_exception), exc_info=True)
except Exception:
error_msg = f'An unexpected error occured when detecting joins for {dataset.dataset_slug}'
bar.write(f'Error: {error_msg}')
logger.debug(error_msg, exc_info=True)
finally:
bar.update()
if action_list.is_empty:
echo_info('No joins detected')
return
echo_diff(action_list)
if diff:
# User decided to see the diff only
return
if not yes and not click.confirm('Do you want to proceed?'):
# User decided not to update local models based on join suggestions
return
echo_info('Updating local state...')
executor = LocalExecutor()
for action in action_list.actions:
try:
executor.execute(action)
except Exception:
echo_error(f'Error: Failed to execute action {action.description}')
echo_info(
f'Updated {executor.success_count}/{executor.total_count} models')