def scaffold_missing_fields(target_dataset: Optional[str] = None,
yes: bool = False,
no_remote: bool = True):
"""Scaffold missing field files."""
echo_info('Loading local state...')
state = get_local_state(target_dataset=target_dataset)
errors = []
for dataset, (fields, models) in state.get_objects_by_package().items():
for idx, error in enumerate(
validate_missing_files(fields, models, package_name=dataset)):
if idx == 0:
echo_info(
f'\nFields referenced in models without definition in dataset {dataset}:'
)
echo_info(f' {error.field_slug}')
errors.append(error)
if len(errors) == 0:
echo_info('No issues found')
return
echo_info('')
if not yes and not click.confirm(
'You will not be able to query these fields until you define them. Do you want to do that now?'
):
# User decided not to fix issues
return
loaded_models: Dict[str, PanoModel] = {}
if not no_remote:
connection = Connection.get()
dialect_name = Connection.get_dialect_name(connection)
query_runtime = EnumHelper.from_value_safe(HuskyQueryRuntime,
dialect_name)
scanner_cls = Scanner.get_scanner(query_runtime)
scanner = scanner_cls()
echo_info('Scanning remote storage...')
scanner.scan()
echo_info('Finished scanning remote storage...')
loaded_models = scanner.models
echo_info('Scanning fields...')
fields = scan_fields_for_errors(errors, loaded_models)
action_list = ActionList(
actions=[Action(desired=field) for field in fields])
echo_info('Updating local state...')
executor = LocalExecutor()
for action in action_list.actions:
try:
executor.execute(action)
except Exception:
echo_error(f'Error: Failed to execute action {action.description}')
echo_info(
f'Updated {executor.success_count}/{executor.total_count} fields')
def is_version_supported(current_version: str) -> bool:
"""Check if current version of the CLI is still supported.
If version has been deprecated print warning message notifying user to update the CLI.
Returns bool. If check was successful program can continue otherwise it should be stopped.
"""
try:
minimum_supported_version = __fetch_minimum_supported_version(
current_version)
except requests.exceptions.RequestException:
error_msg = 'Failed to connect to remote server to verify minimum supported CLI version.'
echo_error(error_msg)
logger.debug(error_msg, exc_info=True)
return False
except (KeyError, TypeError):
error_msg = 'Failed to verify minimum supported CLI version.'
echo_error(error_msg)
logger.debug(error_msg, exc_info=True)
return False
if version.parse(current_version) < version.parse(
minimum_supported_version):
upgrade_command = __get_upgrade_command()
message = (
f"WARNING: This version '{current_version}' has been deprecated.\n"
f"Please update to version '{minimum_supported_version}' or higher.\n"
f"To update run: {upgrade_command}.\n")
tqdm.write(message)
return False
return True
def create_command():
echo_info('Scaffolding a new transform...')
name = click.prompt('name')
connections = Connections.load()
connection_names = connections.keys() if connections else []
connection_base_text = 'connection'
if len(connection_names) == 0:
connection_prompt_text = connection_base_text
elif len(connection_names) > 3:
connection_prompt_text = f'{connection_base_text} (Available - {{{",".join(list(connection_names)[:3])}}},...)'
else:
connection_prompt_text = f'{connection_base_text} (Available - {{{",".join(connection_names)}}})'
# Assemble target based on input
connection = click.prompt(connection_prompt_text)
target_view_path = click.prompt(f'target: {connection}.', prompt_suffix="")
target = f'{connection}.{target_view_path}'
transform = PanoTransform(name=name, fields=[], target=target)
writer = FileWriter()
transform_path = Paths.transforms_dir(
) / f'{transform.name}{FileExtension.TRANSFORM_YAML.value}'
if Path.exists(transform_path):
echo_error(f'Transform {transform_path} already exists')
else:
writer.write_transform(transform)
def invoke(self, ctx: Context):
from panoramic.cli.validate import validate_context
try:
validate_context()
return super().invoke(ctx)
except (ValidationError, SourceNotFoundException) as e:
echo_error(str(e))
sys.exit(1)
def invoke(self, ctx: Context):
from panoramic.cli.validate import validate_config
try:
validate_config()
return super().invoke(ctx)
except ValidationError as e:
echo_error(str(e))
sys.exit(1)
def create_command():
echo_info('Scaffolding a new transform...')
name = click.prompt('name')
target = click.prompt('target:', prompt_suffix="")
transform = PanoTransform(name=name, fields=[], target=target)
writer = FileWriter()
transform_path = Paths.transforms_dir(
) / f'{transform.name}{FileExtension.TRANSFORM_YAML.value}'
if Path.exists(transform_path):
echo_error(f'Transform {transform_path} already exists')
else:
writer.write_transform(transform)
def delete_orphaned_fields(target_dataset: Optional[str] = None,
yes: bool = False):
"""Delete orphaned field files."""
echo_info('Loading local state...')
state = get_local_state(target_dataset=target_dataset)
action_list: ActionList[PanoField] = ActionList()
for dataset, (fields, models) in state.get_objects_by_package().items():
fields_by_slug = {f.slug: f for f in fields}
for idx, error in enumerate(
validate_orphaned_files(fields, models, package_name=dataset)):
if idx == 0:
echo_info(
f'\nFields without calculation or reference in a model in dataset {dataset}:'
)
echo_info(f' {error.field_slug}')
# Add deletion action
action_list.add_action(
Action(current=fields_by_slug[error.field_slug], desired=None))
if action_list.is_empty:
echo_info('No issues found')
return
echo_info('')
if not yes and not click.confirm(
'You will not be able to query these fields. Do you want to remove them?'
):
# User decided not to fix issues
return
echo_info('Updating local state...')
executor = LocalExecutor()
for action in action_list.actions:
try:
executor.execute(action)
except Exception:
echo_error(f'Error: Failed to execute action {action.description}')
echo_info(
f'Updated {executor.success_count}/{executor.total_count} fields')
def detect_joins(target_dataset: Optional[str] = None,
diff: bool = False,
overwrite: bool = False,
yes: bool = False):
echo_info('Loading local state...')
local_state = get_local_state(target_dataset=target_dataset)
if local_state.is_empty:
echo_info('No datasets to detect joins on')
return
models_by_virtual_data_source: Dict[Optional[str],
Dict[str,
PanoModel]] = defaultdict(dict)
for model in local_state.models:
# Prepare a mapping for a quick access when reconciling necessary changes later
models_by_virtual_data_source[model.virtual_data_source][
model.model_name] = model
action_list: ActionList[PanoModel] = ActionList()
with tqdm(list(local_state.data_sources)) as bar:
for dataset in bar:
try:
bar.write(
f'Detecting joins for dataset {dataset.dataset_slug}')
joins_by_model = detect_join_for_models([dataset.dataset_slug])
for model_name, joins in joins_by_model.items():
if not joins:
bar.write(
f'No joins detected for {model_name} under dataset {dataset.dataset_slug}'
)
continue
bar.write(
f'Detected {len(joins)} joins for {model_name} under dataset {dataset.dataset_slug}'
)
detected_join_objects = [
PanoModelJoin.from_dict(join_dict)
for join_dict in joins
]
current_model = models_by_virtual_data_source[
dataset.dataset_slug][model_name]
desired_model = deepcopy(current_model)
if overwrite:
desired_model.joins = detected_join_objects
else:
for detected_join in detected_join_objects:
# Only append joins that are not already defined
if detected_join not in current_model.joins:
desired_model.joins.append(detected_join)
action_list.actions.append(
Action(current=current_model, desired=desired_model))
except JoinException as join_exception:
bar.write(f'Error: {str(join_exception)}')
logger.debug(str(join_exception), exc_info=True)
except Exception:
error_msg = f'An unexpected error occured when detecting joins for {dataset.dataset_slug}'
bar.write(f'Error: {error_msg}')
logger.debug(error_msg, exc_info=True)
finally:
bar.update()
if action_list.is_empty:
echo_info('No joins detected')
return
echo_diff(action_list)
if diff:
# User decided to see the diff only
return
if not yes and not click.confirm('Do you want to proceed?'):
# User decided not to update local models based on join suggestions
return
echo_info('Updating local state...')
executor = LocalExecutor()
for action in action_list.actions:
try:
executor.execute(action)
except Exception:
echo_error(f'Error: Failed to execute action {action.description}')
echo_info(
f'Updated {executor.success_count}/{executor.total_count} models')
def invoke(self, ctx: Context):
try:
return super().invoke(ctx)
except Exception as e:
echo_error(str(e))
sys.exit(1)
def wrapped(*args, **kwargs):
try:
return f(*args, **kwargs)
except Exception:
echo_error('Internal error occurred', exc_info=True)
sys.exit(1)