def WorkGroup(self) -> str:
""" Select AWS Athena workgroup """
if not self._WorkGroup:
logger.info('Selecting Athena workgroup...')
workgroups = self.list_work_groups()
logger.info(
f'Found {len(workgroups)} workgroups: {", ".join([wg.get("Name") for wg in workgroups])}'
)
if len(workgroups) == 1:
self._WorkGroup = workgroups.pop().get('Name')
elif len(workgroups) > 1:
# Select default workgroup if present
default_workgroup = next(
iter([
wg.get('Name') for wg in workgroups
if wg['Name'] == self.defaults.get('WorkGroup')
]), None)
if default_workgroup:
logger.info(
f'Found "{default_workgroup}" as a default workgroup')
# Ask user
self._WorkGroup = get_parameter(
param_name='athena-workgroup',
message="Select AWS Athena workgroup to use",
choices=[d['Name'] for d in workgroups],
default=default_workgroup)
logger.info(f'Selected workgroup: "{self._WorkGroup}"')
return self._WorkGroup
def DatabaseName(self) -> str:
""" Check if Athena database exist """
if not self._DatabaseName:
# Get AWS Athena databases
athena_databases = self.list_databases()
if not len(athena_databases):
self._status = 'AWS Athena databases not found'
print(self._status)
exit(1)
if len(athena_databases) == 1:
self._DatabaseName = athena_databases.pop().get('Name')
elif len(athena_databases) > 1:
# Remove empty databases from the list
for d in athena_databases:
tables = self.list_table_metadata(
DatabaseName=d.get('Name'))
if not len(tables):
athena_databases.remove(d)
# Select default database if present
default_database = [
d for d in athena_databases
if d['Name'] == self.defaults.get('DatabaseName')
]
if len(default_database):
self._DatabaseName = default_database.pop().get('Name')
else:
# Ask user
self._DatabaseName = get_parameter(
param_name='athena-database',
message="Select AWS Athena database to use",
choices=[d['Name'] for d in athena_databases],
)
logger.info(f'Using Athena database: {self._DatabaseName}')
return self._DatabaseName
def select_dashboard(self, force=False) -> str:
""" Select from a list of discovered dashboards """
selection = list()
dashboard_id = None
if not self.dashboards:
return None
choices = {}
for dashboard in self.dashboards.values():
health = 'healthy' if dashboard.health else 'unhealthy'
key = f'{dashboard.name} ({dashboard.arn}, {health}, {dashboard.status})'
if ((dashboard.latest or not dashboard.health) and not force):
choices[key] = None
else:
choices[key] = dashboard.id
try:
dashboard_id = get_parameter(
param_name='dashboard-id',
message="Please select installation(s) from the list",
choices=choices,
none_as_disabled=True,
)
except AttributeError as e:
# No updatable dashboards (selection is disabled)
logger.debug(e, exc_info=True, stack_info=True)
except Exception as e:
logger.exception(e)
finally:
return dashboard_id
def metadata(self) -> dict:
if not self._metadata:
try:
# Look other tables
tables = self.athena.list_table_metadata()
# Filter tables with type = 'EXTERNAL_TABLE'
tables = [
v for v in tables if v.get('TableType') == 'EXTERNAL_TABLE'
]
# Filter tables having CUR structure
for table in tables.copy():
columns = [c.get('Name') for c in table.get('Columns')]
if not all([c in columns
for c in self.curRequiredColumns]):
tables.remove(table)
# Sort tables by name (desc)
tables.sort(key=lambda x: x.get('Name'), reverse=True)
if len(tables) == 1:
self._metadata = tables[0]
self._tableName = self._metadata.get('Name')
elif len(tables) > 1:
self._tableName = get_parameter(
param_name='cur-table-name',
message="Multiple CUR tables found, please select one",
choices=[v.get('Name') for v in tables],
)
self._metadata = self.athena.get_table_metadata(
self._tableName)
except Exception as e:
# For other errors dump the message
print(json.dumps(e, indent=4, sort_keys=True, default=str))
return self._metadata
def select_metadata_collection_method(self) -> str:
""" Selects the method to collect metadata """
logger.info('Metadata source selection')
# Ask user which method to use to retreive account list
account_map_sources = {
'CSV file (relative path required)': 'csv',
'AWS Organizations (one time account listing)': 'organization',
'Dummy (CUR account data, no names)': 'dummy',
}
selected_source = get_parameter(
param_name='account-map-source',
message="Please select account metadata collection method",
choices=account_map_sources,
)
logger.info(f'Selected {selected_source}')
self._metadata_source = selected_source
# Collect account list from different sources of user choice
if self._metadata_source == 'csv':
finished = False
while not finished:
mapping_file = get_parameter(
param_name='account-map-file',
message="Enter file path",
)
finished = self.check_file_exists(mapping_file)
if not finished:
click.echo('File not found, ', nl=False)
click.echo('\nCollecting account info...', nl=False)
self._accounts = self.get_csv_accounts(mapping_file)
logger.info(f'Found {len(self._accounts)} accounts')
click.echo(f' {len(self.accounts)} collected')
elif self._metadata_source == 'organization':
click.echo('\nCollecting account info...', nl=False)
self._accounts = self.get_organization_accounts()
logger.info(f'Found {len(self._accounts)} accounts')
click.echo(f' {len(self.accounts)} collected')
elif self._metadata_source == 'dummy':
click.echo('Notice: Dummy account mapping will be created')
else:
print('Unsupported selection')
return False
def select_folder(self):
""" Select a folder from the list of folders """
try:
folderList = self.list_folders()
if not folderList:
return None
except self.client.exceptions.AccessDeniedException:
raise
_folder = get_parameter(
param_name='folder-id',
message="Please select QuickSight folder to use",
choices={
f"{folder.get('Name')} ({folder.get('FolderId')})": folder
for folder in folderList
})
return _folder
def delete_view(self,
name: str,
catalog: str = None,
database: str = None):
if get_parameter(param_name=f'confirm-{name}',
message=f'Delete Athena view {name}?',
choices=['yes', 'no'],
default='no') != 'yes':
return False
try:
res = self.execute_query(f'DROP VIEW IF EXISTS {name};',
catalog=catalog,
database=database,
fail=False)
except Exception as exc:
logger.debug(exc, stack_info=True)
logger.info(f'View {name} cannot be deleted: {exc}')
return False
else:
if name in self._metadata: del self._metadata[name]
logger.info(f'View {name} deleted')
return True
def CatalogName(self) -> str:
""" Check if AWS Datacalog and Athena database exist """
if not self._CatalogName:
# Get AWS Glue DataCatalogs
glue_data_catalogs = [
d for d in self.list_data_catalogs() if d['Type'] == 'GLUE'
]
if not len(glue_data_catalogs):
logger.error('AWS DataCatog of type GLUE not found!')
self._status = 'AWS DataCatog of type GLUE not found'
if len(glue_data_catalogs) == 1:
self._CatalogName = glue_data_catalogs.pop().get('CatalogName')
elif len(glue_data_catalogs) > 1:
# Ask user
self._CatalogName = get_parameter(
param_name='glue-data-catalog',
message="Select AWS DataCatalog to use",
choices=[
catalog.get('CatalogName')
for catalog in glue_data_catalogs
],
)
logger.info(f'Using datacatalog: {self._CatalogName}')
return self._CatalogName
def select_user(self):
""" Select a user from the list of users """
user_list = None
try:
user_list = self.identityClient.list_users(
AwsAccountId=self.account_id,
Namespace='default').get('UserList')
except self.client.exceptions.AccessDeniedException:
logger.info('Access denied listing users')
return None #FIXME: should we rather allow manual entry when no access?
_username = get_parameter(
param_name='quicksight-user',
message="Please select QuickSight user to use",
choices={
f"{user.get('UserName')} ({user.get('Email')}, {user.get('Role')})":
user.get('UserName')
for user in user_list
})
for u in user_list:
if u.get('UserName') == _username:
return u
else:
return None