def _analyze_projects(self, project=None, sortkey=None, limit=None): projects = fetcher.get_project_files(self, project=project) info = [] for p in projects: metadata = list( map( lambda x: 'model' if x['type'] == 'model' else ('view' if x['type'] == 'view' else None), p['files'])) model_count = metadata.count('model') view_count = metadata.count('view') git_tests = fetcher.test_git_connection(self, p['name']) info.append({ 'project': p['name'], 'model_count': model_count, 'view_count': view_count, 'git_connection_status': git_tests, 'pull_request_mode': p['pr_mode'], 'validation_required': p['validation_required'] }) valid_values = list(info[0].keys()) info = styler.sort(info, valid_values, sortkey) info = styler.limit(info, limit=limit) return info
def _analyze_models(self, project=None, model=None, sortkey=None, limit=None, timeframe=90, min_queries=0): models = fetcher.get_models(self, project=project, model=model, verbose=1) used_models = fetcher.get_used_models(self, timeframe, min_queries) info = [] for m in models: explore_count = len(m['explores']) if m['name'] in used_models: query_run_count = used_models[m['name']] else: query_run_count = 0 unused_explores = fetcher.get_unused_explores( self, m['name'], timeframe, min_queries) info.append({ 'project': m['project_name'], 'model': m['name'], 'explore_count': explore_count, 'unused_explores': len(unused_explores), 'query_run_count': query_run_count }) valid_values = list(info[0].keys()) info = styler.sort(info, valid_values, sortkey) info = styler.limit(info, limit=limit) return info
def _analyze_explores(self, model=None, explore=None, sortkey=None, limit=None, min_queries=0, timeframe=90): explores = fetcher.get_explores(self, model=model, explore=explore, verbose=1) explores_usage = {} info = [] for e in explores: # in case explore does not exist (bug - #32748) if e is None: pass else: _used_fields = fetcher.get_used_explore_fields( self, e['model_name'], e['scopes'], timeframe, min_queries) used_fields = list(_used_fields.keys()) exposed_fields = fetcher.get_explore_fields(self, explore=e, scoped_names=1) unused_fields = set(exposed_fields) - set(used_fields) field_count = len(exposed_fields) query_count = fetcher.get_used_explores(self, model=e['model_name'], explore=e['name']) all_joins = set(e['scopes']) all_joins.remove(e['name']) used_joins = set([i.split('.')[2] for i in used_fields]) unused_joins = len(list(all_joins - used_joins)) has_description = 'Yes' if e['description'] else 'No' if query_count.get(e['name']): query_count = query_count[e['name']] else: query_count = 0 info.append({ 'model': e['model_name'], 'explore': e['name'], 'is_hidden': e['hidden'], 'has_description': has_description, 'join_count': len(all_joins), 'unused_joins': unused_joins, 'field_count': field_count, 'unused_fields': len(unused_fields), 'query_count': query_count }) if not info: self.analyze_logger.error('No matching explores found') raise Exception('No matching explores found') valid_values = list(info[0].keys()) info = styler.sort(info, valid_values, sortkey) info = styler.limit(info, limit=limit) return info
def _analyze_fields(self, model=None, explore=None, sortkey=None, limit=None, min_queries=0, timeframe=90): print('Retrieving explores for fields...') explores = fetcher.get_explores(self, model=model, explore=explore, verbose=1) info = [] progress = 1 for e in explores: print('Analyzing {}.{}, {} of {} explores'.format( e['model_name'], e['name'], progress, len(explores))) if e is None: pass else: _used_fields = fetcher.get_used_explore_fields( self, e['model_name'], e['scopes'], timeframe, min_queries) used_fields = list(_used_fields.keys()) exposed_fields = fetcher.get_explore_fields(self, explore=e, scoped_names=1) unused_fields = set(exposed_fields) - set(used_fields) field_count = len(exposed_fields) missing_description = 0 dimensions = 0 measures = 0 for dim in e['fields']['dimensions']: dimensions += 1 if not dim['description']: missing_description += 1 for measure in e['fields']['measures']: measures += 1 if not measure['description']: missing_description += 1 info.append({ 'model': e['model_name'], 'explore': e['name'], 'field_count': field_count, 'unused_fields': len(unused_fields), 'missing_description': missing_description, 'dimensions': dimensions, 'measures': measures }) progress += 1 if not info: self.analyze_logger.error('No matching explores found') raise Exception('No matching explores found') valid_values = list(info[0].keys()) info = styler.sort(info, valid_values, sortkey) info = styler.limit(info, limit=limit) return info
def _analyze_models(self, project=None, model=None, sortkey=None, limit=None, timeframe=90, min_queries=0): print('fetching all models...') models = fetcher.get_models(self, project=project, model=model, verbose=1) print('complete.') print('fetching used models...') used_models = fetcher.get_used_models(self, timeframe, min_queries) print('complete.') info = [] total = len(used_models) complete = 1 for m in models: print('Processing {} of {} models'.format(complete, total)) explore_count = len(m['explores']) if m['name'] in used_models: query_run_count = used_models[m['name']] else: query_run_count = 0 unused_explores = fetcher.get_unused_explores( self, m['name'], timeframe, min_queries) info.append({ 'project': m['project_name'], 'model': m['name'], 'explore_count': explore_count, 'unused_explores': len(unused_explores), 'query_run_count': query_run_count }) complete += 1 valid_values = list(info[0].keys()) info = styler.sort(info, valid_values, sortkey) info = styler.limit(info, limit=limit) return info