def subsystem_id_function_change_metrics(session, _, subsystem_id, formating='json'): ''' Returns an array of change_metrics (for each function) within a specific subsystem See docs/frontends/django_implementation.md for a more detailed description of the format Args: session(sqlalchemy session): session to use for queries request: : django request object subsystem_id(int): : The subsystem_id to be requested formating(str): Indicates wich format that should be returned, [json|csv] ''' filename, = session.query(Subsystem.subsystem).filter(Subsystem.id == int(subsystem_id)).one() query = session.query(File.file, Function.function, ChangeMetric.nloc, ChangeMetric.cyclomatic_complexity) query = query.filter(ChangeMetric.function_id == Function.id, File.id == ChangeMetric.file_id, File.subsystem_id == int(subsystem_id), ChangeMetric.nloc > 0)\ .group_by(ChangeMetric.function_id)\ .having(func.max(ChangeMetric.date))\ .order_by(File.file, Function.function) data_ = [('file', 'function', 'nloc', 'cyclomatic_complexity')] + query.all() return views_utils.dump_data(data_, response_type=formating, filename=filename)
def treemap_data(session, request): ''' Dumps data that represents the system state at some point in time. Currently it dumps the latest snapshot of the system for continuous metrics and data over the past x months for discontinous metrics. QueryDict params: cmp_type (str): The type of component for which child data is gathered. E.g if a subsystem is provided, the returned data is all data for the files in that subsystem. cmp_id (int): The component id. metric (str): A metric that must exist in the global METRICS dict. response_type (str): "application/json"|"text/csv" ''' metric = request.GET['metric'] cmp_type = request.GET['type'] cmp_id = request.GET.get('id') response_type = request.GET.get("response_type", "application/json") if metric not in METRICS.keys(): return HttpResponseNotFound("<h3>Metric not found</h3>") query = MetricsDb_TreemapData.get_query(session, metric, cmp_type, cmp_id) data = [views_utils.get_query_columns(query)] + query.all() return views_utils.dump_data(data, response_type, filename="treemap_data_" + metric)
def function_id_defects(session, request, function_id, formating='json'): ''' Returns an array of defects for a specific function See docs/frontends/django_implementation.md for a more detailed description of the format Args: session(sqlalchemy session): session to use for queries request: : django request object function_id: : the function_id to use as filter formating(str): Indicates wich format that should be returned, [json|csv] ''' all_fields = ['date', 'defect_id'] try: params = _parse_parameters(all_fields, request) except ValueError: return HttpResponseBadRequest() query = _add_defect_query_columns(session, params) query = query.filter(DefectModification.date.between(params['from'], params['to']))\ .filter(DefectModification.function_id == int(function_id))\ .order_by(DefectModification.date) if params['bins'] is not None: query = query.group_by(DefectModification.function_id) return views_utils.dump_data([params['fields']] + query.all(), response_type=formating, filename=function_id_defects.__name__)
def get_subsystems(session, request): ''' Returns all subsystems as an array of [subsystem_id, subsystem] ''' response_type = request.GET.get("response_type", "application/json") query = session.query(Subsystem.id, Subsystem.subsystem).order_by(Subsystem.id) columns = views_utils.get_query_columns(query) data = [columns] + query.all() return views_utils.dump_data(data, response_type, filename="subsystems")
def metric_descriptions(_): ''' Todo: Make the output csv compatible ''' # The frontend shifts the first row because it should be agnostic towards both csv and json output metrics = ["empty entry for csv"] for metric_name, data in METRICS.iteritems(): entry = {'metric': metric_name} entry.update(data) metrics.append(entry) return views_utils.dump_data(metrics, "application/json")
def get_functions_for_file(session, request): ''' Returns all functions for a file as an array of [function_id, function] ''' response_type = request.GET.get("response_type", "application/json") file_id = request.GET['id'] query = session.query(Function.id, Function.function).filter(Function.file_id == file_id).order_by(Function.id) columns = views_utils.get_query_columns(query) data = [columns] + query.all() return views_utils.dump_data(data, response_type, filename="functions")
def files(session, _, formating='json'): ''' Returns an array of all files and file_id:s See docs/frontends/django_implementation.md for a more detailed description of the format Args: session(sqlalchemy session): session to use for queries formating(str): Indicates wich format that should be returned, [json|csv] ''' query = session.query(File.subsystem_id, File.id, File.file).order_by(File.file) return views_utils.dump_data([views_utils.get_query_columns(query)] + query.all(), response_type=formating, filename=files.__name__)
def get_files_for_subsystem(session, request): ''' Returns all files for a subsystem as an array of [file_id, file] ''' response_type = request.GET.get("response_type", "application/json") subsystem_id = request.GET['id'] query = session.query(File.id, File.file).filter(File.subsystem_id == subsystem_id).order_by(File.id) columns = views_utils.get_query_columns(query) data = [columns] + query.all() return views_utils.dump_data(data, response_type, filename="files")
def get_functions_for_file(session, request): ''' Returns all functions for a file as an array of [function_id, function] ''' response_type = request.GET.get("response_type", "application/json") file_id = request.GET['id'] query = session.query(Function.id, Function.function).filter( Function.file_id == file_id).order_by(Function.id) columns = views_utils.get_query_columns(query) data = [columns] + query.all() return views_utils.dump_data(data, response_type, filename="functions")
def subsystem_id_nloc_threshold(session, request, subsystem_id, formating='json'): ''' Returns the number of functions that is above threshold for a certain subsystem Args: session(sqlalchemy session): session to use for queries request: : django request object ''' threshold = int(request.GET.get("threshold", "300")) thresh_query, total_query = _build_query(session, int(subsystem_id)) data = {'above_threshold': thresh_query.filter(ChangeMetric.nloc >= threshold).count(), 'total_functions': total_query.count()} return views_utils.dump_data(data, response_type=formating)
def get_files_for_subsystem(session, request): ''' Returns all files for a subsystem as an array of [file_id, file] ''' response_type = request.GET.get("response_type", "application/json") subsystem_id = request.GET['id'] query = session.query( File.id, File.file).filter(File.subsystem_id == subsystem_id).order_by(File.id) columns = views_utils.get_query_columns(query) data = [columns] + query.all() return views_utils.dump_data(data, response_type, filename="files")
def subsystems_change_metrics(session, request, formating='json'): ''' Returns an array of change_metrics for all subsystems See docs/frontends/django_implementation.md for a more detailed description of the format Args: session(sqlalchemy session): session to use for queries request: : django request object formating(str): Indicates wich format that should be returned, [json|csv] ''' all_fields = ['subsystem_id', 'date', 'added', 'changed', 'deleted', 'nloc', 'token_count', 'parameter_count', 'cyclomatic_complexity'] try: params = _parse_parameters(all_fields, request) except ValueError: return HttpResponseBadRequest() bins = params['bins'] if bins is None: fid_cache, total_values = _get_initial_continous_data(session, params) query = session.query(ChangeMetric.function_id).filter(ChangeMetric.date.between(params['from'], params['to']))\ .order_by(ChangeMetric.date) if 'subsystem_id' in params['fields']: query = query.add_columns(File.subsystem_id) query = query.filter(File.id == ChangeMetric.file_id) total_data = _get_running_totals(query, params['fields'], fid_cache, total_values, [params['fields']]) elif bins > 0: return HttpResponseBadRequest("Not supported yet") elif bins == 0: query = session.query(File.subsystem_id).outerjoin(ChangeMetric, File.id == ChangeMetric.file_id)\ .group_by(File.subsystem_id) total_data = [params['fields']] + _get_change_metric_snapshot(session, params, query) return views_utils.dump_data(total_data, response_type=formating, filename=subsystems_change_metrics.__name__)
def lineview_data(session, request): ''' Dumps data that has accumulated over time. This data is best suited for chart like visualizations. QueryDict params: cmp_type (str): The type of component. "subsystem"|"file"|"function" cmp_id (int): The component id found in the database metric (str): A metric that must exist in the global METRICS dict. response_type (str): "application/json"|"text/csv" Args: request (django.http.HttpRequest) Returns: django.http.HttpResponse ''' cmp_type = request.GET['type'] cmp_id = request.GET['id'] metric = request.GET['metric'] response_type = request.GET.get("response_type", "application/json") if metric not in METRICS.keys(): return HttpResponseNotFound("<h3>Metric not found</h3>") db = settings.METRICSDB cmp_name = "" if cmp_type == "subsystem": cmp_name = session.query( Subsystem.subsystem).filter(Subsystem.id == cmp_id).scalar() data = db.get_changemetric_for_subsystem(session, cmp_id, metric) elif cmp_type == "file": cmp_name = session.query(File.file).filter(File.id == cmp_id).scalar() data = db.get_changemetric_for_file(session, cmp_id, metric) elif cmp_type == "function": cmp_name = session.query( Function.function).filter(Function.id == cmp_id).scalar() data = db.get_changemetric_for_function(session, cmp_id, metric) columns = [["date", "value"]] data = columns + views_utils.adapt_data(data) return views_utils.dump_data(data, response_type, filename=cmp_name + "_" + metric)
def lineview_data(session, request): ''' Dumps data that has accumulated over time. This data is best suited for chart like visualizations. QueryDict params: cmp_type (str): The type of component. "subsystem"|"file"|"function" cmp_id (int): The component id found in the database metric (str): A metric that must exist in the global METRICS dict. response_type (str): "application/json"|"text/csv" Args: request (django.http.HttpRequest) Returns: django.http.HttpResponse ''' cmp_type = request.GET['type'] cmp_id = request.GET['id'] metric = request.GET['metric'] response_type = request.GET.get("response_type", "application/json") if metric not in METRICS.keys(): return HttpResponseNotFound("<h3>Metric not found</h3>") db = settings.METRICSDB cmp_name = "" if cmp_type == "subsystem": cmp_name = session.query(Subsystem.subsystem).filter(Subsystem.id == cmp_id).scalar() data = db.get_changemetric_for_subsystem(session, cmp_id, metric) elif cmp_type == "file": cmp_name = session.query(File.file).filter(File.id == cmp_id).scalar() data = db.get_changemetric_for_file(session, cmp_id, metric) elif cmp_type == "function": cmp_name = session.query(Function.function).filter(Function.id == cmp_id).scalar() data = db.get_changemetric_for_function(session, cmp_id, metric) columns = [["date", "value"]] data = columns + views_utils.adapt_data(data) return views_utils.dump_data(data, response_type, filename=cmp_name + "_" + metric)
def contributors_defects_data(session, request): """ Returns the contributors and defects for all the files between two dates. QueryDict params: response_type (str) 'application/json'|'text/csv', default='application/json' startdate (json date format) As per ISO 8601, default=datetime.fromtimestamp(0) enddate (json date format) As per ISO 8601, default=datetime.now() Args: request (django.http.HttpRequest) Returns: django.http.HttpResponse """ response_type = request.GET.get("response_type", "application/json") startdate = views_utils.str2datetime(request.GET.get('startdate', datetime.fromtimestamp(0).isoformat())) enddate = views_utils.str2datetime(request.GET.get('enddate', datetime.now().isoformat())) # Pylint incorrectly identifies start/enddate as possible tuples (str2datetime makes sure that is not possible) # plus this code is specifically tested in a unittest so its safe. filename = "contributors_defects_files_" + startdate.strftime("%Y%m%d") + "-" + enddate.strftime("%Y%m%d") # pylint: disable=E1101, C0301 cm1 = aliased(ChangeMetric) cm2 = aliased(ChangeMetric) cm_metrics = session.query(cm1.file_id, MetricsDb_TreemapData.replace_null_0(func.sum(cm1.nloc)).label("nloc"), MetricsDb_TreemapData.replace_null_0(func.sum(cm1.cyclomatic_complexity))\ .label("cyclomatic_complexity"))\ .outerjoin(cm2, and_(cm1.function_id == cm2.function_id, cm1.date < cm2.date))\ .filter(cm2.date.is_(None))\ .group_by(cm1.file_id)\ .subquery() cm_contributors = session.query(ChangeMetric.file_id, func.count(distinct(ChangeMetric.user_id)).label('contributors'))\ .filter(ChangeMetric.date.between(startdate, enddate))\ .group_by(ChangeMetric.file_id)\ .subquery() defect_mods = session.query(DefectModification.file_id, func.count(distinct(DefectModification.user_id)).label('contributors'), func.count(distinct(DefectModification.defect_id)).label('defects'))\ .join(DefectMeta, DefectMeta.id == DefectModification.defect_id)\ .filter(DefectModification.date.between(startdate, enddate))\ .group_by(DefectModification.file_id) defects_a = defect_mods.filter(DefectMeta.severity == 'A').subquery() defects_b = defect_mods.filter(DefectMeta.severity == 'B').subquery() defects_c = defect_mods.filter(DefectMeta.severity == 'C').subquery() defects_improvement = defect_mods.filter(DefectMeta.severity == 'Improvement').subquery() defect_mods = defect_mods.subquery() # Note, we are only retrieveing files with a subsystem with an inner join query = session.query(File.subsystem_id, File.id, File.file, Subsystem.subsystem, MetricsDb_TreemapData.replace_null_0(cm_contributors.c.contributors).label("contributors_cm"), MetricsDb_TreemapData.replace_null_0(defect_mods.c.contributors).label("contributors_tr"), MetricsDb_TreemapData.replace_null_0(defect_mods.c.defects).label("defects"), MetricsDb_TreemapData.replace_null_0(defects_a.c.defects).label("defects_a"), MetricsDb_TreemapData.replace_null_0(defects_b.c.defects).label("defects_b"), MetricsDb_TreemapData.replace_null_0(defects_c.c.defects).label("defects_c"), MetricsDb_TreemapData.replace_null_0(defects_improvement.c.defects)\ .label("defects_improvement"), MetricsDb_TreemapData.replace_null_0(cm_metrics.c.cyclomatic_complexity)\ .label("cyclomatic_complexity"), MetricsDb_TreemapData.replace_null_0(cm_metrics.c.nloc).label("nloc"))\ .join(Subsystem, File.subsystem_id == Subsystem.id)\ .outerjoin(cm_metrics, cm_metrics.c.file_id == File.id)\ .outerjoin(cm_contributors, cm_contributors.c.file_id == File.id)\ .outerjoin(defect_mods, defect_mods.c.file_id == File.id)\ .outerjoin(defects_a, defects_a.c.file_id == File.id)\ .outerjoin(defects_b, defects_b.c.file_id == File.id)\ .outerjoin(defects_c, defects_c.c.file_id == File.id)\ .outerjoin(defects_improvement, defects_improvement.c.file_id == File.id)\ .order_by(File.id) data = [views_utils.get_query_columns(query)] + query.all() return views_utils.dump_data(data, response_type, filename=filename)
def contributors_defects_data(session, request): """ Returns the contributors and defects for all the files between two dates. QueryDict params: response_type (str) 'application/json'|'text/csv', default='application/json' startdate (json date format) As per ISO 8601, default=datetime.fromtimestamp(0) enddate (json date format) As per ISO 8601, default=datetime.now() Args: request (django.http.HttpRequest) Returns: django.http.HttpResponse """ response_type = request.GET.get("response_type", "application/json") startdate = views_utils.str2datetime( request.GET.get('startdate', datetime.fromtimestamp(0).isoformat())) enddate = views_utils.str2datetime( request.GET.get('enddate', datetime.now().isoformat())) # Pylint incorrectly identifies start/enddate as possible tuples (str2datetime makes sure that is not possible) # plus this code is specifically tested in a unittest so its safe. filename = "contributors_defects_files_" + startdate.strftime( "%Y%m%d") + "-" + enddate.strftime("%Y%m%d") # pylint: disable=E1101, C0301 cm1 = aliased(ChangeMetric) cm2 = aliased(ChangeMetric) cm_metrics = session.query(cm1.file_id, MetricsDb_TreemapData.replace_null_0(func.sum(cm1.nloc)).label("nloc"), MetricsDb_TreemapData.replace_null_0(func.sum(cm1.cyclomatic_complexity))\ .label("cyclomatic_complexity"))\ .outerjoin(cm2, and_(cm1.function_id == cm2.function_id, cm1.date < cm2.date))\ .filter(cm2.date.is_(None))\ .group_by(cm1.file_id)\ .subquery() cm_contributors = session.query(ChangeMetric.file_id, func.count(distinct(ChangeMetric.user_id)).label('contributors'))\ .filter(ChangeMetric.date.between(startdate, enddate))\ .group_by(ChangeMetric.file_id)\ .subquery() defect_mods = session.query(DefectModification.file_id, func.count(distinct(DefectModification.user_id)).label('contributors'), func.count(distinct(DefectModification.defect_id)).label('defects'))\ .join(DefectMeta, DefectMeta.id == DefectModification.defect_id)\ .filter(DefectModification.date.between(startdate, enddate))\ .group_by(DefectModification.file_id) defects_a = defect_mods.filter(DefectMeta.severity == 'A').subquery() defects_b = defect_mods.filter(DefectMeta.severity == 'B').subquery() defects_c = defect_mods.filter(DefectMeta.severity == 'C').subquery() defects_improvement = defect_mods.filter( DefectMeta.severity == 'Improvement').subquery() defect_mods = defect_mods.subquery() # Note, we are only retrieveing files with a subsystem with an inner join query = session.query(File.subsystem_id, File.id, File.file, Subsystem.subsystem, MetricsDb_TreemapData.replace_null_0(cm_contributors.c.contributors).label("contributors_cm"), MetricsDb_TreemapData.replace_null_0(defect_mods.c.contributors).label("contributors_tr"), MetricsDb_TreemapData.replace_null_0(defect_mods.c.defects).label("defects"), MetricsDb_TreemapData.replace_null_0(defects_a.c.defects).label("defects_a"), MetricsDb_TreemapData.replace_null_0(defects_b.c.defects).label("defects_b"), MetricsDb_TreemapData.replace_null_0(defects_c.c.defects).label("defects_c"), MetricsDb_TreemapData.replace_null_0(defects_improvement.c.defects)\ .label("defects_improvement"), MetricsDb_TreemapData.replace_null_0(cm_metrics.c.cyclomatic_complexity)\ .label("cyclomatic_complexity"), MetricsDb_TreemapData.replace_null_0(cm_metrics.c.nloc).label("nloc"))\ .join(Subsystem, File.subsystem_id == Subsystem.id)\ .outerjoin(cm_metrics, cm_metrics.c.file_id == File.id)\ .outerjoin(cm_contributors, cm_contributors.c.file_id == File.id)\ .outerjoin(defect_mods, defect_mods.c.file_id == File.id)\ .outerjoin(defects_a, defects_a.c.file_id == File.id)\ .outerjoin(defects_b, defects_b.c.file_id == File.id)\ .outerjoin(defects_c, defects_c.c.file_id == File.id)\ .outerjoin(defects_improvement, defects_improvement.c.file_id == File.id)\ .order_by(File.id) data = [views_utils.get_query_columns(query)] + query.all() return views_utils.dump_data(data, response_type, filename=filename)