def get(self, request): metric = request.GET["metric"] cache = redis_conn.get(f"analysis_and_training:{request.user.id}") filters = loads(cache)["filters"] if cache else None settings = get_training_settings(request.user) issues = get_issues( fields=[ metric.split()[0], settings.get("mark_up_source"), "Description_tr", "Assignee", "Reporter", ], filters=filters, ) df = pd.DataFrame.from_records(issues) if metric.split()[0] not in ("Resolution", "Priority"): if settings["mark_up_source"] and settings["mark_up_entities"]: for area in settings["mark_up_entities"]: if area["area_of_testing"] == metric.split()[0]: df = mark_up_series( df, settings["mark_up_source"], metric.split()[0], area["entities"], ) significant_terms = calculate_significance_weights(df, metric) context = {"significant_terms": significant_terms} return Response(context)
def append_predictions(user: User) -> None: """ Appends predictions for each issue. Parameters: ---------- user: User instance. """ bugs = pd.DataFrame( get_issues(fields=["Key", "Description_tr"], filters=[UNRESOLVED_BUGS_FILTER])) # Split DF to process its chunks asynchronously. chunk_size = ceil(len(bugs) / multiprocessing.cpu_count()) archive_path = get_archive_path(user) training_parameters = read_from_archive(archive_path, TRAINING_PARAMETERS_FILENAME) with multiprocessing.Pool() as pool: df_predictions = [ pool.apply_async( calculate_predictions, args=(chunk, training_parameters, archive_path), ) for chunk in np.array_split(bugs, chunk_size) ] df_predictions = [prediction.get() for prediction in df_predictions] df_predictions = pd.concat(df_predictions) del df_predictions["Description_tr"] update_issues(df_predictions.T.to_dict().values())
def task_extract_updated_issues(self): logger.info(f"STARTED: issues updating. Task: {self.request}") issues = get_issues(fields=["Key", "Updated"]) if issues: request_args = JAPI().get_update_args(issues) tasks = [request_issues.s(jql=args) for args in request_args] results = group(*tasks).apply_async() while not results.ready(): sleep(60 * 2) logger.info(f"FINISHED: issues updating. Task: {self.request}")
def test_filter_label_negative(self): filter = [{ "name": "Labels", "filtration_type": "string", "current_value": "Nostra", "exact_match": False, }] issues = get_issues(filters=filter) assert len(issues) == 80
def test_filter_resolution_negative(self): filter = [{ "name": "Resolution", "filtration_type": "drop-down", "current_value": ["Won't Do"], "exact_match": False, }] issues = get_issues(filters=filter) assert not issues
def test_up_drop_down_fields(self): new_filters = update_drop_down_fields(filters=self.default_filters, issues=pd.DataFrame( get_issues())) assert all([ new_filters[0]["values"][0] == "Test Project", set(new_filters[2]["values"]) == set(["Minor", "Major"]), new_filters[7]["values"][0] == "Open", set(new_filters[10]["values"]) == set( ["Rejected", "Unresolved", "Done"]), ])
def post(self, request): instance = request.user cache = redis_conn.get(f"analysis_and_training:{request.user.id}") filters = loads(cache)["filters"] if cache else None fields = get_issues_fields(request.user) df = pd.DataFrame(get_issues(filters=filters, fields=fields)) # New predictions will be appended after training. delete_old_predictions() settings = get_training_settings(request.user) if settings["mark_up_source"] not in df.columns: raise InvalidMarkUpSource resolutions = ([ resolution["value"] for resolution in settings["bug_resolution"] ] if len(settings["bug_resolution"]) != 0 else []) areas_of_testing = [] if settings["mark_up_source"]: areas_of_testing = [ area["area_of_testing"] for area in settings["mark_up_entities"] ] + ["Other"] for area in settings["mark_up_entities"]: df = mark_up_series( df, settings["mark_up_source"], area["area_of_testing"], area["entities"], ) df = mark_up_other_data(df, areas_of_testing) delete_training_data(get_archive_path(instance)) train( instance, df, areas_of_testing, resolutions, ) context = { "result": "success", } process = Process(target=append_predictions, args=(request.user, )) process.start() return Response(context, status=200)
def test_filter_resolution_positive(self): filter = [ { "name": "Resolution", "filtration_type": "drop-down", "current_value": ["Rejected", "Unresolved"], "exact_match": False, }, ] issues = get_issues(filters=filter) assert len(issues) == 67
def test_filter_project_negative(self): filter = [ { "name": "Project", "filtration_type": "drop-down", "current_value": ["Nostradamus"], "exact_match": False, }, ] issues = get_issues(filters=filter) assert not issues
def test_filter_project_positive(self): filter = [ { "name": "Project", "filtration_type": "drop-down", "current_value": ["Test Project"], "exact_match": False, }, ] issues = get_issues(filters=filter) assert len(issues) == 100
def test_filter_priority_negative(self): filter = [ { "name": "Priority", "filtration_type": "drop-down", "current_value": ["Critical"], "exact_match": False, }, ] issues = get_issues(filters=filter) assert not issues
def test_filter_priority_positive(self): filter = [ { "name": "Priority", "filtration_type": "drop-down", "current_value": ["Minor"], "exact_match": False, }, ] issues = get_issues(filters=filter) assert len(issues) == 20
def get(self, request): user = request.user check_training_files(user) check_predictions() fields = get_qa_metrics_fields(user) issues = DataFrame.from_records( get_issues(filters=[UNRESOLVED_BUGS_FILTER], fields=fields)) filters = get_qa_metrics_settings(user) filters = update_drop_down_fields(filters, issues) return Response(filters)
def get(self, request): period = request.GET["period"] cache = redis_conn.get(f"analysis_and_training:{request.user.id}") filters = loads(cache)["filters"] if cache else None bugs = pd.DataFrame( get_issues(fields=["Key", "Created"], filters=filters)) if bugs.empty: return Response({}) coordinates = calculate_defect_submission(bugs, period) context = {"submission_chart": coordinates} return Response(context)
def test_filter_created_positive_right(self): filter = [{ "name": "Created", "filtration_type": "date", "current_value": [ None, dt.strftime(dt.now() + td(days=1), "%Y-%m-%dT%H:%M:%S.%f%zZ"), ], "exact_match": False, }] issues = get_issues(filters=filter) assert len(issues) == 100
def test_filter_created_both_negative(self): filter = [{ "name": "Created", "filtration_type": "date", "current_value": [ dt.strftime(dt.now() + td(days=1), "%Y-%m-%dT%H:%M:%S.%f%zZ"), dt.strftime(dt.now() - td(days=1), "%Y-%m-%dT%H:%M:%S.%f%zZ"), ], "exact_match": False, }] issues = get_issues(filters=filter) assert not issues
def post(self, request): fields = get_issues_fields(request.user) issues = get_issues_dataframe(fields=fields) filters = get_filters( request.user, issues=issues, ) if request.data.get("action") == "apply": new_filters = request.data.get("filters") if new_filters: for new_filter in new_filters: for filter_ in filters: if new_filter["name"] == filter_["name"]: filter_.update( { "current_value": new_filter[ "current_value" ], "filtration_type": new_filter[ "filtration_type" ], "exact_match": new_filter["exact_match"], } ) issues = get_issues(filters=filters, fields=fields) issues_count = len(issues) context = { "records_count": { "total": get_issue_count(), "filtered": issues_count, }, "filters": filters, } for element in context: redis_conn.set( f"user:{request.user.id}:analysis_and_training:{element}", dumps(context.get(element)), ) remove_cache_record( "analysis_and_training:defect_submission", request.user.id ) return Response(context)
def get_predictions_table(settings, filters, offset, limit) -> pd.DataFrame: """ Reads bugs predictions for according user settings. Parameters: ---------- settings: Predictions table settings. filters: Filters. offset: Start index to read bugs. limit: Count of rows to be read. Returns: ---------- Bugs predictions. """ filters = [UNRESOLVED_BUGS_FILTER] + filters bugs = pd.DataFrame(get_issues(filters=filters)) if bugs.empty: return pd.DataFrame() if offset is not None and limit is not None: bugs = paginate_bugs(bugs, offset, limit) prediction_table_fields = [field["name"] for field in settings] for field in prediction_table_fields: if field.startswith("Resolution:"): class_ = field.replace("Resolution: ", "") bugs[field] = bugs["Resolution_prediction"].apply( lambda value: max(value.get(class_), key=value.get(class_).get )) else: bugs[field] = bugs[PREDICTIONS_TABLE_FIELD_MAPPING.get( field, field)].apply(lambda value: max(value, key=value.get) if isinstance(value, dict) else value) bugs = bugs[prediction_table_fields] return bugs
def get_issues_dataframe( fields: list = None, filters: list = None ) -> pd.DataFrame: """Converts issues to optimized pandas dataframe. Parameters: ---------- fields: Issue fields. filters: Filters to be applied. Returns: ---------- Optimized issues. """ issues = pd.DataFrame(get_issues(fields=fields, filters=filters)) return preprocess_df(issues)
def get_issues_for_report(fields, filters): """ Query database for issues which will be written to a report. Parameters ---------- fields: Issue fields. filters: Filters to be applied. Returns ------- Issues. """ issues = DataFrame.from_records(get_issues(fields=fields, filters=filters)) issues = issues.reindex(fields, axis=1) return issues
def get(self, request): cache = redis_conn.get(f"analysis_and_training:{request.user.id}") if cache: return Response(loads(cache)) fields = get_issues_fields(request.user) issues = get_issues(fields=fields) if not issues: # TODO: FE shows progress bar when data is empty return Response({}) issues = pd.DataFrame.from_records(issues) freq_terms = calculate_frequently_terms(issues) statistics = calculate_statistics( df=issues, series=["Comments", "Attachments", "Time to Resolve"]) defect_submission = calculate_defect_submission(df=issues, period="Month") significant_terms = get_significant_terms(issues) filters = get_filters(request.user, issues=issues) context = { "records_count": { "total": len(issues), "filtered": len(issues) }, "frequently_terms": freq_terms, "statistics": statistics, "submission_chart": defect_submission, "significant_terms": significant_terms, "filters": filters, } redis_conn.set( name=f"analysis_and_training:{request.user.id}", value=dumps(context), ex=60 * 30, ) return Response(context)
def post(self, request): fields = get_issues_fields(request.user) filters = get_filters( request.user, issues=pd.DataFrame.from_records(get_issues(fields=fields)), ) if request.data.get("action") == "apply": new_filters = request.data.get("filters") if new_filters: for new_filter in new_filters: for filter_ in filters: if new_filter["name"] == filter_["name"]: filter_.update({ "current_value": new_filter["current_value"], "filtration_type": new_filter["filtration_type"], "exact_match": new_filter["exact_match"], }) issues = get_issues(filters=filters, fields=fields) else: issues = get_issues(fields=fields) else: issues = get_issues(fields=fields) if len(issues) == 0: context = { "records_count": { "total": get_issue_count(), "filtered": 0 }, "frequently_terms": [], "statistics": {}, "submission_chart": {}, "significant_terms": {}, "filters": filters, } redis_conn.set(f"analysis_and_training:{request.user.id}", dumps(context)) return Response({}) issues = pd.DataFrame.from_records(issues) freq_terms = calculate_frequently_terms(issues) statistics = calculate_statistics( df=issues, series=["Comments", "Attachments", "Time to Resolve"]) submission_chart = calculate_defect_submission(df=issues, period="Month") significant_terms = get_significant_terms( issues, get_training_settings(request.user)) context = { "records_count": { "total": get_issue_count(), "filtered": len(issues), }, "frequently_terms": freq_terms, "statistics": statistics, "submission_chart": submission_chart, "significant_terms": significant_terms, "filters": filters, } redis_conn.set(f"analysis_and_training:{request.user.id}", dumps(context)) return Response(context)