def get(self, request):
metric = request.GET["metric"]
cache = redis_conn.get(f"analysis_and_training:{request.user.id}")
filters = loads(cache)["filters"] if cache else None
settings = get_training_settings(request.user)
issues = get_issues(
fields=[
metric.split()[0],
settings.get("mark_up_source"),
"Description_tr",
"Assignee",
"Reporter",
],
filters=filters,
)
df = pd.DataFrame.from_records(issues)
if metric.split()[0] not in ("Resolution", "Priority"):
if settings["mark_up_source"] and settings["mark_up_entities"]:
for area in settings["mark_up_entities"]:
if area["area_of_testing"] == metric.split()[0]:
df = mark_up_series(
df,
settings["mark_up_source"],
metric.split()[0],
area["entities"],
)
significant_terms = calculate_significance_weights(df, metric)
context = {"significant_terms": significant_terms}
return Response(context)
def append_predictions(user: User) -> None:
""" Appends predictions for each issue.
Parameters:
----------
user:
User instance.
"""
bugs = pd.DataFrame(
get_issues(fields=["Key", "Description_tr"],
filters=[UNRESOLVED_BUGS_FILTER]))
# Split DF to process its chunks asynchronously.
chunk_size = ceil(len(bugs) / multiprocessing.cpu_count())
archive_path = get_archive_path(user)
training_parameters = read_from_archive(archive_path,
TRAINING_PARAMETERS_FILENAME)
with multiprocessing.Pool() as pool:
df_predictions = [
pool.apply_async(
calculate_predictions,
args=(chunk, training_parameters, archive_path),
) for chunk in np.array_split(bugs, chunk_size)
]
df_predictions = [prediction.get() for prediction in df_predictions]
df_predictions = pd.concat(df_predictions)
del df_predictions["Description_tr"]
update_issues(df_predictions.T.to_dict().values())
def task_extract_updated_issues(self):
logger.info(f"STARTED: issues updating. Task: {self.request}")
issues = get_issues(fields=["Key", "Updated"])
if issues:
request_args = JAPI().get_update_args(issues)
tasks = [request_issues.s(jql=args) for args in request_args]
results = group(*tasks).apply_async()
while not results.ready():
sleep(60 * 2)
logger.info(f"FINISHED: issues updating. Task: {self.request}")
def test_filter_label_negative(self):
filter = [{
"name": "Labels",
"filtration_type": "string",
"current_value": "Nostra",
"exact_match": False,
}]
issues = get_issues(filters=filter)
assert len(issues) == 80
def test_filter_resolution_negative(self):
filter = [{
"name": "Resolution",
"filtration_type": "drop-down",
"current_value": ["Won't Do"],
"exact_match": False,
}]
issues = get_issues(filters=filter)
assert not issues
def test_up_drop_down_fields(self):
new_filters = update_drop_down_fields(filters=self.default_filters,
issues=pd.DataFrame(
get_issues()))
assert all([
new_filters[0]["values"][0] == "Test Project",
set(new_filters[2]["values"]) == set(["Minor", "Major"]),
new_filters[7]["values"][0] == "Open",
set(new_filters[10]["values"]) == set(
["Rejected", "Unresolved", "Done"]),
])
def post(self, request):
instance = request.user
cache = redis_conn.get(f"analysis_and_training:{request.user.id}")
filters = loads(cache)["filters"] if cache else None
fields = get_issues_fields(request.user)
df = pd.DataFrame(get_issues(filters=filters, fields=fields))
# New predictions will be appended after training.
delete_old_predictions()
settings = get_training_settings(request.user)
if settings["mark_up_source"] not in df.columns:
raise InvalidMarkUpSource
resolutions = ([
resolution["value"] for resolution in settings["bug_resolution"]
] if len(settings["bug_resolution"]) != 0 else [])
areas_of_testing = []
if settings["mark_up_source"]:
areas_of_testing = [
area["area_of_testing"]
for area in settings["mark_up_entities"]
] + ["Other"]
for area in settings["mark_up_entities"]:
df = mark_up_series(
df,
settings["mark_up_source"],
area["area_of_testing"],
area["entities"],
)
df = mark_up_other_data(df, areas_of_testing)
delete_training_data(get_archive_path(instance))
train(
instance,
df,
areas_of_testing,
resolutions,
)
context = {
"result": "success",
}
process = Process(target=append_predictions, args=(request.user, ))
process.start()
return Response(context, status=200)
def test_filter_resolution_positive(self):
filter = [
{
"name": "Resolution",
"filtration_type": "drop-down",
"current_value": ["Rejected", "Unresolved"],
"exact_match": False,
},
]
issues = get_issues(filters=filter)
assert len(issues) == 67
def test_filter_project_negative(self):
filter = [
{
"name": "Project",
"filtration_type": "drop-down",
"current_value": ["Nostradamus"],
"exact_match": False,
},
]
issues = get_issues(filters=filter)
assert not issues
def test_filter_project_positive(self):
filter = [
{
"name": "Project",
"filtration_type": "drop-down",
"current_value": ["Test Project"],
"exact_match": False,
},
]
issues = get_issues(filters=filter)
assert len(issues) == 100
def test_filter_priority_negative(self):
filter = [
{
"name": "Priority",
"filtration_type": "drop-down",
"current_value": ["Critical"],
"exact_match": False,
},
]
issues = get_issues(filters=filter)
assert not issues
def test_filter_priority_positive(self):
filter = [
{
"name": "Priority",
"filtration_type": "drop-down",
"current_value": ["Minor"],
"exact_match": False,
},
]
issues = get_issues(filters=filter)
assert len(issues) == 20
def get(self, request):
user = request.user
check_training_files(user)
check_predictions()
fields = get_qa_metrics_fields(user)
issues = DataFrame.from_records(
get_issues(filters=[UNRESOLVED_BUGS_FILTER], fields=fields))
filters = get_qa_metrics_settings(user)
filters = update_drop_down_fields(filters, issues)
return Response(filters)
def get(self, request):
period = request.GET["period"]
cache = redis_conn.get(f"analysis_and_training:{request.user.id}")
filters = loads(cache)["filters"] if cache else None
bugs = pd.DataFrame(
get_issues(fields=["Key", "Created"], filters=filters))
if bugs.empty:
return Response({})
coordinates = calculate_defect_submission(bugs, period)
context = {"submission_chart": coordinates}
return Response(context)
def test_filter_created_positive_right(self):
filter = [{
"name":
"Created",
"filtration_type":
"date",
"current_value": [
None,
dt.strftime(dt.now() + td(days=1), "%Y-%m-%dT%H:%M:%S.%f%zZ"),
],
"exact_match":
False,
}]
issues = get_issues(filters=filter)
assert len(issues) == 100
def test_filter_created_both_negative(self):
filter = [{
"name":
"Created",
"filtration_type":
"date",
"current_value": [
dt.strftime(dt.now() + td(days=1), "%Y-%m-%dT%H:%M:%S.%f%zZ"),
dt.strftime(dt.now() - td(days=1), "%Y-%m-%dT%H:%M:%S.%f%zZ"),
],
"exact_match":
False,
}]
issues = get_issues(filters=filter)
assert not issues
def post(self, request):
fields = get_issues_fields(request.user)
issues = get_issues_dataframe(fields=fields)
filters = get_filters(
request.user,
issues=issues,
)
if request.data.get("action") == "apply":
new_filters = request.data.get("filters")
if new_filters:
for new_filter in new_filters:
for filter_ in filters:
if new_filter["name"] == filter_["name"]:
filter_.update(
{
"current_value": new_filter[
"current_value"
],
"filtration_type": new_filter[
"filtration_type"
],
"exact_match": new_filter["exact_match"],
}
)
issues = get_issues(filters=filters, fields=fields)
issues_count = len(issues)
context = {
"records_count": {
"total": get_issue_count(),
"filtered": issues_count,
},
"filters": filters,
}
for element in context:
redis_conn.set(
f"user:{request.user.id}:analysis_and_training:{element}",
dumps(context.get(element)),
)
remove_cache_record(
"analysis_and_training:defect_submission", request.user.id
)
return Response(context)
def get_predictions_table(settings, filters, offset, limit) -> pd.DataFrame:
""" Reads bugs predictions for according user settings.
Parameters:
----------
settings:
Predictions table settings.
filters:
Filters.
offset:
Start index to read bugs.
limit:
Count of rows to be read.
Returns:
----------
Bugs predictions.
"""
filters = [UNRESOLVED_BUGS_FILTER] + filters
bugs = pd.DataFrame(get_issues(filters=filters))
if bugs.empty:
return pd.DataFrame()
if offset is not None and limit is not None:
bugs = paginate_bugs(bugs, offset, limit)
prediction_table_fields = [field["name"] for field in settings]
for field in prediction_table_fields:
if field.startswith("Resolution:"):
class_ = field.replace("Resolution: ", "")
bugs[field] = bugs["Resolution_prediction"].apply(
lambda value: max(value.get(class_), key=value.get(class_).get
))
else:
bugs[field] = bugs[PREDICTIONS_TABLE_FIELD_MAPPING.get(
field, field)].apply(lambda value: max(value, key=value.get)
if isinstance(value, dict) else value)
bugs = bugs[prediction_table_fields]
return bugs
def get_issues_dataframe(
fields: list = None, filters: list = None
) -> pd.DataFrame:
"""Converts issues to optimized pandas dataframe.
Parameters:
----------
fields:
Issue fields.
filters:
Filters to be applied.
Returns:
----------
Optimized issues.
"""
issues = pd.DataFrame(get_issues(fields=fields, filters=filters))
return preprocess_df(issues)
def get_issues_for_report(fields, filters):
""" Query database for issues which will be written to a report.
Parameters
----------
fields:
Issue fields.
filters:
Filters to be applied.
Returns
-------
Issues.
"""
issues = DataFrame.from_records(get_issues(fields=fields, filters=filters))
issues = issues.reindex(fields, axis=1)
return issues
def get(self, request):
cache = redis_conn.get(f"analysis_and_training:{request.user.id}")
if cache:
return Response(loads(cache))
fields = get_issues_fields(request.user)
issues = get_issues(fields=fields)
if not issues:
# TODO: FE shows progress bar when data is empty
return Response({})
issues = pd.DataFrame.from_records(issues)
freq_terms = calculate_frequently_terms(issues)
statistics = calculate_statistics(
df=issues, series=["Comments", "Attachments", "Time to Resolve"])
defect_submission = calculate_defect_submission(df=issues,
period="Month")
significant_terms = get_significant_terms(issues)
filters = get_filters(request.user, issues=issues)
context = {
"records_count": {
"total": len(issues),
"filtered": len(issues)
},
"frequently_terms": freq_terms,
"statistics": statistics,
"submission_chart": defect_submission,
"significant_terms": significant_terms,
"filters": filters,
}
redis_conn.set(
name=f"analysis_and_training:{request.user.id}",
value=dumps(context),
ex=60 * 30,
)
return Response(context)
def post(self, request):
fields = get_issues_fields(request.user)
filters = get_filters(
request.user,
issues=pd.DataFrame.from_records(get_issues(fields=fields)),
)
if request.data.get("action") == "apply":
new_filters = request.data.get("filters")
if new_filters:
for new_filter in new_filters:
for filter_ in filters:
if new_filter["name"] == filter_["name"]:
filter_.update({
"current_value":
new_filter["current_value"],
"filtration_type":
new_filter["filtration_type"],
"exact_match":
new_filter["exact_match"],
})
issues = get_issues(filters=filters, fields=fields)
else:
issues = get_issues(fields=fields)
else:
issues = get_issues(fields=fields)
if len(issues) == 0:
context = {
"records_count": {
"total": get_issue_count(),
"filtered": 0
},
"frequently_terms": [],
"statistics": {},
"submission_chart": {},
"significant_terms": {},
"filters": filters,
}
redis_conn.set(f"analysis_and_training:{request.user.id}",
dumps(context))
return Response({})
issues = pd.DataFrame.from_records(issues)
freq_terms = calculate_frequently_terms(issues)
statistics = calculate_statistics(
df=issues, series=["Comments", "Attachments", "Time to Resolve"])
submission_chart = calculate_defect_submission(df=issues,
period="Month")
significant_terms = get_significant_terms(
issues, get_training_settings(request.user))
context = {
"records_count": {
"total": get_issue_count(),
"filtered": len(issues),
},
"frequently_terms": freq_terms,
"statistics": statistics,
"submission_chart": submission_chart,
"significant_terms": significant_terms,
"filters": filters,
}
redis_conn.set(f"analysis_and_training:{request.user.id}",
dumps(context))
return Response(context)
