def generate_chart_url(body): #noga: E501
response = None
if connexion.request.is_json:
content = connexion.request.get_json()
app.logger.info("Generate Chart: {}".format(content))
request = PrioritizedRecommendationsRequest.from_dict(content)
if request.unique_key() in CACHED_CHART_URLs:
chart_url = CACHED_CHART_URLs[request.unique_key()]
chart_key = chart_url.split("/")[-1]
if chart_key in CHART_REQUESTs:
return ChartResponse(False, None, CACHED_CHART_URLs[request.unique_key()])
chart_key = "".join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(16))
chart_url = "http://{}:{}/prioritizer/chart/c/{}".format(helper.app_host(), helper.app_port(), chart_key)
limit_bugs = 0 #800
bugzilla_fetcher = bugzillafetcher.BugzillaFetcher("https://bugs.eclipse.org/bugs/rest/bug")
bugs = bugzilla_fetcher.fetch_bugs(request.assignee, request.products, request.components,
"RESOLVED", limit=limit_bugs)
requirements = list(map(lambda b: Requirement.from_bug(b), bugs))
ASSIGNED_RESOLVED_REQUIREMENTS_OF_STAKEHOLDER[request.unique_key()] = requirements
CACHED_CHART_URLs[request.unique_key()] = chart_url
CHART_REQUESTs[chart_key] = request
response = ChartResponse(False, None, chart_url)
return response
def compute_popularity(body): # noqa: E501
"""Retrieve a list with values for given set of requirements indicating their popularity for the crowd on twitter.
# noqa: E501
:param body: Requirement objects for which the social popularity should be measured
:type body: list | bytes
:rtype: List[RequirementPopularity]
"""
response_list = []
if connexion.request.is_json:
content = connexion.request.get_json()
assert isinstance(content, list)
requirements = [Requirement.from_dict(d)
for d in content] # noqa: E501
requirements = list(
map(
lambda r: requirement.Requirement(r.id, r.title, r.description
), requirements))
requirements = preprocessing.preprocess_requirements(
requirements,
enable_pos_tagging=True,
enable_lemmatization=False,
enable_stemming=False)
# Extend stop word list: https://www.wordfrequency.info/free.asp?s=y
maut_results = []
for requ in requirements:
maut_temp = 0
if len(list(requ.title_tokens_pos_tags)) > 0:
for tag in set(requ.title_tokens_pos_tags +
requ.description_tokens_pos_tags):
for matching_pos_classes in ["NN", "NNS", "NE", "FW"]:
if matching_pos_classes in tag:
maut_temp += fetch_twitter(str(tag[0]))
else:
for token in set(requ.title_tokens + requ.description_tokens):
print("Note here!!")
maut_temp += fetch_twitter(token)
maut_results.append(maut_temp)
for idx, requ in enumerate(requirements):
response_list.append(
RequirementPopularity(
id=requ.id,
#'totalNumberOfFavorites': favorite_count,
#'totalNumberOfRetweets': retweet_count,
#'MAUT': (maut_result/num_of_tweets),
#'popularity': "{0:.5f}".format((maut_results[i] / sum(maut_results))*100)
popularity=((maut_results[idx] * 100) / sum(maut_results)
if sum(maut_results) > 0 else 0)))
return response_list
def test_compute_popularity(self):
"""Test case for compute_popularity
Retrieve a list with values for given set of requirements indicating their popularity for the crowd on twitter.
"""
body = [Requirement()]
response = self.client.open('/v1/popularity',
method='POST',
data=json.dumps(body),
content_type='application/json')
self.assert200(response,
'Response body is : ' + response.data.decode('utf-8'))
def perform_svd():
enable_tagging = True
max_distance = 0.6
with open(os.path.join(helper.APP_PATH, "data",
"requirements_en.json")) as f:
requs = json.load(f)
max_distance = 0.4
with open(
os.path.join(helper.APP_PATH, "data",
"siemens_requirements_en.csv")) as f:
enable_tagging = False
plain_requirements = csv_reader(f)
requs = []
for (idx, description) in enumerate(plain_requirements):
if idx > 400:
break
requs += [{'id': idx, 'title': '', 'description': description}]
#print(json.dumps(requs))
#import sys;sys.exit()
#pprint(requs)
requs = list(map(lambda r: Requirement.from_dict(r), requs))
lang = "en"
requs = list(
map(lambda r: requirement.Requirement(r.id, r.title, r.description),
requs))
requs = preprocessing.preprocess_requirements(
requs,
enable_pos_tagging=enable_tagging,
enable_lemmatization=enable_tagging,
enable_stemming=False,
lang=lang)
_logger.info("SVD...")
predictions_map = svd.svd(requs, k=3, max_distance=max_distance)
for subject_requirement, similar_requirements in predictions_map.items():
if len(similar_requirements) == 0:
continue
#print("-" * 80)
#print(subject_requirement.description_tokens)
for similar_requirement in similar_requirements:
print("#{}: {} -> #{}: {}".format(
subject_requirement.id, subject_requirement.description[:80],
similar_requirement.id, similar_requirement.description[:80]))
def recommend_requirement_dependencies(body): # noqa: E501
"""Retrieve a list with values for given set of requirements indicating their popularity for the crowd on twitter.
# noqa: E501
:param body: Requirement objects for which the social popularity should be measured
:type body: list | bytes
:rtype: List[Requirement]
"""
response_list = []
# TODO: introduce parameter to set language
lang = "en"
if connexion.request.is_json:
content = connexion.request.get_json()
assert isinstance(content, list)
requs = [Requirement.from_dict(d) for d in content] # noqa: E501
requs = list(map(lambda r: requirement.Requirement(r.id, r.title, r.description, r.comments), requs))
for r in requs:
r.append_comments_to_description()
requs = preprocessing.preprocess_requirements(requs,
enable_stemming=False,
lang=lang)
requs = list(filter(lambda r: len(r.tokens()) > 0, requs))
if len(requs) == 0:
return []
_logger.info("SVD...")
if len(requs) > 100:
min_distance, max_distance = 0.2, 0.5
k = 10
elif len(requs) > 50:
min_distance, max_distance = 0.2, 0.6
k = 8
elif len(requs) > 30:
min_distance, max_distance = 0.2, 0.65
k = 5
elif len(requs) > 10:
min_distance, max_distance = 0.2, 0.7
k = 3
elif len(requs) > 5:
min_distance, max_distance = 0.2, 0.75
k = 2
else:
min_distance, max_distance = 0.2, 0.8
k = 1
predictions_map = svd.svd(requs, k=k, min_distance=min_distance, max_distance=max_distance)
dependency_pairs = set()
for subject_requirement, dependent_requirements in predictions_map.items():
requ = Requirement.from_dict({
"id": subject_requirement.id,
"title": subject_requirement.title,
"description": subject_requirement.description,
"comments": subject_requirement.comments
})
rx = subject_requirement.id
dependent_requirement_ids = list(set(map(lambda r: r.id, dependent_requirements)))
all_undirected_pairs_of_subject_requirement = map(lambda ry: [(rx, ry), (ry, rx)], dependent_requirement_ids)
dependency_pairs_of_subject_requirement = set(list(itertools.chain(*all_undirected_pairs_of_subject_requirement)))
remaining_dependency_pairs_of_subject_requirement = dependency_pairs_of_subject_requirement - dependency_pairs
dependency_pairs = dependency_pairs.union(remaining_dependency_pairs_of_subject_requirement)
predictions = list(set(map(lambda t: t[0] if t[0] != rx else t[1], remaining_dependency_pairs_of_subject_requirement)))
requ.predictions = predictions
response_list += [requ]
for dependent_requirement in dependent_requirements:
print("{} -> {}".format(subject_requirement, dependent_requirement))
"""
for idx, requ in enumerate(requirements):
response_list.append(Requirement.from_dict({
"id": requ.id,
"title": requ.title,
"description": requ.description
}))
"""
return response_list
def recommend_similar_requirements(body): # noqa: E501
"""Retrieve a list with values for given set of requirements indicating their popularity for the crowd on twitter.
# noqa: E501
:param body: Requirement objects for which the social popularity should be measured
:type body: list | bytes
:rtype: List[Requirement]
"""
response_list = []
# TODO: introduce parameter to set language
lang = "en"
if connexion.request.is_json:
content = connexion.request.get_json()
assert isinstance(content, list)
requs = [Requirement.from_dict(d) for d in content] # noqa: E501
requs = list(
map(
lambda r: requirement.Requirement(r.id, r.title, r.description,
r.comments), requs))
for r in requs:
r.append_comments_to_description()
requs = preprocessing.preprocess_requirements(requs, lang=lang)
requs = list(filter(lambda r: len(r.tokens()) > 0, requs))
if len(requs) == 0:
return []
_logger.info("SVD...")
if len(requs) > 100:
max_distance = 0.4
k = 10
elif len(requs) > 50:
max_distance = 0.5
k = 8
elif len(requs) > 30:
max_distance = 0.55
k = 5
elif len(requs) > 10:
max_distance = 0.6
k = 3
elif len(requs) > 5:
max_distance = 0.6
k = 2
else:
max_distance = 0.6
k = 1
predictions_map = svd.svd(requs, k=k, max_distance=max_distance)
predictions = {}
for subject_requirement, similar_requirements in predictions_map.items(
):
rx = subject_requirement.id
rx_predictions = list(
set(map(lambda r: r.id, similar_requirements)))
if rx not in predictions:
predictions[rx] = set()
predictions[rx] = predictions[rx].union(rx_predictions)
for ry in rx_predictions:
if ry not in predictions:
predictions[ry] = set()
predictions[ry].add(rx)
for subject_requirement, similar_requirements in predictions_map.items(
):
requ = Requirement.from_dict({
"id":
subject_requirement.id,
"title":
subject_requirement.title,
"description":
subject_requirement.description,
"comments":
subject_requirement.comments
})
rx = subject_requirement.id
requ.predictions = list(predictions[rx])
response_list += [requ]
for similar_requirement in similar_requirements:
print("{} -> {}".format(subject_requirement,
similar_requirement))
"""
for idx, requ in enumerate(requirements):
response_list.append(Requirement.from_dict({
"id": requ.id,
"title": requ.title,
"description": requ.description
}))
"""
return response_list