def preprocess():
if not request.json or 'requirements' not in request.json:
abort(
400,
'The input json is empty or it does not contain a requirements array'
)
stemmer = request.args.get('stemmer', '')
if stemmer == "":
abort(400, 'The stemmer parameter is missing')
if stemmer != 'true' and stemmer != 'false':
abort(400, 'The stemmer parameter is not correct')
requirements = []
for json_req in request.json['requirements']:
if 'id' not in json_req:
abort(400, 'There is a requirement without id')
id = json_req['id']
title = ''
description = ''
if 'title' in json_req and json_req['title'] is not None:
title = json_req['title']
if 'description' in json_req and json_req['description'] is not None:
description = json_req['description']
requirements.append(Requirement(id, title, description, ''))
if len(requirements) == 0:
abort(400, 'The input requirements array is empty')
if stemmer == 'true':
preprocessed_requirements = preprocessing.preprocess_requirements(
requirements, True)
else:
preprocessed_requirements = preprocessing.preprocess_requirements(
requirements, False)
result = {'requirements': []}
for requirement in preprocessed_requirements:
result['requirements'].append(encoder(requirement))
return result, 200
def compute_popularity(body): # noqa: E501
"""Retrieve a list with values for given set of requirements indicating their popularity for the crowd on twitter.
# noqa: E501
:param body: Requirement objects for which the social popularity should be measured
:type body: list | bytes
:rtype: List[RequirementPopularity]
"""
response_list = []
if connexion.request.is_json:
content = connexion.request.get_json()
assert isinstance(content, list)
requirements = [Requirement.from_dict(d)
for d in content] # noqa: E501
requirements = list(
map(
lambda r: requirement.Requirement(r.id, r.title, r.description
), requirements))
requirements = preprocessing.preprocess_requirements(
requirements,
enable_pos_tagging=True,
enable_lemmatization=False,
enable_stemming=False)
# Extend stop word list: https://www.wordfrequency.info/free.asp?s=y
maut_results = []
for requ in requirements:
maut_temp = 0
if len(list(requ.title_tokens_pos_tags)) > 0:
for tag in set(requ.title_tokens_pos_tags +
requ.description_tokens_pos_tags):
for matching_pos_classes in ["NN", "NNS", "NE", "FW"]:
if matching_pos_classes in tag:
maut_temp += fetch_twitter(str(tag[0]))
else:
for token in set(requ.title_tokens + requ.description_tokens):
print("Note here!!")
maut_temp += fetch_twitter(token)
maut_results.append(maut_temp)
for idx, requ in enumerate(requirements):
response_list.append(
RequirementPopularity(
id=requ.id,
#'totalNumberOfFavorites': favorite_count,
#'totalNumberOfRetweets': retweet_count,
#'MAUT': (maut_result/num_of_tweets),
#'popularity': "{0:.5f}".format((maut_results[i] / sum(maut_results))*100)
popularity=((maut_results[idx] * 100) / sum(maut_results)
if sum(maut_results) > 0 else 0)))
return response_list
def perform_svd():
enable_tagging = True
max_distance = 0.6
with open(os.path.join(helper.APP_PATH, "data",
"requirements_en.json")) as f:
requs = json.load(f)
max_distance = 0.4
with open(
os.path.join(helper.APP_PATH, "data",
"siemens_requirements_en.csv")) as f:
enable_tagging = False
plain_requirements = csv_reader(f)
requs = []
for (idx, description) in enumerate(plain_requirements):
if idx > 400:
break
requs += [{'id': idx, 'title': '', 'description': description}]
#print(json.dumps(requs))
#import sys;sys.exit()
#pprint(requs)
requs = list(map(lambda r: Requirement.from_dict(r), requs))
lang = "en"
requs = list(
map(lambda r: requirement.Requirement(r.id, r.title, r.description),
requs))
requs = preprocessing.preprocess_requirements(
requs,
enable_pos_tagging=enable_tagging,
enable_lemmatization=enable_tagging,
enable_stemming=False,
lang=lang)
_logger.info("SVD...")
predictions_map = svd.svd(requs, k=3, max_distance=max_distance)
for subject_requirement, similar_requirements in predictions_map.items():
if len(similar_requirements) == 0:
continue
#print("-" * 80)
#print(subject_requirement.description_tokens)
for similar_requirement in similar_requirements:
print("#{}: {} -> #{}: {}".format(
subject_requirement.id, subject_requirement.description[:80],
similar_requirement.id, similar_requirement.description[:80]))
def recommend_requirement_dependencies(body): # noqa: E501
"""Retrieve a list with values for given set of requirements indicating their popularity for the crowd on twitter.
# noqa: E501
:param body: Requirement objects for which the social popularity should be measured
:type body: list | bytes
:rtype: List[Requirement]
"""
response_list = []
# TODO: introduce parameter to set language
lang = "en"
if connexion.request.is_json:
content = connexion.request.get_json()
assert isinstance(content, list)
requs = [Requirement.from_dict(d) for d in content] # noqa: E501
requs = list(map(lambda r: requirement.Requirement(r.id, r.title, r.description, r.comments), requs))
for r in requs:
r.append_comments_to_description()
requs = preprocessing.preprocess_requirements(requs,
enable_stemming=False,
lang=lang)
requs = list(filter(lambda r: len(r.tokens()) > 0, requs))
if len(requs) == 0:
return []
_logger.info("SVD...")
if len(requs) > 100:
min_distance, max_distance = 0.2, 0.5
k = 10
elif len(requs) > 50:
min_distance, max_distance = 0.2, 0.6
k = 8
elif len(requs) > 30:
min_distance, max_distance = 0.2, 0.65
k = 5
elif len(requs) > 10:
min_distance, max_distance = 0.2, 0.7
k = 3
elif len(requs) > 5:
min_distance, max_distance = 0.2, 0.75
k = 2
else:
min_distance, max_distance = 0.2, 0.8
k = 1
predictions_map = svd.svd(requs, k=k, min_distance=min_distance, max_distance=max_distance)
dependency_pairs = set()
for subject_requirement, dependent_requirements in predictions_map.items():
requ = Requirement.from_dict({
"id": subject_requirement.id,
"title": subject_requirement.title,
"description": subject_requirement.description,
"comments": subject_requirement.comments
})
rx = subject_requirement.id
dependent_requirement_ids = list(set(map(lambda r: r.id, dependent_requirements)))
all_undirected_pairs_of_subject_requirement = map(lambda ry: [(rx, ry), (ry, rx)], dependent_requirement_ids)
dependency_pairs_of_subject_requirement = set(list(itertools.chain(*all_undirected_pairs_of_subject_requirement)))
remaining_dependency_pairs_of_subject_requirement = dependency_pairs_of_subject_requirement - dependency_pairs
dependency_pairs = dependency_pairs.union(remaining_dependency_pairs_of_subject_requirement)
predictions = list(set(map(lambda t: t[0] if t[0] != rx else t[1], remaining_dependency_pairs_of_subject_requirement)))
requ.predictions = predictions
response_list += [requ]
for dependent_requirement in dependent_requirements:
print("{} -> {}".format(subject_requirement, dependent_requirement))
"""
for idx, requ in enumerate(requirements):
response_list.append(Requirement.from_dict({
"id": requ.id,
"title": requ.title,
"description": requ.description
}))
"""
return response_list
def recommend_similar_requirements(body): # noqa: E501
"""Retrieve a list with values for given set of requirements indicating their popularity for the crowd on twitter.
# noqa: E501
:param body: Requirement objects for which the social popularity should be measured
:type body: list | bytes
:rtype: List[Requirement]
"""
response_list = []
# TODO: introduce parameter to set language
lang = "en"
if connexion.request.is_json:
content = connexion.request.get_json()
assert isinstance(content, list)
requs = [Requirement.from_dict(d) for d in content] # noqa: E501
requs = list(
map(
lambda r: requirement.Requirement(r.id, r.title, r.description,
r.comments), requs))
for r in requs:
r.append_comments_to_description()
requs = preprocessing.preprocess_requirements(requs, lang=lang)
requs = list(filter(lambda r: len(r.tokens()) > 0, requs))
if len(requs) == 0:
return []
_logger.info("SVD...")
if len(requs) > 100:
max_distance = 0.4
k = 10
elif len(requs) > 50:
max_distance = 0.5
k = 8
elif len(requs) > 30:
max_distance = 0.55
k = 5
elif len(requs) > 10:
max_distance = 0.6
k = 3
elif len(requs) > 5:
max_distance = 0.6
k = 2
else:
max_distance = 0.6
k = 1
predictions_map = svd.svd(requs, k=k, max_distance=max_distance)
predictions = {}
for subject_requirement, similar_requirements in predictions_map.items(
):
rx = subject_requirement.id
rx_predictions = list(
set(map(lambda r: r.id, similar_requirements)))
if rx not in predictions:
predictions[rx] = set()
predictions[rx] = predictions[rx].union(rx_predictions)
for ry in rx_predictions:
if ry not in predictions:
predictions[ry] = set()
predictions[ry].add(rx)
for subject_requirement, similar_requirements in predictions_map.items(
):
requ = Requirement.from_dict({
"id":
subject_requirement.id,
"title":
subject_requirement.title,
"description":
subject_requirement.description,
"comments":
subject_requirement.comments
})
rx = subject_requirement.id
requ.predictions = list(predictions[rx])
response_list += [requ]
for similar_requirement in similar_requirements:
print("{} -> {}".format(subject_requirement,
similar_requirement))
"""
for idx, requ in enumerate(requirements):
response_list.append(Requirement.from_dict({
"id": requ.id,
"title": requ.title,
"description": requ.description
}))
"""
return response_list