def gen_concept_model(self): nlp = self.nlp miner = StoryMiner() counter = Counter() # Keep tracking the number of successes and fails success = 0 fail = 0 # Keeps tracking of all success and failed User Stories user_stories_lst = [] failed_stories_lst = [] success_stories_lst = [] us_id = 1 # Parse every user story (remove punctuation and mine) for story_line in self.messages: try: user_story = UserStory.parse(story_line, us_id, self.system_name, nlp, miner) user_story = counter.count(user_story) success = success + 1 user_stories_lst.append(user_story) success_stories_lst.append(story_line) except ValueError as err: failed_stories_lst.append([us_id, story_line, err.args]) fail = fail + 1 us_id = us_id + 1 # Generate the term-by-user story matrix (m), and additional data in two other matrices matrix = Matrix(self.base_weight, self.weights) matrices = matrix.generate( user_stories_lst, ' '.join([u.sentence for u in user_stories_lst]), nlp) m, count_matrix, stories_list, rme = matrices # Generate the ontology patterns = Constructor(nlp, user_stories_lst, m) out = patterns.make(self.system_name, self.threshold, self.link) output_ontology, output_prolog, output_ontobj, output_prologobj, onto_per_role = out # Return objects so that they can be used as input for other tools return { 'stories': user_stories_lst, 'ontology': output_ontology, 'prolog': output_prolog, 'matrix': m }
def __init__(self, threshold=DEFAULT_THRESHOLD, base=DEFAULT_BASE, weights=DEFAULT_WEIGHTS, link=False, per_role=False, stats=False, prolog=False, json=False, spacy_nlp=None): """Initialize across runs Args: threshold (float): threshold for which classes to select base (int): base weight (multiplier for all weights) weights (dict): weights for type of objects link (bool): link ontology classes to user story they originate from per_role (bool): create an additional conceptual model per role stats (bool): show user story set statistics and output these to a .csv file prolog (bool): generate Prolog output (.pl) json (bool): export user stories as JSON (.json) spacy_nlp (spacy.load object): spacy NLP using spacy.load('en_core_web_md') """ self.time = {} self.threshold = threshold self.base = base self.weights = weights self.link = link self.per_role = per_role self.stats = stats self.prolog = prolog self.json = json self.nlp = spacy_nlp if spacy_nlp is not None else self.initialize_nlp( log_time=self.time) self.matrix = Matrix(self.base, self.weights)
def main(filename, systemname, print_us, print_ont, statistics, link, prolog, json, per_role, threshold, base, weights, spacy_nlp): """General class to run the entire program """ start_nlp_time = timeit.default_timer() nlp = spacy_nlp nlp_time = timeit.default_timer() - start_nlp_time start_parse_time = timeit.default_timer() miner = StoryMiner() # Read the input file set = Reader.parse(filename) us_id = 1 # Keep track of all errors success = 0 fail = 0 list_of_fails = [] errors = "" c = Counter() # Keeps track of all succesfully created User Stories objects us_instances = [] failed_stories = [] success_stories = [] # Parse every user story (remove punctuation and mine) for s in set: try: user_story = parse(s, us_id, systemname, nlp, miner) user_story = c.count(user_story) success = success + 1 us_instances.append(user_story) success_stories.append(s) except ValueError as err: failed_stories.append([us_id, s, err.args]) errors += "\n[User Story " + str(us_id) + " ERROR] " + str( err.args[0]) + "! (\"" + " ".join(str.split(s)) + "\")" fail = fail + 1 us_id = us_id + 1 # Print errors (if found) if errors: Printer.print_head("PARSING ERRORS") print(errors) parse_time = timeit.default_timer() - start_parse_time # Generate the term-by-user story matrix (m), and additional data in two other matrices start_matr_time = timeit.default_timer() matrix = Matrix(base, weights) matrices = matrix.generate(us_instances, ' '.join([u.sentence for u in us_instances]), nlp) m, count_matrix, stories_list, rme = matrices matr_time = timeit.default_timer() - start_matr_time # Print details per user story, if argument '-u'/'--print_us' is chosen if print_us: print("Details:\n") for us in us_instances: Printer.print_us_data(us) # Generate the ontology start_gen_time = timeit.default_timer() patterns = Constructor(nlp, us_instances, m) out = patterns.make(systemname, threshold, link) output_ontology, output_prolog, output_ontobj, output_prologobj, onto_per_role = out # Print out the ontology in the terminal, if argument '-o'/'--print_ont' is chosen if print_ont: Printer.print_head("MANCHESTER OWL") print(output_ontology) gen_time = timeit.default_timer() - start_gen_time # Gather statistics and print the results stats_time = 0 if statistics: start_stats_time = timeit.default_timer() statsarr = Statistics.to_stats_array(us_instances) Printer.print_head("USER STORY STATISTICS") Printer.print_stats(statsarr[0], True) Printer.print_stats(statsarr[1], True) Printer.print_subhead( "Term - by - User Story Matrix ( Terms w/ total weight 0 hidden )") hide_zero = m[(m['sum'] > 0)] print(hide_zero) stats_time = timeit.default_timer() - start_stats_time # Write output files w = Writer() folder = "output/" + str(systemname) reports_folder = folder + "/reports" stats_folder = reports_folder + "/stats" outputfile = w.make_file(folder + "/ontology", str(systemname), "omn", output_ontology) files = [["Manchester Ontology", outputfile]] outputcsv = "" sent_outputcsv = "" matrixcsv = "" if statistics: files.append([ "General statistics", w.make_file(stats_folder, str(systemname), "csv", statsarr[0]) ]) files.append([ "Term-by-User Story matrix", w.make_file(stats_folder, str(systemname) + "-term_by_US_matrix", "csv", m) ]) files.append([ "Sentence statistics", w.make_file(stats_folder, str(systemname) + "-sentences", "csv", statsarr[1]) ]) if prolog: files.append([ "Prolog", w.make_file(folder + "/prolog", str(systemname), "pl", output_prolog) ]) if json: output_json_li = [str(us.toJSON()) for us in us_instances] output_json = "\n".join(output_json_li) files.append([ "JSON", w.make_file(folder + "/json", str(systemname) + "-user_stories", "json", output_json) ]) if per_role: for o in onto_per_role: files.append([ "Individual Ontology for '" + str(o[0]) + "'", w.make_file(folder + "/ontology", str(systemname) + "-" + str(o[0]), "omn", o[1]) ]) # Print the used ontology generation settings Printer.print_gen_settings(matrix, base, threshold) # Print details of the generation Printer.print_details(fail, success, nlp_time, parse_time, matr_time, gen_time, stats_time) report_dict = { "stories": us_instances, "failed_stories": failed_stories, "systemname": systemname, "us_success": success, "us_fail": fail, "times": [[ "Initializing Natural Language Processor (<em>spaCy</em> v" + pkg_resources.get_distribution("spacy").version + ")", nlp_time ], ["Mining User Stories", parse_time], ["Creating Factor Matrix", matr_time], ["Generating Manchester Ontology", gen_time], ["Gathering statistics", stats_time]], "dir": os.path.dirname(os.path.realpath(__file__)), "inputfile": filename, "inputfile_lines": len(set), "outputfiles": files, "threshold": threshold, "base": base, "matrix": matrix, "weights": m['sum'].copy().reset_index().sort_values( ['sum'], ascending=False).values.tolist(), "counts": count_matrix.reset_index().values.tolist(), "classes": output_ontobj.classes, "relationships": output_prologobj.relationships, "types": list(count_matrix.columns.values), "ontology": multiline(output_ontology) } # Finally, generate a report report = w.make_file(reports_folder, str(systemname) + "_REPORT", "html", generate_report(report_dict)) files.append(["Report", report]) # Print the location and name of all output files for file in files: if str(file[1]) != "": print( str(file[0]) + " file succesfully created at: \"" + str(file[1]) + "\"") # Return objects so that they can be used as input for other tools return { 'us_instances': us_instances, 'output_ontobj': output_ontobj, 'output_prologobj': output_prologobj, 'matrix': m }
class VisualNarrator: """General class to run Visual Narrator""" def __init__(self, threshold = DEFAULT_THRESHOLD, base = DEFAULT_BASE, weights = DEFAULT_WEIGHTS, link = False, per_role = False, stats = False, prolog = False, json = False, spacy_nlp = None): """Initialize across runs Args: threshold (float): threshold for which classes to select base (int): base weight (multiplier for all weights) weights (dict): weights for type of objects link (bool): link ontology classes to user story they originate from per_role (bool): create an additional conceptual model per role stats (bool): show user story set statistics and output these to a .csv file prolog (bool): generate Prolog output (.pl) json (bool): export user stories as JSON (.json) spacy_nlp (spacy.load object): spacy NLP using spacy.load('en_core_web_md') """ self.time = {} self.threshold = threshold self.base = base self.weights = weights self.link = link self.per_role = per_role self.stats = stats self.prolog = prolog self.json = json self.nlp = spacy_nlp if spacy_nlp is not None else self.initialize_nlp(log_time=self.time) self.matrix = Matrix(self.base, self.weights) def run(self, filename, systemname, print_us = False, print_ont = False, stories = None, write_local = True): """Single run of Visual Narrator Args: filename (str): File name to read systemname (str): Name of System (for output and in model) print_us (bool): print data per user story in the console print_ont (bool): print ontology in the console stories (list): preprocessed stories (from filename) Returns: dict: dictionary with US objects, Ontology + Prolog + JSON objects, matrix """ if stories is None: stories = Reader.parse(filename) # Mine stories us_instances, failed_stories = self._mine_stories(stories, systemname, log_time=self.time) # Generate the term-by-user story matrix (m), and additional data in two other matrices m, count_matrix = self._get_matrix(us_instances, log_time=self.time) # Print details per user story, if argument '-u'/'--print_us' is chosen if print_us: print("Details:\n") for us in us_instances: Printer.print_us_data(us) # Generate the outputs output_ontology, output_prolog, onto_per_role = \ self._get_gen(us_instances, m, systemname, print_ont, log_time=self.time) # Gather statistics and print the results statsarr = self._get_stats(us_instances, m, log_time=self.time) # Print the used ontology generation settings Printer.print_gen_settings(self.matrix, self.base, self.threshold) # Print details of the generation fail = len(failed_stories) success = len(us_instances) time_nlp = self.time['INITIALIZE_NLP'] time_mine = self.time['_MINE_STORIES'] time_matr = self.time['_GET_MATRIX'] time_gen = self.time['_GET_GEN'] time_stats = self.time['_GET_STATS'] Printer.print_details(fail, success, time_nlp, time_mine, time_matr, time_gen, time_stats) self.time['INITIALIZE_NLP'] = 0 output_json = json.dumps([us.toJSON() for us in us_instances], indent=4) files = [] if write_local: w = Writer files, reports_folder = self.write_files(w, systemname, str(output_ontology), str(output_prolog), output_json, statsarr, m, onto_per_role) report_dict = { "stories": us_instances, "failed_stories": failed_stories, "systemname": systemname, "us_success": success, "us_fail": fail, "times": [["Initializing Natural Language Processor (<em>spaCy</em> v" + pkg_resources.get_distribution("spacy").version + ")" , time_nlp], ["Mining User Stories", time_mine], ["Creating Factor Matrix", time_matr], ["Generating Manchester Ontology / Prolog", time_gen], ["Gathering statistics", time_stats]], "dir": sys.path[0], "inputfile": filename, "inputfile_lines": len(stories), "outputfiles": files, "threshold": self.threshold, "base": self.base, "matrix": self.matrix, "weights": m['sum'].copy().reset_index().sort_values(['sum'], ascending=False).values.tolist(), "counts": count_matrix.reset_index().values.tolist(), "classes": output_ontology.classes, "relationships": output_ontology.relationships, "types": list(count_matrix.columns.values), "ontology": multiline(str(output_ontology)), "print_prolog": self.prolog, "prolog": multiline(str(output_prolog)), "write_local": write_local } # Finally, generate a report output_report = self.generate_report(report_dict) # Write output files if write_local: report = w.make_file(reports_folder, str(systemname) + "_REPORT", "html", output_report) files.append(["Report", report]) # Print the location and name of all output files for file in files: if str(file[1]) != "": print(f"{file[0]} file succesfully created at: \"{file[1]}\"") # Return objects so that they can be used as input for other tools return {'us_instances': us_instances, 'output_ontobj': str(output_ontology), 'output_prologobj': str(output_prolog), 'output_json': output_json, 'matrix': m, 'report': output_report} @timeit def initialize_nlp(self, **kw): """Initialize spaCy just once (this takes most of the time...)""" print("Initializing Natural Language Processor. . .") import spacy return spacy.load('en_core_web_md') def parse(self, text, id, systemname, miner): """Create a new user story object and mines it to map all data in the user story text to a predefined model Args: text: The user story text id: The user story ID, which can later be used to identify the user story systemname: Name of the system this user story belongs to miner: instance of class Miner Returns: `vn.userstory.UserStory`: A new user story object """ no_punct = remove_punct(text) no_double_space = ' '.join(no_punct.split()) doc = self.nlp(no_double_space) user_story = UserStory(id, text, no_double_space) user_story.system.main = self.nlp(systemname)[0] user_story.data = doc #Printer.print_dependencies(user_story) #Printer.print_noun_phrases(user_story) miner.structure(user_story) user_story.old_data = user_story.data user_story.data = self.nlp(user_story.sentence) miner.mine(user_story, self.nlp) return user_story @timeit def _mine_stories(self, stories, systemname, **kw): # Keep track of all errors errors = "" # Keeps track of all succesfully created User Stories objects us_instances = [] failed_stories = [] # Parse every user story (remove punctuation and mine) for us_id, s in enumerate(stories, start=1): try: user_story = self.parse(s, us_id, systemname, StoryMiner()) user_story = Counter.count(user_story) us_instances.append(user_story) except ValueError as err: failed_stories.append([us_id, s, err.args]) errors += "\n[User Story {} ERROR] {}! (\"{}\")".format(us_id, err.args[0], " ".join(str.split(s))) # Print errors (if found) if errors: Printer._print_head("PARSING ERRORS") print(errors) return us_instances, failed_stories @timeit def _get_matrix(self, us_instances, **kw): matrices = self.matrix.generate(us_instances, ' '.join([u.sentence for u in us_instances]), self.nlp) m, count_matrix, _, _ = matrices return m, count_matrix @timeit def _get_gen(self, us_instances, m, systemname, print_ont, **kw): patterns = Constructor(self.nlp, us_instances, m) out = patterns.make(systemname, self.threshold, self.link) # Print out the ontology in the terminal, if argument '-o'/'--print_ont' is chosen if print_ont: Printer._print_head("MANCHESTER OWL") print(out[0]) return out @timeit def _get_stats(self, us_instances, m, **kw): statsarr = None if self.stats: statsarr = Statistics.to_stats_array(us_instances) Printer._print_head("USER STORY STATISTICS") Printer.print_stats(statsarr[0], True) #Printer.print_stats(statsarr[1], True) Printer._print_subhead("Term - by - User Story Matrix ( Terms w/ total weight 0 hidden )") hide_zero = m[(m['sum'] > 0)] print(hide_zero) return statsarr def write_files(self, w, systemname, output_ontology, output_prolog, output_json, statsarr, m, onto_per_role): """Writes Ontology / Prolog / JSON to files Args: w (`vn.io.Writer`): Writer object for I/O systemname (str): System name output_ontology (str): Manchester Ontology text output_prolog (str): Prolog text output_json (str): JSON text statsarr (list): User story statistics m (DataFrame): Term-by-UserStory matrix onto_per_role (list): List of ontologies, one per functional role Returns: list: Files created str: Folder the reports were created in """ s = str(systemname) folder = "output/" + s reports_folder = folder + "/reports" stats_folder = reports_folder + "/stats" ontology_folder = folder + "/ontology" outputfile = w.make_file(ontology_folder, s, "omn", output_ontology) files = [["Manchester Ontology", outputfile]] if self.stats: files.append(["General statistics", w.make_file(stats_folder, s, "csv", statsarr[0])]) files.append(["Term-by-User Story matrix", w.make_file(stats_folder, f"{s}-term_by_US_matrix", "csv", m)]) files.append(["Sentence statistics", w.make_file(stats_folder, f"{s}-sentences", "csv", statsarr[1])]) if self.prolog: files.append(["Prolog", w.make_file(folder + "/prolog", s, "pl", output_prolog)]) if self.json: files.append(["JSON", w.make_file(folder + "/json", f"{s}-user_stories", "json", output_json)]) if self.per_role: for c, o in onto_per_role: files.append([f"Individual Ontology for '{c}'", w.make_file(folder + "/ontology", f"{s}-{c}", "omn", o)]) return files, reports_folder def generate_report(self, report_dict): """Generates a report using Jinja2 Args: report_dict: Dictionary containing all variables used in the report Returns: str: HTML page """ CURR_DIR = os.path.dirname(os.path.abspath(__file__)) loader = FileSystemLoader(searchpath=str(CURR_DIR) + "/templates/") env = Environment(loader=loader, trim_blocks=True, lstrip_blocks=True) env.globals['text'] = t env.globals['is_i'] = is_i env.globals['apply_tab'] = tab env.globals['is_comment'] = is_comment env.globals['occurence_list'] = occurence_list env.tests['is_us'] = is_us template = env.get_template("report.html") return template.render(report_dict)
def main(filename, systemname, print_us, print_ont, statistics, link, prolog, json, per_role, threshold, base, weights, spacy_nlp): """General class to run the entire program """ start_nlp_time = timeit.default_timer() nlp = spacy_nlp nlp_time = timeit.default_timer() - start_nlp_time start_parse_time = timeit.default_timer() miner = StoryMiner() # Read the input file set = Reader.parse(filename) us_id = 1 # Keep track of all errors success = 0 fail = 0 list_of_fails = [] errors = "" c = Counter() # Keeps track of all succesfully created User Stories objects us_instances = [] failed_stories = [] success_stories = [] # Parse every user story (remove punctuation and mine) for s in set: try: user_story = parse(s, us_id, systemname, nlp, miner) user_story = c.count(user_story) success = success + 1 us_instances.append(user_story) success_stories.append(s) except ValueError as err: failed_stories.append([us_id, s, err.args]) errors += "\n[User Story " + str(us_id) + " ERROR] " + str( err.args[0]) + "! (\"" + " ".join(str.split(s)) + "\")" fail = fail + 1 us_id = us_id + 1 # Print errors (if found) if errors: Printer.print_head("PARSING ERRORS") print(errors) parse_time = timeit.default_timer() - start_parse_time # Generate the term-by-user story matrix (m), and additional data in two other matrices start_matr_time = timeit.default_timer() matrix = Matrix(base, weights) matrices = matrix.generate(us_instances, ' '.join([u.sentence for u in us_instances]), nlp) m, count_matrix, stories_list, rme = matrices matr_time = timeit.default_timer() - start_matr_time # Generate the ontology start_gen_time = timeit.default_timer() patterns = Constructor(nlp, us_instances, m) out = patterns.make(systemname, threshold, link) output_ontology, output_prolog, output_ontobj, output_prologobj, onto_per_role = out all_classes_list = [] i = 0 for class_vn in output_ontobj.classes: one_concept = { 'id': i, 'class_name': class_vn.name, 'parent_name': class_vn.parent, 'occurs_in': occurence_list(class_vn.stories), 'weight': '0', 'group': class_vn.is_role } all_classes_list.append(one_concept) i += 1 nodes = [{ "id": cl["id"], "label": cl["class_name"], "weight": cl["weight"] } for cl in all_classes_list] relationships_query = output_prologobj.relationships all_relationships_list = [] for relationship in relationships_query: one_concept = { 'relationship_domain': relationship.domain, 'relationship_name': relationship.name, 'relationship_range': relationship.range } all_relationships_list.append(one_concept) edges_id_list = [] concepts_query = [] concepts_dict = {} concepts_dict_list = [] relationshipslist = [] i = 0 for class_vn in all_classes_list: one_concept = { 'class_id': i, 'class_name': class_vn['class_name'], 'parent_name': class_vn['parent_name'], 'weight': '0', 'group': class_vn['group'] } concepts_query.append(one_concept) i += 1 for concept in concepts_query: # print(concept) concepts_dict[concept['class_id']] = concept['class_name'] concepts_dict_list.append([concept['class_id'], concept['class_name']]) i = 0 for rel in all_relationships_list: #app.py 868 # print(rel) relationshipslist.append([ rel['relationship_domain'], rel['relationship_range'], rel['relationship_name'] ]) for concept in concepts_dict_list: if rel['relationship_domain'] == concept[1]: x = concept[0] for concept in concepts_dict_list: if rel['relationship_range'] == concept[1]: y = concept[0] if rel['relationship_name'] == 'isa': edges_id_dict = { 'from': x, 'to': y, 'label': rel['relationship_name'], 'dashes': "true" } else: edges_id_dict = { 'from': x, 'to': y, 'label': rel['relationship_name'] } i += 1 # ELSE?? edges_id_list.append(edges_id_dict) print({'nodes': nodes, 'edges': edges_id_list}) return ({'nodes': nodes, 'edges': edges_id_list})
def main(filename, systemname, print_us, print_ont, statistics, link, prolog, json, per_role, threshold, base, weights, spacy_nlp): """General class to run the entire program """ start_nlp_time = timeit.default_timer() nlp = spacy_nlp nlp_time = timeit.default_timer() - start_nlp_time start_parse_time = timeit.default_timer() miner = StoryMiner() # Read the input file set = Reader.parse(filename) us_id = 1 # Keep track of all errors success = 0 fail = 0 list_of_fails = [] errors = "" c = Counter() # Keeps track of all succesfully created User Stories objects us_instances = [] failed_stories = [] success_stories = [] # Parse every user story (remove punctuation and mine) for s in set: try: user_story = parse(s, us_id, systemname, nlp, miner) user_story = c.count(user_story) success = success + 1 us_instances.append(user_story) success_stories.append(s) except ValueError as err: failed_stories.append([us_id, s, err.args]) errors += "\n[User Story " + str(us_id) + " ERROR] " + str( err.args[0]) + "! (\"" + " ".join(str.split(s)) + "\")" fail = fail + 1 us_id = us_id + 1 # Print errors (if found) if errors: Printer.print_head("PARSING ERRORS") print(errors) parse_time = timeit.default_timer() - start_parse_time # Generate the term-by-user story matrix (m), and additional data in two other matrices start_matr_time = timeit.default_timer() matrix = Matrix(base, weights) matrices = matrix.generate(us_instances, ' '.join([u.sentence for u in us_instances]), nlp) m, count_matrix, stories_list, rme = matrices matr_time = timeit.default_timer() - start_matr_time # Print details per user story, if argument '-u'/'--print_us' is chosen if print_us: print("Details:\n") for us in us_instances: Printer.print_us_data(us) # Generate the ontology start_gen_time = timeit.default_timer() patterns = Constructor(nlp, us_instances, m) out = patterns.make(systemname, threshold, link) output_ontology, output_prolog, output_ontobj, output_prologobj, onto_per_role = out print("HEY THIS IS THE OUTPUT_ONTOBJ WITH THE CLASSES APPARENTLY???") print(output_ontobj.classes) all_classes_list = [] for class_vn in output_ontobj.classes: one_concept = { 'class_name': class_vn.name, 'parent_name': class_vn.parent, 'occurs_in': occurence_list(class_vn.stories), 'weight': '0', 'group': class_vn.is_role } all_classes_list.append(one_concept) print(all_classes_list) # nodes = [] # for cl in all_classes_list: # print(cl) # nodes.append({"label": cl['class_name']}) # taking out class_id from the nodes. idk if this will bite me later. nodes = [{ "label": cl["class_name"], "weight": cl["weight"] } for cl in all_classes_list] # print(nodes) print('IDK WHAT THIS IS BUT IMMA PRINT IT OUT TOO') relationships_query = output_prologobj.relationships all_relationships_list = [] for relationship in relationships_query: one_concept = { 'relationship_domain': relationship.domain, 'relationship_name': relationship.name, 'relationship_range': relationship.range } all_relationships_list.append(one_concept) print(all_relationships_list) edges_id_list = [] concepts_query = [] concepts_dict = {} concepts_dict_list = [] relationshipslist = [] i = 0 for class_vn in all_classes_list: one_concept = { 'class_id': i, 'class_name': class_vn['class_name'], 'parent_name': class_vn['parent_name'], 'weight': '0', 'group': class_vn['group'] } concepts_query.append(one_concept) i += 1 # print(concepts_query) for concept in concepts_query: print(concept) concepts_dict[concept['class_id']] = concept['class_name'] concepts_dict_list.append([concept['class_id'], concept['class_name']]) print('THIS IS WHAT UR CURRENTLY LOOKING AT') print(concepts_dict_list) i = 0 for rel in all_relationships_list: #app.py 868 # print(rel) relationshipslist.append([ rel['relationship_domain'], rel['relationship_range'], rel['relationship_name'] ]) for concept in concepts_dict_list: if rel['relationship_domain'] == concept[1]: x = concept[0] for concept in concepts_dict_list: if rel['relationship_range'] == concept[1]: y = concept[0] if rel['relationship_name'] == 'isa': edges_id_dict = { 'id': i, 'from': x, 'to': y, 'label': rel['relationship_name'], 'dashes': "true" } else: edges_id_dict = { 'id': i, 'from': x, 'to': y, 'label': rel['relationship_name'] } i += 1 # ELSE?? edges_id_list.append(edges_id_dict) print(edges_id_list) # Print out the ontology in the terminal, if argument '-o'/'--print_ont' is chosen if print_ont: Printer.print_head("MANCHESTER OWL") print(output_ontology) gen_time = timeit.default_timer() - start_gen_time # Gather statistics and print the results stats_time = 0 if statistics: start_stats_time = timeit.default_timer() statsarr = Statistics.to_stats_array(us_instances) Printer.print_head("USER STORY STATISTICS") Printer.print_stats(statsarr[0], True) Printer.print_stats(statsarr[1], True) Printer.print_subhead( "Term - by - User Story Matrix ( Terms w/ total weight 0 hidden )") hide_zero = m[(m['sum'] > 0)] print(hide_zero) stats_time = timeit.default_timer() - start_stats_time # Write output files w = Writer() folder = "output/" + str(systemname) reports_folder = folder + "/reports" stats_folder = reports_folder + "/stats" outputfile = w.make_file(folder + "/ontology", str(systemname), "omn", output_ontology) files = [["Manchester Ontology", outputfile]] outputcsv = "" sent_outputcsv = "" matrixcsv = "" if statistics: files.append([ "General statistics", w.make_file(stats_folder, str(systemname), "csv", statsarr[0]) ]) files.append([ "Term-by-User Story matrix", w.make_file(stats_folder, str(systemname) + "-term_by_US_matrix", "csv", m) ]) files.append([ "Sentence statistics", w.make_file(stats_folder, str(systemname) + "-sentences", "csv", statsarr[1]) ]) if prolog: files.append([ "Prolog", w.make_file(folder + "/prolog", str(systemname), "pl", output_prolog) ]) if json: output_json_li = [str(us.toJSON()) for us in us_instances] output_json = "\n".join(output_json_li) files.append([ "JSON", w.make_file(folder + "/json", str(systemname) + "-user_stories", "json", output_json) ]) if per_role: for o in onto_per_role: files.append([ "Individual Ontology for '" + str(o[0]) + "'", w.make_file(folder + "/ontology", str(systemname) + "-" + str(o[0]), "omn", o[1]) ]) # Print the used ontology generation settings Printer.print_gen_settings(matrix, base, threshold) # Print details of the generation Printer.print_details(fail, success, nlp_time, parse_time, matr_time, gen_time, stats_time) report_dict = { "stories": us_instances, "failed_stories": failed_stories, "systemname": systemname, "us_success": success, "us_fail": fail, "times": [[ "Initializing Natural Language Processor (<em>spaCy</em> v" + pkg_resources.get_distribution("spacy").version + ")", nlp_time ], ["Mining User Stories", parse_time], ["Creating Factor Matrix", matr_time], ["Generating Manchester Ontology", gen_time], ["Gathering statistics", stats_time]], "dir": os.path.dirname(os.path.realpath(__file__)), "inputfile": filename, "inputfile_lines": len(set), "outputfiles": files, "threshold": threshold, "base": base, "matrix": matrix, "weights": m['sum'].copy().reset_index().sort_values( ['sum'], ascending=False).values.tolist(), "counts": count_matrix.reset_index().values.tolist(), "classes": output_ontobj.classes, "relationships": output_prologobj.relationships, "types": list(count_matrix.columns.values), "ontology": multiline(output_ontology) } # Finally, generate a report report = w.make_file(reports_folder, str(systemname) + "_REPORT", "html", generate_report(report_dict)) files.append(["Report", report]) # Print the location and name of all output files for file in files: if str(file[1]) != "": print( str(file[0]) + " file succesfully created at: \"" + str(file[1]) + "\"") # Return objects so that they can be used as input for other tools return { 'us_instances': us_instances, 'output_ontobj': output_ontobj, 'output_prologobj': output_prologobj, 'matrix': m }