def testSubtraction(self):
"""
The binary subtraction operator
should, when applied to two matricies, product
the difference of the two matricies if and only if
their dimensions are equal.
"""
A = Matrix.fromArray([
[0,2,3],
[4,2,6],
[4,5,6]
])
B = Matrix.fromArray([
[4, 9, 6],
[7, 8, 1],
[0, 13,2]
])
C = Matrix.fromArray([
[1,2,3],
[4,5,6]
])
diffAB = Matrix.fromArray([
[-4, -7, -3],
[-3, -6, 5],
[4, -8, 4]
])
self.assertEqual(A - B, diffAB)
self.assertEqual(A - C, None)
class TestUpdate(unittest.TestCase):
def setUp(self):
# Instantiate the class
self.matrix = Matrix()
def test_update_array_error(self):
# Initialize the matrix data
self.matrix.set_size(4)
# Test
self.assertRaises(IndexError, self.matrix.update, 2, 5, 2, 4)
self.assertRaises(IndexError, self.matrix.update, 5, 2, 2, 3)
self.assertRaises(IndexError, self.matrix.update, 2, 2, 5, 1)
self.assertRaises(IndexError, self.matrix.update, 0, 0, 0, 1)
self.assertNotIsInstance(self.matrix.update(1, 1, 1, 4), str)
self.assertNotEqual(self.matrix.update(1, 1, 1, 4), {'value': 4, 'row': 0, 'column': 0, 'page': 0})\
def test_update_array(self):
# Initialize the matrix data
self.matrix.set_size(4)
# Test
self.assertIsInstance(self.matrix.update(1, 1, 1, 3), dict)
self.assertDictEqual(self.matrix.update(2, 2, 2, 4), {'value': 4, 'row': 2, 'column': 2, 'page': 2})
self.assertDictEqual(self.matrix.update(1, 1, 1, 23), {'value': 23, 'row': 1, 'column': 1, 'page': 1})
self.assertDictEqual(self.matrix.update(4, 2, 3, 5), {'value': 5, 'row': 4, 'column': 2, 'page': 3})
def testAddition(self):
"""
The binary addition operator
should, when applied to two matricies, product
the sum of the two matricies if and only if
their dimensions are equal.
"""
A = Matrix.fromArray([
[0,2,3],
[4,2,6],
[4,5,6]
])
B = Matrix.fromArray([
[4, 9, 6],
[7, 8, 1],
[0, 13,2]
])
C = Matrix.fromArray([
[1,2,3],
[4,5,6]
])
sumAB = Matrix.fromArray([
[4, 11, 9],
[11, 10, 7],
[4, 18, 8]
])
self.assertEqual(A + B, sumAB)
self.assertEqual(A + C, None)
def setUp(self):
self.matrix = Matrix.fromArray([
[1, 2, 3],
[4, 5, 6],
[7, 8, 9]
])
self.identity = Matrix.fromArray([
[1, 0, 0],
[0, 1, 0],
[0, 0, 1]
])
def testMultiply(self):
"""
The binary multiplication operator
should, when applied to two matricies, produce
the matrix product if and only if the number of rows in the left
operand is equal to the number of columns in the right operand.
Matrix multiplication is generally not communitive, so
the product AB should differ from the product BA (usually)
"""
matrixA = self.matrix
matrixB = self.matrix.transpose()
productAB = matrixA * matrixB
productBA = matrixB * matrixA
self.assertEqual(productAB, [
[14,32,50],
[32,77,122],
[50,122,194]
])
self.assertEqual(productBA, [
[66,78,90],
[78,93,108],
[90,108,126]
])
matrixC = Matrix.fromArray([
[1,2,3],
[4,5,6]
])
self.assertEqual(matrixA * matrixC, None)
def testIdentity(self):
"""
The static identity method should return a 3 x 3
identity matrix, unless a different number of rows and columns
are specified
"""
self.assertEqual(Matrix.identity(), self.identity)
class TestSetUp(unittest.TestCase):
def setUp(self):
self.matrix = Matrix()
def test_setup_matrix_error(self):
self.assertRaises(IndexError, self.matrix.set_size, 101)
self.assertRaises(IndexError, self.matrix.set_size, 120)
self.assertRaises(IndexError, self.matrix.set_size, 150)
self.assertRaises(IndexError, self.matrix.set_size, 0)
self.assertNotEqual(self.matrix.set_size(40), (0, 40, 40,))
self.assertNotEqual(self.matrix.set_size(50), (50, 50, 0,))
def test_setup_matrix(self):
self.assertEqual(self.matrix.set_size(2), (2, 2, 2,))
self.assertEqual(self.matrix.set_size(9), (9, 9, 9,))
self.assertEqual(self.matrix.set_size(20), (20, 20, 20,))
self.assertEqual(self.matrix.set_size(45), (45, 45, 45,))
self.assertEqual(self.matrix.set_size(100), (100, 100, 100,))
def testTranspose(self):
"""
The transpose method return a copy of the matrix
with the columns and rows exchanged.
"""
transpose = Matrix.fromArray([
[1, 4, 7],
[2, 5, 8],
[3, 6, 9]
])
self.assertEqual(self.matrix.transpose(), transpose)
def testConstructor(self):
"""
A new Matrix instance should be a 3 x 3 by default,
unless a different number of rows and columns are specified.
The matrix should be the zero matrix.
"""
with self.assertRaises(TypeError):
badArguments = Matrix(2.5, 'foo')
matrix1 = Matrix()
self.assertEqual(matrix1.identity(), self.identity)
matrix2 = Matrix(4,2)
self.assertEqual(matrix2.rows(), 4)
self.assertEqual(matrix2.columns(), 2)
def testDeterminant(self):
"""
The determinant method should return None if the
matrix is not square. It should return zero
if the matrix is singular.
"""
nonSquare = Matrix.fromArray([
[1,2,3],
[4,5,6]
])
singularMatrix = self.matrix
self.assertEqual(nonSquare.determinant(), None)
self.assertEqual(singularMatrix.determinant(), 0)
def computeDeterminant(model):
request.det = Matrix.fromArray(model['matrix']).determinant()
def main(filename, systemname, print_us, print_ont, statistics, link, prolog, per_role, threshold, base, weights):
"""General class to run the entire program
"""
# Initialize spaCy just once (this takes most of the time...)
print("Initializing Natural Language Processor . . .")
start_nlp_time = timeit.default_timer()
nlp = English()
nlp_time = timeit.default_timer() - start_nlp_time
start_parse_time = timeit.default_timer()
miner = StoryMiner()
# Read the input file
set = Reader.parse(filename)
us_id = 1
# Keep track of all errors
success = 0
fail = 0
list_of_fails = []
errors = ""
c = Counter()
# Keeps track of all succesfully created User Stories objects
us_instances = []
failed_stories = []
success_stories = []
# Parse every user story (remove punctuation and mine)
for s in set:
try:
user_story = parse(s, us_id, systemname, nlp, miner)
user_story = c.count(user_story)
success = success + 1
us_instances.append(user_story)
success_stories.append(s)
except ValueError as err:
failed_stories.append([us_id, s, err.args])
errors += "\n[User Story " + str(us_id) + " ERROR] " + str(err.args[0]) + "! (\"" + " ".join(str.split(s)) + "\")"
fail = fail + 1
us_id = us_id + 1
# Print errors (if found)
if errors:
Printer.print_head("PARSING ERRORS")
print(errors)
parse_time = timeit.default_timer() - start_parse_time
# Generate the term-by-user story matrix (m), and additional data in two other matrices
start_matr_time = timeit.default_timer()
matrix = Matrix(base, weights)
matrices = matrix.generate(us_instances, ' '.join(success_stories), nlp)
m = matrices[0]
count_matrix = matrices[1]
stories_list = matrices[2]
rme = matrices[3]
matr_time = timeit.default_timer() - start_matr_time
# Print details per user story, if argument '-u'/'--print_us' is chosen
if print_us:
print("Details:\n")
for us in us_instances:
Printer.print_us_data(us)
# Generate the ontology
start_gen_time = timeit.default_timer()
patterns = Constructor(nlp, us_instances, m)
out = patterns.make(systemname, threshold, link)
output_ontology = out[0]
output_prolog = out[1]
output_ontobj = out[2]
output_prologobj = out[3]
onto_per_role = out[4]
# Print out the ontology in the terminal, if argument '-o'/'--print_ont' is chosen
if print_ont:
Printer.print_head("MANCHESTER OWL")
print(output_ontology)
gen_time = timeit.default_timer() - start_gen_time
# Gather statistics and print the results
stats_time = 0
if statistics:
start_stats_time = timeit.default_timer()
statsarr = Statistics.to_stats_array(us_instances)
Printer.print_head("USER STORY STATISTICS")
Printer.print_stats(statsarr[0], True)
Printer.print_stats(statsarr[1], True)
Printer.print_subhead("Term - by - User Story Matrix ( Terms w/ total weight 0 hidden )")
hide_zero = m[(m['sum'] > 0)]
print(hide_zero)
stats_time = timeit.default_timer() - start_stats_time
# Write output files
w = Writer()
folder = "output/" + str(systemname)
reports_folder = folder + "/reports"
stats_folder = reports_folder + "/stats"
outputfile = w.make_file(folder + "/ontology", str(systemname), "omn", output_ontology)
files = [["Manchester Ontology", outputfile]]
outputcsv = ""
sent_outputcsv = ""
matrixcsv = ""
if statistics:
outputcsv = w.make_file(stats_folder, str(systemname), "csv", statsarr[0])
matrixcsv = w.make_file(stats_folder, str(systemname) + "-term_by_US_matrix", "csv", m)
sent_outputcsv = w.make_file(stats_folder, str(systemname) + "-sentences", "csv", statsarr[1])
files.append(["General statistics", outputcsv])
files.append(["Term-by-User Story matrix", matrixcsv])
files.append(["Sentence statistics", sent_outputcsv])
if prolog:
outputpl = w.make_file(folder + "/prolog", str(systemname), "pl", output_prolog)
files.append(["Prolog", outputpl])
if per_role:
for o in onto_per_role:
name = str(systemname) + "-" + str(o[0])
pont = w.make_file(folder + "/ontology", name, "omn", o[1])
files.append(["Individual Ontology for '" + str(o[0]) + "'", pont])
# Print the used ontology generation settings
Printer.print_gen_settings(matrix, base, threshold)
# Print details of the generation
Printer.print_details(fail, success, nlp_time, parse_time, matr_time, gen_time, stats_time)
report_dict = {
"stories": us_instances,
"failed_stories": failed_stories,
"systemname": systemname,
"us_success": success,
"us_fail": fail,
"times": [["Initializing Natural Language Processor (<em>spaCy</em> v" + pkg_resources.get_distribution("spacy").version + ")" , nlp_time], ["Mining User Stories", parse_time], ["Creating Factor Matrix", matr_time], ["Generating Manchester Ontology", gen_time], ["Gathering statistics", stats_time]],
"dir": os.path.dirname(os.path.realpath(__file__)),
"inputfile": filename,
"inputfile_lines": len(set),
"outputfiles": files,
"threshold": threshold,
"base": base,
"matrix": matrix,
"weights": m['sum'].copy().reset_index().sort_values(['sum'], ascending=False).values.tolist(),
"counts": count_matrix.reset_index().values.tolist(),
"classes": output_ontobj.classes,
"relationships": output_prologobj.relationships,
"types": list(count_matrix.columns.values),
"ontology": Utility.multiline(output_ontology)
}
# Finally, generate a report
report = w.make_file(reports_folder, str(systemname) + "_REPORT", "html", generate_report(report_dict))
files.append(["Report", report])
# Print the location and name of all output files
for file in files:
if str(file[1]) != "":
print(str(file[0]) + " file succesfully created at: \"" + str(file[1]) + "\"")
def main(filename, systemname, print_us, print_ont, statistics, link, prolog,
per_role, threshold, base, weights):
"""General class to run the entire program
"""
# Initialize spaCy just once (this takes most of the time...)
print("Initializing Natural Language Processor . . .")
start_nlp_time = timeit.default_timer()
nlp = English()
nlp_time = timeit.default_timer() - start_nlp_time
start_parse_time = timeit.default_timer()
miner = StoryMiner()
# Read the input file
set = Reader.parse(filename)
us_id = 1
# Keep track of all errors
success = 0
fail = 0
list_of_fails = []
errors = ""
c = Counter()
# Keeps track of all succesfully created User Stories objects
us_instances = []
failed_stories = []
success_stories = []
# Parse every user story (remove punctuation and mine)
for s in set:
try:
user_story = parse(s, us_id, systemname, nlp, miner)
user_story = c.count(user_story)
success = success + 1
us_instances.append(user_story)
success_stories.append(s)
except ValueError as err:
failed_stories.append([us_id, s, err.args])
errors += "\n[User Story " + str(us_id) + " ERROR] " + str(
err.args[0]) + "! (\"" + " ".join(str.split(s)) + "\")"
fail = fail + 1
us_id = us_id + 1
# Print errors (if found)
if errors:
Printer.print_head("PARSING ERRORS")
print(errors)
parse_time = timeit.default_timer() - start_parse_time
# Generate the term-by-user story matrix (m), and additional data in two other matrices
start_matr_time = timeit.default_timer()
matrix = Matrix(base, weights)
matrices = matrix.generate(us_instances, ' '.join(success_stories), nlp)
m = matrices[0]
count_matrix = matrices[1]
stories_list = matrices[2]
rme = matrices[3]
matr_time = timeit.default_timer() - start_matr_time
# Print details per user story, if argument '-u'/'--print_us' is chosen
if print_us:
print("Details:\n")
for us in us_instances:
Printer.print_us_data(us)
# Generate the ontology
start_gen_time = timeit.default_timer()
patterns = Constructor(nlp, us_instances, m)
out = patterns.make(systemname, threshold, link)
output_ontology = out[0]
output_prolog = out[1]
output_ontobj = out[2]
output_prologobj = out[3]
onto_per_role = out[4]
# Print out the ontology in the terminal, if argument '-o'/'--print_ont' is chosen
if print_ont:
Printer.print_head("MANCHESTER OWL")
print(output_ontology)
gen_time = timeit.default_timer() - start_gen_time
# Gather statistics and print the results
stats_time = 0
if statistics:
start_stats_time = timeit.default_timer()
statsarr = Statistics.to_stats_array(us_instances)
Printer.print_head("USER STORY STATISTICS")
Printer.print_stats(statsarr[0], True)
Printer.print_stats(statsarr[1], True)
Printer.print_subhead(
"Term - by - User Story Matrix ( Terms w/ total weight 0 hidden )")
hide_zero = m[(m['sum'] > 0)]
print(hide_zero)
stats_time = timeit.default_timer() - start_stats_time
# Write output files
w = Writer()
folder = "output/" + str(systemname)
reports_folder = folder + "/reports"
stats_folder = reports_folder + "/stats"
outputfile = w.make_file(folder + "/ontology", str(systemname), "omn",
output_ontology)
files = [["Manchester Ontology", outputfile]]
outputcsv = ""
sent_outputcsv = ""
matrixcsv = ""
if statistics:
outputcsv = w.make_file(stats_folder, str(systemname), "csv",
statsarr[0])
matrixcsv = w.make_file(stats_folder,
str(systemname) + "-term_by_US_matrix", "csv",
m)
sent_outputcsv = w.make_file(stats_folder,
str(systemname) + "-sentences", "csv",
statsarr[1])
files.append(["General statistics", outputcsv])
files.append(["Term-by-User Story matrix", matrixcsv])
files.append(["Sentence statistics", sent_outputcsv])
if prolog:
outputpl = w.make_file(folder + "/prolog", str(systemname), "pl",
output_prolog)
files.append(["Prolog", outputpl])
if per_role:
for o in onto_per_role:
name = str(systemname) + "-" + str(o[0])
pont = w.make_file(folder + "/ontology", name, "omn", o[1])
files.append(["Individual Ontology for '" + str(o[0]) + "'", pont])
# Print the used ontology generation settings
Printer.print_gen_settings(matrix, base, threshold)
# Print details of the generation
Printer.print_details(fail, success, nlp_time, parse_time, matr_time,
gen_time, stats_time)
report_dict = {
"stories":
us_instances,
"failed_stories":
failed_stories,
"systemname":
systemname,
"us_success":
success,
"us_fail":
fail,
"times": [[
"Initializing Natural Language Processor (<em>spaCy</em> v" +
pkg_resources.get_distribution("spacy").version + ")", nlp_time
], ["Mining User Stories", parse_time],
["Creating Factor Matrix", matr_time],
["Generating Manchester Ontology", gen_time],
["Gathering statistics", stats_time]],
"dir":
os.path.dirname(os.path.realpath(__file__)),
"inputfile":
filename,
"inputfile_lines":
len(set),
"outputfiles":
files,
"threshold":
threshold,
"base":
base,
"matrix":
matrix,
"weights":
m['sum'].copy().reset_index().sort_values(
['sum'], ascending=False).values.tolist(),
"counts":
count_matrix.reset_index().values.tolist(),
"classes":
output_ontobj.classes,
"relationships":
output_prologobj.relationships,
"types":
list(count_matrix.columns.values),
"ontology":
Utility.multiline(output_ontology)
}
# Finally, generate a report
report = w.make_file(reports_folder,
str(systemname) + "_REPORT", "html",
generate_report(report_dict))
files.append(["Report", report])
# Print the location and name of all output files
for file in files:
if str(file[1]) != "":
print(
str(file[0]) + " file succesfully created at: \"" +
str(file[1]) + "\"")
def buildMatricies(model):
request.matrixA = Matrix.fromArray(model['lvalue'])
request.matrixB = Matrix.fromArray(model['rvalue'])
def setUp(self):
# Instantiate the class
self.matrix = Matrix()
class TestQuery(unittest.TestCase):
def setUp(self):
# Instantiate the class
self.matrix = Matrix()
def test_query_array_error(self):
# Initialize the matrix data
self.matrix.set_size(2)
self.matrix.update(2, 2, 2, 1)
# Test
self.assertRaises(IndexError, self.matrix.query, 3, 3, 3, 3, 3, 3)
self.assertRaises(IndexError, self.matrix.query, 0, 0, 0, 0, 0, 0)
self.assertNotEqual(self.matrix.query(1, 1, 1, 1, 1, 1), 1)
self.assertNotEqual(self.matrix.query(2, 2, 2, 2, 2, 2), 0)
def test_query_array1(self):
# Initialize the matrix data
self.matrix.set_size(2)
self.matrix.update(2, 2, 2, 1)
# Test
self.assertIsNotNone(self.matrix.query(1, 1, 1, 1, 1, 1))
self.assertIsInstance(self.matrix.query(1, 1, 1, 1, 1, 1), int)
self.assertEqual(self.matrix.query(1, 1, 1, 1, 1, 1), 0)
self.assertEqual(self.matrix.query(1, 1, 1, 2, 2, 2), 1)
self.assertEqual(self.matrix.query(2, 2, 2, 2, 2, 2), 1)
def test_query_array2(self):
# Initialize the matrix data
self.matrix.set_size(10)
self.matrix.update(1,1,1,1)
self.matrix.update(2,2,2,1)
self.matrix.update(10,10,10,1)
# Test
self.assertEqual(self.matrix.query(1,1,1,1,1,2), 1)
self.assertEqual(self.matrix.query(1,1,2,3,3,3), 1)
self.assertEqual(self.matrix.query(1,1,1,9,9,9), 2)
self.assertEqual(self.matrix.query(1,1,1,10,10,10), 3)
def test_query_array3(self):
# Initialize the matrix data
self.matrix.set_size(30)
self.matrix.update(1,1,1,1)
self.matrix.update(4,4,4,1)
self.matrix.update(30,30,30,1)
# Test
self.assertEqual(self.matrix.query(1,1,1,1,1,2), 1)
self.assertEqual(self.matrix.query(1,1,1,5,5,5), 2)
self.assertEqual(self.matrix.query(4,1,1,5,5,5), 1)
self.assertEqual(self.matrix.query(1,1,1,30,30,30), 3)
self.assertEqual(self.matrix.query(20,20,20,30,30,30), 1)
def setUp(self):
self.matrix = Matrix()
class TestMatrix(unittest.TestCase):
def setUp(self):
self.matrix = Matrix()
def test_create(self):
self.matrix.create([2, 2])
self.assertEqual(self.matrix.elements[0][0], 0)
self.assertEqual(self.matrix.elements[0][1], 0)
self.assertEqual(self.matrix.elements[1][0], 0)
self.assertEqual(self.matrix.elements[1][1], 0)
def test_clean(self):
self.matrix.create([2, 2])
self.matrix.elements[0][0] = "A"
self.matrix.elements[0][1] = "B"
self.matrix.elements[1][0] = "C"
self.matrix.elements[1][1] = "D"
self.assertEqual(self.matrix.elements[0][0], "A")
self.assertEqual(self.matrix.elements[0][1], "B")
self.assertEqual(self.matrix.elements[1][0], "C")
self.assertEqual(self.matrix.elements[1][1], "D")
self.matrix.clean()
self.assertEqual(self.matrix.elements[0][0], 0)
self.assertEqual(self.matrix.elements[0][1], 0)
self.assertEqual(self.matrix.elements[1][0], 0)
self.assertEqual(self.matrix.elements[1][1], 0)
def test_draw_element(self):
self.matrix.create([2, 2])
self.matrix.draw_element([1, 1, "A"])
self.assertEqual(self.matrix.elements[0][0], "A")
self.assertEqual(self.matrix.elements[0][1], 0)
self.assertEqual(self.matrix.elements[1][0], 0)
self.assertEqual(self.matrix.elements[1][1], 0)
def test_draw_column(self):
self.matrix.create([3, 4])
self.matrix.draw_column([2, 1, 3, "A"])
self.assertEqual(self.matrix.elements[0][0], 0)
self.assertEqual(self.matrix.elements[0][1], "A")
self.assertEqual(self.matrix.elements[0][2], 0)
self.assertEqual(self.matrix.elements[1][0], 0)
self.assertEqual(self.matrix.elements[1][1], "A")
self.assertEqual(self.matrix.elements[1][2], 0)
self.assertEqual(self.matrix.elements[2][0], 0)
self.assertEqual(self.matrix.elements[2][1], "A")
self.assertEqual(self.matrix.elements[2][2], 0)
self.assertEqual(self.matrix.elements[3][0], 0)
self.assertEqual(self.matrix.elements[3][1], 0)
self.assertEqual(self.matrix.elements[3][2], 0)
def test_draw_row(self):
self.matrix.create([4, 3])
self.matrix.draw_row([1, 3, 2, "A"])
self.assertEqual(self.matrix.elements[0][0], 0)
self.assertEqual(self.matrix.elements[0][1], 0)
self.assertEqual(self.matrix.elements[0][2], 0)
self.assertEqual(self.matrix.elements[0][3], 0)
self.assertEqual(self.matrix.elements[1][0], "A")
self.assertEqual(self.matrix.elements[1][1], "A")
self.assertEqual(self.matrix.elements[1][2], "A")
self.assertEqual(self.matrix.elements[1][3], 0)
self.assertEqual(self.matrix.elements[2][0], 0)
self.assertEqual(self.matrix.elements[2][1], 0)
self.assertEqual(self.matrix.elements[2][2], 0)
self.assertEqual(self.matrix.elements[2][3], 0)
def test_draw_rectangle(self):
self.matrix.create([4, 3])
self.matrix.draw_rectangle([2, 2, 4, 3, "A"])
self.assertEqual(self.matrix.elements[0][0], 0)
self.assertEqual(self.matrix.elements[0][1], 0)
self.assertEqual(self.matrix.elements[0][2], 0)
self.assertEqual(self.matrix.elements[0][3], 0)
self.assertEqual(self.matrix.elements[1][0], 0)
self.assertEqual(self.matrix.elements[1][1], "A")
self.assertEqual(self.matrix.elements[1][2], "A")
self.assertEqual(self.matrix.elements[1][3], "A")
self.assertEqual(self.matrix.elements[2][0], 0)
self.assertEqual(self.matrix.elements[2][1], "A")
self.assertEqual(self.matrix.elements[2][2], "A")
self.assertEqual(self.matrix.elements[2][3], "A")
def buildIdentityMatrix(model):
request.identity = Matrix(model["rows"], model["columns"]).identity()
def setUp(self):
self.matrix = Matrix()
def buildMatricies(model):
request.matrixA = Matrix.fromArray(model['lvalue'])
request.matrixB = Matrix.fromArray(model['rvalue'])
def testDimension(self):
matrix = Matrix.fromArray([
[1,2,3],
[4,5,6]
])
self.assertEqual(util.dimension(matrix), (3,2))
def computeDeterminant(model):
request.det = Matrix.fromArray(model['matrix']).determinant()
def testDimension(self):
matrix = Matrix.fromArray([[1, 2, 3], [4, 5, 6]])
self.assertEqual(util.dimension(matrix), (3, 2))
