def test_get_loc(self):
path = os.path.join(ASSETS_PATH, 'projekt')
# Test: Get SLOC of source at ./assets/projekt
expected = {
'C': {'cloc': 4, 'sloc': 6},
'C/C++ Header': {'cloc': 0, 'sloc': 4},
'Javascript': {'cloc': 1, 'sloc': 13},
'Python': {'cloc': 1, 'sloc': 2}
}
sloc = utilities.get_loc(path)
self.assertEqual(len(expected), len(sloc))
for (k, v) in sloc.items():
for (_k, _v) in v.items():
self.assertEqual(expected[k][_k], _v)
# Test: Get SLOC of only the directory 'include' for source at
# ./assets/projekt
expected = {
'C/C++ Header': {'cloc': 0, 'sloc': 4},
}
sloc = utilities.get_loc(path, files=['include/projekt.h'])
self.assertEqual(len(expected), len(sloc))
for (k, v) in sloc.items():
for (_k, _v) in v.items():
self.assertEqual(expected[k][_k], _v)
# Test: Get SLOC of only the directories 'include' and 'utilities' for
# source at ./assets/projekt
expected = {
'C/C++ Header': {'cloc': 0, 'sloc': 4},
'Javascript': {'cloc': 1, 'sloc': 10},
}
sloc = utilities.get_loc(
path, files=['include/projekt.h', 'utilities/projekt.js']
)
self.assertEqual(len(expected), len(sloc))
for (k, v) in sloc.items():
for (_k, _v) in v.items():
self.assertEqual(expected[k][_k], _v)
# Test: Missing source directory
self.assertRaises(Exception, utilities.get_loc, '/home/nocturnal')
# Test: Path is not a directory
self.assertRaises(Exception, utilities.get_loc, '/bin/bash')
def discover(self, path):
if not (self.frameworks and self.languages and self.extensions):
raise Exception(
'{0} is not appropriately configured.'.format(
self.__class__.__name__
)
)
# SLOC of source code
_sloc = utilities.get_loc(path)
sloc = 0
for language in self.languages:
if language in _sloc:
sloc += _sloc[language]['sloc']
proportion = 0
if sloc > 0:
# Pass 1: Look for a unit testing frameworks
for framework in self.frameworks:
proportion += framework(path, sloc)
# Pass 2: Look for files in conventional test directories
# NOTE: Second pass is necessary only when the proportion of unit
# tests from Pass 1 is less than 1%
if proportion < 0.01:
_path = None
done = False
for (root, dnames, _) in os.walk(path):
for item in TEST_DIRECTORIES:
if item in dnames:
_path = os.path.join(root, item)
done = True
break
if done:
break
files = None
if _path:
files = utilities.get_files(_path, self.language)
if files:
# SLOC of test code
print('path_init: ',path)
_slotc = utilities.get_loc(path, files=files)
slotc = 0
for language in self.languages:
if language in _slotc:
slotc += _slotc[language]['sloc']
proportion = max(proportion, slotc / sloc)
return proportion
def get_lsloc(path, languages):
_sloc = utilities.get_loc(path)
sloc = 0
for language in languages:
if language in _sloc:
sloc += _sloc[language]['sloc']
return sloc
def discover(self, path):
if not (self.frameworks and self.languages and self.extensions):
raise Exception("{0} is not appropriately configured.".format(self.__class__.__name__))
# SLOC of source code
_sloc = utilities.get_loc(path)
sloc = 0
for language in self.languages:
if language in _sloc:
sloc += _sloc[language]["sloc"]
proportion = 0
if sloc > 0:
# Pass 1: Look for a unit testing frameworks
for framework in self.frameworks:
proportion += framework(path, sloc)
# Pass 2: Look for files in conventional test directories
# NOTE: Second pass is necessary only when the proportion of unit
# tests from Pass 1 is less than 1%
if proportion < 0.01:
_path = None
done = False
for (root, dnames, _) in os.walk(path):
for item in TEST_DIRECTORIES:
if item in dnames:
_path = os.path.join(root, item)
done = True
break
if done:
break
files = None
if _path:
files = utilities.get_files(_path, self.language)
if files:
# SLOC of test code
_slotc = utilities.get_loc(path, files=files)
slotc = 0
for language in self.languages:
if language in _slotc:
slotc += _slotc[language]["sloc"]
proportion = max(proportion, slotc / sloc)
return proportion
def doc():
try:
os.chdir(root_directory)
ratio = 0
util = utilities.get_loc(repo_path)
sloc = 0
cloc = 0
for lang, metrics in util.items():
sloc += metrics['sloc']
cloc += metrics['cloc']
if sloc == 0:
print('DOCUMENTATION,SIZE: ',ratio)
else:
t_loc = sloc + cloc
ratio = (cloc / t_loc) if t_loc > 0 else 0
if(ratio > 0.0):
val = (str(ratio),repo_id)
QUERY = '''UPDATE `repoquester_results` SET `documentation`=? WHERE `repo_id`=?'''
try:
os.chdir(root_directory)
conn = sqlite3.connect('repo_quester.db',timeout=180)
conn.execute(QUERY,val)
conn.commit()
conn.close()
print('DOCUMENTATION: ',ratio)
except Exception as ex:
print(ex)
os.chdir(root_directory)
conn = sqlite3.connect('repo_quester.db',timeout=180)
conn.execute(QUERY,val)
conn.commit()
conn.close()
print('DOCUMENTATION: ',ratio,' trying again.. ok')
if(sloc > 0):
val2 = (str(sloc),repo_id)
QUERY2 = '''UPDATE `repoquester_results` SET `size`=? WHERE `repo_id`=?'''
try:
os.chdir(root_directory)
conn2 = sqlite3.connect('repo_quester.db',timeout=180)
conn2.execute(QUERY2,val2)
conn2.commit()
conn2.close()
print('SIZE: ',str(sloc))
except Exception as ex:
os.chdir(root_directory)
conn2 = sqlite3.connect('repo_quester.db',timeout=180)
conn2.execute(QUERY2,val2)
conn2.commit()
conn2.close()
print('SIZE: ',str(sloc),' trying again.. ok')
except Exception as ex:
print(ex)
def measure(self, path, sloc, pattern, whole=False):
proportion = 0
files = utilities.search(pattern, path, whole=whole, include=self.extensions)
if files:
# SLOC of test code
_slotc = utilities.get_loc(path, files=files)
slotc = 0
for language in self.languages:
if language in _slotc:
slotc += _slotc[language]["sloc"]
proportion = slotc / sloc
return proportion
def run(project_id, repo_path, cursor, **options):
print("----- METRIC: REPOSITORY SIZE -----")
threshold = options.get('threshold', 0)
cursor.execute(QUERY.format(project_id))
record = cursor.fetchone()
language = record[0]
languages = MAP[language] if language in MAP else [language]
_sloc = utilities.get_loc(repo_path)
rresult = sum([int(item['sloc']) for (_, item) in _sloc.items()])
bresult = True if rresult >= threshold else False
print('Repository Size: ', rresult)
return bresult, rresult
def measure(self, path, sloc, pattern, whole=False):
proportion = 0
files = utilities.search(
pattern, path, whole=whole, include=self.extensions
)
if files:
# SLOC of test code
_slotc = utilities.get_loc(path, files=files)
slotc = 0
for language in self.languages:
if language in _slotc:
slotc += _slotc[language]['sloc']
proportion = slotc / sloc
return proportion
def run(project_id, repo_path, cursor, **options):
threshold = options.get('threshold', 0)
query = 'SELECT language FROM projects WHERE id = %d' % project_id
cursor.execute(query)
record = cursor.fetchone()
language = record[0]
languages = MAP[language] if language in MAP else [language]
_sloc = utilities.get_loc(repo_path)
rresult = sum([int(item['sloc']) for (_, item) in _sloc.items()])
bresult = True if rresult >= threshold else False
return bresult, rresult
def run(project_id, repo_path, cursor, **options):
print("----- METRIC: PROJECT_SIZE -----")
threshold = options.get('threshold', 0)
cursor.execute(QUERY.format(project_id))
record = cursor.fetchone()
language = record[0]
languages = MAP[language] if language in MAP else [language]
sloc = utilities.get_loc(repo_path)
rresult = 0
for language in languages:
if language in sloc:
rresult += sloc[language]['sloc']
bresult = True if rresult >= threshold else False
print('Size: ', rresult)
return bresult, rresult
def run(project_id, repo_path, cursor, **options):
threshold = options.get('threshold', 0)
query = 'SELECT language FROM projects WHERE id = %d' % project_id
cursor.execute(query)
record = cursor.fetchone()
language = record[0]
languages = MAP[language] if language in MAP else [language]
sloc = utilities.get_loc(repo_path)
rresult = 0
for language in languages:
if language in sloc:
rresult += sloc[language]['sloc']
bresult = True if rresult >= threshold else False
print("----- METRIC: PROJECT_SIZE -----")
print('project_size: ', rresult, ", ", bresult)
return bresult, rresult
def run(project_id, repo_path, cursor, **options):
print("----- METRIC: DOCUMENTATION -----")
ratio = 0
# Dictionary of language => metrics dictionary
util = utilities.get_loc(repo_path)
sloc = 0
cloc = 0
for lang, metrics in util.items():
sloc += metrics['sloc']
cloc += metrics['cloc']
if sloc == 0: # No source code
return False, ratio
t_loc = sloc + cloc
ratio = (cloc / t_loc) if t_loc > 0 else 0
attr_threshold = options['threshold']
attr_pass = (ratio >= attr_threshold)
print('Documentation: ', ratio)
return (attr_pass, ratio)
def run(project_id, repo_path, cursor, **options):
ratio = 0
# Dictionary of language => metrics dictionary
util = utilities.get_loc(repo_path)
sloc = 0
cloc = 0
for lang, metrics in util.items():
sloc += metrics['sloc']
cloc += metrics['cloc']
if sloc == 0: # No source code
return False, ratio
t_loc = sloc + cloc
ratio = (cloc / t_loc) if t_loc > 0 else 0
attr_threshold = options['threshold']
attr_pass = (ratio >= attr_threshold)
return (attr_pass, ratio)
def run(project_id, repo_path, cursor, **options):
num_core_doc_words = 0
totalNumberOfdocWords = 0
cursor.execute(QUERY.format(project_id))
repoName = cursor.fetchone()[0]
os.chdir("path/" + str(project_id) + "/")
stri = os.getcwd()
for repos in os.listdir():
if (repos == repoName):
os.chdir(repos)
stream = []
print("----- METRIC: DOCUMENTATION QUALITY -----")
print('os path: ', os.getcwd())
for (root, dirs, files) in inner_os.walk(os.getcwd(),
topdown=True):
for fi in files:
if (fi.endswith('.py')):
stream += comment_parser.extract_comments(
os.path.join(root, fi), mime='text/x-python')
elif (fi.endswith('.rb')):
stream += comment_parser.extract_comments(
os.path.join(root, fi), mime='text/x-ruby')
elif (fi.endswith('.c')):
stream += comment_parser.extract_comments(
os.path.join(root, fi), mime='text/x-c')
elif (fi.endswith('.cpp')):
stream += comment_parser.extract_comments(
os.path.join(root, fi), mime='text/x-c++')
elif (fi.endswith('.go')):
stream += comment_parser.extract_comments(
os.path.join(root, fi), mime='text/x-go')
elif (fi.endswith('.html')):
stream += comment_parser.extract_comments(
os.path.join(root, fi), mime='text/html')
elif (fi.endswith('.java')):
stream += comment_parser.extract_comments(
os.path.join(root, fi), mime='text/x-java-source')
elif (fi.endswith('.js')):
stream += comment_parser.extract_comments(
os.path.join(root, fi),
mime='application/javascript')
elif (fi.endswith('.sh')):
stream += comment_parser.extract_comments(
os.path.join(root, fi), mime='text/x-shellscript')
elif (fi.endswith('.xml')):
stream += comment_parser.extract_comments(
os.path.join(root, fi), mime='text/xml')
for docs in stream:
docs = str(docs).lower()
trim_doc = re.sub("[^a-zA-Z ]+", "", docs)
trim_doc = ' '.join(trim_doc.split())
totalNumberOfdocWords += len(trim_doc.split())
spell = SpellChecker()
trim_doc = re.sub(r"\b[a-zA-Z]\b", "", trim_doc)
trim_doc = re.sub(r"\b[a-zA-Z][a-zA-Z]\b", "", trim_doc)
trim_doc = trim_doc.split()
spelled = spell.known(trim_doc)
num_core_doc_words += len(spelled)
docs_ratio = 0
if (totalNumberOfdocWords > 0):
docs_ratio = float(num_core_doc_words) / float(
totalNumberOfdocWords * 1.0)
#print('documentation ratio: ',docs_ratio)
break
ratio = 0
# Dictionary of language => metrics dictionary
util = utilities.get_loc(stri)
sloc = 0
cloc = 0
for lang, metrics in util.items():
sloc += metrics['sloc']
cloc += metrics['cloc']
if sloc == 0: # No source code
return False, ratio
t_loc = sloc + cloc
ratio = (cloc / t_loc) if t_loc > 0 else 0
ratio = ratio * docs_ratio
attr_threshold = options['threshold']
print("----- METRIC: DOCUMENTATION -----")
print('ratio: ', ratio)
threshold = options['threshold']
return (ratio >= threshold, ratio)
def test_get_loc(self):
path = os.path.join(ASSETS_PATH, 'projekt')
# Test: Get SLOC of source at ./assets/projekt
expected = {
'C': {
'cloc': 4,
'sloc': 6
},
'C/C++ Header': {
'cloc': 0,
'sloc': 4
},
'Javascript': {
'cloc': 1,
'sloc': 13
},
'Python': {
'cloc': 1,
'sloc': 2
}
}
sloc = utilities.get_loc(path)
self.assertEqual(len(expected), len(sloc))
for (k, v) in sloc.items():
for (_k, _v) in v.items():
self.assertEqual(expected[k][_k], _v)
# Test: Get SLOC of only the directory 'include' for source at
# ./assets/projekt
expected = {
'C/C++ Header': {
'cloc': 0,
'sloc': 4
},
}
sloc = utilities.get_loc(path, files=['include/projekt.h'])
self.assertEqual(len(expected), len(sloc))
for (k, v) in sloc.items():
for (_k, _v) in v.items():
self.assertEqual(expected[k][_k], _v)
# Test: Get SLOC of only the directories 'include' and 'utilities' for
# source at ./assets/projekt
expected = {
'C/C++ Header': {
'cloc': 0,
'sloc': 4
},
'Javascript': {
'cloc': 1,
'sloc': 10
},
}
sloc = utilities.get_loc(
path, files=['include/projekt.h', 'utilities/projekt.js'])
self.assertEqual(len(expected), len(sloc))
for (k, v) in sloc.items():
for (_k, _v) in v.items():
self.assertEqual(expected[k][_k], _v)
# Test: Missing source directory
self.assertRaises(Exception, utilities.get_loc, '/home/nocturnal')
# Test: Path is not a directory
self.assertRaises(Exception, utilities.get_loc, '/bin/bash')
