def main():
info("CLMT Data Processing Tool")
info("-------------------------")
info("Vassilios Karakoidas (c) 2013")
info("[email protected] (http://bkarak.wizhut.com/)")
info("")
if len(sys.argv) not in [1, 2, 3]:
error('usage: clmt-stats.py [initdb|-d {directory}|-f {file}]')
return
if sys.argv[1] == 'initdb':
from db.internals import create_database
info("Creating the SQLite database ... ")
create_database()
return
if len(sys.argv) == 3:
if sys.argv[1] == '-f' and os.path.isfile(sys.argv[2]):
info('Checking file: %s' % (sys.argv[2], ))
import_file(sys.argv[2])
elif sys.argv[1] == '-d' and os.path.isdir(sys.argv[2]):
info('Checking directory: %s' % (sys.argv[2], ))
import_directory(sys.argv[2])
elif sys.argv[1] == '-dups' and os.path.isdir(sys.argv[2]):
info('Duplicates Check on %s' % (sys.argv[2], ))
check_directory(sys.argv[2])
else:
warning('"%s" is not a valid file to import!' % (sys.argv, ))
def convert_jdepend_to_clmtx(filename):
result = []
try:
fp = open(filename, 'r')
xml_json = xmldict.parse(fp)
packages = xml_json['JDepend']['Packages']['Package']
if isinstance(packages, list):
for p in packages:
try:
package_name = p.get('@name', 'NotSet')
if p.get('error', None) is not None:
continue
stats = p.get('Stats', {})
results = {'id': package_name,
'filename': package_name.replace('.', '/'),
'category': 'module',
'measurement': [
{'name': 'NumberOfClasses', 'result-type': 'integer', 'value': stats.get('TotalClasses', 0)},
{'name': 'NumberOfConcreteClasses', 'result-type': 'integer', 'value': stats.get('ConcreteClasses', 0)},
{'name': 'NumberOfAbstractClasses', 'result-type': 'integer', 'value': stats.get('AbstractClasses', 0)},
{'name': 'AfferentCouplings', 'result-type': 'integer', 'value': stats.get('Ca', 0)},
{'name': 'EfferentCouplings', 'result-type': 'integer', 'value': stats.get('Ce', 0)},
{'name': 'Abstractness', 'result-type': 'float', 'value': stats.get('A', 0.0)},
{'name': 'Instability', 'result-type': 'float', 'value': stats.get('I', 0.0)},
{'name': 'DistanceMainSequence', 'result-type': 'float', 'value': stats.get('D', 0.0)}]}
result.append(results)
except Exception, e:
warning('Conversion error: %s' % (e,))
continue
except Exception, e:
warning('Could not convert JDepend XML to clmtx (%s, %s)' % (filename, e))
def main():
info("CLMT Data Processing Tool")
info("-------------------------")
info("Vassilios Karakoidas (c) 2013")
info("[email protected] (http://bkarak.wizhut.com/)")
info("")
if len(sys.argv) not in [1, 2, 3]:
error('usage: clmt-stats.py [initdb|-d {directory}|-f {file}]')
return
if sys.argv[1] == 'initdb':
from db.internals import create_database
info("Creating the SQLite database ... ")
create_database()
return
if len(sys.argv) == 3:
if sys.argv[1] == '-f' and os.path.isfile(sys.argv[2]):
info('Checking file: %s' % (sys.argv[2],))
import_file(sys.argv[2])
elif sys.argv[1] == '-d' and os.path.isdir(sys.argv[2]):
info('Checking directory: %s' % (sys.argv[2],))
import_directory(sys.argv[2])
elif sys.argv[1] == '-dups' and os.path.isdir(sys.argv[2]):
info('Duplicates Check on %s' % (sys.argv[2],))
check_directory(sys.argv[2])
else:
warning('"%s" is not a valid file to import!' % (sys.argv,))
def load(self):
results = []
if self.is_valid():
if self.generator() == 'clmtx':
try:
fp = open(self.filename(), 'r')
data_json = json.load(fp)
for de in data_json:
# Artifact
results.append(None)
fp.close()
except Exception, e:
warning('JSON is not in clmtx format (%s, %s)' %
(self.filename(), e))
elif self.generator() == 'jdepend':
try:
data_json = jdepend.convert_jdepend_to_clmtx(
self.filename())
for de in data_json:
# artifact
results.append(None)
except Exception, e:
warning('XML is not in jdepend format (%s, %s)' %
(self.filename(), e))
def load_directory(directory):
if not os.path.exists(directory):
warning('Directory does not exist (%s)' % (directory,))
return []
artifacts = []
for (root, dirs, files) in os.walk(directory):
for f in files:
r = {'file' : '%s/%s' % (root, f)}
if f.endswith('.json'):
r['type'] = 'json'
r['generator'] = 'clmtx'
elif f.endswith('.xml'):
r['type'] = 'xml'
r['generator'] = 'jdepend'
else:
continue
artifacts.extend(DataFile(r).load())
return artifacts
def load_directory(directory):
if not os.path.exists(directory):
warning('Directory does not exist (%s)' % (directory, ))
return []
artifacts = []
for (root, dirs, files) in os.walk(directory):
for f in files:
r = {'file': '%s/%s' % (root, f)}
if f.endswith('.json'):
r['type'] = 'json'
r['generator'] = 'clmtx'
elif f.endswith('.xml'):
r['type'] = 'xml'
r['generator'] = 'jdepend'
else:
continue
artifacts.extend(DataFile(r).load())
return artifacts
def load(self):
results = []
if self.is_valid():
if self.generator() == 'clmtx':
try:
fp = open(self.filename(), 'r')
data_json = json.load(fp)
for de in data_json:
# Artifact
results.append(None)
fp.close()
except Exception, e:
warning('JSON is not in clmtx format (%s, %s)' % (self.filename(), e))
elif self.generator() == 'jdepend':
try:
data_json = jdepend.convert_jdepend_to_clmtx(self.filename())
for de in data_json:
# artifact
results.append(None)
except Exception, e:
warning('XML is not in jdepend format (%s, %s)' % (self.filename(), e))
def is_valid(self):
__valid_gentypes = {'clmtx': ['json'], 'jdepend': ['xml']}
generator_types = __valid_gentypes.get(self.generator(), None)
if generator_types is None:
warning('Invalid Generator %s not in %s' % (self.generator(), __valid_gentypes.keys()))
return False
if self.type() in generator_types:
if os.path.exists(self.filename()):
return True
else:
warning('File does not exist: %s' % (self.filename(),))
else:
warning('Invalid File Type %s not in %s' % (self.type(), generator_types))
return False
def is_valid(self):
__valid_gentypes = {'clmtx': ['json'], 'jdepend': ['xml']}
generator_types = __valid_gentypes.get(self.generator(), None)
if generator_types is None:
warning('Invalid Generator %s not in %s' %
(self.generator(), __valid_gentypes.keys()))
return False
if self.type() in generator_types:
if os.path.exists(self.filename()):
return True
else:
warning('File does not exist: %s' % (self.filename(), ))
else:
warning('Invalid File Type %s not in %s' %
(self.type(), generator_types))
return False
def convert_jdepend_to_clmtx(filename):
result = []
try:
fp = open(filename, 'r')
xml_json = xmldict.parse(fp)
packages = xml_json['JDepend']['Packages']['Package']
if isinstance(packages, list):
for p in packages:
try:
package_name = p.get('@name', 'NotSet')
if p.get('error', None) is not None:
continue
stats = p.get('Stats', {})
results = {
'id':
package_name,
'filename':
package_name.replace('.', '/'),
'category':
'module',
'measurement': [{
'name': 'NumberOfClasses',
'result-type': 'integer',
'value': stats.get('TotalClasses', 0)
}, {
'name':
'NumberOfConcreteClasses',
'result-type':
'integer',
'value':
stats.get('ConcreteClasses', 0)
}, {
'name':
'NumberOfAbstractClasses',
'result-type':
'integer',
'value':
stats.get('AbstractClasses', 0)
}, {
'name': 'AfferentCouplings',
'result-type': 'integer',
'value': stats.get('Ca', 0)
}, {
'name': 'EfferentCouplings',
'result-type': 'integer',
'value': stats.get('Ce', 0)
}, {
'name': 'Abstractness',
'result-type': 'float',
'value': stats.get('A', 0.0)
}, {
'name': 'Instability',
'result-type': 'float',
'value': stats.get('I', 0.0)
}, {
'name': 'DistanceMainSequence',
'result-type': 'float',
'value': stats.get('D', 0.0)
}]
}
result.append(results)
except Exception, e:
warning('Conversion error: %s' % (e, ))
continue
except Exception, e:
warning('Could not convert JDepend XML to clmtx (%s, %s)' %
(filename, e))
def import_json(project_name, project_input, json_obj):
from db.models import Filenames, Category, Project, Measurement, MeasurementType
info('Importing Project: %s (%s)' % (project_name, project_input))
_project_obj = Project.get_or_insert(project_name)
if _project_obj == -1:
error('Could not import Project: %s (%s)' % (project_name, project_input))
return
for element in json_obj:
try:
# retrieve id
_id = element['id']
_category = element['category']
_filename = element['filename']
# retrieve ids
_id_obj = Filenames.get_or_insert(_id)
if _id_obj == -1:
warning('Invalid Element (id, filename): (%s, %s)' % (_id, _filename))
continue
# retrieve categories
_category_obj = Category.get_or_insert(_category)
if _category_obj == -1:
warning('Invalid Element (id, filename): (%s, %s)' % (_id, _filename))
continue
# retrieve filenames
_filename_obj = Filenames.get_or_insert(_filename)
if _filename_obj == -1:
warning('Invalid Element (id, filename): (%s, %s)' % (_id, _filename))
continue
from db.internals import Session
db_session = Session()
for measurements in element.get('measurement', []):
try:
try:
_name = measurements[u'name']
except Exception, e:
_name = measurements[u'Name']
_mt_obj = MeasurementType.get_or_insert(_name + '_' + project_input)
if _mt_obj == -1:
warning('Invalid Measurement Type: %s' % (_name + '_' + project_input,))
continue
_value = str(measurements['value'])
meas_object = Measurement()
meas_object.cat_pk = _category_obj
meas_object.prj_pk = _project_obj
meas_object.meas_id = _id_obj
meas_object.meas_filename = _filename_obj
meas_object.meas_value = _value
meas_object.mt_pk = _mt_obj
db_session.add(meas_object)
except Exception, e:
warning('Invalid Measurement: %s (%s)' % (measurements, e.message))
continue
def import_json(project_name, project_input, json_obj):
from db.models import Filenames, Category, Project, Measurement, MeasurementType
info('Importing Project: %s (%s)' % (project_name, project_input))
_project_obj = Project.get_or_insert(project_name)
if _project_obj == -1:
error('Could not import Project: %s (%s)' %
(project_name, project_input))
return
for element in json_obj:
try:
# retrieve id
_id = element['id']
_category = element['category']
_filename = element['filename']
# retrieve ids
_id_obj = Filenames.get_or_insert(_id)
if _id_obj == -1:
warning('Invalid Element (id, filename): (%s, %s)' %
(_id, _filename))
continue
# retrieve categories
_category_obj = Category.get_or_insert(_category)
if _category_obj == -1:
warning('Invalid Element (id, filename): (%s, %s)' %
(_id, _filename))
continue
# retrieve filenames
_filename_obj = Filenames.get_or_insert(_filename)
if _filename_obj == -1:
warning('Invalid Element (id, filename): (%s, %s)' %
(_id, _filename))
continue
from db.internals import Session
db_session = Session()
for measurements in element.get('measurement', []):
try:
try:
_name = measurements[u'name']
except Exception, e:
_name = measurements[u'Name']
_mt_obj = MeasurementType.get_or_insert(_name + '_' +
project_input)
if _mt_obj == -1:
warning('Invalid Measurement Type: %s' %
(_name + '_' + project_input, ))
continue
_value = str(measurements['value'])
meas_object = Measurement()
meas_object.cat_pk = _category_obj
meas_object.prj_pk = _project_obj
meas_object.meas_id = _id_obj
meas_object.meas_filename = _filename_obj
meas_object.meas_value = _value
meas_object.mt_pk = _mt_obj
db_session.add(meas_object)
except Exception, e:
warning('Invalid Measurement: %s (%s)' %
(measurements, e.message))
continue