def _summarize_issues_sql(self, include_filename=False, ignore=False):
snapshot_file_revisions_table = self.backend.get_table(
self.fields['file_revisions'].relationship_class)
fr_table = self.backend.get_table(FileRevision)
issue_table = self.backend.get_table(Issue)
issue_occurrence_table = self.backend.get_table(IssueOccurrence)
project_issue_class_table = self.backend.get_table(ProjectIssueClass)
issue_class_table = self.backend.get_table(self.project.IssueClass)
project_pk_type = self.backend.get_field_type(
self.project.fields['pk'])
snapshot_pk_type = self.backend.get_field_type(self.fields['pk'])
#we group by file revision path, issue code and analyzer
group_columns = [
fr_table.c.language, fr_table.c.path, issue_table.c.code,
issue_table.c.analyzer
]
#we perform a JOIN of the file revision table to the issue tables
table = fr_table\
.join(issue_occurrence_table,fr_table.c.pk == issue_occurrence_table.c.file_revision)\
.join(issue_table, and_(issue_table.c.pk == issue_occurrence_table.c.issue, issue_table.c.ignore == ignore))
#here we make sure that the given issue class is enabled for the project
subselect = select([issue_class_table.c.pk])\
.select_from(issue_class_table.join(project_issue_class_table))\
.where(and_(
issue_table.c.analyzer == issue_class_table.c.analyzer,
issue_table.c.code == issue_class_table.c.code,
issue_table.c.ignore == ignore,
project_issue_class_table.c.project == expression.cast(self.project.pk,project_pk_type),
project_issue_class_table.c.enabled == True))\
file_revisions_select = select([snapshot_file_revisions_table.c.filerevision])\
.where(snapshot_file_revisions_table.c.snapshot == expression.cast(self.pk,snapshot_pk_type))
#we select the aggregated issues for all file revisions in this snapshot
s = select(group_columns+[func.count().label('count')])\
.select_from(table)\
.where(and_(exists(subselect),fr_table.c.pk.in_(file_revisions_select)))\
.group_by(*group_columns)\
.order_by(fr_table.c.path)
#we fetch the result
with self.backend.transaction():
result = self.backend.connection.execute(s).fetchall()
#we aggregate the issues by path fragments
aggregator = lambda f: directory_splitter(
f['path'], include_filename=include_filename)
#we perform a map/reduce on the result
#the resulting items will contain the number of files and the number of issues in the file
map_reducer = IssuesMapReducer(aggregators=[aggregator])
return map_reducer.mapreduce(result)
def run(self):
branches = self.backend.filter(self.project.GitBranch,{'project.pk' : self.project.pk})
branches_by_name = dict([(branch.name,branch) for branch in branches])
if 'default_branch' in self.project and self.project.default_branch in branches_by_name:
master_branch = branches_by_name[self.project.default_branch]
elif 'origin/master' in branches_by_name:
master_branch = branches_by_name['origin/master']
else:
logger.error("No default branch defined!\n")
return -1
stats = {
'contributors' : self.project.repository.get_contributors(),
'branch' : master_branch.name
}
stats['n_commits'] = sum([contributor['n_commits'] for contributor in stats['contributors']])
snapshots = self.backend.filter(self.project.GitSnapshot,{'sha' : {'$in' : master_branch.snapshots}}).sort('committer_date_ts',-1)
logger.info("Found %d snapshots" % len(snapshots))
if snapshots:
latest_snapshot = snapshots[0]
stats['snapshot'] = latest_snapshot.sha
if 'summary' in latest_snapshot:
stats['summary'] = latest_snapshot.summary
query = {
'project.pk' : self.project.pk,
'file_revision.pk' : {'$in' : latest_snapshot['file_revisions']},
}
issues = list(self.backend.filter(self.project.Issue,query))
environment = CodeEnvironment([],analyzers,aggregators = {'path' :
{'mapper' :
lambda f: directory_splitter(f['path'],
include_filename = False)
}})
group_by = ["language","analyzer","code"]
issues_summary = environment.summarize_issues(issues,group_by = group_by)
stats['issues_summary'] = issues_summary
logger.info("Updating statistics...")
self.project.stats = stats
self.backend.save(self.project)
self.backend.commit()
'props': {
'get': 'checkmate.management.commands.props.get.Command',
'set': 'checkmate.management.commands.props.set.Command',
'delete': 'checkmate.management.commands.props.delete.Command'
}
}
models = {
'Project': Project,
'Snapshot': Snapshot,
'DiskSnapshot': DiskSnapshot,
'FileRevision': FileRevision,
'Issue': Issue,
}
aggregators = {
'directory': {
'mapper':
lambda file_revision: directory_splitter(file_revision['path'],
include_filename=True)
}
}
checkignore = """*/site-packages/*
*/dist-packages/*
*/build/*
*/eggs/*
*/migrations/*
*/alembic/versions/*
"""
def _summarize_issues_sql(self, include_filename=False, ignore=False):
diff_issue_occurrence_table = self.backend.get_table(
DiffIssueOccurrence)
issue_occurrence_table = self.backend.get_table(IssueOccurrence)
issue_table = self.backend.get_table(Issue)
file_revision_table = self.backend.get_table(FileRevision)
project_issue_class_table = self.backend.get_table(ProjectIssueClass)
issue_class_table = self.backend.get_table(self.project.IssueClass)
#we group by file revision path, issue code and analyzer
group_columns = [
file_revision_table.c.language,
file_revision_table.c.path,
diff_issue_occurrence_table.c['key'],
#we should not group by pk
# diff_issue_occurrence_table.c['pk'],
issue_table.c.code,
issue_table.c.analyzer
]
project_pk_type = self.backend.get_field_type(
self.project.fields['pk'])
#here we make sure that the given issue class is enabled for the project
subselect = select([issue_class_table.c.pk])\
.select_from(issue_class_table.join(project_issue_class_table))\
.where(and_(
issue_table.c.analyzer == issue_class_table.c.analyzer,
issue_table.c.code == issue_class_table.c.code,
issue_table.c.ignore == ignore,
project_issue_class_table.c.project == expression.cast(self.project.pk,project_pk_type),
project_issue_class_table.c.enabled == True))\
#we perform a JOIN of the file revision table to the issue tables
table = diff_issue_occurrence_table\
.join(issue_occurrence_table,
issue_occurrence_table.c.pk == diff_issue_occurrence_table.c.issue_occurrence)\
.join(issue_table, and_(issue_occurrence_table.c.issue == issue_table.c.pk, issue_table.c.ignore == ignore))\
.join(file_revision_table)
#we select the aggregated issues for all file revisions in this snapshot
s = select(group_columns+[func.count().label('count')])\
.select_from(table)\
.where(and_(exists(subselect),diff_issue_occurrence_table.c.diff == self.pk))\
.group_by(*group_columns)\
.order_by(file_revision_table.c.path)
#we fetch the result
with self.backend.transaction():
result = self.backend.connection.execute(s).fetchall()
#we aggregate the issues by path fragments
aggregator = lambda f: directory_splitter(
f['path'], include_filename=include_filename)
added_issues = []
fixed_issues = []
for row in result:
if row['key'] == 'added':
added_issues.append(row)
else:
fixed_issues.append(row)
#we perform a map/reduce on the result
map_reducer = IssuesMapReducer(
aggregators=[aggregator],
group_by=['language', 'analyzer', 'code'])
return {
'added': map_reducer.mapreduce(added_issues),
'fixed': map_reducer.mapreduce(fixed_issues)
}
}
plugins = {
'pep8' : 'checkmate.contrib.plugins.python.pep8',
'pylint' : 'checkmate.contrib.plugins.python.pylint',
'pyflakes' : 'checkmate.contrib.plugins.python.pyflakes',
'jshint' : 'checkmate.contrib.plugins.javascript.jshint',
'jslint' : 'checkmate.contrib.plugins.javascript.jslint',
'metrics' : 'checkmate.contrib.plugins.python.metrics',
'git' : 'checkmate.contrib.plugins.git',
}
aggregators = {
'directory' :
{
'mapper' : lambda file_revision:directory_splitter(file_revision['path'])
}
}
{
'type' : 'import',
'name' : 'foo.bar',
'imported_names' : ['bar','baz','ball'],
}
default_checkignore = """*/site-packages/*
*/dist-packages/*
*/build/*
*/eggs/*
*/migrations/*
*/alembic/versions/*
'CodeObject' : CodeObject,
}
plugins = {
'pep8' : 'checkmate.contrib.plugins.python.pep8',
'pylint' : 'checkmate.contrib.plugins.python.pylint',
'pyflakes' : 'checkmate.contrib.plugins.python.pyflakes',
'jshint' : 'checkmate.contrib.plugins.javascript.jshint',
'metrics' : 'checkmate.contrib.plugins.python.metrics',
'git' : 'checkmate.contrib.plugins.git',
}
aggregators = {
'directory' :
{
'mapper' : lambda file_revision:directory_splitter(file_revision['path'])
}
}
{
'type' : 'import',
'name' : 'foo.bar',
'imported_names' : ['bar','baz','ball'],
}
default_checkignore = """*/site-packages/*
*/dist-packages/*
*/build/*
*/eggs/*
*/migrations/*
*/alembic/versions/*
