def _summarize_issues_sql(self, include_filename=False, ignore=False): snapshot_file_revisions_table = self.backend.get_table( self.fields['file_revisions'].relationship_class) fr_table = self.backend.get_table(FileRevision) issue_table = self.backend.get_table(Issue) issue_occurrence_table = self.backend.get_table(IssueOccurrence) project_issue_class_table = self.backend.get_table(ProjectIssueClass) issue_class_table = self.backend.get_table(self.project.IssueClass) project_pk_type = self.backend.get_field_type( self.project.fields['pk']) snapshot_pk_type = self.backend.get_field_type(self.fields['pk']) #we group by file revision path, issue code and analyzer group_columns = [ fr_table.c.language, fr_table.c.path, issue_table.c.code, issue_table.c.analyzer ] #we perform a JOIN of the file revision table to the issue tables table = fr_table\ .join(issue_occurrence_table,fr_table.c.pk == issue_occurrence_table.c.file_revision)\ .join(issue_table, and_(issue_table.c.pk == issue_occurrence_table.c.issue, issue_table.c.ignore == ignore)) #here we make sure that the given issue class is enabled for the project subselect = select([issue_class_table.c.pk])\ .select_from(issue_class_table.join(project_issue_class_table))\ .where(and_( issue_table.c.analyzer == issue_class_table.c.analyzer, issue_table.c.code == issue_class_table.c.code, issue_table.c.ignore == ignore, project_issue_class_table.c.project == expression.cast(self.project.pk,project_pk_type), project_issue_class_table.c.enabled == True))\ file_revisions_select = select([snapshot_file_revisions_table.c.filerevision])\ .where(snapshot_file_revisions_table.c.snapshot == expression.cast(self.pk,snapshot_pk_type)) #we select the aggregated issues for all file revisions in this snapshot s = select(group_columns+[func.count().label('count')])\ .select_from(table)\ .where(and_(exists(subselect),fr_table.c.pk.in_(file_revisions_select)))\ .group_by(*group_columns)\ .order_by(fr_table.c.path) #we fetch the result with self.backend.transaction(): result = self.backend.connection.execute(s).fetchall() #we aggregate the issues by path fragments aggregator = lambda f: directory_splitter( f['path'], include_filename=include_filename) #we perform a map/reduce on the result #the resulting items will contain the number of files and the number of issues in the file map_reducer = IssuesMapReducer(aggregators=[aggregator]) return map_reducer.mapreduce(result)
def run(self): branches = self.backend.filter(self.project.GitBranch,{'project.pk' : self.project.pk}) branches_by_name = dict([(branch.name,branch) for branch in branches]) if 'default_branch' in self.project and self.project.default_branch in branches_by_name: master_branch = branches_by_name[self.project.default_branch] elif 'origin/master' in branches_by_name: master_branch = branches_by_name['origin/master'] else: logger.error("No default branch defined!\n") return -1 stats = { 'contributors' : self.project.repository.get_contributors(), 'branch' : master_branch.name } stats['n_commits'] = sum([contributor['n_commits'] for contributor in stats['contributors']]) snapshots = self.backend.filter(self.project.GitSnapshot,{'sha' : {'$in' : master_branch.snapshots}}).sort('committer_date_ts',-1) logger.info("Found %d snapshots" % len(snapshots)) if snapshots: latest_snapshot = snapshots[0] stats['snapshot'] = latest_snapshot.sha if 'summary' in latest_snapshot: stats['summary'] = latest_snapshot.summary query = { 'project.pk' : self.project.pk, 'file_revision.pk' : {'$in' : latest_snapshot['file_revisions']}, } issues = list(self.backend.filter(self.project.Issue,query)) environment = CodeEnvironment([],analyzers,aggregators = {'path' : {'mapper' : lambda f: directory_splitter(f['path'], include_filename = False) }}) group_by = ["language","analyzer","code"] issues_summary = environment.summarize_issues(issues,group_by = group_by) stats['issues_summary'] = issues_summary logger.info("Updating statistics...") self.project.stats = stats self.backend.save(self.project) self.backend.commit()
'props': { 'get': 'checkmate.management.commands.props.get.Command', 'set': 'checkmate.management.commands.props.set.Command', 'delete': 'checkmate.management.commands.props.delete.Command' } } models = { 'Project': Project, 'Snapshot': Snapshot, 'DiskSnapshot': DiskSnapshot, 'FileRevision': FileRevision, 'Issue': Issue, } aggregators = { 'directory': { 'mapper': lambda file_revision: directory_splitter(file_revision['path'], include_filename=True) } } checkignore = """*/site-packages/* */dist-packages/* */build/* */eggs/* */migrations/* */alembic/versions/* """
def _summarize_issues_sql(self, include_filename=False, ignore=False): diff_issue_occurrence_table = self.backend.get_table( DiffIssueOccurrence) issue_occurrence_table = self.backend.get_table(IssueOccurrence) issue_table = self.backend.get_table(Issue) file_revision_table = self.backend.get_table(FileRevision) project_issue_class_table = self.backend.get_table(ProjectIssueClass) issue_class_table = self.backend.get_table(self.project.IssueClass) #we group by file revision path, issue code and analyzer group_columns = [ file_revision_table.c.language, file_revision_table.c.path, diff_issue_occurrence_table.c['key'], #we should not group by pk # diff_issue_occurrence_table.c['pk'], issue_table.c.code, issue_table.c.analyzer ] project_pk_type = self.backend.get_field_type( self.project.fields['pk']) #here we make sure that the given issue class is enabled for the project subselect = select([issue_class_table.c.pk])\ .select_from(issue_class_table.join(project_issue_class_table))\ .where(and_( issue_table.c.analyzer == issue_class_table.c.analyzer, issue_table.c.code == issue_class_table.c.code, issue_table.c.ignore == ignore, project_issue_class_table.c.project == expression.cast(self.project.pk,project_pk_type), project_issue_class_table.c.enabled == True))\ #we perform a JOIN of the file revision table to the issue tables table = diff_issue_occurrence_table\ .join(issue_occurrence_table, issue_occurrence_table.c.pk == diff_issue_occurrence_table.c.issue_occurrence)\ .join(issue_table, and_(issue_occurrence_table.c.issue == issue_table.c.pk, issue_table.c.ignore == ignore))\ .join(file_revision_table) #we select the aggregated issues for all file revisions in this snapshot s = select(group_columns+[func.count().label('count')])\ .select_from(table)\ .where(and_(exists(subselect),diff_issue_occurrence_table.c.diff == self.pk))\ .group_by(*group_columns)\ .order_by(file_revision_table.c.path) #we fetch the result with self.backend.transaction(): result = self.backend.connection.execute(s).fetchall() #we aggregate the issues by path fragments aggregator = lambda f: directory_splitter( f['path'], include_filename=include_filename) added_issues = [] fixed_issues = [] for row in result: if row['key'] == 'added': added_issues.append(row) else: fixed_issues.append(row) #we perform a map/reduce on the result map_reducer = IssuesMapReducer( aggregators=[aggregator], group_by=['language', 'analyzer', 'code']) return { 'added': map_reducer.mapreduce(added_issues), 'fixed': map_reducer.mapreduce(fixed_issues) }
} plugins = { 'pep8' : 'checkmate.contrib.plugins.python.pep8', 'pylint' : 'checkmate.contrib.plugins.python.pylint', 'pyflakes' : 'checkmate.contrib.plugins.python.pyflakes', 'jshint' : 'checkmate.contrib.plugins.javascript.jshint', 'jslint' : 'checkmate.contrib.plugins.javascript.jslint', 'metrics' : 'checkmate.contrib.plugins.python.metrics', 'git' : 'checkmate.contrib.plugins.git', } aggregators = { 'directory' : { 'mapper' : lambda file_revision:directory_splitter(file_revision['path']) } } { 'type' : 'import', 'name' : 'foo.bar', 'imported_names' : ['bar','baz','ball'], } default_checkignore = """*/site-packages/* */dist-packages/* */build/* */eggs/* */migrations/* */alembic/versions/*
'CodeObject' : CodeObject, } plugins = { 'pep8' : 'checkmate.contrib.plugins.python.pep8', 'pylint' : 'checkmate.contrib.plugins.python.pylint', 'pyflakes' : 'checkmate.contrib.plugins.python.pyflakes', 'jshint' : 'checkmate.contrib.plugins.javascript.jshint', 'metrics' : 'checkmate.contrib.plugins.python.metrics', 'git' : 'checkmate.contrib.plugins.git', } aggregators = { 'directory' : { 'mapper' : lambda file_revision:directory_splitter(file_revision['path']) } } { 'type' : 'import', 'name' : 'foo.bar', 'imported_names' : ['bar','baz','ball'], } default_checkignore = """*/site-packages/* */dist-packages/* */build/* */eggs/* */migrations/* */alembic/versions/*