def handle(self, projects=None, refetch=False, **options): if projects: project_list = list(Project.objects.filter(label__in=projects)) assert len(projects) == len(projects), 'one or more project label was not found' else: project_list = Project.objects.filter(label__contains='/') for project in project_list: print "Project %r" % project.label for data in github.iter_commits(project.github_user, project.github_repo): print " Revision %r (%s; %s)" % (data['sha'], data['commit']['author']['name'], data['commit']['committer']['date']) rev, created = Revision.get_or_create(project, data['sha'], data=data) if not created: if not rev.data or refetch: rev.update_from_github(data) rev.save()
def get_git_changes(build, previous_build): # TODO: reenable this when it doesnt hit github on each request return None project = build.project try: results = github.compare_commits(project.github_user, project.github_repo, previous_build.revision.label, build.revision.label) except Exception: # XXX: likely we hit rate limit return None commits = [] for commit in results['commits']: revision = Revision.get_or_create(build.project, commit['sha'])[0] commits.append(revision) return reversed(commits)
def import_build(data, project=None, revision=None): version = int(data.get('version', 1)) timestamp = convert_timestamp(data['time']) project_label = project or data.get('project') if not project_label: raise ValueError('You must specify a project') revision_label = revision or data.get('revision') if not revision_label: raise ValueError('You must specify a revision') # We avoid recreating the core items that might get referenced by an ID in the interface project = Project.objects.get_or_create( label=project_label, )[0] revision = Revision.get_or_create( project=project, label=revision_label, )[0] build, created = Build.objects.get_or_create( project=project, revision=revision, datetime=timestamp, ) # Clean out old tests build.test_set.all().delete() # Clean out old tags build.tags.all().delete() # Add tags tag_list = [ BuildTag.objects.get_or_create(label=tag_name)[0] for tag_name in data.get('tags', []) ] build.tags.add(*tag_list) num_tests = 0 total_duration = 0.0 tests_by_id = {} grouped_tests = regroup_tests(data['tests']) # Eliminate useless parents (parents which only have a single child) leaf_counts = count_leaves_with_tests((t['id'] for t in data['tests'])) def find_parent(label): if '.' not in label: return None key = label.split('.')[:-1] while key: path = '.'.join(key) if path in tests_by_id: return tests_by_id[path] key.pop() return None grouped_tests = [(l, t) for l, t in grouped_tests if leaf_counts.get(l) >= 1] # Create branches for label, _ in grouped_tests: parent = find_parent(label) branch = Test.objects.create( parent=parent, project=project, revision=revision, build=build, label=label, ) tests_by_id[branch.label] = branch for label, tests in reversed(grouped_tests): branch = tests_by_id[label] # Create any leaves which do not exist yet for test_data in (t for t in tests if t['id'] not in tests_by_id): test = create_test_leaf(build, test_data, branch, version) tests_by_id[test.label] = test num_tests += 1 total_duration += test.mean_duration # Update aggregated data group_durations = [] interface_durations = defaultdict(list) for test in (tests_by_id[t['id']] for t in tests): group_durations.append(test.mean_duration) for interface, values in test.data.iteritems(): interface_durations[interface].append(values) group_num_tests = len(group_durations) group_total_duration = sum(group_durations) group_durations.sort() td_data = {} for interface, values in interface_durations.iteritems(): durations = [v['mean_duration'] for v in values] td_data[interface] = { 'mean_calls': sum(v['mean_calls'] for v in values), 'mean_duration': sum(durations), 'upper_duration': durations[-1], 'lower_duration': durations[0], 'upper90_duration': percentile(durations, 90), } tests_by_id[branch.label] = Test.objects.filter(id=branch.id).update( label=label, num_tests=group_num_tests, mean_duration=group_total_duration, upper_duration=group_durations[-1], lower_duration=group_durations[0], upper90_duration=percentile(group_durations, 90), data=td_data, ) Build.objects.filter(id=build.id).update( num_tests=num_tests, total_duration=total_duration, ) return build