def handle(self, projects=None, refetch=False, **options):
if projects:
project_list = list(Project.objects.filter(label__in=projects))
assert len(projects) == len(projects), 'one or more project label was not found'
else:
project_list = Project.objects.filter(label__contains='/')
for project in project_list:
print "Project %r" % project.label
for data in github.iter_commits(project.github_user, project.github_repo):
print " Revision %r (%s; %s)" % (data['sha'], data['commit']['author']['name'],
data['commit']['committer']['date'])
rev, created = Revision.get_or_create(project, data['sha'], data=data)
if not created:
if not rev.data or refetch:
rev.update_from_github(data)
rev.save()
def get_git_changes(build, previous_build):
# TODO: reenable this when it doesnt hit github on each request
return None
project = build.project
try:
results = github.compare_commits(project.github_user, project.github_repo,
previous_build.revision.label, build.revision.label)
except Exception:
# XXX: likely we hit rate limit
return None
commits = []
for commit in results['commits']:
revision = Revision.get_or_create(build.project, commit['sha'])[0]
commits.append(revision)
return reversed(commits)
def import_build(data, project=None, revision=None):
version = int(data.get('version', 1))
timestamp = convert_timestamp(data['time'])
project_label = project or data.get('project')
if not project_label:
raise ValueError('You must specify a project')
revision_label = revision or data.get('revision')
if not revision_label:
raise ValueError('You must specify a revision')
# We avoid recreating the core items that might get referenced by an ID in the interface
project = Project.objects.get_or_create(
label=project_label,
)[0]
revision = Revision.get_or_create(
project=project,
label=revision_label,
)[0]
build, created = Build.objects.get_or_create(
project=project,
revision=revision,
datetime=timestamp,
)
# Clean out old tests
build.test_set.all().delete()
# Clean out old tags
build.tags.all().delete()
# Add tags
tag_list = [
BuildTag.objects.get_or_create(label=tag_name)[0]
for tag_name in data.get('tags', [])
]
build.tags.add(*tag_list)
num_tests = 0
total_duration = 0.0
tests_by_id = {}
grouped_tests = regroup_tests(data['tests'])
# Eliminate useless parents (parents which only have a single child)
leaf_counts = count_leaves_with_tests((t['id'] for t in data['tests']))
def find_parent(label):
if '.' not in label:
return None
key = label.split('.')[:-1]
while key:
path = '.'.join(key)
if path in tests_by_id:
return tests_by_id[path]
key.pop()
return None
grouped_tests = [(l, t) for l, t in grouped_tests if leaf_counts.get(l) >= 1]
# Create branches
for label, _ in grouped_tests:
parent = find_parent(label)
branch = Test.objects.create(
parent=parent,
project=project,
revision=revision,
build=build,
label=label,
)
tests_by_id[branch.label] = branch
for label, tests in reversed(grouped_tests):
branch = tests_by_id[label]
# Create any leaves which do not exist yet
for test_data in (t for t in tests if t['id'] not in tests_by_id):
test = create_test_leaf(build, test_data, branch, version)
tests_by_id[test.label] = test
num_tests += 1
total_duration += test.mean_duration
# Update aggregated data
group_durations = []
interface_durations = defaultdict(list)
for test in (tests_by_id[t['id']] for t in tests):
group_durations.append(test.mean_duration)
for interface, values in test.data.iteritems():
interface_durations[interface].append(values)
group_num_tests = len(group_durations)
group_total_duration = sum(group_durations)
group_durations.sort()
td_data = {}
for interface, values in interface_durations.iteritems():
durations = [v['mean_duration'] for v in values]
td_data[interface] = {
'mean_calls': sum(v['mean_calls'] for v in values),
'mean_duration': sum(durations),
'upper_duration': durations[-1],
'lower_duration': durations[0],
'upper90_duration': percentile(durations, 90),
}
tests_by_id[branch.label] = Test.objects.filter(id=branch.id).update(
label=label,
num_tests=group_num_tests,
mean_duration=group_total_duration,
upper_duration=group_durations[-1],
lower_duration=group_durations[0],
upper90_duration=percentile(group_durations, 90),
data=td_data,
)
Build.objects.filter(id=build.id).update(
num_tests=num_tests,
total_duration=total_duration,
)
return build
