def test_already_notified(self):
org = self.create_organization()
project = self.create_project(organization=org)
release = Release.objects.create(
version='a' * 40,
organization=org,
)
release.add_project(project)
env = Environment.objects.create(
name='production',
organization_id=org.id,
)
deploy = Deploy.objects.create(
release=release,
organization_id=org.id,
environment_id=env.id,
notified=True,
)
Deploy.notify_if_ready(deploy.id)
# make sure no activity has been created
assert not Activity.objects.filter(
type=Activity.DEPLOY,
project=project,
ident=release.version,
).exists()
def test_head_commits_fetch_not_complete(self):
# case where there are not commits, but there are head
# commits, indicating we should wait to notify
org = self.create_organization()
project = self.create_project(organization=org)
release = Release.objects.create(version="a" * 40, organization=org)
release.add_project(project)
ReleaseHeadCommit.objects.create(
release=release,
organization_id=org.id,
repository_id=5,
commit=Commit.objects.create(key="b" * 40,
repository_id=5,
organization_id=org.id),
)
env = Environment.objects.create(name="production",
organization_id=org.id)
deploy = Deploy.objects.create(release=release,
organization_id=org.id,
environment_id=env.id)
Deploy.notify_if_ready(deploy.id)
# make sure activity has been created
assert not Activity.objects.filter(type=Activity.DEPLOY,
project=project,
ident=release.version).exists()
assert Deploy.objects.get(id=deploy.id).notified is False
def test_no_commits_fetch_complete(self):
# case where they've created a deploy and
# we've tried to fetch commits, but there
# weren't any
org = self.create_organization()
project = self.create_project(organization=org)
release = Release.objects.create(version="a" * 40, organization=org)
release.add_project(project)
env = Environment.objects.create(name="production",
organization_id=org.id)
deploy = Deploy.objects.create(release=release,
organization_id=org.id,
environment_id=env.id)
Deploy.notify_if_ready(deploy.id, fetch_complete=True)
# make sure activity has been created
assert Activity.objects.filter(type=Activity.DEPLOY,
project=project,
ident=release.version).exists()
assert (Activity.objects.get(
type=Activity.DEPLOY, project=project,
ident=release.version).data["deploy_id"] == deploy.id)
assert Deploy.objects.get(id=deploy.id).notified is True
def test_already_notified(self):
org = self.create_organization()
project = self.create_project(organization=org)
release = Release.objects.create(
version='a' * 40,
organization=org,
)
release.add_project(project)
env = Environment.objects.create(
name='production',
organization_id=org.id,
)
deploy = Deploy.objects.create(
release=release,
organization_id=org.id,
environment_id=env.id,
notified=True,
)
Deploy.notify_if_ready(deploy.id)
# make sure no activity has been created
assert not Activity.objects.filter(
type=Activity.DEPLOY,
project=project,
ident=release.version,
).exists()
def test_no_commits_no_head_commits(self):
# case where there are not commits, but also no head commit,
# so we shouldn't bother waiting to notify
org = self.create_organization()
project = self.create_project(organization=org)
release = Release.objects.create(
version='a' * 40,
organization=org,
)
release.add_project(project)
env = Environment.objects.create(
name='production',
organization_id=org.id,
)
deploy = Deploy.objects.create(
release=release,
organization_id=org.id,
environment_id=env.id,
)
Deploy.notify_if_ready(deploy.id)
# make sure activity has been created
assert Activity.objects.filter(
type=Activity.DEPLOY,
project=project,
ident=release.version,
).exists()
assert Activity.objects.get(
type=Activity.DEPLOY,
project=project,
ident=release.version,
).data['deploy_id'] == deploy.id
assert Deploy.objects.get(id=deploy.id).notified is True
def test_no_commits_no_head_commits(self):
# case where there are not commits, but also no head commit,
# so we shouldn't bother waiting to notify
org = self.create_organization()
project = self.create_project(organization=org)
release = Release.objects.create(
version='a' * 40,
organization=org,
)
release.add_project(project)
env = Environment.objects.create(
name='production',
organization_id=org.id,
)
deploy = Deploy.objects.create(
release=release,
organization_id=org.id,
environment_id=env.id,
)
Deploy.notify_if_ready(deploy.id)
# make sure activity has been created
assert Activity.objects.filter(
type=Activity.DEPLOY,
project=project,
ident=release.version,
).exists()
assert Activity.objects.get(
type=Activity.DEPLOY,
project=project,
ident=release.version,
).data['deploy_id'] == deploy.id
assert Deploy.objects.get(id=deploy.id).notified is True
def test_notify_if_ready_long_release(self):
org = self.create_organization()
project = self.create_project(organization=org)
release = Release.objects.create(
version='a' * 200,
organization=org,
)
release.add_project(project)
env = Environment.objects.create(
name='production',
organization_id=org.id,
)
deploy = Deploy.objects.create(
release=release,
organization_id=org.id,
environment_id=env.id,
)
Deploy.notify_if_ready(deploy.id)
# make sure activity has been created
record = Activity.objects.get(
type=Activity.DEPLOY,
project=project,
)
assert release.version.startswith(record.ident)
def test_head_commits_fetch_not_complete(self):
# case where there are not commits, but there are head
# commits, indicating we should wait to notify
org = self.create_organization()
project = self.create_project(organization=org)
release = Release.objects.create(
version='a' * 40,
organization=org,
)
release.add_project(project)
ReleaseHeadCommit.objects.create(
release=release,
organization_id=org.id,
repository_id=5,
commit=Commit.objects.create(
key='b' * 40,
repository_id=5,
organization_id=org.id,
)
)
env = Environment.objects.create(
name='production',
organization_id=org.id,
)
deploy = Deploy.objects.create(
release=release,
organization_id=org.id,
environment_id=env.id,
)
Deploy.notify_if_ready(deploy.id)
# make sure activity has been created
assert not Activity.objects.filter(
type=Activity.DEPLOY,
project=project,
ident=release.version,
).exists()
assert Deploy.objects.get(id=deploy.id).notified is False
def test_no_commits_fetch_complete(self):
# case where they've created a deploy and
# we've tried to fetch commits, but there
# weren't any
org = self.create_organization()
project = self.create_project(organization=org)
release = Release.objects.create(
version='a' * 40,
organization=org,
)
release.add_project(project)
env = Environment.objects.create(
name='production',
organization_id=org.id,
)
deploy = Deploy.objects.create(
release=release,
organization_id=org.id,
environment_id=env.id,
)
Deploy.notify_if_ready(deploy.id, fetch_complete=True)
# make sure activity has been created
assert Activity.objects.filter(
type=Activity.DEPLOY,
project=project,
ident=release.version,
).exists()
assert Activity.objects.get(
type=Activity.DEPLOY,
project=project,
ident=release.version,
).data['deploy_id'] == deploy.id
assert Deploy.objects.get(id=deploy.id).notified is True
def fetch_commits(release_id, user_id, refs, prev_release_id=None, **kwargs):
# TODO(dcramer): this function could use some cleanup/refactoring as its a bit unwieldly
commit_list = []
release = Release.objects.get(id=release_id)
user = User.objects.get(id=user_id)
prev_release = None
if prev_release_id is not None:
try:
prev_release = Release.objects.get(id=prev_release_id)
except Release.DoesNotExist:
pass
for ref in refs:
try:
repo = Repository.objects.get(
organization_id=release.organization_id,
name=ref['repository'],
)
except Repository.DoesNotExist:
logger.info(
'repository.missing',
extra={
'organization_id': release.organization_id,
'user_id': user_id,
'repository': ref['repository'],
}
)
continue
binding_key = 'integration-repository.provider' if is_integration_provider(repo.provider) else 'repository.provider'
try:
provider_cls = bindings.get(binding_key).get(repo.provider)
except KeyError:
continue
# if previous commit isn't provided, try to get from
# previous release otherwise, try to get
# recent commits from provider api
start_sha = None
if ref.get('previousCommit'):
start_sha = ref['previousCommit']
elif prev_release:
try:
start_sha = ReleaseHeadCommit.objects.filter(
organization_id=release.organization_id,
release=prev_release,
repository_id=repo.id,
).values_list(
'commit__key', flat=True
)[0]
except IndexError:
pass
end_sha = ref['commit']
provider = provider_cls(id=repo.provider)
try:
if is_integration_provider(provider.id):
repo_commits = provider.compare_commits(repo, start_sha, end_sha)
else:
repo_commits = provider.compare_commits(repo, start_sha, end_sha, actor=user)
except NotImplementedError:
pass
except Exception as exc:
logger.exception(
'fetch_commits.error',
exc_info=True,
extra={
'organization_id': repo.organization_id,
'user_id': user_id,
'repository': repo.name,
'end_sha': end_sha,
'start_sha': start_sha,
}
)
if isinstance(exc, InvalidIdentity) and getattr(exc, 'identity', None):
handle_invalid_identity(identity=exc.identity, commit_failure=True)
elif isinstance(exc, (PluginError, InvalidIdentity)):
msg = generate_fetch_commits_error_email(release, exc.message)
msg.send_async(to=[user.email])
else:
msg = generate_fetch_commits_error_email(
release, 'An internal system error occurred.')
msg.send_async(to=[user.email])
else:
logger.info(
'fetch_commits.complete',
extra={
'organization_id': repo.organization_id,
'user_id': user_id,
'repository': repo.name,
'end_sha': end_sha,
'start_sha': start_sha,
'num_commits': len(repo_commits or []),
}
)
commit_list.extend(repo_commits)
if commit_list:
release.set_commits(commit_list)
deploys = Deploy.objects.filter(
organization_id=release.organization_id,
release=release,
notified=False,
).values_list(
'id', 'environment_id', 'date_finished'
)
# XXX(dcramer): i dont know why this would have multiple environments, but for
# our sanity lets assume it can
pending_notifications = []
last_deploy_per_environment = {}
for deploy_id, environment_id, date_finished in deploys:
last_deploy_per_environment[environment_id] = (deploy_id, date_finished)
pending_notifications.append(deploy_id)
repo_queryset = ReleaseHeadCommit.objects.filter(
organization_id=release.organization_id,
release=release,
).values_list(
'repository_id', 'commit',
)
# we need to mark LatestRelease, but only if there's not a deploy which has completed
# *after* this deploy (given we might process commits out of order)
for repository_id, commit_id in repo_queryset:
for environment_id, (deploy_id, date_finished) in six.iteritems(
last_deploy_per_environment):
if not Deploy.objects.filter(
id__in=LatestRelease.objects.filter(
repository_id=repository_id,
environment_id=environment_id,
).values('deploy_id'),
date_finished__gt=date_finished,
).exists():
LatestRelease.objects.create_or_update(
repository_id=repository_id,
environment_id=environment_id,
values={
'release_id': release.id,
'deploy_id': deploy_id,
'commit_id': commit_id,
}
)
for deploy_id in pending_notifications:
Deploy.notify_if_ready(deploy_id, fetch_complete=True)
def fetch_commits(release_id, user_id, refs, prev_release_id=None, **kwargs):
commit_list = []
release = Release.objects.get(id=release_id)
user = User.objects.get(id=user_id)
prev_release = None
if prev_release_id is not None:
try:
prev_release = Release.objects.get(id=prev_release_id)
except Release.DoesNotExist:
pass
for ref in refs:
try:
repo = Repository.objects.get(
organization_id=release.organization_id,
name=ref['repository'],
)
except Repository.DoesNotExist:
logger.info(
'repository.missing',
extra={
'organization_id': release.organization_id,
'user_id': user_id,
'repository': ref['repository'],
}
)
continue
try:
provider_cls = bindings.get('repository.provider').get(repo.provider)
except KeyError:
continue
# if previous commit isn't provided, try to get from
# previous release otherwise, try to get
# recent commits from provider api
start_sha = None
if ref.get('previousCommit'):
start_sha = ref['previousCommit']
elif prev_release:
try:
start_sha = ReleaseHeadCommit.objects.filter(
organization_id=release.organization_id,
release=prev_release,
repository_id=repo.id,
).values_list(
'commit__key', flat=True
)[0]
except IndexError:
pass
end_sha = ref['commit']
provider = provider_cls(id=repo.provider)
try:
repo_commits = provider.compare_commits(repo, start_sha, end_sha, actor=user)
except NotImplementedError:
pass
except Exception as exc:
logger.exception(
'fetch_commits.error',
exc_info=True,
extra={
'organization_id': repo.organization_id,
'user_id': user_id,
'repository': repo.name,
'end_sha': end_sha,
'start_sha': start_sha,
}
)
if isinstance(exc, InvalidIdentity) and getattr(exc, 'identity', None):
handle_invalid_identity(identity=exc.identity, commit_failure=True)
elif isinstance(exc, (PluginError, InvalidIdentity)):
msg = generate_fetch_commits_error_email(release, exc.message)
msg.send_async(to=[user.email])
else:
msg = generate_fetch_commits_error_email(
release, 'An internal system error occurred.')
msg.send_async(to=[user.email])
else:
logger.info(
'fetch_commits.complete',
extra={
'organization_id': repo.organization_id,
'user_id': user_id,
'repository': repo.name,
'end_sha': end_sha,
'start_sha': start_sha,
'num_commits': len(repo_commits or []),
}
)
commit_list.extend(repo_commits)
if commit_list:
release.set_commits(commit_list)
deploys = Deploy.objects.filter(
organization_id=release.organization_id,
release=release,
notified=False,
).values_list(
'id', flat=True
)
for d_id in deploys:
Deploy.notify_if_ready(d_id, fetch_complete=True)
def post(self, request, organization, version):
"""
Create a Deploy
```````````````
Create a deploy for a given release.
:pparam string organization_slug: the organization short name
:pparam string version: the version identifier of the release.
:param string environment: the environment you're deploying to
:param string name: the optional name of the deploy
:param url url: the optional url that points to the deploy
:param datetime dateStarted: an optional date that indicates when
the deploy started
:param datetime dateFinished: an optional date that indicates when
the deploy ended. If not provided, the
current time is used.
"""
try:
release = Release.objects.get(
version=version,
organization=organization,
)
except Release.DoesNotExist:
raise ResourceDoesNotExist
if not self.has_release_permission(request, organization, release):
raise PermissionDenied
serializer = DeploySerializer(data=request.DATA)
if serializer.is_valid():
projects = list(release.projects.all())
result = serializer.object
env = Environment.objects.get_or_create(
name=result['environment'],
organization_id=organization.id,
)[0]
for project in projects:
env.add_project(project)
try:
with transaction.atomic():
deploy, created = Deploy.objects.create(
organization_id=organization.id,
release=release,
environment_id=env.id,
date_finished=result.get('dateFinished',
timezone.now()),
date_started=result.get('dateStarted'),
name=result.get('name'),
url=result.get('url'),
), True
except IntegrityError:
deploy, created = Deploy.objects.get(
organization_id=organization.id,
release=release,
environment_id=env.id,
), False
deploy.update(
date_finished=result.get('dateFinished', timezone.now()),
date_started=result.get('dateStarted'),
)
Deploy.notify_if_ready(deploy.id)
# This is the closest status code that makes sense, and we want
# a unique 2xx response code so people can understand when
# behavior differs.
# 208 Already Reported (WebDAV; RFC 5842)
status = 201 if created else 208
return Response(serialize(deploy, request.user), status=status)
return Response(serializer.errors, status=400)
def post(self, request, organization, version):
"""
Create a Deploy
```````````````
Create a deploy for a given release.
:pparam string organization_slug: the organization short name
:pparam string version: the version identifier of the release.
:param string environment: the environment you're deploying to
:param string name: the optional name of the deploy
:param url url: the optional url that points to the deploy
:param datetime dateStarted: an optional date that indicates when
the deploy started
:param datetime dateFinished: an optional date that indicates when
the deploy ended. If not provided, the
current time is used.
"""
try:
release = Release.objects.get(
version=version,
organization=organization,
)
except Release.DoesNotExist:
raise ResourceDoesNotExist
if not self.has_release_permission(request, organization, release):
raise PermissionDenied
serializer = DeploySerializer(data=request.DATA)
if serializer.is_valid():
projects = list(release.projects.all())
result = serializer.object
env = Environment.objects.get_or_create(
name=result['environment'],
organization_id=organization.id,
)[0]
for project in projects:
env.add_project(project)
deploy = Deploy.objects.create(
organization_id=organization.id,
release=release,
environment_id=env.id,
date_finished=result.get('dateFinished', timezone.now()),
date_started=result.get('dateStarted'),
name=result.get('name'),
url=result.get('url'),
)
deploy_created.send_robust(deploy=deploy, sender=self.__class__)
# XXX(dcramer): this has a race for most recent deploy, but
# should be unlikely to hit in the real world
Release.objects.filter(id=release.id).update(
total_deploys=F('total_deploys') + 1,
last_deploy_id=deploy.id,
)
for project in projects:
ReleaseProjectEnvironment.objects.create_or_update(
release=release,
environment=env,
project=project,
values={
'last_deploy_id': deploy.id,
}
)
Deploy.notify_if_ready(deploy.id)
return Response(serialize(deploy, request.user), status=201)
return Response(serializer.errors, status=400)
def post(self, request, organization, version):
"""
Create a Deploy
```````````````
Create a deploy for a given release.
:pparam string organization_slug: the organization short name
:pparam string version: the version identifier of the release.
:param string environment: the environment you're deploying to
:param string name: the optional name of the deploy
:param url url: the optional url that points to the deploy
:param datetime dateStarted: an optional date that indicates when
the deploy started
:param datetime dateFinished: an optional date that indicates when
the deploy ended. If not provided, the
current time is used.
"""
try:
release = Release.objects.get(
version=version,
organization=organization,
)
except Release.DoesNotExist:
raise ResourceDoesNotExist
if not self.has_release_permission(request, organization, release):
raise PermissionDenied
serializer = DeploySerializer(data=request.DATA)
if serializer.is_valid():
projects = list(release.projects.all())
result = serializer.object
env = Environment.objects.get_or_create(
name=result['environment'],
organization_id=organization.id,
)[0]
for project in projects:
env.add_project(project)
try:
with transaction.atomic():
deploy, created = Deploy.objects.create(
organization_id=organization.id,
release=release,
environment_id=env.id,
date_finished=result.get('dateFinished', timezone.now()),
date_started=result.get('dateStarted'),
name=result.get('name'),
url=result.get('url'),
), True
except IntegrityError:
deploy, created = Deploy.objects.get(
organization_id=organization.id,
release=release,
environment_id=env.id,
), False
deploy.update(
date_finished=result.get('dateFinished', timezone.now()),
date_started=result.get('dateStarted'),
)
# XXX(dcramer): this has a race for most recent deploy, but
# should be unlikely to hit in the real world
Release.objects.filter(id=release.id).update(
total_deploys=F('total_deploys') + 1,
last_deploy_id=deploy.id,
)
Deploy.notify_if_ready(deploy.id)
# This is the closest status code that makes sense, and we want
# a unique 2xx response code so people can understand when
# behavior differs.
# 208 Already Reported (WebDAV; RFC 5842)
status = 201 if created else 208
return Response(serialize(deploy, request.user), status=status)
return Response(serializer.errors, status=400)
def fetch_commits(release_id, user_id, refs, prev_release_id=None, **kwargs):
# TODO(dcramer): this function could use some cleanup/refactoring as its a bit unwieldly
commit_list = []
release = Release.objects.get(id=release_id)
user = User.objects.get(id=user_id)
prev_release = None
if prev_release_id is not None:
try:
prev_release = Release.objects.get(id=prev_release_id)
except Release.DoesNotExist:
pass
for ref in refs:
try:
repo = Repository.objects.get(
organization_id=release.organization_id,
name=ref['repository'],
)
except Repository.DoesNotExist:
logger.info('repository.missing',
extra={
'organization_id': release.organization_id,
'user_id': user_id,
'repository': ref['repository'],
})
continue
binding_key = 'integration-repository.provider' if repo.provider and repo.provider.startswith(
'integrations:') else 'repository.provider'
try:
provider_cls = bindings.get(binding_key).get(repo.provider)
except KeyError:
continue
# if previous commit isn't provided, try to get from
# previous release otherwise, try to get
# recent commits from provider api
start_sha = None
if ref.get('previousCommit'):
start_sha = ref['previousCommit']
elif prev_release:
try:
start_sha = ReleaseHeadCommit.objects.filter(
organization_id=release.organization_id,
release=prev_release,
repository_id=repo.id,
).values_list('commit__key', flat=True)[0]
except IndexError:
pass
end_sha = ref['commit']
provider = provider_cls(id=repo.provider)
try:
repo_commits = provider.compare_commits(repo,
start_sha,
end_sha,
actor=user)
except NotImplementedError:
pass
except Exception as exc:
logger.exception('fetch_commits.error',
exc_info=True,
extra={
'organization_id': repo.organization_id,
'user_id': user_id,
'repository': repo.name,
'end_sha': end_sha,
'start_sha': start_sha,
})
if isinstance(exc, InvalidIdentity) and getattr(
exc, 'identity', None):
handle_invalid_identity(identity=exc.identity,
commit_failure=True)
elif isinstance(exc, (PluginError, InvalidIdentity)):
msg = generate_fetch_commits_error_email(release, exc.message)
msg.send_async(to=[user.email])
else:
msg = generate_fetch_commits_error_email(
release, 'An internal system error occurred.')
msg.send_async(to=[user.email])
else:
logger.info('fetch_commits.complete',
extra={
'organization_id': repo.organization_id,
'user_id': user_id,
'repository': repo.name,
'end_sha': end_sha,
'start_sha': start_sha,
'num_commits': len(repo_commits or []),
})
commit_list.extend(repo_commits)
if commit_list:
release.set_commits(commit_list)
deploys = Deploy.objects.filter(
organization_id=release.organization_id,
release=release,
notified=False,
).values_list('id', 'environment_id', 'date_finished')
# XXX(dcramer): i dont know why this would have multiple environments, but for
# our sanity lets assume it can
pending_notifications = []
last_deploy_per_environment = {}
for deploy_id, environment_id, date_finished in deploys:
last_deploy_per_environment[environment_id] = (deploy_id,
date_finished)
pending_notifications.append(deploy_id)
repo_queryset = ReleaseHeadCommit.objects.filter(
organization_id=release.organization_id,
release=release,
).values_list(
'repository_id',
'commit',
)
# we need to mark LatestRelease, but only if there's not a deploy which has completed
# *after* this deploy (given we might process commits out of order)
for repository_id, commit_id in repo_queryset:
for environment_id, (deploy_id, date_finished) in six.iteritems(
last_deploy_per_environment):
if not Deploy.objects.filter(
id__in=LatestRelease.objects.filter(
repository_id=repository_id,
environment_id=environment_id,
).values('deploy_id'),
date_finished__gt=date_finished,
).exists():
LatestRelease.objects.create_or_update(
repository_id=repository_id,
environment_id=environment_id,
values={
'release_id': release.id,
'deploy_id': deploy_id,
'commit_id': commit_id,
})
for deploy_id in pending_notifications:
Deploy.notify_if_ready(deploy_id, fetch_complete=True)
def fetch_commits(release_id, user_id, refs, prev_release_id=None, **kwargs):
commit_list = []
release = Release.objects.get(id=release_id)
user = User.objects.get(id=user_id)
prev_release = None
if prev_release_id is not None:
try:
prev_release = Release.objects.get(id=prev_release_id)
except Release.DoesNotExist:
pass
for ref in refs:
try:
repo = Repository.objects.get(
organization_id=release.organization_id,
name=ref['repository'],
)
except Repository.DoesNotExist:
logger.info('repository.missing',
extra={
'organization_id': release.organization_id,
'user_id': user_id,
'repository': ref['repository'],
})
continue
try:
provider_cls = bindings.get('repository.provider').get(
repo.provider)
except KeyError:
continue
# if previous commit isn't provided, try to get from
# previous release otherwise, try to get
# recent commits from provider api
start_sha = None
if ref.get('previousCommit'):
start_sha = ref['previousCommit']
elif prev_release:
try:
start_sha = ReleaseHeadCommit.objects.filter(
organization_id=release.organization_id,
release=prev_release,
repository_id=repo.id,
).values_list('commit__key', flat=True)[0]
except IndexError:
pass
end_sha = ref['commit']
provider = provider_cls(id=repo.provider)
try:
repo_commits = provider.compare_commits(repo,
start_sha,
end_sha,
actor=user)
except NotImplementedError:
pass
except Exception as exc:
logger.exception('fetch_commits.error',
exc_info=True,
extra={
'organization_id': repo.organization_id,
'user_id': user_id,
'repository': repo.name,
'end_sha': end_sha,
'start_sha': start_sha,
})
if isinstance(exc, InvalidIdentity) and getattr(
exc, 'identity', None):
handle_invalid_identity(identity=exc.identity,
commit_failure=True)
elif isinstance(exc, (PluginError, InvalidIdentity)):
msg = generate_fetch_commits_error_email(release, exc.message)
msg.send_async(to=[user.email])
else:
msg = generate_fetch_commits_error_email(
release, 'An internal system error occurred.')
msg.send_async(to=[user.email])
else:
logger.info('fetch_commits.complete',
extra={
'organization_id': repo.organization_id,
'user_id': user_id,
'repository': repo.name,
'end_sha': end_sha,
'start_sha': start_sha,
'num_commits': len(repo_commits or []),
})
commit_list.extend(repo_commits)
if commit_list:
release.set_commits(commit_list)
deploys = Deploy.objects.filter(
organization_id=release.organization_id,
release=release,
notified=False,
).values_list('id', flat=True)
for d_id in deploys:
Deploy.notify_if_ready(d_id, fetch_complete=True)
def get(self, request):
org = Organization(id=1, slug="organization", name="My Company")
projects = [
Project(id=1, organization=org, slug="project", name="My Project"),
Project(id=2,
organization=org,
slug="another-project",
name="Another Project"),
Project(id=3,
organization=org,
slug="yet-another-project",
name="Yet Another Project"),
]
release = Release(
organization_id=org.id,
version="6c998f755f304593a4713abd123eaf8833a2de5e",
date_added=datetime.datetime(2016, 10, 12, 15, 39,
tzinfo=pytz.utc),
)
deploy = Deploy(
release=release,
organization_id=org.id,
environment_id=1,
date_finished=datetime.datetime(2016,
10,
12,
15,
39,
tzinfo=pytz.utc),
)
release_links = [
absolute_uri("/organizations/{}/releases/{}/?project={}".format(
org.slug, release.version, p.id)) for p in projects
]
repos = [
{
"name":
"getsentry/getsentry",
"commits": [
(
Commit(
key="48b86fcd677da3dba5679d7a738240ce6fb74b20",
date_added=datetime.datetime(2016,
10,
11,
15,
39,
tzinfo=pytz.utc),
),
None,
),
(
Commit(
key="a53a2756bb8d111b43196210b34df90b87ed336b",
message="Fix billing",
author=CommitAuthor(name="David Cramer",
email="*****@*****.**"),
date_added=datetime.datetime(2016,
10,
11,
16,
45,
tzinfo=pytz.utc),
),
User(email="*****@*****.**", name="David Cramer"),
),
],
},
{
"name":
"getsentry/sentry",
"commits": [
(
Commit(
key="3c8eb3b4af6ee2a29c68daa188fc730c8e4b39fd",
date_added=datetime.datetime(2016,
10,
10,
15,
39,
tzinfo=pytz.utc),
),
None,
),
(
Commit(
key="373562702009df1692da6eb80a933139f29e094b",
message="Fix padding",
author=CommitAuthor(name="Chris Jennings",
email="*****@*****.**"),
date_added=datetime.datetime(2016,
10,
10,
16,
39,
tzinfo=pytz.utc),
),
None,
),
(
Commit(
key="631cd9096bd9811a046a472bb0aa8b573e86e1f1",
message="Update README.rst",
author=CommitAuthor(name="David Cramer",
email="*****@*****.**"),
date_added=datetime.datetime(2016,
10,
11,
10,
39,
tzinfo=pytz.utc),
),
User(email="*****@*****.**", name="David Cramer"),
),
],
},
]
return MailPreview(
html_template="sentry/emails/activity/release.html",
text_template="sentry/emails/activity/release.txt",
context={
"release":
release,
"projects":
zip(projects, release_links, [6, 1, 0]),
"repos":
repos,
"reason":
GroupSubscriptionReason.descriptions[
GroupSubscriptionReason.committed],
"project_count":
len(projects),
"commit_count":
4,
"author_count":
1,
"file_count":
5,
"environment":
"production",
"deploy":
deploy,
"setup_repo_link":
absolute_uri("/organizations/{}/repos/".format(org.slug)),
},
).render(request)
def fetch_commits(release_id, user_id, refs, prev_release_id=None, **kwargs):
# TODO(dcramer): this function could use some cleanup/refactoring as it's a bit unwieldy
commit_list = []
release = Release.objects.get(id=release_id)
user = User.objects.get(id=user_id)
prev_release = None
if prev_release_id is not None:
try:
prev_release = Release.objects.get(id=prev_release_id)
except Release.DoesNotExist:
pass
for ref in refs:
try:
repo = Repository.objects.get(
organization_id=release.organization_id,
name=ref["repository"])
except Repository.DoesNotExist:
logger.info(
"repository.missing",
extra={
"organization_id": release.organization_id,
"user_id": user_id,
"repository": ref["repository"],
},
)
continue
binding_key = ("integration-repository.provider"
if is_integration_provider(repo.provider) else
"repository.provider")
try:
provider_cls = bindings.get(binding_key).get(repo.provider)
except KeyError:
continue
# if previous commit isn't provided, try to get from
# previous release otherwise, try to get
# recent commits from provider api
start_sha = None
if ref.get("previousCommit"):
start_sha = ref["previousCommit"]
elif prev_release:
try:
start_sha = ReleaseHeadCommit.objects.filter(
organization_id=release.organization_id,
release=prev_release,
repository_id=repo.id,
).values_list("commit__key", flat=True)[0]
except IndexError:
pass
end_sha = ref["commit"]
provider = provider_cls(id=repo.provider)
try:
if is_integration_provider(provider.id):
repo_commits = provider.compare_commits(
repo, start_sha, end_sha)
else:
repo_commits = provider.compare_commits(repo,
start_sha,
end_sha,
actor=user)
except NotImplementedError:
pass
except Exception as e:
logger.info(
"fetch_commits.error",
extra={
"organization_id": repo.organization_id,
"user_id": user_id,
"repository": repo.name,
"provider": provider.id,
"error": str(e),
"end_sha": end_sha,
"start_sha": start_sha,
},
)
if isinstance(e, InvalidIdentity) and getattr(e, "identity", None):
handle_invalid_identity(identity=e.identity,
commit_failure=True)
elif isinstance(e,
(PluginError, InvalidIdentity, IntegrationError)):
msg = generate_fetch_commits_error_email(release, repo, str(e))
emails = get_emails_for_user_or_org(user,
release.organization_id)
msg.send_async(to=emails)
else:
msg = generate_fetch_commits_error_email(
release, repo, "An internal system error occurred.")
emails = get_emails_for_user_or_org(user,
release.organization_id)
msg.send_async(to=emails)
else:
logger.info(
"fetch_commits.complete",
extra={
"organization_id": repo.organization_id,
"user_id": user_id,
"repository": repo.name,
"end_sha": end_sha,
"start_sha": start_sha,
"num_commits": len(repo_commits or []),
},
)
commit_list.extend(repo_commits)
if commit_list:
try:
release.set_commits(commit_list)
except ReleaseCommitError:
# Another task or webworker is currently setting commits on this
# release. Return early as that task will do the remaining work.
logger.info(
"fetch_commits.duplicate",
extra={
"release_id": release.id,
"organization_id": release.organization_id,
"user_id": user_id,
},
)
return
deploys = Deploy.objects.filter(
organization_id=release.organization_id,
release=release,
notified=False).values_list("id", "environment_id",
"date_finished")
# XXX(dcramer): i dont know why this would have multiple environments, but for
# our sanity lets assume it can
pending_notifications = []
last_deploy_per_environment = {}
for deploy_id, environment_id, date_finished in deploys:
last_deploy_per_environment[environment_id] = (deploy_id,
date_finished)
pending_notifications.append(deploy_id)
repo_queryset = ReleaseHeadCommit.objects.filter(
organization_id=release.organization_id,
release=release).values_list("repository_id", "commit")
# for each repo, update (or create if this is the first one) our records
# of the latest commit-associated release in each env
# use deploys as a proxy for ReleaseEnvironment, because they contain
# a timestamp in addition to release and env data
for repository_id, commit_id in repo_queryset:
for environment_id, (
deploy_id,
date_finished) in last_deploy_per_environment.items():
# we need to mark LatestRepoReleaseEnvironment, but only if there's not a
# deploy in the given environment which has completed *after*
# this deploy (given we might process commits out of order)
if not Deploy.objects.filter(
id__in=LatestRepoReleaseEnvironment.objects.filter(
repository_id=repository_id,
environment_id=environment_id).values("deploy_id"),
date_finished__gt=date_finished,
).exists():
LatestRepoReleaseEnvironment.objects.create_or_update(
repository_id=repository_id,
environment_id=environment_id,
values={
"release_id": release.id,
"deploy_id": deploy_id,
"commit_id": commit_id,
},
)
for deploy_id in pending_notifications:
Deploy.notify_if_ready(deploy_id, fetch_complete=True)
def get(self, request):
org = Organization(
id=1,
slug='organization',
name='My Company',
)
projects = [
Project(
id=1,
organization=org,
slug='project',
name='My Project',
),
Project(
id=2,
organization=org,
slug='another-project',
name='Another Project',
),
Project(
id=3,
organization=org,
slug='yet-another-project',
name='Yet Another Project',
),
]
release = Release(organization_id=org.id,
version='6c998f755f304593a4713abd123eaf8833a2de5e',
date_added=datetime.datetime(2016,
10,
12,
15,
39,
tzinfo=pytz.utc))
deploy = Deploy(
release=release,
organization_id=org.id,
environment_id=1,
date_finished=datetime.datetime(2016,
10,
12,
15,
39,
tzinfo=pytz.utc),
)
has_new_links = features.has('organizations:sentry10', org)
if has_new_links:
release_links = [
absolute_uri(
u'/organizations/{}/releases/{}/?project={}'.format(
org.slug,
release.version,
p.id,
)) for p in projects
]
else:
release_links = [
absolute_uri(u'/{}/{}/releases/{}/'.format(
org.slug,
p.slug,
release.version,
)) for p in projects
]
repos = [{
'name':
'getsentry/getsentry',
'commits': [
(Commit(
key='48b86fcd677da3dba5679d7a738240ce6fb74b20',
date_added=datetime.datetime(2016,
10,
11,
15,
39,
tzinfo=pytz.utc),
), None),
(Commit(
key='a53a2756bb8d111b43196210b34df90b87ed336b',
message='Fix billing',
author=CommitAuthor(
name='David Cramer',
email='*****@*****.**',
),
date_added=datetime.datetime(2016,
10,
11,
16,
45,
tzinfo=pytz.utc),
), User(email='*****@*****.**', name='David Cramer')),
],
}, {
'name':
'getsentry/sentry',
'commits': [
(Commit(
key='3c8eb3b4af6ee2a29c68daa188fc730c8e4b39fd',
date_added=datetime.datetime(2016,
10,
10,
15,
39,
tzinfo=pytz.utc),
), None),
(Commit(
key='373562702009df1692da6eb80a933139f29e094b',
message='Fix padding',
author=CommitAuthor(
name='Chris Jennings',
email='*****@*****.**',
),
date_added=datetime.datetime(2016,
10,
10,
16,
39,
tzinfo=pytz.utc),
), None),
(Commit(
key='631cd9096bd9811a046a472bb0aa8b573e86e1f1',
message='Update README.rst',
author=CommitAuthor(
name='David Cramer',
email='*****@*****.**',
),
date_added=datetime.datetime(2016,
10,
11,
10,
39,
tzinfo=pytz.utc),
), User(email='*****@*****.**', name='David Cramer')),
],
}]
return MailPreview(
html_template='sentry/emails/activity/release.html',
text_template='sentry/emails/activity/release.txt',
context={
'release':
release,
'projects':
zip(projects, release_links, [6, 1, 0]),
'repos':
repos,
'reason':
GroupSubscriptionReason.descriptions[
GroupSubscriptionReason.committed],
'project_count':
len(projects),
'commit_count':
4,
'author_count':
1,
'file_count':
5,
'environment':
'production',
'deploy':
deploy,
'setup_repo_link':
absolute_uri('/organizations/{}/repos/'.format(org.slug, )),
},
).render(request)
def post(self, request, organization, version):
"""
Create a Deploy
```````````````
Create a deploy for a given release.
:pparam string organization_slug: the organization short name
:pparam string version: the version identifier of the release.
:param string environment: the environment you're deploying to
:param string name: the optional name of the deploy
:param url url: the optional url that points to the deploy
:param datetime dateStarted: an optional date that indicates when
the deploy started
:param datetime dateFinished: an optional date that indicates when
the deploy ended. If not provided, the
current time is used.
"""
try:
release = Release.objects.get(
version=version,
organization=organization,
)
except Release.DoesNotExist:
raise ResourceDoesNotExist
if not self.has_release_permission(request, organization, release):
raise PermissionDenied
serializer = DeploySerializer(data=request.DATA)
if serializer.is_valid():
projects = list(release.projects.all())
result = serializer.object
env = Environment.objects.get_or_create(
name=result['environment'],
organization_id=organization.id,
)[0]
for project in projects:
env.add_project(project)
deploy = Deploy.objects.create(
organization_id=organization.id,
release=release,
environment_id=env.id,
date_finished=result.get('dateFinished', timezone.now()),
date_started=result.get('dateStarted'),
name=result.get('name'),
url=result.get('url'),
)
# XXX(dcramer): this has a race for most recent deploy, but
# should be unlikely to hit in the real world
Release.objects.filter(id=release.id).update(
total_deploys=F('total_deploys') + 1,
last_deploy_id=deploy.id,
)
ReleaseProjectEnvironment.objects.filter(
release=release,
environment=env,
project__in=projects,
).update(last_deploy_id=deploy.id, )
Deploy.notify_if_ready(deploy.id)
return Response(serialize(deploy, request.user), status=201)
return Response(serializer.errors, status=400)
def fetch_commits(release_id, user_id, refs, prev_release_id=None, **kwargs):
# TODO(dcramer): this function could use some cleanup/refactoring as it's a bit unwieldy
commit_list = []
release = Release.objects.get(id=release_id)
user = User.objects.get(id=user_id)
prev_release = None
if prev_release_id is not None:
try:
prev_release = Release.objects.get(id=prev_release_id)
except Release.DoesNotExist:
pass
for ref in refs:
try:
repo = Repository.objects.get(
organization_id=release.organization_id,
name=ref["repository"])
except Repository.DoesNotExist:
logger.info(
"repository.missing",
extra={
"organization_id": release.organization_id,
"user_id": user_id,
"repository": ref["repository"],
},
)
continue
binding_key = ("integration-repository.provider"
if is_integration_provider(repo.provider) else
"repository.provider")
try:
provider_cls = bindings.get(binding_key).get(repo.provider)
except KeyError:
continue
# if previous commit isn't provided, try to get from
# previous release otherwise, try to get
# recent commits from provider api
start_sha = None
if ref.get("previousCommit"):
start_sha = ref["previousCommit"]
elif prev_release:
try:
start_sha = ReleaseHeadCommit.objects.filter(
organization_id=release.organization_id,
release=prev_release,
repository_id=repo.id,
).values_list("commit__key", flat=True)[0]
except IndexError:
pass
end_sha = ref["commit"]
provider = provider_cls(id=repo.provider)
try:
if is_integration_provider(provider.id):
repo_commits = provider.compare_commits(
repo, start_sha, end_sha)
else:
repo_commits = provider.compare_commits(repo,
start_sha,
end_sha,
actor=user)
except NotImplementedError:
pass
except Exception as exc:
logger.info(
"fetch_commits.error",
extra={
"organization_id": repo.organization_id,
"user_id": user_id,
"repository": repo.name,
"provider": provider.id,
"error": six.text_type(exc),
"end_sha": end_sha,
"start_sha": start_sha,
},
)
if isinstance(exc, InvalidIdentity) and getattr(
exc, "identity", None):
handle_invalid_identity(identity=exc.identity,
commit_failure=True)
elif isinstance(exc,
(PluginError, InvalidIdentity, IntegrationError)):
msg = generate_fetch_commits_error_email(release, exc.message)
msg.send_async(to=[user.email])
else:
msg = generate_fetch_commits_error_email(
release, "An internal system error occurred.")
msg.send_async(to=[user.email])
else:
logger.info(
"fetch_commits.complete",
extra={
"organization_id": repo.organization_id,
"user_id": user_id,
"repository": repo.name,
"end_sha": end_sha,
"start_sha": start_sha,
"num_commits": len(repo_commits or []),
},
)
commit_list.extend(repo_commits)
if commit_list:
release.set_commits(commit_list)
deploys = Deploy.objects.filter(
organization_id=release.organization_id,
release=release,
notified=False).values_list("id", "environment_id",
"date_finished")
# XXX(dcramer): i dont know why this would have multiple environments, but for
# our sanity lets assume it can
pending_notifications = []
last_deploy_per_environment = {}
for deploy_id, environment_id, date_finished in deploys:
last_deploy_per_environment[environment_id] = (deploy_id,
date_finished)
pending_notifications.append(deploy_id)
repo_queryset = ReleaseHeadCommit.objects.filter(
organization_id=release.organization_id,
release=release).values_list("repository_id", "commit")
# we need to mark LatestRelease, but only if there's not a deploy which has completed
# *after* this deploy (given we might process commits out of order)
for repository_id, commit_id in repo_queryset:
for environment_id, (deploy_id, date_finished) in six.iteritems(
last_deploy_per_environment):
if not Deploy.objects.filter(
id__in=LatestRelease.objects.filter(
repository_id=repository_id,
environment_id=environment_id).values("deploy_id"),
date_finished__gt=date_finished,
).exists():
LatestRelease.objects.create_or_update(
repository_id=repository_id,
environment_id=environment_id,
values={
"release_id": release.id,
"deploy_id": deploy_id,
"commit_id": commit_id,
},
)
for deploy_id in pending_notifications:
Deploy.notify_if_ready(deploy_id, fetch_complete=True)
def fetch_commits(release_id, user_id, refs, prev_release_id=None, **kwargs):
commit_list = []
release = Release.objects.get(id=release_id)
user = User.objects.get(id=user_id)
prev_release = None
if prev_release_id is not None:
try:
prev_release = Release.objects.get(id=prev_release_id)
except Release.DoesNotExist:
pass
for ref in refs:
try:
repo = Repository.objects.get(
organization_id=release.organization_id,
name=ref['repository'],
)
except Repository.DoesNotExist:
continue
try:
provider_cls = bindings.get('repository.provider').get(
repo.provider)
except KeyError:
continue
# if previous commit isn't provided, try to get from
# previous release otherwise, try to get
# recent commits from provider api
start_sha = None
if ref.get('previousCommit'):
start_sha = ref['previousCommit']
elif prev_release:
try:
start_sha = ReleaseHeadCommit.objects.filter(
organization_id=release.organization_id,
release=prev_release,
repository_id=repo.id,
).values_list('commit__key', flat=True)[0]
except IndexError:
pass
end_sha = ref['commit']
provider = provider_cls(id=repo.provider)
try:
repo_commits = provider.compare_commits(repo,
start_sha,
end_sha,
actor=user)
except NotImplementedError:
pass
except (PluginError, InvalidIdentity) as e:
logger.exception(six.text_type(e))
else:
commit_list.extend(repo_commits)
if commit_list:
release.set_commits(commit_list)
deploys = Deploy.objects.filter(
organization_id=release.organization_id,
release=release,
notified=False,
).values_list('id', flat=True)
for d_id in deploys:
Deploy.notify_if_ready(d_id, fetch_complete=True)
