def send_webhooks(installation, event, **kwargs):
try:
servicehook = ServiceHook.objects.get(
organization_id=installation.organization_id,
actor_id=installation.id)
except ServiceHook.DoesNotExist:
return
if event not in servicehook.events:
return
# The service hook applies to all projects if there are no
# ServiceHookProject records. Otherwise we want check if
# the event is within the allowed projects.
project_limited = ServiceHookProject.objects.filter(
service_hook_id=servicehook.id).exists()
if not project_limited:
resource, action = event.split(".")
kwargs["resource"] = resource
kwargs["action"] = action
kwargs["install"] = installation
request_data = AppPlatformEvent(**kwargs)
safe_urlopen(
url=servicehook.sentry_app.webhook_url,
data=request_data.body,
headers=request_data.headers,
timeout=5,
)
def process_service_hook(servicehook_id, event, **kwargs):
from sentry.models import ServiceHook
try:
servicehook = ServiceHook.objects.get(id=servicehook_id)
except ServiceHook.DoesNotExist:
return
if servicehook.version == 0:
payload = get_payload_v0(event)
else:
raise NotImplementedError
body = json.dumps(payload)
headers = {
'Content-Type': 'application/json',
'X-ServiceHook-Timestamp': int(time()),
'X-ServiceHook-GUID': servicehook.guid,
'X-ServiceHook-Signature': servicehook.build_signature(body),
}
safe_urlopen(
url=servicehook.url,
body=body,
headers=headers,
timeout=5,
verify_ssl=False,
)
def process_service_hook(servicehook_id, event, **kwargs):
from sentry import tsdb
from sentry.models import ServiceHook
try:
servicehook = ServiceHook.objects.get(id=servicehook_id)
except ServiceHook.DoesNotExist:
return
tsdb.incr(tsdb.models.servicehook_fired, servicehook.id)
if servicehook.version == 0:
payload = get_payload_v0(event)
else:
raise NotImplementedError
body = json.dumps(payload)
headers = {
'Content-Type': 'application/json',
'X-ServiceHook-Timestamp': six.text_type(int(time())),
'X-ServiceHook-GUID': servicehook.guid,
'X-ServiceHook-Signature': servicehook.build_signature(body),
}
safe_urlopen(
url=servicehook.url,
data=body,
headers=headers,
timeout=5,
verify_ssl=False,
)
def process_service_hook(servicehook_id, event, **kwargs):
try:
servicehook = ServiceHook.objects.get(id=servicehook_id)
except ServiceHook.DoesNotExist:
return
if servicehook.version == 0:
payload = get_payload_v0(event)
else:
raise NotImplementedError
from sentry import tsdb
tsdb.incr(tsdb.models.servicehook_fired, servicehook.id)
headers = {
"Content-Type": "application/json",
"X-ServiceHook-Timestamp": str(int(time())),
"X-ServiceHook-GUID": servicehook.guid,
"X-ServiceHook-Signature": servicehook.build_signature(json.dumps(payload)),
}
safe_urlopen(
url=servicehook.url, data=json.dumps(payload), headers=headers, timeout=5, verify_ssl=False
)
def send_webhooks(installation, event, **kwargs):
project_ids = Project.objects.filter(
organization_id=installation.organization_id, ).values_list('id',
flat=True)
servicehooks = ServiceHook.objects.filter(project_id__in=project_ids, )
for servicehook in filter(lambda s: event in s.events, servicehooks):
if not servicehook.created_by_sentry_app:
continue
if servicehook.sentry_app != installation.sentry_app:
continue
resource, action = event.split('.')
kwargs['resource'] = resource
kwargs['action'] = action
kwargs['install'] = installation
request_data = AppPlatformEvent(**kwargs)
safe_urlopen(
url=servicehook.sentry_app.webhook_url,
data=request_data.body,
headers=request_data.headers,
timeout=5,
)
def process_resource_change(action, sender, instance_id, *args, **kwargs):
model = None
name = None
# Previous method signature.
if inspect.isclass(sender):
model = sender
else:
model = TYPES[sender]
name = RESOURCE_RENAMES.get(model.__name__, model.__name__.lower())
# We may run into a race condition where this task executes before the
# transaction that creates the Group has committed.
try:
instance = model.objects.get(id=instance_id)
except model.DoesNotExist as e:
# Explicitly requeue the task, so we don't report this to Sentry until
# we hit the max number of retries.
return current.retry(exc=e)
event = '{}.{}'.format(name, action)
if event not in ALLOWED_EVENTS:
return
project = None
if isinstance(instance, Group):
project = instance.project
if not project:
return
servicehooks = ServiceHook.objects.filter(
project_id=project.id,
)
for servicehook in filter(lambda s: event in s.events, servicehooks):
# For now, these ``post_save`` callbacks are only valid for service
# hooks created by a Sentry App.
if not servicehook.created_by_sentry_app:
continue
request_data = AppPlatformEvent(
resource=name,
action=action,
install=SentryAppInstallation.objects.get(id=servicehook.actor_id),
data=serialize(instance),
)
safe_urlopen(
url=servicehook.url,
data=request_data.body,
headers=request_data.headers,
timeout=5,
)
def process_resource_change(action, sender, instance_id, *args, **kwargs):
model = None
name = None
# Previous method signature.
if inspect.isclass(sender):
model = sender
else:
model = TYPES[sender]
name = RESOURCE_RENAMES.get(model.__name__, model.__name__.lower())
# We may run into a race condition where this task executes before the
# transaction that creates the Group has committed.
try:
instance = model.objects.get(id=instance_id)
except model.DoesNotExist as e:
# Explicitly requeue the task, so we don't report this to Sentry until
# we hit the max number of retries.
return current.retry(exc=e)
event = '{}.{}'.format(name, action)
if event not in ALLOWED_EVENTS:
return
project = None
if isinstance(instance, Group):
project = instance.project
if not project:
return
servicehooks = ServiceHook.objects.filter(project_id=project.id, )
for servicehook in filter(lambda s: event in s.events, servicehooks):
# For now, these ``post_save`` callbacks are only valid for service
# hooks created by a Sentry App.
if not servicehook.created_by_sentry_app:
continue
request_data = AppPlatformEvent(
resource=name,
action=action,
install=SentryAppInstallation.objects.get(id=servicehook.actor_id),
data=serialize(instance),
)
safe_urlopen(
url=servicehook.url,
data=request_data.body,
headers=request_data.headers,
timeout=5,
)
def notify_users(self, group, event, **kwargs):
project = group.project
body = 'Sentry [{0}] {1}: {2}'.format(
project.name.encode('utf-8'),
event.group.get_level_display().upper().encode('utf-8'),
event.title.encode('utf-8').splitlines()[0]
)
body = body[:MAX_SMS_LENGTH]
account_sid = self.get_option('account_sid', project)
auth_token = self.get_option('auth_token', project)
sms_from = clean_phone(self.get_option('sms_from', project))
endpoint = twilio_sms_endpoint.format(account_sid)
sms_to = self.get_option('sms_to', project)
if not sms_to:
return
sms_to = split_sms_to(sms_to)
headers = {
'Authorization': basic_auth(account_sid, auth_token),
}
errors = []
for phone in sms_to:
if not phone:
continue
try:
phone = clean_phone(phone)
http.safe_urlopen(
endpoint,
method='POST',
headers=headers,
data={
'From': sms_from,
'To': phone,
'Body': body,
},
).raise_for_status()
except Exception as e:
errors.append(e)
if errors:
if len(errors) == 1:
raise errors[0]
# TODO: multi-exception
raise Exception(errors)
def notify_users(self, group, event, **kwargs):
project = group.project
body = 'Sentry [{0}] {1}: {2}'.format(
project.name.encode('utf-8'),
event.get_level_display().upper().encode('utf-8'),
event.error().encode('utf-8').splitlines()[0]
)
body = body[:MAX_SMS_LENGTH]
account_sid = self.get_option('account_sid', project)
auth_token = self.get_option('auth_token', project)
sms_from = clean_phone(self.get_option('sms_from', project))
endpoint = twilio_sms_endpoint.format(account_sid)
sms_to = self.get_option('sms_to', project)
if not sms_to:
return
sms_to = split_sms_to(sms_to)
headers = {
'Authorization': basic_auth(account_sid, auth_token),
}
errors = []
for phone in sms_to:
if not phone:
continue
try:
phone = clean_phone(phone)
http.safe_urlopen(
endpoint,
method='POST',
headers=headers,
data={
'From': sms_from,
'To': phone,
'Body': body,
},
).raise_for_status()
except Exception as e:
errors.append(e)
if errors:
if len(errors) == 1:
raise errors[0]
# TODO: multi-exception
raise Exception(errors)
def notify_users(self, group, event, **kwargs):
project = group.project
body = "Sentry [{0}] {1}: {2}".format(
project.name.encode("utf-8"),
event.group.get_level_display().upper().encode("utf-8"),
event.title.encode("utf-8").splitlines()[0],
)
body = body[:MAX_SMS_LENGTH]
account_sid = self.get_option("account_sid", project)
auth_token = self.get_option("auth_token", project)
sms_from = clean_phone(self.get_option("sms_from", project))
endpoint = twilio_sms_endpoint.format(account_sid)
sms_to = self.get_option("sms_to", project)
if not sms_to:
return
sms_to = split_sms_to(sms_to)
headers = {"Authorization": basic_auth(account_sid, auth_token)}
errors = []
for phone in sms_to:
if not phone:
continue
try:
phone = clean_phone(phone)
http.safe_urlopen(
endpoint,
method="POST",
headers=headers,
data={
"From": sms_from,
"To": phone,
"Body": body
},
).raise_for_status()
except Exception as e:
errors.append(e)
if errors:
if len(errors) == 1:
raise errors[0]
# TODO: multi-exception
raise Exception(errors)
def send_alert_event(event, rule, sentry_app_id):
group = event.group
project = Project.objects.get_from_cache(id=group.project_id)
organization = Organization.objects.get_from_cache(
id=project.organization_id)
extra = {
'sentry_app_id': sentry_app_id,
'project': project.slug,
'organization': organization.slug,
'rule': rule,
}
try:
sentry_app = SentryApp.objects.get(id=sentry_app_id)
except SentryApp.DoesNotExist:
logger.info('event_alert_webhook.missing_sentry_app', extra=extra)
return
try:
install = SentryAppInstallation.objects.get(
organization=organization.id,
sentry_app=sentry_app,
)
except SentryAppInstallation.DoesNotExist:
logger.info('event_alert_webhook.missing_installation', extra=extra)
return
event_context = _webhook_event_data(event, group.id, project.id)
data = {
'event': event_context,
'triggered_rule': rule,
}
request_data = AppPlatformEvent(
resource='event_alert',
action='triggered',
install=install,
data=data,
)
safe_urlopen(
url=sentry_app.webhook_url,
data=request_data.body,
headers=request_data.headers,
timeout=5,
)
def dispatch(self, request, helper):
access_token = helper.fetch_state('data')['access_token']
req = safe_urlopen('{0}?{1}&alt=json'.format(
USER_DETAILS_ENDPOINT,
urlencode({
'access_token': access_token,
})
))
body = safe_urlread(req)
data = json.loads(body)
if not data.get('data'):
logger.error('Invalid response: %s' % body)
return helper.error(ERR_INVALID_RESPONSE)
if not data.get('data').get('email'):
logger.error('Invalid response: %s' % body)
return helper.error(ERR_INVALID_RESPONSE)
domain = extract_domain(data.get('data').get('email'))
if domain in DOMAIN_BLOCKLIST:
return helper.error(ERR_INVALID_DOMAIN % (domain,))
if self.domain and self.domain != domain:
return helper.error(ERR_INVALID_DOMAIN % (domain,))
helper.bind_state('domain', domain)
helper.bind_state('user', data.get('data'))
return helper.next_step()
def notify_users(self, group, event, fail_silently=False):
if not self.is_configured(group.project):
return
api_key = self.get_option('api_key', group.project)
recipients = self.get_option('recipients', group.project)
alert_url = self.get_option('alert_url', group.project)
message = getattr(group, 'message_short', group.message).encode('utf-8')
payload = {
'apiKey': api_key,
'message': message,
'source': 'Sentry',
'details': self.get_group_data(group, event)
}
if recipients:
payload['recipients'] = recipients
req = http.safe_urlopen(alert_url, json=payload)
resp = req.json()
if resp.get('status') != 'successful':
raise Exception('Unsuccessful response from OpsGenie: %s' % resp)
def refresh_identity(self, auth_identity):
refresh_token = auth_identity.data.get("refresh_token")
if not refresh_token:
raise IdentityNotValid("Missing refresh token")
data = self.get_refresh_token_params(refresh_token=refresh_token)
req = safe_urlopen(self.get_refresh_token_url(), data=data)
try:
body = safe_urlread(req)
payload = json.loads(body)
except Exception:
payload = {}
error = payload.get("error", "unknown_error")
error_description = payload.get("error_description", "no description available")
formatted_error = "HTTP {} ({}): {}".format(req.status_code, error, error_description)
if req.status_code == 401:
raise IdentityNotValid(formatted_error)
if req.status_code == 400:
# this may not be common, but at the very least Google will return
# an invalid grant when a user is suspended
if error == "invalid_grant":
raise IdentityNotValid(formatted_error)
if req.status_code != 200:
raise Exception(formatted_error)
auth_identity.data.update(self.get_oauth_data(payload))
auth_identity.update(data=auth_identity.data)
def send_yach(self, url, payload):
return safe_urlopen(
url=url,
json=payload,
timeout=5,
verify_ssl=False,
)
def _make_request(self):
req = safe_urlopen(
url=self._build_url(),
headers=self._build_headers(),
)
try:
body = safe_urlread(req)
response = json.loads(body)
except Exception:
logger.info(
'select-requester.error',
extra={
'sentry_app': self.sentry_app.slug,
'install': self.install.uuid,
'project': self.project.slug,
'uri': self.uri,
}
)
response = {}
if not self._validate_response(response):
raise APIError()
return self._format_response(response)
def exchange_token(self, request, helper, code):
# TODO: this needs the auth yet
data = self.get_token_params(code=code, redirect_uri=absolute_uri(helper.get_redirect_url()))
req = safe_urlopen(self.access_token_url, data=data)
body = safe_urlread(req)
return json.loads(body)
def _make_request(self):
req = safe_urlopen(url=self._build_url(),
headers=self._build_headers(),
method="POST",
data=self.body)
try:
body = safe_urlread(req)
response = json.loads(body)
except Exception:
logger.info(
"issue-link-requester.error",
extra={
"sentry_app": self.sentry_app.slug,
"install": self.install.uuid,
"project": self.group.project.slug,
"group": self.group.id,
"uri": self.uri,
},
)
response = {}
if not self._validate_response(response):
raise APIError()
return response
def exchange_token(self, request, pipeline, code):
from sentry.http import safe_urlopen, safe_urlread
from sentry.utils.http import absolute_uri
from six.moves.urllib.parse import parse_qsl
from sentry.utils import json
req = safe_urlopen(
url=self.access_token_url,
headers={
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': '1322',
},
data={
'client_assertion_type':
'urn:ietf:params:oauth:client-assertion-type:jwt-bearer',
'client_assertion': self.client_secret,
'grant_type': 'urn:ietf:params:oauth:grant-type:jwt-bearer',
'assertion': code,
'redirect_uri': absolute_uri(pipeline.redirect_url()),
},
)
body = safe_urlread(req)
if req.headers['Content-Type'].startswith(
'application/x-www-form-urlencoded'):
return dict(parse_qsl(body))
return json.loads(body)
def dispatch(self, request, helper):
access_token = helper.fetch_state('data')['access_token']
req = safe_urlopen('{0}?{1}&alt=json'.format(
USER_DETAILS_ENDPOINT, urlencode({
'access_token': access_token,
})))
body = safe_urlread(req)
data = json.loads(body)
if not data.get('data'):
logger.error('Invalid response: %s' % body)
return helper.error(ERR_INVALID_RESPONSE)
if not data.get('data').get('email'):
logger.error('Invalid response: %s' % body)
return helper.error(ERR_INVALID_RESPONSE)
domain = extract_domain(data.get('data').get('email'))
if domain in DOMAIN_BLOCKLIST:
return helper.error(ERR_INVALID_DOMAIN % (domain, ))
if self.domain and self.domain != domain:
return helper.error(ERR_INVALID_DOMAIN % (domain, ))
helper.bind_state('domain', domain)
helper.bind_state('user', data.get('data'))
return helper.next_step()
def send_and_save_sentry_app_request(url, sentry_app, org_id, event, **kwargs):
"""
Send a webhook request, and save the request into the Redis buffer for the app dashboard request log
Returns the response of the request
kwargs ends up being the arguments passed into safe_urlopen
"""
buffer = SentryAppWebhookRequestsBuffer(sentry_app)
try:
resp = safe_urlopen(url=url, **kwargs)
except RequestException:
# Response code of 0 represents timeout
buffer.add_request(response_code=0,
org_id=org_id,
event=event,
url=url)
# Re-raise the exception because some of these tasks might retry on the exception
raise
buffer.add_request(response_code=resp.status_code,
org_id=org_id,
event=event,
url=url)
return resp
def send_webhook(self, payload, webhook, project):
self.logger.debug('Sending webhook to url: %s ' % webhook)
api_origin = self.get_option('api_origin', project)
api_type = self.get_option('api_type', project)
response = safe_urlopen(method=api_type, url=api_origin, json=payload)
self.logger.debug('Response code: %s, content: %s' %
(response.status_code, response.content))
def exchange_token(self, request, pipeline, code):
# TODO: this needs the auth yet
data = self.get_token_params(
code=code,
redirect_uri=absolute_uri(pipeline.redirect_url()),
)
verify_ssl = pipeline.config.get('verify_ssl', True)
try:
req = safe_urlopen(self.access_token_url, data=data, verify_ssl=verify_ssl)
body = safe_urlread(req)
if req.headers.get('Content-Type', '').startswith('application/x-www-form-urlencoded'):
return dict(parse_qsl(body))
return json.loads(body)
except SSLError:
logger.info('identity.oauth2.ssl-error', extra={
'url': self.access_token_url,
'verify_ssl': verify_ssl,
})
url = self.access_token_url
return {
'error': 'Could not verify SSL certificate',
'error_description': u'Ensure that {} has a valid SSL certificate'.format(url)
}
except JSONDecodeError:
logger.info('identity.oauth2.json-error', extra={
'url': self.access_token_url,
})
return {
'error': 'Could not decode a JSON Response',
'error_description': u'We were not able to parse a JSON response, please try again.'
}
def _make_request(self):
req = safe_urlopen(
url=self._build_url(),
headers=self._build_headers(),
method='POST',
data=self.body,
)
try:
body = safe_urlread(req)
response = json.loads(body)
except Exception:
logger.info(
'issue-link-requester.error',
extra={
'sentry_app': self.sentry_app.slug,
'install': self.install.uuid,
'project': self.group.project.slug,
'group': self.group.id,
'uri': self.uri,
}
)
response = {}
if not self._validate_response(response):
raise APIError()
return response
def exchange_token(self, request, pipeline, code):
from urllib.parse import parse_qsl
from sentry.http import safe_urlopen, safe_urlread
from sentry.utils import json
from sentry.utils.http import absolute_uri
req = safe_urlopen(
url=self.access_token_url,
headers={
"Content-Type": "application/x-www-form-urlencoded",
"Content-Length": "1322"
},
data={
"client_assertion_type":
"urn:ietf:params:oauth:client-assertion-type:jwt-bearer",
"client_assertion": self.client_secret,
"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
"assertion": code,
"redirect_uri": absolute_uri(pipeline.redirect_url()),
},
)
body = safe_urlread(req)
if req.headers["Content-Type"].startswith(
"application/x-www-form-urlencoded"):
return dict(parse_qsl(body))
return json.loads(body)
def send_and_save_sentry_app_request(url, sentry_app, org_id, event, **kwargs):
"""
Send a webhook request, and save the request into the Redis buffer for the app dashboard request log
Returns the response of the request
kwargs ends up being the arguments passed into safe_urlopen
"""
buffer = SentryAppWebhookRequestsBuffer(sentry_app)
slug = sentry_app.slug_for_metrics
try:
resp = safe_urlopen(url=url, **kwargs)
except RequestException:
track_response_code("timeout", slug, event)
# Response code of 0 represents timeout
buffer.add_request(response_code=0,
org_id=org_id,
event=event,
url=url)
# Re-raise the exception because some of these tasks might retry on the exception
raise
track_response_code(resp.status_code, slug, event)
buffer.add_request(
response_code=resp.status_code,
org_id=org_id,
event=event,
url=url,
error_id=resp.headers.get("Sentry-Hook-Error"),
project_id=resp.headers.get("Sentry-Hook-Project"),
)
return resp
def refresh_identity(self, identity, *args, **kwargs):
refresh_token = identity.data.get('refresh_token')
refresh_token_url = kwargs.get('refresh_token_url')
if not refresh_token:
raise IdentityNotValid('Missing refresh token')
if not refresh_token_url:
raise IdentityNotValid('Missing refresh token url')
data = self.get_refresh_token_params(refresh_token, *args, **kwargs)
req = safe_urlopen(
url=refresh_token_url,
headers={},
data=data,
)
try:
body = safe_urlread(req)
payload = json.loads(body)
except Exception as e:
self.logger('gitlab.refresh-identity-failure',
extra={
'identity_id': identity.id,
'error_status': e.code,
'error_message': e.message,
})
payload = {}
self.handle_refresh_error(req, payload)
identity.data.update(get_oauth_data(payload))
return identity.update(data=identity.data)
def send_webhook(self, url, payload):
return safe_urlopen(
url=url,
json=payload,
timeout=self.timeout,
verify_ssl=False,
)
def notify(self, notification):
event = notification.event
group = event.group
project = group.project
title = '%s: %s' % (project.name, group.title)
link = group.get_absolute_url()
tags = event.get_tags()
message = event.message + '\n'
if tags:
message = 'Tags: %s\n' % (', '.join('%s=%s' % (k, v)
for (k, v) in tags))
# see https://pushover.net/api
# We can no longer send JSON because pushover disabled incoming
# JSON data: http://updates.pushover.net/post/39822700181/
data = {
'user': self.get_option('userkey', project),
'token': self.get_option('apikey', project),
'message': message,
'title': title,
'url': link,
'url_title': 'Details',
'priority': 1 if self.get_option('priority', project) else 0,
}
rv = safe_urlopen('https://api.pushover.net/1/messages.json',
data=data)
if not rv.ok:
raise RuntimeError('Failed to notify: %s' % rv)
def refresh_identity(self, identity, *args, **kwargs):
refresh_token = identity.data.get('refresh_token')
refresh_token_url = kwargs.get('refresh_token_url')
if not refresh_token:
raise IdentityNotValid('Missing refresh token')
if not refresh_token_url:
raise IdentityNotValid('Missing refresh token url')
data = self.get_refresh_token_params(refresh_token, *args, **kwargs)
req = safe_urlopen(
url=refresh_token_url,
headers={},
data=data,
)
try:
body = safe_urlread(req)
payload = json.loads(body)
except Exception:
payload = {}
self.handle_refresh_error(req, payload)
identity.data.update(get_oauth_data(payload))
return identity.update(data=identity.data)
def _make_request(self):
req = safe_urlopen(
url=self._build_url(),
headers=self._build_headers(),
method='POST',
data=self.body,
)
try:
body = safe_urlread(req)
response = json.loads(body)
except Exception:
logger.info('issue-link-requester.error',
extra={
'sentry_app': self.sentry_app.slug,
'install': self.install.uuid,
'project': self.group.project.slug,
'group': self.group.id,
'uri': self.uri,
})
response = {}
if not self._validate_response(response):
raise APIError()
return response
def send_webhook(self, url, data):
return safe_urlopen(
url=url,
data=data,
timeout=self.timeout,
user_agent=self.user_agent,
)
def send_webhook(self, url, data):
return safe_urlopen(
url=url,
data=data,
timeout=self.timeout,
user_agent=self.user_agent,
)
def _make_request(self):
try:
body = safe_urlread(
safe_urlopen(
url=self._build_url(),
headers=self._build_headers(),
)
)
response = json.loads(body)
except Exception as e:
logger.info(
'select-requester.error',
extra={
'sentry_app': self.sentry_app.slug,
'install': self.install.uuid,
'project': self.project and self.project.slug,
'uri': self.uri,
'error_message': e.message,
}
)
response = {}
if not self._validate_response(response):
raise APIError()
return self._format_response(response)
def exchange_token(self, request, pipeline, code):
# TODO: this needs the auth yet
data = self.get_token_params(code=code, redirect_uri=absolute_uri(pipeline.redirect_url()))
verify_ssl = pipeline.config.get("verify_ssl", True)
try:
req = safe_urlopen(self.access_token_url, data=data, verify_ssl=verify_ssl)
body = safe_urlread(req)
if req.headers.get("Content-Type", "").startswith("application/x-www-form-urlencoded"):
return dict(parse_qsl(body))
return json.loads(body)
except SSLError:
logger.info(
"identity.oauth2.ssl-error",
extra={"url": self.access_token_url, "verify_ssl": verify_ssl},
)
url = self.access_token_url
return {
"error": "Could not verify SSL certificate",
"error_description": u"Ensure that {} has a valid SSL certificate".format(url),
}
except JSONDecodeError:
logger.info("identity.oauth2.json-error", extra={"url": self.access_token_url})
return {
"error": "Could not decode a JSON Response",
"error_description": u"We were not able to parse a JSON response, please try again.",
}
def send_and_save_webhook_request(sentry_app, app_platform_event, url=None):
"""
Notify a SentryApp's webhook about an incident and log response on redis.
:param sentry_app: The SentryApp to notify via a webhook.
:param app_platform_event: Incident data. See AppPlatformEvent.
:param url: The URL to hit for this webhook if it is different from `sentry_app.webhook_url`.
:return: Webhook response
"""
buffer = SentryAppWebhookRequestsBuffer(sentry_app)
org_id = app_platform_event.install.organization_id
event = f"{app_platform_event.resource}.{app_platform_event.action}"
slug = sentry_app.slug_for_metrics
url = url or sentry_app.webhook_url
try:
resp = safe_urlopen(
url=url, data=app_platform_event.body, headers=app_platform_event.headers, timeout=5
)
except (Timeout, ConnectionError) as e:
error_type = e.__class__.__name__.lower()
logger.info(
"send_and_save_webhook_request.timeout",
extra={
"error_type": error_type,
"organization_id": org_id,
"integration_slug": sentry_app.slug,
},
)
track_response_code(error_type, slug, event)
# Response code of 0 represents timeout
buffer.add_request(response_code=0, org_id=org_id, event=event, url=url)
# Re-raise the exception because some of these tasks might retry on the exception
raise
else:
track_response_code(resp.status_code, slug, event)
buffer.add_request(
response_code=resp.status_code,
org_id=org_id,
event=event,
url=url,
error_id=resp.headers.get("Sentry-Hook-Error"),
project_id=resp.headers.get("Sentry-Hook-Project"),
)
if resp.status_code == 503:
raise ApiHostError.from_request(resp.request)
elif resp.status_code == 504:
raise ApiTimeoutError.from_request(resp.request)
if 400 <= resp.status_code < 500:
raise ClientError(resp.status_code, url, response=resp)
resp.raise_for_status()
return resp
def create_snuba_subscription(project, dataset, query, aggregations,
time_window, resolution):
"""
Creates a subscription to a snuba query.
:param project: The project we're applying the query to
:param dataset: The snuba dataset to query and aggregate over
:param query: An event search query that we can parse and convert into a
set of Snuba conditions
:param aggregations: A list of aggregations to calculate over the time
window
:param time_window: The time window to aggregate over
:param resolution: How often to receive updates/bucket size
:return: A uuid representing the subscription id.
"""
# TODO: Might make sense to move this into snuba if we have wider use for
# it.
resp = safe_urlopen(
settings.SENTRY_SNUBA + "/subscriptions",
"POST",
json={
"project_id": project.id,
"dataset": dataset.value,
# We only care about conditions here. Filter keys only matter for
# filtering to project and groups. Projects are handled with an
# explicit param, and groups can't be queried here.
"conditions": get_snuba_query_args(query)["conditions"],
"aggregates":
[alert_aggregation_to_snuba[agg] for agg in aggregations],
"time_window": time_window,
"resolution": resolution,
},
)
resp.raise_for_status()
return uuid.UUID(resp.json()["subscription_id"])
def send_and_save_webhook_request(url, sentry_app, app_platform_event):
buffer = SentryAppWebhookRequestsBuffer(sentry_app)
org_id = app_platform_event.install.organization_id
event = "{}.{}".format(app_platform_event.resource,
app_platform_event.action)
try:
resp = safe_urlopen(url=url,
data=app_platform_event.body,
headers=app_platform_event.headers,
timeout=5)
except RequestException:
# Response code of 0 represents timeout
buffer.add_request(response_code=0,
org_id=org_id,
event=event,
url=url)
# Re-raise the exception because some of these tasks might retry on the exception
raise
buffer.add_request(
response_code=resp.status_code,
org_id=org_id,
event=event,
url=url,
error_id=resp.headers.get("Sentry-Hook-Error"),
project_id=resp.headers.get("Sentry-Hook-Project"),
)
return resp
def send_webhook(self, url, payload):
return safe_urlopen(
url=url,
json=payload,
timeout=self.timeout,
verify_ssl=False,
)
def refresh_identity(self, identity, *args, **kwargs):
refresh_token = identity.data.get("refresh_token")
refresh_token_url = kwargs.get("refresh_token_url")
if not refresh_token:
raise IdentityNotValid("Missing refresh token")
if not refresh_token_url:
raise IdentityNotValid("Missing refresh token url")
data = self.get_refresh_token_params(refresh_token, *args, **kwargs)
req = safe_urlopen(url=refresh_token_url, headers={}, data=data)
try:
body = safe_urlread(req)
payload = json.loads(body)
except Exception as e:
self.logger(
"gitlab.refresh-identity-failure",
extra={
"identity_id": identity.id,
"error_status": e.code,
"error_message": str(e),
},
)
payload = {}
self.handle_refresh_error(req, payload)
identity.data.update(get_oauth_data(payload))
return identity.update(data=identity.data)
def refresh_identity(self, auth_identity):
refresh_token = auth_identity.data.get("refresh_token")
if not refresh_token:
raise IdentityNotValid("Missing refresh token")
data = self.get_refresh_token_params(refresh_token=refresh_token)
req = safe_urlopen(self.get_refresh_token_url(), data=data)
try:
body = safe_urlread(req)
payload = json.loads(body)
except Exception:
payload = {}
error = payload.get("error", "unknown_error")
error_description = payload.get("error_description", "no description available")
formatted_error = f"HTTP {req.status_code} ({error}): {error_description}"
if req.status_code == 401:
raise IdentityNotValid(formatted_error)
if req.status_code == 400:
# this may not be common, but at the very least Google will return
# an invalid grant when a user is suspended
if error == "invalid_grant":
raise IdentityNotValid(formatted_error)
if req.status_code != 200:
raise Exception(formatted_error)
auth_identity.data.update(self.get_oauth_data(payload))
auth_identity.update(data=auth_identity.data)
def github_request(self, request, url, **kwargs):
"""
Make a GitHub request on behalf of the logged in user. Return JSON
response on success or raise forms.ValidationError on any exception
"""
auth = self.get_auth_for_user(user=request.user)
if auth is None:
raise forms.ValidationError(_("You have not yet associated GitHub with your account."))
headers = kwargs.pop("headers", None) or {}
headers["Authorization"] = "token %s" % auth.tokens["access_token"]
try:
req = safe_urlopen(url, headers=headers, **kwargs)
body = safe_urlread(req)
except requests.RequestException as e:
msg = unicode(e)
raise forms.ValidationError(_("Error communicating with GitHub: %s") % (msg,))
try:
json_resp = json.loads(body)
except ValueError as e:
msg = unicode(e)
raise forms.ValidationError(_("Error communicating with GitHub: %s") % (msg,))
if req.status_code > 399:
raise forms.ValidationError(json_resp["message"])
return json_resp
def exchange_token(self, request, pipeline, code):
# TODO: this needs the auth yet
data = self.get_token_params(
code=code,
redirect_uri=absolute_uri(pipeline.redirect_url()),
)
verify_ssl = pipeline.config.get('verify_ssl', True)
try:
req = safe_urlopen(self.access_token_url, data=data, verify_ssl=verify_ssl)
body = safe_urlread(req)
if req.headers.get('Content-Type', '').startswith('application/x-www-form-urlencoded'):
return dict(parse_qsl(body))
return json.loads(body)
except SSLError:
logger.info('identity.oauth2.ssl-error', extra={
'url': self.access_token_url,
'verify_ssl': verify_ssl,
})
url = self.access_token_url
return {
'error': 'Could not verify SSL certificate',
'error_description': u'Ensure that {} has a valid SSL certificate'.format(url)
}
except JSONDecodeError:
logger.info('identity.oauth2.json-error', extra={
'url': self.access_token_url,
})
return {
'error': 'Could not decode a JSON Response',
'error_description': u'We were not able to parse a JSON response, please try again.'
}
def send_webhook(self, url, data):
return safe_urlopen(
url=url,
data=data,
timeout=self.timeout,
user_agent=self.user_agent,
headers=(('Accept-Encoding', 'gzip'), ('Content-type', 'application/json')),
)
def exchange_token(self, request, helper, code):
# TODO: this needs the auth yet
data = self.get_token_params(code=code, redirect_uri=absolute_uri(helper.get_redirect_url()))
req = safe_urlopen(self.access_token_url, data=data)
body = safe_urlread(req)
if req.headers["Content-Type"].startswith("application/x-www-form-urlencoded"):
return dict(parse_qsl(body))
return json.loads(body)
def make_api_request(self, user, url, json_data=None):
auth = self.get_auth_for_user(user=user)
if auth is None:
raise forms.ValidationError(_('You have not yet associated GitHub with your account.'))
req_headers = {
'Authorization': 'token %s' % auth.tokens['access_token'],
}
return safe_urlopen(url, json=json_data, headers=req_headers, allow_redirects=True)
def _send_webhook(self):
safe_urlread(
safe_urlopen(
url=self.sentry_app.webhook_url,
data=self.request.body,
headers=self.request.headers,
timeout=5,
)
)
def send_request(servicehook, payload, verify_ssl=None):
from sentry import tsdb
tsdb.incr(tsdb.models.servicehook_fired, servicehook.id)
headers = {
'Content-Type': 'application/json',
'X-ServiceHook-Timestamp': six.text_type(int(time())),
'X-ServiceHook-GUID': servicehook.guid,
'X-ServiceHook-Signature': servicehook.build_signature(json.dumps(payload)),
}
safe_urlopen(
url=servicehook.url,
data=json.dumps(payload),
headers=headers,
timeout=5,
verify_ssl=(verify_ssl or False),
)
def test_ip_blacklist(self):
http.DISALLOWED_IPS = set([IPNetwork('127.0.0.1'), IPNetwork('::1'), IPNetwork('10.0.0.0/8')])
with pytest.raises(SuspiciousOperation):
http.safe_urlopen('http://127.0.0.1')
with pytest.raises(SuspiciousOperation):
http.safe_urlopen('http://10.0.0.10')
with pytest.raises(SuspiciousOperation):
# '2130706433' is dword for '127.0.0.1'
http.safe_urlopen('http://2130706433')
with pytest.raises(SuspiciousOperation):
# ipv6
http.safe_urlopen('http://[::1]')
def test_simple(self):
responses.add(responses.GET, 'http://example.com', body='foo bar')
resp = safe_urlopen('http://example.com')
data = safe_urlread(resp)
assert data == 'foo bar'
request = responses.calls[0].request
assert 'User-Agent' in request.headers
assert 'gzip' in request.headers.get('Accept-Encoding', '')
def _make_request(self, endpoint, method, **kwargs):
url = self.api_url + endpoint
request_kwargs = {'method': method, 'headers': self.request_headers}
body = kwargs.get('body')
if body:
request_kwargs['json'] = body
return safe_urlopen(url, **request_kwargs)
def test_simple(self, mock_getaddrinfo):
mock_getaddrinfo.return_value = [(2, 1, 6, '', ('81.0.0.1', 0))]
responses.add(responses.GET, 'http://example.com', body='foo bar')
resp = http.safe_urlopen('http://example.com')
data = http.safe_urlread(resp)
assert data.decode('utf-8') == 'foo bar'
request = responses.calls[0].request
assert 'User-Agent' in request.headers
assert 'gzip' in request.headers.get('Accept-Encoding', '')
def fetch_url(url, project=None):
"""
Pull down a URL, returning a UrlResult object.
Attempts to fetch from the cache.
"""
cache_key = 'source:%s' % (
hashlib.md5(url.encode('utf-8')).hexdigest(),)
result = cache.get(cache_key)
if result is None:
# lock down domains that are problematic
domain = urlparse(url).netloc
domain_key = 'source:%s' % (hashlib.md5(domain.encode('utf-8')).hexdigest(),)
domain_result = cache.get(domain_key)
if domain_result:
return BAD_SOURCE
headers = []
if project and is_valid_origin(url, project=project):
token = project.get_option('sentry:token')
if token:
headers.append(('X-Sentry-Token', token))
try:
request = safe_urlopen(
url,
allow_redirects=True,
headers=headers,
timeout=settings.SENTRY_SOURCE_FETCH_TIMEOUT,
)
except HTTPError:
result = BAD_SOURCE
except Exception:
# it's likely we've failed due to a timeout, dns, etc so let's
# ensure we can't cascade the failure by pinning this for 5 minutes
cache.set(domain_key, 1, 300)
logger.warning('Disabling sources to %s for %ss', domain, 300,
exc_info=True)
return BAD_SOURCE
else:
try:
body = safe_urlread(request)
except Exception:
result = BAD_SOURCE
else:
result = (dict(request.headers), body)
cache.set(cache_key, result, 60)
if result == BAD_SOURCE:
return result
return UrlResult(url, *result)
def test_simple(self, mock_gethostbyname):
mock_gethostbyname.return_value = '81.0.0.1'
responses.add(responses.GET, 'http://example.com', body='foo bar')
resp = http.safe_urlopen('http://example.com')
data = http.safe_urlread(resp)
assert data == 'foo bar'
request = responses.calls[0].request
assert 'User-Agent' in request.headers
assert 'gzip' in request.headers.get('Accept-Encoding', '')
def exchange_token(self, request, pipeline, code):
# TODO: this needs the auth yet
data = self.get_token_params(
code=code,
redirect_uri=absolute_uri(pipeline.redirect_url()),
)
verify_ssl = pipeline.config.get('verify_ssl', True)
req = safe_urlopen(self.access_token_url, data=data, verify_ssl=verify_ssl)
body = safe_urlread(req)
if req.headers['Content-Type'].startswith('application/x-www-form-urlencoded'):
return dict(parse_qsl(body))
return json.loads(body)
def send_beacon():
"""
Send a Beacon to a remote server operated by the Sentry team.
See the documentation for more details.
"""
from sentry import options
from sentry.models import Organization, Project, Team, User
if not settings.SENTRY_BEACON:
logger.info('Not sending beacon (disabled)')
return
# TODO(dcramer): move version code off of PyPi and into beacon
install_id = options.get('sentry:install-id')
if not install_id:
logger.info('Generated installation ID: %s', install_id)
install_id = sha1(uuid4().hex).hexdigest()
options.set('sentry:install-id', install_id)
internal_project_ids = filter(bool, [
settings.SENTRY_PROJECT, settings.SENTRY_FRONTEND_PROJECT,
])
platform_list = list(set(Project.objects.exclude(
id__in=internal_project_ids,
).values_list('platform', flat=True)))
payload = {
'install_id': install_id,
'version': sentry.get_version(),
'admin_email': settings.SENTRY_ADMIN_EMAIL,
'data': {
# TODO(dcramer): we'd also like to get an idea about the throughput
# of the system (i.e. events in 24h)
'platforms': platform_list,
'users': User.objects.count(),
'projects': Project.objects.count(),
'teams': Team.objects.count(),
'organizations': Organization.objects.count(),
}
}
# TODO(dcramer): relay the response 'notices' as admin broadcasts
try:
request = safe_urlopen(BEACON_URL, json=payload, timeout=5)
response = safe_urlread(request)
except Exception:
logger.warning('Failed sending beacon', exc_info=True)
return
data = json.loads(response)
if 'version' in data:
options.set('sentry:latest_version', data['version']['stable'])
def notify_users(self, group, event, fail_silently=False):
if not self.is_configured(group.project):
return
webhook = self.get_option('webhook', event.project)
channel = self.get_option('channel', event.project)
username = self.get_option('username', event.project)
icon = self.get_option('icon', event.project)
project = event.project
team = event.team
title = '%s on <%s|%s %s>' % (
'New event' if group.times_seen == 1 else 'Regression',
group.get_absolute_url(),
escape(team.name.encode('utf-8')),
escape(project.name.encode('utf-8')),
)
message = group.message_short.encode('utf-8')
culprit = group.title.encode('utf-8')
# They can be the same if there is no culprit
# So we set culprit to an empty string instead of duplicating the text
if message == culprit:
culprit = ''
payload = {
'parse': 'none',
'text': title,
'channel': channel,
'attachments': [{
'color': self.color_for_group(group),
'fields': [{
'title': escape(message),
'value': escape(culprit),
'short': False,
}]
}]
}
if username:
payload['username'] = username
if icon:
urlparts = urlparse(icon)
if urlparts.scheme and urlparts.netloc:
payload['icon_url'] = icon
else:
payload['icon_emoji'] = icon
data = {'payload': json.dumps(payload)}
return http.safe_urlopen(webhook, method='POST', data=data)
def send_beacon():
"""
Send a Beacon to a remote server operated by the Sentry team.
See the documentation for more details.
"""
from sentry import options
from sentry.models import Organization, Project, Team, User
if not settings.SENTRY_BEACON:
logger.info('Not sending beacon (disabled)')
return
install_id = options.get('sentry:install-id')
if not install_id:
logger.info('Generated installation ID: %s', install_id)
install_id = sha1(uuid4().hex).hexdigest()
options.set('sentry:install-id', install_id)
end = timezone.now()
events_24h = tsdb.get_sums(
model=tsdb.models.internal,
keys=['events.total'],
start=end - timedelta(hours=24),
end=end,
)['events.total']
payload = {
'install_id': install_id,
'version': sentry.get_version(),
'admin_email': settings.SENTRY_ADMIN_EMAIL,
'data': {
# TODO(dcramer): we'd also like to get an idea about the throughput
# of the system (i.e. events in 24h)
'users': User.objects.count(),
'projects': Project.objects.count(),
'teams': Team.objects.count(),
'organizations': Organization.objects.count(),
'events.24h': events_24h,
}
}
# TODO(dcramer): relay the response 'notices' as admin broadcasts
try:
request = safe_urlopen(BEACON_URL, json=payload, timeout=5)
response = safe_urlread(request)
except Exception:
logger.warning('Failed sending beacon', exc_info=True)
return
data = json.loads(response)
if 'version' in data:
options.set('sentry:latest_version', data['version']['stable'])
def test_ip_blacklist_ipv4(self):
with pytest.raises(SuspiciousOperation):
http.safe_urlopen('http://127.0.0.1')
with pytest.raises(SuspiciousOperation):
http.safe_urlopen('http://10.0.0.10')
with pytest.raises(SuspiciousOperation):
# '2130706433' is dword for '127.0.0.1'
http.safe_urlopen('http://2130706433')
def test_simple(self, mock_gethostbyname):
responses.add(responses.GET, 'http://example.com', body='foo bar')
# this test fails if you dont have working DNS as it resolves it to
# localhost, so we patch gethostbyname
mock_gethostbyname.return_value = '208.1.41.1'
resp = safe_urlopen('http://example.com')
data = safe_urlread(resp)
assert data == 'foo bar'
request = responses.calls[0].request
assert 'User-Agent' in request.headers
assert 'gzip' in request.headers.get('Accept-Encoding', '')
