def _kwarg_value(self, key, **kwargs): """ Support a dot notation shortcut for deeply nested dicts or just look up the value if passed a normal key. >>> self.kwargs = {'foo': {'bar': {'baz': 1}}} >>> self._kwarg_value('foo.bar.baz') 1 >>> self._kwarg_value('foo') {'bar': {'baz': 1}} """ if '.' in key: keys = deque(key.split('.')) else: kwarg = self.kwargs[key] if kwargs.get('format') == 'json': return json.loads(kwarg) return kwarg kwarg = dict(self.kwargs) if kwargs.get('format') == 'json': kwarg = json.loads(kwarg[keys.popleft()]) while keys: kwarg = kwarg[keys.popleft()] return kwarg
def test_compare_commits_no_start(self): responses.add( responses.GET, 'https://example.gitlab.com/api/v4/projects/%s/repository/commits/xyz' % self.gitlab_id, json={'created_at': '2018-09-19T13:14:15Z'} ) responses.add( responses.GET, 'https://example.gitlab.com/api/v4/projects/%s/repository/commits?until=2018-09-19T13:14:15Z' % self.gitlab_id, json=json.loads(COMMIT_LIST_RESPONSE) ) responses.add( responses.GET, 'https://example.gitlab.com/api/v4/projects/%s/repository/commits/ed899a2f4b50b4370feeea94676502b42383c746/diff' % self.gitlab_id, json=json.loads(COMMIT_DIFF_RESPONSE) ) responses.add( responses.GET, 'https://example.gitlab.com/api/v4/projects/%s/repository/commits/6104942438c14ec7bd21c6cd5bd995272b3faff6/diff' % self.gitlab_id, json=json.loads(COMMIT_DIFF_RESPONSE) ) response = self.create_repository(self.default_repository_config, self.integration.id) repo = Repository.objects.get(pk=response.data['id']) commits = self.provider.compare_commits(repo, None, 'xyz') for commit in commits: assert_commit_shape(commit)
def put(self, request): """ Verify a User ````````````` This endpoint verifies the currently authenticated user (for example, to gain superuser). :auth: required """ if not request.user.is_authenticated(): return Response(status=status.HTTP_401_UNAUTHORIZED) validator = AuthVerifyValidator(data=request.DATA) if not validator.is_valid(): return self.respond(validator.errors, status=status.HTTP_400_BAD_REQUEST) authenticated = False # See if we have a u2f challenge/response if 'challenge' in validator.object and 'response' in validator.object: try: interface = Authenticator.objects.get_interface(request.user, 'u2f') if not interface.is_enrolled: raise LookupError() challenge = json.loads(validator.object['challenge']) response = json.loads(validator.object['response']) authenticated = interface.validate_response(request, challenge, response) except ValueError: pass except LookupError: pass # attempt password authentication else: authenticated = request.user.check_password(validator.object['password']) # UI treats 401s by redirecting, this 401 should be ignored if not authenticated: return Response({'detail': {'code': 'ignore'}}, status=status.HTTP_403_FORBIDDEN) try: # Must use the real request object that Django knows about auth.login(request._request, request.user) except auth.AuthUserPasswordExpired: return Response( { 'code': 'password-expired', 'message': 'Cannot sign-in with basic auth because password has expired.', }, status=status.HTTP_403_FORBIDDEN ) request.user = request._request.user return self.get(request)
def test_resolve_issue(self): status_action = { 'name': 'resolve_dialog', 'value': 'resolve_dialog', } # Expect request to open dialog on slack responses.add( method=responses.POST, url='https://slack.com/api/dialog.open', body='{"ok": true}', status=200, content_type='application/json', ) resp = self.post_webhook(action_data=[status_action]) assert resp.status_code == 200, resp.content # Opening dialog should *not* cause the current message to be updated assert resp.content == '' data = parse_qs(responses.calls[0].request.body) assert data['token'][0] == self.integration.metadata['access_token'] assert data['trigger_id'][0] == self.trigger_id assert 'dialog' in data dialog = json.loads(data['dialog'][0]) callback_data = json.loads(dialog['callback_id']) assert int(callback_data['issue']) == self.group1.id assert callback_data['orig_response_url'] == self.response_url # Completing the dialog will update the message responses.add( method=responses.POST, url=self.response_url, body='{"ok": true}', status=200, content_type='application/json', ) resp = self.post_webhook( type='dialog_submission', callback_id=dialog['callback_id'], data={'submission': {'resolve_type': 'resolved'}} ) self.group1 = Group.objects.get(id=self.group1.id) assert resp.status_code == 200, resp.content assert self.group1.get_status() == GroupStatus.RESOLVED update_data = json.loads(responses.calls[1].request.body) expect_status = u'*Issue resolved by <@{}>*'.format(self.identity.external_id) assert update_data['text'].endswith(expect_status)
def enroll(self, request, interface, insecure=False): challenge = request.POST.get('challenge') if challenge: interface.enrollment_data = json.loads(challenge) response = request.POST.get('response') if response: interface.try_enroll(json.loads(response)) return TwoFactorSettingsView.enroll(self, request, interface) context = self.make_context(request, interface) return render_to_response('sentry/account/twofactor/enroll_u2f.html', context, request)
def handle(self, request): user = auth.get_pending_2fa_user(request) if user is None or request.user.is_authenticated(): return HttpResponseRedirect(auth.get_login_url()) interfaces = Authenticator.objects.all_interfaces_for_user(user) # If for whatever reason we ended up here but the user has no 2FA # enabled, we just continue successfully. if not interfaces: return self.perform_signin(request, user) challenge = activation = None interface = self.negotiate_interface(request, interfaces) if request.method == "GET": activation = interface.activate(request) if activation is not None and activation.type == "challenge": challenge = activation.challenge elif "challenge" in request.POST: challenge = json.loads(request.POST["challenge"]) form = TwoFactorForm() # If an OTP response was supplied, we try to make it pass. otp = request.POST.get("otp") if otp: used_interface = self.validate_otp(otp, interface, interfaces) if used_interface is not None: return self.perform_signin(request, user, used_interface) self.fail_signin(request, user, form) # If a challenge and response exists, validate if challenge: response = request.POST.get("response") if response: response = json.loads(response) if interface.validate_response(request, challenge, response): return self.perform_signin(request, user, interface) self.fail_signin(request, user, form) return render_to_response( ["sentry/twofactor_%s.html" % interface.interface_id, "sentry/twofactor.html"], { "form": form, "interface": interface, "other_interfaces": self.get_other_interfaces(interface, interfaces), "activation": activation, }, request, status=200, )
def handle_sudo(self, request, redirect_to, context): interface = Authenticator.objects.get_interface(request.user, 'u2f') if interface.is_available and interface.is_enrolled: challenge = interface.activate(request).challenge if request.method == 'POST': if 'challenge' in request.POST: challenge = json.loads(request.POST['challenge']) if 'response' in request.POST: response = json.loads(request.POST['response']) if interface.validate_response(request, challenge, response): return True context['u2f_challenge'] = challenge return BaseSudoView.handle_sudo(self, request, redirect_to, context)
def test_compare_commits_no_start(self): stub_installation_token() responses.add( responses.GET, 'https://api.github.com/repos/getsentry/example-repo/commits?sha=abcdef', json=json.loads(GET_LAST_COMMITS_EXAMPLE) ) responses.add( responses.GET, 'https://api.github.com/repos/getsentry/example-repo/commits/6dcb09b5b57875f334f61aebed695e2e4193db5e', json=json.loads(GET_COMMIT_EXAMPLE) ) result = self.provider.compare_commits(self.repository, None, 'abcdef') for commit in result: assert_commit_shape(commit)
def handle(self, request): user = auth.get_pending_2fa_user(request) if user is None or request.user.is_authenticated(): return HttpResponseRedirect(reverse('sentry')) interfaces = Authenticator.objects.all_interfaces_for_user(user) # If for whatever reason we ended up here but the user has no 2FA # enabled, we just continue successfully. if not interfaces: return self.perform_signin(request, user) challenge = activation = None interface = self.negotiate_interface(request, interfaces) if request.method == 'GET': activation = interface.activate(request) if activation is not None and activation.type == 'challenge': challenge = activation.challenge elif 'challenge' in request.POST: challenge = json.loads(request.POST['challenge']) form = TwoFactorForm() # If an OTP response was supplied, we try to make it pass. otp = request.POST.get('otp') if otp: used_interface = self.validate_otp(otp, interface, interfaces) if used_interface is not None: return self.perform_signin(request, user, used_interface) self.fail_signin(request, user, form) # If a challenge and response exists, validate if challenge: response = request.POST.get('response') if response: response = json.loads(response) if interface.validate_response(request, challenge, response): return self.perform_signin(request, user, interface) self.fail_signin(request, user, form) return render_to_response(['sentry/twofactor_%s.html' % interface.interface_id, 'sentry/twofactor.html'], { 'form': form, 'interface': interface, 'other_interfaces': self.get_other_interfaces(interface, interfaces), 'activation': activation, }, request, status=200)
def dispatch(self, request, helper): access_token = helper.fetch_state('data')['access_token'] req = safe_urlopen('{0}?{1}&alt=json'.format( USER_DETAILS_ENDPOINT, urlencode({ 'access_token': access_token, }) )) body = safe_urlread(req) data = json.loads(body) if not data.get('data'): logger.error('Invalid response: %s' % body) return helper.error(ERR_INVALID_RESPONSE) if not data.get('data').get('email'): logger.error('Invalid response: %s' % body) return helper.error(ERR_INVALID_RESPONSE) domain = extract_domain(data.get('data').get('email')) if domain in DOMAIN_BLOCKLIST: return helper.error(ERR_INVALID_DOMAIN % (domain,)) if self.domain and self.domain != domain: return helper.error(ERR_INVALID_DOMAIN % (domain,)) helper.bind_state('domain', domain) helper.bind_state('user', data.get('data')) return helper.next_step()
def request(self, method, path, user, auth=None, params=None, data=None, is_sudo=False): full_path = self.prefix + path resolver_match = resolve(full_path) callback, callback_args, callback_kwargs = resolver_match if data: # we encode to ensure compatibility data = json.loads(json.dumps(data)) rf = APIRequestFactory() mock_request = getattr(rf, method.lower())(full_path, data) mock_request.auth = auth mock_request.user = user mock_request.is_sudo = lambda: is_sudo if params: mock_request.GET._mutable = True mock_request.GET.update(params) mock_request.GET._mutable = False if data: mock_request.POST._mutable = True mock_request.POST.update(data) mock_request.POST._mutable = False response = callback(mock_request, *callback_args, **callback_kwargs) if 200 <= response.status_code < 400: return response raise self.ApiError(response.status_code, response.data)
def refresh_identity(self, auth_identity): refresh_token = auth_identity.data.get("refresh_token") if not refresh_token: raise IdentityNotValid("Missing refresh token") data = self.get_refresh_token_params(refresh_token=refresh_token) req = safe_urlopen(self.get_refresh_token_url(), data=data) try: body = safe_urlread(req) payload = json.loads(body) except Exception: payload = {} error = payload.get("error", "unknown_error") error_description = payload.get("error_description", "no description available") formatted_error = "HTTP {} ({}): {}".format(req.status_code, error, error_description) if req.status_code == 401: raise IdentityNotValid(formatted_error) if req.status_code == 400: # this may not be common, but at the very least Google will return # an invalid grant when a user is suspended if error == "invalid_grant": raise IdentityNotValid(formatted_error) if req.status_code != 200: raise Exception(formatted_error) auth_identity.data.update(self.get_oauth_data(payload)) auth_identity.update(data=auth_identity.data)
def sourcemap_to_index(sourcemap): smap = json.loads(sourcemap) state_list = [] key_list = [] src_list = set() content = None if 'sourcesContent' in smap: content = {} for idx, source in enumerate(smap['sources']): if smap['sourcesContent'][idx]: content[source] = smap['sourcesContent'][idx].splitlines() else: content[source] = [] for state in parse_sourcemap(smap): state_list.append(state) key_list.append((state.dst_line, state.dst_col)) # Apparently it's possible to not have a src # specified in the vlq segments if state.src is not None: src_list.add(state.src) return SourceMapIndex(state_list, key_list, src_list, content)
def exchange_token(self, request, helper, code): # TODO: this needs the auth yet data = self.get_token_params(code=code, redirect_uri=absolute_uri(helper.get_redirect_url())) req = safe_urlopen(self.access_token_url, data=data) body = safe_urlread(req) return json.loads(body)
def test_simple_notification(self): responses.add( 'POST', 'https://alert.victorops.com/integrations/generic/20131114/alert/secret-api-key/everyone', body=SUCCESS ) self.plugin.set_option('api_key', 'secret-api-key', self.project) self.plugin.set_option('routing_key', 'everyone', self.project) group = self.create_group(message='Hello world', culprit='foo.bar') event = self.create_event(group=group, message='Hello world', tags={'level': 'warning'}) rule = Rule.objects.create(project=self.project, label='my rule') notification = Notification(event=event, rule=rule) with self.options({'system.url-prefix': 'http://example.com'}): self.plugin.notify(notification) request = responses.calls[0].request payload = json.loads(request.body) assert { 'message_type': 'WARNING', 'entity_id': group.id, 'entity_display_name': 'Hello world', 'monitoring_tool': 'sentry', 'state_message': 'Stacktrace\n-----------\n\nStacktrace (most recent call last):\n\n File "sentry/models/foo.py", line 29, in build_msg\n string_max_length=self.string_max_length)\n\nMessage\n-----------\n\nHello world', 'timestamp': int(event.datetime.strftime('%s')), 'issue_url': 'http://example.com/baz/bar/issues/%s/' % group.id, } == payload
def test_link_issue(self): responses.add( responses.GET, 'https://gitlab.com/api/v4/projects/getsentry%2Fsentry/issues/1', body='{"iid": 1, "id": "10", "title": "Hello world"}', match_querystring=True ) responses.add( responses.POST, 'https://gitlab.com/api/v4/projects/getsentry%2Fsentry/issues/1/notes', body='{"body": "Hello"}' ) self.plugin.set_option('gitlab_url', 'https://gitlab.com', self.project) self.plugin.set_option('gitlab_repo', 'getsentry/sentry', self.project) self.plugin.set_option('gitlab_token', 'abcdefg', self.project) group = self.create_group(message='Hello world', culprit='foo.bar') request = self.request.get('/') request.user = self.user form_data = { 'comment': 'Hello', 'issue_id': '1', } self.login_as(self.user) assert self.plugin.link_issue(request, group, form_data) == { 'title': 'Hello world', } request = responses.calls[-1].request payload = json.loads(request.body) assert payload == { 'body': 'Hello', }
def _make_request(self): try: body = safe_urlread( safe_urlopen( url=self._build_url(), headers=self._build_headers(), ) ) response = json.loads(body) except Exception as e: logger.info( 'select-requester.error', extra={ 'sentry_app': self.sentry_app.slug, 'install': self.install.uuid, 'project': self.project and self.project.slug, 'uri': self.uri, 'error_message': e.message, } ) response = {} if not self._validate_response(response): raise APIError() return self._format_response(response)
def sourcemap_to_index(sourcemap): smap = json.loads(sourcemap) state_list = [] key_list = [] src_list = set() content = {} sourceRoot = smap.get('sourceRoot') # turn /foo/bar into /foo/bar/ so urljoin doesnt strip the last path if sourceRoot and not sourceRoot.endswith('/'): sourceRoot = sourceRoot + '/' if 'sourcesContent' in smap: for idx, source in enumerate(smap['sources']): # Ensure we handle null files that may be specified outside of # sourcesContent if smap['sourcesContent'][idx]: # Apply the root to the source before shoving into the index # so we can look it up correctly later source = urljoin(sourceRoot, source) content[source] = smap['sourcesContent'][idx].splitlines() for state in parse_sourcemap(smap): state_list.append(state) key_list.append((state.dst_line, state.dst_col)) # Apparently it's possible to not have a src # specified in the vlq segments if state.src is not None: src_list.add(state.src) return SourceMapIndex(state_list, key_list, src_list, content)
def after_link_issue(self, mock_get_jwt): responses.add( responses.POST, 'https://api.github.com/installations/github_external_id/access_tokens', json={'token': 'token_1', 'expires_at': '2018-10-11T22:14:10Z'} ) responses.add( responses.POST, 'https://api.github.com/repos/getsentry/sentry/issues/321/comments', json={'body': 'hello'} ) data = {'comment': 'hello'} external_issue = ExternalIssue.objects.create( organization_id=self.organization.id, integration_id=self.model.id, key='hello#321', ) self.integration.after_link_issue(external_issue, data=data) request = responses.calls[0].request assert request.headers['Authorization'] == 'Bearer jwt_token_1' request = responses.calls[1].request assert request.headers['Authorization'] == 'token token_1' payload = json.loads(request.body) assert payload == {'body': 'hello'}
def view(self, request, group, **kwargs): if request.GET.get('autocomplete_query'): query = request.GET.get('q') if not query: return JSONResponse({'issues': []}) repo = self.get_option('repo', group.project) query = 'repo:%s %s' % (repo, query) url = 'https://api.github.com/search/issues?%s' % (urlencode({'q': query}),) try: req = self.make_api_request(request.user, url) body = safe_urlread(req) except requests.RequestException as e: msg = unicode(e) self.handle_api_error(request, msg) return JSONResponse({}, status=502) try: json_resp = json.loads(body) except ValueError as e: msg = unicode(e) self.handle_api_error(request, msg) return JSONResponse({}, status=502) issues = [{ 'text': '(#%s) %s' % (i['number'], i['title']), 'id': i['number'] } for i in json_resp.get('items', [])] return JSONResponse({'issues': issues}) return super(GitHubPlugin, self).view(request, group, **kwargs)
def get_issue_title_by_id(self, request, group, issue_id): url = '%s/%s' % (self.build_api_url(group, 'issues'), issue_id) req = self.make_api_request(request.user, url) body = safe_urlread(req) json_resp = json.loads(body) return json_resp['title']
def github_request(self, request, url, **kwargs): """ Make a GitHub request on behalf of the logged in user. Return JSON response on success or raise forms.ValidationError on any exception """ auth = self.get_auth_for_user(user=request.user) if auth is None: raise forms.ValidationError(_("You have not yet associated GitHub with your account.")) headers = kwargs.pop("headers", None) or {} headers["Authorization"] = "token %s" % auth.tokens["access_token"] try: req = safe_urlopen(url, headers=headers, **kwargs) body = safe_urlread(req) except requests.RequestException as e: msg = unicode(e) raise forms.ValidationError(_("Error communicating with GitHub: %s") % (msg,)) try: json_resp = json.loads(body) except ValueError as e: msg = unicode(e) raise forms.ValidationError(_("Error communicating with GitHub: %s") % (msg,)) if req.status_code > 399: raise forms.ValidationError(json_resp["message"]) return json_resp
def test_create_issue(self, mock_get_jwt): responses.add( responses.POST, 'https://api.github.com/installations/github_external_id/access_tokens', json={'token': 'token_1', 'expires_at': '2018-10-11T22:14:10Z'} ) responses.add( responses.POST, 'https://api.github.com/repos/getsentry/sentry/issues', json={'number': 321, 'title': 'hello', 'body': 'This is the description', 'html_url': 'https://github.com/getsentry/sentry/issues/231'} ) form_data = { 'repo': 'getsentry/sentry', 'title': 'hello', 'description': 'This is the description', } assert self.integration.create_issue(form_data) == { 'key': 321, 'description': 'This is the description', 'title': 'hello', 'url': 'https://github.com/getsentry/sentry/issues/231', 'repo': 'getsentry/sentry', } request = responses.calls[0].request assert request.headers['Authorization'] == 'Bearer jwt_token_1' request = responses.calls[1].request assert request.headers['Authorization'] == 'token token_1' payload = json.loads(request.body) assert payload == {'body': 'This is the description', 'assignee': None, 'title': 'hello'}
def create_issue(self, request, group, form_data, **kwargs): # TODO: support multiple identities via a selection input in the form? json_data = { "title": form_data['title'], "body": form_data['description'], "assignee": form_data.get('assignee'), } try: url = self.build_api_url(group, 'issues') req = self.make_api_request(request.user, url, json_data=json_data) body = safe_urlread(req) except requests.RequestException as e: msg = unicode(e) raise forms.ValidationError(_('Error communicating with GitHub: %s') % (msg,)) try: json_resp = json.loads(body) except ValueError as e: msg = unicode(e) raise forms.ValidationError(_('Error communicating with GitHub: %s') % (msg,)) if req.status_code > 399: raise forms.ValidationError(json_resp['message']) return json_resp['number']
def _handle_builtin(self, request, project): endpoint = '/projects/{}/{}/releases/'.format( project.organization.slug, project.slug, ) try: # Ideally the API client would support some kind of god-mode here # as we've already confirmed credentials and simply want to execute # the view code. Instead we hack around it with an ApiKey instance god = ApiKey( organization=project.organization, scopes=getattr(ApiKey.scopes, 'project:write'), ) resp = client.post( endpoint, data=json.loads(request.body), auth=god, ) except client.ApiError as exc: return HttpResponse( status=exc.status_code, content=exc.body, content_type='application/json', ) return HttpResponse( status=resp.status_code, content=json.dumps(resp.data), content_type='application/json', )
def exchange_token(self, request, pipeline, code): # TODO: this needs the auth yet data = self.get_token_params( code=code, redirect_uri=absolute_uri(pipeline.redirect_url()), ) verify_ssl = pipeline.config.get('verify_ssl', True) try: req = safe_urlopen(self.access_token_url, data=data, verify_ssl=verify_ssl) body = safe_urlread(req) if req.headers.get('Content-Type', '').startswith('application/x-www-form-urlencoded'): return dict(parse_qsl(body)) return json.loads(body) except SSLError: logger.info('identity.oauth2.ssl-error', extra={ 'url': self.access_token_url, 'verify_ssl': verify_ssl, }) url = self.access_token_url return { 'error': 'Could not verify SSL certificate', 'error_description': u'Ensure that {} has a valid SSL certificate'.format(url) } except JSONDecodeError: logger.info('identity.oauth2.json-error', extra={ 'url': self.access_token_url, }) return { 'error': 'Could not decode a JSON Response', 'error_description': u'We were not able to parse a JSON response, please try again.' }
def test_compare_commits(self, mock_compare_commits): organization = self.create_organization() integration = Integration.objects.create( provider='github', external_id='1', ) repo = Repository.objects.create( name='example-repo', provider='integrations:github', organization_id=organization.id, integration_id=integration.id, config={'name': 'example-repo'}, ) res = self.provider._format_commits(repo, json.loads(COMPARE_COMMITS_EXAMPLE)['commits']) assert res == [ { 'author_email': '*****@*****.**', 'author_name': 'Monalisa Octocat', 'message': 'Fix all the bugs', 'id': '6dcb09b5b57875f334f61aebed695e2e4193db5e', 'repository': 'example-repo' } ] self.provider.compare_commits(repo, 'b' * 40, 'a' * 40) assert mock_compare_commits.called
def test_valid_params(self): resp = self.client.post(self.path, data=json.dumps({ 'version': 'a', }), content_type='application/json') assert resp.status_code == 201, resp.content data = json.loads(resp.content) assert data['version'] == 'a'
def sourcemap_to_index(sourcemap): smap = json.loads(sourcemap) state_list = [] key_list = [] src_list = set() content = None root = smap.get('sourceRoot') if 'sourcesContent' in smap: content = {} for idx, source in enumerate(smap['sources']): # Apply the root to the source before shoving into the index # so we can look it up correctly later source = urljoin(root, source) if smap['sourcesContent'][idx]: content[source] = smap['sourcesContent'][idx].splitlines() else: content[source] = [] for state in parse_sourcemap(smap): state_list.append(state) key_list.append((state.dst_line, state.dst_col)) # Apparently it's possible to not have a src # specified in the vlq segments if state.src is not None: src_list.add(state.src) return SourceMapIndex(state_list, key_list, src_list, content)
def load_data(platform): json_path = os.path.join(DATA_ROOT, 'samples', '%s.json' % (platform.encode('utf-8'),)) if not os.path.exists(json_path): return with open(json_path) as fp: data = json.loads(fp.read()) data['platform'] = platform data['message'] = 'This is an example %s exception' % ( PLATFORM_TITLES.get(platform, platform.title()),) data['sentry.interfaces.User'] = { "username": "******", "id": "1671", "email": "*****@*****.**" } data['tags'] = [ ('foo', 'bar'), ('version', '1.0'), ] data['sentry.interfaces.Http'] = { "cookies": {}, "url": "http://example.com/foo", "headers": { "Referer": "http://example.com", "User-Agent": "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36" }, "env": {}, "query_string": "", "data": {}, "method": "GET" } return data
def process_message(self, message): return json.loads(message.value())
def bulk_raw_query(snuba_param_list, referrer=None): headers = {} if referrer: headers["referer"] = referrer query_param_list = map(_prepare_query_params, snuba_param_list) def snuba_query(params): query_params, forward, reverse, thread_hub = params try: with timer("snuba_query"): referrer = headers.get("referer", "<unknown>") if SNUBA_INFO: logger.info("{}.body: {}".format(referrer, json.dumps(query_params))) query_params["debug"] = True body = json.dumps(query_params) with thread_hub.start_span( op="snuba", description=u"query {}".format(referrer)) as span: span.set_tag("referrer", referrer) for param_key, param_data in six.iteritems(query_params): span.set_data(param_key, param_data) return ( _snuba_pool.urlopen("POST", "/query", body=body, headers=headers), forward, reverse, ) except urllib3.exceptions.HTTPError as err: raise SnubaError(err) with sentry_sdk.start_span( op="start_snuba_query", description=u"running {} snuba queries".format( len(snuba_param_list)), ) as span: span.set_tag("referrer", headers.get("referer", "<unknown>")) if len(snuba_param_list) > 1: query_results = list( _query_thread_pool.map(snuba_query, [ params + (Hub(Hub.current), ) for params in query_param_list ])) else: # No need to submit to the thread pool if we're just performing a # single query query_results = [ snuba_query(query_param_list[0] + (Hub(Hub.current), )) ] results = [] for response, _, reverse in query_results: try: body = json.loads(response.data) if SNUBA_INFO: if "sql" in body: logger.info("{}.sql: {}".format( headers.get("referer", "<unknown>"), body["sql"])) if "error" in body: logger.info("{}.err: {}".format( headers.get("referer", "<unknown>"), body["error"])) except ValueError: if response.status != 200: logger.error("snuba.query.invalid-json") raise SnubaError("Failed to parse snuba error response") raise UnexpectedResponseError( u"Could not decode JSON response: {}".format(response.data)) if response.status != 200: if body.get("error"): error = body["error"] if response.status == 429: raise RateLimitExceeded(error["message"]) elif error["type"] == "schema": raise SchemaValidationError(error["message"]) elif error["type"] == "clickhouse": raise clickhouse_error_codes_map.get( error["code"], QueryExecutionError)(error["message"]) else: raise SnubaError(error["message"]) else: raise SnubaError(u"HTTP {}".format(response.status)) # Forward and reverse translation maps from model ids to snuba keys, per column body["data"] = [reverse(d) for d in body["data"]] results.append(body) return results
def post(self, request, project, **kwargs): # Minidump request payloads do not have the same structure as # usual events from other SDKs. Most notably, the event needs # to be transfered in the `sentry` form field. All other form # fields are assumed "extra" information. The only exception # to this is `upload_file_minidump`, which contains the minidump. if any(key.startswith('sentry[') for key in request.POST): # First, try to parse the nested form syntax `sentry[key][key]` # This is required for the Breakpad client library, which only # supports string values of up to 64 characters. extra = parser.parse(request.POST.urlencode()) data = extra.pop('sentry', {}) else: # Custom clients can submit longer payloads and should JSON # encode event data into the optional `sentry` field. extra = request.POST json_data = extra.pop('sentry', None) data = json.loads(json_data[0]) if json_data else {} # Merge additional form fields from the request with `extra` # data from the event payload and set defaults for processing. extra.update(data.get('extra', {})) data['extra'] = extra # Assign our own UUID so we can track this minidump. We cannot trust the # uploaded filename, and if reading the minidump fails there is no way # we can ever retrieve the original UUID from the minidump. event_id = data.get('event_id') or uuid.uuid4().hex data['event_id'] = event_id # At this point, we only extract the bare minimum information # needed to continue processing. This requires to process the # minidump without symbols and CFI to obtain an initial stack # trace (most likely via stack scanning). If all validations # pass, the event will be inserted into the database. try: minidump = request.FILES['upload_file_minidump'] except KeyError: raise APIError('Missing minidump upload') # Breakpad on linux sometimes stores the entire HTTP request body as # dump file instead of just the minidump. The Electron SDK then for # example uploads a multipart formdata body inside the minidump file. # It needs to be re-parsed, to extract the actual minidump before # continuing. minidump.seek(0) if minidump.read(2) == b'--': # The remaining bytes of the first line are the form boundary. We # have already read two bytes, the remainder is the form boundary # (excluding the initial '--'). boundary = minidump.readline().rstrip() minidump.seek(0) # Next, we have to fake a HTTP request by specifying the form # boundary and the content length, or otherwise Django will not try # to parse our form body. Also, we need to supply new upload # handlers since they cannot be reused from the current request. meta = { 'CONTENT_TYPE': b'multipart/form-data; boundary=%s' % boundary, 'CONTENT_LENGTH': minidump.size, } handlers = [ uploadhandler.load_handler(handler, request) for handler in settings.FILE_UPLOAD_HANDLERS ] _, files = MultiPartParser(meta, minidump, handlers).parse() try: minidump = files['upload_file_minidump'] except KeyError: raise APIError('Missing minidump upload') if minidump.size == 0: raise APIError('Empty minidump upload received') if settings.SENTRY_MINIDUMP_CACHE: if not os.path.exists(settings.SENTRY_MINIDUMP_PATH): os.mkdir(settings.SENTRY_MINIDUMP_PATH, 0o744) with open('%s/%s.dmp' % (settings.SENTRY_MINIDUMP_PATH, event_id), 'wb') as out: for chunk in minidump.chunks(): out.write(chunk) # Always store the minidump in attachments so we can access it during # processing, regardless of the event-attachments feature. This will # allow us to stack walk again with CFI once symbols are loaded. attachments = [] minidump.seek(0) attachments.append(CachedAttachment.from_upload(minidump, type=MINIDUMP_ATTACHMENT_TYPE)) # Append all other files as generic attachments. We can skip this if the # feature is disabled since they won't be saved. if features.has('organizations:event-attachments', project.organization, actor=request.user): for name, file in six.iteritems(request.FILES): if name == 'upload_file_minidump': continue # Known attachment: msgpack event if name == "__sentry-event": merge_attached_event(file, data) continue if name == "__sentry-breadcrumb1" or name == "__sentry-breadcrumb2": merge_attached_breadcrumbs(file, data) continue # Add any other file as attachment attachments.append(CachedAttachment.from_upload(file)) try: state = process_minidump(minidump) merge_process_state_event(data, state) except ProcessMinidumpError as e: minidumps_logger.exception(e) raise APIError(e.message.split('\n', 1)[0]) event_id = self.process( request, attachments=attachments, data=data, project=project, **kwargs) # Return the formatted UUID of the generated event. This is # expected by the Electron http uploader on Linux and doesn't # break the default Breakpad client library. return HttpResponse( six.text_type(uuid.UUID(event_id)), content_type='text/plain' )
def __decode(self, value): return Report(*json.loads(zlib.decompress(value)))
def test_bot_message_im(self): resp = self.post_webhook(event_data=json.loads(MESSAGE_IM_BOT_EVENT)) assert resp.status_code == 200, resp.content
class GitHubAppsProviderTest(PluginTestCase): @fixture def provider(self): return GitHubAppsRepositoryProvider("github_apps") @patch.object( GitHubAppsClient, "get_repositories", return_value=json.loads(INTSTALLATION_REPOSITORIES_API_RESPONSE), ) @patch.object(GitHubClient, "get_installations", return_value=json.loads(LIST_INSTALLATION_API_RESPONSE)) def test_link_auth(self, *args): user = self.create_user() organization = self.create_organization() UserSocialAuth.objects.create(user=user, provider="github_apps", extra_data={"access_token": "abcdefg"}) integration = Integration.objects.create(provider="github_apps", external_id="1") self.provider.link_auth(user, organization, {"integration_id": integration.id}) assert OrganizationIntegration.objects.filter( organization=organization, integration=integration).exists() def test_delete_repository(self): user = self.create_user() organization = self.create_organization() integration = Integration.objects.create(provider="github_apps", external_id="1") repo = Repository.objects.create( name="example-repo", provider="github_apps", organization_id=organization.id, integration_id=integration.id, ) # just check that it doesn't throw / try to delete a webhook assert self.provider.delete_repository(repo=repo, actor=user) is None @patch.object(GitHubAppsClient, "get_last_commits", return_value=[]) def test_compare_commits_no_start(self, mock_get_last_commits): organization = self.create_organization() integration = Integration.objects.create(provider="github_apps", external_id="1") repo = Repository.objects.create( name="example-repo", provider="github_apps", organization_id=organization.id, integration_id=integration.id, config={"name": "example-repo"}, ) self.provider.compare_commits(repo, None, "a" * 40) assert mock_get_last_commits.called @patch.object(GitHubAppsClient, "compare_commits", return_value={"commits": []}) def test_compare_commits(self, mock_compare_commits): organization = self.create_organization() integration = Integration.objects.create(provider="github_apps", external_id="1") repo = Repository.objects.create( name="example-repo", provider="github_apps", organization_id=organization.id, integration_id=integration.id, config={"name": "example-repo"}, ) self.provider.compare_commits(repo, "b" * 40, "a" * 40) assert mock_compare_commits.called
def process_event(message, projects): payload = message["payload"] start_time = float(message["start_time"]) event_id = message["event_id"] project_id = int(message["project_id"]) remote_addr = message.get("remote_addr") attachments = message.get("attachments") or () # check that we haven't already processed this event (a previous instance of the forwarder # died before it could commit the event queue offset) deduplication_key = "ev:{}:{}".format(project_id, event_id) if cache.get(deduplication_key) is not None: logger.warning( "pre-process-forwarder detected a duplicated event" " with id:%s for project:%s.", event_id, project_id, ) return # message already processed do not reprocess try: project = projects[project_id] except KeyError: logger.error("Project for ingested event does not exist: %s", project_id) return # Parse the JSON payload. This is required to compute the cache key and # call process_event. The payload will be put into Kafka raw, to avoid # serializing it again. # XXX: Do not use CanonicalKeyDict here. This may break preprocess_event # which assumes that data passed in is a raw dictionary. data = json.loads(payload) cache_key = cache_key_for_event(data) default_cache.set(cache_key, data, CACHE_TIMEOUT) if attachments: attachment_objects = [ CachedAttachment(type=attachment.pop("attachment_type"), **attachment) for attachment in attachments ] attachment_cache.set(cache_key, attachments=attachment_objects, timeout=CACHE_TIMEOUT) # Preprocess this event, which spawns either process_event or # save_event. Pass data explicitly to avoid fetching it again from the # cache. preprocess_event(cache_key=cache_key, data=data, start_time=start_time, event_id=event_id, project=project) # remember for an 1 hour that we saved this event (deduplication protection) cache.set(deduplication_key, "", CACHE_TIMEOUT) # emit event_accepted once everything is done event_accepted.send_robust(ip=remote_addr, data=data, project=project, sender=process_event)
def request_callback(request): payload = json.loads(request.body) status_code = 400 if payload.get("refs") else 200 return (status_code, {}, json.dumps({}))
def load_data(platform, default=None): # NOTE: Before editing this data, make sure you understand the context # in which its being used. It is NOT only used for local development and # has production consequences. # * bin/load-mocks to generate fake data for local testing # * When a new project is created, a fake event is generated as a "starter" # event so it's not an empty project. # * When a user clicks Test Configuration from notification plugin settings page, # a fake event is generated to go through the pipeline. # now = datetime.now() data = None for platform in (platform, default): if platform is None: continue json_path = os.path.join(DATA_ROOT, 'samples', '%s.json' % (platform.encode('utf-8'), )) if not os.path.exists(json_path): continue with open(json_path) as fp: data = json.loads(fp.read()) break if data is None: return if platform == 'csp': return data data['platform'] = platform data['message'] = 'This is an example %s exception' % (platform, ) data['sentry.interfaces.User'] = { "username": "******", "id": "1671", "email": "*****@*****.**" } data['extra'] = { 'session': { 'foo': 'bar', }, 'results': [1, 2, 3, 4, 5], 'emptyList': [], 'emptyMap': {}, 'length': 10837790, 'unauthorized': False, 'url': 'http://example.org/foo/bar/', } data['modules'] = { 'my.package': '1.0.0', } data['sentry.interfaces.Http'] = { "cookies": 'foo=bar;biz=baz', "url": "http://example.com/foo", "headers": { "Referer": "http://example.com", "Content-Type": "application/json", "User-Agent": "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36" }, "env": { 'ENV': 'prod', }, "query_string": "foo=bar", "data": '{"hello": "world"}', "method": "GET" } # We can't send Breadcrumb data as a part of the sample event. # This gets used for all new projects as the "starter" event. # # data['sentry.interfaces.Breadcrumbs'] = { # "values": [ # { # "type": "navigation", # "dt": 8200, # "timestamp": milliseconds_ago(now, 5200), # "data": { # "to": "http://example.com/dashboard/", # "from": "http://example.com/login/" # } # }, # { # "type": "message", # "dt": 5000, # "timestamp": milliseconds_ago(now, 4000), # "data": { # "message": "This is a message breadcrumb", # "level": "info" # } # }, # { # "type": "message", # "dt": 4000, # "timestamp": milliseconds_ago(now, 3300), # "data": { # "message": "This is a warning message", # "level": "warning" # } # }, # { # "type": "message", # "dt": 3500, # "timestamp": milliseconds_ago(now, 2700), # "data": { # "message": "This is an error message", # "level": "error" # } # }, # { # "type": "http_request", # "dt": 3000, # "timestamp": milliseconds_ago(now, 1300), # "data": { # "url": "http://example.com/foo", # "statusCode": 200, # "method": "POST", # "headers": { # "Referer": "http://example.com", # "Content-Type": "application/json" # } # } # }, # { # "type": "ui_event", # "dt": 1500, # "timestamp": milliseconds_ago(now, 1000), # "data": { # "type": "click", # "target": "<button name=\"submit\" class=\"btn btn-small\"/>" # } # } # ] # } return data
def process_refresh_token_response(cls, response): return json.loads(response)
def raw_query(start, end, groupby=None, conditions=None, filter_keys=None, aggregations=None, rollup=None, arrayjoin=None, limit=None, offset=None, orderby=None, having=None, referrer=None, is_grouprelease=False, selected_columns=None, totals=None): """ Sends a query to snuba. `conditions`: A list of (column, operator, literal) conditions to be passed to the query. Conditions that we know will not have to be translated should be passed this way (eg tag[foo] = bar). `filter_keys`: A dictionary of {col: [key, ...]} that will be converted into "col IN (key, ...)" conditions. These are used to restrict the query to known sets of project/issue/environment/release etc. Appropriate translations (eg. from environment model ID to environment name) are performed on the query, and the inverse translation performed on the result. The project_id(s) to restrict the query to will also be automatically inferred from these keys. `aggregations` a list of (aggregation_function, column, alias) tuples to be passed to the query. """ # convert to naive UTC datetimes, as Snuba only deals in UTC # and this avoids offset-naive and offset-aware issues start = naiveify_datetime(start) end = naiveify_datetime(end) groupby = groupby or [] conditions = conditions or [] having = having or [] aggregations = aggregations or [] filter_keys = filter_keys or {} selected_columns = selected_columns or [] with timer('get_snuba_map'): forward, reverse = get_snuba_translators( filter_keys, is_grouprelease=is_grouprelease) if 'project_id' in filter_keys: # If we are given a set of project ids, use those directly. project_ids = filter_keys['project_id'] elif filter_keys: # Otherwise infer the project_ids from any related models with timer('get_related_project_ids'): ids = [ get_related_project_ids(k, filter_keys[k]) for k in filter_keys ] project_ids = list(set.union(*map(set, ids))) else: project_ids = [] for col, keys in six.iteritems(forward(filter_keys.copy())): if keys: if len(keys) == 1 and keys[0] is None: conditions.append((col, 'IS NULL', None)) else: conditions.append((col, 'IN', keys)) if not project_ids: raise SnubaError( "No project_id filter, or none could be inferred from other filters." ) # any project will do, as they should all be from the same organization project = Project.objects.get(pk=project_ids[0]) retention = quotas.get_event_retention( organization=Organization(project.organization_id)) if retention: start = max(start, datetime.utcnow() - timedelta(days=retention)) if start > end: raise QueryOutsideRetentionError # If the grouping, aggregation, or any of the conditions reference `issue` # we need to fetch the issue definitions (issue -> fingerprint hashes) aggregate_cols = [a[1] for a in aggregations] condition_cols = all_referenced_columns(conditions) all_cols = groupby + aggregate_cols + condition_cols + selected_columns get_issues = 'issue' in all_cols with timer('get_project_issues'): issues = get_project_issues( project_ids, filter_keys.get('issue')) if get_issues else None start, end = shrink_time_window(issues, start, end) # if `shrink_time_window` pushed `start` after `end` it means the user queried # a Group for T1 to T2 when the group was only active for T3 to T4, so the query # wouldn't return any results anyway if start > end: raise QueryOutsideGroupActivityError request = { k: v for k, v in six.iteritems({ 'from_date': start.isoformat(), 'to_date': end.isoformat(), 'conditions': conditions, 'having': having, 'groupby': groupby, 'totals': totals, 'project': project_ids, 'aggregations': aggregations, 'granularity': rollup, 'issues': issues, 'arrayjoin': arrayjoin, 'limit': limit, 'offset': offset, 'orderby': orderby, 'selected_columns': selected_columns, }) if v is not None } headers = {} if referrer: headers['referer'] = referrer try: with timer('snuba_query'): response = _snuba_pool.urlopen('POST', '/query', body=json.dumps(request), headers=headers) except urllib3.exceptions.HTTPError as err: raise SnubaError(err) try: body = json.loads(response.data) except ValueError: raise SnubaError(u"Could not decode JSON response: {}".format( response.data)) if response.status != 200: if body.get('error'): raise SnubaError(body['error']) else: raise SnubaError(u'HTTP {}'.format(response.status)) # Forward and reverse translation maps from model ids to snuba keys, per column body['data'] = [reverse(d) for d in body['data']] return body
def __decode(self, value): if value is None: return None return Report(*json.loads(zlib.decompress(value)))
def test_produce_step() -> None: topic = Topic("snuba-metrics") partition = Partition(topic, 0) clock = Clock() broker_storage: MemoryMessageStorage[KafkaPayload] = MemoryMessageStorage() broker: Broker[KafkaPayload] = Broker(broker_storage, clock) broker.create_topic(topic, partitions=1) producer = broker.get_producer() commit = Mock() produce_step = ProduceStep(commit_function=commit, producer=producer) message_payloads = [counter_payload, distribution_payload, set_payload] message_batch = [ Message( Partition(Topic("topic"), 0), i + 1, KafkaPayload( None, json.dumps(__translated_payload( message_payloads[i])).encode("utf-8"), []), datetime.now(), ) for i, payload in enumerate(message_payloads) ] # the outer message uses the last message's partition, offset, and timestamp last = message_batch[-1] outer_message = Message(last.partition, last.offset, message_batch, last.timestamp) # 1. Submit the message (that would have been generated from process_messages) produce_step.submit(outer_message=outer_message) # 2. Check that submit created the same number of futures as # messages in the outer_message (3 in this test). Also check # that the produced message payloads are as expected. assert len(produce_step._ProduceStep__futures) == 3 first_message = broker_storage.consume(partition, 0) assert first_message is not None second_message = broker_storage.consume(partition, 1) assert second_message is not None third_message = broker_storage.consume(partition, 2) assert third_message is not None assert broker_storage.consume(partition, 3) is None produced_messages = [ json.loads(msg.payload.value.decode("utf-8"), use_rapid_json=True) for msg in [first_message, second_message, third_message] ] expected_produced_messages = [] for payload in message_payloads: translated = __translated_payload(payload) tags: Mapping[str, int] = { str(k): v for k, v in translated["tags"].items() } translated.update(**{"tags": tags}) expected_produced_messages.append(translated) assert produced_messages == expected_produced_messages # 3. Call poll method, and check that doing so checked that # futures were ready and successful and therefore messages # were committed. produce_step.poll() expected_commit_calls = [ call({message.partition: Position(message.offset, message.timestamp)}) for message in message_batch ] assert commit.call_args_list == expected_commit_calls produce_step.close() produce_step.join()
def test_limit_as_2_with_paging(self): self.setup_project_and_rules() # Test Limit as 2, no cursor: with self.feature( ["organizations:incidents", "organizations:performance-view"]): request_data = {"per_page": "2", "project": self.project_ids} response = self.client.get(path=self.combined_rules_url, data=request_data, content_type="application/json") assert response.status_code == 200 result = json.loads(response.content) assert len(result) == 2 self.assert_alert_rule_serialized(self.yet_another_alert_rule, result[0], skip_dates=True) assert result[1]["id"] == str(self.issue_rule.id) assert result[1]["type"] == "rule" links = requests.utils.parse_header_links( response.get("link").rstrip(">").replace(">,<", ",<")) next_cursor = links[1]["cursor"] # Test Limit 2, next page of previous request: with self.feature( ["organizations:incidents", "organizations:performance-view"]): request_data = { "cursor": next_cursor, "per_page": "2", "project": self.project_ids } response = self.client.get(path=self.combined_rules_url, data=request_data, content_type="application/json") assert response.status_code == 200 result = json.loads(response.content) assert len(result) == 2 self.assert_alert_rule_serialized(self.other_alert_rule, result[0], skip_dates=True) self.assert_alert_rule_serialized(self.alert_rule, result[1], skip_dates=True) links = requests.utils.parse_header_links( response.get("link").rstrip(">").replace(">,<", ",<")) next_cursor = links[1]["cursor"] # Test Limit 2, next page of previous request - should get no results since there are only 4 total: with self.feature( ["organizations:incidents", "organizations:performance-view"]): request_data = { "cursor": next_cursor, "per_page": "2", "project": self.project_ids } response = self.client.get(path=self.combined_rules_url, data=request_data, content_type="application/json") assert response.status_code == 200 result = json.loads(response.content) assert len(result) == 0
def create_full_event(group, event_id='a', **kwargs): payload = """ { "event_id": "f5dd88e612bc406ba89dfebd09120769", "project": 11276, "release": "e1b5d1900526feaf20fe2bc9cad83d392136030a", "platform": "javascript", "culprit": "app/components/events/eventEntries in map", "logentry": {"formatted": "TypeError: Cannot read property '1' of null"}, "tags": [ ["environment", "prod"], ["sentry_version", "e1b5d1900526feaf20fe2bc9cad83d392136030a"], ["level", "error"], ["logger", "javascript"], ["sentry:release", "e1b5d1900526feaf20fe2bc9cad83d392136030a"], ["browser", "Chrome 48.0"], ["device", "Other"], ["os", "Windows 10"], ["url", "https://sentry.io/katon-direct/localhost/issues/112734598/"], ["sentry:user", "id:41656"] ], "errors": [{ "url": "<anonymous>", "type": "js_no_source" }], "extra": { "session:duration": 40364 }, "exception": { "exc_omitted": null, "values": [{ "stacktrace": { "frames": [{ "function": "batchedUpdates", "abs_path": "webpack:////usr/src/getsentry/src/sentry/~/react/lib/ReactUpdates.js", "pre_context": [" // verify that that's the case. (This is called by each top-level update", " // function, like setProps, setState, forceUpdate, etc.; creation and", " // destruction of top-level components is guarded in ReactMount.)", "", " if (!batchingStrategy.isBatchingUpdates) {"], "post_context": [" return;", " }", "", " dirtyComponents.push(component);", "}"], "filename": "~/react/lib/ReactUpdates.js", "module": "react/lib/ReactUpdates", "colno": 0, "in_app": false, "data": { "orig_filename": "/_static/29e365f8b0d923bc123e8afa38d890c3/sentry/dist/vendor.js", "orig_abs_path": "https://media.sentry.io/_static/29e365f8b0d923bc123e8afa38d890c3/sentry/dist/vendor.js", "sourcemap": "https://media.sentry.io/_static/29e365f8b0d923bc123e8afa38d890c3/sentry/dist/vendor.js.map", "orig_lineno": 37, "orig_function": "Object.s [as enqueueUpdate]", "orig_colno": 16101 }, "context_line": " batchingStrategy.batchedUpdates(enqueueUpdate, component);", "lineno": 176 }], "frames_omitted": null }, "type": "TypeError", "value": "Cannot read property '1' of null", "module": null }] }, "request": { "url": "https://sentry.io/katon-direct/localhost/issues/112734598/", "headers": [ ["Referer", "https://sentry.io/welcome/"], ["User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36"] ] }, "user": { "ip_address": "0.0.0.0", "id": "41656", "email": "*****@*****.**" }, "version": "7", "breadcrumbs": { "values": [ { "category": "xhr", "timestamp": 1496395011.63, "type": "http", "data": { "url": "/api/path/here", "status_code": "500", "method": "POST" } } ] } }""" event = Factories.create_event( group=group, event_id=event_id, platform='javascript', data=json.loads(payload), # This payload already went through sourcemap # processing, normalizing it would remove # frame.data (orig_filename, etc) normalize=False) return event
def assert_response_json(response, data): """ Normalizes unicode strings by encoding/decoding expected output """ assert json.loads(response.content) == json.loads(json.dumps(data))
def load_data(platform, default=None, sample_name=None): # NOTE: Before editing this data, make sure you understand the context # in which its being used. It is NOT only used for local development and # has production consequences. # * bin/load-mocks to generate fake data for local testing # * When a new project is created, a fake event is generated as a "starter" # event so it's not an empty project. # * When a user clicks Test Configuration from notification plugin settings page, # a fake event is generated to go through the pipeline. data = None language = None platform_data = INTEGRATION_ID_TO_PLATFORM_DATA.get(platform) if platform_data is not None and platform_data["type"] != "language": language = platform_data["language"] for platform in (platform, language, default): if not platform: continue json_path = os.path.join(DATA_ROOT, "samples", "%s.json" % (platform.encode("utf-8"), )) if not os.path.exists(json_path): continue if not sample_name: try: sample_name = INTEGRATION_ID_TO_PLATFORM_DATA[platform]["name"] except KeyError: pass with open(json_path) as fp: data = json.loads(fp.read()) break if data is None: return data = CanonicalKeyDict(data) if platform in ("csp", "hkpk", "expectct", "expectstaple"): return data # Transaction events need timestamp data set to something current. if platform == "transaction": now = timezone.now() now_time = to_timestamp(now) start_time = to_timestamp(now - timedelta(seconds=2)) data.setdefault("timestamp", now_time) data.setdefault("start_timestamp", start_time) for span in data["spans"]: span.setdefault("timestamp", now_time) span.setdefault("start_timestamp", start_time) data["platform"] = platform # XXX: Message is a legacy alias for logentry. Do not overwrite if set. if "message" not in data: data["message"] = "This is an example %s exception" % (sample_name or platform, ) data.setdefault( "user", generate_user(ip_address="127.0.0.1", username="******", id=1, email="*****@*****.**"), ) data.setdefault( "extra", { "session": { "foo": "bar" }, "results": [1, 2, 3, 4, 5], "emptyList": [], "emptyMap": {}, "length": 10837790, "unauthorized": False, "url": "http://example.org/foo/bar/", }, ) data.setdefault("modules", {"my.package": "1.0.0"}) data.setdefault( "request", { "cookies": "foo=bar;biz=baz", "url": "http://example.com/foo", "headers": { "Referer": "http://example.com", "Content-Type": "application/json", "User-Agent": "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36", }, "env": { "ENV": "prod" }, "query_string": "foo=bar", "data": '{"hello": "world"}', "method": "GET", }, ) return data
def load_data(platform, default=None, sample_name=None): # NOTE: Before editing this data, make sure you understand the context # in which its being used. It is NOT only used for local development and # has production consequences. # * bin/load-mocks to generate fake data for local testing # * When a new project is created, a fake event is generated as a "starter" # event so it's not an empty project. # * When a user clicks Test Configuration from notification plugin settings page, # a fake event is generated to go through the pipeline. data = None language = None platform_data = INTEGRATION_ID_TO_PLATFORM_DATA.get(platform) if platform_data is not None and platform_data['type'] != 'language': language = platform_data['language'] for platform in (platform, language, default): if not platform: continue json_path = os.path.join(DATA_ROOT, 'samples', '%s.json' % (platform.encode('utf-8'), )) if not os.path.exists(json_path): continue if not sample_name: try: sample_name = INTEGRATION_ID_TO_PLATFORM_DATA[platform]['name'] except KeyError: pass with open(json_path) as fp: data = json.loads(fp.read()) break if data is None: return data = CanonicalKeyDict(data) if platform in ('csp', 'hkpk', 'expectct', 'expectstaple'): return data data['platform'] = platform data['message'] = 'This is an example %s exception' % (sample_name or platform, ) data['user'] = generate_user( ip_address='127.0.0.1', username='******', id=1, email='*****@*****.**', ) data['extra'] = { 'session': { 'foo': 'bar', }, 'results': [1, 2, 3, 4, 5], 'emptyList': [], 'emptyMap': {}, 'length': 10837790, 'unauthorized': False, 'url': 'http://example.org/foo/bar/', } data['modules'] = { 'my.package': '1.0.0', } data['request'] = { "cookies": 'foo=bar;biz=baz', "url": "http://example.com/foo", "headers": { "Referer": "http://example.com", "Content-Type": "application/json", "User-Agent": "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36" }, "env": { 'ENV': 'prod', }, "query_string": "foo=bar", "data": '{"hello": "world"}', "method": "GET" } return data
def test_basic_resolving(self): url = reverse( "sentry-api-0-dsym-files", kwargs={ "organization_slug": self.project.organization.slug, "project_slug": self.project.slug, }, ) self.login_as(user=self.user) out = BytesIO() f = zipfile.ZipFile(out, "w") f.writestr("proguard/%s.txt" % PROGUARD_UUID, PROGUARD_SOURCE) f.writestr("ignored-file.txt", b"This is just some stuff") f.close() response = self.client.post( url, { "file": SimpleUploadedFile("symbols.zip", out.getvalue(), content_type="application/zip") }, format="multipart", ) assert response.status_code == 201, response.content assert len(response.data) == 1 event_data = { "user": { "ip_address": "31.172.207.97" }, "extra": {}, "project": self.project.id, "platform": "java", "debug_meta": { "images": [{ "type": "proguard", "uuid": PROGUARD_UUID }] }, "exception": { "values": [{ "stacktrace": { "frames": [ { "function": "a", "abs_path": None, "module": "org.a.b.g$a", "filename": None, "lineno": 67, }, { "function": "a", "abs_path": None, "module": "org.a.b.g$a", "filename": None, "lineno": 69, }, ] }, "type": "RuntimeException", "value": "Shit broke yo", }] }, "timestamp": iso_format(before_now(seconds=1)), } # We do a preflight post, because there are many queries polluting the array # before the actual "processing" happens (like, auth_user) resp = self._postWithHeader(event_data) with self.assertWriteQueries({ "nodestore_node": 2, "sentry_eventuser": 1, "sentry_groupedmessage": 1, "sentry_userreport": 1, }): self._postWithHeader(event_data) assert resp.status_code == 200 event_id = json.loads(resp.content)["id"] event = eventstore.get_event_by_id(self.project.id, event_id) bt = event.interfaces["exception"].values[0].stacktrace frames = bt.frames assert frames[0].function == "getClassContext" assert frames[ 0].module == "org.slf4j.helpers.Util$ClassContextSecurityManager" assert frames[1].function == "getExtraClassContext" assert frames[ 1].module == "org.slf4j.helpers.Util$ClassContextSecurityManager" assert event.culprit == ( "org.slf4j.helpers.Util$ClassContextSecurityManager " "in getExtraClassContext")
def test_send_alert_event(self, safe_urlopen): event = self.store_event(data={}, project_id=self.project.id) group = event.group rule_future = RuleFuture(rule=self.rule, kwargs={"sentry_app": self.sentry_app}) with self.tasks(): notify_sentry_app(event, [rule_future]) data = json.loads(faux(safe_urlopen).kwargs["data"]) assert data == { "action": "triggered", "installation": { "uuid": self.install.uuid }, "data": { "event": DictContaining( event_id=event.event_id, url=absolute_uri( reverse( "sentry-api-0-project-event-details", args=[ self.organization.slug, self.project.slug, event.event_id ], )), web_url=absolute_uri( reverse( "sentry-organization-event-detail", args=[ self.organization.slug, group.id, event.event_id ], )), issue_url=absolute_uri(f"/api/0/issues/{group.id}/"), issue_id=str(group.id), ), "triggered_rule": self.rule.label, }, "actor": { "type": "application", "id": "sentry", "name": "Sentry" }, } assert faux(safe_urlopen).kwarg_equals( "headers", DictContaining( "Content-Type", "Request-ID", "Sentry-Hook-Resource", "Sentry-Hook-Timestamp", "Sentry-Hook-Signature", ), ) buffer = SentryAppWebhookRequestsBuffer(self.sentry_app) requests = buffer.get_requests() assert len(requests) == 1 assert requests[0]["response_code"] == 200 assert requests[0]["event_type"] == "event_alert.triggered"
def test_issue_alert_team_fallback(self, mock_func): """Test that issue alerts are sent to each member of a team in Slack.""" user2 = self.create_user(is_superuser=False) self.create_member(teams=[self.team], user=user2, organization=self.organization) self.identity = Identity.objects.create( external_id="UXXXXXXX2", idp=self.idp, user=user2, status=IdentityStatus.VALID, scopes=[], ) NotificationSetting.objects.update_settings( ExternalProviders.SLACK, NotificationSettingTypes.ISSUE_ALERTS, NotificationSettingOptionValues.ALWAYS, user=user2, ) event = self.store_event(data={ "message": "Hello world", "level": "error" }, project_id=self.project.id) action_data = { "id": "sentry.mail.actions.NotifyEmailAction", "targetType": "Team", "targetIdentifier": str(self.team.id), } rule = Rule.objects.create( project=self.project, label="ja rule", data={ "match": "all", "actions": [action_data], }, ) notification = Notification(event=event, rule=rule) with self.options({"system.url-prefix": "http://example.com"}), self.tasks(): self.adapter.notify(notification, ActionTargetType.TEAM, self.team.id) assert len(responses.calls) == 2 # check that self.user got a notification data = parse_qs(responses.calls[0].request.body) assert data["channel"] == ["UXXXXXXX1"] assert "attachments" in data attachments = json.loads(data["attachments"][0]) assert len(attachments) == 1 assert attachments[0]["title"] == "Hello world" assert ( attachments[0]["footer"] == f"{self.project.slug} | <http://example.com/settings/account/notifications/alerts/?referrer=AlertRuleSlack|Notification Settings>" ) # check that user2 got a notification as well data2 = parse_qs(responses.calls[1].request.body) assert data2["channel"] == ["UXXXXXXX2"] assert "attachments" in data2 attachments = json.loads(data2["attachments"][0]) assert len(attachments) == 1 assert attachments[0]["title"] == "Hello world" assert ( attachments[0]["footer"] == f"{self.project.slug} | <http://example.com/settings/account/notifications/alerts/?referrer=AlertRuleSlack|Notification Settings>" )
def test_error_on_resolving(self): url = reverse( "sentry-api-0-dsym-files", kwargs={ "organization_slug": self.project.organization.slug, "project_slug": self.project.slug, }, ) self.login_as(user=self.user) out = BytesIO() f = zipfile.ZipFile(out, "w") f.writestr("proguard/%s.txt" % PROGUARD_BUG_UUID, PROGUARD_BUG_SOURCE) f.close() response = self.client.post( url, { "file": SimpleUploadedFile("symbols.zip", out.getvalue(), content_type="application/zip") }, format="multipart", ) assert response.status_code == 201, response.content assert len(response.data) == 1 event_data = { "user": { "ip_address": "31.172.207.97" }, "extra": {}, "project": self.project.id, "platform": "java", "debug_meta": { "images": [{ "type": "proguard", "uuid": PROGUARD_BUG_UUID }] }, "exception": { "values": [{ "stacktrace": { "frames": [ { "function": "a", "abs_path": None, "module": "org.a.b.g$a", "filename": None, "lineno": 67, }, { "function": "a", "abs_path": None, "module": "org.a.b.g$a", "filename": None, "lineno": 69, }, ] }, "type": "RuntimeException", "value": "Shit broke yo", }] }, "timestamp": iso_format(before_now(seconds=1)), } resp = self._postWithHeader(event_data) assert resp.status_code == 200 event_id = json.loads(resp.content)["id"] event = eventstore.get_event_by_id(self.project.id, event_id) assert len(event.data["errors"]) == 1 assert event.data["errors"][0] == { "mapping_uuid": u"071207ac-b491-4a74-957c-2c94fd9594f2", "type": "proguard_missing_lineno", }
def test_sync_assignee_outbound_with_paging(self): vsts_work_item_id = 5 responses.add( responses.PATCH, "https://fabrikam-fiber-inc.visualstudio.com/_apis/wit/workitems/%d" % vsts_work_item_id, body=WORK_ITEM_RESPONSE, content_type="application/json", ) responses.add( responses.GET, "https://fabrikam-fiber-inc.vssps.visualstudio.com/_apis/graph/users", json={ "value": [ { "mailAddress": "*****@*****.**" }, { "mailAddress": "*****@*****.**" }, { "mailAddress": "*****@*****.**" }, ] }, headers={"X-MS-ContinuationToken": "continuation-token"}, match_querystring=True, ) responses.add( responses.GET, "https://fabrikam-fiber-inc.vssps.visualstudio.com/_apis/graph/users?continuationToken=continuation-token", body=GET_USERS_RESPONSE, content_type="application/json", match_querystring=True, ) user = self.create_user("*****@*****.**") external_issue = ExternalIssue.objects.create( organization_id=self.organization.id, integration_id=self.integration.model.id, key=vsts_work_item_id, title="I'm a title!", description="I'm a description.", ) self.integration.sync_assignee_outbound(external_issue, user, assign=True) assert len(responses.calls) == 3 assert ( responses.calls[0].request.url == "https://fabrikam-fiber-inc.vssps.visualstudio.com/_apis/graph/users" ) assert responses.calls[0].response.status_code == 200 assert ( responses.calls[1].request.url == "https://fabrikam-fiber-inc.vssps.visualstudio.com/_apis/graph/users?continuationToken=continuation-token" ) assert responses.calls[1].response.status_code == 200 assert ( responses.calls[2].request.url == "https://fabrikam-fiber-inc.visualstudio.com/_apis/wit/workitems/%d" % vsts_work_item_id) request_body = json.loads(responses.calls[2].request.body) assert len(request_body) == 1 assert request_body[0]["path"] == "/fields/System.AssignedTo" assert request_body[0]["value"] == "*****@*****.**" assert request_body[0]["op"] == "replace" assert responses.calls[2].response.status_code == 200
def raw_query(start, end, groupby=None, conditions=None, filter_keys=None, aggregations=None, rollup=None, referrer=None, is_grouprelease=False, **kwargs): """ Sends a query to snuba. `start` and `end`: The beginning and end of the query time window (required) `groupby`: A list of column names to group by. `conditions`: A list of (column, operator, literal) conditions to be passed to the query. Conditions that we know will not have to be translated should be passed this way (eg tag[foo] = bar). `filter_keys`: A dictionary of {col: [key, ...]} that will be converted into "col IN (key, ...)" conditions. These are used to restrict the query to known sets of project/issue/environment/release etc. Appropriate translations (eg. from environment model ID to environment name) are performed on the query, and the inverse translation performed on the result. The project_id(s) to restrict the query to will also be automatically inferred from these keys. `aggregations` a list of (aggregation_function, column, alias) tuples to be passed to the query. The rest of the args are passed directly into the query JSON unmodified. See the snuba schema for details. """ # convert to naive UTC datetimes, as Snuba only deals in UTC # and this avoids offset-naive and offset-aware issues start = naiveify_datetime(start) end = naiveify_datetime(end) groupby = groupby or [] conditions = conditions or [] aggregations = aggregations or [] filter_keys = filter_keys or {} with timer('get_snuba_map'): forward, reverse = get_snuba_translators( filter_keys, is_grouprelease=is_grouprelease) if 'project_id' in filter_keys: # If we are given a set of project ids, use those directly. project_ids = list(set(filter_keys['project_id'])) elif filter_keys: # Otherwise infer the project_ids from any related models with timer('get_related_project_ids'): ids = [ get_related_project_ids(k, filter_keys[k]) for k in filter_keys ] project_ids = list(set.union(*map(set, ids))) else: project_ids = [] for col, keys in six.iteritems(forward(filter_keys.copy())): if keys: if len(keys) == 1 and None in keys: conditions.append((col, 'IS NULL', None)) else: conditions.append((col, 'IN', keys)) if not project_ids: raise UnqualifiedQueryError( "No project_id filter, or none could be inferred from other filters." ) # any project will do, as they should all be from the same organization project = Project.objects.get(pk=project_ids[0]) retention = quotas.get_event_retention( organization=Organization(project.organization_id)) if retention: start = max(start, datetime.utcnow() - timedelta(days=retention)) if start > end: raise QueryOutsideRetentionError # if `shrink_time_window` pushed `start` after `end` it means the user queried # a Group for T1 to T2 when the group was only active for T3 to T4, so the query # wouldn't return any results anyway new_start = shrink_time_window(filter_keys.get('issue'), start) # TODO (alexh) this is a quick emergency fix for an occasion where a search # results in only 1 django candidate, which is then passed to snuba to # check and we raised because of it. Remove this once we figure out why the # candidate was returned from django at all if it existed only outside the # time range of the query if new_start <= end: start = new_start if start > end: raise QueryOutsideGroupActivityError kwargs.update({ 'from_date': start.isoformat(), 'to_date': end.isoformat(), 'groupby': groupby, 'conditions': conditions, 'aggregations': aggregations, 'project': project_ids, 'granularity': rollup, # TODO name these things the same }) kwargs = {k: v for k, v in six.iteritems(kwargs) if v is not None} kwargs.update(OVERRIDE_OPTIONS) headers = {} if referrer: headers['referer'] = referrer try: with timer('snuba_query'): response = _snuba_pool.urlopen('POST', '/query', body=json.dumps(kwargs), headers=headers) except urllib3.exceptions.HTTPError as err: raise SnubaError(err) try: body = json.loads(response.data) except ValueError: raise UnexpectedResponseError( u"Could not decode JSON response: {}".format(response.data)) if response.status != 200: if body.get('error'): error = body['error'] if response.status == 429: raise RateLimitExceeded(error['message']) elif error['type'] == 'schema': raise SchemaValidationError(error['message']) elif error['type'] == 'clickhouse': raise clickhouse_error_codes_map.get( error['code'], QueryExecutionError, )(error['message']) else: raise SnubaError(error['message']) else: raise SnubaError(u'HTTP {}'.format(response.status)) # Forward and reverse translation maps from model ids to snuba keys, per column body['data'] = [reverse(d) for d in body['data']] return body
def post(self, request, organization, version): """ Handle an artifact bundle and merge it into the release ``````````````````````````````````````````````````````` :auth: required """ try: release = Release.objects.get(organization_id=organization.id, version=version) except Release.DoesNotExist: raise ResourceDoesNotExist if not self.has_release_permission(request, organization, release): raise ResourceDoesNotExist schema = { "type": "object", "properties": { "checksum": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, "chunks": { "type": "array", "items": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, }, }, "required": ["checksum", "chunks"], "additionalProperties": False, } try: data = json.loads(request.body) jsonschema.validate(data, schema) except jsonschema.ValidationError as e: return Response({"error": str(e).splitlines()[0]}, status=400) except BaseException: return Response({"error": "Invalid json body"}, status=400) checksum = data.get("checksum", None) chunks = data.get("chunks", []) state, detail = get_assemble_status(AssembleTask.ARTIFACTS, organization.id, checksum) if state == ChunkFileState.OK: return Response({"state": state, "detail": None, "missingChunks": []}, status=200) elif state is not None: return Response({"state": state, "detail": detail, "missingChunks": []}) # There is neither a known file nor a cached state, so we will # have to create a new file. Assure that there are checksums. # If not, we assume this is a poll and report NOT_FOUND if not chunks: return Response({"state": ChunkFileState.NOT_FOUND, "missingChunks": []}, status=200) set_assemble_status( AssembleTask.ARTIFACTS, organization.id, checksum, ChunkFileState.CREATED ) from sentry.tasks.assemble import assemble_artifacts assemble_artifacts.apply_async( kwargs={ "org_id": organization.id, "version": version, "checksum": checksum, "chunks": chunks, } ) return Response({"state": ChunkFileState.CREATED, "missingChunks": []}, status=200)
def _read_manifest(self) -> dict: manifest_bytes = self.read("manifest.json") return json.loads(manifest_bytes.decode("utf-8"))
def test_issue_alert_team_new_project(self, mock_func): """Test that issue alerts are sent to a team in Slack when the team has added a new project""" # add a second user to the team so we can be sure it's only # sent once (to the team, and not to each individual user) user2 = self.create_user(is_superuser=False) self.create_member(teams=[self.team], user=user2, organization=self.organization) self.idp = IdentityProvider.objects.create(type="slack", external_id="TXXXXXXX2", config={}) self.identity = Identity.objects.create( external_id="UXXXXXXX2", idp=self.idp, user=user2, status=IdentityStatus.VALID, scopes=[], ) NotificationSetting.objects.update_settings( ExternalProviders.SLACK, NotificationSettingTypes.ISSUE_ALERTS, NotificationSettingOptionValues.ALWAYS, user=user2, ) # update the team's notification settings ExternalActor.objects.create( actor=self.team.actor, organization=self.organization, integration=self.integration, provider=ExternalProviders.SLACK.value, external_name="goma", external_id="CXXXXXXX2", ) NotificationSetting.objects.update_settings( ExternalProviders.SLACK, NotificationSettingTypes.ISSUE_ALERTS, NotificationSettingOptionValues.ALWAYS, team=self.team, ) # add a new project project2 = self.create_project(name="hellboy", organization=self.organization, teams=[self.team]) event = self.store_event(data={ "message": "Hello world", "level": "error" }, project_id=project2.id) action_data = { "id": "sentry.mail.actions.NotifyEmailAction", "targetType": "Team", "targetIdentifier": str(self.team.id), } rule = Rule.objects.create( project=project2, label="ja rule", data={ "match": "all", "actions": [action_data], }, ) notification = Notification(event=event, rule=rule) with self.options({"system.url-prefix": "http://example.com"}), self.tasks(): self.adapter.notify(notification, ActionTargetType.TEAM, self.team.id) # check that only one was sent out - more would mean each user is being notified # rather than the team assert len(responses.calls) == 1 # check that the team got a notification data = parse_qs(responses.calls[0].request.body) assert data["channel"] == ["CXXXXXXX2"] assert "attachments" in data attachments = json.loads(data["attachments"][0]) assert len(attachments) == 1 assert attachments[0]["title"] == "Hello world" assert ( attachments[0]["footer"] == f"{project2.slug} | <http://example.com/settings/{self.organization.slug}/teams/{self.team.slug}/notifications/?referrer=AlertRuleSlack|Notification Settings>" )
def get_value(self): key = self._get_redis_key() value = self.client.get(key) return json.loads(value)
def load_data(platform, default=None, timestamp=None): # NOTE: Before editing this data, make sure you understand the context # in which its being used. It is NOT only used for local development and # has production consequences. # * bin/load-mocks to generate fake data for local testing # * When a new project is created, a fake event is generated as a "starter" # event so it's not an empty project. # * When a user clicks Test Configuration from notification plugin settings page, # a fake event is generated to go through the pipeline. data = None for platform in (platform, default): if platform is None: continue json_path = os.path.join(DATA_ROOT, 'samples', '%s.json' % (platform.encode('utf-8'), )) if not os.path.exists(json_path): continue with open(json_path) as fp: data = json.loads(fp.read()) break if data is None: return if platform == 'csp': return data data['platform'] = platform data['message'] = 'This is an example %s exception' % (platform, ) data['sentry.interfaces.User'] = { "username": "******", "id": "1671", "email": "*****@*****.**" } data['extra'] = { 'session': { 'foo': 'bar', }, 'results': [1, 2, 3, 4, 5], 'emptyList': [], 'emptyMap': {}, 'length': 10837790, 'unauthorized': False, 'url': 'http://example.org/foo/bar/', } data['modules'] = { 'my.package': '1.0.0', } data['sentry.interfaces.Http'] = { "cookies": 'foo=bar;biz=baz', "url": "http://example.com/foo", "headers": { "Referer": "http://example.com", "Content-Type": "application/json", "User-Agent": "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36" }, "env": { 'ENV': 'prod', }, "query_string": "foo=bar", "data": '{"hello": "world"}', "method": "GET" } start = datetime.utcnow() if timestamp: try: start = datetime.utcfromtimestamp(timestamp) except TypeError: pass # Make breadcrumb timestamps relative to right now so they make sense breadcrumbs = data.get('sentry.interfaces.Breadcrumbs') if breadcrumbs is not None: duration = 1000 values = breadcrumbs['values'] for value in reversed(values): value['timestamp'] = milliseconds_ago(start, duration) # Every breadcrumb is 1s apart duration += 1000 return data
def post(self, request, project): """ Assmble one or multiple chunks (FileBlob) into debug files `````````````````````````````````````````````````````````` :auth: required """ schema = { "type": "object", "patternProperties": { "^[0-9a-f]{40}$": { "type": "object", "required": ["name", "chunks"], "properties": { "name": { "type": "string" }, "chunks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": False } }, "additionalProperties": False } try: files = json.loads(request.body) jsonschema.validate(files, schema) except jsonschema.ValidationError as e: return Response({'error': str(e).splitlines()[0]}, status=400) except BaseException as e: return Response({'error': 'Invalid json body'}, status=400) file_response = {} from sentry.tasks.assemble import assemble_dif for checksum, file_to_assemble in six.iteritems(files): name = file_to_assemble.get('name', None) chunks = file_to_assemble.get('chunks', []) # First, check the cached assemble status. During assembling, a # ProjectDebugFile will be created and we need to prevent a race # condition. state, detail = get_assemble_status(project, checksum) if state is not None: file_response[checksum] = { 'state': state, 'detail': detail, 'missingChunks': [], } continue # Next, check if this project already owns the ProjectDebugFile. # This can under rare circumstances yield more than one file # which is why we use first() here instead of get(). dif = ProjectDebugFile.objects \ .filter(project=project, file__checksum=checksum) \ .select_related('file') \ .order_by('-id') \ .first() if dif is not None: file_response[checksum] = { 'state': ChunkFileState.OK, 'detail': None, 'missingChunks': [], 'dif': serialize(dif), } continue # There is neither a known file nor a cached state, so we will # have to create a new file. Assure that there are checksums. # If not, we assume this is a poll and report NOT_FOUND if not chunks: file_response[checksum] = { 'state': ChunkFileState.NOT_FOUND, 'missingChunks': [], } continue # Check if all requested chunks have been uploaded. missing_chunks = find_missing_chunks(project.organization, chunks) if missing_chunks: file_response[checksum] = { 'state': ChunkFileState.NOT_FOUND, 'missingChunks': missing_chunks, } continue # We don't have a state yet, this means we can now start # an assemble job in the background. set_assemble_status(project, checksum, state) assemble_dif.apply_async( kwargs={ 'project_id': project.id, 'name': name, 'checksum': checksum, 'chunks': chunks, }) file_response[checksum] = { 'state': ChunkFileState.CREATED, 'missingChunks': [], } return Response(file_response, status=200)