def capture_exception(*args, **kwargs): exc_info = sys.exc_info() if not exc_info[0] is capnp.lib.capnp.KjException: sentry_sdk.capture_exception(*args, **kwargs) sentry_sdk.flush( ) # https://github.com/getsentry/sentry-python/issues/291 cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1))
def submit(self, data: dict, exception: Optional[Exception]) -> bool: """ Temporarily enables sentry integration to send error and then disables it again. """ try: sentry_sdk.init( self._sentryDsn, release=self._version, attach_stacktrace=True, with_locals=True, ) with configure_scope() as scope: scope.level = "error" scope.user = {"id": self._profile.id} if "debug" in data and type(data["debug"]) is dict: for k, v in data["debug"].items(): scope.set_extra(k, v) if "message" in data: scope.set_extra("message", data["message"]) if isinstance(exception, Exception): capture_exception(exception) elif "message" in data: capture_message(data["message"], "error") sentry_sdk.flush() return True except Exception as e: # something went wrong safePrint(e) return False finally: # Calling init with empty string disables sentry again # Otherwise we would log other Anki exceptions sentry_sdk.init("")
def closed(self, reason): if self.settings['SENTRY_DSN']: with sentry_sdk.push_scope() as scope: scope.set_extra("reason", reason) sentry_sdk.capture_message('Scraper %s finished' % (self.name,)) sentry_sdk.flush()
def save_article(self, article: dict, to: Union[Collection, str], push_lowercase_to_meta=True): """ Save a processed article. Capitalized fields will be saved as is. Others will be treated as scrapy meta. :param article: The processed article item. :param to: The collection to save to. :param push_lowercase_to_meta: Whether the lower case keys should be pushed into meta document. :return: """ if push_lowercase_to_meta: meta_dict = {} for key in list(article): if key[0].islower(): meta_dict[key] = article[key] del article[key] article['_scrapy_meta'] = meta_dict article['last_updated'] = datetime.now() result = self.get_col(to).insert_one(article) if self.settings['SENTRY_DSN']: with sentry_sdk.push_scope() as scope: scope.set_extra('item_id', str(result.inserted_id)) sentry_sdk.capture_message('Scraper %s: Article download' % (self.name,)) sentry_sdk.flush()
def save_pdf(self, pdf_bytes, pdf_fn, pdf_link, fs: Union[gridfs.GridFS, str]): """ Process PDF bytes and save it into a GridFS collection. :param pdf_bytes: Bytes data of PDF file. :param pdf_fn: PDF filename. :param pdf_link: Link to PDF file. :param fs: GridFS in which PDF files are saved, or a name. :return: The ObjectId for this object in the GridFS. """ parsing_result = self.parse_pdf(pdf_bytes, pdf_fn) meta = parsing_result.copy() meta.update({ 'filename': pdf_fn, 'page_link': pdf_link, }) file_id = self.get_gridfs(fs).put(pdf_bytes, **meta) if self.settings['SENTRY_DSN']: with sentry_sdk.push_scope() as scope: scope.set_extra('filename', pdf_fn) scope.set_extra('page_link', pdf_link) scope.set_extra('file_id', str(file_id)) sentry_sdk.capture_message('Scraper %s: PDF download' % (self.name,)) sentry_sdk.flush() return file_id
def test_aggregates_explicitly_disabled_session_tracking_request_mode( sentry_init, capture_envelopes): sentry_init(release="fun-release", environment="not-fun-env", auto_session_tracking=False) envelopes = capture_envelopes() hub = Hub.current with auto_session_tracking(session_mode="request"): with sentry_sdk.push_scope(): try: raise Exception("all is wrong") except Exception: sentry_sdk.capture_exception() with auto_session_tracking(session_mode="request"): pass hub.start_session(session_mode="request") hub.end_session() sentry_sdk.flush() sess = envelopes[1] assert len(sess.items) == 1 sess_event = sess.items[0].payload.json aggregates = sorted_aggregates(sess_event) assert len(aggregates) == 1 assert aggregates[0]["exited"] == 1 assert "errored" not in aggregates[0]
def test_session_mode_defaults_to_request_mode_in_wsgi_handler( capture_envelopes, sentry_init): """ Test that ensures that even though the default `session_mode` for auto_session_tracking is `application`, that flips to `request` when we are in the WSGI handler """ def app(environ, start_response): start_response("200 OK", []) return ["Go get the ball! Good dog!"] traces_sampler = mock.Mock(return_value=True) sentry_init(send_default_pii=True, traces_sampler=traces_sampler) app = SentryWsgiMiddleware(app) envelopes = capture_envelopes() client = Client(app) client.get("/dogs/are/great/") sentry_sdk.flush() sess = envelopes[1] assert len(sess.items) == 1 sess_event = sess.items[0].payload.json aggregates = sess_event["aggregates"] assert len(aggregates) == 1 assert aggregates[0]["exited"] == 1
def report_tombstone(fn: str, message: str, contents: str) -> None: cloudlog.error({'tombstone': message}) with sentry_sdk.configure_scope() as scope: scope.set_extra("tombstone_fn", fn) scope.set_extra("tombstone", contents) sentry_sdk.capture_message(message=message) sentry_sdk.flush()
def sentry_report(fn, message, contents): cloudlog.error({'tombstone': message}) with sentry_sdk.configure_scope() as scope: scope.set_extra("tombstone_fn", fn) scope.set_extra("tombstone", contents) sentry_sdk.capture_message(message=message) sentry_sdk.flush()
def wrapper(*args, **kwargs): with hub: try: return func(*args, **kwargs) except: sentry_sdk.capture_exception() sentry_sdk.flush() raise
def capture_exception(*args, **kwargs) -> None: cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1)) try: sentry_sdk.capture_exception(*args, **kwargs) sentry_sdk.flush() # https://github.com/getsentry/sentry-python/issues/291 except Exception: cloudlog.exception("sentry exception")
def capture_exception(err: Exception, level: str = "error", extra_tags: dict = {}): """ Capture an exception. Optionally set the log level of the event. Extra tags accepted. """ with sentry_sdk.push_scope() as scope: scope.set_level(level) for k, v in extra_tags: scope.set_tag(k, v) sentry_sdk.capture_exception(err, scope) sentry_sdk.flush()
def capture_exception(*args, **kwargs) -> None: cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1)) try: with open('/data/log/last_exception', 'w') as f: now = datetime.datetime.now() f.write( now.strftime('[%Y-%m-%d %H:%M:%S]') + "\n\n" + str(traceback.format_exc())) except Exception: pass try: sentry_sdk.capture_exception(*args, **kwargs) sentry_sdk.flush( ) # https://github.com/getsentry/sentry-python/issues/291 except Exception: cloudlog.exception("sentry exception")
def test_aggregates(sentry_init, capture_envelopes): sentry_init( release="fun-release", environment="not-fun-env", _experiments={"auto_session_tracking": True, "session_mode": "request"}, ) envelopes = capture_envelopes() hub = Hub.current with auto_session_tracking(): with sentry_sdk.push_scope(): try: with sentry_sdk.configure_scope() as scope: scope.set_user({"id": "42"}) raise Exception("all is wrong") except Exception: sentry_sdk.capture_exception() with auto_session_tracking(): pass hub.start_session() hub.end_session() sentry_sdk.flush() assert len(envelopes) == 2 assert envelopes[0].get_event() is not None sess = envelopes[1] assert len(sess.items) == 1 sess_event = sess.items[0].payload.json assert sess_event["attrs"] == { "release": "fun-release", "environment": "not-fun-env", } aggregates = sorted_aggregates(sess_event) assert len(aggregates) == 1 assert aggregates[0]["exited"] == 2 assert aggregates[0]["errored"] == 1
def test_new_versions(self, response): def url_to_doi(u): return '/'.join(u.split('?')[0].split('/')[-2:]) version_urls = list( map( lambda x: urljoin(response.request.url, x), response.xpath( '//div[contains(@class, "hw-versions")]//li/a/@href'). extract())) versions = list(map(url_to_doi, version_urls)) this_version = url_to_doi(response.meta['Link']) new_version_url = None for url, version in zip(version_urls, versions): if int(version.split('v')[-1]) > int(this_version.split('v')[-1]): new_version_url = url this_version = version if new_version_url is not None: site = 'biorxiv' if 'biorxiv.org' in new_version_url else 'medrxiv' self.logger.info('Registering new update job for DOI: %s Link: %s', response.meta['Doi'], new_version_url) new_job = { 'scrapy_url': new_version_url, 'scrapy_site': site, 'Doi': response.meta['Doi'], 'Journal': site, 'Publication_Date': response.meta['Publication_Date'], 'Origin': response.meta['Origin'] } with sentry_sdk.push_scope() as scope: scope.set_extra('DOI', response.meta['Doi']) sentry_sdk.capture_message('Scraper biorxiv: Updating article') sentry_sdk.flush() self.get_col( 'Scraper_connect_biorxiv_org_new_versions').insert_one(new_job)
def run_server_coroutine(config_values, host, port, manager, idx, shared_stats, enable_sentry: bool = True, extra_web_config = {}): """Run the server coroutine in a synchronous way. Used as target of multiprocessing.Process. """ if enable_sentry: sentry_sdk.init( dsn=config_values['sentry']['dsn'], traces_sample_rate=config_values['sentry']['traces_sample_rate'], ignore_errors=[KeyboardInterrupt], ) # Use a try-catch-capture_exception to work with multiprocessing, see # https://github.com/getsentry/raven-python/issues/1110 try: loop, tasks = prepare_loop(config_values, manager, idx=idx) loop.run_until_complete( asyncio.gather(*tasks, run_server(host, port, shared_stats, extra_web_config))) except Exception as e: if enable_sentry: sentry_sdk.capture_exception(e) sentry_sdk.flush() raise
def runner(pipe, job, *args, **kwargs): # This is the function called in a new process. # Sentry needs to be initialized here (in addition to the main process). sentry_sdk.init(dsn=os.environ.get("SENTRY_DSN")) # Call the wrapped function with the given arguments and pass the return value # back through a pipe. try: retval = function(job, *args, **kwargs) except Exception as exception: # Send a null value to stop the main process from blocking on receiving. pipe.send(None) # Update the job status. job.save(status="FAILURE") logger.exception(exception) sentry_sdk.capture_exception(exception) # Wait for the sentry event to be sent; otherwise we'd exit the process too # soon. sentry_sdk.flush() sys.exit(-1) else: pipe.send(retval)
def send_error(redis, doc_id, exc=None, message=None): """Sends an error to the API server specified as a string message""" if doc_id and not still_processing(redis, doc_id): return if message is None: message = str(exc) # Send the error to the server if doc_id: request(redis, "post", f"documents/{doc_id}/errors/", {"message": message}) if exc is not None: logging.error(message, exc_info=exc) capture_exception(exc) flush() else: logging.error(message) capture_message(message) # Clean out Redis if doc_id: clean_up(redis, doc_id)
def wrapper(*args, **kwargs): try: func(*args, **kwargs) except Abort as err: if not err.warn_only: if err.log_message is not None: logger.error(err.log_message) if err.original_error is not None: logger.error(f"Original exception: {err.original_error}") if settings.SENTRY_DSN: with sentry_sdk.push_scope() as scope: if err.sentry_context is not None: scope.set_context(**err.sentry_context) sentry_sdk.capture_exception(err.original_error if err.original_error is not None else err) sentry_sdk.flush() panel_kwargs = dict() if err.subject is not None: panel_kwargs["title"] = f"[red]{err.subject}" message = dedent(err.message) if err.support: support_message = unwrap( f""" [yellow]If the problem persists, please contact [bold]{OV_CONTACT}[/bold] for support and trouble-shooting """ ) message = f"{message}\n\n{support_message}" console = Console() console.print() console.print(Panel(message, **panel_kwargs)) console.print() raise typer.Exit(code=1)
def test_auto_session_tracking_with_aggregates(app, sentry_init, capture_envelopes): """ Test for correct session aggregates in auto session tracking. """ @app.route("/dogs/are/great/") @app.route("/trigger/an/error/") def great_dogs_handler(request): if request["path"] != "/dogs/are/great/": 1 / 0 return PlainTextResponse("dogs are great") sentry_init(traces_sample_rate=1.0) envelopes = capture_envelopes() app = SentryAsgiMiddleware(app) client = TestClient(app, raise_server_exceptions=False) client.get("/dogs/are/great/") client.get("/dogs/are/great/") client.get("/trigger/an/error/") sentry_sdk.flush() count_item_types = Counter() for envelope in envelopes: count_item_types[envelope.items[0].type] += 1 assert count_item_types["transaction"] == 3 assert count_item_types["event"] == 1 assert count_item_types["sessions"] == 1 assert len(envelopes) == 5 session_aggregates = envelopes[-1].items[0].payload.json["aggregates"] assert session_aggregates[0]["exited"] == 2 assert session_aggregates[0]["crashed"] == 1 assert len(session_aggregates) == 1
def clean_outputs_wrapper(*args, **kwargs): from polyaxon import settings cli_config = settings.CLI_CONFIG if cli_config and cli_config.log_handler and cli_config.log_handler.dsn: import sentry_sdk try: sentry_sdk.flush() return fn(*args, **kwargs) except Exception as e: sentry_sdk.capture_exception(e) sentry_sdk.flush() raise e finally: sentry_sdk.flush() else: return fn(*args, **kwargs)
def flush(self): import sentry_sdk sentry_sdk.flush()
def flush(self): sentry_sdk.flush()
def flush_sentry(): client = sentry_sdk.Hub.current.client if client is not None: client.close() sentry_sdk.flush() logger.info("Sentry client has been closed")
def fetch_page_timeout(self, urllib_fallback=False, requests_exception=None): html = None feed_link = self.feed.feed_link if not feed_link: self.save_no_page(reason="No feed link") return if feed_link.startswith('www'): self.feed.feed_link = 'http://' + feed_link try: if any(feed_link.startswith(s) for s in BROKEN_PAGES): self.save_no_page(reason="Broken page") return elif any(s in feed_link.lower() for s in BROKEN_PAGE_URLS): self.save_no_page(reason="Broke page url") return elif feed_link.startswith('http'): if urllib_fallback: request = urllib.request.Request(feed_link, headers=self.headers) response = urllib.request.urlopen(request) time.sleep(0.01) # Grrr, GIL. data = response.read().decode( response.headers.get_content_charset() or 'utf-8') else: try: response = requests.get(feed_link, headers=self.headers, timeout=10) response.connection.close() except requests.exceptions.TooManyRedirects: response = requests.get(feed_link, timeout=10) except (AttributeError, SocketError, OpenSSLError, PyAsn1Error, TypeError, requests.adapters.ReadTimeout) as e: logging.debug( ' ***> [%-30s] Page fetch failed using requests: %s' % (self.feed.log_title[:30], e)) self.save_no_page(reason="Page fetch failed") return data = response.text if response.encoding and response.encoding.lower( ) != 'utf-8': logging.debug( f" -> ~FBEncoding is {response.encoding}, re-encoding..." ) try: data = data.encode('utf-8').decode('utf-8') except (LookupError, UnicodeEncodeError): logging.debug(f" -> ~FRRe-encoding failed!") pass else: try: data = open(feed_link, 'r').read() except IOError: self.feed.feed_link = 'http://' + feed_link self.fetch_page(urllib_fallback=True) return if data: html = self.rewrite_page(data) if html: self.save_page(html) else: self.save_no_page(reason="No HTML found") return else: self.save_no_page(reason="No data found") return except (ValueError, urllib.error.URLError, http.client.BadStatusLine, http.client.InvalidURL, requests.exceptions.ConnectionError) as e: self.feed.save_page_history(401, "Bad URL", e) try: fp = feedparser.parse(self.feed.feed_address) except (urllib.error.HTTPError, urllib.error.URLError) as e: return html feed_link = fp.feed.get('link', "") self.feed.save() logging.debug(' ***> [%-30s] Page fetch failed: %s' % (self.feed.log_title[:30], e)) except (urllib.error.HTTPError) as e: self.feed.save_page_history(e.code, e.msg, e.fp.read()) except (http.client.IncompleteRead) as e: self.feed.save_page_history(500, "IncompleteRead", e) except (requests.exceptions.RequestException, requests.packages.urllib3.exceptions.HTTPError) as e: logging.debug( ' ***> [%-30s] Page fetch failed using requests: %s' % (self.feed.log_title[:30], e)) # mail_feed_error_to_admin(self.feed, e, local_vars=locals()) return self.fetch_page(urllib_fallback=True, requests_exception=e) except Exception as e: logging.debug('[%d] ! -------------------------' % (self.feed.id, )) tb = traceback.format_exc() logging.debug(tb) logging.debug('[%d] ! -------------------------' % (self.feed.id, )) self.feed.save_page_history(500, "Error", tb) # mail_feed_error_to_admin(self.feed, e, local_vars=locals()) if (not settings.DEBUG and hasattr(settings, 'SENTRY_DSN') and settings.SENTRY_DSN): capture_exception(e) flush() if not urllib_fallback: self.fetch_page(urllib_fallback=True) else: self.feed.save_page_history(200, "OK") return html
def publish(self, strict_mode: bool = False): """ Publish all graphs in the current package Args: strict_mode (bool): Whether to terminate if dependency cannot be resolved. """ sys.path.append(os.getcwd()) try: config = get_config(self._path) except KeyboardInterrupt: return "Manual exit" except Exception as e: if self._raise_errors: raise e if self._report_errors: sentry_sdk.capture_exception(e) sentry_sdk.flush() return f"Could not load config ({error_name(e)})" logger.info("Running 'setup'") try: create_graph_func = getattr( config, "setup", getattr( config, "create_system_graph", getattr(config, "bootstrap", None), ), ) if create_graph_func is None: return "You have to define a `setup` function in your config." try: graph = create_graph_func(strict_mode=strict_mode) except TypeError: logger.warning( "Please pass kwargs into `setup`! this will be deprecated soon!" ) graph = create_graph_func() except KeyboardInterrupt: return "Manual exit" except Exception as e: if self._raise_errors: raise e if self._report_errors: sentry_sdk.capture_exception(e) sentry_sdk.flush() return f"Could not run `setup` ({error_name(e)})" logger.info("Graph loaded") api_key = os.getenv("CODOC_API_KEY") if not api_key: return NO_API_ERROR logger.info("Loading views") views = get_views_in_folder(self._path) resp = [] for view in views: logger.info(f"Publishing {view.label}...") try: pk = view(graph=graph, api_key=api_key) except KeyboardInterrupt: return "Manual exit" except Exception as e: if self._raise_errors: raise e error = f"An unexpected error occurred when running `{view.label}` ({error_name(e)})" if self._report_errors: sentry_sdk.capture_exception(e) sentry_sdk.flush() return f"{error}\n\nThe error has been reported" return f"{error}\n\nRerun with `--report_errors` to report the errors" url = get_url_for_graph(pk) resp.append(f"'{view.label}' published at:\n{url}") return "\n".join(resp)
import os from datetime import datetime, timezone import sentry_sdk import setup_logging from src.scheduler import UpdateScheduler logger = setup_logging.setup() if 'SENTRY_URL' in os.environ: sentry_sdk.init(os.environ['SENTRY_URL'], traces_sample_rate=1.0) else: logger.info('Skipping sentry initialization') scheduler = UpdateScheduler() logger.debug( "Next update in %s", scheduler.run_once() - datetime.utcnow().replace(tzinfo=timezone.utc).astimezone(tz=timezone.utc)) scheduler.pool.closeall() sentry_sdk.flush()