def write(*args, stream=sys.stdout): for a in args: if isinstance(a, Exception): def fmt(fs): return ('./' + relpath(fs.filename) + f':{fs.lineno}' + f' in {fs.name}\n' + f' {fs.line}\n') trace = traceback.extract_tb(a.__traceback__) fmt_trace = ''.join(fmt(f) for f in trace) stack = traceback.extract_stack() for i, f in enumerate(reversed(stack)): if (f.filename, f.name) == (trace[0].filename, trace[0].name): stack = stack[:-i] break # skip the log.py part of stack for i, f in enumerate(reversed(stack)): if 'site-packages' in f.filename: stack = stack[-i:] break # skip the flask part of stack fmt_stack = ''.join(fmt(f) for f in stack) a = (fmt_stack + '--- printed exception w/ trace ---\n' + fmt_trace + utils.format_exception_only(a)) pid = getpid() print(f'[{pid}] {a}', file=stream, flush=True)
def wrapper(*args, **kwargs): try: result = f(*args, **kwargs) result.setdefault('success', True) except Exception as e: log.error(e) result = { 'success': False, 'errorMessage': format_exception_only(e), } return jsonify(result)
def record_metadata(metadata, table, e=None): ctx = connect() if e is None and 'EXCEPTION' in metadata: e = metadata['EXCEPTION'] del metadata['EXCEPTION'] if e is not None: exception_only = utils.format_exception_only(e) metadata['ERROR'] = { 'EXCEPTION': utils.format_exception(e), 'EXCEPTION_ONLY': exception_only, } if exception_only.startswith( 'snowflake.connector.errors.ProgrammingError: '): metadata['ERROR']['PROGRAMMING_ERROR'] = exception_only[45:] metadata.setdefault('ROW_COUNT', { 'INSERTED': 0, 'UPDATED': 0, 'SUPPRESSED': 0, 'PASSED': 0 }) metadata['END_TIME'] = datetime.utcnow() metadata['DURATION'] = str(metadata['END_TIME'] - metadata['START_TIME']) metadata['START_TIME'] = str(metadata['START_TIME']) metadata['END_TIME'] = str(metadata['END_TIME']) record_type = metadata.get('QUERY_NAME', 'RUN') metadata_json_sql = "'" + json.dumps(metadata).replace( '\\', '\\\\').replace("'", "\\'") + "'" sql = f''' INSERT INTO {table}(event_time, v) SELECT '{metadata['START_TIME']}' , PARSE_JSON(column1) FROM VALUES({metadata_json_sql}) ''' try: ctx.cursor().execute(sql) log.info(f"{record_type} metadata recorded.") except Exception as e: log.error(f"{record_type} metadata failed to log.", e)
def record_metadata(metadata, table, e=None): ctx = connect() if e is None and 'EXCEPTION' in metadata: e = metadata['EXCEPTION'] del metadata['EXCEPTION'] if e is not None: exception_only = utils.format_exception_only(e) metadata['ERROR'] = { 'EXCEPTION': utils.format_exception(e), 'EXCEPTION_ONLY': exception_only, } if exception_only.startswith( 'snowflake.connector.errors.ProgrammingError: '): metadata['ERROR']['PROGRAMMING_ERROR'] = exception_only[45:] metadata.setdefault('ROW_COUNT', { 'INSERTED': 0, 'UPDATED': 0, 'SUPPRESSED': 0, 'PASSED': 0 }) metadata['START_TIME'] = str(metadata['START_TIME']) metadata['END_TIME'] = str(datetime.utcnow()) metadata['DURATION'] = ( datetime.fromisoformat(metadata['END_TIME']) - datetime.fromisoformat(metadata['START_TIME'])).total_seconds() record_type = metadata.get('QUERY_NAME', 'RUN') metadata_json_sql = value_to_sql(metadata) sql = f''' INSERT INTO {table}(event_time, v) SELECT '{metadata['START_TIME']}' , {metadata_json_sql} ''' try: ctx.cursor().execute(sql) log.info(f"{record_type} metadata recorded.") except Exception as e: log.error(f"{record_type} metadata failed to log.", e)
def connect(flush_cache=False, set_cache=False, oauth={}): account = oauth.get('account') role = oauth.get('role') db = oauth.get('database') wh = oauth.get('warehouse') oauth_refresh_token = oauth.get('refresh_token') oauth_access_token = (oauth_refresh(account, oauth_refresh_token) if oauth_refresh_token else None) if oauth_refresh_token and not oauth_access_token: log.error('failed to connect with oauth creds provided') return oauth_username = oauth.get('username') oauth_account = oauth.get('account') cached_connection = getattr(CACHE, CONNECTION, None) if cached_connection and not flush_cache and not oauth_access_token: return cached_connection connect_db, authenticator, pk = ( (snowflake.connector.connect, OAUTH_AUTHENTICATOR, None) if oauth_access_token else (snowflake_connect, 'EXTERNALBROWSER', None) if PRIVATE_KEY is None else ( snowflake.connector.connect, None, load_pkb(PRIVATE_KEY, PRIVATE_KEY_PASSWORD), )) db = (db or environ.get('OAUTH_CONNECTION_DATABASE', None) if oauth_account else DATABASE) wh = (wh or environ.get('OAUTH_CONNECTION_WAREHOUSE', None) if oauth_access_token else WAREHOUSE) rl = (role or environ.get('OAUTH_CONNECTION_ROLE', None) if oauth_access_token else ROLE) def connect(): return connect_db( # Role, Warehouse, and Database connection values are defined in the following order: # 1) If not using OAuth, use the SA_* env variables with defaults to "snowalert" (see dbconfig.py) # 2) If using OAuth and OAUTH_CONNECTION_* env vars have been set, use these # 3) Else, use OAuth'd user's default namespace (see ALTER USER docs) account=oauth_account or ACCOUNT, port=PORT, protocol=PROTOCOL, database=db, user=oauth_username or USER, warehouse=wh, role=rl, token=oauth_access_token, private_key=pk, authenticator=authenticator, ocsp_response_cache_filename= '/tmp/.cache/snowflake/ocsp_response_cache', network_timeout=TIMEOUT, ) try: connection = retry( connect, loggers=[( snowflake.connector.errors.DatabaseError, lambda e: print('db.retry:', utils.format_exception_only(e)), )], ) # see SP-1116 for why set_cache=False by default if set_cache and not cached_connection: setattr(CACHE, CONNECTION, connection) return connection except Exception as e: log.error(e, "Failed to connect.")