def get(self, **kwargs):
elasticapm.set_user_context(username=kwargs['key_app'],
user_id=kwargs['key_id'])
args = taxonomy_query.parse_args()
q = request.args.get('q', None)
parent_id = args.get('parent-id') if args.get('parent-id') else []
concept_type = args.get('type') or []
offset = request.args.get(settings.OFFSET, 0)
limit = request.args.get(settings.LIMIT, 10)
response = taxonomy.find_concepts(elastic, q, parent_id, concept_type,
offset, limit)
show_count = request.args.get(settings.SHOW_COUNT)
statistics = platsannonser.get_stats_for(concept_type) \
if concept_type and show_count else {}
if not response:
abort(500, custom="The server failed to respond properly")
query_dict = {}
if q:
query_dict['filter'] = q
if parent_id:
query_dict['parentId'] = parent_id
if concept_type:
query_dict['type'] = concept_type
query_dict['offset'] = offset
query_dict['limit'] = limit
return self._build_response(query_dict, response, statistics)
def test_transaction_keyword_truncation(sending_elasticapm_client):
too_long = 'x' * (KEYWORD_MAX_LENGTH + 1)
expected = encoding.keyword_field(too_long)
assert too_long != expected
assert len(expected) == KEYWORD_MAX_LENGTH
assert expected[-1] != 'x'
sending_elasticapm_client.begin_transaction(too_long)
elasticapm.tag(val=too_long)
elasticapm.set_user_context(username=too_long, email=too_long, user_id=too_long)
with elasticapm.capture_span(name=too_long, span_type=too_long):
pass
sending_elasticapm_client.end_transaction(too_long, too_long)
sending_elasticapm_client.close()
assert sending_elasticapm_client.httpserver.responses[0]['code'] == 202
transaction = sending_elasticapm_client.httpserver.payloads[0]['transactions'][0]
span = transaction['spans'][0]
assert transaction['name'] == expected
assert transaction['type'] == expected
assert transaction['result'] == expected
assert transaction['context']['user']['id'] == expected
assert transaction['context']['user']['username'] == expected
assert transaction['context']['user']['email'] == expected
assert transaction['context']['tags']['val'] == expected
assert span['type'] == expected
assert span['name'] == expected
def test_transaction_keyword_truncation(elasticapm_client):
too_long = "x" * (KEYWORD_MAX_LENGTH + 1)
expected = encoding.keyword_field(too_long)
assert too_long != expected
assert len(expected) == KEYWORD_MAX_LENGTH
assert expected[-1] != "x"
elasticapm_client.begin_transaction(too_long)
elasticapm.tag(val=too_long)
elasticapm.set_user_context(username=too_long, email=too_long, user_id=too_long)
with elasticapm.capture_span(name=too_long, span_type=too_long):
pass
elasticapm_client.end_transaction(too_long, too_long)
elasticapm_client.close()
span = elasticapm_client.events["span"][0]
transaction = elasticapm_client.events["transaction"][0]
assert transaction["name"] == expected
assert transaction["type"] == expected
assert transaction["result"] == expected
assert transaction["context"]["user"]["id"] == expected
assert transaction["context"]["user"]["username"] == expected
assert transaction["context"]["user"]["email"] == expected
assert transaction["context"]["tags"]["val"] == expected
assert span["type"] == expected
assert span["name"] == expected
def process_response(self, req, resp, resource, req_succeeded=None):
if self.client and req.user_agent != 'mcod-heartbeat':
rule = route_to_name(req.uri_template,
prefix='api',
method=req.method)
elasticapm.set_context(
lambda: get_data_from_request(
req,
capture_body=self.client.config.capture_body in
("transactions", "all"),
capture_headers=self.client.config.capture_headers,
),
"request",
)
elasticapm.set_context(
lambda: get_data_from_response(
resp, capture_headers=self.client.config.capture_headers),
"response")
result = resp.status
elasticapm.set_transaction_name(rule, override=False)
if hasattr(req, 'user') and req.user.is_authenticated:
elasticapm.set_user_context(email=req.user.email,
user_id=req.user.id)
elasticapm.set_transaction_result(result, override=False)
# Instead of calling end_transaction here, we defer the call until the response is closed.
# This ensures that we capture things that happen until the WSGI server closes the response.
self.client.end_transaction(rule, result)
def test_set_user_context(elasticapm_client):
elasticapm_client.begin_transaction("test")
elasticapm.set_user_context(username="******", email="*****@*****.**", user_id=42)
elasticapm_client.end_transaction("foo", 200)
transactions = elasticapm_client.transaction_store.get_all()
assert transactions[0]["context"]["user"] == {"username": "******", "email": "*****@*****.**", "id": 42}
def test_set_user_context_merge(elasticapm_client):
elasticapm_client.begin_transaction("test")
elasticapm.set_user_context(username="******", email="*****@*****.**")
elasticapm.set_user_context(email="*****@*****.**", user_id=42)
elasticapm_client.end_transaction("foo", 200)
transactions = elasticapm_client.events[TRANSACTION]
assert transactions[0]["context"]["user"] == {"username": "******", "email": "*****@*****.**", "id": 42}
def _patched(self, app, response):
if not self.app.debug or self.client.config.debug:
# 在 context 的 user_id 中记录匿名 user id，在 username 中记录当前登录的用户学号
if session.get('user_id', None):
user_id = session['user_id']
elasticapm.set_user_context(user_id=user_id)
if session.get(SESSION_CURRENT_USER, None):
elasticapm.set_user_context(username=session[SESSION_CURRENT_USER])
# execute the original `request_finished` function
original_func(self, app, response)
def test_transaction_context_is_used_in_errors(elasticapm_client):
elasticapm_client.begin_transaction("test")
elasticapm.tag(foo="baz")
elasticapm.set_custom_context({"a": "b"})
elasticapm.set_user_context(username="******", email="*****@*****.**", user_id=42)
elasticapm_client.capture_message("x", custom={"foo": "bar"})
transaction = elasticapm_client.end_transaction("test", "OK")
message = elasticapm_client.events[ERROR][0]
assert message["context"]["custom"] == {"a": "b", "foo": "bar"}
assert message["context"]["user"] == {"username": "******", "email": "*****@*****.**", "id": 42}
assert message["context"]["tags"] == {"foo": "baz"}
assert "a" in transaction.context["custom"]
assert "foo" not in transaction.context["custom"]
def test_transaction_context_is_used_in_errors(elasticapm_client):
elasticapm_client.begin_transaction('test')
elasticapm.tag(foo='baz')
elasticapm.set_custom_context({'a': 'b'})
elasticapm.set_user_context(username='******', email='*****@*****.**', user_id=42)
elasticapm_client.capture_message('x', custom={'foo': 'bar'})
transaction = elasticapm_client.end_transaction('test', 'OK')
message = elasticapm_client.events[0]['errors'][0]
assert message['context']['custom'] == {'a': 'b', 'foo': 'bar'}
assert message['context']['user'] == {'username': '******', 'email': '*****@*****.**', 'id': 42}
assert message['context']['tags'] == {'foo': 'baz'}
assert 'a' in transaction.context['custom']
assert 'foo' not in transaction.context['custom']
def _patched(self, app, response):
if not self.app.debug or self.client.config.debug:
# 在 context 中记录 cookie 中的 user id
if session.get('user_id', None):
user_id = session['user_id']
else:
user_id = request.cookies.get('UM_distinctid', None)
elasticapm.set_user_context(user_id=user_id,
username=session.get(
'stu_id', None))
# execute the original `request_finished` function
original_func(self, app, response)
def get(self, **kwargs):
elasticapm.set_user_context(username=kwargs['key_app'],
user_id=kwargs['key_id'])
start_time = int(time.time() * 1000)
args = bulk_zip_query.parse_args()
bytes_result = repository.zip_ads(args.get(settings.DATE), start_time)
filename = "ads_%s.zip" % args.get(settings.DATE)
log.debug("Elapsed time for completion: %d" %
int((time.time() * 1000) - start_time))
return send_file(bytes_result,
attachment_filename=filename,
cache_timeout=60,
as_attachment=True)
def current_payload(
self,
credentials: HTTPAuthorizationCredentials = Security(
HTTPBearerWithCookie())
) -> JWTPayload:
token = credentials.credentials
payload = self.decode_token(token)
if self.scope:
if self.scope not in payload.scope:
raise HTTPException(status_code=401, detail='Invalid scope')
elasticapm.set_user_context(username=payload.name,
email=payload.email,
user_id=payload.sub)
return payload
def test_set_user_context_merge(elasticapm_client):
elasticapm_client.begin_transaction('test')
elasticapm.set_user_context(username='******', email='*****@*****.**')
elasticapm.set_user_context(email='*****@*****.**', user_id=42)
elasticapm_client.end_transaction('foo', 200)
transactions = elasticapm_client.instrumentation_store.get_all()
assert transactions[0]['context']['user'] == {
'username': '******',
'email': '*****@*****.**',
'id': 42
}
async def _instrument_request(request: Request):
if not self._client.should_ignore_url(url=request.path):
trace_parent = TraceParent.from_headers(headers=request.headers)
self._client.begin_transaction("request", trace_parent=trace_parent)
await set_context(
lambda: get_request_info(config=self._client.config, request=request),
"request",
)
self._setup_transaction_name(request=request)
if self._user_context_callback:
name, email, uid = await self._user_context_callback(request)
set_user_context(username=name, email=email, user_id=uid)
await self._setup_custom_context(request=request)
if self._label_info_callback:
labels = await self._label_info_callback(request)
label(**labels)
def __call__(self, request):
self.client.begin_transaction('request')
try:
response = self.handler(request)
transaction_result = response.status[0] + 'xx'
elasticapm.set_context(
lambda: self.get_data_from_response(response), 'response')
return response
except Exception:
transaction_result = '5xx'
self.client.capture_exception(
context={'request': self.get_data_from_request(request)},
handled=False,
)
reraise(*sys.exc_info())
finally:
transaction_name = self.get_transaction_name(request)
elasticapm.set_context(lambda: self.get_data_from_request(request),
'request')
elasticapm.set_user_context(user_id=request.authenticated_userid, )
self.client.end_transaction(transaction_name, transaction_result)
def base(*args, **kwargs):
# Before anything else, check that the API key is set
apikey = request.environ.get('HTTP_X_APIKEY', None)
if AUTH_KEY != apikey:
client_id = request.headers.get('container_id', 'Unknown Client')
header_dump = '; '.join(f'{k}={v}' for k, v in request.headers.items())
wsgi_dump = '; '.join(f'{k}={v}' for k, v in request.environ.items())
LOGGER.warning(f'Client [{client_id}] provided wrong api key [{apikey}] '
f'headers: {header_dump}; wsgi: {wsgi_dump}')
return make_api_response("", "Unauthorized access denied", 401)
client_info = dict(
client_id=request.headers['container_id'],
service_name=request.headers['service_name'],
service_version=request.headers['service_version'],
service_tool_version=request.headers.get('service_tool_version'),
)
if config.core.metrics.apm_server.server_url is not None:
elasticapm.set_user_context(username=client_info['service_name'])
kwargs['client_info'] = client_info
return func(*args, **kwargs)
def get(self, **kwargs):
elasticapm.set_user_context(username=kwargs['key_app'],
user_id=kwargs['key_id'])
args = bulk_stream_query.parse_args()
return Response(repository.load_all(args.get(settings.DATE)),
mimetype='application/json')
from flask import Flask, jsonify, redirect, render_template
from elasticapm.contrib.flask import ElasticAPM
from elasticapm.handlers.logging import LoggingHandler
#from elasticapm.contrib.opentracing import Tracer
#tracer = Tracer(config={"server_url":"http://172.23.0.1:8200","service_name":"ap-mflask"});
names = [
'ruan', 'stefan', 'philip', 'norman', 'frank', 'pete', 'johnny', 'peter',
'adam'
]
cities = [
'cape town', 'johannesburg', 'pretoria', 'dublin', 'kroonstad',
'bloemfontein', 'port elizabeth', 'auckland', 'sydney'
]
lastnames = ['smith', 'bekker', 'admams', 'phillips', 'james', 'adamson']
elasticapm.set_user_context(username='******',
email='*****@*****.**',
user_id=1234)
conn = sqlite3.connect('database.db')
conn.execute(
'CREATE TABLE IF NOT EXISTS people (name STRING, age INTEGER, surname STRING, city STRING)'
)
#sqlquery_write = conn.execute('INSERT INTO people VALUES("{}", "{}", "{}", "{}")'.format(random.choice(names), random.randint(18,40), random.choice(lastnames), random.choice(cities)))
seconds = [
0.002, 0.003, 0.004, 0.01, 0.3, 0.2, 0.009, 0.015, 0.02, 0.225, 0.009,
0.001, 0.25, 0.030, 0.018
]
app = Flask(__name__)
apm = ElasticAPM(app,
server_url='http://172.30.0.1:8200',
def _check_token():
''' Check the validity of the token provided '''
auth_header = request.headers.get('Authorization')
current_user = None
if auth_header:
try:
access_token = auth_header.split(' ')[1]
expired = ExpiredToken.search().filter(
'term', token=access_token).execute()
if expired:
abort(401, 'Token retired.')
try:
token = jwt.decode(access_token,
current_app.config['SECRET_KEY'],
algorithms=['HS256'])
# If this is an agents token pull the agents information and
# set it to current_user
if 'type' in token and token['type'] == 'agent':
current_user = Agent.get_by_uuid(uuid=token['uuid'])
# Refresh and Password Reset tokens should not be used to access the API
# only to refresh an access token or reset the password
elif 'type' in token and token['type'] in [
'refresh', 'password_reset'
]:
abort(401, 'Unauthorized')
# The pairing token can only be used on the add_agent endpoint
# and because the token is signed we don't have to worry about
# someone adding a the pairing type to their token
elif 'type' in token and token['type'] == 'pairing':
current_user = token
else:
try:
current_user = User.get_by_uuid(token['uuid'])
except:
abort(401, 'Unknown user error.')
# If the user is currently locked
# reject the accesss_token and expire it
if hasattr(current_user, 'locked') and current_user.locked:
expired = ExpiredToken(token=access_token)
expired.save()
abort(401, 'Unauthorized')
if 'default_org' in token and token['default_org']:
current_user.default_org = True
except ValueError:
abort(401, 'Token retired.')
except jwt.ExpiredSignatureError:
abort(401, 'Access token expired.')
except (jwt.DecodeError, jwt.InvalidTokenError) as e:
abort(401, 'Invalid access token.')
except Exception as e:
print(e)
abort(401, 'Unknown token error.')
except IndexError:
abort(401, 'Invalid access token.')
raise jwt.InvalidTokenError
else:
abort(403, 'Access token required.')
if current_app.config['ELASTIC_APM_ENABLED']:
if isinstance(current_user, (Agent, User)):
username = None
email = None
if isinstance(current_user, Agent):
username = '******'
email = 'agent'
else:
username = current_user.username
email = current_user.email
elasticapm.set_user_context(username=username + '-' +
current_user.organization,
user_id=current_user.uuid,
email=email)
return current_user
def base(*args, **kwargs):
if 'user' in kwargs:
if kwargs['user'].get('authenticated', False):
return func(*args, **kwargs)
else:
abort(403, "Invalid pre-authenticated user")
return
self.test_readonly("API")
logged_in_uname = self.get_logged_in_user()
user = login(logged_in_uname)
# Terms of Service
if request.path not in ["/api/v4/help/tos/", "/api/v4/user/whoami/",
f"/api/v4/user/tos/{logged_in_uname}/",
"/api/v4/auth/logout/"] \
and not user.get('agrees_with_tos', False) and config.ui.tos is not None:
abort(
403,
"Agree to Terms of Service before you can make any API calls"
)
return
self.test_require_type(user, "API")
#############################################
# Special username api query validation
#
# If an API call requests a username, the username as to match
# the logged in user or the user has to be ADMIN
#
# API that needs this special validation need to make sure their
# variable name for the username is as an optional parameter
# inside 'username_key'. Default: 'username'
if self.username_key in kwargs:
if kwargs[self.username_key] != user['uname'] \
and not kwargs[self.username_key] == "__global__" \
and not kwargs[self.username_key] == "__workflow__" \
and not kwargs[self.username_key].lower() == "__current__" \
and 'admin' not in user['type']:
return make_api_response(
{}, "Your username does not match requested username",
403)
self.audit_if_required(args, kwargs, logged_in_uname, user, func)
# Save user credential in user kwarg for future reference
kwargs['user'] = user
if config.core.metrics.apm_server.server_url is not None:
elasticapm.set_user_context(username=user.get('name', None),
email=user.get('email', None),
user_id=user.get('uname', None))
# Check current user quota
quota_user = user['uname']
flsk_session['quota_user'] = quota_user
flsk_session['quota_set'] = True
quota = user.get('api_quota', 10)
if not QUOTA_TRACKER.begin(quota_user, quota):
if config.ui.enforce_quota:
LOGGER.info(
f"User {quota_user} was prevented from using the api due to exceeded quota."
)
return make_api_response(
"", f"You've exceeded your maximum quota of {quota}",
503)
else:
LOGGER.debug(
f"Quota of {quota} exceeded for user {quota_user}.")
else:
LOGGER.debug(
f"{quota_user}'s quota is under or equal its limit of {quota}"
)
return func(*args, **kwargs)
def apm_user_context():
# importing over here as it won't work outside request context
from flask import request
initiator = request.headers.get('X-Initiator')
if initiator:
set_user_context(user_id=initiator)
def base(*args, **kwargs):
self.test_readonly("Page")
# Validate User-Agent
UNK = "__UNKNOWN__"
user_agent = request.environ.get("HTTP_USER_AGENT", UNK)
if UNK == user_agent or "MSIE 8" in user_agent or "MSIE 9" in user_agent or "MSIE 7" in user_agent \
or "MSIE 6" in user_agent:
return redirect(redirect_helper("/unsupported.html"))
# Create Path
path = f"{request.path}?{request.query_string.decode('UTF-8')}"
# Login
logged_in_uname = self.get_logged_in_user()
user = login(logged_in_uname, path)
self.test_require_type(user, "Url")
self.audit_if_required(args, kwargs, logged_in_uname, user, func)
# Dump Generic KWARGS
kwargs['build'] = f"{BUILD_MASTER}.{BUILD_LOWER}.{BUILD_NO}"
kwargs['user'] = user
if config.core.metrics.apm_server.server_url is not None:
elasticapm.set_user_context(username=user.get('name', None),
email=user.get('email', None),
user_id=user.get('uname', None))
settings = STORAGE.user_settings.get(user['uname'], as_obj=False)
if settings:
if config.ui.ui4_path is not None:
if settings.get('ui4', False):
return redirect(redirect_helper(config.ui.ui4_path))
user['ui4_ask'] = settings.get('ui4_ask', True)
else:
user['ui4_ask'] = settings.get('ui4_ask', False)
else:
user['ui4_ask'] = False
user['ui4_allow'] = config.ui.ui4_path is not None
kwargs['user_js'] = json.dumps(user)
kwargs['debug'] = str(DEBUG).lower()
kwargs['menu'] = create_menu(user, path)
kwargs['avatar'] = STORAGE.user_avatar.get(user['uname'])
kwargs['is_prod'] = SYSTEM_TYPE == "production"
kwargs['is_readonly'] = config.ui.read_only
if not request.path == "/terms.html":
if not user.get('agrees_with_tos',
False) and config.ui.tos is not None:
return redirect(redirect_helper("/terms.html"))
if not settings and not request.path == "/settings.html":
return redirect(redirect_helper("/settings.html?forced"))
if self.load_settings:
kwargs['settings'] = json.dumps(settings)
return func(*args, **kwargs)
def base(*args, **kwargs):
if 'user' in kwargs:
if kwargs['user'].get('authenticated', False):
return func(*args, **kwargs)
else:
abort(403, "Invalid pre-authenticated user")
return
self.test_readonly("API")
logged_in_uname = self.get_logged_in_user()
impersonator = None
# Impersonate
authorization = request.environ.get("HTTP_AUTHORIZATION", None)
if authorization:
# noinspection PyBroadException
try:
bearer_token = authorization.split(" ")[-1]
headers = jwt.get_unverified_header(bearer_token)
decoded = jwt.decode(
bearer_token,
hashlib.sha256(f"{SECRET_KEY}_{headers['token_id']}".
encode()).hexdigest(),
algorithms=[headers.get('alg', "HS256")])
except Exception:
abort(400, "Malformed bearer token")
return
target_user = STORAGE.user.get(headers['user'], as_obj=False)
if target_user:
target_token = target_user.get('apps', {}).get(
headers['token_id'], {})
if target_token == decoded and target_token[
'client_id'] == logged_in_uname:
impersonator = logged_in_uname
logged_in_uname = headers['user']
LOGGER.info(
f"{impersonator} is impersonating {logged_in_uname} for query: {request.path}"
)
if not set(self.required_priv).intersection(
set(SCOPES[decoded["scope"]])):
abort(
403,
"The method you've used to login does not give you access to this API"
)
return
else:
abort(403, "Invalid bearer token")
return
else:
abort(404, "User not found")
return
user = login(logged_in_uname)
# Terms of Service
if request.path not in ["/api/v4/help/tos/", "/api/v4/user/whoami/",
f"/api/v4/user/tos/{logged_in_uname}/",
"/api/v4/auth/logout/"] \
and not user.get('agrees_with_tos', False) and config.ui.tos is not None:
abort(
403,
"Agree to Terms of Service before you can make any API calls"
)
return
self.test_require_type(user, "API")
#############################################
# Special username api query validation
#
# If an API call requests a username, the username as to match
# the logged in user or the user has to be ADMIN
#
# API that needs this special validation need to make sure their
# variable name for the username is as an optional parameter
# inside 'username_key'. Default: 'username'
if self.username_key in kwargs:
if kwargs[self.username_key] != user['uname'] \
and not kwargs[self.username_key] == "__global__" \
and not kwargs[self.username_key] == "__workflow__" \
and not kwargs[self.username_key].lower() == "__current__" \
and 'admin' not in user['type']:
return make_api_response(
{}, "Your username does not match requested username",
403)
self.audit_if_required(args,
kwargs,
logged_in_uname,
user,
func,
impersonator=impersonator)
# Save user credential in user kwarg for future reference
kwargs['user'] = user
if config.core.metrics.apm_server.server_url is not None:
elasticapm.set_user_context(username=user.get('name', None),
email=user.get('email', None),
user_id=user.get('uname', None))
# Check current user quota
quota_user = user['uname']
flsk_session['quota_user'] = quota_user
flsk_session['quota_set'] = True
quota = user.get('api_quota', 10)
if not QUOTA_TRACKER.begin(quota_user, quota):
if config.ui.enforce_quota:
LOGGER.info(
f"User {quota_user} was prevented from using the api due to exceeded quota."
)
return make_api_response(
"", f"You've exceeded your maximum quota of {quota}",
503)
else:
LOGGER.debug(
f"Quota of {quota} exceeded for user {quota_user}.")
else:
LOGGER.debug(
f"{quota_user}'s quota is under or equal its limit of {quota}"
)
return func(*args, **kwargs)
