def get_governance_info_from_request(self, json_params=None):
# Default values for governance headers.
actor_id = DEFAULT_ACTOR_ID
expiry = DEFAULT_EXPIRY
authtoken = ""
user_session = get_auth()
if user_session.get("actor_id", None) and user_session.get("valid_until", 0):
# Get info from current server session
# NOTE: Actor id may be inside server session
actor_id = user_session["actor_id"]
expiry = str(int(user_session.get("valid_until", 0)) * 1000)
log.info("Request associated with session actor_id=%s, expiry=%s", actor_id, expiry)
# Developer access using api_key
if self.develop_mode and "api_key" in request.args and request.args["api_key"]:
actor_id = str(request.args["api_key"])
expiry = str(int(user_session.get("valid_until", 0)) * 1000)
if 0 < int(expiry) < current_time_millis():
expiry = str(current_time_millis() + 10000)
# flask.session["valid_until"] = int(expiry / 1000)
log.info("Request associated with actor_id=%s, expiry=%s from developer api_key", actor_id, expiry)
# Check in headers for OAuth2 bearer token
auth_hdr = request.headers.get("authorization", None)
if auth_hdr:
valid, req = self.process.oauth.verify_request([self.process.oauth_scope])
if valid:
actor_id = flask.g.oauth_user.get("actor_id", "")
if actor_id:
log.info("Request associated with actor_id=%s, expiry=%s from OAuth token", actor_id, expiry)
return actor_id, DEFAULT_EXPIRY
# Try to find auth token override
if not authtoken:
if json_params:
if "authtoken" in json_params:
authtoken = json_params["authtoken"]
else:
if "authtoken" in request.args:
authtoken = str(request.args["authtoken"])
# Enable temporary authentication tokens to resolve to actor ids
if authtoken:
try:
token_info = self.idm_client.check_authentication_token(authtoken, headers=self._get_gateway_headers())
actor_id = token_info.get("actor_id", actor_id)
expiry = token_info.get("expiry", expiry)
log.info("Resolved token %s into actor_id=%s expiry=%s", authtoken, actor_id, expiry)
except NotFound:
log.info("Provided authentication token not found: %s", authtoken)
except Unauthorized:
log.info("Authentication token expired or invalid: %s", authtoken)
except Exception as ex:
log.exception("Problem resolving authentication token")
return actor_id, expiry
def get_session(self):
try:
# Get session based on OAuth token
auth_hdr = request.headers.get("authorization", None)
if auth_hdr:
valid, req = self.oauth.verify_request([self.oauth_scope])
if valid:
actor_id = flask.g.oauth_user.get("actor_id", "")
actor_user = self.idm_client.read_actor_identity(actor_id)
session_attrs = dict(is_logged_in=True, is_registered=True, attributes={"roles":actor_user.details.contact.roles}, roles={})
if actor_user.session:
session_attrs.update(actor_user.session)
return build_json_response(session_attrs)
# Support quick reload
access_token = flask.session.get("access_token", None)
actor_id = flask.session.get("actor_id", None)
if access_token and actor_id:
actor_user = self.idm_client.read_actor_identity(actor_id)
session_attrs = dict(access_token=access_token, is_logged_in=True, is_registered=True, attributes={"roles":actor_user.details.contact.roles}, roles={})
if actor_user.session:
session_attrs.update(actor_user.session)
return build_json_response(session_attrs)
# Get session from Flask session and cookie
user_info = get_auth()
if 0 < int(user_info.get("valid_until", 0)) * 1000 < current_time_millis():
clear_auth()
user_info = get_auth()
return build_json_response(user_info)
except Exception:
return build_json_error()
def validate_request(ion_actor_id, expiry):
# There is no point in looking up an anonymous user - so return default values.
if ion_actor_id == DEFAULT_ACTOR_ID:
expiry = DEFAULT_EXPIRY # Since this is now an anonymous request, there really is no expiry associated with it
return ion_actor_id, expiry
idm_client = IdentityManagementServiceProcessClient(process=service_gateway_instance)
try:
user = idm_client.read_actor_identity(actor_id=ion_actor_id,
headers={"ion-actor-id": service_gateway_instance.name,
'expiry': DEFAULT_EXPIRY})
except NotFound as e:
ion_actor_id = DEFAULT_ACTOR_ID # If the user isn't found default to anonymous
expiry = DEFAULT_EXPIRY # Since this is now an anonymous request, there really is no expiry associated with it
return ion_actor_id, expiry
# Need to convert to a float first in order to compare against current time.
try:
int_expiry = int(expiry)
except Exception as e:
raise Inconsistent("Unable to read the expiry value in the request '%s' as an int" % expiry)
# The user has been validated as being known in the system, so not check the expiry and raise exception if
# the expiry is not set to 0 and less than the current time.
if 0 < int_expiry < current_time_millis():
raise Unauthorized('The certificate associated with the user and expiry time in the request has expired.')
return ion_actor_id, expiry
def __init__(self, originating_container, actor_id, requesting_message, token):
self.originator = originating_container
self.actor_id = actor_id
self.requesting_message = requesting_message
self.token = token
timeout = CFG.get_safe('container.messaging.timeout.receive', 30)
self.expire_time = current_time_millis() + (timeout * 1000) # Set the expire time to current time + timeout in ms
def _retrieve_attribute_values(self):
"""
Retrieves the attribute values using the given function and calls
_values_retrieved.
"""
# determine from_time for the request:
if self._last_ts is None:
# This is the very first retrieval request, so pick a from_time
# that makes sense. At the moment, setting from_time to be current
# system minus the monitoring rate.
# TODO: determine actual criteria here.
from_time = current_time_millis() - self._rate_millis
# TODO: Also note that the "from_time" parameter for the request was
# influenced by the RSN case (see CI-OMS interface). Need to see
# whether it also applies to CGSN so eventually adjustments may be needed.
#
else:
# note that int(x) returns a long object if needed.
from_time = int(self._last_ts) + _DELTA_TIME
log.debug("%r: _retrieve_attribute_values: attr_ids=%r from_time=%s",
self._platform_id, self._attr_ids, from_time)
retrieved_vals = self._get_attribute_values(self._attr_ids, from_time)
log.debug(
"%r: _retrieve_attribute_values: _get_attribute_values "
"for attr_ids=%r and from_time=%s returned %s", self._platform_id,
self._attr_ids, from_time, retrieved_vals)
# vals_dict: attributes with non-empty reported values:
vals_dict = {}
for attr_id in self._attr_ids:
if not attr_id in retrieved_vals:
log.warn(
"%r: _retrieve_attribute_values: unexpected: "
"response does not include requested attribute %r. "
"Response is: %s", self._platform_id, attr_id,
retrieved_vals)
continue
attr_vals = retrieved_vals[attr_id]
if not attr_vals:
log.debug(
"%r: No values reported for attribute=%r from_time=%f",
self._platform_id, attr_id, from_time)
continue
if log.isEnabledFor(logging.DEBUG):
self._debug_values_retrieved(attr_id, attr_vals)
# ok, include this attribute for the notification:
vals_dict[attr_id] = attr_vals
if vals_dict:
self._values_retrieved(vals_dict)
def _retrieve_attribute_values(self):
"""
Retrieves the attribute values using the given function and calls
_values_retrieved.
"""
# determine from_time for the request:
if self._last_ts is None:
# This is the very first retrieval request, so pick a from_time
# that makes sense. At the moment, setting from_time to be current
# system minus the monitoring rate.
# TODO: determine actual criteria here.
from_time = current_time_millis() - self._rate_millis
# TODO: Also note that the "from_time" parameter for the request was
# influenced by the RSN case (see CI-OMS interface). Need to see
# whether it also applies to CGSN so eventually adjustments may be needed.
#
else:
# note that int(x) returns a long object if needed.
from_time = int(self._last_ts) + _DELTA_TIME
log.debug("%r: _retrieve_attribute_values: attr_ids=%r from_time=%s",
self._platform_id, self._attr_ids, from_time)
retrieved_vals = self._get_attribute_values(self._attr_ids, from_time)
log.debug("%r: _retrieve_attribute_values: _get_attribute_values "
"for attr_ids=%r and from_time=%s returned %s",
self._platform_id,
self._attr_ids, from_time, retrieved_vals)
# vals_dict: attributes with non-empty reported values:
vals_dict = {}
for attr_id in self._attr_ids:
if not attr_id in retrieved_vals:
log.warn("%r: _retrieve_attribute_values: unexpected: "
"response does not include requested attribute %r. "
"Response is: %s",
self._platform_id, attr_id, retrieved_vals)
continue
attr_vals = retrieved_vals[attr_id]
if not attr_vals:
log.debug("%r: No values reported for attribute=%r from_time=%f",
self._platform_id, attr_id, from_time)
continue
if log.isEnabledFor(logging.DEBUG):
self._debug_values_retrieved(attr_id, attr_vals)
# ok, include this attribute for the notification:
vals_dict[attr_id] = attr_vals
if vals_dict:
self._values_retrieved(vals_dict)
def validate_request(self, ion_actor_id, expiry, in_whitelist=False):
# There is no point in looking up an anonymous user - so return default values.
if ion_actor_id == DEFAULT_ACTOR_ID:
# Since this is an anonymous request, there really is no expiry associated with it
if not in_whitelist and self.require_login:
raise Unauthorized("Anonymous access not permitted")
else:
return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY
try:
user = self.idm_client.read_actor_identity(
actor_id=ion_actor_id, headers=self._get_gateway_headers())
except NotFound as e:
if not in_whitelist and self.require_login:
# This could be a restart of the system with a new preload.
# TODO: Invalidate Flask sessions on relaunch/bootstrap with creating new secret
user_session = get_auth()
if user_session.get("actor_id", None) == ion_actor_id:
clear_auth()
raise Unauthorized("Invalid identity", exc_id="01.10")
else:
# If the user isn't found default to anonymous
return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY
# Need to convert to int first in order to compare against current time.
try:
int_expiry = int(expiry)
except Exception as ex:
raise Inconsistent(
"Unable to read the expiry value in the request '%s' as an int"
% expiry)
# The user has been validated as being known in the system, so not check the expiry and raise exception if
# the expiry is not set to 0 and less than the current time.
if 0 < int_expiry < current_time_millis():
if not in_whitelist and self.require_login:
raise Unauthorized("User authentication expired")
else:
log.warn("User authentication expired")
return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY
return ion_actor_id, expiry
def validate_request(self, ion_actor_id, expiry, in_whitelist=False):
# There is no point in looking up an anonymous user - so return default values.
if ion_actor_id == DEFAULT_ACTOR_ID:
# Since this is an anonymous request, there really is no expiry associated with it
if not in_whitelist and self.require_login:
raise Unauthorized("Anonymous access not permitted")
else:
return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY
try:
user = self.idm_client.read_actor_identity(actor_id=ion_actor_id, headers=self._get_gateway_headers())
except NotFound as e:
if not in_whitelist and self.require_login:
# This could be a restart of the system with a new preload.
# TODO: Invalidate Flask sessions on relaunch/bootstrap with creating new secret
user_session = get_auth()
if user_session.get("actor_id", None) == ion_actor_id:
clear_auth()
raise Unauthorized("Invalid identity", exc_id="01.10")
else:
# If the user isn't found default to anonymous
return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY
# Need to convert to int first in order to compare against current time.
try:
int_expiry = int(expiry)
except Exception as ex:
raise Inconsistent("Unable to read the expiry value in the request '%s' as an int" % expiry)
# The user has been validated as being known in the system, so not check the expiry and raise exception if
# the expiry is not set to 0 and less than the current time.
if 0 < int_expiry < current_time_millis():
if not in_whitelist and self.require_login:
raise Unauthorized("User authentication expired")
else:
log.warn("User authentication expired")
return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY
return ion_actor_id, expiry
def get_session(self):
"""
Returns user session information for current authentication.
This can be polled regularly by client code to detect changes in session state and expiration.
"""
def call_extend_session_attrs(session_attrs, actor_user):
""" Call UI extensions to make additions to user session """
for ext_obj in self.extension_objs:
func = getattr(ext_obj, "extend_user_session_attributes", None)
if func:
try:
func(session_attrs, actor_user)
except Exception:
log.exception("Error calling UI extension extend_user_session_attributes()")
try:
# Get user session from OAuth access token in HTTP Authorization header
auth_hdr = request.headers.get("authorization", None)
if auth_hdr:
valid, req = self.oauth.verify_request([self.oauth_scope]) # Note: Do NOT extend session timeout here!
if valid:
actor_id = flask.g.oauth_user.get("actor_id", "")
actor_user = self.idm_client.read_actor_identity(actor_id)
session_attrs = dict(is_logged_in=True, is_registered=True,
attributes={"roles": actor_user.details.contact.roles}, roles={})
if actor_user.session:
session_attrs.update(actor_user.session)
call_extend_session_attrs(session_attrs, actor_user)
return build_json_response(session_attrs)
if self.remember_user:
# Get user session from user_id/access_token placed inside server session (Cookie)
# This is a feature to allow returning users to resume a session if still valid
access_token = flask.session.get("access_token", None)
actor_id = flask.session.get("actor_id", None)
if access_token and actor_id:
actor_user = self.idm_client.read_actor_identity(actor_id)
session_attrs = dict(access_token=access_token, is_logged_in=True, is_registered=True,
attributes={"roles": actor_user.details.contact.roles}, roles={})
if actor_user.session:
# Check validity in persisted user session
if 0 < int(actor_user.session.get("valid_until", 0)) * 1000 < current_time_millis():
clear_auth()
return build_json_response(get_auth())
session_attrs.update(actor_user.session)
else:
# No trace of existing session in user object
clear_auth()
return build_json_response(get_auth())
call_extend_session_attrs(session_attrs, actor_user)
return build_json_response(session_attrs)
# Get user session from Flask session and cookie (non-token mode)
user_info = get_auth()
if 0 < int(user_info.get("valid_until", 0)) * 1000 < current_time_millis():
clear_auth() # Clear expired session
user_info = get_auth()
call_extend_session_attrs(user_info, None)
return build_json_response(user_info)
except Exception:
return build_json_error()
def get_session(self):
"""
Returns user session information for current authentication.
This can be polled regularly by client code to detect changes in session state and expiration.
"""
def call_extend_session_attrs(session_attrs, actor_user):
""" Call UI extensions to make additions to user session """
for ext_obj in self.extension_objs:
func = getattr(ext_obj, "extend_user_session_attributes", None)
if func:
try:
func(session_attrs, actor_user)
except Exception:
log.exception("Error calling UI extension extend_user_session_attributes()")
try:
# Get user session from OAuth access token in HTTP Authorization header
auth_hdr = request.headers.get("authorization", None)
if auth_hdr:
valid, req = self.oauth.verify_request([self.oauth_scope]) # Note: Do NOT extend session timeout here!
if valid:
actor_id = flask.g.oauth_user.get("actor_id", "")
actor_user = self.idm_client.read_actor_identity(actor_id)
session_attrs = dict(is_logged_in=True, is_registered=True,
attributes={"roles": actor_user.details.contact.roles}, roles={})
if actor_user.session:
session_attrs.update(actor_user.session)
call_extend_session_attrs(session_attrs, actor_user)
return build_json_response(session_attrs)
if self.remember_user:
# Get user session from user_id/access_token placed inside server session (Cookie)
# This is a feature to allow returning users to resume a session if still valid
access_token = flask.session.get("access_token", None)
actor_id = flask.session.get("actor_id", None)
if access_token and actor_id:
actor_user = self.idm_client.read_actor_identity(actor_id)
session_attrs = dict(access_token=access_token, is_logged_in=True, is_registered=True,
attributes={"roles": actor_user.details.contact.roles}, roles={})
if actor_user.session:
# Check validity in persisted user session
if 0 < int(actor_user.session.get("valid_until", 0)) * 1000 < current_time_millis():
clear_auth()
return build_json_response(get_auth())
session_attrs.update(actor_user.session)
else:
# No trace of existing session in user object
clear_auth()
return build_json_response(get_auth())
call_extend_session_attrs(session_attrs, actor_user)
return build_json_response(session_attrs)
# Get user session from Flask session and cookie (non-token mode)
user_info = get_auth()
if 0 < int(user_info.get("valid_until", 0)) * 1000 < current_time_millis():
clear_auth() # Clear expired session
user_info = get_auth()
call_extend_session_attrs(user_info, None)
return build_json_response(user_info)
except Exception:
return build_json_error()
def test_resource_monitoring_recent(self):
#
# https://jira.oceanobservatories.org/tasks/browse/OOIION-1372
#
# Verifies that the requests for attribute values are always for
# the most recent ones, meaning that the retrieved values should *not*
# be older than a small multiple of the nominal monitoring rate, even
# after a long period in non-monitoring state.
# See ResourceMonitor._retrieve_attribute_values
#
# start this test as in test_resource_monitoring()
self.test_resource_monitoring()
# which completes right after stopping monitoring. We want that initial
# start/stop-monitoring phase to make this test more comprehensive.
self._assert_state(PlatformAgentState.COMMAND)
# now, the rest of this test does the following:
# - pick an attribute to use as a basis for the time parameters to
# be used in the test
# - wait for a while in the current non-monitoring mode
# - re-enable monitoring
# - wait for a sample to be published
# - verify that new received data sample is "recent"
# - stop monitoring
# first, use an attribute (from the root platform being tested) with
# a minimal monitoring rate, since that attribute should be reported
# in a first sample received after re-enabling the monitoring.
attr = None
for attr_id, plat_attr in self._platform_attributes[self.PLATFORM_ID].iteritems():
if attr is None or \
float(plat_attr['monitor_cycle_seconds']) < float(attr['monitor_cycle_seconds']):
attr = plat_attr
self.assertIsNotNone(attr,
"some attribute expected to be defined for %r to "
"actually proceed with this test" % self.PLATFORM_ID)
attr_id = attr['attr_id']
monitor_cycle_seconds = attr['monitor_cycle_seconds']
log.info("test_resource_monitoring_recent: using attr_id=%r: monitor_cycle_seconds=%s",
attr_id, monitor_cycle_seconds)
# sleep for twice the interval defining "recent":
from ion.agents.platform.resource_monitor import _MULT_INTERVAL
time_to_sleep = 2 * (_MULT_INTERVAL * monitor_cycle_seconds)
log.info("test_resource_monitoring_recent: sleeping for %s secs "
"before resuming monitoring", time_to_sleep)
sleep(time_to_sleep)
# reset the variables associated with the _wait_for_a_data_sample call below:
self._samples_received = []
self._async_data_result = AsyncResult()
#################################################
# re-start monitoring and wait for new sample:
log.info("test_resource_monitoring_recent: re-starting monitoring")
self._start_resource_monitoring(recursion=False)
# should also work with recursion to children but set recursion=False
# to avoid wasting the extra time in this test.
try:
self._wait_for_a_data_sample()
# get current time here (right after receiving sample) for comparison below:
curr_time_millis = current_time_millis()
# verify that the timestamp of the received sample is not too old.
# For this, use the minimum of the reported timestamps:
rdt = RecordDictionaryTool.load_from_granule(self._samples_received[0])
log.trace("test_resource_monitoring_recent: rdt:\n%s", rdt.pretty_print())
temporal_parameter_name = rdt.temporal_parameter
times = rdt[temporal_parameter_name]
log.trace("test_resource_monitoring_recent: times:\n%s", self._pp.pformat(times))
# minimum reported timestamp (note the NTP -> ION_time conversion):
min_reported_time_ntp = min(times)
min_reported_time_millis = float(ntp_2_ion_ts(min_reported_time_ntp))
log.info("test_resource_monitoring_recent: sample received, min_reported_time_millis=%s",
int(min_reported_time_millis))
# finally verify that it is actually not older than the small multiple
# of monitor_cycle_seconds plus some additional tolerance (which is
# arbitrarily set here to 10 secs):
lower_limit_millis = \
curr_time_millis - 1000 * (_MULT_INTERVAL * monitor_cycle_seconds + 10)
self.assertGreaterEqual(
min_reported_time_millis, lower_limit_millis,
"min_reported_time_millis=%s must be >= %s. Diff=%s millis" % (
min_reported_time_millis, lower_limit_millis,
min_reported_time_millis - lower_limit_millis))
finally:
self._stop_resource_monitoring(recursion=False)
def _retrieve_attribute_values(self):
"""
Retrieves the attribute values using the given function and calls
_values_retrieved.
"""
# TODO: note that the "from_time" parameters for the request below
# as well as the expected response are influenced by the RSN case
# (see CI-OMS interface). Need to see whether it also applies to
# CGSN so eventually adjustments may be needed.
#
current_time_secs = current_time_millis() / 1000.0
# determine each from_time for the request:
attrs = []
for attr_id in self._attr_ids:
if self._last_ts[attr_id] is None:
# Arbitrarily setting from_time to current system time minus a few seconds:
# TODO: determine actual criteria here.
win_size_secs = 5
from_time = current_time_secs - win_size_secs
else:
# note that int(x) returns a long object if needed.
from_time = int(self._last_ts[attr_id]) + _DELTA_TIME
attrs.append((attr_id, from_time))
log.debug("%r: _retrieve_attribute_values: attrs=%s",
self._platform_id, attrs)
retrieved_vals = self._get_attribute_values(attrs)
if retrieved_vals is None:
# lost connection; nothing else to do here:
return
good_retrieved_vals = {}
# do validation: we expect an array of tuples (val, timestamp) for
# each attribute. If not, log a warning for the attribute and
# continue processing with the other valid attributes:
for attr_id, vals in retrieved_vals.iteritems():
if not isinstance(vals, (list, tuple)):
log.warn("%r: expecting an array for attribute %r, but got: %r",
self._platform_id, attr_id, vals)
continue
if len(vals):
if not isinstance(vals[0], (tuple, list)):
log.warn("%r: expecting elements in array to be tuples "
"(val, ts) for attribute %r, but got: %r",
self._platform_id, attr_id, vals[0])
continue
good_retrieved_vals[attr_id] = vals # even if empty array.
if not good_retrieved_vals:
# nothing else to do. TODO perhaps an additional warning?
return
if log.isEnabledFor(logging.DEBUG): # pragma: no cover
summary = {attr_id: "(%d vals)" % len(vals)
for attr_id, vals in good_retrieved_vals.iteritems()}
log.debug("%r: _retrieve_attribute_values: _get_attribute_values "
"for attrs=%s returned %s",
self._platform_id, attrs, summary)
elif log.isEnabledFor(logging.TRACE): # pragma: no cover
# show good_retrieved_vals as retrieved (might be large)
log.trace("%r: _retrieve_attribute_values: _get_attribute_values "
"for attrs=%s returned %s",
self._platform_id, attrs, good_retrieved_vals)
# vals_dict: attributes with non-empty reported values:
vals_dict = {}
for attr_id, from_time in attrs:
if not attr_id in good_retrieved_vals:
log.warn("%r: _retrieve_attribute_values: unexpected: "
"response does not include requested attribute %r. "
"Response is: %s",
self._platform_id, attr_id, good_retrieved_vals)
continue
attr_vals = good_retrieved_vals[attr_id]
if not attr_vals:
log.debug("%r: No values reported for attribute=%r from_time=%f",
self._platform_id, attr_id, from_time)
continue
if log.isEnabledFor(logging.DEBUG):
self._debug_values_retrieved(attr_id, attr_vals)
# ok, include this attribute for the notification:
vals_dict[attr_id] = attr_vals
if vals_dict:
self._values_retrieved(vals_dict)
def is_expired(self):
if current_time_millis() > self.expire_time:
return True
return False
def _retrieve_attribute_values(self):
"""
Retrieves the attribute values using the given function and calls
_values_retrieved.
"""
# TODO: note that the "from_time" parameters for the request below
# as well as the expected response are influenced by the RSN case
# (see CI-OMS interface). Need to see whether it also applies to
# CGSN so eventually adjustments may be needed.
#
current_time_secs = current_time_millis() / 1000.0
# determine each from_time for the request:
attrs = []
for attr_id in self._attr_ids:
if self._last_ts[attr_id] is None:
# Arbitrarily setting from_time to current system time minus a few seconds:
# TODO: determine actual criteria here.
win_size_secs = 5
from_time = current_time_secs - win_size_secs
else:
# note that int(x) returns a long object if needed.
from_time = int(self._last_ts[attr_id]) + _DELTA_TIME
attrs.append((attr_id, from_time))
log.debug("%r: _retrieve_attribute_values: attrs=%s",
self._platform_id, attrs)
retrieved_vals = self._get_attribute_values(attrs)
if retrieved_vals is None:
# lost connection; nothing else to do here:
return
good_retrieved_vals = {}
# do validation: we expect an array of tuples (val, timestamp) for
# each attribute. If not, log a warning for the attribute and
# continue processing with the other valid attributes:
for attr_id, vals in retrieved_vals.iteritems():
if not isinstance(vals, (list, tuple)):
log.warn(
"%r: expecting an array for attribute %r, but got: %r",
self._platform_id, attr_id, vals)
continue
if len(vals):
if not isinstance(vals[0], (tuple, list)):
log.warn(
"%r: expecting elements in array to be tuples "
"(val, ts) for attribute %r, but got: %r",
self._platform_id, attr_id, vals[0])
continue
good_retrieved_vals[attr_id] = vals # even if empty array.
if not good_retrieved_vals:
# nothing else to do. TODO perhaps an additional warning?
return
if log.isEnabledFor(logging.TRACE): # pragma: no cover
# show good_retrieved_vals as retrieved (might be large)
log.trace(
"%r: _retrieve_attribute_values: _get_attribute_values "
"for attrs=%s returned\n%s", self._platform_id, attrs,
self._pp.pformat(good_retrieved_vals))
elif log.isEnabledFor(logging.DEBUG): # pragma: no cover
summary = {
attr_id: "(%d vals)" % len(vals)
for attr_id, vals in good_retrieved_vals.iteritems()
}
log.debug(
"%r: _retrieve_attribute_values: _get_attribute_values "
"for attrs=%s returned %s", self._platform_id, attrs, summary)
# vals_dict: attributes with non-empty reported values:
vals_dict = {}
for attr_id, from_time in attrs:
if not attr_id in good_retrieved_vals:
log.warn(
"%r: _retrieve_attribute_values: unexpected: "
"response does not include requested attribute %r. "
"Response is: %s", self._platform_id, attr_id,
good_retrieved_vals)
continue
attr_vals = good_retrieved_vals[attr_id]
if not attr_vals:
log.debug(
"%r: No values reported for attribute=%r from_time=%f",
self._platform_id, attr_id, from_time)
continue
if log.isEnabledFor(logging.DEBUG):
self._debug_values_retrieved(attr_id, attr_vals)
# ok, include this attribute for the notification:
vals_dict[attr_id] = attr_vals
if vals_dict:
self._values_retrieved(vals_dict)
def _retrieve_attribute_values(self):
"""
Retrieves the attribute values using the given function and calls
_values_retrieved.
"""
# TODO: note that the "from_time" parameters for the request below
# as well as the expected response are influenced by the RSN case
# (see CI-OMS interface). Need to see whether it also applies to
# CGSN so eventually adjustments may be needed.
#
# note that the "from_time" parameter in each pair (attr_id, from_time)
# for the _get_attribute_values call below, is in millis in UNIX epoch.
curr_time_millis = current_time_millis()
# minimum value for the from_time parameter (OOIION-1372):
min_from_time = curr_time_millis - 1000 * _MULT_INTERVAL * self._rate_secs
# this corresponds to (_MULT_INTERVAL * self._rate_secs) ago.
# determine each from_time for the request:
attrs = []
for attr_id in self._attr_ids:
if self._last_ts_millis[attr_id] is None:
# Very first request for this attribute. Use min_from_time:
from_time = min_from_time
else:
# We've already got values for this attribute. Use the latest
# timestamp + _DELTA_TIME as a basis for the new request:
from_time = int(self._last_ts_millis[attr_id]) + _DELTA_TIME
# but adjust it if it goes too far in the past:
if from_time < min_from_time:
from_time = min_from_time
log.trace("test_resource_monitoring_recent: attr_id=%s, from_time=%s, %s millis ago",
attr_id, from_time, curr_time_millis - from_time)
attrs.append((attr_id, from_time))
log.debug("%r: _retrieve_attribute_values: attrs=%s",
self._platform_id, attrs)
retrieved_vals = self._get_attribute_values(attrs)
if retrieved_vals is None:
# lost connection; nothing else to do here:
return
good_retrieved_vals = {}
# do validation: we expect an array of tuples (val, timestamp) for
# each attribute. If not, log a warning for the attribute and
# continue processing with the other valid attributes:
for attr_id, vals in retrieved_vals.iteritems():
if not isinstance(vals, (list, tuple)):
log.warn("%r: expecting an array for attribute %r, but got: %r",
self._platform_id, attr_id, vals)
continue
if len(vals):
if not isinstance(vals[0], (tuple, list)):
log.warn("%r: expecting elements in array to be tuples "
"(val, ts) for attribute %r, but got: %r",
self._platform_id, attr_id, vals[0])
continue
good_retrieved_vals[attr_id] = vals # even if empty array.
if not good_retrieved_vals:
# nothing else to do. TODO perhaps an additional warning?
return
if log.isEnabledFor(logging.TRACE): # pragma: no cover
# show good_retrieved_vals as retrieved (might be large)
log.trace("%r: _retrieve_attribute_values: _get_attribute_values "
"for attrs=%s returned\n%s",
self._platform_id, attrs, self._pp.pformat(good_retrieved_vals))
elif log.isEnabledFor(logging.DEBUG): # pragma: no cover
summary = {attr_id: "(%d vals)" % len(vals)
for attr_id, vals in good_retrieved_vals.iteritems()}
log.debug("%r: _retrieve_attribute_values: _get_attribute_values "
"for attrs=%s returned %s",
self._platform_id, attrs, summary)
# vals_dict: attributes with non-empty reported values:
vals_dict = {}
for attr_id, from_time in attrs:
if not attr_id in good_retrieved_vals:
log.warn("%r: _retrieve_attribute_values: unexpected: "
"response does not include requested attribute %r. "
"Response is: %s",
self._platform_id, attr_id, good_retrieved_vals)
continue
attr_vals = good_retrieved_vals[attr_id]
if not attr_vals:
log.debug("%r: No values reported for attribute=%r from_time=%f",
self._platform_id, attr_id, from_time)
continue
if log.isEnabledFor(logging.DEBUG):
self._debug_values_retrieved(attr_id, attr_vals)
# ok, include this attribute for the notification:
vals_dict[attr_id] = attr_vals
if vals_dict:
self._values_retrieved(vals_dict)
def _retrieve_attribute_values(self):
"""
Retrieves the attribute values using the given function and calls
_values_retrieved.
"""
# TODO: note that the "from_time" parameters for the request below was
# influenced by the RSN case (see CI-OMS interface). Need to see
# whether it also applies to CGSN so eventually adjustments may be needed.
#
current_time_secs = current_time_millis() / 1000.0
# determine each from_time for the request:
attrs = []
for attr_id in self._attr_ids:
if self._last_ts[attr_id] is None:
# Arbitrarily setting from_time to current system time minus a few seconds:
# TODO: determine actual criteria here.
win_size_secs = 5
from_time = current_time_secs - win_size_secs
else:
# note that int(x) returns a long object if needed.
from_time = int(self._last_ts[attr_id]) + _DELTA_TIME
attrs.append((attr_id, from_time))
log.debug("%r: _retrieve_attribute_values: attrs=%s",
self._platform_id, attrs)
retrieved_vals = self._get_attribute_values(attrs)
log.debug("%r: _retrieve_attribute_values: _get_attribute_values "
"for attrs=%s returned %s",
self._platform_id, attrs, retrieved_vals)
# vals_dict: attributes with non-empty reported values:
vals_dict = {}
for attr_id, from_time in attrs:
if not attr_id in retrieved_vals:
log.warn("%r: _retrieve_attribute_values: unexpected: "
"response does not include requested attribute %r. "
"Response is: %s",
self._platform_id, attr_id, retrieved_vals)
continue
attr_vals = retrieved_vals[attr_id]
if not attr_vals:
log.debug("%r: No values reported for attribute=%r from_time=%f",
self._platform_id, attr_id, from_time)
continue
if log.isEnabledFor(logging.DEBUG):
self._debug_values_retrieved(attr_id, attr_vals)
# ok, include this attribute for the notification:
vals_dict[attr_id] = attr_vals
if vals_dict:
self._values_retrieved(vals_dict)
def _retrieve_attribute_values(self):
"""
Retrieves the attribute values using the given function and calls
_values_retrieved.
"""
# TODO: note that the "from_time" parameters for the request below
# as well as the expected response are influenced by the RSN case
# (see CI-OMS interface). Need to see whether it also applies to
# CGSN so eventually adjustments may be needed.
#
# note that the "from_time" parameter in each pair (attr_id, from_time)
# for the _get_attribute_values call below, is in millis in UNIX epoch.
curr_time_millis = current_time_millis()
# minimum value for the from_time parameter (OOIION-1372):
min_from_time = curr_time_millis - 1000 * _MULT_INTERVAL * self._rate_secs
# this corresponds to (_MULT_INTERVAL * self._rate_secs) ago.
# determine each from_time for the request:
attrs = []
for attr_id in self._attr_ids:
if self._last_ts_millis[attr_id] is None:
# Very first request for this attribute. Use min_from_time:
from_time = min_from_time
else:
# We've already got values for this attribute. Use the latest
# timestamp + _DELTA_TIME as a basis for the new request:
from_time = int(self._last_ts_millis[attr_id]) + _DELTA_TIME
# but adjust it if it goes too far in the past:
if from_time < min_from_time:
from_time = min_from_time
log.trace(
"test_resource_monitoring_recent: attr_id=%s, from_time=%s, %s millis ago",
attr_id, from_time, curr_time_millis - from_time)
attrs.append((attr_id, from_time))
log.debug("%r: _retrieve_attribute_values: attrs=%s",
self._platform_id, attrs)
retrieved_vals = self._get_attribute_values(attrs)
if retrieved_vals is None:
# lost connection; nothing else to do here:
return
good_retrieved_vals = {}
# do validation: we expect an array of tuples (val, timestamp) for
# each attribute. If not, log a warning for the attribute and
# continue processing with the other valid attributes:
for attr_id, vals in retrieved_vals.iteritems():
if not isinstance(vals, (list, tuple)):
log.warn(
"%r: expecting an array for attribute %r, but got: %r",
self._platform_id, attr_id, vals)
continue
if len(vals):
if not isinstance(vals[0], (tuple, list)):
log.warn(
"%r: expecting elements in array to be tuples "
"(val, ts) for attribute %r, but got: %r",
self._platform_id, attr_id, vals[0])
continue
good_retrieved_vals[attr_id] = vals # even if empty array.
if not good_retrieved_vals:
# nothing else to do. TODO perhaps an additional warning?
return
if log.isEnabledFor(logging.TRACE): # pragma: no cover
# show good_retrieved_vals as retrieved (might be large)
log.trace(
"%r: _retrieve_attribute_values: _get_attribute_values "
"for attrs=%s returned\n%s", self._platform_id, attrs,
self._pp.pformat(good_retrieved_vals))
elif log.isEnabledFor(logging.DEBUG): # pragma: no cover
summary = {
attr_id: "(%d vals)" % len(vals)
for attr_id, vals in good_retrieved_vals.iteritems()
}
log.debug(
"%r: _retrieve_attribute_values: _get_attribute_values "
"for attrs=%s returned %s", self._platform_id, attrs, summary)
# vals_dict: attributes with non-empty reported values:
vals_dict = {}
for attr_id, from_time in attrs:
if not attr_id in good_retrieved_vals:
log.warn(
"%r: _retrieve_attribute_values: unexpected: "
"response does not include requested attribute %r. "
"Response is: %s", self._platform_id, attr_id,
good_retrieved_vals)
continue
attr_vals = good_retrieved_vals[attr_id]
if not attr_vals:
log.debug(
"%r: No values reported for attribute=%r from_time=%f",
self._platform_id, attr_id, from_time)
continue
if log.isEnabledFor(logging.DEBUG):
self._debug_values_retrieved(attr_id, attr_vals)
# ok, include this attribute for the notification:
vals_dict[attr_id] = attr_vals
if vals_dict:
self._values_retrieved(vals_dict)
def get_governance_info_from_request(self, json_params=None):
# Default values for governance headers.
actor_id = DEFAULT_ACTOR_ID
expiry = DEFAULT_EXPIRY
authtoken = ""
user_session = get_auth()
#if user_session.get("actor_id", None) and user_session.get("valid_until", 0):
if user_session.get("actor_id", None):
# Get info from current server session
# NOTE: Actor id may be inside server session
expiry = int(user_session.get("valid_until", 0)) * 1000
if expiry:
# This was a proper non-token server session authentication
expiry = str(expiry)
actor_id = user_session["actor_id"]
log.info(
"Request associated with session actor_id=%s, expiry=%s",
actor_id, expiry)
else:
# We are just taking the user_id out of the session
# TODO: Need to check access token here
expiry = str(expiry)
if self.token_from_session:
actor_id = user_session["actor_id"]
log.info(
"Request associated with actor's token from session; actor_id=%s, expiry=%s",
actor_id, expiry)
# Developer access using api_key
if self.develop_mode and "api_key" in request.args and request.args[
"api_key"]:
actor_id = str(request.args["api_key"])
expiry = str(int(user_session.get("valid_until", 0)) * 1000)
if 0 < int(expiry) < current_time_millis():
expiry = str(current_time_millis() + 10000)
# flask.session["valid_until"] = int(expiry / 1000)
log.info(
"Request associated with actor_id=%s, expiry=%s from developer api_key",
actor_id, expiry)
# Check in headers for OAuth2 bearer token
auth_hdr = request.headers.get("authorization", None)
if auth_hdr:
valid, req = self.process.oauth.verify_request(
[self.process.oauth_scope])
if valid:
actor_id = flask.g.oauth_user.get("actor_id", "")
if actor_id:
log.info(
"Request associated with actor_id=%s, expiry=%s from OAuth token",
actor_id, expiry)
return actor_id, DEFAULT_EXPIRY
# Try to find auth token override
if not authtoken:
if json_params:
if "authtoken" in json_params:
authtoken = json_params["authtoken"]
else:
if "authtoken" in request.args:
authtoken = str(request.args["authtoken"])
# Enable temporary authentication tokens to resolve to actor ids
if authtoken:
try:
token_info = self.idm_client.check_authentication_token(
authtoken, headers=self._get_gateway_headers())
actor_id = token_info.get("actor_id", actor_id)
expiry = token_info.get("expiry", expiry)
log.info("Resolved token %s into actor_id=%s expiry=%s",
authtoken, actor_id, expiry)
except NotFound:
log.info("Provided authentication token not found: %s",
authtoken)
except Unauthorized:
log.info("Authentication token expired or invalid: %s",
authtoken)
except Exception as ex:
log.exception("Problem resolving authentication token")
return actor_id, expiry
def test_resource_monitoring_recent(self):
#
# https://jira.oceanobservatories.org/tasks/browse/OOIION-1372
#
# Verifies that the requests for attribute values are always for
# the most recent ones, meaning that the retrieved values should *not*
# be older than a small multiple of the nominal monitoring rate, even
# after a long period in non-monitoring state.
# See ResourceMonitor._retrieve_attribute_values
#
# start this test as in test_resource_monitoring()
self.test_resource_monitoring()
# which completes right after stopping monitoring. We want that initial
# start/stop-monitoring phase to make this test more comprehensive.
self._assert_state(PlatformAgentState.COMMAND)
# now, the rest of this test does the following:
# - pick an attribute to use as a basis for the time parameters to
# be used in the test
# - wait for a while in the current non-monitoring mode
# - re-enable monitoring
# - wait for a sample to be published
# - verify that new received data sample is "recent"
# - stop monitoring
# first, use an attribute (from the root platform being tested) with
# a minimal monitoring rate, since that attribute should be reported
# in a first sample received after re-enabling the monitoring.
attr = None
for attr_id, plat_attr in self._platform_attributes[
self.PLATFORM_ID].iteritems():
if attr is None or \
float(plat_attr['monitor_cycle_seconds']) < float(attr['monitor_cycle_seconds']):
attr = plat_attr
self.assertIsNotNone(
attr, "some attribute expected to be defined for %r to "
"actually proceed with this test" % self.PLATFORM_ID)
attr_id = attr['attr_id']
monitor_cycle_seconds = attr['monitor_cycle_seconds']
log.info(
"test_resource_monitoring_recent: using attr_id=%r: monitor_cycle_seconds=%s",
attr_id, monitor_cycle_seconds)
# sleep for twice the interval defining "recent":
from ion.agents.platform.resource_monitor import _MULT_INTERVAL
time_to_sleep = 2 * (_MULT_INTERVAL * monitor_cycle_seconds)
log.info(
"test_resource_monitoring_recent: sleeping for %s secs "
"before resuming monitoring", time_to_sleep)
sleep(time_to_sleep)
# reset the variables associated with the _wait_for_a_data_sample call below:
self._samples_received = []
self._async_data_result = AsyncResult()
#################################################
# re-start monitoring and wait for new sample:
log.info("test_resource_monitoring_recent: re-starting monitoring")
self._start_resource_monitoring(recursion=False)
# should also work with recursion to children but set recursion=False
# to avoid wasting the extra time in this test.
try:
self._wait_for_a_data_sample()
# get current time here (right after receiving sample) for comparison below:
curr_time_millis = current_time_millis()
# verify that the timestamp of the received sample is not too old.
# For this, use the minimum of the reported timestamps:
rdt = RecordDictionaryTool.load_from_granule(
self._samples_received[0])
log.trace("test_resource_monitoring_recent: rdt:\n%s",
rdt.pretty_print())
temporal_parameter_name = rdt.temporal_parameter
times = rdt[temporal_parameter_name]
log.trace("test_resource_monitoring_recent: times:\n%s",
self._pp.pformat(times))
# minimum reported timestamp (note the NTP -> ION_time conversion):
min_reported_time_ntp = min(times)
min_reported_time_millis = float(
ntp_2_ion_ts(min_reported_time_ntp))
log.info(
"test_resource_monitoring_recent: sample received, min_reported_time_millis=%s",
int(min_reported_time_millis))
# finally verify that it is actually not older than the small multiple
# of monitor_cycle_seconds plus some additional tolerance (which is
# arbitrarily set here to 10 secs):
lower_limit_millis = \
curr_time_millis - 1000 * (_MULT_INTERVAL * monitor_cycle_seconds + 10)
self.assertGreaterEqual(
min_reported_time_millis, lower_limit_millis,
"min_reported_time_millis=%s must be >= %s. Diff=%s millis" %
(min_reported_time_millis, lower_limit_millis,
min_reported_time_millis - lower_limit_millis))
finally:
self._stop_resource_monitoring(recursion=False)
except NotFound, e:
ion_actor_id = DEFAULT_ACTOR_ID # If the user isn't found default to anonymous
expiry = DEFAULT_EXPIRY #Since this is now an anonymous request, there really is no expiry associated with it
return ion_actor_id, expiry
#need to convert to a float first in order to compare against current time.
try:
int_expiry = int(expiry)
except Exception, e:
raise Inconsistent(
"Unable to read the expiry value in the request '%s' as an int" %
expiry)
#The user has been validated as being known in the system, so not check the expiry and raise exception if
# the expiry is not set to 0 and less than the current time.
if int_expiry > 0 and int_expiry < current_time_millis():
raise Unauthorized(
'The certificate associated with the user and expiry time in the request has expired.'
)
return ion_actor_id, expiry
def build_message_headers(ion_actor_id, expiry):
headers = dict()
headers['ion-actor-id'] = ion_actor_id
headers['expiry'] = expiry
#If this is an anonymous requester then there are no roles associated with the request
def is_expired(self):
if current_time_millis() > self.expire_time:
return True
return False
def get_governance_info_from_request(self, json_params=None):
# Default values for governance headers.
actor_id = DEFAULT_ACTOR_ID
expiry = DEFAULT_EXPIRY
authtoken = ""
user_session = get_auth()
#if user_session.get("actor_id", None) and user_session.get("valid_until", 0):
if user_session.get("actor_id", None):
# Get info from current server session
# NOTE: Actor id may be inside server session
expiry = int(user_session.get("valid_until", 0)) * 1000
if expiry:
# This was a proper non-token server session authentication
expiry = str(expiry)
actor_id = user_session["actor_id"]
log.info("Request associated with session actor_id=%s, expiry=%s", actor_id, expiry)
else:
# We are just taking the user_id out of the session
# TODO: Need to check access token here
expiry = str(expiry)
if self.token_from_session:
actor_id = user_session["actor_id"]
log.info("Request associated with actor's token from session; actor_id=%s, expiry=%s", actor_id, expiry)
# Developer access using api_key
if self.develop_mode and "api_key" in request.args and request.args["api_key"]:
actor_id = str(request.args["api_key"])
expiry = str(int(user_session.get("valid_until", 0)) * 1000)
if 0 < int(expiry) < current_time_millis():
expiry = str(current_time_millis() + 10000)
# flask.session["valid_until"] = int(expiry / 1000)
log.info("Request associated with actor_id=%s, expiry=%s from developer api_key", actor_id, expiry)
# Check in headers for OAuth2 bearer token
auth_hdr = request.headers.get("authorization", None)
if auth_hdr:
valid, req = self.process.oauth.verify_request([self.process.oauth_scope])
if valid:
actor_id = flask.g.oauth_user.get("actor_id", "")
if actor_id:
log.info("Request associated with actor_id=%s, expiry=%s from OAuth token", actor_id, expiry)
return actor_id, DEFAULT_EXPIRY
# Try to find auth token override
if not authtoken:
if json_params:
if "authtoken" in json_params:
authtoken = json_params["authtoken"]
else:
if "authtoken" in request.args:
authtoken = str(request.args["authtoken"])
# Enable temporary authentication tokens to resolve to actor ids
if authtoken:
try:
if authtoken.startswith(("Bearer_")):
# Backdoor way for OAuth2 access tokens as request args for GET URLs
authtoken = authtoken[7:]
token_id = "access_token_" + str(authtoken)
token_obj = self.process.container.object_store.read(token_id)
token = OAuthTokenObj.from_security_token(token_obj)
if token.is_valid(check_expiry=True):
actor_id = token.user["actor_id"]
expiry = str(token._token_obj.expires)
log.info("Resolved OAuth2 token %s into actor_id=%s expiry=%s", authtoken, actor_id, expiry)
else:
token_info = self.idm_client.check_authentication_token(authtoken, headers=self._get_gateway_headers())
actor_id = token_info.get("actor_id", actor_id)
expiry = token_info.get("expiry", expiry)
log.info("Resolved token %s into actor_id=%s expiry=%s", authtoken, actor_id, expiry)
except NotFound:
log.info("Provided authentication token not found: %s", authtoken)
except Unauthorized:
log.info("Authentication token expired or invalid: %s", authtoken)
except Exception as ex:
log.exception("Problem resolving authentication token")
return actor_id, expiry
try:
user = idm_client.read_actor_identity(actor_id=ion_actor_id, headers={"ion-actor-id": service_gateway_instance.name, 'expiry': DEFAULT_EXPIRY })
except NotFound, e:
ion_actor_id = DEFAULT_ACTOR_ID # If the user isn't found default to anonymous
expiry = DEFAULT_EXPIRY #Since this is now an anonymous request, there really is no expiry associated with it
return ion_actor_id, expiry
#need to convert to a float first in order to compare against current time.
try:
int_expiry = int(expiry)
except Exception, e:
raise Inconsistent("Unable to read the expiry value in the request '%s' as an int" % expiry)
#The user has been validated as being known in the system, so not check the expiry and raise exception if
# the expiry is not set to 0 and less than the current time.
if int_expiry > 0 and int_expiry < current_time_millis():
raise Unauthorized('The certificate associated with the user and expiry time in the request has expired.')
return ion_actor_id, expiry
def build_message_headers( ion_actor_id, expiry):
headers = dict()
headers['ion-actor-id'] = ion_actor_id
headers['expiry'] = expiry
#If this is an anonymous requester then there are no roles associated with the request
if ion_actor_id == DEFAULT_ACTOR_ID:
headers['ion-actor-roles'] = dict()
def is_expired(self):
return current_time_millis() > self.expire_time
