def get_governance_info_from_request(self, json_params=None): # Default values for governance headers. actor_id = DEFAULT_ACTOR_ID expiry = DEFAULT_EXPIRY authtoken = "" user_session = get_auth() if user_session.get("actor_id", None) and user_session.get("valid_until", 0): # Get info from current server session # NOTE: Actor id may be inside server session actor_id = user_session["actor_id"] expiry = str(int(user_session.get("valid_until", 0)) * 1000) log.info("Request associated with session actor_id=%s, expiry=%s", actor_id, expiry) # Developer access using api_key if self.develop_mode and "api_key" in request.args and request.args["api_key"]: actor_id = str(request.args["api_key"]) expiry = str(int(user_session.get("valid_until", 0)) * 1000) if 0 < int(expiry) < current_time_millis(): expiry = str(current_time_millis() + 10000) # flask.session["valid_until"] = int(expiry / 1000) log.info("Request associated with actor_id=%s, expiry=%s from developer api_key", actor_id, expiry) # Check in headers for OAuth2 bearer token auth_hdr = request.headers.get("authorization", None) if auth_hdr: valid, req = self.process.oauth.verify_request([self.process.oauth_scope]) if valid: actor_id = flask.g.oauth_user.get("actor_id", "") if actor_id: log.info("Request associated with actor_id=%s, expiry=%s from OAuth token", actor_id, expiry) return actor_id, DEFAULT_EXPIRY # Try to find auth token override if not authtoken: if json_params: if "authtoken" in json_params: authtoken = json_params["authtoken"] else: if "authtoken" in request.args: authtoken = str(request.args["authtoken"]) # Enable temporary authentication tokens to resolve to actor ids if authtoken: try: token_info = self.idm_client.check_authentication_token(authtoken, headers=self._get_gateway_headers()) actor_id = token_info.get("actor_id", actor_id) expiry = token_info.get("expiry", expiry) log.info("Resolved token %s into actor_id=%s expiry=%s", authtoken, actor_id, expiry) except NotFound: log.info("Provided authentication token not found: %s", authtoken) except Unauthorized: log.info("Authentication token expired or invalid: %s", authtoken) except Exception as ex: log.exception("Problem resolving authentication token") return actor_id, expiry
def get_session(self): try: # Get session based on OAuth token auth_hdr = request.headers.get("authorization", None) if auth_hdr: valid, req = self.oauth.verify_request([self.oauth_scope]) if valid: actor_id = flask.g.oauth_user.get("actor_id", "") actor_user = self.idm_client.read_actor_identity(actor_id) session_attrs = dict(is_logged_in=True, is_registered=True, attributes={"roles":actor_user.details.contact.roles}, roles={}) if actor_user.session: session_attrs.update(actor_user.session) return build_json_response(session_attrs) # Support quick reload access_token = flask.session.get("access_token", None) actor_id = flask.session.get("actor_id", None) if access_token and actor_id: actor_user = self.idm_client.read_actor_identity(actor_id) session_attrs = dict(access_token=access_token, is_logged_in=True, is_registered=True, attributes={"roles":actor_user.details.contact.roles}, roles={}) if actor_user.session: session_attrs.update(actor_user.session) return build_json_response(session_attrs) # Get session from Flask session and cookie user_info = get_auth() if 0 < int(user_info.get("valid_until", 0)) * 1000 < current_time_millis(): clear_auth() user_info = get_auth() return build_json_response(user_info) except Exception: return build_json_error()
def validate_request(ion_actor_id, expiry): # There is no point in looking up an anonymous user - so return default values. if ion_actor_id == DEFAULT_ACTOR_ID: expiry = DEFAULT_EXPIRY # Since this is now an anonymous request, there really is no expiry associated with it return ion_actor_id, expiry idm_client = IdentityManagementServiceProcessClient(process=service_gateway_instance) try: user = idm_client.read_actor_identity(actor_id=ion_actor_id, headers={"ion-actor-id": service_gateway_instance.name, 'expiry': DEFAULT_EXPIRY}) except NotFound as e: ion_actor_id = DEFAULT_ACTOR_ID # If the user isn't found default to anonymous expiry = DEFAULT_EXPIRY # Since this is now an anonymous request, there really is no expiry associated with it return ion_actor_id, expiry # Need to convert to a float first in order to compare against current time. try: int_expiry = int(expiry) except Exception as e: raise Inconsistent("Unable to read the expiry value in the request '%s' as an int" % expiry) # The user has been validated as being known in the system, so not check the expiry and raise exception if # the expiry is not set to 0 and less than the current time. if 0 < int_expiry < current_time_millis(): raise Unauthorized('The certificate associated with the user and expiry time in the request has expired.') return ion_actor_id, expiry
def __init__(self, originating_container, actor_id, requesting_message, token): self.originator = originating_container self.actor_id = actor_id self.requesting_message = requesting_message self.token = token timeout = CFG.get_safe('container.messaging.timeout.receive', 30) self.expire_time = current_time_millis() + (timeout * 1000) # Set the expire time to current time + timeout in ms
def _retrieve_attribute_values(self): """ Retrieves the attribute values using the given function and calls _values_retrieved. """ # determine from_time for the request: if self._last_ts is None: # This is the very first retrieval request, so pick a from_time # that makes sense. At the moment, setting from_time to be current # system minus the monitoring rate. # TODO: determine actual criteria here. from_time = current_time_millis() - self._rate_millis # TODO: Also note that the "from_time" parameter for the request was # influenced by the RSN case (see CI-OMS interface). Need to see # whether it also applies to CGSN so eventually adjustments may be needed. # else: # note that int(x) returns a long object if needed. from_time = int(self._last_ts) + _DELTA_TIME log.debug("%r: _retrieve_attribute_values: attr_ids=%r from_time=%s", self._platform_id, self._attr_ids, from_time) retrieved_vals = self._get_attribute_values(self._attr_ids, from_time) log.debug( "%r: _retrieve_attribute_values: _get_attribute_values " "for attr_ids=%r and from_time=%s returned %s", self._platform_id, self._attr_ids, from_time, retrieved_vals) # vals_dict: attributes with non-empty reported values: vals_dict = {} for attr_id in self._attr_ids: if not attr_id in retrieved_vals: log.warn( "%r: _retrieve_attribute_values: unexpected: " "response does not include requested attribute %r. " "Response is: %s", self._platform_id, attr_id, retrieved_vals) continue attr_vals = retrieved_vals[attr_id] if not attr_vals: log.debug( "%r: No values reported for attribute=%r from_time=%f", self._platform_id, attr_id, from_time) continue if log.isEnabledFor(logging.DEBUG): self._debug_values_retrieved(attr_id, attr_vals) # ok, include this attribute for the notification: vals_dict[attr_id] = attr_vals if vals_dict: self._values_retrieved(vals_dict)
def _retrieve_attribute_values(self): """ Retrieves the attribute values using the given function and calls _values_retrieved. """ # determine from_time for the request: if self._last_ts is None: # This is the very first retrieval request, so pick a from_time # that makes sense. At the moment, setting from_time to be current # system minus the monitoring rate. # TODO: determine actual criteria here. from_time = current_time_millis() - self._rate_millis # TODO: Also note that the "from_time" parameter for the request was # influenced by the RSN case (see CI-OMS interface). Need to see # whether it also applies to CGSN so eventually adjustments may be needed. # else: # note that int(x) returns a long object if needed. from_time = int(self._last_ts) + _DELTA_TIME log.debug("%r: _retrieve_attribute_values: attr_ids=%r from_time=%s", self._platform_id, self._attr_ids, from_time) retrieved_vals = self._get_attribute_values(self._attr_ids, from_time) log.debug("%r: _retrieve_attribute_values: _get_attribute_values " "for attr_ids=%r and from_time=%s returned %s", self._platform_id, self._attr_ids, from_time, retrieved_vals) # vals_dict: attributes with non-empty reported values: vals_dict = {} for attr_id in self._attr_ids: if not attr_id in retrieved_vals: log.warn("%r: _retrieve_attribute_values: unexpected: " "response does not include requested attribute %r. " "Response is: %s", self._platform_id, attr_id, retrieved_vals) continue attr_vals = retrieved_vals[attr_id] if not attr_vals: log.debug("%r: No values reported for attribute=%r from_time=%f", self._platform_id, attr_id, from_time) continue if log.isEnabledFor(logging.DEBUG): self._debug_values_retrieved(attr_id, attr_vals) # ok, include this attribute for the notification: vals_dict[attr_id] = attr_vals if vals_dict: self._values_retrieved(vals_dict)
def validate_request(self, ion_actor_id, expiry, in_whitelist=False): # There is no point in looking up an anonymous user - so return default values. if ion_actor_id == DEFAULT_ACTOR_ID: # Since this is an anonymous request, there really is no expiry associated with it if not in_whitelist and self.require_login: raise Unauthorized("Anonymous access not permitted") else: return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY try: user = self.idm_client.read_actor_identity( actor_id=ion_actor_id, headers=self._get_gateway_headers()) except NotFound as e: if not in_whitelist and self.require_login: # This could be a restart of the system with a new preload. # TODO: Invalidate Flask sessions on relaunch/bootstrap with creating new secret user_session = get_auth() if user_session.get("actor_id", None) == ion_actor_id: clear_auth() raise Unauthorized("Invalid identity", exc_id="01.10") else: # If the user isn't found default to anonymous return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY # Need to convert to int first in order to compare against current time. try: int_expiry = int(expiry) except Exception as ex: raise Inconsistent( "Unable to read the expiry value in the request '%s' as an int" % expiry) # The user has been validated as being known in the system, so not check the expiry and raise exception if # the expiry is not set to 0 and less than the current time. if 0 < int_expiry < current_time_millis(): if not in_whitelist and self.require_login: raise Unauthorized("User authentication expired") else: log.warn("User authentication expired") return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY return ion_actor_id, expiry
def validate_request(self, ion_actor_id, expiry, in_whitelist=False): # There is no point in looking up an anonymous user - so return default values. if ion_actor_id == DEFAULT_ACTOR_ID: # Since this is an anonymous request, there really is no expiry associated with it if not in_whitelist and self.require_login: raise Unauthorized("Anonymous access not permitted") else: return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY try: user = self.idm_client.read_actor_identity(actor_id=ion_actor_id, headers=self._get_gateway_headers()) except NotFound as e: if not in_whitelist and self.require_login: # This could be a restart of the system with a new preload. # TODO: Invalidate Flask sessions on relaunch/bootstrap with creating new secret user_session = get_auth() if user_session.get("actor_id", None) == ion_actor_id: clear_auth() raise Unauthorized("Invalid identity", exc_id="01.10") else: # If the user isn't found default to anonymous return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY # Need to convert to int first in order to compare against current time. try: int_expiry = int(expiry) except Exception as ex: raise Inconsistent("Unable to read the expiry value in the request '%s' as an int" % expiry) # The user has been validated as being known in the system, so not check the expiry and raise exception if # the expiry is not set to 0 and less than the current time. if 0 < int_expiry < current_time_millis(): if not in_whitelist and self.require_login: raise Unauthorized("User authentication expired") else: log.warn("User authentication expired") return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY return ion_actor_id, expiry
def get_session(self): """ Returns user session information for current authentication. This can be polled regularly by client code to detect changes in session state and expiration. """ def call_extend_session_attrs(session_attrs, actor_user): """ Call UI extensions to make additions to user session """ for ext_obj in self.extension_objs: func = getattr(ext_obj, "extend_user_session_attributes", None) if func: try: func(session_attrs, actor_user) except Exception: log.exception("Error calling UI extension extend_user_session_attributes()") try: # Get user session from OAuth access token in HTTP Authorization header auth_hdr = request.headers.get("authorization", None) if auth_hdr: valid, req = self.oauth.verify_request([self.oauth_scope]) # Note: Do NOT extend session timeout here! if valid: actor_id = flask.g.oauth_user.get("actor_id", "") actor_user = self.idm_client.read_actor_identity(actor_id) session_attrs = dict(is_logged_in=True, is_registered=True, attributes={"roles": actor_user.details.contact.roles}, roles={}) if actor_user.session: session_attrs.update(actor_user.session) call_extend_session_attrs(session_attrs, actor_user) return build_json_response(session_attrs) if self.remember_user: # Get user session from user_id/access_token placed inside server session (Cookie) # This is a feature to allow returning users to resume a session if still valid access_token = flask.session.get("access_token", None) actor_id = flask.session.get("actor_id", None) if access_token and actor_id: actor_user = self.idm_client.read_actor_identity(actor_id) session_attrs = dict(access_token=access_token, is_logged_in=True, is_registered=True, attributes={"roles": actor_user.details.contact.roles}, roles={}) if actor_user.session: # Check validity in persisted user session if 0 < int(actor_user.session.get("valid_until", 0)) * 1000 < current_time_millis(): clear_auth() return build_json_response(get_auth()) session_attrs.update(actor_user.session) else: # No trace of existing session in user object clear_auth() return build_json_response(get_auth()) call_extend_session_attrs(session_attrs, actor_user) return build_json_response(session_attrs) # Get user session from Flask session and cookie (non-token mode) user_info = get_auth() if 0 < int(user_info.get("valid_until", 0)) * 1000 < current_time_millis(): clear_auth() # Clear expired session user_info = get_auth() call_extend_session_attrs(user_info, None) return build_json_response(user_info) except Exception: return build_json_error()
def test_resource_monitoring_recent(self): # # https://jira.oceanobservatories.org/tasks/browse/OOIION-1372 # # Verifies that the requests for attribute values are always for # the most recent ones, meaning that the retrieved values should *not* # be older than a small multiple of the nominal monitoring rate, even # after a long period in non-monitoring state. # See ResourceMonitor._retrieve_attribute_values # # start this test as in test_resource_monitoring() self.test_resource_monitoring() # which completes right after stopping monitoring. We want that initial # start/stop-monitoring phase to make this test more comprehensive. self._assert_state(PlatformAgentState.COMMAND) # now, the rest of this test does the following: # - pick an attribute to use as a basis for the time parameters to # be used in the test # - wait for a while in the current non-monitoring mode # - re-enable monitoring # - wait for a sample to be published # - verify that new received data sample is "recent" # - stop monitoring # first, use an attribute (from the root platform being tested) with # a minimal monitoring rate, since that attribute should be reported # in a first sample received after re-enabling the monitoring. attr = None for attr_id, plat_attr in self._platform_attributes[self.PLATFORM_ID].iteritems(): if attr is None or \ float(plat_attr['monitor_cycle_seconds']) < float(attr['monitor_cycle_seconds']): attr = plat_attr self.assertIsNotNone(attr, "some attribute expected to be defined for %r to " "actually proceed with this test" % self.PLATFORM_ID) attr_id = attr['attr_id'] monitor_cycle_seconds = attr['monitor_cycle_seconds'] log.info("test_resource_monitoring_recent: using attr_id=%r: monitor_cycle_seconds=%s", attr_id, monitor_cycle_seconds) # sleep for twice the interval defining "recent": from ion.agents.platform.resource_monitor import _MULT_INTERVAL time_to_sleep = 2 * (_MULT_INTERVAL * monitor_cycle_seconds) log.info("test_resource_monitoring_recent: sleeping for %s secs " "before resuming monitoring", time_to_sleep) sleep(time_to_sleep) # reset the variables associated with the _wait_for_a_data_sample call below: self._samples_received = [] self._async_data_result = AsyncResult() ################################################# # re-start monitoring and wait for new sample: log.info("test_resource_monitoring_recent: re-starting monitoring") self._start_resource_monitoring(recursion=False) # should also work with recursion to children but set recursion=False # to avoid wasting the extra time in this test. try: self._wait_for_a_data_sample() # get current time here (right after receiving sample) for comparison below: curr_time_millis = current_time_millis() # verify that the timestamp of the received sample is not too old. # For this, use the minimum of the reported timestamps: rdt = RecordDictionaryTool.load_from_granule(self._samples_received[0]) log.trace("test_resource_monitoring_recent: rdt:\n%s", rdt.pretty_print()) temporal_parameter_name = rdt.temporal_parameter times = rdt[temporal_parameter_name] log.trace("test_resource_monitoring_recent: times:\n%s", self._pp.pformat(times)) # minimum reported timestamp (note the NTP -> ION_time conversion): min_reported_time_ntp = min(times) min_reported_time_millis = float(ntp_2_ion_ts(min_reported_time_ntp)) log.info("test_resource_monitoring_recent: sample received, min_reported_time_millis=%s", int(min_reported_time_millis)) # finally verify that it is actually not older than the small multiple # of monitor_cycle_seconds plus some additional tolerance (which is # arbitrarily set here to 10 secs): lower_limit_millis = \ curr_time_millis - 1000 * (_MULT_INTERVAL * monitor_cycle_seconds + 10) self.assertGreaterEqual( min_reported_time_millis, lower_limit_millis, "min_reported_time_millis=%s must be >= %s. Diff=%s millis" % ( min_reported_time_millis, lower_limit_millis, min_reported_time_millis - lower_limit_millis)) finally: self._stop_resource_monitoring(recursion=False)
def _retrieve_attribute_values(self): """ Retrieves the attribute values using the given function and calls _values_retrieved. """ # TODO: note that the "from_time" parameters for the request below # as well as the expected response are influenced by the RSN case # (see CI-OMS interface). Need to see whether it also applies to # CGSN so eventually adjustments may be needed. # current_time_secs = current_time_millis() / 1000.0 # determine each from_time for the request: attrs = [] for attr_id in self._attr_ids: if self._last_ts[attr_id] is None: # Arbitrarily setting from_time to current system time minus a few seconds: # TODO: determine actual criteria here. win_size_secs = 5 from_time = current_time_secs - win_size_secs else: # note that int(x) returns a long object if needed. from_time = int(self._last_ts[attr_id]) + _DELTA_TIME attrs.append((attr_id, from_time)) log.debug("%r: _retrieve_attribute_values: attrs=%s", self._platform_id, attrs) retrieved_vals = self._get_attribute_values(attrs) if retrieved_vals is None: # lost connection; nothing else to do here: return good_retrieved_vals = {} # do validation: we expect an array of tuples (val, timestamp) for # each attribute. If not, log a warning for the attribute and # continue processing with the other valid attributes: for attr_id, vals in retrieved_vals.iteritems(): if not isinstance(vals, (list, tuple)): log.warn("%r: expecting an array for attribute %r, but got: %r", self._platform_id, attr_id, vals) continue if len(vals): if not isinstance(vals[0], (tuple, list)): log.warn("%r: expecting elements in array to be tuples " "(val, ts) for attribute %r, but got: %r", self._platform_id, attr_id, vals[0]) continue good_retrieved_vals[attr_id] = vals # even if empty array. if not good_retrieved_vals: # nothing else to do. TODO perhaps an additional warning? return if log.isEnabledFor(logging.DEBUG): # pragma: no cover summary = {attr_id: "(%d vals)" % len(vals) for attr_id, vals in good_retrieved_vals.iteritems()} log.debug("%r: _retrieve_attribute_values: _get_attribute_values " "for attrs=%s returned %s", self._platform_id, attrs, summary) elif log.isEnabledFor(logging.TRACE): # pragma: no cover # show good_retrieved_vals as retrieved (might be large) log.trace("%r: _retrieve_attribute_values: _get_attribute_values " "for attrs=%s returned %s", self._platform_id, attrs, good_retrieved_vals) # vals_dict: attributes with non-empty reported values: vals_dict = {} for attr_id, from_time in attrs: if not attr_id in good_retrieved_vals: log.warn("%r: _retrieve_attribute_values: unexpected: " "response does not include requested attribute %r. " "Response is: %s", self._platform_id, attr_id, good_retrieved_vals) continue attr_vals = good_retrieved_vals[attr_id] if not attr_vals: log.debug("%r: No values reported for attribute=%r from_time=%f", self._platform_id, attr_id, from_time) continue if log.isEnabledFor(logging.DEBUG): self._debug_values_retrieved(attr_id, attr_vals) # ok, include this attribute for the notification: vals_dict[attr_id] = attr_vals if vals_dict: self._values_retrieved(vals_dict)
def is_expired(self): if current_time_millis() > self.expire_time: return True return False
def _retrieve_attribute_values(self): """ Retrieves the attribute values using the given function and calls _values_retrieved. """ # TODO: note that the "from_time" parameters for the request below # as well as the expected response are influenced by the RSN case # (see CI-OMS interface). Need to see whether it also applies to # CGSN so eventually adjustments may be needed. # current_time_secs = current_time_millis() / 1000.0 # determine each from_time for the request: attrs = [] for attr_id in self._attr_ids: if self._last_ts[attr_id] is None: # Arbitrarily setting from_time to current system time minus a few seconds: # TODO: determine actual criteria here. win_size_secs = 5 from_time = current_time_secs - win_size_secs else: # note that int(x) returns a long object if needed. from_time = int(self._last_ts[attr_id]) + _DELTA_TIME attrs.append((attr_id, from_time)) log.debug("%r: _retrieve_attribute_values: attrs=%s", self._platform_id, attrs) retrieved_vals = self._get_attribute_values(attrs) if retrieved_vals is None: # lost connection; nothing else to do here: return good_retrieved_vals = {} # do validation: we expect an array of tuples (val, timestamp) for # each attribute. If not, log a warning for the attribute and # continue processing with the other valid attributes: for attr_id, vals in retrieved_vals.iteritems(): if not isinstance(vals, (list, tuple)): log.warn( "%r: expecting an array for attribute %r, but got: %r", self._platform_id, attr_id, vals) continue if len(vals): if not isinstance(vals[0], (tuple, list)): log.warn( "%r: expecting elements in array to be tuples " "(val, ts) for attribute %r, but got: %r", self._platform_id, attr_id, vals[0]) continue good_retrieved_vals[attr_id] = vals # even if empty array. if not good_retrieved_vals: # nothing else to do. TODO perhaps an additional warning? return if log.isEnabledFor(logging.TRACE): # pragma: no cover # show good_retrieved_vals as retrieved (might be large) log.trace( "%r: _retrieve_attribute_values: _get_attribute_values " "for attrs=%s returned\n%s", self._platform_id, attrs, self._pp.pformat(good_retrieved_vals)) elif log.isEnabledFor(logging.DEBUG): # pragma: no cover summary = { attr_id: "(%d vals)" % len(vals) for attr_id, vals in good_retrieved_vals.iteritems() } log.debug( "%r: _retrieve_attribute_values: _get_attribute_values " "for attrs=%s returned %s", self._platform_id, attrs, summary) # vals_dict: attributes with non-empty reported values: vals_dict = {} for attr_id, from_time in attrs: if not attr_id in good_retrieved_vals: log.warn( "%r: _retrieve_attribute_values: unexpected: " "response does not include requested attribute %r. " "Response is: %s", self._platform_id, attr_id, good_retrieved_vals) continue attr_vals = good_retrieved_vals[attr_id] if not attr_vals: log.debug( "%r: No values reported for attribute=%r from_time=%f", self._platform_id, attr_id, from_time) continue if log.isEnabledFor(logging.DEBUG): self._debug_values_retrieved(attr_id, attr_vals) # ok, include this attribute for the notification: vals_dict[attr_id] = attr_vals if vals_dict: self._values_retrieved(vals_dict)
def _retrieve_attribute_values(self): """ Retrieves the attribute values using the given function and calls _values_retrieved. """ # TODO: note that the "from_time" parameters for the request below # as well as the expected response are influenced by the RSN case # (see CI-OMS interface). Need to see whether it also applies to # CGSN so eventually adjustments may be needed. # # note that the "from_time" parameter in each pair (attr_id, from_time) # for the _get_attribute_values call below, is in millis in UNIX epoch. curr_time_millis = current_time_millis() # minimum value for the from_time parameter (OOIION-1372): min_from_time = curr_time_millis - 1000 * _MULT_INTERVAL * self._rate_secs # this corresponds to (_MULT_INTERVAL * self._rate_secs) ago. # determine each from_time for the request: attrs = [] for attr_id in self._attr_ids: if self._last_ts_millis[attr_id] is None: # Very first request for this attribute. Use min_from_time: from_time = min_from_time else: # We've already got values for this attribute. Use the latest # timestamp + _DELTA_TIME as a basis for the new request: from_time = int(self._last_ts_millis[attr_id]) + _DELTA_TIME # but adjust it if it goes too far in the past: if from_time < min_from_time: from_time = min_from_time log.trace("test_resource_monitoring_recent: attr_id=%s, from_time=%s, %s millis ago", attr_id, from_time, curr_time_millis - from_time) attrs.append((attr_id, from_time)) log.debug("%r: _retrieve_attribute_values: attrs=%s", self._platform_id, attrs) retrieved_vals = self._get_attribute_values(attrs) if retrieved_vals is None: # lost connection; nothing else to do here: return good_retrieved_vals = {} # do validation: we expect an array of tuples (val, timestamp) for # each attribute. If not, log a warning for the attribute and # continue processing with the other valid attributes: for attr_id, vals in retrieved_vals.iteritems(): if not isinstance(vals, (list, tuple)): log.warn("%r: expecting an array for attribute %r, but got: %r", self._platform_id, attr_id, vals) continue if len(vals): if not isinstance(vals[0], (tuple, list)): log.warn("%r: expecting elements in array to be tuples " "(val, ts) for attribute %r, but got: %r", self._platform_id, attr_id, vals[0]) continue good_retrieved_vals[attr_id] = vals # even if empty array. if not good_retrieved_vals: # nothing else to do. TODO perhaps an additional warning? return if log.isEnabledFor(logging.TRACE): # pragma: no cover # show good_retrieved_vals as retrieved (might be large) log.trace("%r: _retrieve_attribute_values: _get_attribute_values " "for attrs=%s returned\n%s", self._platform_id, attrs, self._pp.pformat(good_retrieved_vals)) elif log.isEnabledFor(logging.DEBUG): # pragma: no cover summary = {attr_id: "(%d vals)" % len(vals) for attr_id, vals in good_retrieved_vals.iteritems()} log.debug("%r: _retrieve_attribute_values: _get_attribute_values " "for attrs=%s returned %s", self._platform_id, attrs, summary) # vals_dict: attributes with non-empty reported values: vals_dict = {} for attr_id, from_time in attrs: if not attr_id in good_retrieved_vals: log.warn("%r: _retrieve_attribute_values: unexpected: " "response does not include requested attribute %r. " "Response is: %s", self._platform_id, attr_id, good_retrieved_vals) continue attr_vals = good_retrieved_vals[attr_id] if not attr_vals: log.debug("%r: No values reported for attribute=%r from_time=%f", self._platform_id, attr_id, from_time) continue if log.isEnabledFor(logging.DEBUG): self._debug_values_retrieved(attr_id, attr_vals) # ok, include this attribute for the notification: vals_dict[attr_id] = attr_vals if vals_dict: self._values_retrieved(vals_dict)
def _retrieve_attribute_values(self): """ Retrieves the attribute values using the given function and calls _values_retrieved. """ # TODO: note that the "from_time" parameters for the request below was # influenced by the RSN case (see CI-OMS interface). Need to see # whether it also applies to CGSN so eventually adjustments may be needed. # current_time_secs = current_time_millis() / 1000.0 # determine each from_time for the request: attrs = [] for attr_id in self._attr_ids: if self._last_ts[attr_id] is None: # Arbitrarily setting from_time to current system time minus a few seconds: # TODO: determine actual criteria here. win_size_secs = 5 from_time = current_time_secs - win_size_secs else: # note that int(x) returns a long object if needed. from_time = int(self._last_ts[attr_id]) + _DELTA_TIME attrs.append((attr_id, from_time)) log.debug("%r: _retrieve_attribute_values: attrs=%s", self._platform_id, attrs) retrieved_vals = self._get_attribute_values(attrs) log.debug("%r: _retrieve_attribute_values: _get_attribute_values " "for attrs=%s returned %s", self._platform_id, attrs, retrieved_vals) # vals_dict: attributes with non-empty reported values: vals_dict = {} for attr_id, from_time in attrs: if not attr_id in retrieved_vals: log.warn("%r: _retrieve_attribute_values: unexpected: " "response does not include requested attribute %r. " "Response is: %s", self._platform_id, attr_id, retrieved_vals) continue attr_vals = retrieved_vals[attr_id] if not attr_vals: log.debug("%r: No values reported for attribute=%r from_time=%f", self._platform_id, attr_id, from_time) continue if log.isEnabledFor(logging.DEBUG): self._debug_values_retrieved(attr_id, attr_vals) # ok, include this attribute for the notification: vals_dict[attr_id] = attr_vals if vals_dict: self._values_retrieved(vals_dict)
def _retrieve_attribute_values(self): """ Retrieves the attribute values using the given function and calls _values_retrieved. """ # TODO: note that the "from_time" parameters for the request below # as well as the expected response are influenced by the RSN case # (see CI-OMS interface). Need to see whether it also applies to # CGSN so eventually adjustments may be needed. # # note that the "from_time" parameter in each pair (attr_id, from_time) # for the _get_attribute_values call below, is in millis in UNIX epoch. curr_time_millis = current_time_millis() # minimum value for the from_time parameter (OOIION-1372): min_from_time = curr_time_millis - 1000 * _MULT_INTERVAL * self._rate_secs # this corresponds to (_MULT_INTERVAL * self._rate_secs) ago. # determine each from_time for the request: attrs = [] for attr_id in self._attr_ids: if self._last_ts_millis[attr_id] is None: # Very first request for this attribute. Use min_from_time: from_time = min_from_time else: # We've already got values for this attribute. Use the latest # timestamp + _DELTA_TIME as a basis for the new request: from_time = int(self._last_ts_millis[attr_id]) + _DELTA_TIME # but adjust it if it goes too far in the past: if from_time < min_from_time: from_time = min_from_time log.trace( "test_resource_monitoring_recent: attr_id=%s, from_time=%s, %s millis ago", attr_id, from_time, curr_time_millis - from_time) attrs.append((attr_id, from_time)) log.debug("%r: _retrieve_attribute_values: attrs=%s", self._platform_id, attrs) retrieved_vals = self._get_attribute_values(attrs) if retrieved_vals is None: # lost connection; nothing else to do here: return good_retrieved_vals = {} # do validation: we expect an array of tuples (val, timestamp) for # each attribute. If not, log a warning for the attribute and # continue processing with the other valid attributes: for attr_id, vals in retrieved_vals.iteritems(): if not isinstance(vals, (list, tuple)): log.warn( "%r: expecting an array for attribute %r, but got: %r", self._platform_id, attr_id, vals) continue if len(vals): if not isinstance(vals[0], (tuple, list)): log.warn( "%r: expecting elements in array to be tuples " "(val, ts) for attribute %r, but got: %r", self._platform_id, attr_id, vals[0]) continue good_retrieved_vals[attr_id] = vals # even if empty array. if not good_retrieved_vals: # nothing else to do. TODO perhaps an additional warning? return if log.isEnabledFor(logging.TRACE): # pragma: no cover # show good_retrieved_vals as retrieved (might be large) log.trace( "%r: _retrieve_attribute_values: _get_attribute_values " "for attrs=%s returned\n%s", self._platform_id, attrs, self._pp.pformat(good_retrieved_vals)) elif log.isEnabledFor(logging.DEBUG): # pragma: no cover summary = { attr_id: "(%d vals)" % len(vals) for attr_id, vals in good_retrieved_vals.iteritems() } log.debug( "%r: _retrieve_attribute_values: _get_attribute_values " "for attrs=%s returned %s", self._platform_id, attrs, summary) # vals_dict: attributes with non-empty reported values: vals_dict = {} for attr_id, from_time in attrs: if not attr_id in good_retrieved_vals: log.warn( "%r: _retrieve_attribute_values: unexpected: " "response does not include requested attribute %r. " "Response is: %s", self._platform_id, attr_id, good_retrieved_vals) continue attr_vals = good_retrieved_vals[attr_id] if not attr_vals: log.debug( "%r: No values reported for attribute=%r from_time=%f", self._platform_id, attr_id, from_time) continue if log.isEnabledFor(logging.DEBUG): self._debug_values_retrieved(attr_id, attr_vals) # ok, include this attribute for the notification: vals_dict[attr_id] = attr_vals if vals_dict: self._values_retrieved(vals_dict)
def get_governance_info_from_request(self, json_params=None): # Default values for governance headers. actor_id = DEFAULT_ACTOR_ID expiry = DEFAULT_EXPIRY authtoken = "" user_session = get_auth() #if user_session.get("actor_id", None) and user_session.get("valid_until", 0): if user_session.get("actor_id", None): # Get info from current server session # NOTE: Actor id may be inside server session expiry = int(user_session.get("valid_until", 0)) * 1000 if expiry: # This was a proper non-token server session authentication expiry = str(expiry) actor_id = user_session["actor_id"] log.info( "Request associated with session actor_id=%s, expiry=%s", actor_id, expiry) else: # We are just taking the user_id out of the session # TODO: Need to check access token here expiry = str(expiry) if self.token_from_session: actor_id = user_session["actor_id"] log.info( "Request associated with actor's token from session; actor_id=%s, expiry=%s", actor_id, expiry) # Developer access using api_key if self.develop_mode and "api_key" in request.args and request.args[ "api_key"]: actor_id = str(request.args["api_key"]) expiry = str(int(user_session.get("valid_until", 0)) * 1000) if 0 < int(expiry) < current_time_millis(): expiry = str(current_time_millis() + 10000) # flask.session["valid_until"] = int(expiry / 1000) log.info( "Request associated with actor_id=%s, expiry=%s from developer api_key", actor_id, expiry) # Check in headers for OAuth2 bearer token auth_hdr = request.headers.get("authorization", None) if auth_hdr: valid, req = self.process.oauth.verify_request( [self.process.oauth_scope]) if valid: actor_id = flask.g.oauth_user.get("actor_id", "") if actor_id: log.info( "Request associated with actor_id=%s, expiry=%s from OAuth token", actor_id, expiry) return actor_id, DEFAULT_EXPIRY # Try to find auth token override if not authtoken: if json_params: if "authtoken" in json_params: authtoken = json_params["authtoken"] else: if "authtoken" in request.args: authtoken = str(request.args["authtoken"]) # Enable temporary authentication tokens to resolve to actor ids if authtoken: try: token_info = self.idm_client.check_authentication_token( authtoken, headers=self._get_gateway_headers()) actor_id = token_info.get("actor_id", actor_id) expiry = token_info.get("expiry", expiry) log.info("Resolved token %s into actor_id=%s expiry=%s", authtoken, actor_id, expiry) except NotFound: log.info("Provided authentication token not found: %s", authtoken) except Unauthorized: log.info("Authentication token expired or invalid: %s", authtoken) except Exception as ex: log.exception("Problem resolving authentication token") return actor_id, expiry
def test_resource_monitoring_recent(self): # # https://jira.oceanobservatories.org/tasks/browse/OOIION-1372 # # Verifies that the requests for attribute values are always for # the most recent ones, meaning that the retrieved values should *not* # be older than a small multiple of the nominal monitoring rate, even # after a long period in non-monitoring state. # See ResourceMonitor._retrieve_attribute_values # # start this test as in test_resource_monitoring() self.test_resource_monitoring() # which completes right after stopping monitoring. We want that initial # start/stop-monitoring phase to make this test more comprehensive. self._assert_state(PlatformAgentState.COMMAND) # now, the rest of this test does the following: # - pick an attribute to use as a basis for the time parameters to # be used in the test # - wait for a while in the current non-monitoring mode # - re-enable monitoring # - wait for a sample to be published # - verify that new received data sample is "recent" # - stop monitoring # first, use an attribute (from the root platform being tested) with # a minimal monitoring rate, since that attribute should be reported # in a first sample received after re-enabling the monitoring. attr = None for attr_id, plat_attr in self._platform_attributes[ self.PLATFORM_ID].iteritems(): if attr is None or \ float(plat_attr['monitor_cycle_seconds']) < float(attr['monitor_cycle_seconds']): attr = plat_attr self.assertIsNotNone( attr, "some attribute expected to be defined for %r to " "actually proceed with this test" % self.PLATFORM_ID) attr_id = attr['attr_id'] monitor_cycle_seconds = attr['monitor_cycle_seconds'] log.info( "test_resource_monitoring_recent: using attr_id=%r: monitor_cycle_seconds=%s", attr_id, monitor_cycle_seconds) # sleep for twice the interval defining "recent": from ion.agents.platform.resource_monitor import _MULT_INTERVAL time_to_sleep = 2 * (_MULT_INTERVAL * monitor_cycle_seconds) log.info( "test_resource_monitoring_recent: sleeping for %s secs " "before resuming monitoring", time_to_sleep) sleep(time_to_sleep) # reset the variables associated with the _wait_for_a_data_sample call below: self._samples_received = [] self._async_data_result = AsyncResult() ################################################# # re-start monitoring and wait for new sample: log.info("test_resource_monitoring_recent: re-starting monitoring") self._start_resource_monitoring(recursion=False) # should also work with recursion to children but set recursion=False # to avoid wasting the extra time in this test. try: self._wait_for_a_data_sample() # get current time here (right after receiving sample) for comparison below: curr_time_millis = current_time_millis() # verify that the timestamp of the received sample is not too old. # For this, use the minimum of the reported timestamps: rdt = RecordDictionaryTool.load_from_granule( self._samples_received[0]) log.trace("test_resource_monitoring_recent: rdt:\n%s", rdt.pretty_print()) temporal_parameter_name = rdt.temporal_parameter times = rdt[temporal_parameter_name] log.trace("test_resource_monitoring_recent: times:\n%s", self._pp.pformat(times)) # minimum reported timestamp (note the NTP -> ION_time conversion): min_reported_time_ntp = min(times) min_reported_time_millis = float( ntp_2_ion_ts(min_reported_time_ntp)) log.info( "test_resource_monitoring_recent: sample received, min_reported_time_millis=%s", int(min_reported_time_millis)) # finally verify that it is actually not older than the small multiple # of monitor_cycle_seconds plus some additional tolerance (which is # arbitrarily set here to 10 secs): lower_limit_millis = \ curr_time_millis - 1000 * (_MULT_INTERVAL * monitor_cycle_seconds + 10) self.assertGreaterEqual( min_reported_time_millis, lower_limit_millis, "min_reported_time_millis=%s must be >= %s. Diff=%s millis" % (min_reported_time_millis, lower_limit_millis, min_reported_time_millis - lower_limit_millis)) finally: self._stop_resource_monitoring(recursion=False)
except NotFound, e: ion_actor_id = DEFAULT_ACTOR_ID # If the user isn't found default to anonymous expiry = DEFAULT_EXPIRY #Since this is now an anonymous request, there really is no expiry associated with it return ion_actor_id, expiry #need to convert to a float first in order to compare against current time. try: int_expiry = int(expiry) except Exception, e: raise Inconsistent( "Unable to read the expiry value in the request '%s' as an int" % expiry) #The user has been validated as being known in the system, so not check the expiry and raise exception if # the expiry is not set to 0 and less than the current time. if int_expiry > 0 and int_expiry < current_time_millis(): raise Unauthorized( 'The certificate associated with the user and expiry time in the request has expired.' ) return ion_actor_id, expiry def build_message_headers(ion_actor_id, expiry): headers = dict() headers['ion-actor-id'] = ion_actor_id headers['expiry'] = expiry #If this is an anonymous requester then there are no roles associated with the request
def get_governance_info_from_request(self, json_params=None): # Default values for governance headers. actor_id = DEFAULT_ACTOR_ID expiry = DEFAULT_EXPIRY authtoken = "" user_session = get_auth() #if user_session.get("actor_id", None) and user_session.get("valid_until", 0): if user_session.get("actor_id", None): # Get info from current server session # NOTE: Actor id may be inside server session expiry = int(user_session.get("valid_until", 0)) * 1000 if expiry: # This was a proper non-token server session authentication expiry = str(expiry) actor_id = user_session["actor_id"] log.info("Request associated with session actor_id=%s, expiry=%s", actor_id, expiry) else: # We are just taking the user_id out of the session # TODO: Need to check access token here expiry = str(expiry) if self.token_from_session: actor_id = user_session["actor_id"] log.info("Request associated with actor's token from session; actor_id=%s, expiry=%s", actor_id, expiry) # Developer access using api_key if self.develop_mode and "api_key" in request.args and request.args["api_key"]: actor_id = str(request.args["api_key"]) expiry = str(int(user_session.get("valid_until", 0)) * 1000) if 0 < int(expiry) < current_time_millis(): expiry = str(current_time_millis() + 10000) # flask.session["valid_until"] = int(expiry / 1000) log.info("Request associated with actor_id=%s, expiry=%s from developer api_key", actor_id, expiry) # Check in headers for OAuth2 bearer token auth_hdr = request.headers.get("authorization", None) if auth_hdr: valid, req = self.process.oauth.verify_request([self.process.oauth_scope]) if valid: actor_id = flask.g.oauth_user.get("actor_id", "") if actor_id: log.info("Request associated with actor_id=%s, expiry=%s from OAuth token", actor_id, expiry) return actor_id, DEFAULT_EXPIRY # Try to find auth token override if not authtoken: if json_params: if "authtoken" in json_params: authtoken = json_params["authtoken"] else: if "authtoken" in request.args: authtoken = str(request.args["authtoken"]) # Enable temporary authentication tokens to resolve to actor ids if authtoken: try: if authtoken.startswith(("Bearer_")): # Backdoor way for OAuth2 access tokens as request args for GET URLs authtoken = authtoken[7:] token_id = "access_token_" + str(authtoken) token_obj = self.process.container.object_store.read(token_id) token = OAuthTokenObj.from_security_token(token_obj) if token.is_valid(check_expiry=True): actor_id = token.user["actor_id"] expiry = str(token._token_obj.expires) log.info("Resolved OAuth2 token %s into actor_id=%s expiry=%s", authtoken, actor_id, expiry) else: token_info = self.idm_client.check_authentication_token(authtoken, headers=self._get_gateway_headers()) actor_id = token_info.get("actor_id", actor_id) expiry = token_info.get("expiry", expiry) log.info("Resolved token %s into actor_id=%s expiry=%s", authtoken, actor_id, expiry) except NotFound: log.info("Provided authentication token not found: %s", authtoken) except Unauthorized: log.info("Authentication token expired or invalid: %s", authtoken) except Exception as ex: log.exception("Problem resolving authentication token") return actor_id, expiry
try: user = idm_client.read_actor_identity(actor_id=ion_actor_id, headers={"ion-actor-id": service_gateway_instance.name, 'expiry': DEFAULT_EXPIRY }) except NotFound, e: ion_actor_id = DEFAULT_ACTOR_ID # If the user isn't found default to anonymous expiry = DEFAULT_EXPIRY #Since this is now an anonymous request, there really is no expiry associated with it return ion_actor_id, expiry #need to convert to a float first in order to compare against current time. try: int_expiry = int(expiry) except Exception, e: raise Inconsistent("Unable to read the expiry value in the request '%s' as an int" % expiry) #The user has been validated as being known in the system, so not check the expiry and raise exception if # the expiry is not set to 0 and less than the current time. if int_expiry > 0 and int_expiry < current_time_millis(): raise Unauthorized('The certificate associated with the user and expiry time in the request has expired.') return ion_actor_id, expiry def build_message_headers( ion_actor_id, expiry): headers = dict() headers['ion-actor-id'] = ion_actor_id headers['expiry'] = expiry #If this is an anonymous requester then there are no roles associated with the request if ion_actor_id == DEFAULT_ACTOR_ID: headers['ion-actor-roles'] = dict()
def is_expired(self): return current_time_millis() > self.expire_time