def get_secure_service_def(self, service_name):
"""Checks whether the service indicated by given service_name exists and/or
is exposed after white and black listing. Returns service registry entry.
"""
if self.service_whitelist:
if service_name not in self.service_whitelist:
raise Unauthorized("Service access not permitted")
if self.service_blacklist:
if service_name in self.service_blacklist:
raise Unauthorized("Service access not permitted")
# Retrieve service definition
target_service = get_service_registry().get_service_by_name(
service_name)
if not target_service:
raise BadRequest("The requested service (%s) is not available" %
service_name)
if not target_service.client:
raise Inconsistent(
"Cannot find a client class for the specified service: %s" %
service_name)
if not target_service.schema:
raise Inconsistent(
"Cannot find a schema for the specified service: %s" %
service_name)
return target_service
def _analyze_objects(self, resources_objs):
for obj_id, key, obj in resources_objs:
if obj_id.startswith("_design"):
continue
if not isinstance(obj, dict):
raise Inconsistent("Object of bad type found: %s" % type(obj))
obj_type = obj.get("type_", None)
if obj_type == "Association":
self._associations[obj_id] = obj
elif obj_type == "DirEntry":
self._directory[obj_id] = obj
elif obj_type:
self._resources[obj_id] = obj
if obj_type not in self._res_by_type:
self._res_by_type[obj_type] = []
self._res_by_type[obj_type].append(obj_id)
for attr, value in obj.iteritems():
if obj_type not in self._attr_by_type:
self._attr_by_type[obj_type] = set()
self._attr_by_type[obj_type].add(attr)
else:
raise Inconsistent("Object with no type_ found: %s" % obj)
for assoc in self._associations.values():
key = (assoc['s'], assoc['p'])
if key not in self._assoc_by_sub:
self._assoc_by_sub[key] = []
self._assoc_by_sub[key].append(assoc['o'])
def find_user_info_by_subject(self, subject=''):
# Find UserCredentials
objects, matches = self.clients.resource_registry.find_resources(
RT.UserCredentials, None, subject, id_only=False)
if not objects:
raise NotFound("UserCredentials with subject %s does not exist" %
subject)
if len(objects) > 1:
raise Inconsistent(
"Multiple UserCredentials with subject %s exist" % subject)
user_credentials_id = objects[0]._id
subjects, assocs = self.clients.resource_registry.find_subjects(
RT.ActorIdentity, PRED.hasCredentials, user_credentials_id)
if not subjects or len(subjects) == 0:
raise NotFound(
"ActorIdentity to UserCredentials association for subject %s does not exist"
% subject)
if len(subjects) > 1:
raise Inconsistent(
"Multiple ActorIdentity to UserCredentials associations for subject %s exist"
% subject)
actor_identity_id = subjects[0]._id
# Look up UserInfo via association with ActorIdentity
objects, assocs = self.clients.resource_registry.find_objects(
actor_identity_id, PRED.hasInfo, RT.UserInfo)
if not objects:
raise NotFound("UserInfo for subject %s does not exist" % subject)
if len(objects) > 1:
raise Inconsistent("Multiple UserInfos for subject %s exist" %
subject)
user_info = objects[0]
return user_info
def check_authentication_token(self, token_string=''):
"""Checks given token and returns a dict with actor id if valid.
"""
token_id = "token_%s" % token_string
token = self.container.object_store.read(token_id)
if not isinstance(token, SecurityToken):
raise Inconsistent("Token illegal type")
if token.token_type != TokenTypeEnum.ACTOR_AUTH:
raise BadRequest("Illegal token type")
if token.token_string != token_string:
raise Inconsistent("Found token's token_string does not match")
cur_time = get_ion_ts_millis()
if token.status != "OPEN":
raise Unauthorized("Token status invalid")
if cur_time >= int(token.expires):
raise Unauthorized("Token expired")
token_info = dict(actor_id=token.actor_id,
expiry=token.expires,
token=token,
token_id=token_id)
log.info("Authentication token %s resolved to actor %s, expiry %s",
token_string, token.actor_id, token.expires)
return token_info
def set_object_field(self, obj, field, field_val):
"""Recursively set sub object field values.
TODO: This may be an expensive operation. May also be redundant with object code
"""
if isinstance(field_val, dict) and field != "kwargs":
sub_obj = getattr(obj, field)
if isinstance(sub_obj, IonObjectBase):
if "type_" in field_val and field_val["type_"] != sub_obj.type_:
if issubtype(field_val["type_"], sub_obj.type_):
sub_obj = IonObject(field_val["type_"])
setattr(obj, field, sub_obj)
else:
raise Inconsistent(
"Unable to walk the field %s - types don't match: %s %s"
% (field, sub_obj.type_, field_val["type_"]))
for sub_field in field_val:
self.set_object_field(sub_obj, sub_field,
field_val.get(sub_field))
elif isinstance(sub_obj, dict):
setattr(obj, field, field_val)
else:
for sub_field in field_val:
self.set_object_field(sub_obj, sub_field,
field_val.get(sub_field))
else:
# type_ already exists in the class.
if field != "type_":
setattr(obj, field, field_val)
def create_policy(self, policy=None):
"""Persists the provided Policy object. Returns the policy id.
"""
self._validate_resource_obj("policy",
policy,
RT.Policy,
checks="noid,name")
if not is_basic_identifier(policy.name):
raise BadRequest(
"The policy name '%s' can only contain alphanumeric and underscore characters"
% policy.name)
try:
# If there is a policy_rule field then try to add the policy name and description to the rule text
if policy.definition:
rule_tokens = dict(rule_id=policy.name,
description=policy.description)
policy.definition = policy.definition.format(**rule_tokens)
except Exception as e:
raise Inconsistent(
"Missing the elements in the policy rule to set the description: "
+ e.message)
policy_id, _ = self.clients.resource_registry.create(policy)
policy._id = policy_id
log.debug('Policy created: ' + policy.name)
self._publish_policy_event(policy)
return policy_id
def add_service_operation_precondition_policy(self,
service_name='',
op='',
policy_content=''):
"""Boilerplate operation for adding a precondition policy for a specific service operation.
The precondition method must return a tuple (boolean, string).
"""
if not service_name:
raise BadRequest("The service_name argument is missing")
if not op:
raise BadRequest("The op argument is missing")
if not policy_content:
raise BadRequest("The policy_content argument is missing")
policy_name = "Service_" + service_name + "_" + op + "_Precondition_Policies"
policies, _ = self.clients.resource_registry.find_resources(
restype=RT.Policy, name=policy_name)
if policies:
# Update existing policy by adding to list
if len(policies) > 1:
raise Inconsistent(
'There should only be one Policy object per service and operation'
)
policy_obj = policies[0]
if policy_obj.details.type_ != OT.ServiceOperationPreconditionPolicyDetails or policy_obj.details.op != op:
raise Inconsistent(
'The Policy %s does not match the requested service operation %s: %s'
% (policy_obj.name, service_name, op))
policy_obj.details.preconditions.append(policy_content)
self.update_policy(policy_obj)
return policy_obj._id
else:
policy_obj = IonObject(
RT.Policy,
name=policy_name,
description=
'List of operation precondition policies for service ' +
service_name,
policy_type=PolicyTypeEnum.SERVICE_OP_PRECOND,
details=IonObject(OT.ProcessOperationPreconditionPolicyDetails,
service_name=service_name,
op=op,
preconditions=[policy_content]))
return self.create_policy(policy_obj)
def read_authentication_token(self, token_string=''):
"""Returns the token object for given actor authentication token string.
"""
token_id = "token_%s" % token_string
token = self.container.object_store.read(token_id)
if not isinstance(token, SecurityToken):
raise Inconsistent("Token illegal type")
return token
def create_negotiation(self, sap=None):
if sap is None or (sap.type_ != OT.ServiceAgreementProposal and not issubtype(sap.type_, OT.ServiceAgreementProposal)):
raise BadRequest('The sap parameter must be a valid Service Agreement Proposal object')
if sap.proposal_status != ProposalStatusEnum.INITIAL or sap.sequence_num != 0:
raise Inconsistent('The specified Service Agreement Proposal has inconsistent status fields')
if sap.negotiation_id != '':
raise Inconsistent('The specified Service Agreement Proposal cannot have a negotiation_id for an initial proposal')
if sap.type_ in self.negotiation_rules:
# validate preconditions before creating
for pc in self.negotiation_rules[sap.type_]['pre_conditions']:
if pc.startswith('not '):
pre_condition_met = not eval("self.service_provider." + pc.lstrip('not ')) #Strip off the 'not ' part
else:
pre_condition_met = eval("self.service_provider."+pc)
if not pre_condition_met:
raise BadRequest("A precondition for this request has not been satisfied: %s" % pc)
# Should be able to determine the negotiation type based on the intial originator
neg_type = NegotiationTypeEnum.REQUEST
if sap.originator == ProposalOriginatorEnum.PROVIDER:
neg_type = NegotiationTypeEnum.INVITATION
elif sap.originator == ProposalOriginatorEnum.BROKER:
neg_type = NegotiationTypeEnum.BROKERED
neg_obj = IonObject(RT.Negotiation, negotiation_type=neg_type)
# If there is a description in the initial proposal, then set the negotiation description with it.
if sap.description != '':
neg_obj.description = sap.description
neg_id,_ = self.service_provider.clients.resource_registry.create(neg_obj)
# Create associations between the parties
self.service_provider.clients.resource_registry.create_association(sap.consumer, PRED.hasNegotiation, neg_id)
self.service_provider.clients.resource_registry.create_association(sap.provider, PRED.hasNegotiation, neg_id)
if sap.broker != "":
self.service_provider.clients.resource_registry.create_association(sap.broker, PRED.hasNegotiation, neg_id)
return neg_id
def find_user_info_by_name(self, name=''):
objects, matches = self.clients.resource_registry.find_resources(
RT.UserInfo, None, name, id_only=False)
if not objects:
raise NotFound("UserInfo with name %s does not exist" % name)
if len(objects) > 1:
raise Inconsistent("Multiple UserInfo objects with name %s exist" %
name)
return objects[0]
def sample_ping(self, name='name', time='2011-07-27T02:59:43.1Z', an_int=0, a_float=0.0, a_str='',
none=None, a_dict=None, a_list=None):
log.info("StatefulTestProcess trying to force store and load the state")
state_vector = self._get_state_vector()
self._set_state("othervalue", "TOKEN")
if not "othervalue" in state_vector:
raise Inconsistent("state value not found in state vector")
self._flush_state()
state_vector['othervalue'] = "FUZZ"
self._load_state()
state_vector1 = self._get_state_vector()
if not state_vector == state_vector1:
raise Inconsistent("State vectors are different after load")
if not state_vector['othervalue'] == "TOKEN":
raise Inconsistent("state vector not restored. content: %s" % state_vector)
def invalidate_authentication_token(self, token_string=''):
"""Invalidates an authentication token, but leaves it in place for auditing purposes.
"""
token_id = "token_%s" % token_string
token = self.container.object_store.read(token_id)
if not isinstance(token, SecurityToken):
raise Inconsistent("Token illegal type")
if token.token_type != TokenTypeEnum.ACTOR_AUTH:
raise BadRequest("Illegal token type")
token.status = "INVALID"
self.container.object_store.update(token)
log.info("Invalidated security auth token: %s", token.token_string)
def find_actor_identity_by_name(self, name=''):
"""Return the ActorIdentity object whose name attribute matches the passed value.
"""
objects, _ = self.rr.find_resources(RT.ActorIdentity,
None,
name,
id_only=False)
if not objects:
raise NotFound("ActorIdentity with name %s does not exist" % name)
if len(objects) > 1:
raise Inconsistent(
"Multiple ActorIdentity objects with name %s exist" % name)
return objects[0]
def read_authentication_token(self, token_string=''):
"""Returns the token object for given actor authentication token string.
@param token_string str
@retval token SecurityToken
@throws BadRequest Illegal parameter type of value
@throws NotFound Token string not found
"""
token_id = "token_%s" % token_string
token = self.container.object_store.read(token_id)
if not isinstance(token, SecurityToken):
raise Inconsistent("Token illegal type")
return token
def create_resources_snapshot(self, persist=False, filename=None):
ds = DatastoreManager.get_datastore_instance(
DataStore.DS_RESOURCES, DataStore.DS_PROFILE.RESOURCES)
all_objs = ds.find_docs_by_view("_all_docs", None, id_only=False)
log.info("Found %s objects in datastore resources", len(all_objs))
resources = {}
associations = {}
snapshot = dict(resources=resources, associations=associations)
for obj_id, key, obj in all_objs:
if obj_id.startswith("_design"):
continue
if not isinstance(obj, dict):
raise Inconsistent("Object of bad type found: %s" % type(obj))
obj_type = obj.get("type_", None)
if obj_type == "Association":
associations[obj_id] = obj.get("ts", None)
elif obj_type:
resources[obj_id] = obj.get("ts_updated", None)
else:
raise Inconsistent("Object with no type_ found: %s" % obj)
if persist:
dtstr = datetime.datetime.today().strftime('%Y%m%d_%H%M%S')
path = filename or "interface/rrsnapshot_%s.json" % dtstr
snapshot_json = json.dumps(snapshot)
with open(path, "w") as f:
#yaml.dump(snapshot, f, default_flow_style=False)
f.write(snapshot_json)
log.debug(
"Created resource registry snapshot. %s resources, %s associations",
len(resources), len(associations))
return snapshot
def update_negotiation(self, sap=None, reason=None):
# Find the Negotiation resource associated with this proposal
if sap is None or sap.negotiation_id == '':
raise Inconsistent('The Service Agreement Proposal must have a negotiation resource id associated with it')
neg_obj = self.service_provider.clients.resource_registry.read(sap.negotiation_id)
if sap.sequence_num != len(neg_obj.proposals):
raise Inconsistent('The Service Agreement Proposal does not have the correct sequence_num value (%d) for this negotiation (%d)' % (sap.sequence_num, len(neg_obj.proposals)))
# Synchronize negotiation status based on proposals
if sap.proposal_status == ProposalStatusEnum.REJECTED:
neg_obj.negotiation_status = NegotiationStatusEnum.REJECTED
elif sap.proposal_status == ProposalStatusEnum.ACCEPTED:
# Look for an previously Accepted proposal from the other party
for prop in neg_obj.proposals:
if prop.proposal_status == ProposalStatusEnum.ACCEPTED and prop.originator != sap.originator:
neg_obj.negotiation_status = NegotiationStatusEnum.ACCEPTED
if reason is not None:
neg_obj.reason = reason
# Add the current proposal to the Negotiation object to keep a record of it - then save it
neg_obj.proposals.append(sap)
neg_id,_ = self.service_provider.clients.resource_registry.update(neg_obj)
self._publish_status_event(neg_obj)
if neg_obj.negotiation_status == NegotiationStatusEnum.ACCEPTED:
self._execute_accept_action(neg_obj.proposals[-1])
# Publish request granted notification
self._publish_status_event(neg_obj, ProposalStatusEnum.GRANTED)
return neg_id
def find_actor_identity_by_name(self, name=''):
"""Return the ActorIdentity object whose name attribute matches the passed value.
@param name str
@retval user_info ActorIdentity
@throws NotFound failed to find ActorIdentity
@throws Inconsistent Multiple ActorIdentity objects matched name
"""
objects, matches = self.clients.resource_registry.find_resources(
RT.ActorIdentity, None, name, id_only=False)
if not objects:
raise NotFound("ActorIdentity with name %s does not exist" % name)
if len(objects) > 1:
raise Inconsistent(
"Multiple ActorIdentity objects with name %s exist" % name)
return objects[0]
def validate_request(self, ion_actor_id, expiry, in_whitelist=False):
# There is no point in looking up an anonymous user - so return default values.
if ion_actor_id == DEFAULT_ACTOR_ID:
# Since this is an anonymous request, there really is no expiry associated with it
if not in_whitelist and self.require_login:
raise Unauthorized("Anonymous access not permitted")
else:
return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY
try:
user = self.idm_client.read_actor_identity(
actor_id=ion_actor_id, headers=self._get_gateway_headers())
except NotFound as e:
if not in_whitelist and self.require_login:
# This could be a restart of the system with a new preload.
# TODO: Invalidate Flask sessions on relaunch/bootstrap with creating new secret
user_session = get_auth()
if user_session.get("actor_id", None) == ion_actor_id:
clear_auth()
raise Unauthorized("Invalid identity", exc_id="01.10")
else:
# If the user isn't found default to anonymous
return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY
# Need to convert to int first in order to compare against current time.
try:
int_expiry = int(expiry)
except Exception as ex:
raise Inconsistent(
"Unable to read the expiry value in the request '%s' as an int"
% expiry)
# The user has been validated as being known in the system, so not check the expiry and raise exception if
# the expiry is not set to 0 and less than the current time.
if 0 < int_expiry < current_time_millis():
if not in_whitelist and self.require_login:
raise Unauthorized("User authentication expired")
else:
log.warn("User authentication expired")
return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY
return ion_actor_id, expiry
def dump_dicts_as_xlsx(self, objects, filename=None):
self._clear()
"""Dumps a dict of dicts. Tab names will be the names of keys in the objects dict"""
self._wb = xlwt.Workbook()
self._worksheets = {}
for cat_name, cat_objects in objects.iteritems():
for obj_id, obj in cat_objects.iteritems():
if not isinstance(obj, dict):
raise Inconsistent("Object of bad type found: %s" %
type(obj))
self._resources[obj_id] = obj
self._res_by_type.setdefault(cat_name, []).append(obj_id)
self._resobj_by_type.setdefault(cat_name, {})[obj_id] = obj
self._attr_by_type.setdefault(cat_name,
set()).update(obj.keys())
for restype in sorted(self._res_by_type.keys()):
self._dump_resource_type(restype)
dtstr = datetime.datetime.today().strftime('%Y%m%d_%H%M%S')
path = filename or "interface/objects_%s.xls" % dtstr
self._wb.save(path)
def create_object_from_cfg(self, cfg, objtype, key="resource", prefix="", existing_obj=None):
"""
Construct an IonObject of a determined type from given config dict with attributes.
Convert all attributes according to their schema target type. Supports nested objects.
Supports edit of objects of same type.
"""
log.trace("Create object type=%s, prefix=%s", objtype, prefix)
if objtype == "dict":
schema = None
else:
schema = self._get_object_class(objtype)._schema
obj_fields = {} # Attributes for IonObject creation as dict
nested_done = set() # Names of attributes with nested objects already created
obj_cfg = get_safe(cfg, key)
for subkey, value in obj_cfg.iteritems():
if subkey.startswith(prefix):
attr = subkey[len(prefix):]
if '.' in attr: # We are a parent entry
# TODO: Make sure to not create nested object multiple times
slidx = attr.find('.')
nested_obj_field = attr[:slidx]
parent_field = attr[:slidx+1]
nested_prefix = prefix + parent_field # prefix plus nested object name
if '[' in nested_obj_field and nested_obj_field[-1] == ']':
sqidx = nested_obj_field.find('[')
nested_obj_type = nested_obj_field[sqidx+1:-1]
nested_obj_field = nested_obj_field[:sqidx]
elif objtype == "dict":
nested_obj_type = "dict"
else:
nested_obj_type = schema[nested_obj_field]['type']
# Make sure to not create the same nested object twice
if parent_field in nested_done:
continue
# Support direct indexing in a list
list_idx = -1
if nested_obj_type.startswith("list/"):
_, list_idx, nested_obj_type = nested_obj_type.split("/")
list_idx = int(list_idx)
log.trace("Get nested object field=%s type=%s, prefix=%s", nested_obj_field, nested_obj_type, prefix)
nested_obj = self.create_object_from_cfg(cfg, nested_obj_type, key, nested_prefix)
if list_idx >= 0:
my_list = obj_fields.setdefault(nested_obj_field, [])
if list_idx >= len(my_list):
my_list[len(my_list):list_idx] = [None]*(list_idx-len(my_list)+1)
my_list[list_idx] = nested_obj
else:
obj_fields[nested_obj_field] = nested_obj
nested_done.add(parent_field)
elif objtype == "dict":
# TODO: What about type?
obj_fields[attr] = value
elif attr in schema: # We are the leaf attribute
try:
if value:
fieldvalue = get_typed_value(value, schema[attr])
obj_fields[attr] = fieldvalue
except Exception:
log.warn("Object type=%s, prefix=%s, field=%s cannot be converted to type=%s. Value=%s",
objtype, prefix, attr, schema[attr]['type'], value, exc_info=True)
#fieldvalue = str(fieldvalue)
else:
# warn about unknown fields just once -- not on each row
log.warn("Skipping unknown field in %s: %s%s", objtype, prefix, attr)
if objtype == "dict":
obj = obj_fields
else:
if existing_obj:
# Edit attributes
if existing_obj.type_ != objtype:
raise Inconsistent("Cannot edit resource. Type mismatch old=%s, new=%s" % (existing_obj.type_, objtype))
# TODO: Don't edit empty nested attributes
for attr in list(obj_fields.keys()):
if not obj_fields[attr]:
del obj_fields[attr]
for attr in ('alt_ids','_id','_rev','type_'):
if attr in obj_fields:
del obj_fields[attr]
existing_obj.__dict__.update(obj_fields)
log.trace("Update object type %s using field names %s", objtype, obj_fields.keys())
obj = existing_obj
else:
if cfg.get(KEY_ID, None) and 'alt_ids' in schema:
if 'alt_ids' in obj_fields:
obj_fields['alt_ids'].append("PRE:"+cfg[KEY_ID])
else:
obj_fields['alt_ids'] = ["PRE:"+cfg[KEY_ID]]
log.trace("Create object type %s from field names %s", objtype, obj_fields.keys())
obj = IonObject(objtype, **obj_fields)
return obj
