def create_extended_resource_container(self, extended_resource_type, resource_id, computed_resource_type=None,
ext_associations=None, ext_exclude=None):
if not self.service_provider or not self.resource_registry:
raise Inconsistent("This class is not initialized properly")
if extended_resource_type not in getextends(OT.ResourceContainer):
raise BadRequest('Requested resource %s is not extended from %s' % ( extended_resource_type, OT.ResourceContainer) )
if computed_resource_type and computed_resource_type not in getextends(OT.ComputedAttributes):
raise BadRequest('Requested resource %s is not extended from %s' % ( computed_resource_type, OT.ComputedAttributes) )
resource_object = self.resource_registry.read(resource_id)
if not resource_object:
raise NotFound("Resource %s does not exist" % resource_id)
res_container = IonObject(extended_resource_type)
res_container._id = resource_object._id
res_container.resource = resource_object
self.set_container_field_values(res_container, ext_exclude)
self.set_computed_attributes(res_container, computed_resource_type, ext_exclude)
self.set_extended_associations(res_container, ext_associations, ext_exclude)
res_container.ts_created = get_ion_ts()
return res_container
def has_valid_resource_commitments(actor_id, resource_id):
'''
Returns a ResourceCommitmentStatus object indicating the commitment status between this resource/actor
Can only have an exclusive commitment if actor already has a shared commitment.
@param actor_id:
@param resource_id:
@return:
'''
ret_status = IonObject(OT.ResourceCommitmentStatus)
commitments = get_valid_resource_commitments(resource_id, actor_id)
if commitments is None:
#No commitments were found between this resource_id and actor_id - so return default object with
#fields set to False
return ret_status
ret_status.shared = True
for com in commitments:
if com.commitment.exclusive == True:
#Found an exclusive commitment
ret_status.exclusive = True
return ret_status
#Only a shared commitment was found
return ret_status
def test_get_valid_resource_commitment(self):
from pyon.util.containers import get_ion_ts_millis
# create ION org and an actor
ion_org = IonObject(RT.Org, name='ION')
ion_org_id, _ = self.rr.create(ion_org)
ion_org._id = ion_org_id
actor = IonObject(RT.ActorIdentity, name='actor1')
actor_id, _ = self.rr.create(actor)
# create an expired commitment in the org
ts = get_ion_ts_millis() - 50000
com_obj = IonObject(RT.Commitment, provider=ion_org_id, consumer=actor_id, commitment=True, expiration=ts)
com_id, _ = self.rr.create(com_obj)
id = self.rr.create_association(ion_org_id, PRED.hasCommitment, com_id)
c = get_valid_resource_commitments(ion_org_id, actor_id)
#verify that the commitment is not returned
self.assertIsNone(c)
# create a commitment that has not expired yet
ts = get_ion_ts_millis() + 50000
com_obj = IonObject(RT.Commitment, provider=ion_org_id, consumer=actor_id, commitment=True, expiration=ts)
com_id, _ = self.rr.create(com_obj)
id = self.rr.create_association(ion_org_id, PRED.hasCommitment, com_id)
c = get_valid_resource_commitments(ion_org_id, actor_id)
#verify that the commitment is returned
self.assertIsNotNone(c)
def test_rr_create_with_id(self):
res_obj1 = IonObject(RT.ActorIdentity)
newid = uuid.uuid4().hex
rid1, _ = self.rr.create(res_obj1, object_id=newid)
self.assertEqual(rid1, newid)
res_list = [
IonObject(RT.TestInstrument, name="ID1"),
IonObject(RT.TestInstrument, name="ID2"),
IonObject(RT.TestInstrument, name="ID3"),
]
rid_list = self.rr.create_mult(res_list)
self.assertEquals(len(rid_list), 3)
owned_list, _ = self.rr.find_subjects(subject_type=RT.TestInstrument, predicate=PRED.hasOwner, object=rid1, id_only=True)
self.assertEquals(len(owned_list), 0)
self.rr.rr_store.delete_mult([rid for (rid, rrv) in rid_list])
rid_list = self.rr.create_mult(res_list, actor_id=rid1)
self.assertEquals(len(rid_list), 3)
owned_list, _ = self.rr.find_subjects(subject_type=RT.TestInstrument, predicate=PRED.hasOwner, object=rid1, id_only=True)
self.assertEquals(len(owned_list), 3)
self.rr.rr_store.delete_mult([rid for (rid, rrv) in rid_list])
self.rr.delete(rid1)
def test_create_counter_proposal(self):
with self.assertRaises(BadRequest) as cm:
consumer_accept_sap = Negotiation.create_counter_proposal(proposal_status=ProposalStatusEnum.INITIAL)
self.assertIn('The negotiation parameter must be a valid Negotiation object',cm.exception.message)
negotiation_handler = Negotiation(self)
sap = IonObject(OT.EnrollmentProposal,consumer=self.actor_identity._id, provider=self.org._id )
negotiation = Mock()
negotiation._id = '456'
negotiation.type_ = RT.Negotiation
negotiation.proposals = [sap]
self.mock_read.return_value = negotiation
self.mock_create.return_value = ['456', 2]
neg_id = negotiation_handler.create_negotiation(sap)
sap.negotiation_id = neg_id
consumer_accept_sap = Negotiation.create_counter_proposal(negotiation, proposal_status=ProposalStatusEnum.COUNTER,
originator=ProposalOriginatorEnum.PROVIDER)
self.assertEqual(consumer_accept_sap.negotiation_id, negotiation._id)
self.assertEqual(len(negotiation.proposals),1)
self.assertEqual(consumer_accept_sap.sequence_num, len(negotiation.proposals))
self.assertEqual(consumer_accept_sap.proposal_status, ProposalStatusEnum.COUNTER)
self.assertEqual(consumer_accept_sap.originator, ProposalOriginatorEnum.PROVIDER)
def test_read_negotiation(self):
negotiation_handler = Negotiation(self)
with self.assertRaises(BadRequest) as cm:
negotiation_handler.read_negotiation()
self.assertIn(
'The sap parameter must be a valid Service Agreement Proposal object',
cm.exception.message)
sap = IonObject(OT.EnrollmentProposal,
consumer=self.actor_identity._id,
provider=self.org._id)
with self.assertRaises(BadRequest) as cm:
negotiation_handler.read_negotiation(sap)
self.assertIn(
'The Service Agreement Proposal object (sap) is missing a negotiation_id value',
cm.exception.message)
negotiation = Mock()
negotiation._id = '456'
sap.negotiation_id = negotiation._id
self.mock_read.return_value = negotiation
neg_obj = negotiation_handler.read_negotiation(sap)
self.assertEqual(neg_obj, negotiation)
def create_test_col(level=0, ot=dict, no_ion=False):
if level == 0:
return get_value(0, 15, uvals)
if ot == dict:
res_dict = {}
num_kinds = 1 if do_ion and no_ion else (1 if do_dict else 0) + (1 if do_list else 0)
for i in xrange(breadth / num_kinds):
if do_ion and not no_ion:
key = get_key()
res_obj = IonObject(restype, name="TestObject %s.%s" % (level, key))
res_obj.addl = create_test_col(level-1, dict, no_ion=True)
res_dict[key] = res_obj
else:
if do_dict:
res_dict[get_key()] = create_test_col(level-1, dict)
if do_list:
res_dict[get_key()] = create_test_col(level-1, list)
return res_dict
elif ot == list:
res_list = []
num_kinds = 1 if do_ion and no_ion else (1 if do_dict else 0) + (1 if do_list else 0)
for i in xrange(breadth / num_kinds):
if do_ion and not no_ion:
res_obj = IonObject(restype, name="TestObject %s.%s" % (level, random.randint(1000, 9999)))
res_obj.addl = create_test_col(level-1, dict, no_ion=True)
res_list.append(res_obj)
else:
if do_dict:
res_list.append(create_test_col(level-1, dict))
if do_list:
res_list.append(create_test_col(level-1, list))
return res_list
elif ot == "IonObject":
res_obj = IonObject(restype, name="TestObject %s.%s" % (level, random.randint(1000, 9999)))
return res_obj
def _create_resource(self, restype, name, *args, **kwargs):
res_obj = IonObject(restype, dict(name=name, **kwargs))
res_obj_res = self.data_store.create(res_obj,
create_unique_resource_id())
res_obj._id = res_obj_res[0]
self.resources[name] = res_obj
return res_obj_res[0]
def test_get_resource_extension(self):
#Testing multiple instrument owners
subject1 = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
actor_identity_obj1 = IonObject("ActorIdentity", {"name": subject1})
user_id1,_ = self.rr.create(actor_identity_obj1)
user_info_obj1 = IonObject("UserInfo", {"name": "Foo"})
user_info_id1,_ = self.rr.create(user_info_obj1)
self.rr.create_association(user_id1, PRED.hasInfo, user_info_id1)
subject2 = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256"
actor_identity_obj2 = IonObject("ActorIdentity", {"name": subject2})
user_id2,_ = self.rr.create(actor_identity_obj2)
user_info_obj2 = IonObject("UserInfo", {"name": "Foo2"})
user_info_id2,_ = self.rr.create(user_info_obj2)
self.rr.create_association(user_id2, PRED.hasInfo, user_info_id2)
test_obj = IonObject('InformationResource', {"name": "TestResource"})
test_obj_id,_ = self.rr.create(test_obj)
self.rr.create_association(test_obj_id, PRED.hasOwner, user_id1)
self.rr.create_association(test_obj_id, PRED.hasOwner, user_id2)
extended_resource = self.rr.get_resource_extension(test_obj_id, 'ExtendedInformationResource')
self.assertEqual(test_obj_id,extended_resource._id)
self.assertEqual(len(extended_resource.owners),2)
extended_resource_list = self.rr.get_resource_extension(str([user_info_id1,user_info_id2]), 'ExtendedInformationResource')
self.assertEqual(len(extended_resource_list), 2)
def fun():
#ret = Mock()
ret = IonObject(impl.iontype)
ret.name = "sample %s" % impl.iontype
ret.description = "description of sample %s" % impl.iontype
for k, v in resource_params.iteritems():
setattr(ret, k, v)
return ret
def fun():
#ret = Mock()
self.log.debug("Creating sample %s" % impl.iontype)
ret = IonObject(impl.iontype)
ret.name = "sample %s" % impl.iontype
ret.description = "description of sample %s" % impl.iontype
for k, v in resource_params.iteritems():
setattr(ret, k, v)
return ret
def fun():
#ret = Mock()
self.log.debug("Creating sample %s" % iontype)
ret = IonObject(iontype)
ret.name = "sample %s" % iontype
ret.description = "description of sample %s" % iontype
for k, v in resource_params.iteritems():
setattr(ret, k, v)
return ret
def test_rr_read_assoc(self):
res_obj1 = IonObject(RT.Org)
rid1, _ = self.rr.create(res_obj1)
res_obj2 = IonObject(RT.InstrumentDevice)
rid2, _ = self.rr.create(res_obj2)
with self.assertRaises(NotFound) as ex:
read_obj2 = self.rr.read_object(rid1, PRED.hasResource)
aid1, _ = self.rr.create_association(rid1, PRED.hasResource, rid2)
read_obj2 = self.rr.read_object(rid1, PRED.hasResource)
self.assertEquals(read_obj2._id, rid2)
read_obj2 = self.rr.read_object(rid1, PRED.hasResource, id_only=True)
self.assertEquals(read_obj2, rid2)
read_obj2 = self.rr.read_object(assoc=aid1)
self.assertEquals(read_obj2._id, rid2)
read_obj1 = self.rr.read_subject(None, PRED.hasResource, rid2)
self.assertEquals(read_obj1._id, rid1)
read_obj1 = self.rr.read_subject(None,
PRED.hasResource,
rid2,
id_only=True)
self.assertEquals(read_obj1, rid1)
read_obj1 = self.rr.read_subject(assoc=aid1)
self.assertEquals(read_obj1._id, rid1)
res_obj3 = IonObject(RT.InstrumentDevice)
rid3, _ = self.rr.create(res_obj3)
res_obj4 = IonObject(RT.Org)
rid4, _ = self.rr.create(res_obj4)
aid2, _ = self.rr.create_association(rid1, PRED.hasResource, rid3)
aid3, _ = self.rr.create_association(rid4, PRED.hasResource, rid3)
with self.assertRaises(Inconsistent) as ex:
read_obj2 = self.rr.read_object(rid1, PRED.hasResource)
with self.assertRaises(Inconsistent) as ex:
read_obj1 = self.rr.read_subject(None, PRED.hasResource, rid3)
res_obj5 = IonObject(RT.PlatformDevice)
rid5, _ = self.rr.create(res_obj5)
aid4, _ = self.rr.create_association(rid1, PRED.hasResource, rid5)
read_obj5 = self.rr.read_object(rid1, PRED.hasResource,
RT.PlatformDevice)
def _execute(self, cprefix, command):
if not command:
raise iex.BadRequest("execute argument 'command' not present")
if not command.command:
raise iex.BadRequest("command not set")
cmd_res = IonObject("AgentCommandResult", command_id=command.command_id, command=command.command)
cmd_func = getattr(self, cprefix + str(command.command), None)
if cmd_func:
cmd_res.ts_execute = get_ion_ts()
try:
res = cmd_func(*command.args, **command.kwargs)
cmd_res.status = 0
cmd_res.result = res
except iex.IonException as ex:
# TODO: Distinguish application vs. uncaught exception
cmd_res.status = getattr(ex, 'status_code', -1)
cmd_res.result = str(ex)
log.warn("Agent command %s failed with trace=%s" % (command.command, traceback.format_exc()))
else:
log.info("Agent command not supported: %s" % (command.command))
ex = iex.NotFound("Command not supported: %s" % command.command)
cmd_res.status = iex.NotFound.status_code
cmd_res.result = str(ex)
sub_type = "%s.%s" % (command.command, cmd_res.status)
post_event = self._event_publisher._create_event(event_type=self.COMMAND_EVENT_TYPE,
origin=self.resource_id, origin_type=self.ORIGIN_TYPE,
sub_type=sub_type, command=command, result=cmd_res)
post_event = self._post_execute_event_hook(post_event)
success = self._event_publisher._publish_event(post_event, origin=post_event.origin)
return cmd_res
def obtain_agent_calculation(self, device_id, result_container):
ret = IonObject(result_container)
h_agent, reason = self.get_device_agent(device_id)
if None is h_agent:
ret.status = ComputedValueAvailability.NOTAVAILABLE
ret.reason = reason
else:
ret.status = ComputedValueAvailability.PROVIDED
return h_agent, ret
def obtain_agent_calculation(self, device_id, result_container):
ret = IonObject(result_container)
h_agent, reason = self.get_device_agent(device_id)
if None is h_agent:
ret.status = ComputedValueAvailability.NOTAVAILABLE
ret.reason = reason
else:
ret.status = ComputedValueAvailability.PROVIDED
return h_agent, ret
def execute_resource(self, resource_id='', command=None):
"""
"""
if not command:
iex = BadRequest('Execute argument "command" not set.')
self._on_command_error('execute_resource', None, None, None, iex)
raise iex
# Grab command syntax.
id = command.command_id
cmd = command.command
args = command.args or []
kwargs = command.kwargs or {}
if not command.command:
iex = BadRequest('Command name not set.')
self._on_command_error('execute_resource', cmd, args, kwargs, iex)
raise iex
# Construct a command result object.
cmd_result = IonObject("AgentCommandResult",
command_id=id,
command=cmd,
ts_execute=get_ion_ts(),
status=0)
try:
result = self._fsm.on_event(
ResourceAgentEvent.EXECUTE_RESOURCE, cmd, *args, **kwargs)
cmd_result.result = result
self._on_command('execute_resource', cmd, args, kwargs, result)
except FSMStateError as ex:
iex = Conflict(*(ex.args))
self._on_command_error('execute_resource', cmd, args, kwargs, iex)
raise iex
except FSMCommandUnknownError as ex:
iex = BadRequest(*(ex.args))
self._on_command_error('execute_resource', cmd, args, kwargs, iex)
raise iex
except IonException as iex:
self._on_command_error('execute_resource', cmd, args, kwargs, iex)
raise iex
except Exception as ex:
iex = ServerError(*(ex.args))
self._on_command_error('execute_resource', cmd, args, kwargs, iex)
raise iex
return cmd_result
def create_extended_resource_container(self, extended_resource_type, resource_id, computed_resource_type=None,
ext_associations=None, ext_exclude=None):
overall_start_time = time.time()
if not isinstance(resource_id, types.StringType):
raise Inconsistent("The parameter resource_id is not a single resource id string")
if not self.service_provider or not self._rr:
raise Inconsistent("This class is not initialized properly")
if extended_resource_type not in getextends(OT.ResourceContainer):
raise BadRequest('The requested resource %s is not extended from %s' % (extended_resource_type, OT.ResourceContainer))
if computed_resource_type and computed_resource_type not in getextends(OT.ComputedAttributes):
raise BadRequest('The requested resource %s is not extended from %s' % (computed_resource_type, OT.ComputedAttributes))
resource_object = self._rr.read(resource_id)
if not resource_object:
raise NotFound("The Resource %s does not exist" % resource_id)
res_container = IonObject(extended_resource_type)
# @TODO - replace with object level decorators and raise exceptions
if not hasattr(res_container, 'origin_resource_type'):
log.error('The requested resource %s does not contain a properly set origin_resource_type field.' , extended_resource_type)
#raise Inconsistent('The requested resource %s does not contain a properly set origin_resource_type field.' % extended_resource_type)
if hasattr(res_container, 'origin_resource_type') and res_container.origin_resource_type != resource_object.type_\
and not issubtype(resource_object.type_, res_container.origin_resource_type):
log.error('The origin_resource_type of the requested resource %s(%s) does not match the type of the specified resource id(%s).' % (
extended_resource_type, res_container.origin_resource_type, resource_object.type_))
#raise Inconsistent('The origin_resource_type of the requested resource %s(%s) does not match the type of the specified resource id(%s).' % (extended_resource_type, res_container.origin_resource_type, resource_object.type_))
res_container._id = resource_object._id
res_container.resource = resource_object
self.set_container_lcstate_info(res_container)
self.set_container_field_values(res_container, ext_exclude)
self.set_computed_attributes(res_container, computed_resource_type, ext_exclude)
self.set_extended_associations(res_container, ext_associations, ext_exclude)
res_container.ts_created = get_ion_ts()
overall_stop_time = time.time()
log.debug("Time to process extended resource container %s %f secs", extended_resource_type, overall_stop_time - overall_start_time )
return res_container
def obtain_agent_calculation(self, device_id, result_container):
ret = IonObject(result_container)
a_client = None
try:
a_client = self.obtain_agent_handle(device_id)
ret.status = ComputedValueAvailability.PROVIDED
except NotFound:
ret.status = ComputedValueAvailability.NOTAVAILABLE
ret.reason = "Could not connect to instrument agent instance -- may not be running"
except Exception as e:
raise e
return a_client, ret
def _execute(self, cprefix, command):
if not command:
raise iex.BadRequest("execute argument 'command' not present")
if not command.command:
raise iex.BadRequest("command not set")
cmd_res = IonObject("AgentCommandResult", command_id=command.command_id, command=command.command)
cmd_func = getattr(self, cprefix + str(command.command), None)
if cmd_func:
cmd_res.ts_execute = get_ion_ts()
try:
res = cmd_func(*command.args, **command.kwargs)
cmd_res.status = 0
cmd_res.result = res
except Exception as ex:
# TODO: Distinguish application vs. uncaught exception
cmd_res.status = getattr(ex, 'status_code', -1)
cmd_res.result = str(ex)
log.info("Agent function failed with ex=%s msg=%s" % (type(ex), str(ex)))
else:
log.info("Agent command not supported: %s" % (command.command))
ex = iex.NotFound("Command not supported: %s" % command.command)
cmd_res.status = iex.NotFound.status_code
cmd_res.result = str(ex)
return cmd_res
def _start_raw_ingestion(self):
dpsc_cli = DataProductManagementServiceClient()
rrclient = ResourceRegistryServiceClient()
RR2 = EnhancedResourceRegistryClient(rrclient)
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
log.info("Create data product... raw stream id: %s",
self._raw_stream_id)
dp_id = dpsc_cli.create_data_product_(data_product=dp_obj)
dataset_id = self.create_dataset(self._raw_stream_pdict_id)
RR2.assign_stream_definition_to_data_product_with_has_stream_definition(
self._raw_stream_def_id, dp_id)
RR2.assign_stream_to_data_product_with_has_stream(
self._raw_stream_id, dp_id)
RR2.assign_dataset_to_data_product_with_has_dataset(dataset_id, dp_id)
self._raw_dataset_id = dataset_id
log.info("Create data product...Complete")
# Assert that the data product has an associated stream at this stage
stream_ids, _ = rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream,
True)
self.assertNotEquals(len(stream_ids), 0)
# Assert that the data product has an associated stream def at this stage
stream_ids, _ = rrclient.find_objects(dp_id, PRED.hasStreamDefinition,
RT.StreamDefinition, True)
self.assertNotEquals(len(stream_ids), 0)
log.info("Activate data product persistence")
dpsc_cli.activate_data_product_persistence(dp_id)
log.info("Read data product")
dp_obj = dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Created data product %s', dp_obj)
def _start_raw_ingestion(self):
dpsc_cli = DataProductManagementServiceClient()
rrclient = ResourceRegistryServiceClient()
RR2 = EnhancedResourceRegistryClient(rrclient)
# Generic time-series data domain creation
tdom, sdom = time_series_domain()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
log.info("Create data product... raw stream id: %s", self._raw_stream_id)
dp_id = dpsc_cli.create_data_product_(data_product= dp_obj)
dataset_id = self.create_dataset(self._raw_stream_pdict_id)
RR2.assign_stream_definition_to_data_product_with_has_stream_definition(self._raw_stream_def_id, dp_id)
RR2.assign_stream_to_data_product_with_has_stream(self._raw_stream_id, dp_id)
RR2.assign_dataset_to_data_product_with_has_dataset(dataset_id, dp_id)
self._raw_dataset_id = dataset_id
log.info("Create data product...Complete")
# Assert that the data product has an associated stream at this stage
stream_ids, _ = rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True)
self.assertNotEquals(len(stream_ids), 0)
# Assert that the data product has an associated stream def at this stage
stream_ids, _ = rrclient.find_objects(dp_id, PRED.hasStreamDefinition, RT.StreamDefinition, True)
self.assertNotEquals(len(stream_ids), 0)
log.info("Activate data product persistence")
dpsc_cli.activate_data_product_persistence(dp_id)
log.info("Read data product")
dp_obj = dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Created data product %s', dp_obj)
def get_resource_object(self,
my_resource_id,
resource_name='TestSystem_Resource'):
'''
Method used for testing
'''
return IonObject(RT.SystemResource, name=resource_name)
def set_configuration(self, config):
log.warn("DRIVER: set_configuration")
"""
expect configuration to have:
- parser module/class
- directory, wildcard to find data files
- optional timestamp of last granule
- optional poll rate
- publish info
"""
log.error("Log level: %s", log.getEffectiveLevel())
log.debug('using configuration: %s', config)
self.config = config
self.max_records = get_safe(config, 'max_records', 100)
self.stream_config = self.CFG.get('stream_config', {})
if len(self.stream_config) == 1:
stream_cfg = self.stream_config.values()[0]
elif len(self.stream_config) > 1:
stream_cfg = self.stream_config.values()[0]
stream_id = stream_cfg['stream_id']
stream_route = IonObject(OT.StreamRoute,
routing_key=stream_cfg['routing_key'],
exchange_point=stream_cfg['exchange_point'])
param_dict = stream_cfg['stream_def_dict']['parameter_dictionary']
self.publisher = StandaloneStreamPublisher(stream_id=stream_id,
stream_route=stream_route)
self.parameter_dictionary = ParameterDictionary.load(param_dict)
self.time_field = self.parameter_dictionary.get_temporal_context()
self.latest_granule_time = get_safe(config, 'last_time', 0)
def test_rr_create_with_id(self):
res_obj1 = IonObject(RT.Org)
newid = uuid.uuid4().hex
rid1,_ = self.rr.create(res_obj1, object_id=newid)
self.assertEqual(rid1, newid)
def reset_policy_cache(self, headers=None, timeout=None):
"""Clears and reloads the policy caches in all of the containers.
@throws BadRequest None
"""
self.perform_action(ALL_CONTAINERS_INSTANCE,
IonObject(OT.ResetPolicyCache))
def set_log_level(self, logger='', level='', recursive=False):
self.perform_action(
ALL_CONTAINERS_INSTANCE,
IonObject(OT.ChangeLogLevel,
logger=logger,
level=level,
recursive=recursive))
def test_garbage_collection_handler(self):
""" initial log level for ion.process.event is INFO -- test we can change it to TRACE """
#
handler = pyon.container.management.GarbageCollectionHandler()
action = IonObject(OT.TriggerGarbageCollection)
handler.handle_request(action)
def xtest_policy_cache_handler(self):
""" initial log level for ion.process.event is INFO -- test we can change it to TRACE """
#
handler = pyon.container.management.PolicyCacheHandler()
action = IonObject(OT.ResetPolicyCache)
handler.handle_request(action)
def trigger_garbage_collection(self):
"""Triggers a garbage collection in all containers
@throws BadRequest None
"""
self.perform_action(ALL_CONTAINERS_INSTANCE,
IonObject(OT.TriggerGarbageCollection))
def test_get_sytsem_actor_header(self):
actor = IonObject(RT.ActorIdentity, name='ionsystem')
actor_id, _ = self.rr.create(actor)
system_actor_header = get_system_actor_header()
self.assertDictEqual(system_actor_header['ion-actor-roles'],
{'ION': [MEMBER_ROLE]})
def test_crud_list_obj(self):
""" crud batch operations using lists of objects where possible """
obj1 = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
obj2 = IonObject(RT.Observatory, name='mount spaghetti', description='covered with snow')
objs = [obj1,obj2]
tuples = self.repo.insert('sample', objs)
for t in tuples:
self.assertTrue(t[0])
self.assertTrue(t[1] is not None)
ids = [ tuples[0][1], tuples[1][1], 'howdy' ]
tuples = self.repo.read('sample', ids)
self.assertTrue(tuples[0][0])
self.assertTrue(tuples[1][0])
self.assertFalse(tuples[2][0])
obj3 = tuples[0][2]
obj4 = tuples[1][2]
obj3.name = 'no longer SBE'
obj4.description = 'no more snow'
obj1._id = 'abc123'
objs = [ obj3, obj4, obj1 ]
tuples = self.repo.update('sample', objs)
self.assertTrue(tuples[0][0])
self.assertTrue(tuples[1][0])
self.assertFalse(tuples[2][0])
tuples = self.repo.read('sample', objs)
self.assertTrue(tuples[0][0])
self.assertTrue(tuples[1][0])
self.assertFalse(tuples[2][0])
obj5 = tuples[0][2]
obj6 = tuples[1][2]
for key in ['_id', 'name', 'description', 'serial_number']:
self.assertEqual(obj3.__dict__[key],obj5.__dict__[key], msg='objects do not have the same '+key)
objs = [ obj4, obj5 ] # 4 has obsolete _rev
tuples = self.repo.delete('sample', objs)
self.assertFalse(tuples[0][0])
self.assertTrue(tuples[1][0], msg='failed: '+str(tuples[1][2]) +'\nobj: ' + repr(obj5.__dict__))
objs = [ obj5, obj6 ] # 5 is already deleted
tuples = self.repo.delete('sample', objs)
self.assertFalse(tuples[0][0])
self.assertTrue(tuples[1][0], msg='failed: '+str(tuples[1][2]) +'\nobj: ' + repr(obj5.__dict__))
def create_negotiation(self, sap=None):
if sap is None or ( sap.type_ != OT.ServiceAgreementProposal and not issubtype(sap.type_, OT.ServiceAgreementProposal)):
raise BadRequest('The sap parameter must be a valid Service Agreement Proposal object')
if sap.proposal_status != ProposalStatusEnum.INITIAL or sap.sequence_num != 0:
raise Inconsistent('The specified Service Agreement Proposal has inconsistent status fields')
if sap.negotiation_id != '':
raise Inconsistent('The specified Service Agreement Proposal cannot have a negotiation_id for an initial proposal')
if self.negotiation_rules.has_key(sap.type_):
#validate preconditions before creating
for pc in self.negotiation_rules[sap.type_]['pre_conditions']:
if pc.startswith('not '):
pre_condition_met = not eval("self.service_provider." + pc.lstrip('not ')) #Strip off the 'not ' part
else:
pre_condition_met = eval("self.service_provider."+pc)
if not pre_condition_met:
raise BadRequest("A precondition for this request has not been satisfied: %s" % pc)
#Should be able to determine the negotiation type based on the intial originator
neg_type = NegotiationTypeEnum.REQUEST
if sap.originator == ProposalOriginatorEnum.PROVIDER:
neg_type = NegotiationTypeEnum.INVITATION
elif sap.originator == ProposalOriginatorEnum.BROKER:
neg_type = NegotiationTypeEnum.BROKERED
neg_obj = IonObject(RT.Negotiation, negotiation_type=neg_type)
#If there is a description in the initial proposal, then set the negotiation description with it.
if sap.description != '':
neg_obj.description = sap.description
neg_id,_ = self.service_provider.clients.resource_registry.create(neg_obj)
#Create associations between the parties
self.service_provider.clients.resource_registry.create_association(sap.consumer, PRED.hasNegotiation, neg_id)
self.service_provider.clients.resource_registry.create_association(sap.provider, PRED.hasNegotiation, neg_id)
if sap.broker != "":
self.service_provider.clients.resource_registry.create_association(sap.broker, PRED.hasNegotiation, neg_id)
return neg_id
def create_negotiation(self, sap=None):
if sap is None or ( sap.type_ != OT.ServiceAgreementProposal and not issubtype(sap.type_, OT.ServiceAgreementProposal)):
raise BadRequest('The sap parameter must be a valid Service Agreement Proposal object')
if sap.proposal_status != ProposalStatusEnum.INITIAL or sap.sequence_num != 0:
raise Inconsistent('The specified Service Agreement Proposal has inconsistent status fields')
if sap.negotiation_id != '':
raise Inconsistent('The specified Service Agreement Proposal cannot have a negotiation_id for an initial proposal')
if self.negotiation_rules.has_key(sap.type_):
#validate preconditions before creating
for pc in self.negotiation_rules[sap.type_]['pre_conditions']:
if pc.startswith('not '):
pre_condition_met = not eval("self.service_provider." + pc.lstrip('not ')) #Strip off the 'not ' part
else:
pre_condition_met = eval("self.service_provider."+pc)
if not pre_condition_met:
raise BadRequest("A precondition for this request has not been satisfied: %s" % pc)
#Should be able to determine the negotiation type based on the intial originator
neg_type = NegotiationTypeEnum.REQUEST
if sap.originator == ProposalOriginatorEnum.PROVIDER:
neg_type = NegotiationTypeEnum.INVITATION
elif sap.originator == ProposalOriginatorEnum.BROKER:
neg_type = NegotiationTypeEnum.BROKERED
neg_obj = IonObject(RT.Negotiation, negotiation_type=neg_type)
#If there is a description in the initial proposal, then set the negotiation description with it.
if sap.description != '':
neg_obj.description = sap.description
neg_id,_ = self.service_provider.clients.resource_registry.create(neg_obj)
#Create associations between the parties
self.service_provider.clients.resource_registry.create_association(sap.consumer, PRED.hasNegotiation, neg_id)
self.service_provider.clients.resource_registry.create_association(sap.provider, PRED.hasNegotiation, neg_id)
if sap.broker != "":
self.service_provider.clients.resource_registry.create_association(sap.broker, PRED.hasNegotiation, neg_id)
return neg_id
def has_resource_commitments(self, actor_id, resource_id):
ret_status = IonObject(OT.ResourceCommitmentStatus)
commitments = self.get_resource_commitments(actor_id, resource_id)
if commitments is None:
#No commitments were found between this resource_id and actor_id
return ret_status
ret_status.shared = True
for com in commitments:
if com.commitment.exclusive == True:
#Found an exclusive commitment
ret_status.exclusive = True
return ret_status
#Only a shared commitment was found
return ret_status
def has_resource_commitments(self, actor_id, resource_id):
ret_status = IonObject(OT.ResourceCommitmentStatus)
commitments = self.get_resource_commitments(actor_id, resource_id)
if commitments is None:
#No commitments were found between this resource_id and actor_id
return ret_status
ret_status.shared = True
for com in commitments:
if com.commitment.exclusive == True:
#Found an exclusive commitment
ret_status.exclusive = True
return ret_status
#Only a shared commitment was found
return ret_status
def svcmethod(self, *args, **kwargs):
assert len(
args
) == 0, "You MUST used named keyword args when calling a dynamically generated remote method" # we have no way of getting correct order
headers = kwargs.pop(
'headers', None
) # pull headers off, cannot put this in the signature due to *args for ordering
ionobj = IonObject(in_obj, **kwargs)
return self.request(ionobj, op=name, headers=headers)
def on_init(self):
IonObject("Resource") # suppress pyflakes error
self.override_clients(self.clients)
self.init_module_uploader()
self.get_unique_id = (lambda: uuid4().hex)
self.data_product_management = DataProductManagementServiceClient()
def set_computed_attributes(self, res_container, computed_resource_type,
ext_exclude):
if not computed_resource_type or computed_resource_type is None:
return
res_container.computed = IonObject(computed_resource_type)
self.set_object_field_values(res_container.computed,
res_container.resource, ext_exclude)
def set_computed_attributes(self, res_container, computed_resource_type, ext_exclude, **kwargs):
"""
Creates the specified ComputedAttributes object if given and iterate over the fields
to set the computed values.
"""
if not computed_resource_type or computed_resource_type is None:
return
res_container.computed = IonObject(computed_resource_type)
self.set_object_field_values(res_container.computed, res_container.resource, ext_exclude, **kwargs)
def create_enrollment_request(self, org_id, user_id):
request_object = IonObject(
RT.EnrollmentRequest,
name='Enroll Request',
org_id=org_id,
user_id=user_id,
status="Initialized",
description='%s Org Enrollment Request at %s' %
(user_id, str(now)))
return request_object
def create_role_request(self, org_id, user_id, role_name):
request_object = IonObject(RT.RoleRequest,
name='Role Request',
org_id=org_id,
user_id=user_id,
role_name=role_name,
status="Initialized",
description='%s Role Request at %s' %
(user_id, str(now)))
return request_object
def test_logging_handler(self):
""" initial log level for ion.process.event is INFO -- test we can change it to TRACE """
config.replace_configuration(os.path.join(TEST_DIR, 'logging.yml'))
log.debug('this should probably not be logged')
self.assertFalse(log.isEnabledFor(TRACE))
#
handler = pyon.container.management.LogLevelHandler()
action = IonObject(OT.ChangeLogLevel, logger='pyon.container', level='TRACE')
handler.handle_request(action)
#
self.assertTrue(log.isEnabledFor(TRACE))
def xtest_logging_root(self):
""" initial root log level is WARN -- test that we can change it to ERROR """
config.replace_configuration(os.path.join(TEST_DIR, 'logging.yml'))
otherlog = logging.getLogger('pyon.container')
self.assertTrue(otherlog.isEnabledFor(logging.WARN))
#
handler = pyon.container.management.LogLevelHandler()
action = IonObject(OT.ChangeLogLevel, logger='pyon', level='ERROR')
handler.handle_request(action)
#
self.assertFalse(otherlog.isEnabledFor(logging.WARN))
def test_expected_exceptions(self):
""" test uses that are expected to throw exceptions (and probably need code fixes) """
obj1 = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345")
# invalid name for repository
try:
self.repo.insert('bad name', obj1)
self.fail('should fail')
except:
pass
# invalid argument types
try:
self.repo.insert('sample', None)
self.fail('should fail')
except:
pass
try:
self.repo.insert('sample', "not ion obj")
self.fail('should fail')
except:
pass
# cannot reinsert with same id
_,id,__ = self.repo.insert('sample', obj1)
_,__,obj2 = self.repo.read('sample', id)
try:
success,_,__ = self.repo.insert('sample', obj2)
self.fail('should not succeed')
except:
pass
# cannot insert with _rev
obj1._rev = obj2._rev
try:
success,_,__ = self.repo.insert('sample', obj1)
self.fail('should not succeed')
except:
pass
def test_create_counter_proposal(self):
with self.assertRaises(BadRequest) as cm:
consumer_accept_sap = Negotiation.create_counter_proposal(
proposal_status=ProposalStatusEnum.INITIAL)
self.assertIn(
'The negotiation parameter must be a valid Negotiation object',
cm.exception.message)
negotiation_handler = Negotiation(self)
sap = IonObject(OT.EnrollmentProposal,
consumer=self.actor_identity._id,
provider=self.org._id)
negotiation = Mock()
negotiation._id = '456'
negotiation.type_ = RT.Negotiation
negotiation.proposals = [sap]
self.mock_read.return_value = negotiation
self.mock_create.return_value = ['456', 2]
neg_id = negotiation_handler.create_negotiation(sap)
sap.negotiation_id = neg_id
consumer_accept_sap = Negotiation.create_counter_proposal(
negotiation,
proposal_status=ProposalStatusEnum.COUNTER,
originator=ProposalOriginatorEnum.PROVIDER)
self.assertEqual(consumer_accept_sap.negotiation_id, negotiation._id)
self.assertEqual(len(negotiation.proposals), 1)
self.assertEqual(consumer_accept_sap.sequence_num,
len(negotiation.proposals))
self.assertEqual(consumer_accept_sap.proposal_status,
ProposalStatusEnum.COUNTER)
self.assertEqual(consumer_accept_sap.originator,
ProposalOriginatorEnum.PROVIDER)
def test_encode_decode(self):
obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
d = self.subject.encode(obj)
print 'keys: ' + repr(d)
self.assertFalse('_id' in d.keys())
new_obj = self.subject.decode(d)
self.assertEqual(obj, new_obj)
d2 = self.subject.encode(new_obj, add_id=True)
self.assertTrue('_id' in d2.keys(), msg=repr(d2.keys()))
# retains key once added
d3 = self.subject.encode(self.subject.decode(d2))
self.assertTrue('_id' in d3.keys())
def test_create_negotiation(self):
self.preconditions.check_method1.return_value = True
self.preconditions.check_method2.return_value = False
self.accept_actions.accept_method.return_value = None
negotiation_rules = {
OT.EnrollmentProposal: {
'pre_conditions': ['preconditions.check_method1(sap.consumer)', 'not preconditions.check_method2(sap.provider,sap.consumer)'],
'accept_action': 'accept_actions.accept_method(sap.provider,sap.consumer)'
}}
negotiation_handler = Negotiation(self, negotiation_rules)
with self.assertRaises(BadRequest) as cm:
negotiation_handler.create_negotiation()
self.assertIn('The sap parameter must be a valid Service Agreement Proposal object',cm.exception.message)
sap = IonObject(OT.EnrollmentProposal,consumer=self.actor_identity._id, provider=self.org._id )
sap.sequence_num = 1 #Force an error
with self.assertRaises(Inconsistent) as cm:
negotiation_handler.create_negotiation(sap)
self.assertIn('The specified Service Agreement Proposal has inconsistent status fields',cm.exception.message)
sap = IonObject(OT.EnrollmentProposal,consumer=self.actor_identity._id, provider=self.org._id )
sap.proposal_status = ProposalStatusEnum.COUNTER #Force an error
with self.assertRaises(Inconsistent) as cm:
negotiation_handler.create_negotiation(sap)
self.assertIn('The specified Service Agreement Proposal has inconsistent status fields',cm.exception.message)
sap = IonObject(OT.EnrollmentProposal,consumer=self.actor_identity._id, provider=self.org._id )
sap.negotiation_id = 'efefeff' #Force an error
with self.assertRaises(Inconsistent) as cm:
negotiation_handler.create_negotiation(sap)
self.assertIn('The specified Service Agreement Proposal cannot have a negotiation_id for an initial proposal',cm.exception.message)
sap = IonObject(OT.EnrollmentProposal,consumer=self.actor_identity._id, provider=self.org._id )
negotiation = Mock()
negotiation._id = '456'
negotiation.type_ = RT.Negotiation
negotiation.proposals = []
self.mock_read.return_value = negotiation
self.mock_create.return_value = ['456', 2]
neg_id = negotiation_handler.create_negotiation(sap)
self.assertEqual(neg_id, negotiation._id)
self.assertEqual(len(negotiation.proposals),0)
self.assertEqual(self.preconditions.check_method1.called,True)
self.assertEqual(self.preconditions.check_method2.called,True)
self.assertEqual(self.accept_actions.accept_method.called,False)
def create_prepare_resource_support(self, resource_id="", prepare_resource_type=None, origin_resource_type=None):
if not isinstance(resource_id, types.StringType):
raise Inconsistent("The parameter resource_id is not a single resource id string")
if not self.service_provider or not self._rr:
raise Inconsistent("This class is not initialized properly")
if prepare_resource_type is not None and prepare_resource_type not in getextends(OT.ResourcePrepareSupport):
raise BadRequest('The requested resource %s is not extended from %s' % (prepare_resource_type, OT.ResourcePrepareSupport))
resource_data = IonObject(prepare_resource_type)
# Check to make sure the extended resource decorator raise OriginResourceType matches the type of the resource type
origin_resource_decorator = resource_data.get_class_decorator_value('OriginResourceType')
if origin_resource_decorator is None and origin_resource_type is None:
raise NotFound('OriginResourceType decorator not found in object specification %s', prepare_resource_type)
origin_resource_type = origin_resource_type if origin_resource_type is not None else origin_resource_decorator
if origin_resource_type is None:
raise NotFound('OriginResourceType decorator not found in object specification %s', prepare_resource_type)
resource_object = None
if resource_id:
resource_object = self._rr.read(resource_id)
if origin_resource_type != resource_object.type_ and not issubtype(resource_object.type_, origin_resource_type):
raise Inconsistent('The OriginResourceType decorator of the requested resource %s(%s) does not match the type of the specified resource id(%s).' % (
prepare_resource_type, origin_resource_type, resource_object.type_))
resource_data._id = resource_object._id
else:
resource_object = IonObject(origin_resource_type)
resource_data.resource = resource_object
resource_data.resource_schema = get_object_schema(origin_resource_type)
for field in resource_data._schema:
deco_value = resource_data.get_decorator_value(field, 'AssociatedResources')
assoc_dict = {}
if deco_value is not None:
if deco_value.find(',') == -1:
associated_resources = [deco_value]
else:
associated_resources = deco_value.split(',')
for res in associated_resources:
assoc = self.get_associated_resource_info(origin_resource_type, resource_id, res)
assoc_dict[assoc.key] = assoc
setattr(resource_data, field, assoc_dict)
continue
return resource_data
def test_read_negotiation(self):
negotiation_handler = Negotiation(self)
with self.assertRaises(BadRequest) as cm:
negotiation_handler.read_negotiation()
self.assertIn('The sap parameter must be a valid Service Agreement Proposal object',cm.exception.message)
sap = IonObject(OT.EnrollmentProposal,consumer=self.actor_identity._id, provider=self.org._id )
with self.assertRaises(BadRequest) as cm:
negotiation_handler.read_negotiation(sap)
self.assertIn('The Service Agreement Proposal object (sap) is missing a negotiation_id value',cm.exception.message)
negotiation = Mock()
negotiation._id = '456'
sap.negotiation_id = negotiation._id
self.mock_read.return_value = negotiation
neg_obj = negotiation_handler.read_negotiation(sap)
self.assertEqual(neg_obj, negotiation)
def _execute(self, cprefix, command):
if not command:
raise iex.BadRequest("execute argument 'command' not present")
if not command.command:
raise iex.BadRequest("command not set")
cmd_res = IonObject("AgentCommandResult", command_id=command.command_id, command=command.command)
cmd_func = getattr(self, cprefix + str(command.command), None)
if cmd_func:
cmd_res.ts_execute = get_ion_ts()
try:
res = cmd_func(*command.args, **command.kwargs)
cmd_res.status = 0
cmd_res.result = res
except Exception as ex:
cmd_res.status = getattr(ex, 'status_code', -1)
cmd_res.result = str(ex)
else:
ex = iex.NotFound("Command not supported: %s" % command.command)
cmd_res.status = iex.NotFound.status_code
cmd_res.result = str(ex)
return cmd_res
def test_datastore_basic(self):
data_store = self.ds_class(datastore_name='ion_test_ds', profile=DataStore.DS_PROFILE.RESOURCES, scope=get_sys_name())
self.data_store = data_store
self.resources = {}
# Just in case previous run failed without cleaning up,
# delete data store
try:
data_store.delete_datastore()
except NotFound:
pass
# Create should succeed and not throw exception
data_store.create_datastore()
# Create should throw exception the second time
with self.assertRaises(BadRequest):
data_store.create_datastore()
# Call ops with wrong object type and make sure exception is thrown
with self.assertRaises(BadRequest):
data_store.create({"foo": "bar"})
with self.assertRaises(BadRequest):
data_store.create_mult([{"foo": "bar"}])
with self.assertRaises(BadRequest):
data_store.read({"foo": "bar"})
with self.assertRaises(BadRequest):
data_store.read_mult([{"foo": "bar"}])
with self.assertRaises(BadRequest):
data_store.update({"foo": "bar"})
with self.assertRaises(BadRequest):
data_store.delete({"foo": "bar"})
# Should see new data
self.assertTrue([1 for dsn in data_store.list_datastores() if dsn in ('ion_test_ds', '%s_%s' % (get_sys_name(), 'ion_test_ds'))])
# Something should be returned
self.assertTrue(data_store.info_datastore() is not None)
res = data_store.list_objects()
numcoredocs = len(res)
# Construct user role objects
admin_role = {
"name":"Admin",
"description":"Super user"
}
admin_role_obj = IonObject('UserRole', admin_role)
admin_role_tuple = data_store.create(admin_role_obj)
self.assertTrue(len(admin_role_tuple) == 2)
admin_role_ooi_id = admin_role_tuple[0]
# Check that a create fails with an existing ID
admin_role_obj2 = IonObject('UserRole', admin_role)
with self.assertRaises(BadRequest):
data_store.create(admin_role_obj2, object_id=admin_role_ooi_id)
data_provider_role = {
"name": "Data Provider",
"description": "User allowed to ingest data sets"
}
data_provider_role_obj = IonObject('UserRole', data_provider_role)
data_provider_role_tuple = data_store.create(data_provider_role_obj)
self.assertTrue(len(data_provider_role_tuple) == 2)
data_provider_role_ooi_id = data_provider_role_tuple[0]
marine_operator_role = {
"name": "Marine Operator",
"description": "User allowed to administer instruments"
}
marine_operator_role_obj = IonObject('UserRole', marine_operator_role)
marine_operator_role_tuple = data_store.create(marine_operator_role_obj)
self.assertTrue(len(marine_operator_role_tuple) == 2)
marine_operator_role_ooi_id = marine_operator_role_tuple[0]
role_objs = data_store.read_mult([admin_role_ooi_id, data_provider_role_ooi_id, marine_operator_role_ooi_id])
self.assertTrue(len(role_objs) == 3)
self.assertTrue(role_objs[0]._id == admin_role_ooi_id)
self.assertTrue(role_objs[1]._id == data_provider_role_ooi_id)
self.assertTrue(role_objs[2]._id == marine_operator_role_ooi_id)
with self.assertRaises(NotFound):
data_store.read_mult([admin_role_ooi_id, data_provider_role_ooi_id, "NONEXISTENT"])
role_objs1 = data_store.read_mult([admin_role_ooi_id, "NONEXISTENT", data_provider_role_ooi_id], strict=False)
self.assertTrue(len(role_objs1) == 3)
self.assertTrue(role_objs1[0]._id == admin_role_ooi_id)
self.assertTrue(role_objs1[1] is None)
self.assertTrue(role_objs1[2]._id == data_provider_role_ooi_id)
# Construct three user info objects and assign them roles
hvl_contact_info = {
"individual_names_given": "Heitor Villa-Lobos",
"email": "*****@*****.**",
"variables": [
{"name": "Claim To Fame", "value": "Legendary Brazilian composer"}
]
}
hvl_contact_info_obj = IonObject('ContactInformation', hvl_contact_info)
hvl_user_info = {
"name": "Heitor Villa-Lobos",
#"contact": hvl_contact_info_obj
}
hvl_user_info_obj = IonObject('ActorIdentity', hvl_user_info)
hvl_user_info_tuple = data_store.create(hvl_user_info_obj)
self.assertTrue(len(hvl_user_info_tuple) == 2)
heitor_villa_lobos_ooi_id = hvl_user_info_tuple[0]
ats_contact_info = {
"individual_names_given": "Andres Torres Segovia",
"email": "*****@*****.**",
"variables": [
{"name": "Claim To Fame", "value": "Legendary Concert Guitarist"}
]
}
ats_contact_info_obj = IonObject('ContactInformation', ats_contact_info)
ats_user_info = {
"name": "Andres Torres Segovia",
#"contact": ats_contact_info_obj
}
ats_user_info_obj = IonObject('ActorIdentity', ats_user_info)
ats_user_info_tuple = data_store.create(ats_user_info_obj)
self.assertTrue(len(ats_user_info_tuple) == 2)
pok_contact_info = {
"individual_names_given": "Per-Olov Kindgren",
"email": "*****@*****.**",
"variables": [
{"name": "Claim To Fame", "value": "Composer and YouTube star"}
]
}
pok_contact_info_obj = IonObject('ContactInformation', pok_contact_info)
pok_user_info = {
"name": "Per-Olov Kindgren",
#"contact": pok_contact_info_obj
}
pok_user_info_obj = IonObject('ActorIdentity', pok_user_info)
pok_user_info_tuple = data_store.create(pok_user_info_obj)
self.assertTrue(len(pok_user_info_tuple) == 2)
# List all objects in data store and confirm there are six docs
res = data_store.list_objects()
# There are indices. Therefore can't could all docs
self.assertTrue(len(res) == 6 + numcoredocs)
# Create an Ion object with default values set (if any)
data_set = IonObject('Commitment')
self.assertTrue(isinstance(data_set, interface.objects.Commitment))
# Assign values to object fields
data_set.description = "Real-time water data for Choptank River near Greensboro, MD"
# Write Dataset object"
write_tuple_1 = data_store.create(data_set)
self.assertTrue(len(write_tuple_1) == 2)
# Save off the object UUID
data_set_uuid = write_tuple_1[0]
# Read back the HEAD version of the object and validate fields
data_set_read_obj = data_store.read(data_set_uuid)
self.assertTrue(data_set_read_obj._id == data_set_uuid)
self.assertTrue(isinstance(data_set_read_obj, interface.objects.Commitment))
self.assertTrue(data_set_read_obj.description == "Real-time water data for Choptank River near Greensboro, MD")
self.assertTrue('type_' in data_set_read_obj)
data_set_read_obj2 = data_store.read(data_set_uuid)
# Update Dataset's Description field and write
data_set_read_obj.description = "Updated Description"
write_tuple_2 = data_store.update(data_set_read_obj)
self.assertTrue(len(write_tuple_2) == 2)
# Retrieve the updated Dataset
data_set_read_obj_2 = data_store.read(data_set_uuid)
self.assertTrue(data_set_read_obj_2._id == data_set_uuid)
self.assertTrue(data_set_read_obj_2.description == "Updated Description")
# Check that a second update fails
self.assertNotEqual(data_set_read_obj_2._rev, data_set_read_obj2._rev)
with self.assertRaises(Conflict):
data_store.update(data_set_read_obj2)
# Test update with non-existing object
spurious_obj = IonObject('Commitment')
with self.assertRaises(BadRequest):
data_store.update(spurious_obj)
spurious_obj._rev = data_set_read_obj._rev
with self.assertRaises(BadRequest):
data_store.update(spurious_obj)
#spurious_obj._id = "NON EXISTING ID"
#with self.assertRaises(NotFound):
# data_store.update(spurious_obj)
# Do another update to the object
data_set_read_obj_2.description = "USGS instantaneous value data for station 01491000"
write_tuple_3 = data_store.update(data_set_read_obj_2)
# Retrieve HEAD version of Dataset
head = data_store.read(data_set_uuid)
self.assertTrue(head._id == data_set_uuid)
self.assertTrue(head.description == "USGS instantaneous value data for station 01491000")
# Delete Dataset by object id
data_store.delete(head)
xobj = dict(some="content")
xoid, _ = data_store.create_doc(xobj)
xobj2 = data_store.read_doc(xoid)
xobj2["_deleted"] = True
data_store.update_doc(xobj2)
with self.assertRaises(NotFound):
data_store.read_doc(xoid)
xobj = dict(some="content1")
xoid, _ = data_store.create_doc(xobj)
xobj2 = data_store.read_doc(xoid)
xobj2["_deleted"] = True
data_store.update_doc_mult([xobj2])
with self.assertRaises(NotFound):
data_store.read_doc(xoid)
# Try to re-delete Dataset by object id. Should throw exception.
with self.assertRaises(NotFound):
data_store.delete(head._id)
# List all objects in data store, should be back to six
res = data_store.list_objects()
self.assertTrue(len(res) == 6 + numcoredocs)
# List revisions of now deleted Dataset, should be empty list
res = data_store.list_object_revisions(data_set_uuid)
self.assertTrue(len(res) == 0)
o1 = IonObject("Commitment", name="One more")
o2 = IonObject("Commitment", name="Another one")
res = data_store.create_mult((o1, o2))
self.assertTrue(all([success for success, oid, rev in res]))
oids = [oid for success, oid, rev in res]
res = data_store.list_objects()
self.assertTrue(len(res) == 8 + numcoredocs)
o1 = dict(type_="Resource", name="name1xxx", visibility=1, lcstate=LCS.DRAFT, availability=AS.AVAILABLE)
o2 = dict(type_="DirEntry", key="key1", parent="/Parent", org="ION")
o3 = dict(type_="Association", s=oids[0], o=oids[1], st="Dataset", ot="Dataset", p="some", retired=False)
res = data_store.create_doc_mult([o1, o2, o3])
start_key = ["ION", "/Parent", 0]
end_key = ["ION", "/Parent"]
res1 = data_store.find_docs_by_view("directory", "by_parent", start_key=start_key, end_key=end_key, id_only=True)
self.assertEquals(len(res1), 1)
res1 = data_store.find_associations(predicate="some", id_only=True)
self.assertEquals(len(res1), 1)
res1,_ = data_store.find_resources(name="name1xxx", id_only=True)
self.assertEquals(len(res1), 1)
res = data_store.list_objects()
self.assertTrue(len(res) == 11 + numcoredocs)
# Delete data store to clean up
data_store.delete_datastore()
# Assert data store is now gone
self.assertFalse([1 for dsn in data_store.list_datastores() if dsn in ('ion_test_ds', '%s_%s' % (get_sys_name(), 'ion_test_ds'))])
def test_get_actor_header(self):
#Setup data
actor = IonObject(RT.ActorIdentity, name='actor1')
actor_id, _ = self.rr.create(actor)
ion_org = IonObject(RT.Org, name='ION', org_governance_name='ION')
ion_org_id, _ = self.rr.create(ion_org)
ion_org._id = ion_org_id
manager_role = IonObject(RT.UserRole, name='Org Manager', governance_name=ORG_MANAGER_ROLE, description='Org Manager')
manager_role_id = self.add_user_role(ion_org, manager_role)
member_role = IonObject(RT.UserRole, name='Org Member', governance_name=ORG_MEMBER_ROLE, description='Org Member')
# all actors have a defaul org_member_role
actor_roles = find_roles_by_actor(actor_id)
self.assertDictEqual(actor_roles, {'ION': [ORG_MEMBER_ROLE]})
actor_header = get_actor_header(actor_id)
self.assertDictEqual(actor_header, {'ion-actor-id': actor_id, 'ion-actor-roles': {'ION': [ORG_MEMBER_ROLE]}})
#Add Org Manager Role
self.rr.create_association(actor_id, PRED.hasRole, manager_role_id)
actor_roles = find_roles_by_actor(actor_id)
role_header = get_role_message_headers({'ION': [manager_role, member_role]})
self.assertDictEqual(actor_roles, role_header)
org2 = IonObject(RT.Org, name='Org 2', org_governance_name='Second_Org')
org2_id, _ = self.rr.create(org2)
org2._id = org2_id
member2_role = IonObject(RT.UserRole, governance_name=ORG_MEMBER_ROLE, name='Org Member', description='Org Member')
member2_role_id = self.add_user_role(org2, member2_role)
operator2_role = IonObject(RT.UserRole, governance_name='INSTRUMENT_OPERATOR', name='Instrument Operator',
description='Instrument Operator')
operator2_role_id = self.add_user_role(org2, operator2_role)
self.rr.create_association(actor_id, PRED.hasRole, member2_role_id)
self.rr.create_association(actor_id, PRED.hasRole, operator2_role_id)
actor_roles = find_roles_by_actor(actor_id)
role_header = get_role_message_headers({'ION': [manager_role, member_role], 'Second_Org': [operator2_role, member2_role]})
self.assertEqual(len(actor_roles), 2)
self.assertEqual(len(role_header), 2)
self.assertIn('Second_Org', actor_roles)
self.assertIn('Second_Org', role_header)
self.assertEqual(len(actor_roles['Second_Org']), 2)
self.assertEqual(len(role_header['Second_Org']), 2)
self.assertIn('INSTRUMENT_OPERATOR', actor_roles['Second_Org'])
self.assertIn('INSTRUMENT_OPERATOR', role_header['Second_Org'])
self.assertIn(ORG_MEMBER_ROLE, actor_roles['Second_Org'])
self.assertIn(ORG_MEMBER_ROLE, role_header['Second_Org'])
self.assertIn('ION', actor_roles)
self.assertIn('ION', role_header)
self.assertIn(ORG_MANAGER_ROLE, actor_roles['ION'])
self.assertIn(ORG_MEMBER_ROLE, actor_roles['ION'])
self.assertIn(ORG_MANAGER_ROLE, role_header['ION'])
self.assertIn(ORG_MEMBER_ROLE, role_header['ION'])
actor_header = get_actor_header(actor_id)
self.assertEqual(actor_header['ion-actor-id'], actor_id)
self.assertEqual(actor_header['ion-actor-roles'], actor_roles)
#Now make sure we can change the name of the Org and not affect the headers
org2 = self.rr.read(org2_id)
org2.name = 'Updated Org 2'
org2_id, _ = self.rr.update(org2)
actor_roles = find_roles_by_actor(actor_id)
self.assertEqual(len(actor_roles), 2)
self.assertEqual(len(role_header), 2)
self.assertIn('Second_Org', actor_roles)
self.assertIn('Second_Org', role_header)
self.assertEqual(len(actor_roles['Second_Org']), 2)
self.assertEqual(len(role_header['Second_Org']), 2)
self.assertIn('INSTRUMENT_OPERATOR', actor_roles['Second_Org'])
self.assertIn('INSTRUMENT_OPERATOR', role_header['Second_Org'])
self.assertIn(ORG_MEMBER_ROLE, actor_roles['Second_Org'])
self.assertIn(ORG_MEMBER_ROLE, role_header['Second_Org'])
self.assertIn('ION', actor_roles)
self.assertIn('ION', role_header)
self.assertIn(ORG_MANAGER_ROLE, actor_roles['ION'])
self.assertIn(ORG_MEMBER_ROLE, actor_roles['ION'])
self.assertIn(ORG_MANAGER_ROLE, role_header['ION'])
self.assertIn(ORG_MEMBER_ROLE, role_header['ION'])
actor_header = get_actor_header(actor_id)
self.assertEqual(actor_header['ion-actor-id'], actor_id)
self.assertEqual(actor_header['ion-actor-roles'], actor_roles)
def test_update_negotiation(self):
self.preconditions.check_method1.return_value = True
self.preconditions.check_method2.return_value = False
self.accept_actions.accept_method.return_value = None
negotiation_rules = {
OT.EnrollmentProposal: {
'pre_conditions': ['preconditions.check_method1(sap.consumer)', 'not preconditions.check_method2(sap.provider,sap.consumer)'],
'accept_action': 'accept_actions.accept_method(sap.provider,sap.consumer)'
}}
negotiation_handler = Negotiation(self, negotiation_rules, self.event_pub)
with self.assertRaises(Inconsistent) as cm:
negotiation_handler.update_negotiation()
self.assertIn('The Service Agreement Proposal must have a negotiation resource id associated with it',cm.exception.message)
sap = IonObject(OT.EnrollmentProposal,consumer=self.actor_identity._id, provider=self.org._id )
with self.assertRaises(Inconsistent) as cm:
negotiation_handler.update_negotiation(sap)
self.assertIn('The Service Agreement Proposal must have a negotiation resource id associated with it',cm.exception.message)
negotiation = Mock()
negotiation._id = '456'
negotiation.type_ = RT.Negotiation
negotiation.proposals = []
sap.negotiation_id = negotiation._id
self.mock_read.return_value = negotiation
self.mock_update.return_value = ['456', 2]
neg_id = negotiation_handler.update_negotiation(sap)
self.assertEqual(self.event_pub.publish_event.called,True)
self.assertEqual(neg_id, negotiation._id)
self.assertEqual(len(negotiation.proposals),1)
counter_sap = Negotiation.create_counter_proposal(negotiation, ProposalStatusEnum.REJECTED, ProposalOriginatorEnum.PROVIDER)
neg_id = negotiation_handler.update_negotiation(counter_sap, 'Fake rejection reason')
self.assertEqual(len(negotiation.proposals),2)
self.assertEqual(negotiation.negotiation_status, NegotiationStatusEnum.REJECTED)
self.assertEquals(negotiation.reason, 'Fake rejection reason' )
counter_sap = Negotiation.create_counter_proposal(negotiation, ProposalStatusEnum.ACCEPTED, ProposalOriginatorEnum.PROVIDER)
neg_id = negotiation_handler.update_negotiation(counter_sap)
self.assertEqual(len(negotiation.proposals),3)
self.assertEqual(negotiation.negotiation_status, NegotiationStatusEnum.REJECTED)
counter_sap = Negotiation.create_counter_proposal(negotiation, ProposalStatusEnum.ACCEPTED, ProposalOriginatorEnum.CONSUMER)
neg_id = negotiation_handler.update_negotiation(counter_sap)
self.assertEqual(len(negotiation.proposals),4)
self.assertEqual(negotiation.negotiation_status, NegotiationStatusEnum.ACCEPTED)
self.assertEqual(self.accept_actions.accept_method.called,True)
def test_decorator_validation(self):
#
# Test required values
#
obj = IonObject('Deco_Example', {"list1": [1], "list2": ["One element"], "dict1": {"key1": 1}, "dict2": {"key1": 1}, "us_phone_number": "555-555-5555"})
# Should fail because required value not provided
self.assertRaises(AttributeError, obj._validate)
obj.an_important_value = {"key": "value"}
# Should work now that we have set a value for the required field
obj._validate
#
# Test collection content types validation
#
# List
obj = IonObject('Deco_Example', {"list1": ["Should be a numeric type"], "list2": ["One element"], "dict1": {"key1": 1}, "dict2": {"key1": 1}, "an_important_value": "good value", "us_phone_number": "555-555-5555"})
self.assertRaises(AttributeError, obj._validate)
obj.list1 = [1, 2]
# Should work now that list content has been corrected
obj._validate
# Dict
obj = IonObject('Deco_Example', {"list1": [1], "list2": ["One element"], "dict1": {"key1": "Should be a numeric type"}, "dict2": {"key1": 1}, "an_important_value": "good value", "us_phone_number": "555-555-5555"})
# Should fail because dict value contains non-numeric value
self.assertRaises(AttributeError, obj._validate)
obj.dict1 = {"key1": 1}
# Should work now that dict value content has been corrected
obj._validate
#
# Test collection length
#
# List
obj = IonObject('Deco_Example', {"list1": [1,2], "list2": [], "dict1": {"key1": 1}, "dict2": {"key1": 1}, "an_important_value": "good value", "us_phone_number": "555-555-5555"})
# Should fail since list has zero length
self.assertRaises(AttributeError, obj._validate)
obj.list2 = ["Item 1", "Item 2"]
# Should work new that item length of list has been corrected
obj._validate
# Dict
obj = IonObject('Deco_Example', {"list1": [1,2], "list2": [1,2], "dict1": {"key1": 1}, "dict2": {}, "an_important_value": "good value", "us_phone_number": "555-555-5555"})
# Should fail since dict has zero length
self.assertRaises(AttributeError, obj._validate)
obj.dict2 = {"key1": 1}
# Should work new that item length of dict has been corrected
obj._validate
#
# Test numeric value range
#
# int
obj = IonObject('Deco_Example', {"list1": [1,2], "list2": ["One element"], "dict1": {"key1": 1}, "dict2": {"key1": 1}, "unsigned_short_int": -1, "an_important_value": "good value", "us_phone_number": "555-555-5555"})
self.assertRaises(AttributeError, obj._validate)
obj.unsigned_short_int = 256
# Should work
obj._validate
# float
obj = IonObject('Deco_Example', {"list1": [1,2], "list2": ["One element"], "dict1": {"key1": 1}, "dict2": {"key1": 1}, "a_float": 10.11, "an_important_value": "good value", "us_phone_number": "555-555-5555"})
self.assertRaises(AttributeError, obj._validate)
obj.a_float = 1.234
# Should work
obj._validate
#
# Test string pattern matching
#
obj = IonObject('Deco_Example', {"list1": [1,2], "list2": ["One element"], "dict1": {"key1": 1}, "dict2": {"key1": 1}, "an_important_value": "good value", "us_phone_number": "5555555555"})
self.assertRaises(AttributeError, obj._validate)
obj.us_phone_number = "555-555-5555"
# Should work
obj._validate
def _create_resource(self, restype, name, *args, **kwargs):
res_obj = IonObject(restype, dict(name=name, **kwargs))
res_obj_res = self.data_store.create(res_obj)
res_obj._id = res_obj_res[0]
self.resources[name] = res_obj
return res_obj_res[0]
def side_effect(_def, _dict=None, **kwargs):
test_obj = IonObject(_def, _dict, **kwargs)
test_obj._validate()
return DEFAULT
def _do_test(self, data_store):
self.data_store = data_store
self.resources = {}
# Just in case previous run failed without cleaning up,
# delete data store
data_store.delete_datastore()
# Create should succeed and not throw error
data_store.create_datastore()
# Create should throw exception the second time
with self.assertRaises(BadRequest):
data_store.create_datastore()
# Call ops with wrong object type and make sure exception is thrown
with self.assertRaises(BadRequest):
data_store.create({"foo": "bar"})
with self.assertRaises(BadRequest):
data_store.create_mult([{"foo": "bar"}])
with self.assertRaises(BadRequest):
data_store.read({"foo": "bar"})
with self.assertRaises(BadRequest):
data_store.read_mult([{"foo": "bar"}])
with self.assertRaises(BadRequest):
data_store.update({"foo": "bar"})
with self.assertRaises(BadRequest):
data_store.delete({"foo": "bar"})
# Should see new data
self.assertIn("ion_test_ds", data_store.list_datastores())
# Something should be returned
self.assertTrue(data_store.info_datastore() is not None)
res = data_store.list_objects()
numcoredocs = len(res)
# Construct user role objects
admin_role = {"name": "Admin", "description": "Super user"}
admin_role_obj = IonObject("UserRole", admin_role)
admin_role_tuple = data_store.create(admin_role_obj)
self.assertTrue(len(admin_role_tuple) == 2)
admin_role_ooi_id = admin_role_tuple[0]
data_provider_role = {"name": "Data Provider", "description": "User allowed to ingest data sets"}
data_provider_role_obj = IonObject("UserRole", data_provider_role)
data_provider_role_tuple = data_store.create(data_provider_role_obj)
self.assertTrue(len(data_provider_role_tuple) == 2)
data_provider_role_ooi_id = data_provider_role_tuple[0]
marine_operator_role = {"name": "Marine Operator", "description": "User allowed to administer instruments"}
marine_operator_role_obj = IonObject("UserRole", marine_operator_role)
marine_operator_role_tuple = data_store.create(marine_operator_role_obj)
self.assertTrue(len(marine_operator_role_tuple) == 2)
marine_operator_role_ooi_id = marine_operator_role_tuple[0]
role_objs = data_store.read_mult([admin_role_ooi_id, data_provider_role_ooi_id, marine_operator_role_ooi_id])
self.assertTrue(len(role_objs) == 3)
self.assertTrue(role_objs[0]._id == admin_role_ooi_id)
self.assertTrue(role_objs[1]._id == data_provider_role_ooi_id)
self.assertTrue(role_objs[2]._id == marine_operator_role_ooi_id)
# Construct three user info objects and assign them roles
hvl_contact_info = {
"name": "Heitor Villa-Lobos",
"email": "*****@*****.**",
"variables": [{"name": "Claim To Fame", "value": "Legendary Brazilian composer"}],
}
hvl_contact_info_obj = IonObject("ContactInformation", hvl_contact_info)
hvl_user_info = {"name": "Heitor Villa-Lobos", "contact": hvl_contact_info_obj}
hvl_user_info_obj = IonObject("UserInfo", hvl_user_info)
hvl_user_info_tuple = data_store.create(hvl_user_info_obj)
self.assertTrue(len(hvl_user_info_tuple) == 2)
heitor_villa_lobos_ooi_id = hvl_user_info_tuple[0]
ats_contact_info = {
"name": "Andres Torres Segovia",
"email": "*****@*****.**",
"variables": [{"name": "Claim To Fame", "value": "Legendary Concert Guitarist"}],
}
ats_contact_info_obj = IonObject("ContactInformation", ats_contact_info)
ats_user_info = {"name": "Andres Torres Segovia", "contact": ats_contact_info_obj}
ats_user_info_obj = IonObject("UserInfo", ats_user_info)
ats_user_info_tuple = data_store.create(ats_user_info_obj)
self.assertTrue(len(ats_user_info_tuple) == 2)
pok_contact_info = {
"name": "Per-Olov Kindgren",
"email": "*****@*****.**",
"variables": [{"name": "Claim To Fame", "value": "Composer and YouTube star"}],
}
pok_contact_info_obj = IonObject("ContactInformation", pok_contact_info)
pok_user_info = {"name": "Per-Olov Kindgren", "contact": pok_contact_info_obj}
pok_user_info_obj = IonObject("UserInfo", pok_user_info)
pok_user_info_tuple = data_store.create(pok_user_info_obj)
self.assertTrue(len(pok_user_info_tuple) == 2)
# List all objects in data store and confirm there are six docs
res = data_store.list_objects()
# There are indices. Therefore can't could all docs
self.assertTrue(len(res) == 6 + numcoredocs)
# Create an Ion object with default values set (if any)
data_set = IonObject("DataSet")
self.assertTrue(isinstance(data_set, interface.objects.DataSet))
# Assign values to object fields
data_set.description = "Real-time water data for Choptank River near Greensboro, MD"
# data_set.min_datetime = "2011-08-04T13:15:00Z"
# data_set.max_datetime = "2011-08-09T19:15:00Z"
# data_set.variables = [
# {"name":"water_height", "value":"ft"}
# ]
data_set.contact.name = "Heitor Villa-Lobos"
# Write DataSet object"
write_tuple_1 = data_store.create(data_set)
self.assertTrue(len(write_tuple_1) == 2)
# Save off the object UUID
data_set_uuid = write_tuple_1[0]
# Read back the HEAD version of the object and validate fields
data_set_read_obj = data_store.read(data_set_uuid)
self.assertTrue(data_set_read_obj._id == data_set_uuid)
self.assertTrue(isinstance(data_set_read_obj, interface.objects.DataSet))
self.assertTrue(data_set_read_obj.description == "Real-time water data for Choptank River near Greensboro, MD")
self.assertTrue("type_" in data_set_read_obj)
# Update DataSet's Description field and write
data_set_read_obj.description = "Updated Description"
write_tuple_2 = data_store.update(data_set_read_obj)
self.assertTrue(len(write_tuple_2) == 2)
# Retrieve the updated DataSet
data_set_read_obj_2 = data_store.read(data_set_uuid)
self.assertTrue(data_set_read_obj_2._id == data_set_uuid)
self.assertTrue(data_set_read_obj_2.description == "Updated Description")
# List all the revisions of DataSet in data store, should be two
res = data_store.list_object_revisions(data_set_uuid)
self.assertTrue(len(res) == 2)
# Do another update to the object
data_set_read_obj_2.description = "USGS instantaneous value data for station 01491000"
write_tuple_3 = data_store.update(data_set_read_obj_2)
# List revisions of DataSet in data store, should now be three
res = data_store.list_object_revisions(data_set_uuid)
self.assertTrue(len(res) == 3)
# Retrieve original version of DataSet
obj1 = data_store.read(data_set_uuid, rev_id=write_tuple_1[1])
self.assertTrue(obj1._id == data_set_uuid)
self.assertTrue(obj1.description == "Real-time water data for Choptank River near Greensboro, MD")
# Retrieve second version of DataSet
obj2 = data_store.read(data_set_uuid, rev_id=write_tuple_2[1])
self.assertTrue(obj2._id == data_set_uuid)
self.assertTrue(obj2.description == "Updated Description")
# Retrieve third version of DataSet
obj3 = data_store.read(data_set_uuid, rev_id=write_tuple_3[1])
self.assertTrue(obj3._id == data_set_uuid)
self.assertTrue(obj3.description == "USGS instantaneous value data for station 01491000")
# Retrieve HEAD version of DataSet
head = data_store.read(data_set_uuid)
self.assertTrue(head._id == data_set_uuid)
self.assertTrue(head.description == "USGS instantaneous value data for station 01491000")
# Delete DataSet by object id
data_store.delete(head)
# Try to re-delete DataSet by object id. Should throw exception.
with self.assertRaises(NotFound):
data_store.delete(head._id)
# List all objects in data store, should be back to six
res = data_store.list_objects()
self.assertTrue(len(res) == 6 + numcoredocs)
# List revisions of now deleted DataSet, should be empty list
res = data_store.list_object_revisions(data_set_uuid)
self.assertTrue(len(res) == 0)
o1 = IonObject("DataSet", name="One more")
o2 = IonObject("DataSet", name="Another one")
res = data_store.create_mult((o1, o2))
self.assertTrue(all([success for success, oid, rev in res]))
res = data_store.list_objects()
self.assertTrue(len(res) == 8 + numcoredocs)
# Delete data store to clean up
data_store.delete_datastore()
# Assert data store is now gone
self.assertNotIn("ion_test_ds", data_store.list_datastores())
def _do_test_attach(self, data_store):
self.data_store = data_store
self.resources = {}
# Create an Ion object with default values set (if any)
data_set = IonObject('Dataset')
self.assertTrue(isinstance(data_set, interface.objects.Dataset))
# Assign values to object fields
data_set.description = "Real-time water data for Choptank River near Greensboro, MD"
# Write Dataset object"
write_tuple_1 = data_store.create(data_set)
# Save off the object UUID
data_set_uuid = write_tuple_1[0]
# Read back the HEAD version of the object
data_set_read_obj = data_store.read(data_set_uuid)
# Update Dataset's Description field and write
data_set_read_obj.description = "Updated Description"
write_tuple_2 = data_store.update(data_set_read_obj)
# test attachment related stuff
# create attachment
ds_id_and_rev = {}
attachment_name = 'resource.attachment'
ds_id_and_rev['_id'] = write_tuple_2[0]
ds_id_and_rev['_rev'] = write_tuple_2[1]
data = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x03\x00\x00\x00(-\x0fS\x00\x00\x00\x03sBIT\x08\x08\x08\xdb\xe1O\xe0\x00\x00\x00~PLTEf3\x00\xfc\xf7\xe0\xee\xcc\x00\xd3\xa0\x00\xcc\x99\x00\xec\xcdc\x9fl\x00\xdd\xb2\x00\xff\xff\xff|I\x00\xf9\xdb\x00\xdd\xb5\x19\xd9\xad\x10\xb6\x83\x00\xf8\xd6\x00\xf2\xc5\x00\xd8\xab\x00n;\x00\xff\xcc\x00\xd6\xa4\t\xeb\xb8\x00\x83Q\x00\xadz\x00\xff\xde\x00\xff\xd6\x00\xd6\xa3\x00\xdf\xaf\x00\xde\xad\x10\xbc\x8e\x00\xec\xbe\x00\xec\xd4d\xff\xe3\x00tA\x00\xf6\xc4\x00\xf6\xce\x00\xa5u\x00\xde\xa5\x00\xf7\xbd\x00\xd6\xad\x08\xdd\xaf\x19\x8cR\x00\xea\xb7\x00\xee\xe9\xdf\xc5\x00\x00\x00\tpHYs\x00\x00\n\xf0\x00\x00\n\xf0\x01B\xac4\x98\x00\x00\x00\x1ctEXtSoftware\x00Adobe Fireworks CS4\x06\xb2\xd3\xa0\x00\x00\x00\x15tEXtCreation Time\x0029/4/09Oq\xfdE\x00\x00\x00\xadIDAT\x18\x95M\x8f\x8d\x0e\x820\x0c\x84;ZdC~f\x07\xb2\x11D\x86\x89\xe8\xfb\xbf\xa0+h\xe2\x97\\\xd2^\x93\xb6\x07:1\x9f)q\x9e\xa5\x06\xad\xd5\x13\x8b\xac,\xb3\x02\x9d\x12C\xa1-\xef;M\x08*\x19\xce\x0e?\x1a\xeb4\xcc\xd4\x0c\x831\x87V\xca\xa1\x1a\xd3\x08@\xe4\xbd\xb7\x15P;\xc8\xd4{\x91\xbf\x11\x90\xffg\xdd\x8di\xfa\xb6\x0bs2Z\xff\xe8yg2\xdc\x11T\x96\xc7\x05\xa5\xef\x96+\xa7\xa59E\xae\xe1\x84cm^1\xa6\xb3\xda\x85\xc8\xd8/\x17se\x0eN^'\x8c\xc7\x8e\x88\xa8\xf6p\x8e\xc2;\xc6.\xd0\x11.\x91o\x12\x7f\xcb\xa5\xfe\x00\x89]\x10:\xf5\x00\x0e\xbf\x00\x00\x00\x00IEND\xaeB`\x82"
some_text = "SOME TEXT"
# create attachment with no data
with self.assertRaises(BadRequest):
data_store.create_attachment(doc=ds_id_and_rev, data=None,
attachment_name=attachment_name,
content_type=None, datastore_name="")
# create attachment with no attachment
with self.assertRaises(BadRequest):
data_store.create_attachment(doc=ds_id_and_rev, data=data,
attachment_name=None, content_type=None, datastore_name="")
#create attachment by passing a doc parameter that
# is a dictionary containing _rev and _id elements
data_store.create_attachment(doc=ds_id_and_rev, data=data,
attachment_name=attachment_name,
content_type=None, datastore_name="")
# read attachment by passing a doc parameter that is a dictionary
# containing _rev and _id elements and verify that the content read
# is same as the content put in
content_read = data_store.read_attachment(doc=ds_id_and_rev,
attachment_name=attachment_name,
datastore_name="")
self.assertEquals(data, content_read)
# update attachment by passing a doc parameter that is a dictionary
# containing _rev and _id elements
data_store.update_attachment(ds_id_and_rev, attachment_name, data=some_text)
# read the attachment passing only the doc _id and verify that content has changed
content_read = data_store.read_attachment(doc=ds_id_and_rev['_id'],
attachment_name=attachment_name,
datastore_name="")
self.assertNotEquals(data, content_read)
self.assertEquals(some_text, content_read)
# delete attachment by passing a doc parameter that is a dictionary containing _rev
# and _id elements
data_store.delete_attachment(doc=ds_id_and_rev, attachment_name=attachment_name)
# interestingly, deleting an attachment that does not exist works
# pass a doc parameter that is a dictionary containing _rev and _id elements
data_store.delete_attachment(doc=ds_id_and_rev['_id'], attachment_name='no_such_file')
#create attachment by passing a doc parameter that is string indicating _id
data_store.create_attachment(doc=ds_id_and_rev['_id'], data=data,
attachment_name=attachment_name, content_type=None,
datastore_name="")
# read attachment by passing a doc parameter that is a string indicating _id
# and verify that the content read is same as the content put in
content_read = data_store.read_attachment(doc=ds_id_and_rev['_id'],
attachment_name=attachment_name,
datastore_name="")
self.assertEquals(data, content_read)
# update attachment by passing a doc parameter that is a string indicating _id
data_store.update_attachment(ds_id_and_rev['_id'], attachment_name, data=some_text)
# create another attachment and
# list attachments by passing a doc parameter that is a dictionary
# containing _rev and _id elements
data_store.create_attachment(doc=ds_id_and_rev['_id'], data=data,
attachment_name=attachment_name+"_01", content_type=None,
datastore_name="")
_attachments = data_store.list_attachments(doc=ds_id_and_rev)
#refer to a previous version of the document
updated_ds_id_and_rev = {}
updated_ds_id_and_rev['_id'] = write_tuple_1[0]
updated_ds_id_and_rev['_rev'] = write_tuple_1[1]
# deleting attachment from the previous (wrong) revision raises document update conflict
with self.assertRaises(Exception):
data_store.delete_attachment(doc=updated_ds_id_and_rev, attachment_name=attachment_name)
# operations on previous versions are not allowed
with self.assertRaises(Exception):
data_store.create_attachment(doc=updated_ds_id_and_rev, data=some_text,
attachment_name=attachment_name, content_type=None,
datastore_name="")
# send in an incorrect document _id
with self.assertRaises(NotFound):
data_store.create_attachment(doc="incorrect_id", data=data,
attachment_name=attachment_name, content_type=None,
datastore_name="")
# send in an incorrect document _id
with self.assertRaises(NotFound):
data_store.read_attachment(doc="incorrect_id", attachment_name=attachment_name,
datastore_name="")
# send in an incorrect attachment_name
with self.assertRaises(NotFound):
data_store.read_attachment(doc=ds_id_and_rev['_id'],
attachment_name="incorrect_attachment", datastore_name="")
# send in an incorrect document_id
with self.assertRaises(NotFound):
data_store.update_attachment(doc="incorrect_id", attachment_name=attachment_name,
data=some_text)
# send in an incorrect attachment_name; this should work because update creates an
# attachment when it can't find an attachment to update
data_store.update_attachment(ds_id_and_rev['_id'], attachment_name="incorrect_attachment",
data=some_text)
# send in an incorrect attachment_name; interestingly, this is not an error
data_store.delete_attachment(doc=ds_id_and_rev['_id'], attachment_name='no_such_file')
# send in an incorrect document_id
with self.assertRaises(NotFound):
data_store.delete_attachment(doc="incorrect_id", attachment_name='no_such_file')
