def _construct_stream_and_publisher(self, stream_name, stream_config):
if log.isEnabledFor(logging.TRACE): # pragma: no cover
log.trace("%r: _construct_stream_and_publisher: "
"stream_name:%r, stream_config:\n%s",
self._platform_id, stream_name,
self._pp.pformat(stream_config))
decoder = IonObjectDeserializer(obj_registry=get_obj_registry())
if 'stream_def_dict' not in stream_config:
# should not happen: PlatformAgent._validate_configuration validates this.
log.error("'stream_def_dict' key not in configuration for stream %r" % stream_name)
return
stream_def_dict = stream_config['stream_def_dict']
stream_def_dict['type_'] = 'StreamDefinition'
stream_def_obj = decoder.deserialize(stream_def_dict)
self._stream_defs[stream_name] = stream_def_obj
routing_key = stream_config['routing_key']
stream_id = stream_config['stream_id']
exchange_point = stream_config['exchange_point']
parameter_dictionary = stream_def_dict['parameter_dictionary']
log.debug("%r: got parameter_dictionary from stream_def_dict", self._platform_id)
self._data_streams[stream_name] = stream_id
self._param_dicts[stream_name] = ParameterDictionary.load(parameter_dictionary)
stream_route = StreamRoute(exchange_point=exchange_point, routing_key=routing_key)
publisher = self._create_publisher(stream_id, stream_route)
self._data_publishers[stream_name] = publisher
log.debug("%r: created publisher for stream_name=%r", self._platform_id, stream_name)
def test_persisted_version(self):
# create an initial version of SampleResource
io_serializer = IonObjectSerializer()
obj = IonObject('SampleResource', {'num': 9, 'other_field': 'test value'})
obj_dict = io_serializer.serialize(obj,True)
self.assertEquals(obj_dict['persisted_version'], 1)
# verify that the simulated previous version does not have new_attribute
self.assertEquals('new_attribute' in obj_dict, False)
# simulate version increment to SampleResource that adds new_attribute
obj_dict['type_'] = 'SampleResource_V2'
# simulate reading the previous version after version increment
io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
obj = io_deserializer.deserialize(obj_dict)
# verify that “new_attribute” is added and initialized with default value
self.assertEquals(obj.new_attribute['key'], 'value')
# verify that old attributes are still there and retain values
self.assertEquals(obj.num, 9)
# verify that old attributes are still there and retain values
self.assertEquals(obj.other_field, 'test value')
# verify that persisted_version is not updated at read
self.assertEquals(obj_dict['persisted_version'], 1)
# simulate update
obj_dict = io_serializer.serialize(obj,True)
# verify that version is updated
self.assertEquals(obj_dict['persisted_version'], 2)
def test_complex_version(self):
io_serializer = IonObjectSerializer()
obj = IonObject('SampleComplexEvent', {'num': 9, 'other_field': 'test value'})
obj_dict = io_serializer.serialize(obj,True)
self.assertEquals(obj_dict['persisted_version'], 1)
# simulate a previous version data of SampleComplexEvent_V2
obj_dict['type_'] = 'SampleComplexEvent_V2'
# verify that the simulated previous version data has resource
self.assertEquals('resource' in obj_dict, True)
# verify that the simulated previous version data does not have new_attribute
self.assertEquals('new_resource' in obj_dict, False)
# simulate reading the previous version that does not have new_attribute
io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
obj = io_deserializer.deserialize(obj_dict)
# verify that new attribute is added and initialized with default value
self.assertEquals(obj.new_resource.new_attribute['key'], 'value')
# verify that old attributes are still there
self.assertEquals(obj.num, 9)
# verify that old attributes are still there
self.assertEquals(obj.other_field, 'test value')
# verify that on read version is not yet updated
self.assertEquals(obj_dict['persisted_version'], 1)
# simulate create/update
obj_dict = io_serializer.serialize(obj,True)
# verify that version is updated
self.assertEquals(obj_dict['persisted_version'], 2)
def validate_driver_configuration(self, driver_config):
"""
Driver config must include 'oms_uri' entry.
"""
if not 'oms_uri' in driver_config:
log.error("'oms_uri' not present in driver_config = %s", driver_config)
raise PlatformDriverException(msg="driver_config does not indicate 'oms_uri'")
# validate and process ports
if not 'ports' in driver_config:
log.error("port information not present in driver_config = %s", driver_config)
raise PlatformDriverException(msg="driver_config does not indicate 'ports'")
# Create an IonObjectDeserializer
ior = IonObjectRegistry()
ion_deserializer = IonObjectDeserializer(obj_registry=ior)
port_info_dict = driver_config['ports']
for device_id, platform_port_serialized in port_info_dict.iteritems():
platform_port = ion_deserializer.deserialize(platform_port_serialized)
ooi_rd = OOIReferenceDesignator(platform_port.reference_designator)
if ooi_rd.error or not ooi_rd.port:
log.error("Invalid port information in driver_config. Reference designator: %s", platform_port.reference_designator)
else:
#strip leading zeros from port numbers as OMS stores as strings w/o leading zeros
port_string = str( int(ooi_rd.port) )
self._active_ports.append(port_string)
def test_config(self):
"""
test_initialize
Test agent initialize command. This causes creation of
driver process and transition to inactive.
"""
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
decoder = IonObjectDeserializer(obj_registry=get_obj_registry())
# Grab the alarms defined in the config.
retval = decoder.deserialize(
self._ia_client.get_agent(['alarms'])['alarms'])
"""
{'status': None, 'stream_name': 'parsed', 'name': 'test_sim_warning',
'upper_bound': 5.0, 'expr': 'x<5.0', 'upper_rel_op': '<',
'lower_rel_op': None, 'type_': 'IntervalAlarmDef', 'value_id': 'temp',
'lower_bound': None, 'message': 'Temperature is above test range of 5.0.',
'current_val': None, 'type': 1}
"""
self.assertEqual(retval[0].type_, 'IntervalAlarmDef')
self.assertEqual(retval[0].upper_bound, 5.0)
self.assertEqual(retval[0].expr, 'x<5.0')
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
def test_attribute_version(self):
io_serializer = IonObjectSerializer()
# verify that extraneous fields given while creating an IonObject raises an error.
with self.assertRaises(AttributeError):
IonObject('SampleComplexEvent_V2', {'num': 9, 'other_field': 'test value','more_new_resource':
{'key':'value'}})
obj = IonObject('SampleComplexEvent_V2', {'num': 9, 'other_field': 'test value','new_resource':
{'num': 9, 'other_field': 'test value','new_attribute':{'key':'value'}}})
obj_dict = io_serializer.serialize(obj,True)
self.assertEquals(obj_dict['persisted_version'], 2)
# verify that the simulated previous version data does have new_resource
self.assertEquals('new_resource' in obj_dict, True)
# verify that the new_resource has type SampleResource_V2
self.assertEquals(obj_dict['new_resource']['type_'],"SampleResource_V2")
# set type to SampleComplexEvent_V3
obj_dict['type_']="SampleComplexEvent_V3"
obj_dict['persisted_version']=3
# set new_resource's type to SampleResource_V3
# so we pretend that version, not the type, of the attribute has been changed
obj_dict['new_resource']['type_']="SampleResource_V3"
# simulate reading SampleComplexEvent_V3 after a new version of new_resource has been introduced
io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
obj = io_deserializer.deserialize(obj_dict)
# verify that new resource is not deleted
self.assertTrue('new_resource' in obj)
# verify that new resource does not have new_attribute
self.assertFalse('new_attribute' in obj.new_resource)
# verify that the next version of new_resource has default data in the another_new_attribute
self.assertEquals(obj.new_resource.another_new_attribute['key'], 'new_value')
# verify that old attributes values of new_resource have not been thrown away
self.assertEquals(obj.new_resource.num, 9)
# verify that values from old attributes of SampleComplexEvent_V2 are still there
self.assertEquals(obj.num, 9)
self.assertEquals(obj.other_field, 'test value')
# verify that on read version is not yet updated for the subsumed object
self.assertEquals(obj.new_resource.persisted_version, 2)
# simulate create/update
obj_dict = io_serializer.serialize(obj,True)
# verify that versions are unchanged
self.assertEquals(obj_dict['persisted_version'], 3)
# verify that versions are updated in the subsumed object
self.assertEquals(obj_dict['new_resource']['persisted_version'], 3)
def test_complex_version_del_attrib(self):
io_serializer = IonObjectSerializer()
# verify that extraneous fields given while creating an IonObject raises an error.
with self.assertRaises(AttributeError):
IonObject('SampleComplexEvent_V2', {'num': 9, 'other_field': 'test value','more_new_resource': {'key':'value'}})
obj = IonObject('SampleComplexEvent_V2', {'num': 9, 'other_field': 'test value','new_resource': {'num': 9, 'other_field': 'test value','new_attribute':{'key':'value'}}})
# create simulated saved data
obj_dict = io_serializer.serialize(obj,True)
self.assertEquals(obj_dict['persisted_version'], 2)
# simulate a next version data of SampleComplexEvent_V2
obj_dict['type_'] = 'SampleComplexEvent_V3'
# verify that the simulated previous version data does have new_resource
self.assertEquals('new_resource' in obj_dict, True)
# note the schema version of new_resource
self.assertEquals(obj_dict['new_resource']['persisted_version'], 2)
# simulate reading the next version that has a new type of new_resource
io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
obj = io_deserializer.deserialize(obj_dict)
# verify that new_resource exists
self.assertTrue('new_resource' in obj)
# however, verify that new_resource does not have new_attribute since type of new_resource has changed
self.assertFalse('new_attribute' in obj.new_resource)
# verify that the new type of new_resource has another_new_attribute that is initialized to default data
self.assertEquals(obj.new_resource.another_new_attribute['key'], 'new_value')
# verify on read that the schema version of new_resource replaces the old persisted_version
self.assertEquals(obj.new_resource.persisted_version, 3)
# verify that old attributes values of new_resource have been thrown away
self.assertNotEquals(obj.new_resource.num, 9)
# verify that attributes values of new_resource have been initialized to default values
self.assertEquals(obj.new_resource.num, 0)
# However, verify that old attributes of the resource (SampleComplexEvent) are still there
self.assertEquals(obj.num, 9)
# verify that old attributes are still there
self.assertEquals(obj.other_field, 'test value')
# verify that on read, version is not yet updated
self.assertEquals(obj.persisted_version, 2)
# simulate create/update
obj_dict = io_serializer.serialize(obj,True)
# verify that version is updated
self.assertEquals(obj_dict['persisted_version'], 3)
# verify that version is updated fo the subsumed object
self.assertEquals(obj_dict['new_resource']['persisted_version'], 3)
def __init__(self, container, datastore_name=""):
self.container = container
self.datastore_name = datastore_name
# Object serialization/deserialization
self._io_serializer = IonObjectSerializer()
self._io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
class CodecInterceptor(Interceptor):
"""
Transforms IonObject <-> dict
"""
def __init__(self):
Interceptor.__init__(self)
self._io_serializer = IonObjectSerializer()
self._io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
def outgoing(self, invocation):
#log.debug("CodecInterceptor.outgoing: %s", invocation)
#log.debug("Payload, pre-transform: %s", invocation.message)
invocation.message = self._io_serializer.serialize(invocation.message)
#log.debug("Payload, post-transform: %s", invocation.message)
return invocation
def incoming(self, invocation):
#log.debug("CodecInterceptor.incoming: %s", invocation)
payload = invocation.message
#log.debug("Payload, pre-transform: %s", payload)
invocation.message = self._io_deserializer.deserialize(payload)
#log.debug("Payload, post-transform: %s", invocation.message)
return invocation
def __init__(self, datastore_name='prototype'):
self.datastore_name = datastore_name
log.debug('Creating in-memory dict of dicts that will simulate data stores')
self.root = {}
# serializers
self._io_serializer = IonObjectSerializer()
self._io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
class IonSerializerDictionaryRepresentation(Representation):
def __init__(self, id_factory):
self.encoder = IonObjectSerializer()
self.decoder = IonObjectDeserializer(obj_registry=get_obj_registry())
self.id_factory = id_factory
def encode(self, obj, add_id=False):
out = self.encoder.serialize(obj)
if add_id and '_id' not in out.keys():
out['_id'] = self.id_factory.create_id()
return out
def decode(self, data):
return self.decoder.deserialize(data)
def test_version_del_attrib(self):
io_serializer = IonObjectSerializer()
# verify that extraneous fields given while creating an IonObject raises an error.
with self.assertRaises(AttributeError):
IonObject('SampleResource_V2', {'num': 9, 'other_field': 'test value','more_new_attribute': {'key':'value'}})
# simulate creating a version 2 of SampleResource that has "new_attribute"
obj = IonObject('SampleResource_V2', {'num': 9, 'other_field': 'test value','new_attribute': {'key':'value'}})
obj_dict = io_serializer.serialize(obj,True)
# verify that version is 2
self.assertEquals(obj_dict['persisted_version'], 2)
# verify that the simulated version 2 data does have new_attribute
self.assertEquals('new_attribute' in obj_dict, True)
# simulate incrementing to version 3 that does not have "new_attribute"
obj_dict['type_'] = 'SampleResource_V3'
# simulate reading after version increment to 3
io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
obj = io_deserializer.deserialize(obj_dict)
# verify that new attribute is deleted
self.assertFalse('new_attribute' in obj)
# verify that the simulated next version data does have more_new_attribute
self.assertEquals(obj.another_new_attribute['key'], 'new_value')
# verify that old attributes are still there and retain their data
self.assertEquals(obj.num, 9)
# verify that old attributes are still there and retain their data
self.assertEquals(obj.other_field, 'test value')
# verify that persisted_version is not yet updated i.e. it is still 2
self.assertEquals(obj_dict['persisted_version'], 2)
# simulate update
obj_dict = io_serializer.serialize(obj,True)
# verify that version is updated
self.assertEquals(obj_dict['persisted_version'], 3)
def _construct_stream_and_publisher(self, stream_name, stream_config):
if log.isEnabledFor(logging.TRACE): # pragma: no cover
log.trace(
"%r: _construct_stream_and_publisher: "
"stream_name:%r, stream_config:\n%s", self._platform_id,
stream_name, self._pp.pformat(stream_config))
decoder = IonObjectDeserializer(obj_registry=get_obj_registry())
if 'stream_def_dict' not in stream_config:
# should not happen: PlatformAgent._validate_configuration validates this.
log.error(
"'stream_def_dict' key not in configuration for stream %r" %
stream_name)
return
stream_def_dict = stream_config['stream_def_dict']
stream_def_dict['type_'] = 'StreamDefinition'
stream_def_obj = decoder.deserialize(stream_def_dict)
self._stream_defs[stream_name] = stream_def_obj
routing_key = stream_config['routing_key']
stream_id = stream_config['stream_id']
exchange_point = stream_config['exchange_point']
parameter_dictionary = stream_def_dict['parameter_dictionary']
log.debug("%r: got parameter_dictionary from stream_def_dict",
self._platform_id)
self._data_streams[stream_name] = stream_id
self._param_dicts[stream_name] = ParameterDictionary.load(
parameter_dictionary)
stream_route = StreamRoute(exchange_point=exchange_point,
routing_key=routing_key)
publisher = self._create_publisher(stream_id, stream_route)
self._data_publishers[stream_name] = publisher
log.debug("%r: created publisher for stream_name=%r",
self._platform_id, stream_name)
def test_stream_ingestion_worker(self):
self.start_ingestion_worker()
context_ids, time_ctxt = self._create_param_contexts()
pdict_id = self.dataset_management_client.create_parameter_dictionary(name='stream_ingestion_pdict', parameter_context_ids=context_ids, temporal_context='ingestion_timestamp')
self.addCleanup(self.dataset_management_client.delete_parameter_dictionary, pdict_id)
dataset_id = self.dataset_management_client.create_dataset(name='fake_dataset', description='fake_dataset', stream_id=self.stream_id, parameter_dictionary_id=pdict_id)
self.addCleanup(self.dataset_management_client.delete_dataset, dataset_id)
self.cov = self._create_coverage(dataset_id=dataset_id, parameter_dict_id=pdict_id, time_dom=self.time_dom, spatial_dom=self.spatial_dom)
self.addCleanup(self.cov.close)
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['conductivity'] = 1
rdt['pressure'] = 2
rdt['salinity'] = 3
self.start_listener(dataset_id)
self.publisher.publish(rdt.to_granule())
self.data_modified = Event()
self.data_modified.wait(30)
cov = self.get_coverage(dataset_id)
self.assertIsNotNone(cov.get_parameter_values('raw'))
deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
granule = retrieve_stream(dataset_id)
rdt_complex = RecordDictionaryTool.load_from_granule(granule)
rdt_complex['raw'] = [deserializer.deserialize(i) for i in rdt_complex['raw']]
for gran in rdt_complex['raw']:
rdt_new = RecordDictionaryTool.load_from_granule(gran)
self.assertIn(1, rdt_new['conductivity'])
self.assertIn(2, rdt_new['pressure'])
self.assertIn(3, rdt_new['salinity'])
cov.close()
def _construct_streams(self, stream_info):
decoder = IonObjectDeserializer(obj_registry=get_obj_registry())
for (stream_name, config) in stream_info.iteritems():
try:
if config.has_key('stream_def_dict'):
stream_def_dict = config['stream_def_dict']
stream_def_dict['type_'] = 'StreamDefinition'
stream_def_obj = decoder.deserialize(stream_def_dict)
self._stream_defs[stream_name] = stream_def_obj
rdt = RecordDictionaryTool(stream_definition=stream_def_obj)
else:
stream_def = config['stream_definition_ref']
self._stream_defs[stream_name] = stream_def
rdt = RecordDictionaryTool(stream_definition_id=stream_def)
self._agent.aparam_streams[stream_name] = rdt.fields
self._agent.aparam_pubrate[stream_name] = 0
except Exception as e:
errmsg = 'Instrument agent %s' % self._agent._proc_name
errmsg += 'error constructing stream %s. ' % stream_name
errmsg += str(e)
log.error(errmsg)
self._agent.aparam_set_pubrate = self.aparam_set_pubrate
def _construct_streams(self, stream_info):
decoder = IonObjectDeserializer(obj_registry=get_obj_registry())
for (stream_name, config) in stream_info.iteritems():
try:
if config.has_key('stream_def_dict'):
stream_def_dict = config['stream_def_dict']
stream_def_dict['type_'] = 'StreamDefinition'
stream_def_obj = decoder.deserialize(stream_def_dict)
self._stream_defs[stream_name] = stream_def_obj
rdt = RecordDictionaryTool(stream_definition=stream_def_obj)
else:
stream_def = config['stream_definition_ref']
self._stream_defs[stream_name] = stream_def
rdt = RecordDictionaryTool(stream_definition_id=stream_def)
self._agent.aparam_streams[stream_name] = rdt.fields
self._agent.aparam_pubrate[stream_name] = 0
except Exception as e:
errmsg = 'Instrument agent %s' % self._agent._proc_name
errmsg += 'error constructing stream %s. ' % stream_name
errmsg += str(e)
log.error(errmsg)
self._agent.aparam_set_pubrate = self.aparam_set_pubrate
class IonSerializerDictionaryRepresentation(Representation):
def __init__(self, id_factory):
self.encoder = IonObjectSerializer()
self.decoder = IonObjectDeserializer(obj_registry=get_obj_registry())
self.id_factory = id_factory
def encode(self, obj, add_id=False):
out = self.encoder.serialize(obj)
if add_id and '_id' not in out.keys():
out['_id'] = self.id_factory.create_id()
return out
def decode(self, data):
return self.decoder.deserialize(data)
def __init__(self,
datastore_name=None,
config=None,
scope=None,
profile=None):
"""
@param datastore_name Name of datastore within server. May be scoped to sysname
@param config A server config dict with connection params
@param scope Prefix for the datastore name (e.g. sysname) to separate multiple systems
"""
PostgresDataStore.__init__(self,
datastore_name=datastore_name,
config=config
or CFG.get_safe("server.postgresql"),
profile=profile
or DataStore.DS_PROFILE.BASIC,
scope=scope)
# IonObject Serializers
self._io_serializer = IonObjectSerializer()
self._io_deserializer = IonObjectDeserializer(
obj_registry=get_obj_registry())
def _construct_stream_and_publisher(self, stream_name, stream_config):
# granule_publish_rate
# records_per_granule
if log.isEnabledFor(logging.TRACE): # pragma: no cover
log.trace("%r: _construct_stream_and_publisher: "
"stream_name:%r, stream_config:\n%s",
self._platform_id, stream_name,
self._pp.pformat(stream_config))
decoder = IonObjectDeserializer(obj_registry=get_obj_registry())
if 'stream_def_dict' in stream_config:
stream_def_dict = stream_config['stream_def_dict']
stream_def_dict['type_'] = 'StreamDefinition'
stream_def_obj = decoder.deserialize(stream_def_dict)
self._stream_defs[stream_name] = stream_def_obj
log.debug("%r: using stream_def_dict", self._platform_id)
else: # TODO this case to be removed.
stream_definition_ref = stream_config['stream_definition_ref']
self._stream_defs[stream_name] = stream_definition_ref
log.debug("%r: using stream_definition_ref", self._platform_id)
routing_key = stream_config['routing_key']
stream_id = stream_config['stream_id']
exchange_point = stream_config['exchange_point']
parameter_dictionary = stream_config['parameter_dictionary']
self._data_streams[stream_name] = stream_id
self._param_dicts[stream_name] = ParameterDictionary.load(parameter_dictionary)
stream_route = StreamRoute(exchange_point=exchange_point, routing_key=routing_key)
publisher = self._create_publisher(stream_id, stream_route)
self._data_publishers[stream_name] = publisher
log.debug("%r: created publisher for stream_name=%r", self._platform_id, stream_name)
def __init__(self, datastore_name=None, config=None, scope=None, profile=None):
"""
@param datastore_name Name of datastore within server. May be scoped to sysname
@param config A server config dict with connection params
@param scope Prefix for the datastore name (e.g. sysname) to separate multiple systems
"""
PostgresDataStore.__init__(self, datastore_name=datastore_name,
config=config or CFG.get_safe("server.postgresql"),
profile=profile or DataStore.DS_PROFILE.BASIC,
scope=scope)
# IonObject Serializers
self._io_serializer = IonObjectSerializer()
self._io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
def test_event_version_del_attrib(self):
io_serializer = IonObjectSerializer()
# verify that extraneous fields given while creating an IonObject raises an error.
with self.assertRaises(AttributeError):
IonObject('SampleEvent_V2', {'num': 9, 'other_field': 'test value','more_new_attribute': {'key':'value'}})
obj = IonObject('SampleEvent_V2', {'num': 9, 'other_field': 'test value','new_attribute': {'key':'value'}})
obj_dict = io_serializer.serialize(obj,True)
self.assertEquals(obj_dict['persisted_version'], 2)
# simulate a next version data of SampleEvent_V2
obj_dict['type_'] = 'SampleEvent_V3'
# verify that the simulated previous version data does have new_attribute
self.assertEquals('new_attribute' in obj_dict, True)
# simulate reading the next version that does not have new_attribute
io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
obj = io_deserializer.deserialize(obj_dict)
# verify that new attribute is deleted
self.assertFalse('new_attribute' in obj)
# verify that the simulated next version data does have more_new_attribute
self.assertEquals(obj.another_new_attribute['key'], 'new_value')
# verify that old attributes are still there
self.assertEquals(obj.num, 9)
# verify that old attributes are still there
self.assertEquals(obj.other_field, 'test value')
# verify that on read version is not yet updated
self.assertEquals(obj_dict['persisted_version'], 2)
# simulate create/update
obj_dict = io_serializer.serialize(obj,True)
# verify that version is updated
self.assertEquals(obj_dict['persisted_version'], 3)
def _results_from_response(self, response, id_only):
deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
if not (response.has_key('hits') and response['hits'].has_key('hits')):
return []
hits = response['hits']['hits']
if len(hits) > 0:
if len(hits) >= SEARCH_BUFFER_SIZE:
log.warning("Query results exceeded search buffer limitations")
self.raise_search_buffer_exceeded()
if id_only:
return [str(i['_id']) for i in hits]
results = map(deserializer.deserialize, hits)
return results
else:
return []
def __init__(self,
host=None,
port=None,
datastore_name='prototype',
options="",
profile=DataStore.DS_PROFILE.BASIC):
log.debug('__init__(host=%s, port=%s, datastore_name=%s, options=%s)',
host, port, datastore_name, options)
self.host = host or CFG.server.couchdb.host
self.port = port or CFG.server.couchdb.port
# The scoped name of the datastore
self.datastore_name = datastore_name
self.auth_str = ""
try:
if CFG.server.couchdb.username and CFG.server.couchdb.password:
self.auth_str = "%s:%s@" % (CFG.server.couchdb.username,
CFG.server.couchdb.password)
log.debug(
"Using username:password authentication to connect to datastore"
)
except AttributeError:
log.error(
"CouchDB username:password not configured correctly. Trying anonymous..."
)
connection_str = "http://%s%s:%s" % (self.auth_str, self.host,
self.port)
#connection_str = "http://%s:%s" % (self.host, self.port)
# TODO: Security risk to emit password into log. Remove later.
log.info('Connecting to CouchDB server: %s' % connection_str)
self.server = couchdb.Server(connection_str)
# Datastore specialization (views)
self.profile = profile
# serializers
self._io_serializer = IonObjectSerializer()
# TODO: Not nice to have this class depend on ION objects
self._io_deserializer = IonObjectDeserializer(
obj_registry=get_obj_registry())
self._datastore_cache = {}
class CodecInterceptor(Interceptor):
"""
Transforms IonObject <-> dict
"""
def __init__(self):
Interceptor.__init__(self)
self._io_serializer = IonObjectSerializer()
self._io_deserializer = IonObjectDeserializer(obj_registry=obj_registry)
def outgoing(self, invocation):
log.debug("CodecInterceptor.outgoing: %s", invocation)
log.debug("Payload, pre-transform: %s", invocation.message)
invocation.message = self._io_serializer.serialize(invocation.message)
log.debug("Payload, post-transform: %s", invocation.message)
return invocation
def incoming(self, invocation):
log.debug("CodecInterceptor.incoming: %s", invocation)
payload = invocation.message
log.debug("Payload, pre-transform: %s", payload)
# Horrible, hacky workaround for msgpack issue
# See http://jira.msgpack.org/browse/MSGPACK-15
#@todo replace this with use_list in msgpack.unpackb !!!
def convert_tuples_to_lists(obj):
if isinstance(obj, tuple):
res = list(obj)
return res
return obj
payload = walk(payload, convert_tuples_to_lists)
invocation.message = self._io_deserializer.deserialize(payload)
log.debug("Payload, post-transform: %s", invocation.message)
return invocation
class MockDB_DataStore(DataStore):
"""
Data store implementation utilizing in-memory dict of dicts
to persist documents.
"""
def __init__(self, datastore_name='prototype'):
self.datastore_name = datastore_name
log.debug(
'Creating in-memory dict of dicts that will simulate data stores')
self.root = {}
# serializers
self._io_serializer = IonObjectSerializer()
self._io_deserializer = IonObjectDeserializer(
obj_registry=obj_registry)
def create_datastore(self, datastore_name="", create_indexes=True):
if not datastore_name:
datastore_name = self.datastore_name
log.info('Creating data store %s' % datastore_name)
if self.datastore_exists(datastore_name):
raise BadRequest("Data store with name %s already exists" %
datastore_name)
if datastore_name not in self.root:
self.root[datastore_name] = {}
def delete_datastore(self, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
log.info('Deleting data store %s' % datastore_name)
if datastore_name in self.root:
del self.root[datastore_name]
else:
log.info('Data store %s does not exist' % datastore_name)
def list_datastores(self):
log.debug('Listing all data stores')
dsList = self.root.keys()
log.debug('Data stores: %s' % str(dsList))
return dsList
def info_datastore(self, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
log.debug('Listing information about data store %s' % datastore_name)
if datastore_name in self.root:
info = 'Data store exists'
else:
raise BadRequest("Data store with name %s does not exist" %
datastore_name)
log.debug('Data store info: %s' % str(info))
return info
def datastore_exists(self, datastore_name=""):
return datastore_name in self.root
def list_objects(self, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
log.debug('Listing all objects in data store %s' % datastore_name)
objs = []
for key, value in self.root[datastore_name].items():
if key.find('_version_counter') == -1 and key.find(
'_version_') == -1:
objs.append(key)
log.debug('Objects: %s' % str(objs))
return objs
def list_object_revisions(self, object_id, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
log.debug('Listing all versions of object %s/%s' %
(datastore_name, str(object_id)))
res = []
for key, value in self.root[datastore_name].items():
if (key.find('_version_counter') == -1
and (key.find(object_id + '_version_') == 0)):
res.append(key)
log.debug('Versions: %s' % str(res))
return res
def create(self, obj, object_id=None, datastore_name=""):
if not isinstance(obj, IonObjectBase):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.create_doc(self._ion_object_to_persistence_dict(obj),
object_id=object_id,
datastore_name=datastore_name)
def create_doc(self, doc, object_id=None, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
if '_id' in doc:
raise BadRequest("Doc must not have '_id'")
if '_rev' in doc:
raise BadRequest("Doc must not have '_rev'")
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name +
' does not exist.')
if object_id:
if object_id in datastore_dict:
raise BadRequest("Object with id %s already exist" % object_id)
# Assign an id to doc
doc["_id"] = object_id or uuid4().hex
object_id = doc["_id"]
log.debug('Creating new object %s/%s' % (datastore_name, object_id))
# Create key for version counter entry. Will be used
# on update to increment version number easily.
version_counter_key = '__' + object_id + '_version_counter'
version_counter = 1
# Assign initial version to doc
doc["_rev"] = str(version_counter)
# Write HEAD, version and version counter dicts
datastore_dict[object_id] = doc
datastore_dict[version_counter_key] = version_counter
datastore_dict[object_id + '_version_' + str(version_counter)] = doc
# Return list that identifies the id of the new doc and its version
res = [object_id, str(version_counter)]
log.debug('Create result: %s' % str(res))
return res
def create_mult(self, objects, object_ids=None):
if any([not isinstance(obj, IonObjectBase) for obj in objects]):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.create_doc_mult(
[self._ion_object_to_persistence_dict(obj) for obj in objects],
object_ids)
def create_doc_mult(self, docs, object_ids=None):
if any(["_id" in doc for doc in docs]):
raise BadRequest("Docs must not have '_id'")
if any(["_rev" in doc for doc in docs]):
raise BadRequest("Docs must not have '_rev'")
if object_ids and len(object_ids) != len(docs):
raise BadRequest("Invalid object_ids")
# Assign an id to doc (recommended in CouchDB documentation)
object_ids = object_ids or [uuid4().hex for i in xrange(len(docs))]
res = []
for doc, oid in zip(docs, object_ids):
oid, rev = self.create_doc(doc, oid)
res.append((True, oid, rev))
return res
def read(self, object_id, rev_id="", datastore_name=""):
if not isinstance(object_id, str):
raise BadRequest("Object id param is not string")
doc = self.read_doc(object_id, rev_id, datastore_name)
# Convert doc into Ion object
obj = self._persistence_dict_to_ion_object(doc)
log.debug('Ion object: %s' % str(obj))
return obj
def read_doc(self, object_id, rev_id="", datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name +
' does not exist.')
try:
key = object_id
if rev_id != None and rev_id != "":
log.debug('Reading version %s of object %s/%s' %
(str(rev_id), datastore_name, str(object_id)))
key += '_version_' + str(rev_id)
else:
log.debug('Reading head version of object %s/%s' %
(datastore_name, str(object_id)))
doc = datastore_dict[key]
except KeyError:
raise NotFound('Object with id %s does not exist.' %
str(object_id))
log.debug('Read result: %s' % str(doc))
return doc
def read_mult(self, object_ids, datastore_name=""):
if any([not isinstance(object_id, str) for object_id in object_ids]):
raise BadRequest("Object id param is not string")
docs = self.read_doc_mult(object_ids, datastore_name)
# Convert docs into Ion objects
obj_list = [self._persistence_dict_to_ion_object(doc) for doc in docs]
return obj_list
def read_doc_mult(self, object_ids, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name +
' does not exist.')
doc_list = []
try:
for object_id in object_ids:
log.debug('Reading head version of object %s/%s' %
(datastore_name, str(object_id)))
doc = datastore_dict[object_id]
doc_list.append(doc.copy())
except KeyError:
raise NotFound('Object with id %s does not exist.' %
str(object_id))
return doc_list
def update(self, obj, datastore_name=""):
if not isinstance(obj, IonObjectBase):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.update_doc(self._ion_object_to_persistence_dict(obj))
def update_doc(self, doc, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
if '_id' not in doc:
raise BadRequest("Doc must have '_id'")
if '_rev' not in doc:
raise BadRequest("Doc must have '_rev'")
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name +
' does not exist.')
try:
object_id = doc["_id"]
# Find the next doc version
version_counter_key = '__' + object_id + '_version_counter'
baseVersion = doc["_rev"]
version_counter = datastore_dict[version_counter_key] + 1
if baseVersion != str(version_counter - 1):
raise Conflict('Object not based on most current version')
except KeyError:
raise BadRequest("Object missing required _id and/or _rev values")
log.debug('Saving new version of object %s/%s' %
(datastore_name, doc["_id"]))
doc["_rev"] = str(version_counter)
# Overwrite HEAD and version counter dicts, add new version dict
datastore_dict[object_id] = doc
datastore_dict[version_counter_key] = version_counter
datastore_dict[object_id + '_version_' + str(version_counter)] = doc
res = [object_id, str(version_counter)]
log.debug('Update result: %s' % str(res))
return res
def delete(self, obj, datastore_name=""):
if not isinstance(obj, IonObjectBase) and not isinstance(obj, str):
raise BadRequest(
"Obj param is not instance of IonObjectBase or string id")
if type(obj) is str:
return self.delete_doc(obj, datastore_name=datastore_name)
return self.delete_doc(self._ion_object_to_persistence_dict(obj),
datastore_name=datastore_name)
def delete_doc(self, doc, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name +
' does not exist.')
if type(doc) is str:
object_id = doc
else:
object_id = doc["_id"]
log.info('Deleting object %s/%s' % (datastore_name, object_id))
if object_id in datastore_dict.keys():
if self._is_in_association(object_id, datastore_name):
obj = self.read(object_id, "", datastore_name)
log.warn(
"XXXXXXX Attempt to delete object %s that still has associations"
% str(obj))
# raise BadRequest("Object cannot be deleted until associations are broken")
# Find all version dicts and delete them
for key in datastore_dict.keys():
if key.find(object_id + '_version_') == 0:
del datastore_dict[key]
# Delete the HEAD dict
del datastore_dict[object_id]
# Delete the version counter dict
del datastore_dict['__' + object_id + '_version_counter']
else:
raise NotFound('Object with id ' + object_id + ' does not exist.')
log.info('Delete result: True')
def _is_in_association(self, obj_id, datastore_name=""):
log.debug("_is_in_association(%s)" % obj_id)
if not obj_id:
raise BadRequest("Must provide object id")
if not datastore_name:
datastore_name = self.datastore_name
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name +
' does not exist.')
for objname, obj in datastore_dict.iteritems():
if (objname.find('_version_') > 0) or (not type(obj) is dict):
continue
if 'type_' in obj and obj['type_'] == "Association":
association = obj
if association["s"] == obj_id or association["o"] == obj_id:
log.debug("association found(%s)" % association)
return True
return False
def find_objects(self,
subject,
predicate=None,
object_type=None,
id_only=False):
log.debug(
"find_objects(subject=%s, predicate=%s, object_type=%s, id_only=%s"
% (subject, predicate, object_type, id_only))
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' %
type(id_only))
if not subject:
raise BadRequest("Must provide subject")
try:
datastore_dict = self.root[self.datastore_name]
except KeyError:
raise BadRequest('Data store ' + self.datastore_name +
' does not exist.')
if type(subject) is str:
subject_id = subject
else:
if "_id" not in subject:
raise BadRequest("Object id not available in subject")
else:
subject_id = subject._id
assoc_list = []
target_id_list = []
target_list = []
for objname, obj in datastore_dict.iteritems():
if (objname.find('_version_') > 0) or (not type(obj) is dict):
continue
if 'type_' in obj and obj['type_'] == "Association":
if obj['s'] == subject_id:
if predicate and obj['p'] == predicate:
if (object_type and obj['ot']
== object_type) or not object_type:
assoc_list.append(obj)
target_id_list.append(obj['o'])
target_list.append(self.read(obj['o']))
elif not predicate:
assoc_list.append(obj)
target_id_list.append(obj['o'])
target_list.append(self.read(obj['o']))
log.debug("find_objects() found %s objects" % (len(target_list)))
if id_only:
return (target_id_list, assoc_list)
else:
return (target_list, assoc_list)
def find_subjects(self,
subject_type=None,
predicate=None,
obj=None,
id_only=False):
log.debug(
"find_subjects(subject_type=%s, predicate=%s, object=%s, id_only=%s"
% (subject_type, predicate, obj, id_only))
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' %
type(id_only))
if not obj:
raise BadRequest("Must provide object")
try:
datastore_dict = self.root[self.datastore_name]
except KeyError:
raise BadRequest('Data store ' + self.datastore_name +
' does not exist.')
if type(obj) is str:
object_id = obj
else:
if "_id" not in obj:
raise BadRequest("Object id not available in object")
else:
object_id = obj._id
assoc_list = []
target_id_list = []
target_list = []
for objname, obj in datastore_dict.iteritems():
if (objname.find('_version_') > 0) or (not type(obj) is dict):
continue
if 'type_' in obj and obj['type_'] == "Association":
if obj['o'] == object_id:
if predicate and obj['p'] == predicate:
if (subject_type and obj['st']
== subject_type) or not subject_type:
assoc_list.append(obj)
target_id_list.append(obj['s'])
target_list.append(self.read(obj['s']))
elif not predicate:
assoc_list.append(obj)
target_id_list.append(obj['s'])
target_list.append(self.read(obj['s']))
log.debug("find_subjects() found %s subjects" % (len(target_list)))
if id_only:
return (target_id_list, assoc_list)
else:
return (target_list, assoc_list)
def find_associations(self,
subject=None,
predicate=None,
obj=None,
assoc_type=None,
id_only=True):
log.debug(
"find_associations(subject=%s, predicate=%s, object=%s, assoc_type=%s)"
% (subject, predicate, obj, assoc_type))
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' %
type(id_only))
if subject and obj or predicate:
pass
else:
raise BadRequest("Illegal parameters")
try:
datastore_dict = self.root[self.datastore_name]
except KeyError:
raise BadRequest('Data store ' + self.datastore_name +
' does not exist.')
if subject and obj:
if type(subject) is str:
subject_id = subject
else:
if "_id" not in subject:
raise BadRequest("Object id not available in subject")
else:
subject_id = subject._id
if type(obj) is str:
object_id = obj
else:
if "_id" not in obj:
raise BadRequest("Object id not available in object")
else:
object_id = obj._id
target_list = []
for objname, obj in datastore_dict.iteritems():
if (objname.find('_version_') > 0) or (not type(obj) is dict):
continue
if 'type_' in obj and obj['type_'] == "Association":
if obj['s'] == subject_id and obj['o'] == object_id:
if assoc_type:
if obj['at'] == assoc_type:
target_list.append(obj)
else:
target_list.append(obj)
else:
target_list = []
for objname, obj in datastore_dict.iteritems():
if (objname.find('_version_') > 0) or (not type(obj) is dict):
continue
if 'type_' in obj and obj['type_'] == "Association":
if obj['p'] == predicate:
target_list.append(obj)
if id_only:
assocs = [row['_id'] for row in target_list]
else:
assocs = [
self._persistence_dict_to_ion_object(row)
for row in target_list
]
log.debug("find_associations() found %s associations" % (len(assocs)))
return assocs
def find_res_by_type(self, restype, lcstate=None, id_only=False):
log.debug("find_res_by_type(restype=%s, lcstate=%s)" %
(restype, lcstate))
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' %
type(id_only))
try:
datastore_dict = self.root[self.datastore_name]
except KeyError:
raise BadRequest('Data store ' + self.datastore_name +
' does not exist.')
assoc_list = []
target_id_list = []
target_list = []
for objname, obj in datastore_dict.iteritems():
if (objname.find('_version_') > 0) or (not type(obj) is dict):
continue
if 'type_' in obj and (obj['type_'] == restype or
(not restype
and obj['type_'] != "Association")):
if (lcstate and 'lcstate' in obj and obj['lcstate']
== lcstate) or not lcstate or not restype:
target_id_list.append(obj['_id'])
target_list.append(
self._persistence_dict_to_ion_object(obj))
assoc_list.append([])
log.debug("find_res_by_type() found %s resources" % (len(target_list)))
if id_only:
return (target_id_list, assoc_list)
else:
return (target_list, assoc_list)
def find_res_by_lcstate(self, lcstate, restype=None, id_only=False):
log.debug("find_res_by_type(lcstate=%s, restype=%s)" %
(lcstate, restype))
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' %
type(id_only))
try:
datastore_dict = self.root[self.datastore_name]
except KeyError:
raise BadRequest('Data store ' + self.datastore_name +
' does not exist.')
if lcstate in CommonResourceLifeCycleSM.STATE_ALIASES:
lcstate_match = CommonResourceLifeCycleSM.STATE_ALIASES[lcstate]
else:
lcstate_match = [lcstate]
assoc_list = []
target_id_list = []
target_list = []
for objname, obj in datastore_dict.iteritems():
if (objname.find('_version_') > 0) or (not type(obj) is dict):
continue
if 'lcstate' in obj and obj['lcstate'] in lcstate_match:
if (restype and obj['type_'] == restype) or not restype:
target_id_list.append(obj['_id'])
target_list.append(
self._persistence_dict_to_ion_object(obj))
assoc_list.append([])
log.debug("find_res_by_lcstate() found %s resources" %
(len(target_list)))
if id_only:
return (target_id_list, assoc_list)
else:
return (target_list, assoc_list)
def _pass(self):
pass
def find_res_by_name(self, name, restype=None, id_only=False):
log.debug("find_res_by_name(name=%s, restype=%s)" % (name, restype))
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' %
type(id_only))
try:
datastore_dict = self.root[self.datastore_name]
except KeyError:
raise BadRequest('Data store ' + self.datastore_name +
' does not exist.')
assoc_list = []
target_id_list = []
target_list = []
for objname, obj in datastore_dict.iteritems():
if (objname.find('_version_') > 0) or (not type(obj) is dict):
continue
if 'name' in obj and obj['name'] == name:
if (restype and obj['type_'] == restype) or not restype:
target_id_list.append(obj['_id'])
target_list.append(
self._persistence_dict_to_ion_object(obj))
assoc_list.append([])
log.debug("find_res_by_name() found %s resources" % (len(target_list)))
if id_only:
return (target_id_list, assoc_list)
else:
return (target_list, assoc_list)
def find_dir_entries(self, qname):
raise NotImplementedError()
def _ion_object_to_persistence_dict(self, ion_object):
if ion_object is None: return None
obj_dict = self._io_serializer.serialize(ion_object)
return obj_dict
def _persistence_dict_to_ion_object(self, obj_dict):
if obj_dict is None: return None
ion_object = self._io_deserializer.deserialize(obj_dict)
return ion_object
def __init__(self, *args, **kwargs):
super(ReplayProcess,self).__init__(*args,**kwargs)
self.deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
self.publishing = Event()
self.play = Event()
self.end = Event()
class PostgresPyonDataStore(PostgresDataStore):
"""
Base class common to both CouchDB and Couchbase datastores.
"""
def __init__(self,
datastore_name=None,
config=None,
scope=None,
profile=None):
"""
@param datastore_name Name of datastore within server. May be scoped to sysname
@param config A server config dict with connection params
@param scope Prefix for the datastore name (e.g. sysname) to separate multiple systems
"""
PostgresDataStore.__init__(self,
datastore_name=datastore_name,
config=config
or CFG.get_safe("server.postgresql"),
profile=profile
or DataStore.DS_PROFILE.BASIC,
scope=scope)
# IonObject Serializers
self._io_serializer = IonObjectSerializer()
self._io_deserializer = IonObjectDeserializer(
obj_registry=get_obj_registry())
# -------------------------------------------------------------------------
# Couch document operations
def create(self, obj, object_id=None, attachments=None, datastore_name=""):
"""
Converts ion objects to python dictionary before persisting them using the optional
suggested identifier and creates attachments to the object.
Returns an identifier and revision number of the object
"""
if not isinstance(obj, IonObjectBase):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.create_doc(self._ion_object_to_persistence_dict(obj),
object_id=object_id,
datastore_name=datastore_name,
attachments=attachments)
def create_mult(self, objects, object_ids=None, allow_ids=None):
if any([not isinstance(obj, IonObjectBase) for obj in objects]):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.create_doc_mult(
[self._ion_object_to_persistence_dict(obj) for obj in objects],
object_ids)
def update(self, obj, datastore_name=""):
if not isinstance(obj, IonObjectBase):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.update_doc(self._ion_object_to_persistence_dict(obj))
def update_mult(self, objects):
if any([not isinstance(obj, IonObjectBase) for obj in objects]):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.update_doc_mult(
[self._ion_object_to_persistence_dict(obj) for obj in objects])
def read(self, object_id, rev_id="", datastore_name="", object_type=None):
if not isinstance(object_id, str):
raise BadRequest("Object id param is not string")
doc = self.read_doc(object_id,
rev_id,
datastore_name=datastore_name,
object_type=object_type)
obj = self._persistence_dict_to_ion_object(doc)
return obj
def read_mult(self, object_ids, datastore_name="", strict=True):
if any([not isinstance(object_id, str) for object_id in object_ids]):
raise BadRequest("Object ids are not string: %s" % str(object_ids))
docs = self.read_doc_mult(object_ids, datastore_name, strict=strict)
obj_list = [
self._persistence_dict_to_ion_object(doc)
if doc is not None else None for doc in docs
]
return obj_list
def delete(self, obj, datastore_name="", object_type=None):
if not isinstance(obj, IonObjectBase) and not isinstance(obj, str):
raise BadRequest(
"Obj param is not instance of IonObjectBase or string id")
if type(obj) is str:
self.delete_doc(obj,
datastore_name=datastore_name,
object_type=object_type)
else:
if '_id' not in obj:
raise BadRequest("Doc must have '_id'")
if '_rev' not in obj:
raise BadRequest("Doc must have '_rev'")
self.delete_doc(self._ion_object_to_persistence_dict(obj),
datastore_name=datastore_name,
object_type=object_type)
def delete_mult(self, object_ids, datastore_name=None):
return self.delete_doc_mult(object_ids, datastore_name)
# -------------------------------------------------------------------------
# View operations
def find_objects_mult(self,
subjects,
id_only=False,
predicate=None,
access_args=None):
"""
Returns a list of associations for a given list of subjects
"""
# TODO: Port this implementation to Postgres single query
res_list = [[], []]
if not subjects:
return res_list
for sub in subjects:
res_ids, res_assocs = self.find_objects(subject=sub,
id_only=id_only,
predicate=predicate,
access_args=access_args)
res_list[0].extend(res_ids)
res_list[1].extend(res_assocs)
return res_list
def find_subjects_mult(self,
objects,
id_only=False,
predicate=None,
access_args=None):
"""
Returns a list of associations for a given list of objects
"""
# TODO: Port this implementation to Postgres single query
res_list = [[], []]
if not objects:
return res_list
for obj in objects:
res_ids, res_assocs = self.find_subjects(obj=obj,
id_only=id_only,
predicate=predicate,
access_args=access_args)
res_list[0].extend(res_ids)
res_list[1].extend(res_assocs)
return res_list
def find_objects(self,
subject,
predicate=None,
object_type=None,
id_only=False,
access_args=None,
**kwargs):
#log.debug("find_objects(subject=%s, predicate=%s, object_type=%s, id_only=%s", subject, predicate, object_type, id_only)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' %
type(id_only))
if not subject:
raise BadRequest("Must provide subject")
if object_type and not predicate:
raise BadRequest("Cannot provide object type without a predicate")
if type(subject) is str:
subject_id = subject
else:
if "_id" not in subject:
raise BadRequest("Object id not available in subject")
else:
subject_id = subject._id
qual_ds_name = self._get_datastore_name()
assoc_table_name = qual_ds_name + "_assoc"
table_names = dict(ds=qual_ds_name, dsa=assoc_table_name)
view_args = self._get_view_args(kwargs, access_args)
if id_only:
#query = "SELECT o, doc FROM %(dsa)s WHERE retired<>true " % table_names
query = "SELECT %(dsa)s.o, %(dsa)s.doc FROM %(dsa)s, %(ds)s WHERE retired<>true AND %(dsa)s.o=%(ds)s.id " % table_names
else:
query = "SELECT %(ds)s.doc, %(dsa)s.doc FROM %(dsa)s, %(ds)s WHERE retired<>true AND %(dsa)s.o=%(ds)s.id " % table_names
query_args = dict(s=subject_id, ot=object_type, p=predicate)
query_clause = "AND s=%(s)s"
if predicate:
query_clause += " AND p=%(p)s"
if object_type:
query_clause += " AND ot=%(ot)s"
query_clause = self._add_access_filter(access_args, qual_ds_name,
query_clause, query_args)
extra_clause = view_args.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
obj_assocs = [
self._persistence_dict_to_ion_object(row[-1]) for row in rows
]
#log.debug("find_objects() found %s objects", len(obj_assocs))
if id_only:
res_ids = [self._prep_id(row[0]) for row in rows]
return res_ids, obj_assocs
else:
res_objs = [
self._persistence_dict_to_ion_object(row[0]) for row in rows
]
return res_objs, obj_assocs
def find_subjects(self,
subject_type=None,
predicate=None,
obj=None,
id_only=False,
access_args=None,
**kwargs):
#log.debug("find_subjects(subject_type=%s, predicate=%s, object=%s, id_only=%s", subject_type, predicate, obj, id_only)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' %
type(id_only))
if not obj:
raise BadRequest("Must provide object")
if subject_type and not predicate:
raise BadRequest("Cannot provide subject type without a predicate")
if type(obj) is str:
object_id = obj
else:
if "_id" not in obj:
raise BadRequest("Object id not available in object")
else:
object_id = obj._id
qual_ds_name = self._get_datastore_name()
assoc_table_name = qual_ds_name + "_assoc"
table_names = dict(ds=qual_ds_name, dsa=assoc_table_name)
view_args = self._get_view_args(kwargs, access_args)
if id_only:
#query = "SELECT s, doc FROM %(dsa)s WHERE retired<>true " % table_names
query = "SELECT %(dsa)s.s, %(dsa)s.doc FROM %(dsa)s, %(ds)s WHERE retired<>true AND %(dsa)s.s=%(ds)s.id " % table_names
else:
query = "SELECT %(ds)s.doc, %(dsa)s.doc FROM %(dsa)s, %(ds)s WHERE retired<>true AND %(dsa)s.s=%(ds)s.id " % table_names
query_args = dict(o=object_id, st=subject_type, p=predicate)
query_clause = "AND o=%(o)s"
if predicate:
query_clause += " AND p=%(p)s"
if subject_type:
query_clause += " AND st=%(st)s"
query_clause = self._add_access_filter(access_args, qual_ds_name,
query_clause, query_args)
extra_clause = view_args.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
obj_assocs = [
self._persistence_dict_to_ion_object(row[-1]) for row in rows
]
#log.debug("find_subjects() found %s subjects", len(obj_assocs))
if id_only:
res_ids = [self._prep_id(row[0]) for row in rows]
return res_ids, obj_assocs
else:
res_objs = [
self._persistence_dict_to_ion_object(row[0]) for row in rows
]
return res_objs, obj_assocs
def find_associations(self,
subject=None,
predicate=None,
obj=None,
assoc_type=None,
id_only=True,
anyside=None,
query=None,
**kwargs):
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' %
type(id_only))
if not (subject or obj or predicate or anyside or query):
raise BadRequest("Illegal parameters: No S/P/O or anyside")
if anyside and (subject or obj):
raise BadRequest(
"Illegal parameters: anyside cannot be combined with S/O")
if anyside and predicate and type(anyside) in (list, tuple):
raise BadRequest(
"Illegal parameters: anyside list cannot be combined with P")
if query:
query["query_args"]["id_only"] = id_only
query["query_args"]["ds_sub"] = "assoc"
# TODO: filter out retired
return self.find_by_query(query)
subject_id, object_id, anyside_ids = None, None, None
if subject:
if type(subject) is str:
subject_id = subject
else:
if "_id" not in subject:
raise BadRequest("Object id not available in subject")
else:
subject_id = subject._id
if obj:
if type(obj) is str:
object_id = obj
else:
if "_id" not in obj:
raise BadRequest("Object id not available in object")
else:
object_id = obj._id
if anyside:
if type(anyside) is str:
anyside_ids = [anyside]
elif type(anyside) in (list, tuple):
if not all([type(o) in (str, list, tuple) for o in anyside]):
raise BadRequest(
"List of object ids or (object id, predicate) expected"
)
anyside_ids = anyside
else:
if "_id" not in anyside:
raise BadRequest("Object id not available in anyside")
else:
anyside_ids = [anyside._id]
#log.debug("find_associations(subject=%s, predicate=%s, object=%s, anyside=%s)", subject_id, predicate, object_id, anyside_ids)
qual_ds_name = self._get_datastore_name()
table = qual_ds_name + "_assoc"
view_args = self._get_view_args(kwargs)
if id_only:
query = "SELECT id FROM " + table
else:
query = "SELECT id, doc, s, st, p, o, ot FROM " + table
query_clause = " WHERE retired<>true AND "
query_args = dict(s=subject_id, o=object_id, p=predicate)
if subject and obj:
query_clause += "s=%(s)s AND o=%(o)s"
if predicate:
query_clause += " AND p=%(p)s"
elif subject:
query_clause += "s=%(s)s"
if predicate:
query_clause += " AND p=%(p)s"
elif obj:
query_clause += "o=%(o)s"
if predicate:
query_clause += " AND p=%(p)s"
elif anyside:
if predicate:
query_clause += "p=%(p)s AND (s=%(any)s OR o=%(any)s)"
query_args["any"] = anyside
elif type(anyside_ids[0]) is str:
# keys are IDs of resources
for i, key in enumerate(anyside_ids):
if i > 0:
query_clause += " OR "
argname = "id%s" % i
query_args[argname] = key
query_clause += "(s=%(" + argname + ")s OR o=%(" + argname + ")s)"
else:
# keys are tuples of (id, pred)
for i, (key, pred) in enumerate(anyside_ids):
if i > 0:
query_clause += " OR "
argname_id = "id%s" % i
argname_p = "p%s" % i
query_args[argname_id] = key
query_args[argname_p] = pred
query_clause += "(p=%(" + argname_p + ")s AND (s=%(" + argname_id + ")s OR o=%(" + argname_id + ")s))"
elif predicate:
if predicate == "*":
query_clause += "p is not null"
else:
query_clause += "p=%(p)s"
else:
raise BadRequest("Illegal arguments")
extra_clause = view_args.get("extra_clause", "")
sql = query + query_clause + extra_clause
#print "find_associations(): SQL=", sql, query_args
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(sql, query_args)
rows = cur.fetchall()
if id_only:
assocs = [self._prep_id(row[0]) for row in rows]
else:
assocs = [
self._persistence_dict_to_ion_object(row[1]) for row in rows
]
#log.debug("find_associations() found %s associations", len(assocs))
return assocs
def _prepare_find_return(self,
rows,
res_assocs=None,
id_only=True,
**kwargs):
if id_only:
res_ids = [self._prep_id(row[0]) for row in rows]
return res_ids, res_assocs
else:
res_docs = [
self._persistence_dict_to_ion_object(row[-1]) for row in rows
]
return res_docs, res_assocs
def _add_access_filter(self,
view_args,
tablename,
query_clause,
query_args,
add_where=True,
tablealias=None):
"""Returns a Postgres SQL filter clause and referenced values for resource queries filtered
by resource visibility and current actor role/facility membership/superuser status"""
view_args = view_args if view_args is not None else {}
current_actor_id = view_args.get("current_actor_id", None)
superuser_actor_ids = view_args.get("superuser_actor_ids", None) or []
tablealias = tablealias or tablename
access_filter = ""
access_args = {}
access_args["current_actor_id"] = current_actor_id
assoc_tablename = tablename + "_assoc"
if current_actor_id in superuser_actor_ids:
# Current user is a superuser - no additional filter
pass
elif current_actor_id and current_actor_id != "anonymous":
# Registered actor
# - Return all PUBLIC, REGISTERED
access_filter += tablealias + ".visibility NOT IN (3,4)" # 1, 2, null and other values
# - Return all owned by user independent of visibility
access_filter += " OR (" + tablealias + ".id IN (SELECT s FROM " + assoc_tablename + \
" WHERE p='hasOwner' AND o=%(current_actor_id)s))"
# - Return all FACILITY if user is in same facility
access_filter += " OR (" + tablealias + ".visibility=3 AND " + tablealias + ".id IN (SELECT o FROM " + assoc_tablename + \
" WHERE p='hasResource' AND st='Org' AND s IN (SELECT s FROM " + assoc_tablename + \
" WHERE p='hasMember' AND st='Org' AND o=%(current_actor_id)s)))"
else:
# Anonymous access
# All public resources
access_filter += tablealias + ".visibility NOT IN (2,3,4)"
if query_clause and access_filter:
query_clause += " AND (" + access_filter + ")"
elif not query_clause and access_filter:
if add_where:
query_clause = " WHERE " + access_filter
else:
query_clause = access_filter
query_args.update(access_args)
return query_clause
def _add_deleted_filter(self,
tablename,
ds_sub,
query_clause,
query_args,
with_deleted=False):
if with_deleted:
return query_clause
deleted_filter = ""
if not ds_sub:
deleted_filter = tablename + ".lcstate<>'DELETED'"
elif ds_sub == "assoc":
deleted_filter = tablename + ".retired<>true"
if query_clause and deleted_filter:
query_clause += " AND " + deleted_filter
elif not query_clause and deleted_filter:
query_clause = deleted_filter
return query_clause
def find_resources(self,
restype="",
lcstate="",
name="",
id_only=True,
access_args=None):
return self.find_resources_ext(restype=restype,
lcstate=lcstate,
name=name,
id_only=id_only,
access_args=access_args)
def find_resources_ext(self,
restype="",
lcstate="",
name="",
keyword=None,
nested_type=None,
attr_name=None,
attr_value=None,
alt_id=None,
alt_id_ns=None,
limit=None,
skip=None,
descending=None,
id_only=True,
query=None,
access_args=None):
filter_kwargs = self._get_view_args(
dict(limit=limit, skip=skip, descending=descending), access_args)
if query:
qargs = query["query_args"]
if id_only is not None:
qargs["id_only"] = id_only
if limit is not None and limit != 0:
qargs["limit"] = limit
if skip is not None and skip != 0:
qargs["skip"] = skip
return self.find_by_query(query, access_args=access_args)
elif name:
if lcstate:
raise BadRequest("find by name does not support lcstate")
return self.find_res_by_name(name,
restype,
id_only,
filter=filter_kwargs)
elif keyword:
return self.find_res_by_keyword(keyword,
restype,
id_only,
filter=filter_kwargs)
elif alt_id or alt_id_ns:
return self.find_res_by_alternative_id(alt_id,
alt_id_ns,
id_only,
filter=filter_kwargs)
elif nested_type:
return self.find_res_by_nested_type(nested_type,
restype,
id_only,
filter=filter_kwargs)
elif restype and attr_name:
return self.find_res_by_attribute(restype,
attr_name,
attr_value,
id_only=id_only,
filter=filter_kwargs)
elif restype and lcstate:
return self.find_res_by_lcstate(lcstate,
restype,
id_only,
filter=filter_kwargs)
elif restype:
return self.find_res_by_type(restype,
lcstate,
id_only,
filter=filter_kwargs)
elif lcstate:
return self.find_res_by_lcstate(lcstate,
restype,
id_only,
filter=filter_kwargs)
elif not restype and not lcstate and not name:
return self.find_res_by_type(None,
None,
id_only,
filter=filter_kwargs)
def find_res_by_type(self,
restype,
lcstate=None,
id_only=False,
filter=None):
log.debug("find_res_by_type(restype=%s, lcstate=%s)", restype, lcstate)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' %
type(id_only))
if lcstate:
raise BadRequest(
'lcstate not supported anymore in find_res_by_type')
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, name, type_, lcstate FROM " + qual_ds_name
else:
query = "SELECT id, name, type_, lcstate, doc FROM " + qual_ds_name
query_clause = " WHERE lcstate<>'DELETED' "
query_args = dict(type_=restype, lcstate=lcstate)
if restype:
query_clause += "AND type_=%(type_)s"
else:
# Returns ALL documents, only limited by filter
query_clause = ""
query_clause = self._add_access_filter(filter, qual_ds_name,
query_clause, query_args)
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [
dict(id=self._prep_id(row[0]), name=row[1], type=row[2])
for row in rows
]
log.debug("find_res_by_type() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_lcstate(self,
lcstate,
restype=None,
id_only=False,
filter=None):
log.debug("find_res_by_lcstate(lcstate=%s, restype=%s)", lcstate,
restype)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' %
type(id_only))
if '_' in lcstate:
log.warn("Search for compound lcstate restricted to maturity: %s",
lcstate)
lcstate, _ = lcstate.split("_", 1)
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, name, type_, lcstate, availability FROM " + qual_ds_name
else:
query = "SELECT id, name, type_, lcstate, availability, doc FROM " + qual_ds_name
query_clause = " WHERE "
query_args = dict(type_=restype, lcstate=lcstate)
is_maturity = lcstate not in AvailabilityStates
if is_maturity:
query_clause += "lcstate=%(lcstate)s"
else:
query_clause += "availability=%(lcstate)s"
if restype:
query_clause += " AND type_=%(type_)s"
query_clause = self._add_access_filter(filter, qual_ds_name,
query_clause, query_args)
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [
dict(id=self._prep_id(row[0]),
name=row[1],
type=row[2],
lcstate=row[3] if is_maturity else row[4]) for row in rows
]
log.debug("find_res_by_lcstate() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_name(self, name, restype=None, id_only=False, filter=None):
log.debug("find_res_by_name(name=%s, restype=%s)", name, restype)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' %
type(id_only))
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, name, type_ FROM " + qual_ds_name
else:
query = "SELECT id, name, type_, doc FROM " + qual_ds_name
query_clause = " WHERE lcstate<>'DELETED' "
query_args = dict(name=name, type_=restype)
query_clause += "AND name=%(name)s"
if restype:
query_clause += " AND type_=%(type_)s"
query_clause = self._add_access_filter(filter, qual_ds_name,
query_clause, query_args)
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [
dict(id=self._prep_id(row[0]), name=row[1], type=row[2])
for row in rows
]
log.debug("find_res_by_name() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_keyword(self,
keyword,
restype=None,
id_only=False,
filter=None):
log.debug("find_res_by_keyword(keyword=%s, restype=%s)", keyword,
restype)
if not keyword or type(keyword) is not str:
raise BadRequest('Argument keyword illegal')
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' %
type(id_only))
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, type_ FROM " + qual_ds_name
else:
query = "SELECT id, type_, doc FROM " + qual_ds_name
query_clause = " WHERE lcstate<>'DELETED' "
query_args = dict(type_=restype, kw=[keyword])
query_clause += "AND %(kw)s <@ json_keywords(doc)"
if restype:
query_clause += " AND type_=%(type_)s"
query_clause = self._add_access_filter(filter, qual_ds_name,
query_clause, query_args)
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [
dict(id=self._prep_id(row[0]), type=row[1], keyword=keyword)
for row in rows
]
log.debug("find_res_by_keyword() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_nested_type(self,
nested_type,
restype=None,
id_only=False,
filter=None):
log.debug("find_res_by_nested_type(nested_type=%s, restype=%s)",
nested_type, restype)
if not nested_type or type(nested_type) is not str:
raise BadRequest('Argument nested_type illegal')
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' %
type(id_only))
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, type_ FROM " + qual_ds_name
else:
query = "SELECT id, type_, doc FROM " + qual_ds_name
query_clause = " WHERE lcstate<>'DELETED' "
query_args = dict(type_=restype, nest=[nested_type])
query_clause += "AND %(nest)s <@ json_nested(doc)"
if restype:
query_clause += " AND type_=%(type_)s"
query_clause = self._add_access_filter(filter, qual_ds_name,
query_clause, query_args)
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [
dict(id=self._prep_id(row[0]),
type=row[1],
nested_type=nested_type) for row in rows
]
log.debug("find_res_by_nested_type() found %s objects",
len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_attribute(self,
restype,
attr_name,
attr_value=None,
id_only=False,
filter=None):
log.debug(
"find_res_by_attribute(restype=%s, attr_name=%s, attr_value=%s)",
restype, attr_name, attr_value)
if not attr_name or type(attr_name) is not str:
raise BadRequest('Argument attr_name illegal')
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' %
type(id_only))
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, type_, json_specialattr(doc) FROM " + qual_ds_name
else:
query = "SELECT id, type_, json_specialattr(doc), doc FROM " + qual_ds_name
query_clause = " WHERE lcstate<>'DELETED' "
query_args = dict(type_=restype, att=attr_name, val=attr_value)
if attr_value: # Note: cannot make None test here (and allow empty string because of default service args "")
query_clause += "AND json_specialattr(doc)=%(spc)s"
query_args['spc'] = "%s=%s" % (attr_name, attr_value)
else:
query_clause += "AND json_specialattr(doc) LIKE %(spc)s"
query_args['spc'] = "%s=%%" % (attr_name, )
if restype:
query_clause += " AND type_=%(type_)s"
query_clause = self._add_access_filter(filter, qual_ds_name,
query_clause, query_args)
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [
dict(id=self._prep_id(row[0]),
type=row[1],
attr_name=attr_name,
attr_value=row[2].split("=", 1)[-1]) for row in rows
]
log.debug("find_res_by_attribute() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_alternative_id(self,
alt_id=None,
alt_id_ns=None,
id_only=False,
filter=None):
log.debug("find_res_by_alternative_id(restype=%s, alt_id_ns=%s)",
alt_id, alt_id_ns)
if alt_id and type(alt_id) is not str:
raise BadRequest('Argument alt_id illegal')
if alt_id_ns and type(alt_id_ns) is not str:
raise BadRequest('Argument alt_id_ns illegal')
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' %
type(id_only))
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
query = "SELECT id, type_, doc FROM " + qual_ds_name
query_args = dict(aid=[alt_id], ans=[alt_id_ns])
query_clause = " WHERE lcstate<>'DELETED' "
if not alt_id and not alt_id_ns:
query_clause += "AND json_altids_ns(doc) is not null"
elif alt_id and not alt_id_ns:
query_clause += "AND %(aid)s <@ json_altids_id(doc)"
elif alt_id_ns and not alt_id:
query_clause += "AND %(ans)s <@ json_altids_ns(doc)"
else:
query_clause += "AND %(aid)s <@ json_altids_id(doc) AND %(ans)s <@ json_altids_ns(doc)"
query_clause = self._add_access_filter(filter, qual_ds_name,
query_clause, query_args)
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
# Need to fake the return format of the Couch view for alt_ids. One record per alt_id, not one per resource.
res_assocs = []
res_rows = []
for row in rows:
doc_id = self._prep_id(row[0])
doc = row[-1]
for aid in doc.get("alt_ids", []):
aid_parts = aid.split(":", 1)
aid_ns = aid_parts[0] if len(aid_parts) > 1 else "_"
aid_id = aid_parts[-1]
if alt_id_ns and alt_id:
if alt_id_ns == aid_ns and alt_id == aid_id:
res_assocs.append(
dict(id=doc_id, alt_id_ns=aid_ns, alt_id=aid_id))
res_rows.append((doc_id, doc))
elif (not alt_id_ns
and not alt_id) or (alt_id_ns and alt_id_ns
== aid_ns) or (alt_id
and alt_id == aid_id):
res_assocs.append(
dict(id=doc_id, alt_id_ns=aid_ns, alt_id=aid_id))
res_rows.append((doc_id, doc))
log.debug("find_res_by_alternative_id() found %s objects",
len(res_assocs))
return self._prepare_find_return(res_rows, res_assocs, id_only=id_only)
def find_by_view(self,
design_name,
view_name,
key=None,
keys=None,
start_key=None,
end_key=None,
id_only=True,
convert_doc=True,
**kwargs):
"""
Generic find function using a defined index
@param design_name design document
@param view_name view name
@param key specific key to find
@param keys list of keys to find
@param start_key find range start value
@param end_key find range end value
@param id_only if True, the 4th element of each triple is the document
@param convert_doc if True, make IonObject out of doc
@retval Returns a list of 3-tuples: (document id, index key, index value or document)
"""
res_rows = self.find_docs_by_view(design_name=design_name,
view_name=view_name,
key=key,
keys=keys,
start_key=start_key,
end_key=end_key,
id_only=id_only,
**kwargs)
res_rows = [(rid, key, self._persistence_dict_to_ion_object(doc)
if convert_doc and isinstance(doc, dict) else doc)
for rid, key, doc in res_rows]
log.debug("find_by_view() found %s objects" % (len(res_rows)))
return res_rows
def find_by_query(self, query, access_args=None):
"""
Find resources given a datastore query expression dict.
@param query a dict representation of a datastore query
@retval list of resource ids or resource objects matching query (dependent on id_only value)
"""
qual_ds_name = self._get_datastore_name()
query_ds_sub = query["query_args"].get("ds_sub", None)
query_format = query["query_args"].get("format", "")
pqb = PostgresQueryBuilder(query, qual_ds_name)
if self.profile == DataStore.DS_PROFILE.RESOURCES and not query_ds_sub:
table_alias = qual_ds_name if query_format != "complex" else "base"
pqb.where = self._add_access_filter(access_args,
qual_ds_name,
pqb.where,
pqb.values,
add_where=False,
tablealias=table_alias)
if self.profile == DataStore.DS_PROFILE.RESOURCES:
pqb.where = self._add_deleted_filter(
pqb.table_aliases[0],
query_ds_sub,
pqb.where,
pqb.values,
with_deleted=query["query_args"].get("with_deleted",
False) is True)
with self.pool.cursor(**self.cursor_args) as cur:
exec_query = pqb.get_query()
cur.execute(exec_query, pqb.get_values())
rows = cur.fetchall()
log.info("find_by_query() QUERY: %s (%s rows)", cur.query,
cur.rowcount)
query_res = {}
query["_result"] = query_res
query_res["statement_gen"] = exec_query
query_res["statement_sql"] = cur.query
query_res["rowcount"] = cur.rowcount
id_only = query["query_args"].get("id_only", True)
if query_format == "complex" and pqb.has_basic_cols:
# Return format is list of lists
if id_only:
res_vals = [[self._prep_id(row[0])] + list(row[1:])
for row in rows]
else:
res_vals = [[self._persistence_dict_to_ion_object(row[1])] +
list(rows[2:]) for row in rows]
elif query_format == "complex":
res_vals = [list(row) for row in rows]
else:
if id_only:
res_vals = [self._prep_id(row[0]) for row in rows]
else:
res_vals = [
self._persistence_dict_to_ion_object(row[-1])
for row in rows
]
return res_vals
# -------------------------------------------------------------------------
# Internal operations
def _ion_object_to_persistence_dict(self, ion_object):
if ion_object is None:
return None
obj_dict = self._io_serializer.serialize(ion_object,
update_version=True)
return obj_dict
def _persistence_dict_to_ion_object(self, obj_dict):
if obj_dict is None:
return None
ion_object = self._io_deserializer.deserialize(obj_dict)
return ion_object
class PostgresPyonDataStore(PostgresDataStore):
"""
Base class common to both CouchDB and Couchbase datastores.
"""
def __init__(self, datastore_name=None, config=None, scope=None, profile=None):
"""
@param datastore_name Name of datastore within server. May be scoped to sysname
@param config A server config dict with connection params
@param scope Prefix for the datastore name (e.g. sysname) to separate multiple systems
"""
PostgresDataStore.__init__(self, datastore_name=datastore_name,
config=config or CFG.get_safe("server.postgresql"),
profile=profile or DataStore.DS_PROFILE.BASIC,
scope=scope)
# IonObject Serializers
self._io_serializer = IonObjectSerializer()
self._io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
# -------------------------------------------------------------------------
# Couch document operations
def create(self, obj, object_id=None, attachments=None, datastore_name=""):
"""
Converts ion objects to python dictionary before persisting them using the optional
suggested identifier and creates attachments to the object.
Returns an identifier and revision number of the object
"""
if not isinstance(obj, IonObjectBase):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.create_doc(self._ion_object_to_persistence_dict(obj),
object_id=object_id, datastore_name=datastore_name,
attachments=attachments)
def create_mult(self, objects, object_ids=None, allow_ids=None):
if any([not isinstance(obj, IonObjectBase) for obj in objects]):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.create_doc_mult([self._ion_object_to_persistence_dict(obj) for obj in objects], object_ids)
def update(self, obj, datastore_name=""):
if not isinstance(obj, IonObjectBase):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.update_doc(self._ion_object_to_persistence_dict(obj))
def update_mult(self, objects):
if any([not isinstance(obj, IonObjectBase) for obj in objects]):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.update_doc_mult([self._ion_object_to_persistence_dict(obj) for obj in objects])
def read(self, object_id, rev_id="", datastore_name="", object_type=None):
if not isinstance(object_id, str):
raise BadRequest("Object id param is not string")
doc = self.read_doc(object_id, rev_id, datastore_name=datastore_name, object_type=object_type)
obj = self._persistence_dict_to_ion_object(doc)
return obj
def read_mult(self, object_ids, datastore_name="", strict=True):
if any([not isinstance(object_id, str) for object_id in object_ids]):
raise BadRequest("Object ids are not string: %s" % str(object_ids))
docs = self.read_doc_mult(object_ids, datastore_name, strict=strict)
obj_list = [self._persistence_dict_to_ion_object(doc) if doc is not None else None for doc in docs]
return obj_list
def delete(self, obj, datastore_name="", object_type=None):
if not isinstance(obj, IonObjectBase) and not isinstance(obj, str):
raise BadRequest("Obj param is not instance of IonObjectBase or string id")
if type(obj) is str:
self.delete_doc(obj, datastore_name=datastore_name, object_type=object_type)
else:
if '_id' not in obj:
raise BadRequest("Doc must have '_id'")
if '_rev' not in obj:
raise BadRequest("Doc must have '_rev'")
self.delete_doc(self._ion_object_to_persistence_dict(obj),
datastore_name=datastore_name, object_type=object_type)
def delete_mult(self, object_ids, datastore_name=None):
return self.delete_doc_mult(object_ids, datastore_name)
# -------------------------------------------------------------------------
# View operations
def find_objects_mult(self, subjects, id_only=False, predicate=None, access_args=None):
"""
Returns a list of associations for a given list of subjects
"""
# TODO: Port this implementation to Postgres single query
res_list = [[], []]
if not subjects:
return res_list
for sub in subjects:
res_ids, res_assocs = self.find_objects(subject=sub, id_only=id_only, predicate=predicate, access_args=access_args)
res_list[0].extend(res_ids)
res_list[1].extend(res_assocs)
return res_list
def find_subjects_mult(self, objects, id_only=False, predicate=None, access_args=None):
"""
Returns a list of associations for a given list of objects
"""
# TODO: Port this implementation to Postgres single query
res_list = [[], []]
if not objects:
return res_list
for obj in objects:
res_ids, res_assocs = self.find_subjects(obj=obj, id_only=id_only, predicate=predicate, access_args=access_args)
res_list[0].extend(res_ids)
res_list[1].extend(res_assocs)
return res_list
def find_objects(self, subject, predicate=None, object_type=None, id_only=False, access_args=None, **kwargs):
#log.debug("find_objects(subject=%s, predicate=%s, object_type=%s, id_only=%s", subject, predicate, object_type, id_only)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
if not subject:
raise BadRequest("Must provide subject")
if object_type and not predicate:
raise BadRequest("Cannot provide object type without a predicate")
if type(subject) is str:
subject_id = subject
else:
if "_id" not in subject:
raise BadRequest("Object id not available in subject")
else:
subject_id = subject._id
qual_ds_name = self._get_datastore_name()
assoc_table_name = qual_ds_name+"_assoc"
table_names = dict(ds=qual_ds_name, dsa=assoc_table_name)
view_args = self._get_view_args(kwargs, access_args)
if id_only:
#query = "SELECT o, doc FROM %(dsa)s WHERE retired<>true " % table_names
query = "SELECT %(dsa)s.o, %(dsa)s.doc FROM %(dsa)s, %(ds)s WHERE retired<>true AND %(dsa)s.o=%(ds)s.id " % table_names
else:
query = "SELECT %(ds)s.doc, %(dsa)s.doc FROM %(dsa)s, %(ds)s WHERE retired<>true AND %(dsa)s.o=%(ds)s.id " % table_names
query_args = dict(s=subject_id, ot=object_type, p=predicate)
query_clause = "AND s=%(s)s"
if predicate:
query_clause += " AND p=%(p)s"
if object_type:
query_clause += " AND ot=%(ot)s"
query_clause = self._add_access_filter(access_args, qual_ds_name, query_clause, query_args)
extra_clause = view_args.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
obj_assocs = [self._persistence_dict_to_ion_object(row[-1]) for row in rows]
#log.debug("find_objects() found %s objects", len(obj_assocs))
if id_only:
res_ids = [self._prep_id(row[0]) for row in rows]
return res_ids, obj_assocs
else:
res_objs = [self._persistence_dict_to_ion_object(row[0]) for row in rows]
return res_objs, obj_assocs
def find_subjects(self, subject_type=None, predicate=None, obj=None, id_only=False, access_args=None, **kwargs):
#log.debug("find_subjects(subject_type=%s, predicate=%s, object=%s, id_only=%s", subject_type, predicate, obj, id_only)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
if not obj:
raise BadRequest("Must provide object")
if subject_type and not predicate:
raise BadRequest("Cannot provide subject type without a predicate")
if type(obj) is str:
object_id = obj
else:
if "_id" not in obj:
raise BadRequest("Object id not available in object")
else:
object_id = obj._id
qual_ds_name = self._get_datastore_name()
assoc_table_name = qual_ds_name+"_assoc"
table_names = dict(ds=qual_ds_name, dsa=assoc_table_name)
view_args = self._get_view_args(kwargs, access_args)
if id_only:
#query = "SELECT s, doc FROM %(dsa)s WHERE retired<>true " % table_names
query = "SELECT %(dsa)s.s, %(dsa)s.doc FROM %(dsa)s, %(ds)s WHERE retired<>true AND %(dsa)s.s=%(ds)s.id " % table_names
else:
query = "SELECT %(ds)s.doc, %(dsa)s.doc FROM %(dsa)s, %(ds)s WHERE retired<>true AND %(dsa)s.s=%(ds)s.id " % table_names
query_args = dict(o=object_id, st=subject_type, p=predicate)
query_clause = "AND o=%(o)s"
if predicate:
query_clause += " AND p=%(p)s"
if subject_type:
query_clause += " AND st=%(st)s"
query_clause = self._add_access_filter(access_args, qual_ds_name, query_clause, query_args)
extra_clause = view_args.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
obj_assocs = [self._persistence_dict_to_ion_object(row[-1]) for row in rows]
#log.debug("find_subjects() found %s subjects", len(obj_assocs))
if id_only:
res_ids = [self._prep_id(row[0]) for row in rows]
return res_ids, obj_assocs
else:
res_objs = [self._persistence_dict_to_ion_object(row[0]) for row in rows]
return res_objs, obj_assocs
def find_associations(self, subject=None, predicate=None, obj=None, assoc_type=None, id_only=True,
anyside=None, query=None, **kwargs):
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
if not (subject or obj or predicate or anyside or query):
raise BadRequest("Illegal parameters: No S/P/O or anyside")
if anyside and (subject or obj):
raise BadRequest("Illegal parameters: anyside cannot be combined with S/O")
if anyside and predicate and type(anyside) in (list, tuple):
raise BadRequest("Illegal parameters: anyside list cannot be combined with P")
if query:
query["query_args"]["id_only"] = id_only
query["query_args"]["ds_sub"] = "assoc"
# TODO: filter out retired
return self.find_by_query(query)
subject_id, object_id, anyside_ids = None, None, None
if subject:
if type(subject) is str:
subject_id = subject
else:
if "_id" not in subject:
raise BadRequest("Object id not available in subject")
else:
subject_id = subject._id
if obj:
if type(obj) is str:
object_id = obj
else:
if "_id" not in obj:
raise BadRequest("Object id not available in object")
else:
object_id = obj._id
if anyside:
if type(anyside) is str:
anyside_ids = [anyside]
elif type(anyside) in (list, tuple):
if not all([type(o) in (str, list, tuple) for o in anyside]):
raise BadRequest("List of object ids or (object id, predicate) expected")
anyside_ids = anyside
else:
if "_id" not in anyside:
raise BadRequest("Object id not available in anyside")
else:
anyside_ids = [anyside._id]
#log.debug("find_associations(subject=%s, predicate=%s, object=%s, anyside=%s)", subject_id, predicate, object_id, anyside_ids)
qual_ds_name = self._get_datastore_name()
table = qual_ds_name + "_assoc"
view_args = self._get_view_args(kwargs)
if id_only:
query = "SELECT id FROM " + table
else:
query = "SELECT id, doc, s, st, p, o, ot FROM " + table
query_clause = " WHERE retired<>true AND "
query_args = dict(s=subject_id, o=object_id, p=predicate)
if subject and obj:
query_clause += "s=%(s)s AND o=%(o)s"
if predicate:
query_clause += " AND p=%(p)s"
elif subject:
query_clause += "s=%(s)s"
if predicate:
query_clause += " AND p=%(p)s"
elif obj:
query_clause += "o=%(o)s"
if predicate:
query_clause += " AND p=%(p)s"
elif anyside:
if predicate:
query_clause += "p=%(p)s AND (s=%(any)s OR o=%(any)s)"
query_args["any"] = anyside
elif type(anyside_ids[0]) is str:
# keys are IDs of resources
for i, key in enumerate(anyside_ids):
if i > 0:
query_clause += " OR "
argname = "id%s" % i
query_args[argname] = key
query_clause += "(s=%("+argname+")s OR o=%("+argname+")s)"
else:
# keys are tuples of (id, pred)
for i, (key, pred) in enumerate(anyside_ids):
if i > 0:
query_clause += " OR "
argname_id = "id%s" % i
argname_p = "p%s" % i
query_args[argname_id] = key
query_args[argname_p] = pred
query_clause += "(p=%("+argname_p+")s AND (s=%("+argname_id+")s OR o=%("+argname_id+")s))"
elif predicate:
if predicate == "*":
query_clause += "p is not null"
else:
query_clause += "p=%(p)s"
else:
raise BadRequest("Illegal arguments")
extra_clause = view_args.get("extra_clause", "")
sql = query + query_clause + extra_clause
#print "find_associations(): SQL=", sql, query_args
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(sql, query_args)
rows = cur.fetchall()
if id_only:
assocs = [self._prep_id(row[0]) for row in rows]
else:
assocs = [self._persistence_dict_to_ion_object(row[1]) for row in rows]
#log.debug("find_associations() found %s associations", len(assocs))
return assocs
def _prepare_find_return(self, rows, res_assocs=None, id_only=True, **kwargs):
if id_only:
res_ids = [self._prep_id(row[0]) for row in rows]
return res_ids, res_assocs
else:
res_docs = [self._persistence_dict_to_ion_object(row[-1]) for row in rows]
return res_docs, res_assocs
def _add_access_filter(self, view_args, tablename, query_clause, query_args, add_where=True, tablealias=None):
"""Returns a Postgres SQL filter clause and referenced values for resource queries filtered
by resource visibility and current actor role/facility membership/superuser status"""
view_args = view_args if view_args is not None else {}
current_actor_id = view_args.get("current_actor_id", None)
superuser_actor_ids = view_args.get("superuser_actor_ids", None) or []
tablealias = tablealias or tablename
access_filter = ""
access_args = {}
access_args["current_actor_id"] = current_actor_id
assoc_tablename = tablename + "_assoc"
if current_actor_id in superuser_actor_ids:
# Current user is a superuser - no additional filter
pass
elif current_actor_id and current_actor_id != "anonymous":
# Registered actor
# - Return all PUBLIC, REGISTERED
access_filter += tablealias + ".visibility NOT IN (3,4)" # 1, 2, null and other values
# - Return all owned by user independent of visibility
access_filter += " OR (" + tablealias + ".id IN (SELECT s FROM " + assoc_tablename + \
" WHERE p='hasOwner' AND o=%(current_actor_id)s))"
# - Return all FACILITY if user is in same facility
access_filter += " OR (" + tablealias + ".visibility=3 AND " + tablealias + ".id IN (SELECT o FROM " + assoc_tablename + \
" WHERE p='hasResource' AND st='Org' AND s IN (SELECT s FROM " + assoc_tablename + \
" WHERE p='hasMember' AND st='Org' AND o=%(current_actor_id)s)))"
else:
# Anonymous access
# All public resources
access_filter += tablealias + ".visibility NOT IN (2,3,4)"
if query_clause and access_filter:
query_clause += " AND (" + access_filter + ")"
elif not query_clause and access_filter:
if add_where:
query_clause = " WHERE " + access_filter
else:
query_clause = access_filter
query_args.update(access_args)
return query_clause
def _add_deleted_filter(self, tablename, ds_sub, query_clause, query_args, show_all=False):
if show_all:
return query_clause
deleted_filter = ""
if not ds_sub:
deleted_filter = tablename + ".lcstate<>'DELETED'"
elif ds_sub == "assoc":
deleted_filter = tablename + ".retired<>true"
if query_clause and deleted_filter:
query_clause += " AND " + deleted_filter
elif not query_clause and deleted_filter:
query_clause = deleted_filter
return query_clause
def find_resources(self, restype="", lcstate="", name="", id_only=True, access_args=None):
return self.find_resources_ext(restype=restype, lcstate=lcstate, name=name, id_only=id_only, access_args=access_args)
def find_resources_ext(self, restype="", lcstate="", name="",
keyword=None, nested_type=None,
attr_name=None, attr_value=None, alt_id=None, alt_id_ns=None,
limit=None, skip=None, descending=None, id_only=True, query=None, access_args=None):
filter_kwargs = self._get_view_args(dict(limit=limit, skip=skip, descending=descending), access_args)
if query:
qargs = query["query_args"]
if id_only is not None:
qargs["id_only"] = id_only
if limit is not None and limit != 0:
qargs["limit"] = limit
if skip is not None and skip != 0:
qargs["skip"] = skip
return self.find_by_query(query, access_args=access_args)
elif name:
if lcstate:
raise BadRequest("find by name does not support lcstate")
return self.find_res_by_name(name, restype, id_only, filter=filter_kwargs)
elif keyword:
return self.find_res_by_keyword(keyword, restype, id_only, filter=filter_kwargs)
elif alt_id or alt_id_ns:
return self.find_res_by_alternative_id(alt_id, alt_id_ns, id_only, filter=filter_kwargs)
elif nested_type:
return self.find_res_by_nested_type(nested_type, restype, id_only, filter=filter_kwargs)
elif restype and attr_name:
return self.find_res_by_attribute(restype, attr_name, attr_value, id_only=id_only, filter=filter_kwargs)
elif restype and lcstate:
return self.find_res_by_lcstate(lcstate, restype, id_only, filter=filter_kwargs)
elif restype:
return self.find_res_by_type(restype, lcstate, id_only, filter=filter_kwargs)
elif lcstate:
return self.find_res_by_lcstate(lcstate, restype, id_only, filter=filter_kwargs)
elif not restype and not lcstate and not name:
return self.find_res_by_type(None, None, id_only, filter=filter_kwargs)
def find_res_by_type(self, restype, lcstate=None, id_only=False, filter=None):
log.debug("find_res_by_type(restype=%s, lcstate=%s)", restype, lcstate)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
if lcstate:
raise BadRequest('lcstate not supported anymore in find_res_by_type')
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, name, type_, lcstate FROM " + qual_ds_name
else:
query = "SELECT id, name, type_, lcstate, doc FROM " + qual_ds_name
query_clause = " WHERE lcstate<>'DELETED' "
query_args = dict(type_=restype, lcstate=lcstate)
if restype:
query_clause += "AND type_=%(type_)s"
else:
# Returns ALL documents, only limited by filter
query_clause = ""
query_clause = self._add_access_filter(filter, qual_ds_name, query_clause, query_args)
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [dict(id=self._prep_id(row[0]), name=row[1], type=row[2]) for row in rows]
log.debug("find_res_by_type() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_lcstate(self, lcstate, restype=None, id_only=False, filter=None):
log.debug("find_res_by_lcstate(lcstate=%s, restype=%s)", lcstate, restype)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
if '_' in lcstate:
log.warn("Search for compound lcstate restricted to maturity: %s", lcstate)
lcstate,_ = lcstate.split("_", 1)
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, name, type_, lcstate, availability FROM " + qual_ds_name
else:
query = "SELECT id, name, type_, lcstate, availability, doc FROM " + qual_ds_name
query_clause = " WHERE "
query_args = dict(type_=restype, lcstate=lcstate)
is_maturity = lcstate not in AvailabilityStates
if is_maturity:
query_clause += "lcstate=%(lcstate)s"
else:
query_clause += "availability=%(lcstate)s"
if restype:
query_clause += " AND type_=%(type_)s"
query_clause = self._add_access_filter(filter, qual_ds_name, query_clause, query_args)
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [dict(id=self._prep_id(row[0]), name=row[1], type=row[2], lcstate=row[3] if is_maturity else row[4]) for row in rows]
log.debug("find_res_by_lcstate() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_name(self, name, restype=None, id_only=False, filter=None):
log.debug("find_res_by_name(name=%s, restype=%s)", name, restype)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, name, type_ FROM " + qual_ds_name
else:
query = "SELECT id, name, type_, doc FROM " + qual_ds_name
query_clause = " WHERE lcstate<>'DELETED' "
query_args = dict(name=name, type_=restype)
query_clause += "AND name=%(name)s"
if restype:
query_clause += " AND type_=%(type_)s"
query_clause = self._add_access_filter(filter, qual_ds_name, query_clause, query_args)
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [dict(id=self._prep_id(row[0]), name=row[1], type=row[2]) for row in rows]
log.debug("find_res_by_name() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_keyword(self, keyword, restype=None, id_only=False, filter=None):
log.debug("find_res_by_keyword(keyword=%s, restype=%s)", keyword, restype)
if not keyword or type(keyword) is not str:
raise BadRequest('Argument keyword illegal')
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, type_ FROM " + qual_ds_name
else:
query = "SELECT id, type_, doc FROM " + qual_ds_name
query_clause = " WHERE lcstate<>'DELETED' "
query_args = dict(type_=restype, kw=[keyword])
query_clause += "AND %(kw)s <@ json_keywords(doc)"
if restype:
query_clause += " AND type_=%(type_)s"
query_clause = self._add_access_filter(filter, qual_ds_name, query_clause, query_args)
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [dict(id=self._prep_id(row[0]), type=row[1], keyword=keyword) for row in rows]
log.debug("find_res_by_keyword() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_nested_type(self, nested_type, restype=None, id_only=False, filter=None):
log.debug("find_res_by_nested_type(nested_type=%s, restype=%s)", nested_type, restype)
if not nested_type or type(nested_type) is not str:
raise BadRequest('Argument nested_type illegal')
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, type_ FROM " + qual_ds_name
else:
query = "SELECT id, type_, doc FROM " + qual_ds_name
query_clause = " WHERE lcstate<>'DELETED' "
query_args = dict(type_=restype, nest=[nested_type])
query_clause += "AND %(nest)s <@ json_nested(doc)"
if restype:
query_clause += " AND type_=%(type_)s"
query_clause = self._add_access_filter(filter, qual_ds_name, query_clause, query_args)
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [dict(id=self._prep_id(row[0]), type=row[1], nested_type=nested_type) for row in rows]
log.debug("find_res_by_nested_type() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_attribute(self, restype, attr_name, attr_value=None, id_only=False, filter=None):
log.debug("find_res_by_attribute(restype=%s, attr_name=%s, attr_value=%s)", restype, attr_name, attr_value)
if not attr_name or type(attr_name) is not str:
raise BadRequest('Argument attr_name illegal')
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, type_, json_specialattr(doc) FROM " + qual_ds_name
else:
query = "SELECT id, type_, json_specialattr(doc), doc FROM " + qual_ds_name
query_clause = " WHERE lcstate<>'DELETED' "
query_args = dict(type_=restype, att=attr_name, val=attr_value)
if attr_value: # Note: cannot make None test here (and allow empty string because of default service args "")
query_clause += "AND json_specialattr(doc)=%(spc)s"
query_args['spc'] = "%s=%s" % (attr_name, attr_value)
else:
query_clause += "AND json_specialattr(doc) LIKE %(spc)s"
query_args['spc'] = "%s=%%" % (attr_name, )
if restype:
query_clause += " AND type_=%(type_)s"
query_clause = self._add_access_filter(filter, qual_ds_name, query_clause, query_args)
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [dict(id=self._prep_id(row[0]), type=row[1], attr_name=attr_name, attr_value=row[2].split("=",1)[-1]) for row in rows]
log.debug("find_res_by_attribute() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_alternative_id(self, alt_id=None, alt_id_ns=None, id_only=False, filter=None):
log.debug("find_res_by_alternative_id(restype=%s, alt_id_ns=%s)", alt_id, alt_id_ns)
if alt_id and type(alt_id) is not str:
raise BadRequest('Argument alt_id illegal')
if alt_id_ns and type(alt_id_ns) is not str:
raise BadRequest('Argument alt_id_ns illegal')
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
query = "SELECT id, type_, doc FROM " + qual_ds_name
query_args = dict(aid=[alt_id], ans=[alt_id_ns])
query_clause = " WHERE lcstate<>'DELETED' "
if not alt_id and not alt_id_ns:
query_clause += "AND json_altids_ns(doc) is not null"
elif alt_id and not alt_id_ns:
query_clause += "AND %(aid)s <@ json_altids_id(doc)"
elif alt_id_ns and not alt_id:
query_clause += "AND %(ans)s <@ json_altids_ns(doc)"
else:
query_clause += "AND %(aid)s <@ json_altids_id(doc) AND %(ans)s <@ json_altids_ns(doc)"
query_clause = self._add_access_filter(filter, qual_ds_name, query_clause, query_args)
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
# Need to fake the return format of the Couch view for alt_ids. One record per alt_id, not one per resource.
res_assocs = []
res_rows = []
for row in rows:
doc_id = self._prep_id(row[0])
doc = row[-1]
for aid in doc.get("alt_ids", []):
aid_parts = aid.split(":", 1)
aid_ns = aid_parts[0] if len(aid_parts)>1 else "_"
aid_id = aid_parts[-1]
if alt_id_ns and alt_id:
if alt_id_ns == aid_ns and alt_id == aid_id:
res_assocs.append(dict(id=doc_id, alt_id_ns=aid_ns, alt_id=aid_id))
res_rows.append((doc_id, doc))
elif (not alt_id_ns and not alt_id) or (alt_id_ns and alt_id_ns == aid_ns) or (alt_id and alt_id == aid_id):
res_assocs.append(dict(id=doc_id, alt_id_ns=aid_ns, alt_id=aid_id))
res_rows.append((doc_id, doc))
log.debug("find_res_by_alternative_id() found %s objects", len(res_assocs))
return self._prepare_find_return(res_rows, res_assocs, id_only=id_only)
def find_by_view(self, design_name, view_name, key=None, keys=None, start_key=None, end_key=None,
id_only=True, convert_doc=True, **kwargs):
"""
Generic find function using a defined index
@param design_name design document
@param view_name view name
@param key specific key to find
@param keys list of keys to find
@param start_key find range start value
@param end_key find range end value
@param id_only if True, the 4th element of each triple is the document
@param convert_doc if True, make IonObject out of doc
@retval Returns a list of 3-tuples: (document id, index key, index value or document)
"""
res_rows = self.find_docs_by_view(design_name=design_name, view_name=view_name, key=key, keys=keys,
start_key=start_key, end_key=end_key, id_only=id_only, **kwargs)
res_rows = [(rid, key,
self._persistence_dict_to_ion_object(doc) if convert_doc and isinstance(doc, dict) else doc)
for rid, key, doc in res_rows]
log.debug("find_by_view() found %s objects" % (len(res_rows)))
return res_rows
def find_by_query(self, query, access_args=None):
"""
Find resources given a datastore query expression dict.
@param query a dict representation of a datastore query
@retval list of resource ids or resource objects matching query (dependent on id_only value)
"""
qual_ds_name = self._get_datastore_name()
query_ds_sub = query["query_args"].get("ds_sub", None)
query_format = query["query_args"].get("format", "")
pqb = PostgresQueryBuilder(query, qual_ds_name)
if self.profile == DataStore.DS_PROFILE.RESOURCES and not query_ds_sub:
table_alias = qual_ds_name if query_format != "complex" else "base"
pqb.where = self._add_access_filter(access_args, qual_ds_name, pqb.where, pqb.values,
add_where=False, tablealias=table_alias)
if self.profile == DataStore.DS_PROFILE.RESOURCES:
pqb.where = self._add_deleted_filter(pqb.table_aliases[0], query_ds_sub,
pqb.where, pqb.values,
show_all=query["query_args"].get("show_all", False))
with self.pool.cursor(**self.cursor_args) as cur:
exec_query = pqb.get_query()
cur.execute(exec_query, pqb.get_values())
rows = cur.fetchall()
log.info("find_by_query() QUERY: %s (%s rows)", cur.query, cur.rowcount)
query_res = {}
query["_result"] = query_res
query_res["statement_gen"] = exec_query
query_res["statement_sql"] = cur.query
query_res["rowcount"] = cur.rowcount
id_only = query["query_args"].get("id_only", True)
if query_format == "complex" and pqb.has_basic_cols:
# Return format is list of lists
if id_only:
res_vals = [[self._prep_id(row[0])] + list(row[1:]) for row in rows]
else:
res_vals = [[self._persistence_dict_to_ion_object(row[1])] + list(rows[2:]) for row in rows]
elif query_format == "complex":
res_vals = [list(row) for row in rows]
else:
if id_only:
res_vals = [self._prep_id(row[0]) for row in rows]
else:
res_vals = [self._persistence_dict_to_ion_object(row[-1]) for row in rows]
return res_vals
# -------------------------------------------------------------------------
# Internal operations
def _ion_object_to_persistence_dict(self, ion_object):
if ion_object is None:
return None
obj_dict = self._io_serializer.serialize(ion_object, update_version=True)
return obj_dict
def _persistence_dict_to_ion_object(self, obj_dict):
if obj_dict is None:
return None
ion_object = self._io_deserializer.deserialize(obj_dict)
return ion_object
def test_perf(self):
_io_serializer = IonObjectSerializer()
_io_deserializer = IonObjectDeserializer(
obj_registry=get_obj_registry())
def time_serialize(test_obj, name="?", has_ion=False):
with time_it(name + ", serialize"):
os = _io_serializer.serialize(test_obj)
with time_it(name + ", deserialize"):
os2 = _io_deserializer.deserialize(os)
count_objs(os)
if has_ion:
test_obj = os
with time_it(name + ", json.dumps"):
oj = json.dumps(test_obj)
with time_it(name + ", json.loads"):
o2 = json.loads(oj)
log.info(" len(json): %s", len(oj))
with time_it(name + ", simplejson.dumps"):
oj = simplejson.dumps(test_obj)
with time_it(name + ", simplejson.loads"):
o2 = simplejson.loads(oj)
log.info(" len(simplejson): %s", len(oj))
with time_it(name + ", msgpack.packb"):
o1 = msgpack.packb(test_obj)
with time_it(name + ", msgpack.unpackb"):
o2 = msgpack.unpackb(o1, use_list=1)
log.info(" len(msgpack): %s", len(o1))
# with time_it(name + ", pickle.dumps"):
# op = pickle.dumps(test_obj)
#
# with time_it(name + ", pickle.loads"):
# o2 = pickle.loads(op)
# log.info(" len(pickle): %s", len(op))
#
# with time_it(name + ", cPickle.dumps"):
# op = cPickle.dumps(test_obj)
#
# with time_it(name + ", cPickle.loads"):
# o2 = cPickle.loads(op)
# log.info(" len(cPickle): %s", len(op))
log.info("----------------")
# Large nested
with time_it("highly nested dict/list, create"):
test_obj = create_test_object(4,
4,
do_list=False,
uvals=True,
ukeys=True)
time_serialize(test_obj, "highly nested dict/list")
# Nested
with time_it("nested dict/list, create"):
test_obj = create_test_object(3,
40,
do_list=True,
uvals=False,
ukeys=False)
time_serialize(test_obj, "nested dict/list")
# Large string
#value = ''.join(random.choice(allowed_chars) for x in xrange(1460000))
value = ''.join(random.choice(allowed_chars) for x in xrange(500000))
time_serialize(value, "long string")
# ION
with time_it("create ion"):
test_obj1 = create_test_object(2,
200,
do_ion=True,
do_list=False,
do_dict=True,
obj_validate=False)
count_objs(test_obj1)
time_serialize(test_obj1, "dict of ion nested", has_ion=True)
from pyon.core.interceptor.interceptor import Invocation
from pyon.core.interceptor.encode import EncodeInterceptor
encode = EncodeInterceptor()
invocation = Invocation()
invocation.message = test_obj1
with time_it("ion object, encode"):
encode.outgoing(invocation)
with time_it("ion object, decode"):
encode.incoming(invocation)
count_objs(invocation.message)
# ION
with time_it("create ion unicode"):
test_obj1 = create_test_object(2,
200,
do_ion=True,
do_list=False,
do_dict=True,
obj_validate=False,
uvals=True,
ukeys=True)
count_objs(test_obj1)
time_serialize(test_obj1, "dict of ion nested unicode", has_ion=True)
# Create objects with validation on
with time_it("create ion calidated"):
test_obj1 = create_test_object(2,
200,
do_ion=True,
do_list=False,
do_dict=True,
obj_validate=True)
count_objs(test_obj1)
time_serialize(test_obj1, "dict of ion nested validated", has_ion=True)
def __init__(self):
Interceptor.__init__(self)
self._io_serializer = IonObjectSerializer()
self._io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
class FileDataStore(object):
def __init__(self, container, datastore_name=""):
self.container = container
self.datastore_name = datastore_name
# Object serialization/deserialization
self._io_serializer = IonObjectSerializer()
self._io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
def start(self):
if self.container.has_capability(self.container.CCAP.FILE_SYSTEM):
self.datastore_dir = FileSystem.get_url(FS.FILESTORE, self.datastore_name)
else:
self.datastore_dir = "./tmp/%s" % self.datastore_name
def stop(self):
pass
def _get_filename(self, object_id):
return "%s/%s" % (self.datastore_dir, object_id)
def create(self, obj, object_id=None, attachments=None, datastore_name=""):
"""
Converts ion objects to python dictionary before persisting them using the optional
suggested identifier and creates attachments to the object.
Returns an identifier and revision number of the object
"""
if not isinstance(obj, IonObjectBase):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.create_doc(self._ion_object_to_persistence_dict(obj),
object_id=object_id, datastore_name=datastore_name,
attachments=attachments)
def create_doc(self, doc, object_id=None, attachments=None, datastore_name=""):
"""
Persists the document using the optionally suggested doc_id, and creates attachments to it.
Returns the identifier and version number of the document
"""
if '_id' in doc:
raise BadRequest("Doc must not have '_id'")
# Assign an id to doc (recommended in CouchDB documentation)
doc["_id"] = object_id or uuid4().hex
log.debug('Creating new object %s/%s' % (datastore_name, doc["_id"]))
log.debug('create doc contents: %s', doc)
filename = self._get_filename(doc["_id"])
doc_json = json.dumps(doc)
with open(filename, "w") as f:
f.write(doc_json)
return doc["_id"], 1
def update(self, obj, datastore_name=""):
if not isinstance(obj, IonObjectBase):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.update_doc(self._ion_object_to_persistence_dict(obj))
def update_doc(self, doc, datastore_name=""):
if '_id' not in doc:
raise BadRequest("Doc must have '_id'")
log.debug('update doc contents: %s', doc)
filename = self._get_filename(doc["_id"])
doc_json = json.dumps(doc)
with open(filename, "w") as f:
f.write(doc_json)
return doc["_id"], 2
def read(self, object_id, rev_id="", datastore_name=""):
if not isinstance(object_id, str):
raise BadRequest("Object id param is not string")
doc = self.read_doc(object_id, rev_id, datastore_name)
# Convert doc into Ion object
obj = self._persistence_dict_to_ion_object(doc)
log.debug('Ion object: %s', str(obj))
return obj
def read_doc(self, doc_id, rev_id="", datastore_name=""):
log.debug('Reading head version of object %s/%s', datastore_name, doc_id)
filename = self._get_filename(doc_id)
doc = None
with open(filename, "r") as f:
doc_json = f.read()
doc = json.loads(doc_json)
if doc is None:
raise NotFound('Object with id %s does not exist.' % str(doc_id))
log.debug('read doc contents: %s', doc)
return doc
def delete(self, obj, datastore_name="", del_associations=False):
if not isinstance(obj, IonObjectBase) and not isinstance(obj, str):
raise BadRequest("Obj param is not instance of IonObjectBase or string id")
if type(obj) is str:
self.delete_doc(obj, datastore_name=datastore_name, del_associations=del_associations)
else:
if '_id' not in obj:
raise BadRequest("Doc must have '_id'")
self.delete_doc(self._ion_object_to_persistence_dict(obj), datastore_name=datastore_name, del_associations=del_associations)
def delete_doc(self, doc, datastore_name="", del_associations=False):
doc_id = doc if type(doc) is str else doc["_id"]
log.debug('Deleting object %s/%s', datastore_name, doc_id)
filename = self._get_filename(doc_id)
try:
os.remove(filename)
except OSError:
raise NotFound('Object with id %s does not exist.' % doc_id)
def _ion_object_to_persistence_dict(self, ion_object):
if ion_object is None: return None
obj_dict = self._io_serializer.serialize(ion_object)
return obj_dict
def _persistence_dict_to_ion_object(self, obj_dict):
if obj_dict is None: return None
ion_object = self._io_deserializer.deserialize(obj_dict)
return ion_object
class MockDB_DataStore(DataStore):
"""
Data store implementation utilizing in-memory dict of dicts
to persist documents.
"""
def __init__(self, datastore_name='prototype'):
self.datastore_name = datastore_name
log.debug('Creating in-memory dict of dicts that will simulate data stores')
self.root = {}
# serializers
self._io_serializer = IonObjectSerializer()
self._io_deserializer = IonObjectDeserializer(obj_registry=obj_registry)
def create_datastore(self, datastore_name="", create_indexes=True):
if not datastore_name:
datastore_name = self.datastore_name
log.info('Creating data store %s' % datastore_name)
if self.datastore_exists(datastore_name):
raise BadRequest("Data store with name %s already exists" % datastore_name)
if datastore_name not in self.root:
self.root[datastore_name] = {}
def delete_datastore(self, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
log.info('Deleting data store %s' % datastore_name)
if datastore_name in self.root:
del self.root[datastore_name]
else:
log.info('Data store %s does not exist' % datastore_name)
def list_datastores(self):
log.debug('Listing all data stores')
dsList = self.root.keys()
log.debug('Data stores: %s' % str(dsList))
return dsList
def info_datastore(self, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
log.debug('Listing information about data store %s' % datastore_name)
if datastore_name in self.root:
info = 'Data store exists'
else:
raise BadRequest("Data store with name %s does not exist" % datastore_name)
log.debug('Data store info: %s' % str(info))
return info
def datastore_exists(self, datastore_name=""):
return datastore_name in self.root
def list_objects(self, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
log.debug('Listing all objects in data store %s' % datastore_name)
objs = []
for key, value in self.root[datastore_name].items():
if key.find('_version_counter') == -1 and key.find('_version_') == -1:
objs.append(key)
log.debug('Objects: %s' % str(objs))
return objs
def list_object_revisions(self, object_id, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
log.debug('Listing all versions of object %s/%s' % (datastore_name, str(object_id)))
res = []
for key, value in self.root[datastore_name].items():
if (key.find('_version_counter') == -1
and (key.find(object_id + '_version_') == 0)):
res.append(key)
log.debug('Versions: %s' % str(res))
return res
def create(self, obj, object_id=None, datastore_name=""):
if not isinstance(obj, IonObjectBase):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.create_doc(self._ion_object_to_persistence_dict(obj),
object_id=object_id, datastore_name=datastore_name)
def create_doc(self, doc, object_id=None, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
if '_id' in doc:
raise BadRequest("Doc must not have '_id'")
if '_rev' in doc:
raise BadRequest("Doc must not have '_rev'")
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
if object_id:
if object_id in datastore_dict:
raise BadRequest("Object with id %s already exist" % object_id)
# Assign an id to doc
doc["_id"] = object_id or uuid4().hex
object_id = doc["_id"]
log.debug('Creating new object %s/%s' % (datastore_name, object_id))
# Create key for version counter entry. Will be used
# on update to increment version number easily.
versionCounterKey = '__' + object_id + '_version_counter'
versionCounter = 1
# Assign initial version to doc
doc["_rev"] = str(versionCounter)
# Write HEAD, version and version counter dicts
datastore_dict[object_id] = doc
datastore_dict[versionCounterKey] = versionCounter
datastore_dict[object_id + '_version_' + str(versionCounter)] = doc
# Return list that identifies the id of the new doc and its version
res = [object_id, str(versionCounter)]
log.debug('Create result: %s' % str(res))
return res
def create_mult(self, objects, object_ids=None):
if any([not isinstance(obj, IonObjectBase) for obj in objects]):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.create_doc_mult([self._ion_object_to_persistence_dict(obj) for obj in objects],
object_ids)
def create_doc_mult(self, docs, object_ids=None):
if any(["_id" in doc for doc in docs]):
raise BadRequest("Docs must not have '_id'")
if any(["_rev" in doc for doc in docs]):
raise BadRequest("Docs must not have '_rev'")
if object_ids and len(object_ids) != len(docs):
raise BadRequest("Invalid object_ids")
# Assign an id to doc (recommended in CouchDB documentation)
object_ids = object_ids or [uuid4().hex for i in xrange(len(docs))]
res = []
for doc, oid in zip(docs, object_ids):
oid,rev = self.create_doc(doc, oid)
res.append((True,oid,rev))
return res
def read(self, object_id, rev_id="", datastore_name=""):
if not isinstance(object_id, str):
raise BadRequest("Object id param is not string")
doc = self.read_doc(object_id, rev_id, datastore_name)
# Convert doc into Ion object
obj = self._persistence_dict_to_ion_object(doc)
log.debug('Ion object: %s' % str(obj))
return obj
def read_doc(self, object_id, rev_id="", datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
try:
key = object_id
if rev_id == "":
log.debug('Reading head version of object %s/%s' % (datastore_name, str(object_id)))
else:
log.debug('Reading version %s of object %s/%s' % (str(rev_id), datastore_name, str(object_id)))
key += '_version_' + str(rev_id)
doc = datastore_dict[key]
except KeyError:
raise NotFound('Object with id %s does not exist.' % str(object_id))
log.debug('Read result: %s' % str(doc))
return doc
def read_mult(self, object_ids, datastore_name=""):
if any([not isinstance(object_id, str) for object_id in object_ids]):
raise BadRequest("Object id param is not string")
docs = self.read_doc_mult(object_ids, datastore_name)
# Convert docs into Ion objects
obj_list = [self._persistence_dict_to_ion_object(doc) for doc in docs]
return obj_list
def read_doc_mult(self, object_ids, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
doc_list = []
try:
for object_id in object_ids:
log.debug('Reading head version of object %s/%s' % (datastore_name, str(object_id)))
doc = datastore_dict[object_id]
doc_list.append(doc.copy())
except KeyError:
raise NotFound('Object with id %s does not exist.' % str(object_id))
return doc_list
def update(self, obj, datastore_name=""):
if not isinstance(obj, IonObjectBase):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.update_doc(self._ion_object_to_persistence_dict(obj))
def update_doc(self, doc, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
if '_id' not in doc:
raise BadRequest("Doc must have '_id'")
if '_rev' not in doc:
raise BadRequest("Doc must have '_rev'")
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
try:
object_id = doc["_id"]
# Find the next doc version
versionCounterKey = '__' + object_id + '_version_counter'
baseVersion = doc["_rev"]
versionCounter = datastore_dict[versionCounterKey] + 1
if baseVersion != str(versionCounter - 1):
raise Conflict('Object not based on most current version')
except KeyError:
raise BadRequest("Object missing required _id and/or _rev values")
log.debug('Saving new version of object %s/%s' % (datastore_name, doc["_id"]))
doc["_rev"] = str(versionCounter)
# Overwrite HEAD and version counter dicts, add new version dict
datastore_dict[object_id] = doc
datastore_dict[versionCounterKey] = versionCounter
datastore_dict[object_id + '_version_' + str(versionCounter)] = doc
res = [object_id, str(versionCounter)]
log.debug('Update result: %s' % str(res))
return res
def delete(self, obj, datastore_name=""):
if not isinstance(obj, IonObjectBase) and not isinstance(obj, str):
raise BadRequest("Obj param is not instance of IonObjectBase or string id")
if type(obj) is str:
return self.delete_doc(obj, datastore_name=datastore_name)
return self.delete_doc(self._ion_object_to_persistence_dict(obj), datastore_name=datastore_name)
def delete_doc(self, doc, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
if type(doc) is str:
object_id = doc
else:
object_id = doc["_id"]
log.info('Deleting object %s/%s' % (datastore_name, object_id))
if object_id in datastore_dict.keys():
if self._is_in_association(object_id, datastore_name):
obj = self.read(object_id, datastore_name)
log.warn("XXXXXXX Attempt to delete object %s that still has associations" % str(obj))
# raise BadRequest("Object cannot be deleted until associations are broken")
# Find all version dicts and delete them
for key in datastore_dict.keys():
if key.find(object_id + '_version_') == 0:
del datastore_dict[key]
# Delete the HEAD dict
del datastore_dict[object_id]
# Delete the version counter dict
del datastore_dict['__' + object_id + '_version_counter']
else:
raise NotFound('Object ' + object_id + ' does not exist.')
log.info('Delete result: True')
def find(self, criteria=[], datastore_name=""):
docList = self.find_doc(criteria, datastore_name)
results = []
# Convert each returned doc to its associated Ion object
for doc in docList:
obj = self._persistence_dict_to_ion_object(doc)
log.debug('Ion object: %s' % str(obj))
results.append(obj)
return results
def find_doc(self, criteria=[], datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
results = []
log_string = "Searching for objects matching criteria list: " + str(criteria)
log.debug(log_string)
# Traverse entire data store, checking each HEAD version for equality
# with specified criterion
for obj_id in self.list_objects(datastore_name):
try:
doc = self.read_doc(obj_id, rev_id="", datastore_name=datastore_name)
log.debug("Doc: %s" % str(doc))
if len(criteria) == 0:
results.append(doc)
else:
criteria_satisfied = False
for criterion in criteria:
if isinstance(criterion, list):
if len(criterion) != 3:
raise BadRequest("Insufficient criterion values specified. Much match [<field>, <logical constant>, <value>]")
for item in criterion:
if not isinstance(item, str):
raise BadRequest("All criterion values must be strings")
key = criterion[0]
logical_operation = criterion[1]
value = criterion[2]
if key in doc:
if logical_operation == DataStore.EQUAL:
if doc[key] == value:
criteria_satisfied = True
else:
criteria_satisfied = False
elif logical_operation == DataStore.NOT_EQUAL:
if doc[key] != value:
criteria_satisfied = True
else:
criteria_satisfied = False
elif logical_operation == DataStore.GREATER_THAN:
if doc[key] > value:
criteria_satisfied = True
else:
criteria_satisfied = False
elif logical_operation == DataStore.GREATER_THAN_OR_EQUAL:
if doc[key] >= value:
criteria_satisfied = True
else:
criteria_satisfied = False
elif logical_operation == DataStore.LESS_THAN:
if doc[key] < value:
criteria_satisfied = True
else:
criteria_satisfied = False
elif logical_operation == DataStore.LESS_THAN_OR_EQUAL:
if doc[key] <= value:
criteria_satisfied = True
else:
criteria_satisfied = False
else:
if criterion == DataStore.AND:
# Can shortcut the query at this point if the
# previous criterion failed
if criteria_satisfied == False:
break
if criteria_satisfied:
results.append(doc)
except KeyError:
pass
log.debug('Find results: %s' % str(results))
if len(results) == 0:
raise NotFound('No objects matched criteria %s' % criteria)
return results
def find_by_idref(self, criteria=[], association="", datastore_name=""):
doc_list = self.find_by_idref_doc(criteria, association, datastore_name)
results = []
# Convert each returned doc to its associated Ion object
for doc in doc_list:
obj = self._persistence_dict_to_ion_object(doc)
log.debug('Ion object: %s' % str(obj))
results.append(obj)
return results
def find_by_idref_doc(self, criteria=[], association="", datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
ids = []
log_string = "Searching for objects matching criteria list: " + str(criteria)
log.debug(log_string)
# Traverse entire data store, checking each HEAD version for equality
# with specified criterion
for obj_id in self.list_objects(datastore_name):
try:
doc = self.read_doc(obj_id, rev_id="", datastore_name=datastore_name)
log.debug("Doc: %s" % str(doc))
if len(criteria) == 0:
if association in doc:
for id in doc[association]:
ids.append(id)
else:
criteria_satisfied = False
for criterion in criteria:
if isinstance(criterion, list):
key = criterion[0]
logical_operation = criterion[1]
value = criterion[2]
if key in doc:
if logical_operation == DataStore.EQUAL:
if doc[key] == value:
criteria_satisfied = True
else:
criteria_satisfied = False
elif logical_operation == DataStore.NOT_EQUAL:
if doc[key] != value:
criteria_satisfied = True
else:
criteria_satisfied = False
elif logical_operation == DataStore.GREATER_THAN:
if doc[key] > value:
criteria_satisfied = True
else:
criteria_satisfied = False
elif logical_operation == DataStore.GREATER_THAN_OR_EQUAL:
if doc[key] >= value:
criteria_satisfied = True
else:
criteria_satisfied = False
elif logical_operation == DataStore.LESS_THAN:
if doc[key] < value:
criteria_satisfied = True
else:
criteria_satisfied = False
elif logical_operation == DataStore.LESS_THAN_OR_EQUAL:
if doc[key] <= value:
criteria_satisfied = True
else:
criteria_satisfied = False
else:
if criterion == DataStore.AND:
# Can shortcut the query at this point if the
# previous criterion failed
if criteria_satisfied == False:
break
if criteria_satisfied:
if association in doc:
for id in doc[association]:
ids.append(id)
except KeyError:
pass
results = []
for id in ids:
doc = self.read_doc(id, "", datastore_name)
results.append(doc)
log.debug('Find results: %s' % str(results))
if len(results) == 0:
raise NotFound('No objects matched criteria %s' % criteria)
return results
def resolve_idref(self, subject="", predicate="", obj="", datastore_name=""):
res_list = self.resolve_idref_doc(subject, predicate, obj, datastore_name)
results = []
# Convert each returned doc to its associated Ion object
for item in res_list:
subject_dict = item[0]
object_dict = item[2]
subject = self._persistence_dict_to_ion_object(subject_dict)
log.debug('Subject Ion object: %s' % str(subject))
obj = self._persistence_dict_to_ion_object(object_dict)
log.debug('Object Ion object: %s' % str(obj))
results.append([subject, item[1], obj])
return results
def resolve_idref_doc(self, subject="", predicate="", obj="", datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
if subject == "":
if predicate == "":
if obj == "":
# throw exception
raise BadRequest("Data store query does not specify subject, predicate or object")
else:
# Find all subjects with any association to object
object_doc = self.read_doc(obj, "", datastore_name)
res = []
all_doc_ids = self.list_objects(datastore_name)
for subject_doc_id in all_doc_ids:
if subject_doc_id == obj:
continue
subject_doc = self.read_doc(subject_doc_id, "", datastore_name)
for key in subject_doc:
if isinstance(subject_doc[key], list):
if obj in subject_doc[key]:
res.append([subject_doc, key, object_doc])
else:
if obj == subject_doc[key]:
res.append([subject_doc, key, object_doc])
if len(res) == 0:
raise NotFound("Data store query for association %s/%s/%s failed" % (subject, predicate, obj))
else:
return res
else:
# Find all subjects with association to object
object_doc = self.read_doc(obj, "", datastore_name)
res = []
all_doc_ids = self.list_objects(datastore_name)
for subject_doc_id in all_doc_ids:
if subject_doc_id == obj:
continue
subject_doc = self.read_doc(subject_doc_id, "", datastore_name)
if predicate in subject_doc:
if obj in subject_doc[predicate]:
res.append([subject_doc, predicate, object_doc])
if len(res) == 0:
raise NotFound("Data store query for association %s/%s/%s failed" % (subject, predicate, obj))
else:
return res
else:
if predicate == "":
if obj == "":
# Find all objects with any association to subject
# TODO would need some way to indicate a key is an association predicate
pass
else:
# Find all associations between subject and object
subject_doc = self.read_doc(subject, "", datastore_name)
object_doc = self.read_doc(obj, "", datastore_name)
res = []
for key in subject_doc:
if isinstance(subject_doc[key], list):
if obj in subject_doc[key]:
res.append([subject_doc, key, object_doc])
else:
if obj == subject_doc[key]:
res.append([subject_doc, key, object_doc])
if len(res) == 0:
raise NotFound("Data store query for association %s/%s/%s failed" % (subject, predicate, obj))
else:
return res
else:
if obj == "":
# Find all associated objects
subject_doc = self.read_doc(subject, "", datastore_name)
res = []
if predicate in subject_doc:
for id in subject_doc[predicate]:
object_doc = self.read_doc(id, "", datastore_name)
res.append([subject_doc, predicate, object_doc])
return res
raise NotFound("Data store query for association %s/%s/%s failed" % (subject, predicate, obj))
else:
# Determine if association exists
subject_doc = self.read_doc(subject, "", datastore_name)
object_doc = self.read_doc(obj, "", datastore_name)
if predicate in subject_doc:
if obj in subject_doc[predicate]:
return [[subject_doc, predicate, object_doc]]
raise NotFound("Data store query for association %s/%s/%s failed" % (subject, predicate, obj))
def _is_in_association(self, obj_id, datastore_name=""):
log.debug("_is_in_association(%s)" % obj_id)
if not obj_id:
raise BadRequest("Must provide object id")
if not datastore_name:
datastore_name = self.datastore_name
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
for objname,obj in datastore_dict.iteritems():
if (objname.find('_version_')>0) or (not type(obj) is dict): continue
if 'type_' in obj and obj['type_'] == "Association":
association = obj
if association["s"] == obj_id or association["o"] == obj_id:
log.debug("association found(%s)" % association)
return True
return False
def find_objects(self, subject, predicate=None, object_type=None, id_only=False):
log.debug("find_objects(subject=%s, predicate=%s, object_type=%s, id_only=%s" % (subject, predicate, object_type, id_only))
if not subject:
raise BadRequest("Must provide subject")
try:
datastore_dict = self.root[self.datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
if type(subject) is str:
subject_id = subject
else:
if "_id" not in subject:
raise BadRequest("Object id not available in subject")
else:
subject_id = subject._id
assoc_list = []
target_id_list = []
target_list = []
for objname,obj in datastore_dict.iteritems():
if (objname.find('_version_')>0) or (not type(obj) is dict): continue
if 'type_' in obj and obj['type_'] == "Association":
if obj['s'] == subject_id:
if predicate and obj['p'] == predicate:
if (object_type and obj['ot'] == object_type) or not object_type:
assoc_list.append(obj)
target_id_list.append(obj['o'])
target_list.append(self.read(obj['o']))
elif not predicate:
assoc_list.append(obj)
target_id_list.append(obj['o'])
target_list.append(self.read(obj['o']))
log.debug("find_objects() found %s objects" % (len(target_list)))
if id_only:
return (target_id_list, assoc_list)
else:
return (target_list, assoc_list)
def find_subjects(self, subject_type=None, predicate=None, obj=None, id_only=False):
log.debug("find_subjects(subject_type=%s, predicate=%s, object=%s, id_only=%s" % (subject_type, predicate, obj, id_only))
if not obj:
raise BadRequest("Must provide object")
try:
datastore_dict = self.root[self.datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
if type(obj) is str:
object_id = obj
else:
if "_id" not in obj:
raise BadRequest("Object id not available in object")
else:
object_id = obj._id
assoc_list = []
target_id_list = []
target_list = []
for objname,obj in datastore_dict.iteritems():
if (objname.find('_version_')>0) or (not type(obj) is dict): continue
if 'type_' in obj and obj['type_'] == "Association":
if obj['o'] == object_id:
if predicate and obj['p'] == predicate:
if (subject_type and obj['st'] == subject_type) or not subject_type:
assoc_list.append(obj)
target_id_list.append(obj['s'])
target_list.append(self.read(obj['s']))
elif not predicate:
assoc_list.append(obj)
target_id_list.append(obj['s'])
target_list.append(self.read(obj['s']))
log.debug("find_subjects() found %s subjects" % (len(target_list)))
if id_only:
return (target_id_list, assoc_list)
else:
return (target_list, assoc_list)
def find_associations(self, subject=None, predicate=None, obj=None, id_only=True):
log.debug("find_associations(subject=%s, predicate=%s, object=%s)" % (subject, predicate, obj))
if subject and obj or predicate:
pass
else:
raise BadRequest("Illegal parameters")
try:
datastore_dict = self.root[self.datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
if subject and obj:
if type(subject) is str:
subject_id = subject
else:
if "_id" not in subject:
raise BadRequest("Object id not available")
else:
subject_id = subject._id
if type(obj) is str:
object_id = obj
else:
if "_id" not in obj:
raise BadRequest("Object id not available in object")
else:
object_id = obj._id
target_list = []
for objname,obj in datastore_dict.iteritems():
if (objname.find('_version_')>0) or (not type(obj) is dict): continue
if 'type_' in obj and obj['type_'] == "Association":
if obj['s'] == subject_id and obj['o'] == object_id:
target_list.append(obj)
else:
target_list = []
for objname,obj in datastore_dict.iteritems():
if (objname.find('_version_')>0) or (not type(obj) is dict): continue
if 'type_' in obj and obj['type_'] == "Association":
if obj['p'] == predicate:
target_list.append(obj)
if id_only:
assocs = [row['_id'] for row in target_list]
else:
assocs = [self._persistence_dict_to_ion_object(row) for row in target_list]
log.debug("find_associations() found %s associations" % (len(assocs)))
return assocs
def find_res_by_type(self, restype, lcstate=None, id_only=False):
log.debug("find_res_by_type(restype=%s, lcstate=%s)" % (restype, lcstate))
try:
datastore_dict = self.root[self.datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
assoc_list = []
target_id_list = []
target_list = []
for objname,obj in datastore_dict.iteritems():
if (objname.find('_version_')>0) or (not type(obj) is dict): continue
if 'type_' in obj and (obj['type_'] == restype or (not restype and obj['type_'] != "Association")):
if (lcstate and 'lcstate' in obj and obj['lcstate'] == lcstate) or not lcstate or not restype:
target_id_list.append(obj['_id'])
target_list.append(self._persistence_dict_to_ion_object(obj))
assoc_list.append([])
log.debug("find_res_by_type() found %s resources" % (len(target_list)))
if id_only:
return (target_id_list, assoc_list)
else:
return (target_list, assoc_list)
def find_res_by_lcstate(self, lcstate, restype=None, id_only=False):
log.debug("find_res_by_type(lcstate=%s, restype=%s)" % (lcstate, restype))
try:
datastore_dict = self.root[self.datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
if lcstate in ResourceLifeCycleSM.STATE_ALIASES:
lcstate_match = ResourceLifeCycleSM.STATE_ALIASES[lcstate]
else:
lcstate_match = [lcstate]
assoc_list = []
target_id_list = []
target_list = []
for objname,obj in datastore_dict.iteritems():
if (objname.find('_version_')>0) or (not type(obj) is dict): continue
if 'lcstate' in obj and obj['lcstate'] in lcstate_match:
if (restype and obj['type_'] == restype) or not restype:
target_id_list.append(obj['_id'])
target_list.append(self._persistence_dict_to_ion_object(obj))
assoc_list.append([])
log.debug("find_res_by_lcstate() found %s resources" % (len(target_list)))
if id_only:
return (target_id_list, assoc_list)
else:
return (target_list, assoc_list)
def _pass(self):
pass
def find_res_by_name(self, name, restype=None, id_only=False):
log.debug("find_res_by_name(name=%s, restype=%s)" % (name, restype))
try:
datastore_dict = self.root[self.datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
assoc_list = []
target_id_list = []
target_list = []
for objname,obj in datastore_dict.iteritems():
if (objname.find('_version_')>0) or (not type(obj) is dict): continue
if 'name' in obj and obj['name'] == name:
if (restype and obj['type_'] == restype) or not restype:
target_id_list.append(obj['_id'])
target_list.append(self._persistence_dict_to_ion_object(obj))
assoc_list.append([])
log.debug("find_res_by_name() found %s resources" % (len(target_list)))
if id_only:
return (target_id_list, assoc_list)
else:
return (target_list, assoc_list)
def find_dir_entries(self, qname):
raise NotImplementedError()
def _ion_object_to_persistence_dict(self, ion_object):
if ion_object is None: return None
obj_dict = self._io_serializer.serialize(ion_object)
return obj_dict
def _persistence_dict_to_ion_object(self, obj_dict):
if obj_dict is None: return None
ion_object = self._io_deserializer.deserialize(obj_dict)
return ion_object
class MockDB_DataStore(DataStore):
"""
Data store implementation utilizing in-memory dict of dicts
to persist documents.
"""
def __init__(self, datastore_name='prototype'):
self.datastore_name = datastore_name
log.debug('Creating in-memory dict of dicts that will simulate data stores')
self.root = {}
# serializers
self._io_serializer = IonObjectSerializer()
self._io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
def create_datastore(self, datastore_name="", create_indexes=True):
if not datastore_name:
datastore_name = self.datastore_name
log.info('Creating data store %s' % datastore_name)
if self.datastore_exists(datastore_name):
raise BadRequest("Data store with name %s already exists" % datastore_name)
if datastore_name not in self.root:
self.root[datastore_name] = {}
def delete_datastore(self, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
log.info('Deleting data store %s' % datastore_name)
if datastore_name in self.root:
del self.root[datastore_name]
else:
log.info('Data store %s does not exist' % datastore_name)
def list_datastores(self):
log.debug('Listing all data stores')
dsList = self.root.keys()
log.debug('Data stores: %s' % str(dsList))
return dsList
def info_datastore(self, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
log.debug('Listing information about data store %s' % datastore_name)
if datastore_name in self.root:
info = 'Data store exists'
else:
raise BadRequest("Data store with name %s does not exist" % datastore_name)
log.debug('Data store info: %s' % str(info))
return info
def datastore_exists(self, datastore_name=""):
return datastore_name in self.root
def list_objects(self, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
log.debug('Listing all objects in data store %s' % datastore_name)
objs = []
for key, value in self.root[datastore_name].items():
if key.find('_version_counter') == -1 and key.find('_version_') == -1:
objs.append(key)
log.debug('Objects: %s' % str(objs))
return objs
def list_object_revisions(self, object_id, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
log.debug('Listing all versions of object %s/%s' % (datastore_name, str(object_id)))
res = []
for key, value in self.root[datastore_name].items():
if (key.find('_version_counter') == -1
and (key.find(object_id + '_version_') == 0)):
res.append(key)
log.debug('Versions: %s' % str(res))
return res
def create(self, obj, object_id=None, datastore_name=""):
if not isinstance(obj, IonObjectBase):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.create_doc(self._ion_object_to_persistence_dict(obj),
object_id=object_id, datastore_name=datastore_name)
def create_doc(self, doc, object_id=None, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
if '_id' in doc:
raise BadRequest("Doc must not have '_id'")
if '_rev' in doc:
raise BadRequest("Doc must not have '_rev'")
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
if object_id:
if object_id in datastore_dict:
raise BadRequest("Object with id %s already exist" % object_id)
# Assign an id to doc
doc["_id"] = object_id or uuid4().hex
object_id = doc["_id"]
log.debug('Creating new object %s/%s' % (datastore_name, object_id))
# Create key for version counter entry. Will be used
# on update to increment version number easily.
version_counter_key = '__' + object_id + '_version_counter'
version_counter = 1
# Assign initial version to doc
doc["_rev"] = str(version_counter)
# Write HEAD, version and version counter dicts
datastore_dict[object_id] = doc
datastore_dict[version_counter_key] = version_counter
datastore_dict[object_id + '_version_' + str(version_counter)] = doc
# Return list that identifies the id of the new doc and its version
res = [object_id, str(version_counter)]
log.debug('Create result: %s' % str(res))
return res
def create_mult(self, objects, object_ids=None):
if any([not isinstance(obj, IonObjectBase) for obj in objects]):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.create_doc_mult([self._ion_object_to_persistence_dict(obj) for obj in objects],
object_ids)
def create_doc_mult(self, docs, object_ids=None):
if any(["_id" in doc for doc in docs]):
raise BadRequest("Docs must not have '_id'")
if any(["_rev" in doc for doc in docs]):
raise BadRequest("Docs must not have '_rev'")
if object_ids and len(object_ids) != len(docs):
raise BadRequest("Invalid object_ids")
# Assign an id to doc (recommended in CouchDB documentation)
object_ids = object_ids or [uuid4().hex for i in xrange(len(docs))]
res = []
for doc, oid in zip(docs, object_ids):
oid,rev = self.create_doc(doc, oid)
res.append((True,oid,rev))
return res
def read(self, object_id, rev_id="", datastore_name=""):
if not isinstance(object_id, str):
raise BadRequest("Object id param is not string")
doc = self.read_doc(object_id, rev_id, datastore_name)
# Convert doc into Ion object
obj = self._persistence_dict_to_ion_object(doc)
log.debug('Ion object: %s' % str(obj))
return obj
def read_doc(self, object_id, rev_id="", datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
try:
key = object_id
if rev_id != None and rev_id != "":
log.debug('Reading version %s of object %s/%s' % (str(rev_id), datastore_name, str(object_id)))
key += '_version_' + str(rev_id)
else:
log.debug('Reading head version of object %s/%s' % (datastore_name, str(object_id)))
doc = datastore_dict[key]
except KeyError:
raise NotFound('Object with id %s does not exist.' % str(object_id))
log.debug('Read result: %s' % str(doc))
return doc
def read_mult(self, object_ids, datastore_name=""):
if any([not isinstance(object_id, str) for object_id in object_ids]):
raise BadRequest("Object id param is not string")
docs = self.read_doc_mult(object_ids, datastore_name)
# Convert docs into Ion objects
obj_list = [self._persistence_dict_to_ion_object(doc) for doc in docs]
return obj_list
def read_doc_mult(self, object_ids, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
doc_list = []
try:
for object_id in object_ids:
log.debug('Reading head version of object %s/%s' % (datastore_name, str(object_id)))
doc = datastore_dict[object_id]
doc_list.append(doc.copy())
except KeyError:
raise NotFound('Object with id %s does not exist.' % str(object_id))
return doc_list
def update(self, obj, datastore_name=""):
if not isinstance(obj, IonObjectBase):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.update_doc(self._ion_object_to_persistence_dict(obj))
def update_doc(self, doc, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
if '_id' not in doc:
raise BadRequest("Doc must have '_id'")
if '_rev' not in doc:
raise BadRequest("Doc must have '_rev'")
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
try:
object_id = doc["_id"]
# Find the next doc version
version_counter_key = '__' + object_id + '_version_counter'
baseVersion = doc["_rev"]
version_counter = datastore_dict[version_counter_key] + 1
if baseVersion != str(version_counter - 1):
raise Conflict('Object not based on most current version')
except KeyError:
raise BadRequest("Object missing required _id and/or _rev values")
log.debug('Saving new version of object %s/%s' % (datastore_name, doc["_id"]))
doc["_rev"] = str(version_counter)
# Overwrite HEAD and version counter dicts, add new version dict
datastore_dict[object_id] = doc
datastore_dict[version_counter_key] = version_counter
datastore_dict[object_id + '_version_' + str(version_counter)] = doc
res = [object_id, str(version_counter)]
log.debug('Update result: %s' % str(res))
return res
def delete(self, obj, datastore_name=""):
if not isinstance(obj, IonObjectBase) and not isinstance(obj, str):
raise BadRequest("Obj param is not instance of IonObjectBase or string id")
if type(obj) is str:
return self.delete_doc(obj, datastore_name=datastore_name)
return self.delete_doc(self._ion_object_to_persistence_dict(obj), datastore_name=datastore_name)
def delete_doc(self, doc, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
if type(doc) is str:
object_id = doc
else:
object_id = doc["_id"]
log.info('Deleting object %s/%s' % (datastore_name, object_id))
if object_id in datastore_dict.keys():
if self._is_in_association(object_id, datastore_name):
obj = self.read(object_id, "", datastore_name)
log.warn("XXXXXXX Attempt to delete object %s that still has associations" % str(obj))
# raise BadRequest("Object cannot be deleted until associations are broken")
# Find all version dicts and delete them
for key in datastore_dict.keys():
if key.find(object_id + '_version_') == 0:
del datastore_dict[key]
# Delete the HEAD dict
del datastore_dict[object_id]
# Delete the version counter dict
del datastore_dict['__' + object_id + '_version_counter']
else:
raise NotFound('Object with id ' + object_id + ' does not exist.')
log.info('Delete result: True')
def _is_in_association(self, obj_id, datastore_name=""):
log.debug("_is_in_association(%s)" % obj_id)
if not obj_id:
raise BadRequest("Must provide object id")
if not datastore_name:
datastore_name = self.datastore_name
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name + ' does not exist.')
for objname,obj in datastore_dict.iteritems():
if (objname.find('_version_')>0) or (not type(obj) is dict): continue
if 'type_' in obj and obj['type_'] == "Association":
association = obj
if association["s"] == obj_id or association["o"] == obj_id:
log.debug("association found(%s)" % association)
return True
return False
def find_objects(self, subject, predicate=None, object_type=None, id_only=False):
log.debug("find_objects(subject=%s, predicate=%s, object_type=%s, id_only=%s" % (subject, predicate, object_type, id_only))
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
if not subject:
raise BadRequest("Must provide subject")
try:
datastore_dict = self.root[self.datastore_name]
except KeyError:
raise BadRequest('Data store ' + self.datastore_name + ' does not exist.')
if type(subject) is str:
subject_id = subject
else:
if "_id" not in subject:
raise BadRequest("Object id not available in subject")
else:
subject_id = subject._id
assoc_list = []
target_id_list = []
target_list = []
for objname,obj in datastore_dict.iteritems():
if (objname.find('_version_')>0) or (not type(obj) is dict): continue
if 'type_' in obj and obj['type_'] == "Association":
if obj['s'] == subject_id:
if predicate and obj['p'] == predicate:
if (object_type and obj['ot'] == object_type) or not object_type:
assoc_list.append(obj)
target_id_list.append(obj['o'])
target_list.append(self.read(obj['o']))
elif not predicate:
assoc_list.append(obj)
target_id_list.append(obj['o'])
target_list.append(self.read(obj['o']))
log.debug("find_objects() found %s objects" % (len(target_list)))
if id_only:
return (target_id_list, assoc_list)
else:
return (target_list, assoc_list)
def find_subjects(self, subject_type=None, predicate=None, obj=None, id_only=False):
log.debug("find_subjects(subject_type=%s, predicate=%s, object=%s, id_only=%s" % (subject_type, predicate, obj, id_only))
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
if not obj:
raise BadRequest("Must provide object")
try:
datastore_dict = self.root[self.datastore_name]
except KeyError:
raise BadRequest('Data store ' + self.datastore_name + ' does not exist.')
if type(obj) is str:
object_id = obj
else:
if "_id" not in obj:
raise BadRequest("Object id not available in object")
else:
object_id = obj._id
assoc_list = []
target_id_list = []
target_list = []
for objname,obj in datastore_dict.iteritems():
if (objname.find('_version_')>0) or (not type(obj) is dict): continue
if 'type_' in obj and obj['type_'] == "Association":
if obj['o'] == object_id:
if predicate and obj['p'] == predicate:
if (subject_type and obj['st'] == subject_type) or not subject_type:
assoc_list.append(obj)
target_id_list.append(obj['s'])
target_list.append(self.read(obj['s']))
elif not predicate:
assoc_list.append(obj)
target_id_list.append(obj['s'])
target_list.append(self.read(obj['s']))
log.debug("find_subjects() found %s subjects" % (len(target_list)))
if id_only:
return (target_id_list, assoc_list)
else:
return (target_list, assoc_list)
def find_associations(self, subject=None, predicate=None, obj=None, assoc_type=None, id_only=True):
log.debug("find_associations(subject=%s, predicate=%s, object=%s, assoc_type=%s)" % (subject, predicate, obj, assoc_type))
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
if subject and obj or predicate:
pass
else:
raise BadRequest("Illegal parameters")
try:
datastore_dict = self.root[self.datastore_name]
except KeyError:
raise BadRequest('Data store ' + self.datastore_name + ' does not exist.')
if subject and obj:
if type(subject) is str:
subject_id = subject
else:
if "_id" not in subject:
raise BadRequest("Object id not available in subject")
else:
subject_id = subject._id
if type(obj) is str:
object_id = obj
else:
if "_id" not in obj:
raise BadRequest("Object id not available in object")
else:
object_id = obj._id
target_list = []
for objname,obj in datastore_dict.iteritems():
if (objname.find('_version_')>0) or (not type(obj) is dict): continue
if 'type_' in obj and obj['type_'] == "Association":
if obj['s'] == subject_id and obj['o'] == object_id:
if assoc_type:
if obj['at'] == assoc_type:
target_list.append(obj)
else:
target_list.append(obj)
else:
target_list = []
for objname,obj in datastore_dict.iteritems():
if (objname.find('_version_')>0) or (not type(obj) is dict): continue
if 'type_' in obj and obj['type_'] == "Association":
if obj['p'] == predicate:
target_list.append(obj)
if id_only:
assocs = [row['_id'] for row in target_list]
else:
assocs = [self._persistence_dict_to_ion_object(row) for row in target_list]
log.debug("find_associations() found %s associations" % (len(assocs)))
return assocs
def find_res_by_type(self, restype, lcstate=None, id_only=False):
log.debug("find_res_by_type(restype=%s, lcstate=%s)" % (restype, lcstate))
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
try:
datastore_dict = self.root[self.datastore_name]
except KeyError:
raise BadRequest('Data store ' + self.datastore_name + ' does not exist.')
assoc_list = []
target_id_list = []
target_list = []
for objname,obj in datastore_dict.iteritems():
if (objname.find('_version_')>0) or (not type(obj) is dict): continue
if 'type_' in obj and (obj['type_'] == restype or (not restype and obj['type_'] != "Association")):
if (lcstate and 'lcstate' in obj and obj['lcstate'] == lcstate) or not lcstate or not restype:
target_id_list.append(obj['_id'])
target_list.append(self._persistence_dict_to_ion_object(obj))
assoc_list.append([])
log.debug("find_res_by_type() found %s resources" % (len(target_list)))
if id_only:
return (target_id_list, assoc_list)
else:
return (target_list, assoc_list)
def find_res_by_lcstate(self, lcstate, restype=None, id_only=False):
log.debug("find_res_by_type(lcstate=%s, restype=%s)" % (lcstate, restype))
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
try:
datastore_dict = self.root[self.datastore_name]
except KeyError:
raise BadRequest('Data store ' + self.datastore_name + ' does not exist.')
if lcstate in CommonResourceLifeCycleSM.STATE_ALIASES:
lcstate_match = CommonResourceLifeCycleSM.STATE_ALIASES[lcstate]
else:
lcstate_match = [lcstate]
assoc_list = []
target_id_list = []
target_list = []
for objname,obj in datastore_dict.iteritems():
if (objname.find('_version_')>0) or (not type(obj) is dict): continue
if 'lcstate' in obj and obj['lcstate'] in lcstate_match:
if (restype and obj['type_'] == restype) or not restype:
target_id_list.append(obj['_id'])
target_list.append(self._persistence_dict_to_ion_object(obj))
assoc_list.append([])
log.debug("find_res_by_lcstate() found %s resources" % (len(target_list)))
if id_only:
return (target_id_list, assoc_list)
else:
return (target_list, assoc_list)
def _pass(self):
pass
def find_res_by_name(self, name, restype=None, id_only=False):
log.debug("find_res_by_name(name=%s, restype=%s)" % (name, restype))
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
try:
datastore_dict = self.root[self.datastore_name]
except KeyError:
raise BadRequest('Data store ' + self.datastore_name + ' does not exist.')
assoc_list = []
target_id_list = []
target_list = []
for objname,obj in datastore_dict.iteritems():
if (objname.find('_version_')>0) or (not type(obj) is dict): continue
if 'name' in obj and obj['name'] == name:
if (restype and obj['type_'] == restype) or not restype:
target_id_list.append(obj['_id'])
target_list.append(self._persistence_dict_to_ion_object(obj))
assoc_list.append([])
log.debug("find_res_by_name() found %s resources" % (len(target_list)))
if id_only:
return (target_id_list, assoc_list)
else:
return (target_list, assoc_list)
def find_dir_entries(self, qname):
raise NotImplementedError()
def _ion_object_to_persistence_dict(self, ion_object):
if ion_object is None: return None
obj_dict = self._io_serializer.serialize(ion_object)
return obj_dict
def _persistence_dict_to_ion_object(self, obj_dict):
if obj_dict is None: return None
ion_object = self._io_deserializer.deserialize(obj_dict)
return ion_object
class FileDataStore(object):
def __init__(self, container, datastore_name=""):
self.container = container
self.datastore_name = datastore_name
# Object serialization/deserialization
self._io_serializer = IonObjectSerializer()
self._io_deserializer = IonObjectDeserializer(
obj_registry=get_obj_registry())
def start(self):
if self.container.has_capability(self.container.CCAP.FILE_SYSTEM):
self.datastore_dir = FileSystem.get_url(FS.FILESTORE,
self.datastore_name)
else:
self.datastore_dir = "./tmp/%s" % self.datastore_name
def stop(self):
pass
def _get_filename(self, object_id):
return "%s/%s" % (self.datastore_dir, object_id)
def create(self, obj, object_id=None, attachments=None, datastore_name=""):
"""
Converts ion objects to python dictionary before persisting them using the optional
suggested identifier and creates attachments to the object.
Returns an identifier and revision number of the object
"""
if not isinstance(obj, IonObjectBase):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.create_doc(self._ion_object_to_persistence_dict(obj),
object_id=object_id,
datastore_name=datastore_name,
attachments=attachments)
def create_doc(self,
doc,
object_id=None,
attachments=None,
datastore_name=""):
"""
Persists the document using the optionally suggested doc_id, and creates attachments to it.
Returns the identifier and version number of the document
"""
if '_id' in doc:
raise BadRequest("Doc must not have '_id'")
# Assign an id to doc (recommended in CouchDB documentation)
doc["_id"] = object_id or uuid4().hex
log.debug('Creating new object %s/%s' % (datastore_name, doc["_id"]))
log.debug('create doc contents: %s', doc)
filename = self._get_filename(doc["_id"])
doc_json = json.dumps(doc)
with open(filename, "w") as f:
f.write(doc_json)
return doc["_id"], 1
def update(self, obj, datastore_name=""):
if not isinstance(obj, IonObjectBase):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.update_doc(self._ion_object_to_persistence_dict(obj))
def update_doc(self, doc, datastore_name=""):
if '_id' not in doc:
raise BadRequest("Doc must have '_id'")
log.debug('update doc contents: %s', doc)
filename = self._get_filename(doc["_id"])
doc_json = json.dumps(doc)
with open(filename, "w") as f:
f.write(doc_json)
return doc["_id"], 2
def read(self, object_id, rev_id="", datastore_name=""):
if not isinstance(object_id, str):
raise BadRequest("Object id param is not string")
doc = self.read_doc(object_id, rev_id, datastore_name)
# Convert doc into Ion object
obj = self._persistence_dict_to_ion_object(doc)
log.debug('Ion object: %s', str(obj))
return obj
def read_doc(self, doc_id, rev_id="", datastore_name=""):
log.debug('Reading head version of object %s/%s', datastore_name,
doc_id)
filename = self._get_filename(doc_id)
doc = None
with open(filename, "r") as f:
doc_json = f.read()
doc = json.loads(doc_json)
if doc is None:
raise NotFound('Object with id %s does not exist.' % str(doc_id))
log.debug('read doc contents: %s', doc)
return doc
def delete(self, obj, datastore_name="", del_associations=False):
if not isinstance(obj, IonObjectBase) and not isinstance(obj, str):
raise BadRequest(
"Obj param is not instance of IonObjectBase or string id")
if type(obj) is str:
self.delete_doc(obj,
datastore_name=datastore_name,
del_associations=del_associations)
else:
if '_id' not in obj:
raise BadRequest("Doc must have '_id'")
self.delete_doc(self._ion_object_to_persistence_dict(obj),
datastore_name=datastore_name,
del_associations=del_associations)
def delete_doc(self, doc, datastore_name="", del_associations=False):
doc_id = doc if type(doc) is str else doc["_id"]
log.debug('Deleting object %s/%s', datastore_name, doc_id)
filename = self._get_filename(doc_id)
try:
os.remove(filename)
except OSError:
raise NotFound('Object with id %s does not exist.' % doc_id)
def _ion_object_to_persistence_dict(self, ion_object):
if ion_object is None: return None
obj_dict = self._io_serializer.serialize(ion_object)
return obj_dict
def _persistence_dict_to_ion_object(self, obj_dict):
if obj_dict is None: return None
ion_object = self._io_deserializer.deserialize(obj_dict)
return ion_object
@author David Stuebe
@author Don Brittain
@author Tim Giguere
@brief https://confluence.oceanobservatories.org/display/CIDev/R2+Construction+Data+Model
'''
import yaml
from pyon.core.object import ion_serializer, IonObjectDeserializer
from pyon.core.registry import IonObjectRegistry
from interface.objects import Taxonomy
from pyon.util.log import log
# Create an IonObjectDeserializer used in the prototype loads method...
ior = IonObjectRegistry()
ion_deserializer = IonObjectDeserializer(obj_registry=ior)
class TaxyTool(object):
"""
@brief Wraps up a Taxonomy (IONObject) in a class which uses that information
Definition of the Taxonomy Ion Resource:
Taxonomy: !Extends_InformationResource
map: {}
The map is a dictionary which contains handles as keys and name sets as values.
A name set is a set of objects which can be hashed for inverse lookup and should be serializable for transport and persistence
In practice they are strings for nicknames and Taxonomy Description objects for complex definitions
def __init__(self, id_factory):
self.encoder = IonObjectSerializer()
self.decoder = IonObjectDeserializer(obj_registry=get_obj_registry())
self.id_factory = id_factory
def __init__(self):
Interceptor.__init__(self)
self._io_serializer = IonObjectSerializer()
self._io_deserializer = IonObjectDeserializer(
obj_registry=get_obj_registry())
class PostgresPyonDataStore(PostgresDataStore):
"""
Base class common to both CouchDB and Couchbase datastores.
"""
def __init__(self, datastore_name=None, config=None, scope=None, profile=None):
"""
@param datastore_name Name of datastore within server. May be scoped to sysname
@param config A server config dict with connection params
@param scope Prefix for the datastore name (e.g. sysname) to separate multiple systems
"""
PostgresDataStore.__init__(self, datastore_name=datastore_name,
config=config or CFG.get_safe("server.postgresql"),
profile=profile or DataStore.DS_PROFILE.BASIC,
scope=scope)
# IonObject Serializers
self._io_serializer = IonObjectSerializer()
self._io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
# -------------------------------------------------------------------------
# Couch document operations
def create(self, obj, object_id=None, attachments=None, datastore_name=""):
"""
Converts ion objects to python dictionary before persisting them using the optional
suggested identifier and creates attachments to the object.
Returns an identifier and revision number of the object
"""
if not isinstance(obj, IonObjectBase):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.create_doc(self._ion_object_to_persistence_dict(obj),
object_id=object_id, datastore_name=datastore_name,
attachments=attachments)
def create_mult(self, objects, object_ids=None, allow_ids=None):
if any([not isinstance(obj, IonObjectBase) for obj in objects]):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.create_doc_mult([self._ion_object_to_persistence_dict(obj) for obj in objects], object_ids)
def update(self, obj, datastore_name=""):
if not isinstance(obj, IonObjectBase):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.update_doc(self._ion_object_to_persistence_dict(obj))
def update_mult(self, objects):
if any([not isinstance(obj, IonObjectBase) for obj in objects]):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.update_doc_mult([self._ion_object_to_persistence_dict(obj) for obj in objects])
def read(self, object_id, rev_id="", datastore_name="", object_type=None):
if not isinstance(object_id, str):
raise BadRequest("Object id param is not string")
doc = self.read_doc(object_id, rev_id, datastore_name=datastore_name, object_type=object_type)
obj = self._persistence_dict_to_ion_object(doc)
return obj
def read_mult(self, object_ids, datastore_name="", strict=True):
if any([not isinstance(object_id, str) for object_id in object_ids]):
raise BadRequest("Object ids are not string: %s" % str(object_ids))
docs = self.read_doc_mult(object_ids, datastore_name, strict=strict)
obj_list = [self._persistence_dict_to_ion_object(doc) if doc is not None else None for doc in docs]
return obj_list
def delete(self, obj, datastore_name="", object_type=None):
if not isinstance(obj, IonObjectBase) and not isinstance(obj, str):
raise BadRequest("Obj param is not instance of IonObjectBase or string id")
if type(obj) is str:
self.delete_doc(obj, datastore_name=datastore_name, object_type=object_type)
else:
if '_id' not in obj:
raise BadRequest("Doc must have '_id'")
if '_rev' not in obj:
raise BadRequest("Doc must have '_rev'")
self.delete_doc(self._ion_object_to_persistence_dict(obj),
datastore_name=datastore_name, object_type=object_type)
def delete_mult(self, object_ids, datastore_name=None):
return self.delete_doc_mult(object_ids, datastore_name)
# -------------------------------------------------------------------------
# View operations
def find_objects_mult(self, subjects, id_only=False):
"""
Returns a list of associations for a given list of subjects
"""
#ds, datastore_name = self._get_datastore()
#validate_is_instance(subjects, list, 'subjects is not a list of resource_ids')
#view_args = dict(keys=subjects, include_docs=True)
#results = self.query_view(self._get_view_name("association", "by_bulk"), view_args)
#ids = [i['value'] for i in results]
#assocs = [i['doc'] for i in results]
#self._count(find_assocs_mult_call=1, find_assocs_mult_obj=len(ids))
#if id_only:
# return ids, assocs
#else:
# return self.read_mult(ids), assocs
# TODO: Port this implementation to Postgres single query
res_list = [[], []]
if not subjects:
return res_list
for sub in subjects:
res_ids, res_assocs = self.find_objects(subject=sub, id_only=id_only)
res_list[0].extend(res_ids)
res_list[1].extend(res_assocs)
return res_list
def find_subjects_mult(self, objects, id_only=False):
"""
Returns a list of associations for a given list of objects
"""
#ds, datastore_name = self._get_datastore()
#validate_is_instance(objects, list, 'objects is not a list of resource_ids')
#view_args = dict(keys=objects, include_docs=True)
#results = self.query_view(self._get_view_name("association", "by_subject_bulk"), view_args)
#ids = [i['value'] for i in results]
#assocs = [i['doc'] for i in results]
#self._count(find_assocs_mult_call=1, find_assocs_mult_obj=len(ids))
#if id_only:
# return ids, assocs
#else:
# return self.read_mult(ids), assocs
# TODO: Port this implementation to Postgres single query
res_list = [[], []]
if not objects:
return res_list
for obj in objects:
res_ids, res_assocs = self.find_subjects(obj=obj, id_only=id_only)
res_list[0].extend(res_ids)
res_list[1].extend(res_assocs)
return res_list
def find_objects(self, subject, predicate=None, object_type=None, id_only=False, **kwargs):
#log.debug("find_objects(subject=%s, predicate=%s, object_type=%s, id_only=%s", subject, predicate, object_type, id_only)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
if not subject:
raise BadRequest("Must provide subject")
if object_type and not predicate:
raise BadRequest("Cannot provide object type without a predicate")
if type(subject) is str:
subject_id = subject
else:
if "_id" not in subject:
raise BadRequest("Object id not available in subject")
else:
subject_id = subject._id
qual_ds_name = self._get_datastore_name()
view_args = self._get_view_args(kwargs)
if id_only:
query = "SELECT o, doc FROM %(dsa)s WHERE retired<>true " % dict(dsa=qual_ds_name+"_assoc")
else:
query = "SELECT %(ds)s.doc, %(dsa)s.doc FROM %(dsa)s, %(ds)s WHERE retired<>true AND %(dsa)s.o=%(ds)s.id " % dict(ds=qual_ds_name, dsa=qual_ds_name+"_assoc")
query_args = dict(s=subject_id, ot=object_type, p=predicate)
query_clause = "AND s=%(s)s"
if predicate:
query_clause += " AND p=%(p)s"
if object_type:
query_clause += " AND ot=%(ot)s"
extra_clause = view_args.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
obj_assocs = [self._persistence_dict_to_ion_object(row[-1]) for row in rows]
#log.debug("find_objects() found %s objects", len(obj_assocs))
if id_only:
res_ids = [self._prep_id(row[0]) for row in rows]
return res_ids, obj_assocs
else:
res_objs = [self._persistence_dict_to_ion_object(row[0]) for row in rows]
return res_objs, obj_assocs
def find_subjects(self, subject_type=None, predicate=None, obj=None, id_only=False, **kwargs):
#log.debug("find_subjects(subject_type=%s, predicate=%s, object=%s, id_only=%s", subject_type, predicate, obj, id_only)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
if not obj:
raise BadRequest("Must provide object")
if subject_type and not predicate:
raise BadRequest("Cannot provide subject type without a predicate")
if type(obj) is str:
object_id = obj
else:
if "_id" not in obj:
raise BadRequest("Object id not available in object")
else:
object_id = obj._id
qual_ds_name = self._get_datastore_name()
view_args = self._get_view_args(kwargs)
if id_only:
query = "SELECT s, doc FROM %(dsa)s WHERE retired<>true " % dict(dsa=qual_ds_name+"_assoc")
else:
query = "SELECT %(ds)s.doc, %(dsa)s.doc FROM %(dsa)s, %(ds)s WHERE retired<>true AND %(dsa)s.s=%(ds)s.id " % dict(ds=qual_ds_name, dsa=qual_ds_name+"_assoc")
query_args = dict(o=object_id, st=subject_type, p=predicate)
query_clause = "AND o=%(o)s"
if predicate:
query_clause += " AND p=%(p)s"
if subject_type:
query_clause += " AND st=%(st)s"
extra_clause = view_args.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
obj_assocs = [self._persistence_dict_to_ion_object(row[-1]) for row in rows]
#log.debug("find_subjects() found %s subjects", len(obj_assocs))
if id_only:
res_ids = [self._prep_id(row[0]) for row in rows]
return res_ids, obj_assocs
else:
res_objs = [self._persistence_dict_to_ion_object(row[0]) for row in rows]
return res_objs, obj_assocs
def find_associations(self, subject=None, predicate=None, obj=None, assoc_type=None, id_only=True, anyside=None, **kwargs):
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
if not (subject or obj or predicate or anyside):
raise BadRequest("Illegal parameters: No S/P/O or anyside")
#if assoc_type:
# raise BadRequest("Illegal parameters: assoc_type deprecated")
if anyside and (subject or obj):
raise BadRequest("Illegal parameters: anyside cannot be combined with S/O")
if anyside and predicate and type(anyside) in (list, tuple):
raise BadRequest("Illegal parameters: anyside list cannot be combined with P")
subject_id, object_id, anyside_ids = None, None, None
if subject:
if type(subject) is str:
subject_id = subject
else:
if "_id" not in subject:
raise BadRequest("Object id not available in subject")
else:
subject_id = subject._id
if obj:
if type(obj) is str:
object_id = obj
else:
if "_id" not in obj:
raise BadRequest("Object id not available in object")
else:
object_id = obj._id
if anyside:
if type(anyside) is str:
anyside_ids = [anyside]
elif type(anyside) in (list, tuple):
if not all([type(o) in (str, list, tuple) for o in anyside]):
raise BadRequest("List of object ids or (object id, predicate) expected")
anyside_ids = anyside
else:
if "_id" not in anyside:
raise BadRequest("Object id not available in anyside")
else:
anyside_ids = [anyside._id]
#log.debug("find_associations(subject=%s, predicate=%s, object=%s, anyside=%s)", subject_id, predicate, object_id, anyside_ids)
qual_ds_name = self._get_datastore_name()
table = qual_ds_name + "_assoc"
view_args = self._get_view_args(kwargs)
if id_only:
query = "SELECT id FROM " + table
else:
query = "SELECT id, doc, s, st, p, o, ot FROM " + table
query_clause = " WHERE retired<>true AND "
query_args = dict(s=subject_id, o=object_id, p=predicate)
if subject and obj:
query_clause += "s=%(s)s AND o=%(o)s"
if predicate:
query_clause += " AND p=%(p)s"
elif subject:
query_clause += "s=%(s)s"
if predicate:
query_clause += " AND p=%(p)s"
elif obj:
query_clause += "o=%(o)s"
if predicate:
query_clause += " AND p=%(p)s"
elif anyside:
if predicate:
query_clause += "p=%(p)s AND (s=%(any)s OR o=%(any)s)"
query_args["any"] = anyside
elif type(anyside_ids[0]) is str:
# keys are IDs of resources
for i, key in enumerate(anyside_ids):
if i > 0:
query_clause += " OR "
argname = "id%s" % i
query_args[argname] = key
query_clause += "(s=%("+argname+")s OR o=%("+argname+")s)"
else:
# keys are tuples of (id, pred)
for i, (key, pred) in enumerate(anyside_ids):
if i > 0:
query_clause += " OR "
argname_id = "id%s" % i
argname_p = "p%s" % i
query_args[argname_id] = key
query_args[argname_p] = pred
query_clause += "(p=%("+argname_p+")s AND (s=%("+argname_id+")s OR o=%("+argname_id+")s))"
elif predicate:
if predicate == "*":
query_clause += "p is not null"
else:
query_clause += "p=%(p)s"
else:
raise BadRequest("Illegal arguments")
extra_clause = view_args.get("extra_clause", "")
sql = query + query_clause + extra_clause
#print "find_associations(): SQL=", sql, query_args
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(sql, query_args)
rows = cur.fetchall()
if id_only:
assocs = [self._prep_id(row[0]) for row in rows]
else:
assocs = [self._persistence_dict_to_ion_object(row[1]) for row in rows]
#log.debug("find_associations() found %s associations", len(assocs))
return assocs
def _prepare_find_return(self, rows, res_assocs=None, id_only=True, **kwargs):
if id_only:
res_ids = [self._prep_id(row[0]) for row in rows]
return res_ids, res_assocs
else:
res_docs = [self._persistence_dict_to_ion_object(row[-1]) for row in rows]
return res_docs, res_assocs
def find_resources(self, restype="", lcstate="", name="", id_only=True):
return self.find_resources_ext(restype=restype, lcstate=lcstate, name=name, id_only=id_only)
def find_resources_ext(self, restype="", lcstate="", name="",
keyword=None, nested_type=None,
attr_name=None, attr_value=None, alt_id=None, alt_id_ns=None,
limit=None, skip=None, descending=None, id_only=True):
filter_kwargs = self._get_view_args(dict(limit=limit, skip=skip, descending=descending))
if name:
if lcstate:
raise BadRequest("find by name does not support lcstate")
return self.find_res_by_name(name, restype, id_only, filter=filter_kwargs)
elif keyword:
return self.find_res_by_keyword(keyword, restype, id_only, filter=filter_kwargs)
elif alt_id or alt_id_ns:
return self.find_res_by_alternative_id(alt_id, alt_id_ns, id_only, filter=filter_kwargs)
elif nested_type:
return self.find_res_by_nested_type(nested_type, restype, id_only, filter=filter_kwargs)
elif restype and attr_name:
return self.find_res_by_attribute(restype, attr_name, attr_value, id_only=id_only, filter=filter_kwargs)
elif restype and lcstate:
return self.find_res_by_lcstate(lcstate, restype, id_only, filter=filter_kwargs)
elif restype:
return self.find_res_by_type(restype, lcstate, id_only, filter=filter_kwargs)
elif lcstate:
return self.find_res_by_lcstate(lcstate, restype, id_only, filter=filter_kwargs)
elif not restype and not lcstate and not name:
return self.find_res_by_type(None, None, id_only, filter=filter_kwargs)
def find_res_by_type(self, restype, lcstate=None, id_only=False, filter=None):
log.debug("find_res_by_type(restype=%s, lcstate=%s)", restype, lcstate)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
if lcstate:
raise BadRequest('lcstate not supported anymore in find_res_by_type')
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, name, type_, lcstate FROM " + qual_ds_name
else:
query = "SELECT id, name, type_, lcstate, doc FROM " + qual_ds_name
query_clause = " WHERE lcstate<>'RETIRED' "
query_args = dict(type_=restype, lcstate=lcstate)
if restype:
query_clause += "AND type_=%(type_)s"
else:
# Returns ALL documents, only limited by filter
query_clause = ""
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [dict(id=self._prep_id(row[0]), name=row[1], type=row[2]) for row in rows]
log.debug("find_res_by_type() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_lcstate(self, lcstate, restype=None, id_only=False, filter=None):
log.debug("find_res_by_lcstate(lcstate=%s, restype=%s)", lcstate, restype)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
if '_' in lcstate:
log.warn("Search for compound lcstate restricted to maturity: %s", lcstate)
lcstate,_ = lcstate.split("_", 1)
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, name, type_, lcstate, availability FROM " + qual_ds_name
else:
query = "SELECT id, name, type_, lcstate, availability, doc FROM " + qual_ds_name
query_clause = " WHERE "
query_args = dict(type_=restype, lcstate=lcstate)
is_maturity = lcstate not in CommonResourceLifeCycleSM.AVAILABILITY
if is_maturity:
query_clause += "lcstate=%(lcstate)s"
else:
query_clause += "availability=%(lcstate)s"
if restype:
query_clause += " AND type_=%(type_)s"
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [dict(id=self._prep_id(row[0]), name=row[1], type=row[2], lcstate=row[3] if is_maturity else row[4]) for row in rows]
log.debug("find_res_by_lcstate() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_name(self, name, restype=None, id_only=False, filter=None):
log.debug("find_res_by_name(name=%s, restype=%s)", name, restype)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, name, type_ FROM " + qual_ds_name
else:
query = "SELECT id, name, type_, doc FROM " + qual_ds_name
query_clause = " WHERE lcstate<>'RETIRED' "
query_args = dict(name=name, type_=restype)
query_clause += "AND name=%(name)s"
if restype:
query_clause += " AND type_=%(type_)s"
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [dict(id=self._prep_id(row[0]), name=row[1], type=row[2]) for row in rows]
log.debug("find_res_by_name() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_keyword(self, keyword, restype=None, id_only=False, filter=None):
log.debug("find_res_by_keyword(keyword=%s, restype=%s)", keyword, restype)
if not keyword or type(keyword) is not str:
raise BadRequest('Argument keyword illegal')
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, type_ FROM " + qual_ds_name
else:
query = "SELECT id, type_, doc FROM " + qual_ds_name
query_clause = " WHERE lcstate<>'RETIRED' "
query_args = dict(type_=restype, kw=[keyword])
query_clause += "AND %(kw)s <@ json_keywords(doc)"
if restype:
query_clause += " AND type_=%(type_)s"
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [dict(id=self._prep_id(row[0]), type=row[1], keyword=keyword) for row in rows]
log.debug("find_res_by_keyword() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_nested_type(self, nested_type, restype=None, id_only=False, filter=None):
log.debug("find_res_by_nested_type(nested_type=%s, restype=%s)", nested_type, restype)
if not nested_type or type(nested_type) is not str:
raise BadRequest('Argument nested_type illegal')
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, type_ FROM " + qual_ds_name
else:
query = "SELECT id, type_, doc FROM " + qual_ds_name
query_clause = " WHERE lcstate<>'RETIRED' "
query_args = dict(type_=restype, nest=[nested_type])
query_clause += "AND %(nest)s <@ json_nested(doc)"
if restype:
query_clause += " AND type_=%(type_)s"
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [dict(id=self._prep_id(row[0]), type=row[1], nested_type=nested_type) for row in rows]
log.debug("find_res_by_nested_type() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_attribute(self, restype, attr_name, attr_value=None, id_only=False, filter=None):
log.debug("find_res_by_attribute(restype=%s, attr_name=%s, attr_value=%s)", restype, attr_name, attr_value)
if not attr_name or type(attr_name) is not str:
raise BadRequest('Argument attr_name illegal')
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
if id_only:
query = "SELECT id, type_, json_specialattr(doc) FROM " + qual_ds_name
else:
query = "SELECT id, type_, json_specialattr(doc), doc FROM " + qual_ds_name
query_clause = " WHERE lcstate<>'RETIRED' "
query_args = dict(type_=restype, att=attr_name, val=attr_value)
if attr_value is not None:
query_clause += "AND json_specialattr(doc)=%(spc)s"
query_args['spc'] = "%s=%s" % (attr_name, attr_value)
else:
query_clause += "AND json_specialattr(doc) LIKE %(spc)s"
query_args['spc'] = "%s=%%" % (attr_name, )
if restype:
query_clause += " AND type_=%(type_)s"
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
res_assocs = [dict(id=self._prep_id(row[0]), type=row[1], attr_name=attr_name, attr_value=row[2].split("=",1)[-1]) for row in rows]
log.debug("find_res_by_attribute() found %s objects", len(res_assocs))
return self._prepare_find_return(rows, res_assocs, id_only=id_only)
def find_res_by_alternative_id(self, alt_id=None, alt_id_ns=None, id_only=False, filter=None):
log.debug("find_res_by_alternative_id(restype=%s, alt_id_ns=%s)", alt_id, alt_id_ns)
if alt_id and type(alt_id) is not str:
raise BadRequest('Argument alt_id illegal')
if alt_id_ns and type(alt_id_ns) is not str:
raise BadRequest('Argument alt_id_ns illegal')
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
filter = filter if filter is not None else {}
qual_ds_name = self._get_datastore_name()
#if id_only:
# query = "SELECT id, x[1], x[2] FROM (SELECT json_altids(doc) as x, * FROM " + qual_ds_name + ") AS A"
#else:
# query = "SELECT id, x[1], x[2], doc FROM (SELECT json_altids(doc) as x, * FROM " + qual_ds_name + ") AS A"
#query_args = dict(aid=alt_id, ans=alt_id_ns)
#query_clause = " WHERE lcstate<>'RETIRED' "
#
#if not alt_id and not alt_id_ns:
# query_clause += " "
#elif alt_id and not alt_id_ns:
# query_clause += " AND x[2]=%(aid)s"
#elif alt_id_ns and not alt_id:
# query_clause += " AND x[1]=%(ans)s"
#else:
# query_clause += " AND x[1]=%(ans)s AND x[2]=%(aid)s"
query = "SELECT id, type_, doc FROM " + qual_ds_name
query_args = dict(aid=[alt_id], ans=[alt_id_ns])
query_clause = " WHERE lcstate<>'RETIRED' "
if not alt_id and not alt_id_ns:
query_clause += "AND json_altids_ns(doc) is not null"
elif alt_id and not alt_id_ns:
query_clause += "AND %(aid)s <@ json_altids_id(doc)"
elif alt_id_ns and not alt_id:
query_clause += "AND %(ans)s <@ json_altids_ns(doc)"
else:
query_clause += "AND %(aid)s <@ json_altids_id(doc) AND %(ans)s <@ json_altids_ns(doc)"
extra_clause = filter.get("extra_clause", "")
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(query + query_clause + extra_clause, query_args)
rows = cur.fetchall()
# Need to fake the return format of the Couch view for alt_ids. One record per alt_id, not one per resource.
res_assocs = []
res_rows = []
for row in rows:
doc_id = self._prep_id(row[0])
doc = row[-1]
for aid in doc.get("alt_ids", []):
aid_parts = aid.split(":", 1)
aid_ns = aid_parts[0] if len(aid_parts)>1 else "_"
aid_id = aid_parts[-1]
if alt_id_ns and alt_id:
if alt_id_ns == aid_ns and alt_id == aid_id:
res_assocs.append(dict(id=doc_id, alt_id_ns=aid_ns, alt_id=aid_id))
res_rows.append((doc_id, doc))
elif (not alt_id_ns and not alt_id) or (alt_id_ns and alt_id_ns == aid_ns) or (alt_id and alt_id == aid_id):
res_assocs.append(dict(id=doc_id, alt_id_ns=aid_ns, alt_id=aid_id))
res_rows.append((doc_id, doc))
log.debug("find_res_by_alternative_id() found %s objects", len(res_assocs))
return self._prepare_find_return(res_rows, res_assocs, id_only=id_only)
def find_by_view(self, design_name, view_name, key=None, keys=None, start_key=None, end_key=None,
id_only=True, convert_doc=True, **kwargs):
"""
Generic find function using a defined index
@param design_name design document
@param view_name view name
@param key specific key to find
@param keys list of keys to find
@param start_key find range start value
@param end_key find range end value
@param id_only if True, the 4th element of each triple is the document
@param convert_doc if True, make IonObject out of doc
@retval Returns a list of 3-tuples: (document id, index key, index value or document)
"""
res_rows = self.find_docs_by_view(design_name=design_name, view_name=view_name, key=key, keys=keys,
start_key=start_key, end_key=end_key, id_only=id_only, **kwargs)
res_rows = [(rid, key,
self._persistence_dict_to_ion_object(doc) if convert_doc and isinstance(doc, dict) else doc)
for rid, key, doc in res_rows]
log.debug("find_by_view() found %s objects" % (len(res_rows)))
return res_rows
def find_resources_mult(self, query):
"""
Find resources given a datastore query expression dict.
@param query a dict representation of a datastore query
@retval list of resource ids or resource objects matching query (dependent on id_only value)
"""
qual_ds_name = self._get_datastore_name()
pqb = PostgresQueryBuilder(query, qual_ds_name)
with self.pool.cursor(**self.cursor_args) as cur:
cur.execute(pqb.get_query(), pqb.get_values())
rows = cur.fetchall()
log.info("find_resources_mult() QUERY: %s (%s rows)", cur.query, cur.rowcount)
id_only = query["query_args"].get("id_only", True)
if id_only:
res_ids = [self._prep_id(row[0]) for row in rows]
return res_ids
else:
res_docs = [self._persistence_dict_to_ion_object(row[-1]) for row in rows]
return res_docs
# -------------------------------------------------------------------------
# Internal operations
def _ion_object_to_persistence_dict(self, ion_object):
if ion_object is None:
return None
obj_dict = self._io_serializer.serialize(ion_object)
return obj_dict
def _persistence_dict_to_ion_object(self, obj_dict):
if obj_dict is None:
return None
ion_object = self._io_deserializer.deserialize(obj_dict)
return ion_object