def test_datastore_operations_list_values(self):
# Verify prefix filtering
mock_api_client = mock.Mock()
mock_api_client.keys.get_all.return_value = []
self._set_mock_api_client(mock_api_client)
self._datastore_service.list_values(local=True, prefix=None)
mock_api_client.keys.get_all.assert_called_with(
prefix="core.TestSensor:", limit=100, offset=0)
self._datastore_service.list_values(local=True, prefix="ponies")
mock_api_client.keys.get_all.assert_called_with(
prefix="core.TestSensor:ponies", limit=100, offset=0)
self._datastore_service.list_values(local=False, prefix=None)
mock_api_client.keys.get_all.assert_called_with(prefix=None,
limit=100,
offset=0)
self._datastore_service.list_values(local=False, prefix="ponies")
mock_api_client.keys.get_all.assert_called_with(prefix="ponies",
limit=100,
offset=0)
# No values in the datastore
mock_api_client = mock.Mock()
mock_api_client.keys.get_all.return_value = []
self._set_mock_api_client(mock_api_client)
values = self._datastore_service.list_values(local=True)
self.assertEqual(values, [])
values = self._datastore_service.list_values(local=False)
self.assertEqual(values, [])
# Values in the datastore
kvp1 = KeyValuePair()
kvp1.name = "test1"
kvp1.value = "bar"
kvp2 = KeyValuePair()
kvp2.name = "test2"
kvp2.value = "bar"
mock_return_value = [kvp1, kvp2]
mock_api_client.keys.get_all.return_value = mock_return_value
self._set_mock_api_client(mock_api_client)
values = self._datastore_service.list_values(local=True)
self.assertEqual(len(values), 2)
self.assertEqual(values, mock_return_value)
# Test limit
_ = self._datastore_service.list_values(local=True, limit=1)
mock_api_client.keys.get_all.assert_called_with(
prefix="core.TestSensor:", limit=1, offset=0)
# Test offset
_ = self._datastore_service.list_values(local=True, offset=1)
mock_api_client.keys.get_all.assert_called_with(
prefix="core.TestSensor:", limit=100, offset=1)
def run(self, args, **kwargs):
file_path = os.path.normpath(pjoin(os.getcwd(), args.file))
if not os.path.exists(args.file):
raise ValueError('File "%s" doesn\'t exist' % (file_path))
if not os.path.isfile(args.file):
raise ValueError('"%s" is not a file' % (file_path))
with open(file_path, 'r') as f:
kvps = json.loads(f.read())
instances = []
for item in kvps:
name = item['name']
value = item['value']
instance = KeyValuePair()
instance.id = name # TODO: refactor and get rid of id
instance.name = name
instance.value = value
self.manager.update(instance, **kwargs)
instances.append(instance)
return instances
def test_datastore_token_timeout(self):
datastore_service = SensorDatastoreService(logger=mock.Mock(),
pack_name='core',
class_name='TestSensor',
api_username='******')
mock_api_client = mock.Mock()
kvp1 = KeyValuePair()
kvp1.name = 'test1'
kvp1.value = 'bar'
mock_api_client.keys.get_by_id.return_value = kvp1
token_expire_time = get_datetime_utc_now() - timedelta(seconds=5)
datastore_service._client = mock_api_client
datastore_service._token_expire = token_expire_time
self._set_mock_api_client(mock_api_client)
with mock.patch(
'st2common.services.datastore.Client',
return_value=mock_api_client
) as datastore_client:
value = datastore_service.get_value(name='test1', local=False)
self.assertTrue(datastore_client.called)
self.assertEqual(value, kvp1.value)
self.assertGreater(datastore_service._token_expire, token_expire_time)
def delete_value(self, name, local=True):
"""
Delete the provided key.
By default, value is deleted from a namespace local to the sensor. If you want to
delete a global value, pass local=False to this method.
:param name: Name of the key to delete.
:type name: ``str``
:param local: Delete a value in a namespace local to the sensor. Defaults to True.
:type: local: ``bool``
:return: ``True`` on success, ``False`` otherwise.
:rtype: ``bool``
"""
if local:
name = self._get_key_name_with_sensor_prefix(name=name)
client = self._get_api_client()
instance = KeyValuePair()
instance.id = name
instance.name = name
self._logger.audit('Deleting value from the datastore (name=%s)', name)
try:
client.keys.delete(instance=instance)
except Exception:
return False
return True
def run(self, args, **kwargs):
file_path = os.path.normpath(pjoin(os.getcwd(), args.file))
if not os.path.exists(args.file):
raise ValueError('File "%s" doesn\'t exist' % (file_path))
if not os.path.isfile(args.file):
raise ValueError('"%s" is not a file' % (file_path))
with open(file_path, 'r') as f:
kvps = json.loads(f.read())
instances = []
for item in kvps:
name = item['name']
value = item['value']
instance = KeyValuePair()
instance.id = name # TODO: refactor and get rid of id
instance.name = name
instance.value = value
self.manager.update(instance, **kwargs)
instances.append(instance)
return instances
def test_datastore_token_timeout(self):
datastore_service = SensorDatastoreService(
logger=mock.Mock(),
pack_name='core',
class_name='TestSensor',
api_username='******')
mock_api_client = mock.Mock()
kvp1 = KeyValuePair()
kvp1.name = 'test1'
kvp1.value = 'bar'
mock_api_client.keys.get_by_id.return_value = kvp1
token_expire_time = get_datetime_utc_now() - timedelta(seconds=5)
datastore_service._client = mock_api_client
datastore_service._token_expire = token_expire_time
self._set_mock_api_client(mock_api_client)
with mock.patch('st2common.services.datastore.Client',
return_value=mock_api_client) as datastore_client:
value = datastore_service.get_value(name='test1', local=False)
self.assertTrue(datastore_client.called)
self.assertEqual(value, kvp1.value)
self.assertGreater(datastore_service._token_expire,
token_expire_time)
def delete_value(self, name, local=True):
"""
Delete the provided key.
By default, value is deleted from a namespace local to the sensor. If you want to
delete a global value, pass local=False to this method.
:param name: Name of the key to delete.
:type name: ``str``
:param local: Delete a value in a namespace local to the sensor. Defaults to True.
:type: local: ``bool``
:return: ``True`` on success, ``False`` otherwise.
:rtype: ``bool``
"""
if local:
name = self._get_key_name_with_sensor_prefix(name=name)
client = self._get_api_client()
instance = KeyValuePair()
instance.id = name
instance.name = name
self._logger.audit('Deleting value from the datastore (name=%s)', name)
try:
client.keys.delete(instance=instance)
except Exception:
return False
return True
def set_value(self,
name,
value,
ttl=None,
local=True,
scope=SYSTEM_SCOPE,
encrypt=False):
"""
Store a value in a dictionary which is local to this class.
"""
name = self._get_full_key_name(name=name, local=local)
instance = KeyValuePair()
instance.id = name
instance.name = name
instance.value = value
if ttl:
self._logger.warning(
"MockDatastoreService is not able to expire keys based on ttl."
)
instance.ttl = ttl
self._datastore_items[name] = instance
return True
def run(self, args, **kwargs):
# normalize the file path to allow for relative files to be specified
file_path = os.path.normpath(pjoin(os.getcwd(), args.file))
# load the data (JSON/YAML) from the file
kvps = resource.load_meta_file(file_path)
instances = []
# bail out if file was empty
if not kvps:
return instances
# if the data is not a list (ie. it's a single entry)
# then make it a list so our process loop is generic
if not isinstance(kvps, list):
kvps = [kvps]
for item in kvps:
# parse required KeyValuePair properties
name = item['name']
value = item['value']
# parse optional KeyValuePair properties
scope = item.get('scope', DEFAULT_CUD_SCOPE)
user = item.get('user', None)
secret = item.get('secret', False)
ttl = item.get('ttl', None)
# if the value is not a string, convert it to JSON
# all keys in the datastore must strings
if not isinstance(value, six.string_types):
if args.convert:
value = json.dumps(value)
else:
raise ValueError(
("Item '%s' has a value that is not a string."
" Either pass in the -c/--convert option to convert"
" non-string types to JSON strings automatically, or"
" convert the data to a string in the file") % name)
# create the KeyValuePair instance
instance = KeyValuePair()
instance.id = name # TODO: refactor and get rid of id
instance.name = name
instance.value = value
instance.scope = scope
if user:
instance.user = user
if secret:
instance.secret = secret
if ttl:
instance.ttl = ttl
# call the API to create/update the KeyValuePair
self.manager.update(instance, **kwargs)
instances.append(instance)
return instances
def run(self, args, **kwargs):
instance = KeyValuePair()
instance.id = args.name # TODO: refactor and get rid of id
instance.name = args.name
instance.value = args.value
if args.ttl:
instance.ttl = args.ttl
return self.manager.update(instance, **kwargs)
def test_datastore_operations_get_value(self):
mock_api_client = mock.Mock()
kvp1 = KeyValuePair()
kvp1.name = "test1"
kvp1.value = "bar"
mock_api_client.keys.get_by_id.return_value = kvp1
self._set_mock_api_client(mock_api_client)
value = self._sensor_service.get_value(name="test1", local=False)
self.assertEqual(value, kvp1.value)
def test_datastore_operations_get_value(self):
mock_api_client = mock.Mock()
kvp1 = KeyValuePair()
kvp1.name = 'test1'
kvp1.value = 'bar'
mock_api_client.keys.get_by_id.return_value = kvp1
self._set_mock_api_client(mock_api_client)
value = self._datastore_service.get_value(name='test1', local=False)
self.assertEqual(value, kvp1.value)
def run(self, args, **kwargs):
instance = KeyValuePair()
instance.id = args.name # TODO: refactor and get rid of id
instance.name = args.name
instance.value = args.value
if args.ttl:
instance.ttl = args.ttl
return self.manager.update(instance, **kwargs)
def test_datastore_operations_list_values(self):
# Verify prefix filtering
mock_api_client = mock.Mock()
mock_api_client.keys.get_all.return_value = []
self._set_mock_api_client(mock_api_client)
self._datastore_service.list_values(local=True, prefix=None)
mock_api_client.keys.get_all.assert_called_with(
prefix='core.TestSensor:')
self._datastore_service.list_values(local=True, prefix='ponies')
mock_api_client.keys.get_all.assert_called_with(
prefix='core.TestSensor:ponies')
self._datastore_service.list_values(local=False, prefix=None)
mock_api_client.keys.get_all.assert_called_with(prefix=None)
self._datastore_service.list_values(local=False, prefix='ponies')
mock_api_client.keys.get_all.assert_called_with(prefix='ponies')
# No values in the datastore
mock_api_client = mock.Mock()
mock_api_client.keys.get_all.return_value = []
self._set_mock_api_client(mock_api_client)
values = self._datastore_service.list_values(local=True)
self.assertEqual(values, [])
values = self._datastore_service.list_values(local=False)
self.assertEqual(values, [])
# Values in the datastore
kvp1 = KeyValuePair()
kvp1.name = 'test1'
kvp1.value = 'bar'
kvp2 = KeyValuePair()
kvp2.name = 'test2'
kvp2.value = 'bar'
mock_return_value = [kvp1, kvp2]
mock_api_client.keys.get_all.return_value = mock_return_value
self._set_mock_api_client(mock_api_client)
values = self._datastore_service.list_values(local=True)
self.assertEqual(len(values), 2)
self.assertEqual(values, mock_return_value)
def set_value(
self, name, value, ttl=None, local=True, scope=SYSTEM_SCOPE, encrypt=False
):
"""
Set a value for the provided key.
By default, value is set in a namespace local to the pack/class. If you want to
set a global value, pass local=False to this method.
:param name: Key name.
:type name: ``str``
:param value: Key value.
:type value: ``str``
:param ttl: Optional TTL (in seconds).
:type ttl: ``int``
:param local: Set value in a namespace local to the pack/class. Defaults to True.
:type: local: ``bool``
:param scope: Scope under which to place the item. Defaults to system scope.
:type: local: ``str``
:param encrypt: Encrypt the value when saving. Defaults to False.
:type: local: ``bool``
:return: ``True`` on success, ``False`` otherwise.
:rtype: ``bool``
"""
if scope != SYSTEM_SCOPE:
raise ValueError("Scope %s is unsupported." % scope)
name = self._get_full_key_name(name=name, local=local)
value = str(value)
client = self.get_api_client()
self._logger.debug("Setting value in the datastore (name=%s)", name)
instance = KeyValuePair()
instance.id = name
instance.name = name
instance.value = value
instance.scope = scope
if encrypt:
instance.secret = True
if ttl:
instance.ttl = ttl
client.keys.update(instance=instance)
return True
def run(self, args, **kwargs):
# normalize the file path to allow for relative files to be specified
file_path = os.path.normpath(pjoin(os.getcwd(), args.file))
# load the data (JSON/YAML) from the file
kvps = resource.load_meta_file(file_path)
# if the data is not a list (ie. it's a single entry)
# then make it a list so our process loop is generic
if not isinstance(kvps, list):
kvps = [kvps]
instances = []
for item in kvps:
# parse required KeyValuePair properties
name = item['name']
value = item['value']
# parse optional KeyValuePair properties
scope = item.get('scope', DEFAULT_SCOPE)
user = item.get('user', None)
secret = item.get('secret', False)
ttl = item.get('ttl', None)
# if the value is not a string, convert it to JSON
# all keys in the datastore must strings
if not isinstance(value, six.string_types):
if args.convert:
value = json.dumps(value)
else:
raise ValueError(("Item '%s' has a value that is not a string."
" Either pass in the -c/--convert option to convert"
" non-string types to JSON strings automatically, or"
" convert the data to a string in the file") % name)
# create the KeyValuePair instance
instance = KeyValuePair()
instance.id = name # TODO: refactor and get rid of id
instance.name = name
instance.value = value
instance.scope = scope
if user:
instance.user = user
if secret:
instance.secret = secret
if ttl:
instance.ttl = ttl
# call the API to create/update the KeyValuePair
self.manager.update(instance, **kwargs)
instances.append(instance)
return instances
def test_datastore_operations_list_values(self):
# Verify prefix filtering
mock_api_client = mock.Mock()
mock_api_client.keys.get_all.return_value = []
self._set_mock_api_client(mock_api_client)
self._sensor_service.list_values(local=True, prefix=None)
mock_api_client.keys.get_all.assert_called_with(prefix="core.TestSensor:")
self._sensor_service.list_values(local=True, prefix="ponies")
mock_api_client.keys.get_all.assert_called_with(prefix="core.TestSensor:ponies")
self._sensor_service.list_values(local=False, prefix=None)
mock_api_client.keys.get_all.assert_called_with(prefix=None)
self._sensor_service.list_values(local=False, prefix="ponies")
mock_api_client.keys.get_all.assert_called_with(prefix="ponies")
# No values in the datastore
mock_api_client = mock.Mock()
mock_api_client.keys.get_all.return_value = []
self._set_mock_api_client(mock_api_client)
values = self._sensor_service.list_values(local=True)
self.assertEqual(values, [])
values = self._sensor_service.list_values(local=False)
self.assertEqual(values, [])
# Values in the datastore
kvp1 = KeyValuePair()
kvp1.name = "test1"
kvp1.value = "bar"
kvp2 = KeyValuePair()
kvp2.name = "test2"
kvp2.value = "bar"
mock_return_value = [kvp1, kvp2]
mock_api_client.keys.get_all.return_value = mock_return_value
self._set_mock_api_client(mock_api_client)
values = self._sensor_service.list_values(local=True)
self.assertEqual(len(values), 2)
self.assertEqual(values, mock_return_value)
def set_value(self, name, value, ttl=None, local=True):
"""
Store a value in a dictionary which is local to this class.
"""
if ttl:
raise ValueError('MockSensorService.set_value doesn\'t support "ttl" argument')
name = self._get_full_key_name(name=name, local=local)
instance = KeyValuePair()
instance.id = name
instance.name = name
instance.value = value
self._datastore_items[name] = instance
return True
def set_value(self, name, value, ttl=None, local=True, scope=SYSTEM_SCOPE, encrypt=False):
"""
Store a value in a dictionary which is local to this class.
"""
if ttl:
raise ValueError('MockDatastoreService.set_value doesn\'t support "ttl" argument')
name = self._get_full_key_name(name=name, local=local)
instance = KeyValuePair()
instance.id = name
instance.name = name
instance.value = value
self._datastore_items[name] = instance
return True
def run(self, args, **kwargs):
instance = KeyValuePair()
instance.id = args.name # TODO: refactor and get rid of id
instance.name = args.name
instance.value = args.value
instance.scope = args.scope
instance.user = args.user
if args.secret:
instance.secret = args.secret
if args.encrypted:
instance.encrypted = args.encrypted
if args.ttl:
instance.ttl = args.ttl
return self.manager.update(instance, **kwargs)
def run(self, args, **kwargs):
instance = KeyValuePair()
instance.id = args.name # TODO: refactor and get rid of id
instance.name = args.name
instance.value = args.value
instance.scope = args.scope
instance.user = args.user
if args.secret:
instance.secret = args.secret
if args.encrypted:
instance.encrypted = args.encrypted
if args.ttl:
instance.ttl = args.ttl
return self.manager.update(instance, **kwargs)
def delete_value(self, name, local=True, scope=SYSTEM_SCOPE):
"""
Delete the provided key.
By default, value is deleted from a namespace local to the pack/class. If you want to
delete a global value, pass local=False to this method.
:param name: Name of the key to delete.
:type name: ``str``
:param local: Delete a value in a namespace local to the pack/class. Defaults to True.
:type: local: ``bool``
:param scope: Scope under which item is saved. Defaults to system scope.
:type: local: ``str``
:return: ``True`` on success, ``False`` otherwise.
:rtype: ``bool``
"""
if scope != SYSTEM_SCOPE:
raise ValueError("Scope %s is unsupported." % scope)
name = self._get_full_key_name(name=name, local=local)
client = self.get_api_client()
instance = KeyValuePair()
instance.id = name
instance.name = name
self._logger.debug("Deleting value from the datastore (name=%s)", name)
try:
params = {"scope": scope}
client.keys.delete(instance=instance, params=params)
except Exception as e:
self._logger.exception(
"Exception deleting value from datastore (name=%s): %s", name, e
)
return False
return True
def set_value(self, name, value, ttl=None, local=True):
"""
Set a value for the provided key.
By default, value is set in a namespace local to the sensor. If you want to
set a global value, pass local=False to this method.
:param name: Key name.
:type name: ``str``
:param value: Key value.
:type value: ``str``
:param ttl: Optional TTL (in seconds).
:type ttl: ``int``
:param local: Set value in a namespace local to the sensor. Defaults to True.
:type: local: ``bool``
:return: ``True`` on success, ``False`` otherwise.
:rtype: ``bool``
"""
if local:
name = self._get_key_name_with_sensor_prefix(name=name)
value = str(value)
client = self._get_api_client()
self._logger.audit('Setting value in the datastore (name=%s)', name)
instance = KeyValuePair()
instance.id = name
instance.name = name
instance.value = value
if ttl:
instance.ttl = ttl
client.keys.update(instance=instance)
return True
def set_value(self, name, value, ttl=None, local=True):
"""
Set a value for the provided key.
By default, value is set in a namespace local to the sensor. If you want to
set a global value, pass local=False to this method.
:param name: Key name.
:type name: ``str``
:param value: Key value.
:type value: ``str``
:param ttl: Optional TTL (in seconds).
:type ttl: ``int``
:param local: Set value in a namespace local to the sensor. Defaults to True.
:type: local: ``bool``
:return: ``True`` on success, ``False`` otherwise.
:rtype: ``bool``
"""
if local:
name = self._get_key_name_with_sensor_prefix(name=name)
value = str(value)
client = self._get_api_client()
self._logger.audit('Setting value in the datastore (name=%s)', name)
instance = KeyValuePair()
instance.id = name
instance.name = name
instance.value = value
if ttl:
instance.ttl = ttl
client.keys.update(instance=instance)
return True