def test_dt_set_remove_updated_context(self):
btype = self.client.bucket_type('sets')
bucket = btype.bucket(self.bucket_name)
set = datatypes.Set(bucket, self.key_name)
set.add('X')
set.add('Y')
set.store()
set_parallel = datatypes.Set(bucket, self.key_name)
set_parallel.add('Z')
set_parallel.store()
set.reload()
set.discard('Z')
set.store()
set2 = bucket.get(self.key_name)
self.assertItemsEqual(['X', 'Y'], set2.value)
def test_dt_set_remove_without_context(self):
btype = self.client.bucket_type('sets')
bucket = btype.bucket(self.bucket_name)
set = datatypes.Set(bucket, self.key_name)
set.add("X")
set.add("Y")
set.add("Z")
with self.assertRaises(datatypes.ContextRequired):
set.discard("Y")
def test_dt_set_add_twice(self):
btype = self.client.bucket_type('sets')
bucket = btype.bucket(self.bucket_name)
set = datatypes.Set(bucket, self.key_name)
set.add('X')
set.add('Y')
set.store()
set.reload()
set.add('X')
set.store()
set2 = bucket.get(self.key_name)
self.assertItemsEqual(['X', 'Y'], set2.value)
def test_dt_set_remove_fetching_context(self):
btype = self.client.bucket_type('pytest-sets')
bucket = btype.bucket(self.bucket_name)
set = datatypes.Set(bucket, self.key_name)
set.add('X')
set.add('Y')
set.store()
set.reload()
set.discard('bogus')
set.store()
set2 = bucket.get(self.key_name)
self.assertItemsEqual(['X', 'Y'], set2.value)
def readdir(self, path, fh):
full_path = self._full_path(path)
logger.debug('readdir %s - fh: %s' % (path, fh))
dirents = ['.', '..']
# generate the correct names for the buckets and keys
RiakDirectoryBucketNamespace = NameMapping.legacyPathToRiakBucketName(
riak_directory_namespace_prefix, path)
if (use_riak_directory_structure_for_read_access) and (
RiakDirectoryBucketNamespace is not None):
try:
logger.debug('using RIAK directory structure in %s (for %s)' %
(RiakDirectoryBucketNamespace, path))
# Updating the $prefix+$id+$directoryprefix set with the given information
# we shall use our own riak client instance for this
btype = riak.RiakClient(
host=riak_host, pb_port=riak_port,
protocol='pbc').bucket_type(riak_directory_set_buckettype)
# get the bucket for the directory namespace - this is a sets pre-configured bucket
bucket = btype.bucket(RiakDirectoryBucketNamespace)
# get the correct key inside that bucket
myset = datatypes.Set(bucket, riak_directory_set_directorykey)
# fetch the directory in order to be able to output it...
myset.reload()
for r in dirents:
yield r
for id in myset:
yield id
except Exception as e:
# throw controlled exception
logger.error(
'ERROR retrieving directory structure on RIAK bucket %s (Exception: %s)'
% (RiakDirectoryBucketNamespace, str(e)))
raise FuseOSError(errno.EACCES)
else:
# this is not a valid namespace - so pattern did not match on a directory/filename structure known to be mapped
# --> therefore this method will return to the caller now.
if RiakDirectoryBucketNamespace is None:
# apparently this is not a proper path, so just go ahead and unlink the local file and report on that
logger.warning('%s is not a mappable RIAK bucket - ignoring.' %
(path))
# take local information instead
if os.path.isdir(full_path):
dirents.extend(os.listdir(full_path))
for r in dirents:
yield r
def test_dt_set_add_wins_in_same_op_reversed(self):
btype = self.client.bucket_type('sets')
bucket = btype.bucket(self.bucket_name)
set = datatypes.Set(bucket, self.key_name)
set.add('X')
set.add('Y')
set.store()
set.reload()
set.discard('X')
set.add('X')
set.store()
set2 = bucket.get(self.key_name)
self.assertItemsEqual(['X', 'Y'], set2.value)
def test_dt_set(self):
btype = self.client.bucket_type('sets')
bucket = btype.bucket(self.bucket_name)
myset = datatypes.Set(bucket, self.key_name)
myset.add('Sean')
myset.add('Brett')
myset.store()
otherset = bucket.get(self.key_name)
self.assertIn('Sean', otherset)
self.assertIn('Brett', otherset)
otherset.add('Russell')
otherset.discard('Sean')
otherset.store(return_body=True)
myset.reload()
self.assertIn('Russell', myset)
self.assertIn('Brett', myset)
self.assertNotIn('Sean', myset)
def getattr(self, path, fh=None):
full_path = self._full_path(path)
logger.debug('getattr %s' % (path))
if (use_riak_file_contents_for_read_access):
RiakDirectoryBucketNamespace = NameMapping.legacyPathToRiakBucketName(
riak_directory_namespace_prefix, path)
RiakKeyNamespace = NameMapping.legacyPathToRiakKeyName(path)
if RiakKeyNamespace is None:
# apparently this is not a proper path, so just go ahead and unlink the local file and report on that
logger.warning('%s is not a mappable RIAK bucket - ignoring.' %
(path))
# so ignore...
st = os.lstat(full_path)
else:
# we got a valid path, now just return the default file mask
#st = os.lstat('/etc/passwd')
#check if it is existing by trying to get it's size from the value stored under the file key in the directory bucket
logger.debug(
'Retrieving stored object size and status...%s/%s' %
(RiakDirectoryBucketNamespace, RiakKeyNamespace))
# read the set contents...
btype = riak.RiakClient(host=riak_host,
pb_port=riak_port,
protocol='pbc').bucket_type('sets')
bucket = btype.bucket(RiakDirectoryBucketNamespace)
myset = datatypes.Set(bucket, RiakKeyNamespace)
myset.reload()
if len(myset) > 0:
the_file_size = int(next(iter(myset)))
logger.debug('Got size: %s' % (str(the_file_size)))
else:
the_file_size = None
if (the_file_size is None):
logger.debug(
'Requested file %s apparently not existing in %s checking locally...'
% (RiakKeyNamespace, RiakDirectoryBucketNamespace))
if os.path.isfile(full_path):
logger.debug(
'file exists locally - using this one...(%s)' %
(full_path))
return dict(st_mode=(S_IFREG
| riak_contents_file_mask),
st_nlink=1,
st_uid=riak_contents_file_uid,
st_gid=riak_contents_file_gid,
st_size=os.path.getsize(full_path),
st_ctime=time(),
st_mtime=time(),
st_atime=time())
else:
raise FuseOSError(errno.ENOENT)
else:
# You've now got a ``RiakObject``. To get at the binary data, call:
return dict(st_mode=(S_IFREG | riak_contents_file_mask),
st_nlink=1,
st_uid=riak_contents_file_uid,
st_gid=riak_contents_file_gid,
st_size=the_file_size,
st_ctime=time(),
st_mtime=time(),
st_atime=time())
else:
st = os.lstat(full_path)
logger.debug('Attributes: %s' % (st))
return dict((key, getattr(st, key))
for key in ('st_atime', 'st_ctime', 'st_gid', 'st_mode',
'st_mtime', 'st_nlink', 'st_size', 'st_uid'))
def release(self, path, fh):
logger.debug('release %s - fh: %s' % (path, fh))
# First step: get the correct names for buckets and keys
RiakBucketNamespace = NameMapping.legacyPathToRiakBucketName(
riak_namespace_prefix, path)
RiakKeyNamespace = NameMapping.legacyPathToRiakKeyName(path)
RiakDirectoryBucketNamespace = NameMapping.legacyPathToRiakBucketName(
riak_directory_namespace_prefix, path)
# this is not a valid namespace - so pattern did not match on a directory/filename structure known to be mapped
# --> therefore this method will return to the caller now.
if RiakBucketNamespace is None:
# apparently this is not a proper path, so just go ahead and close the handle but report
logger.warning('%s is not a mappable RIAK bucket - ignoring.' %
(path))
return os.close(fh)
# this seems to be a mappable RIAK path - so let's log and move on to actually moving the file to RIAK
logger.debug(
'updating on RIAK bucket %s the key %s (%s bytes to write.)' %
(RiakBucketNamespace, RiakKeyNamespace,
os.path.getsize(self._full_path(path))))
try:
# create RiakClient instance
riakClient = riak.RiakClient(host=riak_host,
pb_port=riak_port,
protocol='pbc')
# get the correct bucket
release_bucket = riakClient.bucket(RiakBucketNamespace)
# open the local file for read access off the filesystem
the_imge_data = open(self._full_path(path), 'rb').read()
# get the key+value and add it to the bucket
riak_image = release_bucket.new(RiakKeyNamespace,
encoded_data=the_imge_data,
content_type=riak_content_type)
# send to RIAK afterall
riak_image.store()
except Exception as e:
logger.error(
'ERROR updating on RIAK bucket %s the key %s (Exception: %s)' %
(RiakBucketNamespace, RiakKeyNamespace, str(e)))
else:
logger.debug('DONE updating on RIAK bucket %s the key %s' %
(RiakBucketNamespace, RiakKeyNamespace))
# is the directory structure (also) maintained in RIAK - if so, go ahead and update properly
if (maintain_riak_directory_structure):
logger.debug('updating %s directory structure for %s' %
(RiakDirectoryBucketNamespace, RiakKeyNamespace))
# Updating the $prefix+$id+$directoryprefix set with the given information
# we shall use our own riak client instance for this
btype = riak.RiakClient(
host=riak_host, pb_port=riak_port,
protocol='pbc').bucket_type(riak_directory_set_buckettype)
# get the bucket for the directory namespace - this is a sets pre-configured bucket
bucket = btype.bucket(RiakDirectoryBucketNamespace)
# get the correct key inside that bucket
myset = datatypes.Set(bucket, riak_directory_set_directorykey)
# add this file to the directory - if it's already there it won't be added (handled by RIAK)
myset.add(RiakKeyNamespace)
logger.debug('updating size (%s) entry in directory %s/%s' %
(str(os.stat(self._full_path(path)).st_size),
RiakDirectoryBucketNamespace, RiakKeyNamespace))
btype.bucket(RiakDirectoryBucketNamespace).new(
RiakKeyNamespace,
encoded_data=str(os.stat(self._full_path(path)).st_size),
content_type=riak_content_type)
mysizeset = datatypes.Set(bucket, RiakKeyNamespace)
mysizeset.add(str(os.stat(self._full_path(path)).st_size))
# send to RIAK afterall
myset.store()
mysizeset.store()
logger.debug('DONE updating directory structure')
# should the local copy of the file be removed or not
if (remove_local_copy_after_successful_mapping):
logger.debug('removing local copy %s' % (path))
# first close it
returnvalueclose = os.close(fh)
# then remove it (just locally)
os.unlink(self._full_path(path))
# return the correct return value as per close
return returnvalueclose
else:
logger.debug('not removing local copy %s' % (path))
return os.close(fh)
def unlink(self, path):
logger.debug('unlink path %s' % (path))
# all Riak interactions only necessary when this module actually is managing directory structures in RIAK
if (maintain_riak_directory_structure):
try:
# generate the correct names for the buckets and keys
RiakBucketNamespace = NameMapping.legacyPathToRiakBucketName(
riak_namespace_prefix, path)
RiakKeyNamespace = NameMapping.legacyPathToRiakKeyName(path)
RiakDirectoryBucketNamespace = NameMapping.legacyPathToRiakBucketName(
riak_directory_namespace_prefix, path)
# this is not a valid namespace - so pattern did not match on a directory/filename structure known to be mapped
# --> therefore this method will return to the caller now.
if RiakKeyNamespace is None:
# apparently this is not a proper path, so just go ahead and unlink the local file and report on that
logger.warning(
'%s is not a mappable RIAK bucket - ignoring.' %
(path))
return os.unlink(self._full_path(path))
logger.debug(
'updating %s directory structure for %s (discarding)' %
(RiakDirectoryBucketNamespace, RiakKeyNamespace))
# Updating the $prefix+$id+$directoryprefix set with the given information
# we shall use our own riak client instance for this
btype = riak.RiakClient(
host=riak_host, pb_port=riak_port,
protocol='pbc').bucket_type(riak_directory_set_buckettype)
# get the bucket for the directory namespace - this is a sets pre-configured bucket
bucket = btype.bucket(RiakDirectoryBucketNamespace)
# get the correct key inside that bucket
myset = datatypes.Set(bucket, riak_directory_set_directorykey)
# fetch the directory in order to be change it...
myset.reload()
# add this file to the directory - if it's already there it won't be added (handled by RIAK)
myset.discard(RiakKeyNamespace)
mysizeset = datatypes.Set(bucket,
RiakKeyNamespace) # this is the set
mysizeset.reload()
logger.debug('SizeSet len %s' % (len(mysizeset)))
if len(mysizeset) > 0:
the_file_size = int(next(iter(mysizeset)))
logger.debug('Discarding object size %s in riak %s' %
(the_file_size, RiakKeyNamespace))
mysizeset.discard(str(the_file_size))
# send to RIAK afterall
#logger.debug('Storing directory structure...')
myset.store()
#logger.debug('Storing size structure...')
mysizeset.store()
logger.debug('DONE updating directory structure')
# now update the bucket itself by removing the key
# create RiakClient instance
riakClient = riak.RiakClient(host=riak_host,
pb_port=riak_port,
protocol='pbc')
# get the correct bucket
release_bucket = riakClient.bucket(RiakBucketNamespace)
# remove that key
release_bucket.delete(RiakKeyNamespace)
except Exception as e:
# throw controlled exception but do not remove the actual local file due to errors in the process...
logger.error(
'ERROR updating directory structure on RIAK bucket %s the key %s (Exception: %s)'
% (RiakDirectoryBucketNamespace, RiakKeyNamespace, str(e)))
raise FuseOSError(errno.EACCES)
# finally do the local unlink when all of the above where successful
if (os.path.exists(self._full_path(path))):
return os.unlink(self._full_path(path))
else:
return
def rename(self, old, new):
logger.debug('rename %s - target: %s' % (old, new))
if (maintain_riak_directory_structure):
# check if this is the directory that matches our patterns
try:
# generate the correct names for the buckets and keys
RiakBucketNamespace = NameMapping.legacyPathToRiakBucketName(
riak_namespace_prefix, old)
RiakNewBucketNamespace = NameMapping.legacyPathToRiakBucketName(
riak_namespace_prefix, new)
RiakKeyNamespace = NameMapping.legacyPathToRiakKeyName(old)
RiakNewKeyNamespace = NameMapping.legacyPathToRiakKeyName(new)
RiakDirectoryBucketNamespace = NameMapping.legacyPathToRiakBucketName(
riak_directory_namespace_prefix, old)
RiakDirectoryNewBucketNamespace = NameMapping.legacyPathToRiakBucketName(
riak_directory_namespace_prefix, new)
logger.debug('rename %s %s %s %s' %
(RiakKeyNamespace, RiakNewKeyNamespace,
RiakDirectoryBucketNamespace,
RiakDirectoryNewBucketNamespace))
# only continue of the Bucket Names match, not supported to rename between buckets
if (RiakDirectoryBucketNamespace ==
RiakDirectoryNewBucketNamespace):
# this is not a valid namespace - so pattern did not match on a directory/filename structure known to be mapped
# --> therefore this method will return to the caller now.
if RiakKeyNamespace is None:
# apparently this is not a proper path, so just go ahead and unlink the local file and report on that
logger.warning(
'%s is not a mappable RIAK bucket - ignoring.' %
(old))
# so do the local rename anways...
return os.rename(self._full_path(old),
self._full_path(new))
logger.debug(
'updating %s directory structure for %s to %s (renaming)'
% (RiakDirectoryBucketNamespace, RiakKeyNamespace,
RiakNewKeyNamespace))
# Updating the $prefix+$id+$directoryprefix set with the given information
# we shall use our own riak client instance for this
btype = riak.RiakClient(host=riak_host,
pb_port=riak_port,
protocol='pbc').bucket_type(
riak_directory_set_buckettype)
# get the bucket for the directory namespace - this is a sets pre-configured bucket
bucket = btype.bucket(RiakDirectoryBucketNamespace)
# get the correct key inside that bucket
myset = datatypes.Set(bucket,
riak_directory_set_directorykey)
# fetch the directory in order to be change it...
myset.reload()
# add this file to the directory - if it's already there it won't be added (handled by RIAK)
myset.discard(RiakKeyNamespace)
myset.add(RiakNewKeyNamespace)
# send to RIAK afterall
myset.store()
logger.debug('DONE updating RIAK directory structure')
# create RiakClient instance
riakClient = riak.RiakClient(host=riak_host,
pb_port=riak_port,
protocol='pbc')
# get the correct bucket
old_bucket = riakClient.bucket(RiakBucketNamespace)
new_bucket = riakClient.bucket(RiakNewBucketNamespace)
# read the old key contents...
the_imge_data = old_bucket.get(RiakKeyNamespace)
logger.debug('Got the old key contents from RIAK (%s/%s)' %
(RiakBucketNamespace, RiakKeyNamespace))
# and write those to the new bucket...
riak_image = new_bucket.new(RiakNewKeyNamespace,
encoded_data=the_imge_data,
content_type=riak_content_type)
logger.debug('Wrote contents to RIAK %s' %
(RiakNewKeyNamespace))
# remove the old one...
old_bucket.delete(RiakKeyNamespace)
logger.debug('Removed old key from RIAK %s' %
(RiakKeyNamespace))
# send to RIAK afterall
riak_image.store()
# if the local copy is removed, it's gone anyways...
if not (remove_local_copy_after_successful_mapping):
# we shall now rename the local file and finish...
return os.rename(self._full_path(old),
self._full_path(new))
else:
logger.debug(
'ERROR unsupported rename of file between buckets (%s -> %s)'
% (RiakDirectoryBucketNamespace,
RiakDirectoryNewBucketNamespace))
raise FuseOSError(errno.ENOTSUP)
except Exception as e:
# throw controlled exception but do not remove the actual local file due to errors in the process...
logger.error(
'ERROR updating directory structure on RIAK bucket %s the key %s (Exception: %s)'
% (RiakDirectoryBucketNamespace, RiakKeyNamespace, str(e)))
raise FuseOSError(errno.EACCES)
else:
# go ahead and rename locally
return os.rename(self._full_path(old), self._full_path(new))
#!/usr/bin/env python
# this tool dumps the directory listing of a specified merchant directory bucket to STDOUT
#
# This takes two parameters:
# parameter 1: the bucket name
import riak
import sys
import riak.datatypes as datatypes
if (len(sys.argv) == 1):
print('Give 1 parameters: <bucket name>')
print('Example:')
print(' python riak-dump-directory.py IMGDIR_test')
print('')
print('This will dump the IMGDIR_test directories contents to STDOUT')
else:
btype = riak.RiakClient(pb_port=8087, protocol='pbc').bucket_type('sets')
bucket = btype.bucket(sys.argv[1])
myset = datatypes.Set(bucket, 'directory')
myset.reload()
for id in myset:
print('%s' % (id))
