def test_disk_file_finalize_put(self):
tmpdir = os.path.join(self.filesystems, self.filesystem)
os.rmdir(tmpdir)
df = self._simple_get_diskfile()
metadata = {'X-metadata-value': 'data'}
file_name = '_'.join([
hash_path(df.account),
hash_path(df.account, df.container), df._name_hash
])
with df.create() as writer:
self.assertTrue(os.path.exists(df._tmpdir))
self.assertTrue(
os.path.exists(os.path.join(df._tmpdir, file_name + ".data")))
self.assertTrue(
os.path.exists(os.path.join(df._tmpdir, file_name + ".meta")))
writer.put(metadata)
self.assertTrue(
os.path.exists(os.path.join(df._datadir, df._name_hash + ".data")))
self.assertTrue(
os.path.exists(os.path.join(df._metadir, df._name_hash + ".meta")))
self.assertTrue(os.path.exists(df._tmpdir))
self.assertFalse(
os.path.exists(os.path.join(df._tmpdir, df._name_hash + ".data")))
self.assertFalse(
os.path.exists(os.path.join(df._tmpdir, df._name_hash + ".meta")))
def acquire_bulk_delete_lock(self, filesystem, acc_dir, cont_dir, account,
container, object_names):
"""
"""
try:
self._logger.debug("Received obj list: %s" % object_names)
req_mode = WRITE_MODE
obj_service_id = socket.gethostname() + "_object-server"
active_records = []
for obj in object_names:
obj_hash = "%s/%s/%s" % (hash_path(account), \
hash_path(account, container), \
hash_path(account, container, obj))
self._logger.debug("Locking obj: %s" % obj_hash)
active_records.append(
ActiveRecord(int(req_mode), obj_hash, 'BULK_DELETE',
obj_service_id, int(0)))
status_obj = self.asyn_helper.call("add_bulk_delete_transaction",
active_records)
ret_val = status_obj.get_value()
self._logger.debug(
"Transaction manager returning acquire_bulk_delete_lock with: %s"
% ret_val)
return status_obj
except Exception as err:
self._logger.error(
'ERROR Transaction method add_bulk_delete_transaction failed')
self._logger.exception(err)
raise err
def create_path(self, filesystem, acc_dir, cont_dir, account, container):
"""
Create hashed path for container library
param: filesystem: Filesystem name
param: account: Account name
param: container: Container name
"""
acc_path = hash_path(account)
con_path = hash_path(account, container)
return '%s/%s/%s/%s/%s/%s' % (self.__fs_base, \
filesystem, acc_dir, acc_path, cont_dir, con_path)
def __init__(self,
mgr,
filesystem,
threadPool_write,
object_lib,
acc_dir,
cont_dir,
obj_dir,
account=None,
container=None,
obj=None):
self._mgr = mgr
self._filesystem = filesystem
self._acc_dir = acc_dir
self._cont_dir = cont_dir
self._obj_dir = obj_dir
self._threadpool_write = threadPool_write or ThreadPool(nthreads=0)
self._object_lib = object_lib
self._logger = mgr.logger
self._disk_chunk_size = mgr.disk_chunk_size
self._object_server_id = mgr.object_server_id
self._datadir = os.path.join(self._filesystem, acc_dir,
hash_path(account), cont_dir,
hash_path(account, container), obj_dir,
DATADIR)
self._metadir = os.path.join(self._filesystem, acc_dir,
hash_path(account), cont_dir,
hash_path(account, container), obj_dir,
METADIR)
self._tmpdir = os.path.join(self._filesystem, TMPDIR,
self._object_server_id)
self._meta_ext = ".meta"
self._data_ext = ".data"
if account and container and obj:
self._name = '/' + '/'.join((account, container, obj))
self._account = account
self._container = container
self._obj = obj
self._name_hash = hash_path(account, container, obj)
else:
_name = None
self._account = None
self._container = None
self._obj = None
self._name_hash = None
self._metadata = None
self._data_file = None
self._meta_file = None
self._fp_meta = None
self._fp = None
self._content_length = None
def test_disk_file_create_tmp_file(self):
tmpdir = os.path.join(self.filesystems, self.filesystem)
os.rmdir(tmpdir)
df = self._simple_get_diskfile()
file_name = '_'.join([
hash_path(df.account),
hash_path(df.account, df.container), df._name_hash
])
with df.create():
self.assert_(os.path.exists(df._tmpdir))
self.assert_(
os.path.exists(os.path.join(df._tmpdir, file_name + ".data")))
self.assert_(
os.path.exists(os.path.join(df._tmpdir, file_name + ".meta")))
def _verify_name_matches_hash(self, data_file):
hash_from_fs = os.path.basename(self._datadir)
hash_from_name = hash_path(self._name.lstrip('/'))
if hash_from_fs != hash_from_name:
raise self._quarantine(
data_file,
"Hash of name in metadata does not match directory name")
def _get_data_file_name(self):
"""
Gives the full path of data file
"""
if self._name_hash:
return os.path.join(self._datadir,
self._name_hash + self._data_ext)
self._name_hash = hash_path(self._account, self._container, self._obj)
return os.path.join(self._datadir, self._name_hash + self._data_ext)
def __get_target_path(self, metadata, file_name):
'''
Get targets path of object file and meta file storage.
:param tmp_dir: tmp directory
:param file_name: file name
:returns: data file target path and meta file target path
'''
data_file = file_name.split('_')[-1] + '.data'
meta_file = file_name.split('_')[-1] + '.meta'
path = metadata['name']
account, container, obj = split_path(path, 1, 3, True)
#_, file_system, directory, global_map_version, comp_no = \
file_system, directory = \
self.__object_ring.get_node(account, container, obj, only_fs_dir_flag=True)
acc_dir, cont_dir, obj_dir = directory.split('/')
data_target_path = join(EXPORT_PATH, file_system, acc_dir, hash_path(account), \
cont_dir, hash_path(account, container), \
obj_dir, DATADIR, data_file)
meta_target_path = join(EXPORT_PATH, file_system, acc_dir, hash_path(account), \
cont_dir, hash_path(account, container), \
obj_dir, METADIR, meta_file)
return data_target_path, meta_target_path
def test_PUT_common(self):
timestamp = normalize_timestamp(time())
req = Request.blank('/fs1/a1/d1/o1/a/c/o',
environ={'REQUEST_METHOD': 'PUT'},
headers={
'X-Timestamp': timestamp,
'Content-Length': '6',
'Content-Type': 'application/octet-stream',
'X-Container-Host': '1.2.3:0000',
'X-GLOBAL-MAP-VERSION': 4.0
})
req.body = 'VERIFY'
print req, "-------------------"
resp = req.get_response(self.object_controller)
print "-------status", resp.status
self.assertEquals(resp.status_int, 201)
print "----hello", self.filesystems
print "----hello2", self.filesystems
objfile = os.path.join(self.filesystems, "fs1", "a1", hash_path("a"),
"d1", hash_path("a", "c"), "o1", "data",
hash_path('a', 'c', 'o')) + ".data"
metafile = os.path.join(self.filesystems, "fs1", "a1", hash_path("a"),
"d1", hash_path("a", "c"), "o1", "meta",
hash_path('a', 'c', 'o')) + ".meta"
print objfile, metafile
try:
self.assert_(os.path.isfile(objfile))
except:
print os.path.isfile(objfile)
try:
self.assert_(os.path.isfile(metafile))
except:
print os.path.isfile(metafile), "------------"
self.assertEquals(open(objfile, 'r').read(), 'VERIFY')
self.assertEquals(
read_metadata(metafile), {
'X-Timestamp': timestamp,
'Content-Length': '6',
'ETag': '0b4c12d7e0a73840c1c4f148fda3b037',
'Content-Type': 'application/octet-stream',
'name': '/a/c/o'
})
def test_DELETE(self):
# Test osd.obj.server.ObjectController.DELETE
req = Request.blank('/fs1/a1/d1/o1/a/c',
environ={
'REQUEST_METHOD': 'DELETE',
'HTTP_X_GLOBAL_MAP_VERSION': 4.0
},
headers={'X-Container-Host': '1.2.3:0000'})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 400)
req = Request.blank('/fs1/a1/d1/o1/a/c/o',
environ={
'REQUEST_METHOD': 'DELETE',
'HTTP_X_GLOBAL_MAP_VERSION': 4.0
},
headers={'X-Container-Host': '1.2.3:0000'})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 400)
timestamp = normalize_timestamp(1002)
req = Request.blank('/fs1/a1/d1/o1/a/c/o',
environ={
'REQUEST_METHOD': 'PUT',
'HTTP_X_GLOBAL_MAP_VERSION': 4.0
},
headers={
'X-Timestamp': timestamp,
'Content-Type': 'application/octet-stream',
'Content-Length': '4',
'X-Container-Host': '1.2.3:0000'
})
req.body = 'test'
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 201)
data_1002_file = os.path.join(self.filesystems, self.filesystem, "a1",
hash_path("a"), "d1", hash_path(
"a", "c"), self.dir, "data",
hash_path('a', 'c', 'o')) + ".data"
meta_1002_file = os.path.join(self.filesystems, self.filesystem, "a1",
hash_path("a"), "d1", hash_path(
"a", "c"), self.dir, "meta",
hash_path('a', 'c', 'o')) + ".meta"
self.assertTrue(os.path.isfile(data_1002_file))
self.assertTrue(os.path.isfile(meta_1002_file))
##check for conflict
timestamp = normalize_timestamp(1001)
req = Request.blank('/fs1/a1/d1/o1/a/c/o',
environ={
'REQUEST_METHOD': 'DELETE',
'HTTP_X_GLOBAL_MAP_VERSION': 4.0
},
headers={
'X-Timestamp': timestamp,
'X-Container-Host': '1.2.3:0000'
})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 409)
self.assertTrue(os.path.isfile(data_1002_file))
self.assertTrue(os.path.isfile(meta_1002_file))
#check for bad request
req = Request.blank('/fs1/a1/d1/o1/a/c/o',
environ={
'REQUEST_METHOD': 'DELETE',
'HTTP_X_GLOBAL_MAP_VERSION': 4.0
},
headers={'X-Container-Host': '1.2.3:0000'})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 400)
#check for successfull deletion
timestamp = normalize_timestamp(1003)
req = Request.blank('/fs1/a1/d1/o1/a/c/o',
environ={
'REQUEST_METHOD': 'DELETE',
'HTTP_X_GLOBAL_MAP_VERSION': 4.0
},
headers={
'X-Timestamp': timestamp,
'X-Container-Host': '1.2.3:0000'
})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 204)
self.assertFalse(os.path.isfile(data_1002_file))
self.assertFalse(os.path.isfile(meta_1002_file))
def test_GET(self):
req = Request.blank('/fs1/a1/d1/o1/a/c',
environ={
'REQUEST_METHOD': 'GET',
'HTTP_X_GLOBAL_MAP_VERSION': 4.0
},
headers={'X-Container-Host': '1.2.3:0000'})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 400)
req = Request.blank('/fs1/a1/d1/o1/a/c/o',
environ={
'REQUEST_METHOD': 'GET',
'HTTP_X_GLOBAL_MAP_VERSION': 4.0
},
headers={'X-Container-Host': '1.2.3:0000'})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 404)
timestamp = normalize_timestamp(time())
req = Request.blank('/fs1/a1/d1/o1/a/c/o',
environ={
'REQUEST_METHOD': 'PUT',
'HTTP_X_GLOBAL_MAP_VERSION': 4.0
},
headers={
'X-Timestamp': timestamp,
'Content-Type': 'application/x-test',
'X-Object-Meta-1': 'One',
'X-Object-Meta-Two': 'Two',
'X-Container-Host': '1.2.3:0000'
})
req.body = 'VERIFY'
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 201)
req = Request.blank('/fs1/a1/d1/o1/a/c/o',
environ={
'REQUEST_METHOD': 'GET',
'HTTP_X_GLOBAL_MAP_VERSION': 4.0
},
headers={'X-Container-Host': '1.2.3:0000'})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.body, 'VERIFY')
self.assertEquals(resp.content_length, 6)
self.assertEquals(resp.content_type, 'application/x-test')
self.assertEquals(resp.headers['content-length'], '6')
self.assertEquals(resp.headers['content-type'], 'application/x-test')
self.assertEquals(
resp.headers['last-modified'],
strftime('%a, %d %b %Y %H:%M:%S GMT',
gmtime(math.ceil(float(timestamp)))))
self.assertEquals(resp.headers['etag'],
'"0b4c12d7e0a73840c1c4f148fda3b037"')
self.assertEquals(resp.headers['x-object-meta-1'], 'One')
self.assertEquals(resp.headers['x-object-meta-two'], 'Two')
req = Request.blank('/fs1/a1/d1/o1/a/c/o',
environ={
'REQUEST_METHOD': 'GET',
'HTTP_X_GLOBAL_MAP_VERSION': 4.0
},
headers={'X-Container-Host': '1.2.3:0000'})
req.range = 'bytes=1-3'
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 206)
self.assertEquals(resp.body, 'ERI')
self.assertEquals(resp.headers['content-length'], '3')
req = Request.blank('/fs1/a1/d1/o1/a/c/o',
environ={
'REQUEST_METHOD': 'GET',
'HTTP_X_GLOBAL_MAP_VERSION': 4.0
},
headers={'X-Container-Host': '1.2.3:0000'})
req.range = 'bytes=1-'
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 206)
self.assertEquals(resp.body, 'ERIFY')
self.assertEquals(resp.headers['content-length'], '5')
req = Request.blank('/fs1/a1/d1/o1/a/c/o',
environ={
'REQUEST_METHOD': 'GET',
'HTTP_X_GLOBAL_MAP_VERSION': 4.0
},
headers={'X-Container-Host': '1.2.3:0000'})
req.range = 'bytes=-2'
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 206)
self.assertEquals(resp.body, 'FY')
self.assertEquals(resp.headers['content-length'], '2')
objfile = os.path.join(self.filesystems, "fs1", 'a1', hash_path("a"),
"d1", hash_path("a", "c"), "o1", "data",
hash_path('a', 'c', 'o')) + ".data"
os.unlink(objfile)
req = Request.blank('/fs1/a1/d1/o1/a/c/o',
environ={
'REQUEST_METHOD': 'GET',
'HTTP_X_GLOBAL_MAP_VERSION': 4.0
},
headers={'X-Container-Host': '1.2.3:0000'})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 404)
sleep(.00001)
timestamp = normalize_timestamp(time())
req = Request.blank('/fs1/a1/d1/o1/a/c/o',
environ={
'REQUEST_METHOD': 'PUT',
'HTTP_X_GLOBAL_MAP_VERSION': 4.0
},
headers={
'X-Container-Host': '1.2.3:0000',
'X-Timestamp': timestamp,
'Content-Type': 'application:octet-stream',
'Content-Length': '6'
})
req.body = 'VERIFY'
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 201)
sleep(.00001)
timestamp = normalize_timestamp(time())
req = Request.blank('/fs1/a1/d1/o1/a/c/o',
environ={
'REQUEST_METHOD': 'DELETE',
'HTTP_X_GLOBAL_MAP_VERSION': 4.0
},
headers={
'X-Timestamp': timestamp,
'X-Container-Host': '1.2.3:0000'
})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 204)
req = Request.blank('/fs1/a1/d1/o1/a/c/o',
environ={
'REQUEST_METHOD': 'GET',
'HTTP_X_GLOBAL_MAP_VERSION': 4.0
},
headers={'X-Container-Host': '1.2.3:0000'})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 404)
def test_get_cont_dir_by_hash(self):
cont_hash = hash_path('a', 'c')
self.assertEquals(self.ring_data_obj.get_cont_dir_by_hash(cont_hash),
'd6')
def test_get_obj_dir_by_hash(self):
obj_hash = hash_path('a', 'c', 'o')
self.assertEquals(self.object_ring_obj.get_obj_dir_by_hash(obj_hash),
(10, 'c10'))
def create(self):
"""
Context manager to create a data file and metadata file .
We create a temporary file first, and
then return a DiskFileWriter object to encapsulate the state.
.. note::
An implementation is not required to perform on-disk
preallocations even if the parameter is specified. But if it does
and it fails, it must raise a `DiskFileNoSpace` exception.
"""
#if not exists(self._tmpdir):
# mkdirs(self._tmpdir)
tmp_data_file = '_'.join([hash_path(self.account), \
hash_path(self.account, self.container), \
self._name_hash]) + self._data_ext
tmp_meta_file = '_'.join([hash_path(self.account), \
hash_path(self.account, self.container), \
self._name_hash]) + self._meta_ext
#need to check if we need to generate the error
def _create_tmp_file(tmpdir, tmp_data_file, tmp_meta_file):
if not exists(tmpdir):
mkdirs(tmpdir)
fd_data, tmppath_data = create_tmp_file(tmpdir, tmp_data_file)
fd_meta, tmppath_meta = create_tmp_file(tmpdir, tmp_meta_file)
return (fd_data, tmppath_data, fd_meta, tmppath_meta)
(fd_data, tmppath_data, fd_meta, tmppath_meta) = \
self._threadpool_write.force_run_in_thread( \
_create_tmp_file, self._tmpdir, tmp_data_file, tmp_meta_file)
_raw_rpipe, wpipe = os.pipe()
rpipe = greenio.GreenPipe(_raw_rpipe, 'rb', bufsize=0)
#fd_data, tmppath_data = create_tmp_file(self._tmpdir, tmp_data_file)
#rfd_meta, tmppath_meta = create_tmp_file(self._tmpdir, tmp_meta_file)
try:
yield DiskFileWriter(self._name, self._name_hash, self._datadir,
self._metadir, fd_data, fd_meta, tmppath_data,
tmppath_meta, rpipe, wpipe,
self._threadpool_write, self._object_lib,
self._logger)
finally:
def _finally(fd_data, fd_meta, tmppath_data, tmppath_meta):
try:
os.close(fd_data)
os.close(fd_meta)
except OSError:
pass
try:
os.unlink(tmppath_data)
os.unlink(tmppath_meta)
except OSError:
pass
self._threadpool_write.force_run_in_thread(_finally, fd_data,
fd_meta, tmppath_data,
tmppath_meta)
rpipe.close()
os.close(wpipe)
def test_get_directories_by_hash(self):
acc_hash = hash_path('a', )
cont_hash = hash_path('a', 'c')
obj_hash = hash_path('a', 'c', 'o')
self.assertEquals(self.object_ring_obj.get_directories_by_hash \
(acc_hash, cont_hash, obj_hash), ('fs5', 'a10/d6/c10'))
def test_get_directory_by_hash(self):
acc_hash = hash_path('a')
cont_hash = hash_path('a', 'c')
self.assertEquals(
self.container_ring_obj.get_directory_by_hash(acc_hash, cont_hash),
'a10/d6')
def test_get_acc_dir_by_hash(self):
acc_hash = hash_path('a')
self.assertEquals(self.ring_data_obj.get_acc_dir_by_hash(acc_hash),
'a10')