def __init__(self, path, device, partition, account, container, obj,
logger, keep_data_fp=False, disk_chunk_size=65536,
uid=DEFAULT_UID, gid=DEFAULT_GID):
self.disk_chunk_size = disk_chunk_size
#Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc
obj = obj.strip('/')
if '/' in obj:
self.obj_path, self.obj = obj.rsplit('/', 1)
else:
self.obj_path = ''
self.obj = obj
if self.obj_path:
self.name = '/'.join((container, self.obj_path))
else:
self.name = container
#Absolute path for obj directory.
self.datadir = os.path.join(path, device,
storage_directory(DATADIR, partition, self.name))
self.device_path = os.path.join(path, device)
self.container_path = os.path.join(path, device, container)
self.tmpdir = os.path.join(path, device, 'tmp')
self.logger = logger
self.metadata = {}
self.meta_file = None
self.data_file = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.keep_cache = False
self.is_dir = False
self.is_valid = True
self.uid = int(uid)
self.gid = int(gid)
if not os.path.exists(self.datadir + '/' + self.obj):
return
self.data_file = os.path.join(self.datadir, self.obj)
self.metadata = read_metadata(self.datadir + '/' + self.obj)
if not self.metadata:
create_object_metadata(self.datadir + '/' + self.obj)
self.metadata = read_metadata(self.datadir + '/' + self.obj)
if not validate_object(self.metadata):
self.logger.error('Metadata validation failed %s %s' % \
(self.data_file, self.metadata))
self.metadata = {}
self.is_valid = False
self.data_file = None
return
if os.path.isdir(self.datadir + '/' + self.obj):
self.is_dir = True
else:
self.fp = do_open(self.data_file, 'rb')
if not keep_data_fp:
self.close(verify_file=False)
def list_account_containers(self, limit, marker, end_marker,
prefix, delimiter,
out_content_type):
"""
Return tuple of name, object_count, bytes_used, 0(is_subdir).
Used by account server.
"""
if delimiter and not prefix:
prefix = ''
containers = []
container_count = 0
account_list = []
containers, container_count = get_account_details(self.datadir, self.memcache)
if int(self.metadata[X_CONTAINER_COUNT]) != container_count:
self.metadata[X_CONTAINER_COUNT] = container_count
self.update_account(self.metadata)
if containers:
containers.sort()
if containers and prefix:
containers = self.filter_prefix(containers, prefix)
if containers and delimiter:
containers = self.filter_delimiter(containers, delimiter, prefix)
if containers and marker:
containers = self.filter_marker(containers, marker)
if containers and end_marker:
containers = self.filter_end_marker(containers, end_marker)
if containers and limit:
if len(containers) > limit:
containers = self.filter_limit(containers, limit)
if containers:
for cont in containers:
list_item = []
metadata = None
list_item.append(cont)
if out_content_type != 'text/plain':
metadata = read_metadata(self.datadir + '/' + cont)
if not metadata:
metadata = create_container_metadata(self.datadir + '/' + cont)
if metadata:
list_item.append(metadata[X_OBJECTS_COUNT])
list_item.append(metadata[X_BYTES_USED])
list_item.append(0)
account_list.append(list_item)
#print 'Gaurav list_containet objs', objects
return account_list
def __init__(self, path, device, partition, account, container, logger, uid=DEFAULT_UID, gid=DEFAULT_GID):
if container:
#self.name = '/'.join((account, container))
self.name = container
else:
self.name = None
if self.name:
self.datadir = os.path.join(path, device,
storage_directory(DATADIR, partition, self.name))
else:
self.datadir = os.path.join(path, device)
#print 'Gaurav DiskDir datadir', account, container, path,\
#device, self.datadir
self.device_path = os.path.join(path, device)
self.logger = logger
self.metadata = {}
self.uid = int(uid)
self.gid = int(gid)
self.dir_exists = os.path.isdir (self.datadir)
if self.dir_exists:
self.metadata = read_metadata(self.datadir)
else:
return
if container:
if not self.metadata:
#objects, object_count, bytes_used = get_container_details(self.datadir)
#self.update_container(object_count, bytes_used)
create_container_metadata(self.datadir)
self.metadata = read_metadata(self.datadir)
ret = validate_container(self.metadata)
else:
if not self.metadata:
#containers, container_count = get_account_details(self.datadir)
#self.update_acocunt(container_count)
create_account_metadata(self.datadir)
self.metadata = read_metadata(self.datadir)
ret = validate_account(self.metadata)
if not ret:
self.dir_exists = False
self.metadata = {}
def __init__(self, path, device, partition, account, container, logger,
memcache=None, uid=DEFAULT_UID, gid=DEFAULT_GID):
if container:
self.name = container
else:
self.name = None
if self.name:
self.datadir = os.path.join(path, device,
storage_directory(DATADIR, partition, self.name))
else:
self.datadir = os.path.join(path, device)
self.device_path = os.path.join(path, device)
self.logger = logger
self.metadata = {}
self.uid = int(uid)
self.gid = int(gid)
self.memcache = memcache
self.dir_exists = os.path.isdir (self.datadir)
if self.dir_exists:
self.metadata = read_metadata(self.datadir)
else:
return
if container:
if not self.metadata:
create_container_metadata(self.datadir, memcache=self.memcache)
self.metadata = read_metadata(self.datadir)
ret = validate_container(self.metadata)
else:
if not self.metadata:
create_account_metadata(self.datadir, memcache=self.memcache)
self.metadata = read_metadata(self.datadir)
ret = validate_account(self.metadata)
if not ret:
self.logger.error('Metadata validation failed %s %s' % \
(self.datadir, self.metadata))
self.dir_exists = False
self.metadata = {}
def list_container_objects(self, limit, marker, end_marker,
prefix, delimiter, path, out_content_type):
"""
Returns tuple of name, created_at, size, content_type, etag.
"""
if path:
prefix = path = path.rstrip('/') + '/'
delimiter = '/'
if delimiter and not prefix:
prefix = ''
#print 'Container_list prefix, del, datadir', prefix,\
#delimiter, self.datadir
objects = []
object_count = 0
bytes_used = 0
container_list = []
objects, object_count, bytes_used = get_container_details(self.datadir,
self.memcache)
#print 'cont', object_count, self.metadata
if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \
int(self.metadata[X_BYTES_USED]) != bytes_used:
self.metadata[X_OBJECTS_COUNT] = object_count
self.metadata[X_BYTES_USED] = bytes_used
self.update_container(self.metadata)
if objects:
objects.sort()
if objects and prefix:
objects = self.filter_prefix(objects, prefix)
if objects and delimiter:
objects = self.filter_delimiter(objects, delimiter, prefix)
if objects and marker:
objects = self.filter_marker(objects, marker)
if objects and end_marker:
objects = self.filter_end_marker(objects, end_marker)
if objects and limit:
if len(objects) > limit:
objects = self.filter_limit(objects, limit)
if objects:
for obj in objects:
list_item = []
metadata = None
list_item.append(obj)
if out_content_type != 'text/plain':
metadata = read_metadata(self.datadir + '/' + obj)
if not metadata:
metadata = create_object_metadata(self.datadir + '/' + obj)
#print 'Gaurav list_obj meta', metadata
if metadata:
list_item.append(metadata[X_TIMESTAMP])
list_item.append(metadata[X_CONTENT_LENGTH])
list_item.append(metadata[X_CONTENT_TYPE])
list_item.append(metadata[X_ETAG])
container_list.append(list_item)
#print 'Gaurav list_container objs', container_list
return container_list
def __init__(self, path, device, partition, account, container, obj,
logger, keep_data_fp=False, disk_chunk_size=65536, uid=DEFAULT_UID,
gid=DEFAULT_GID):
self.disk_chunk_size = disk_chunk_size
#Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc
obj = obj.strip('/')
if '/' in obj:
self.obj_path, self.obj = obj.rsplit('/', 1)
else:
self.obj_path = ''
self.obj = obj
#self.name = '/'.join((account, container, self.obj_path))
if self.obj_path:
self.name = '/'.join((container, self.obj_path))
else:
self.name = container
#name_hash = hash_path(account, container, obj)
self.datadir = os.path.join(path, device,
storage_directory(DATADIR, partition, self.name))
#print 'Gaurav Diskfile datadir', account, container, obj, path,\
#device, self.datadir
self.device_path = os.path.join(path, device)
self.container_path = os.path.join(path, device, container)
self.tmpdir = os.path.join(path, device, 'tmp')
self.logger = logger
self.metadata = {}
self.meta_file = None
self.data_file = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.keep_cache = False
self.is_dir = False
self.is_valid = True
self.uid = int(uid)
self.gid = int(gid)
if not os.path.exists(self.datadir + '/' + self.obj):
return
#TODO: marker objects.
self.data_file = os.path.join(self.datadir, self.obj)
self.metadata = read_metadata(self.datadir + '/' + self.obj)
if not self.metadata:
#self.metadata = get_object_details(self.datadir + '/' + self.obj)
#self.update_object(self.metadata)
create_object_metadata(self.datadir + '/' + self.obj)
self.metadata = read_metadata(self.datadir + '/' + self.obj)
if not validate_object(self.metadata):
self.metadata = {}
self.is_valid = False
self.data_file = None
return
if os.path.isdir(self.datadir + '/' + self.obj):
self.is_dir = True
else:
self.fp = open(self.data_file, 'rb')
if not keep_data_fp:
self.close(verify_file=False)
#name, ext = fname.rsplit('.', 1)
if fname == self.obj:
try:
os.unlink(os.path.join(self.datadir, fname))
except OSError, err:
if err.errno != errno.ENOENT:
raise
if self.obj_path:
obj_dirs = self.obj_path.split('/')
tmp_path = self.obj_path
if len(obj_dirs):
while tmp_path:
#TODO: Check dir is empty (Done in rmdirs)
dir_path = os.path.join(self.container_path, tmp_path)
metadata = read_metadata(dir_path)
#if metadata[X_OBJECT_TYPE] == DIR:
rmdirs(dir_path)
if '/' in tmp_path:
tmp_path = tmp_path.rsplit('/', 1)[0]
else:
break
self.metadata = {}
self.data_file = None
def drop_cache(self, fd, offset, length):
"""Method for no-oping buffer cache drop method."""
if not self.keep_cache: