def __init__(self, path, drive, account, container, logger,
uid=DEFAULT_UID, gid=DEFAULT_GID):
self.root = path
self.container = container
self.datadir = path_std(os.path.join(path, drive, self.container))
self.account = account
assert logger is not None
self.logger = logger
self.metadata = {}
self.container_info = None
self.object_info = None
self.uid = int(uid)
self.gid = int(gid)
self.db_file = _db_file
self.dir_exists = os.path.exists(self.datadir)
self.metauuid = 'ff89f933b2ca8df40'
self.fhr_path = parent_path(self.datadir)
self.metafile = self.fhr_path+ '/' + self.metauuid+'/' + self.datadir.split('/')[-1]
self.meta_fhr_path = parent_path(self.metafile)
if self.meta_fhr_dir_is_deleted():
self.create_dir_object(self.meta_fhr_path)
if not os.path.exists(self.datadir):
return
if self.dir_exists:
try:
self.metadata = meta_read_metadata(self.metafile)
except EOFError:
meta_create_container_metadata(self.datadir,self.metafile)
else:
return
if self.container:
if not self.metadata:
meta_create_container_metadata(self.datadir,self.metafile)
self.metadata = meta_read_metadata(self.metafile)
else:
if not validate_container(self.metadata):
meta_create_container_metadata(self.datadir,self.metafile)
self.metadata = meta_read_metadata(self.metafile)
def get_tree_info(datapath,metauuid):
objects = os.listdir(datapath)
container_list = []
for obj in objects:
if 'ff89f933b2ca8df40' == obj:
continue
list_item = {}
obj_path = path_std(os.path.join(datapath, obj))
obj_meta_path = os.path.join('/'.join(obj_path.split('/')[:-1]),metauuid,obj_path.split('/')[-1])
metadata = meta_read_metadata(obj_meta_path)
if not metadata or not validate_object(metadata):
metadata = meta_create_object_metadata(obj_path,obj_meta_path)
if metadata:
list_item.update({'name':obj})
list_item.update({'modificationTime':str(metadata[X_TIMESTAMP])})
list_item.update({'bytes':int(metadata[X_CONTENT_LENGTH])})
list_item.update({'md5':metadata[X_ETAG]})
list_item.update({'ftype':metadata[X_FILE_TYPE]})
if metadata.get('metadata'):
list_item.update({'metadata':metadata['metadata']})
if os.path.isdir(obj_path):
cList = get_tree_info(obj_path,metauuid)
list_item.update({'list':cList})
container_list.append(list_item)
return container_list
def non_recursive_iter(self):
objects = os.listdir(self.datadir)
if 'ff89f933b2ca8df40' in objects:
objects.remove('ff89f933b2ca8df40')
if objects:
objects.sort()
container_list = []
if objects:
for obj in objects:
list_item = {}
obj_path = path_std(os.path.join(self.datadir, obj))
obj_meta_path = os.path.join('/'.join(obj_path.split('/')[:-1]),self.metauuid,obj_path.split('/')[-1])
metadata = meta_read_metadata(obj_meta_path)
if not metadata or not validate_object(metadata):
metadata = meta_create_object_metadata(obj_path,obj_meta_path)
if metadata:
list_item.update({'name':obj})
list_item.update({'modificationTime':str(metadata[X_TIMESTAMP])})
list_item.update({'bytes':int(metadata[X_CONTENT_LENGTH])})
list_item.update({'md5':metadata[X_ETAG]})
list_item.update({'ftype':metadata[X_FILE_TYPE]})
if metadata.has_key('metadata'):
list_item.update({'metadata':metadata['metadata']})
container_list.append(list_item)
return container_list
def __init__(self, path, drive, account, container,direr, logger,
uid=DEFAULT_UID, gid=DEFAULT_GID,recycle_uuid=''):
self.root = path
self.container = container
self.datadir = os.path.join(path, drive,container,direr)
self.account = account
assert logger is not None
self.logger = logger
self.metadata = {}
self.container_info = None
self.object_info = None
self.uid = int(uid)
self.gid = int(gid)
self.db_file = _db_file
self.dir_exists = os.path.exists(self.datadir)
self.fhr_path = parent_path(self.datadir)
if 'recycle' == container:
self.recycle_uuid = recycle_uuid
self.metafile = os.path.join(path, drive,container +'/'+'meta',self.recycle_uuid)
self.meta_fhr_path = parent_path(self.metafile)
if self.meta_fhr_dir_is_deleted():
# self.create_dir_object(self.meta_fhr_path)
self.tmp_meta_fhr_path = self.meta_fhr_path
while not os.path.exists(self.tmp_meta_fhr_path):
self.create_dir_object(self.tmp_meta_fhr_path)
self.tmp_meta_fhr_path = parent_path(parent_path(self.tmp_meta_fhr_path)) + '/' +self.metauuid
if not os.path.exists(self.datadir):
return
self.metadata = meta_read_metadata(self.metafile)
def prefix_list_objects_iter(self, limit, marker, end_marker,
prefix, delimiter, path):
"""
Returns tuple of name, created_at, size, content_type, etag.
"""
if path:
prefix = path = path.rstrip('/') + '/'
delimiter = '/'
if delimiter and not prefix:
prefix = ''
self.update_object_count()
objects, object_count, bytes_used = self.object_info
if objects:
objects.sort()
if objects and prefix:
objects = self.filter_prefix(objects, prefix)
if objects and delimiter:
objects = self.filter_delimiter(objects, delimiter, prefix)
if objects and marker:
objects = self.filter_marker(objects, marker)
if objects and end_marker:
objects = self.filter_end_marker(objects, end_marker)
if objects and limit:
if len(objects) > limit:
objects = self.filter_limit(objects, limit)
container_list = []
if objects:
for obj in objects:
list_item = {}
obj_path = path_std(os.path.join(self.datadir, obj))
obj_meta_path = os.path.join('/'.join(obj_path.split('/')[:-1]),self.metauuid,obj_path.split('/')[-1])
metadata = meta_read_metadata(obj_meta_path)
if not metadata or not validate_object(metadata):
metadata = meta_create_object_metadata(obj_path,obj_meta_path)
if metadata:
list_item.update({'name':obj})
list_item.update({'modificationTime':str(metadata[X_TIMESTAMP])})
list_item.update({'bytes':int(metadata[X_CONTENT_LENGTH])})
list_item.update({'md5':metadata[X_ETAG]})
list_item.update({'ftype':metadata[X_FILE_TYPE]})
if metadata.has_key('metadata'):
list_item.update({'metadata':metadata['metadata']})
container_list.append(list_item)
return container_list
def list_containers_iter(self, limit, marker, end_marker,
prefix, delimiter):
"""
Return tuple of name, object_count, bytes_used, 0(is_subdir).
Used by account server.
"""
if delimiter and not prefix:
prefix = ''
self.update_container_count()
containers, container_count = self.container_info
if containers:
containers.sort()
if containers and prefix:
containers = self.filter_prefix(containers, prefix)
if containers and delimiter:
containers = self.filter_delimiter(containers, delimiter, prefix)
if containers and marker:
containers = self.filter_marker(containers, marker)
if containers and end_marker:
containers = self.filter_end_marker(containers, end_marker)
if containers and limit:
if len(containers) > limit:
containers = self.filter_limit(containers, limit)
account_list = []
if containers:
for cont in containers:
list_item = []
metadata = None
list_item.append(cont)
cont_path = path_std(os.path.join(self.datadir, cont))
cont_meta_path = '/'.join(cont_path.split('/')[:-1])+ '/' + self.metauuid+ '/' + cont_path.split('/')[-1]
metadata = meta_read_metadata(cont_meta_path)
if not metadata or not validate_container(metadata):
metadata = meta_create_container_metadata(cont_path,cont_meta_path)
if metadata:
list_item.append(metadata[X_OBJECTS_COUNT])
list_item.append(metadata[X_BYTES_USED])
list_item.append(0)
account_list.append(list_item)
return account_list
def list_objects_meta_iter(self,start,limit):
self.update_object_count()
objects, object_count, bytes_used = self.object_info
if objects:
objects.sort()
container_list = []
if objects:
for obj in objects:
list_item = []
list_item.append(obj)
obj_path = os.path.join(self.datadir, obj)
metadata = meta_read_metadata(obj_path)
if metadata:
list_item.append(metadata[X_TIMESTAMP])
list_item.append(int(metadata[X_CONTENT_LENGTH]))
list_item.append(metadata[X_ETAG])
list_item.append(metadata[X_FILE_TYPE])
if 'recycle' == self.container:
list_item.append(metadata['user_path'])
list_item.append(metadata['recycle_uuid'])
list_item.append(metadata['ftype'])
container_list.append(list_item)
if 'recycle' == self.container:
container_list.sort(key=lambda x:float(x[1]))
container_list.reverse()
if start:
start = int(start)
if limit:
limit = int(limit)
return container_list[start:start+limit]
else:
return container_list[start:]
return container_list
def __init__(self, path, device, partition, account, container, obj,
logger, keep_data_fp=False, disk_chunk_size=65536,
uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None,recycle_uuid=''):
self.iter_hook = iter_hook
self.disk_chunk_size = disk_chunk_size
obj = obj.strip('/')
if '/' in obj:
self.obj_path, self.obj = obj.rsplit('/', 1)
else:
self.obj_path = ''
self.obj = obj
if self.obj_path:
self.name = '/'.join((container, self.obj_path))
else:
self.name = container
self.datadir = os.path.join(path, device, self.name)
self.device_path = os.path.join(path, device)
self.container_path = os.path.join(path, device, container)
self.tmpdir = os.path.join(path, device, 'tmp')
self.logger = logger
self.metadata = {}
self.data_file = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.keep_cache = False
self.is_dir = False
self.is_valid = True
self.uid = int(uid)
self.gid = int(gid)
self.data_file = os.path.join(self.datadir, self.obj)
self.fhr_path = parent_path(self.data_file)
self.container = container
if 'recycle' == container:
self.recycle_uuid = recycle_uuid
self.metafile = os.path.join(self.datadir[:-5]+'/'+'meta',self.recycle_uuid)
self.meta_fhr_path = parent_path(self.metafile)
if self.meta_fhr_dir_is_deleted():
#self.create_dir_object(self.meta_fhr_path)
self.tmp_meta_fhr_path = self.meta_fhr_path
while not os.path.exists(self.tmp_meta_fhr_path):
self.create_dir_object(self.tmp_meta_fhr_path)
self.tmp_meta_fhr_path = parent_path(parent_path(self.tmp_meta_fhr_path)) + '/' +self.metauuid
if not os.path.exists(self.datadir + '/' + self.obj):
return
if os.path.isdir(self.datadir + '/' + self.obj):
self.is_dir = True
else:
self.fp = do_open(self.data_file, 'rb')
if not keep_data_fp:
self.close(verify_file=False)
self.metadata = meta_read_metadata(self.metafile)
if not self.metadata:
meta_create_object_metadata(self.data_file,self.metafile)
self.metadata = meta_read_metadata(self.metafile)
if not validate_object(self.metadata):
meta_create_object_metadata(self.data_file,self.metafile)
self.metadata = meta_read_metadata(self.metafile)
self.filter_metadata()
def get_container_timestamp(self, container):
cont_path = path_std(os.path.join(self.datadir, container))
cont_meta_path = '/'.join(cont_path.split('/')[:-1])+ '/' + self.metauuid+ '/' + cont_path.split('/')[-1]
metadata = meta_read_metadata(cont_meta_path)
return int(metadata.get(X_PUT_TIMESTAMP, '0')) or None
def __init__(
self,
path,
device,
partition,
account,
container,
obj,
logger,
keep_data_fp=False,
disk_chunk_size=65536,
uid=DEFAULT_UID,
gid=DEFAULT_GID,
iter_hook=None,
):
self.iter_hook = iter_hook
self.disk_chunk_size = disk_chunk_size
obj = obj.strip("/")
if "/" in obj:
self.obj_path, self.obj = obj.rsplit("/", 1)
else:
self.obj_path = ""
self.obj = obj
if self.obj_path:
self.name = "/".join((container, self.obj_path))
else:
self.name = container
# Absolute path for obj directory.
self.datadir = os.path.join(path, device, self.name)
self.device_path = os.path.join(path, device)
self.container_path = os.path.join(path, device, container)
self.cntpath = os.path.join(path, device, container)
self.tmpdir = os.path.join(path, device, "tmp")
self.logger = logger
self.metadata = {}
self.data_file = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.keep_cache = False
self.is_dir = False
self.is_valid = True
self.uid = int(uid)
self.gid = int(gid)
self.data_file = path_std(os.path.join(self.datadir, self.obj))
self.fhr_path = parent_path(self.data_file)
self.container = container
self.metauuid = "ff89f933b2ca8df40"
self.metafile = self.fhr_path + "/" + self.metauuid + "/" + self.data_file.split("/")[-1]
self.meta_fhr_path = parent_path(self.metafile)
self.cnt_flag = self.cnt_exists()
if not self.cnt_flag:
return
if self.meta_fhr_dir_is_deleted():
# self.create_dir_object(self.meta_fhr_path)
self.tmp_meta_fhr_path = self.meta_fhr_path
while not os.path.exists(self.tmp_meta_fhr_path):
self.create_dir_object(self.tmp_meta_fhr_path)
self.tmp_meta_fhr_path = parent_path(parent_path(self.tmp_meta_fhr_path)) + "/" + self.metauuid
if not os.path.exists(self.datadir + "/" + self.obj):
return
if os.path.isdir(self.datadir + "/" + self.obj):
self.is_dir = True
else:
self.fp = do_open(self.data_file, "rb")
if not keep_data_fp:
self.close(verify_file=False)
self.metadata = meta_read_metadata(self.metafile)
if not self.metadata:
meta_create_object_metadata(self.data_file, self.metafile)
self.metadata = meta_read_metadata(self.metafile)
if not validate_object(self.metadata):
meta_create_object_metadata(self.data_file, self.metafile)
self.metadata = meta_read_metadata(self.data_file)
self.filter_metadata()
