def __init__(self, path, device, partition, account, container, obj, logger, keep_data_fp=False, disk_chunk_size=65536, uid=DEFAULT_UID, gid=DEFAULT_GID): self.disk_chunk_size = disk_chunk_size #Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc obj = obj.strip('/') if '/' in obj: self.obj_path, self.obj = obj.rsplit('/', 1) else: self.obj_path = '' self.obj = obj if self.obj_path: self.name = '/'.join((container, self.obj_path)) else: self.name = container #Absolute path for obj directory. self.datadir = os.path.join(path, device, storage_directory(DATADIR, partition, self.name)) self.device_path = os.path.join(path, device) self.container_path = os.path.join(path, device, container) self.tmpdir = os.path.join(path, device, 'tmp') self.logger = logger self.metadata = {} self.meta_file = None self.data_file = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.keep_cache = False self.is_dir = False self.is_valid = True self.uid = int(uid) self.gid = int(gid) if not os.path.exists(self.datadir + '/' + self.obj): return self.data_file = os.path.join(self.datadir, self.obj) self.metadata = read_metadata(self.datadir + '/' + self.obj) if not self.metadata: create_object_metadata(self.datadir + '/' + self.obj) self.metadata = read_metadata(self.datadir + '/' + self.obj) if not validate_object(self.metadata): self.logger.error('Metadata validation failed %s %s' % \ (self.data_file, self.metadata)) self.metadata = {} self.is_valid = False self.data_file = None return if os.path.isdir(self.datadir + '/' + self.obj): self.is_dir = True else: self.fp = do_open(self.data_file, 'rb') if not keep_data_fp: self.close(verify_file=False)
def create_dir_object(self, dir_path): #TODO: if object already exists??? if os.path.exists(dir_path) and not os.path.isdir(dir_path): self.logger.error("Deleting file %s", dir_path) do_unlink(dir_path) #If dir aleady exist just override metadata. mkdirs(dir_path) do_chown(dir_path, self.uid, self.gid) create_object_metadata(dir_path) return True
def create_dir_object(self, dir_name, timestamp): dir_path = os.path.join(self.container_path, dir_name) #TODO: if object already exists??? if os.path.exists(dir_path) and not os.path.isdir(dir_path): os.unlink(dir_path) #If dir aleady exist just override metadata. mkdirs(dir_path) os.chown(dir_path, self.uid, self.gid) create_object_metadata(dir_path) return True
def list_container_objects(self, limit, marker, end_marker, prefix, delimiter, path, out_content_type): """ Returns tuple of name, created_at, size, content_type, etag. """ if path: prefix = path = path.rstrip('/') + '/' delimiter = '/' if delimiter and not prefix: prefix = '' #print 'Container_list prefix, del, datadir', prefix,\ #delimiter, self.datadir objects = [] object_count = 0 bytes_used = 0 container_list = [] objects, object_count, bytes_used = get_container_details(self.datadir, self.memcache) #print 'cont', object_count, self.metadata if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \ int(self.metadata[X_BYTES_USED]) != bytes_used: self.metadata[X_OBJECTS_COUNT] = object_count self.metadata[X_BYTES_USED] = bytes_used self.update_container(self.metadata) if objects: objects.sort() if objects and prefix: objects = self.filter_prefix(objects, prefix) if objects and delimiter: objects = self.filter_delimiter(objects, delimiter, prefix) if objects and marker: objects = self.filter_marker(objects, marker) if objects and end_marker: objects = self.filter_end_marker(objects, end_marker) if objects and limit: if len(objects) > limit: objects = self.filter_limit(objects, limit) if objects: for obj in objects: list_item = [] metadata = None list_item.append(obj) if out_content_type != 'text/plain': metadata = read_metadata(self.datadir + '/' + obj) if not metadata: metadata = create_object_metadata(self.datadir + '/' + obj) #print 'Gaurav list_obj meta', metadata if metadata: list_item.append(metadata[X_TIMESTAMP]) list_item.append(metadata[X_CONTENT_LENGTH]) list_item.append(metadata[X_CONTENT_TYPE]) list_item.append(metadata[X_ETAG]) container_list.append(list_item) #print 'Gaurav list_container objs', container_list return container_list
def __init__(self, path, device, partition, account, container, obj, logger, keep_data_fp=False, disk_chunk_size=65536, uid=DEFAULT_UID, gid=DEFAULT_GID): self.disk_chunk_size = disk_chunk_size #Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc obj = obj.strip('/') if '/' in obj: self.obj_path, self.obj = obj.rsplit('/', 1) else: self.obj_path = '' self.obj = obj #self.name = '/'.join((account, container, self.obj_path)) if self.obj_path: self.name = '/'.join((container, self.obj_path)) else: self.name = container #name_hash = hash_path(account, container, obj) self.datadir = os.path.join(path, device, storage_directory(DATADIR, partition, self.name)) #print 'Gaurav Diskfile datadir', account, container, obj, path,\ #device, self.datadir self.device_path = os.path.join(path, device) self.container_path = os.path.join(path, device, container) self.tmpdir = os.path.join(path, device, 'tmp') self.logger = logger self.metadata = {} self.meta_file = None self.data_file = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.keep_cache = False self.is_dir = False self.is_valid = True self.uid = int(uid) self.gid = int(gid) if not os.path.exists(self.datadir + '/' + self.obj): return #TODO: marker objects. self.data_file = os.path.join(self.datadir, self.obj) self.metadata = read_metadata(self.datadir + '/' + self.obj) if not self.metadata: #self.metadata = get_object_details(self.datadir + '/' + self.obj) #self.update_object(self.metadata) create_object_metadata(self.datadir + '/' + self.obj) self.metadata = read_metadata(self.datadir + '/' + self.obj) if not validate_object(self.metadata): self.metadata = {} self.is_valid = False self.data_file = None return if os.path.isdir(self.datadir + '/' + self.obj): self.is_dir = True else: self.fp = open(self.data_file, 'rb') if not keep_data_fp: self.close(verify_file=False)