class CloudFilesFD(object):
'''Acts like a file() object, but attached to a cloud files object'''
def __init__(self, cffs, container, obj, mode):
self.cffs = cffs
self.container = container
self.name = obj
self.mode = mode
self.closed = False
self.total_size = 0
self.stream = None
if not all([container, obj]):
self.closed = True
raise IOSError(EPERM, 'Container and object requred')
self.container = self.cffs._get_container(self.container)
if 'r' in self.mode:
self.obj = self.container.get_object(self.name)
logging.debug("read fd obj.name=%r obj.size=%r" % (self.obj.name, self.obj.size))
else: #write
self.obj = ChunkObject(self.container, obj)
self.obj.content_type = mimetypes.guess_type(obj)[0]
self.obj.prepare_chunk()
def write(self, data):
'''Write data to the object'''
if 'r' in self.mode:
raise IOSError(EPERM, "Can't write to stream opened for read")
self.obj.send_chunk(data)
def close(self):
'''Close the object and finish the data transfer'''
if 'r' in self.mode:
return
self.obj.finish_chunk()
def read(self, size=65536):
'''Read data from the object.
We can use just one request because 'seek' is not supported.
NB: It uses the size passed into the first call for all subsequent calls'''
if not self.stream:
self.stream = self.obj.stream(size)
logging.debug("read size=%r, total_size=%r, obj.size=%r" % (size, self.total_size, self.obj.size))
try:
buff = self.stream.next()
self.total_size += len(buff)
except StopIteration:
return ""
else:
return buff
def seek(self, *kargs, **kwargs):
'''Seek in the object: FIXME doesn't work and raises an error'''
logging.debug("seek args=%s, kargs=%s" % (str(kargs), str(kwargs)))
raise IOSError(EPERM, "Seek not implemented")
class CloudFilesFD(object):
"""Acts like a file() object, but attached to a cloud files object"""
def __init__(self, cffs, container, obj, mode):
self.cffs = cffs
self.container = container
self.name = obj
self.mode = mode
self.closed = False
self.total_size = 0
if not all([container, obj]):
self.closed = True
raise IOSError(EPERM, "Container and object requred")
self.container = self.cffs._get_container(self.container)
if "r" in self.mode:
self.obj = self.container.get_object(self.name)
logging.debug("read fd obj.name=%r obj.size=%r" % (self.obj.name, self.obj.size))
else: # write
self.obj = ChunkObject(self.container, obj)
self.obj.content_type = mimetypes.guess_type(obj)[0]
self.obj.prepare_chunk()
def write(self, data):
"""Write data to the object"""
if "r" in self.mode:
raise IOSError(EPERM, "Can't write to stream opened for read")
self.obj.send_chunk(data)
def close(self):
"""Close the object and finish the data transfer"""
if "r" in self.mode:
return
self.obj.finish_chunk()
def read(self, size=65536):
"""Read data from the object"""
logging.debug("read size=%r, total_size=%r, obj.size=%r" % (size, self.total_size, self.obj.size))
readsize = size
if (self.total_size + size) > self.obj.size:
readsize = self.obj.size - self.total_size
if self.total_size >= self.obj.size:
return ""
else:
offset = self.total_size
self.total_size += size
return self.obj.read(size=readsize, offset=offset)
def seek(self, *kargs, **kwargs):
"""Seek in the object: FIXME doesn't work and raises an error"""
logging.debug("seek args=%s, kargs=%s" % (str(kargs), str(kwargs)))
raise IOSError(EPERM, "Seek not implemented")
class ObjectStorageFD(object):
"""File alike object attached to the Object Storage."""
split_size = 0
def __init__(self, cffs, container, obj, mode):
self.cffs = cffs
self.container = container
self.name = obj
self.mode = mode
self.closed = False
self.total_size = 0
self.part_size = 0
self.part = 0
self.headers = dict()
self.content_type = mimetypes.guess_type(self.name)[0]
self.pending_copy_task = None
self.obj = None
# this is only used by `seek`, so we delay the HEAD request until is required
self.size = None
if not all([container, obj]):
self.closed = True
raise IOSError(EPERM, 'Container and object required')
logging.debug("ObjectStorageFD object: %r (mode: %r)" % (obj, mode))
if 'r' in self.mode:
logging.debug("read fd %r" % self.name)
else: # write
logging.debug("write fd %r" % self.name)
self.obj = ChunkObject(self.conn, self.container, self.name, content_type=self.content_type)
@property
def part_base_name(self):
return u"%s.part" % self.name
@property
def part_name(self):
return u"%s/%.6d" % (self.part_base_name, self.part)
@property
def conn(self):
"""Connection to the storage."""
return self.cffs.conn
def _start_copy_task(self):
"""
Copy the first part of a multi-part file to its final location and create
the manifest file.
This happens in the background, pending_copy_task must be cleaned up at
the end.
"""
def copy_task(conn, container, name, part_name, part_base_name):
# open a new connection
conn = ProxyConnection(None, preauthurl=conn.url, preauthtoken=conn.token)
headers = { 'x-copy-from': "/%s/%s" % (container, name) }
logging.debug("copying first part %r/%r, %r" % (container, part_name, headers))
try:
conn.put_object(container, part_name, headers=headers, contents=None)
except ClientException as ex:
logging.error("Failed to copy %s: %s" % (name, ex.http_reason))
sys.exit(1)
# setup the manifest
headers = { 'x-object-manifest': "%s/%s" % (container, part_base_name) }
logging.debug("creating manifest %r/%r, %r" % (container, name, headers))
try:
conn.put_object(container, name, headers=headers, contents=None)
except ClientException as ex:
logging.error("Failed to store the manifest %s: %s" % (name, ex.http_reason))
sys.exit(1)
logging.debug("copy task done")
self.pending_copy_task = multiprocessing.Process(target=copy_task,
args=(self.conn,
self.container,
self.name,
self.part_name,
self.part_base_name,
),
)
self.pending_copy_task.start()
@translate_objectstorage_error
def write(self, data):
"""Write data to the object."""
if 'r' in self.mode:
raise IOSError(EPERM, "File is opened for read")
# large file support
if self.split_size:
# data can be of any size, so we need to split it in split_size chunks
offs = 0
while offs < len(data):
if self.part_size + len(data) - offs > self.split_size:
current_size = self.split_size-self.part_size
logging.debug("data is to large (%r), using %s" % (len(data), current_size))
else:
current_size = len(data)-offs
self.part_size += current_size
if not self.obj:
self.obj = ChunkObject(self.conn, self.container, self.part_name, content_type=self.content_type)
self.obj.send_chunk(data[offs:offs+current_size])
offs += current_size
if self.part_size == self.split_size:
logging.debug("current size is %r, split_file is %r" % (self.part_size, self.split_size))
self.obj.finish_chunk()
# this obj is not valid anymore, will create a new one if a new part is required
self.obj = None
# make it the first part
if self.part == 0:
self._start_copy_task()
self.part_size = 0
self.part += 1
else:
self.obj.send_chunk(data)
@translate_objectstorage_error
def close(self):
"""Close the object and finish the data transfer."""
if 'r' in self.mode:
return
if self.pending_copy_task:
logging.debug("waiting for a pending copy task...")
self.pending_copy_task.join()
logging.debug("wait is over")
if self.pending_copy_task.exitcode != 0:
raise IOSError(EIO, 'Failed to store the file')
if self.obj is not None:
self.obj.finish_chunk()
def read(self, size=65536):
"""
Read data from the object.
We can use just one request because 'seek' is not fully supported.
NB: It uses the size passed into the first call for all subsequent calls.
"""
if self.obj is None:
if self.total_size > 0:
self.conn.range_from = self.total_size
# we need to open a new connection to inject the `Range` header
if self.conn.http_conn:
self.conn.http_conn[1].close()
self.conn.http_conn = None
_, self.obj = self.conn.get_object(self.container, self.name, resp_chunk_size=size)
logging.debug("read size=%r, total_size=%r (range_from: %s)" % (size,
self.total_size, self.conn.range_from))
try:
buff = self.obj.next()
self.total_size += len(buff)
except StopIteration:
return ""
else:
return buff
def seek(self, offset, whence=None):
"""
Seek in the object.
It's supported only for read operations because of object storage limitations.
"""
logging.debug("seek offset=%s, whence=%s" % (str(offset), str(whence)))
if 'r' in self.mode:
if self.size is None:
meta = self.conn.head_object(self.container, self.name)
try:
self.size = int(meta["content-length"])
except ValueError:
raise IOSError(EPERM, "Invalid file size")
if not whence:
offs = offset
elif whence == 1:
offs = self.total_size + offset
elif whence == 2:
offs = self.size - offset
else:
raise IOSError(EPERM, "Invalid file offset")
if offs < 0 or offs > self.size:
raise IOSError(EPERM, "Invalid file offset")
# we need to start over after a seek call
if self.obj is not None:
del self.obj # GC the generator
self.obj = None
self.total_size = offs
else:
raise IOSError(EPERM, "Seek not available for write operations")
class ObjectStorageFD(object):
"""File alike object attached to the Object Storage."""
def __init__(self, cffs, container, obj, mode):
self.cffs = cffs
self.container = container
self.name = obj
self.mode = mode
self.closed = False
self.total_size = 0
self.headers = dict()
self.obj = None
# this is only used by `seek`, so we delay the HEAD request until is required
self.size = None
if not all([container, obj]):
self.closed = True
raise IOSError(EPERM, 'Container and object required')
logging.debug("ObjectStorageFD object: %r (mode: %r)" % (obj, mode))
if 'r' in self.mode:
logging.debug("read fd %r" % self.name)
else: # write
logging.debug("write fd %r" % self.name)
self.obj = ChunkObject(self.conn, self.container, self.name, content_type=mimetypes.guess_type(self.name)[0])
@property
def conn(self):
"""Connection to the storage."""
return self.cffs.conn
def write(self, data):
"""Write data to the object."""
if 'r' in self.mode:
raise IOSError(EPERM, "File is opened for read")
self.obj.send_chunk(data)
def close(self):
"""Close the object and finish the data transfer."""
if 'r' in self.mode:
return
self.obj.finish_chunk()
def read(self, size=65536):
"""
Read data from the object.
We can use just one request because 'seek' is not fully supported.
NB: It uses the size passed into the first call for all subsequent calls.
"""
if self.obj is None:
if self.total_size > 0:
self.conn.range_from = self.total_size
# we need to open a new connection to inject the `Range` header
if self.conn.http_conn:
self.conn.http_conn[1].close()
self.conn.http_conn = None
_, self.obj = self.conn.get_object(self.container, self.name, resp_chunk_size=size)
logging.debug("read size=%r, total_size=%r (range_from: %s)" % (size,
self.total_size, self.conn.range_from))
try:
buff = self.obj.next()
self.total_size += len(buff)
except StopIteration:
return ""
else:
return buff
def seek(self, offset, whence=None):
"""
Seek in the object.
It's supported only for read operations because of object storage limitations.
"""
logging.debug("seek offset=%s, whence=%s" % (str(offset), str(whence)))
if 'r' in self.mode:
if self.size is None:
meta = self.conn.head_object(self.container, self.name)
try:
self.size = int(meta["content-length"])
except ValueError:
raise IOSError(EPERM, "Invalid file size")
if not whence:
offs = offset
elif whence == 1:
offs = self.total_size + offset
elif whence == 2:
offs = self.size - offset
else:
raise IOSError(EPERM, "Invalid file offset")
if offs < 0 or offs > self.size:
raise IOSError(EPERM, "Invalid file offset")
# we need to start over after a seek call
if self.obj is not None:
del self.obj # GC the generator
self.obj = None
self.total_size = offs
else:
raise IOSError(EPERM, "Seek not available for write operations")
class ObjectStorageFD(object):
"""File alike object attached to the Object Storage."""
split_size = 0
def __init__(self, cffs, container, obj, mode):
self.cffs = cffs
self.container = container
self.name = obj
self.mode = mode
self.closed = False
self.total_size = 0
self.part_size = 0
self.part = 0
self.headers = dict()
self.content_type = mimetypes.guess_type(self.name)[0]
self.pending_copy_task = None
self.obj = None
# this is only used by `seek`, so we delay the HEAD request until is required
self.size = None
if not all([container, obj]):
self.closed = True
raise IOSError(EPERM, 'Container and object required')
logging.debug("ObjectStorageFD object: %r (mode: %r)" % (obj, mode))
if 'r' in self.mode:
logging.debug("read fd %r" % self.name)
else: # write
logging.debug("write fd %r" % self.name)
self.obj = ChunkObject(self.conn, self.container, self.name, content_type=self.content_type)
@property
def part_base_name(self):
return u"%s.part" % self.name
@property
def part_name(self):
return u"%s/%.6d" % (self.part_base_name, self.part)
@property
def conn(self):
"""Connection to the storage."""
return self.cffs.conn
def _start_copy_task(self):
"""
Copy the first part of a multi-part file to its final location and create
the manifest file.
This happens in the background, pending_copy_task must be cleaned up at
the end.
"""
def copy_task(conn, container, name, part_name, part_base_name):
# open a new connection
conn = ProxyConnection(None, preauthurl=conn.url, preauthtoken=conn.token)
headers = { 'x-copy-from': "/%s/%s" % (container, name) }
logging.debug("copying first part %r/%r, %r" % (container, part_name, headers))
try:
conn.put_object(container, part_name, headers=headers, contents=None)
except ClientException as ex:
logging.error("Failed to copy %s: %s" % (name, ex.http_reason))
sys.exit(1)
# setup the manifest
headers = { 'x-object-manifest': "%s/%s" % (container, part_base_name) }
logging.debug("creating manifest %r/%r, %r" % (container, name, headers))
try:
conn.put_object(container, name, headers=headers, contents=None)
except ClientException as ex:
logging.error("Failed to store the manifest %s: %s" % (name, ex.http_reason))
sys.exit(1)
logging.debug("copy task done")
self.pending_copy_task = multiprocessing.Process(target=copy_task,
args=(self.conn,
self.container,
self.name,
self.part_name,
self.part_base_name,
),
)
self.pending_copy_task.start()
@translate_objectstorage_error
def write(self, data):
"""Write data to the object."""
if 'r' in self.mode:
raise IOSError(EPERM, "File is opened for read")
# large file support
if self.split_size:
# data can be of any size, so we need to split it in split_size chunks
offs = 0
while offs < len(data):
if self.part_size + len(data) - offs > self.split_size:
current_size = self.split_size-self.part_size
logging.debug("data is to large (%r), using %s" % (len(data), current_size))
else:
current_size = len(data)-offs
self.part_size += current_size
if not self.obj:
self.obj = ChunkObject(self.conn, self.container, self.part_name, content_type=self.content_type)
self.obj.send_chunk(data[offs:offs+current_size])
offs += current_size
if self.part_size == self.split_size:
logging.debug("current size is %r, split_file is %r" % (self.part_size, self.split_size))
self.obj.finish_chunk()
# this obj is not valid anymore, will create a new one if a new part is required
self.obj = None
# make it the first part
if self.part == 0:
self._start_copy_task()
self.part_size = 0
self.part += 1
else:
self.obj.send_chunk(data)
@translate_objectstorage_error
def close(self):
"""Close the object and finish the data transfer."""
if 'r' in self.mode:
return
if self.pending_copy_task:
logging.debug("waiting for a pending copy task...")
self.pending_copy_task.join()
logging.debug("wait is over")
if self.pending_copy_task.exitcode != 0:
raise IOSError(EIO, 'Failed to store the file')
if self.obj is not None:
self.obj.finish_chunk()
def read(self, size=65536):
"""
Read data from the object.
We can use just one request because 'seek' is not fully supported.
NB: It uses the size passed into the first call for all subsequent calls.
"""
if self.obj is None:
if self.total_size > 0:
self.conn.range_from = self.total_size
# we need to open a new connection to inject the `Range` header
if self.conn.http_conn:
self.conn.http_conn[1].close()
self.conn.http_conn = None
_, self.obj = self.conn.get_object(self.container, self.name, resp_chunk_size=size)
logging.debug("read size=%r, total_size=%r (range_from: %s)" % (size,
self.total_size, self.conn.range_from))
try:
buff = self.obj.next()
self.total_size += len(buff)
except StopIteration:
return ""
else:
return buff
def seek(self, offset, whence=None):
"""
Seek in the object.
It's supported only for read operations because of object storage limitations.
"""
logging.debug("seek offset=%s, whence=%s" % (str(offset), str(whence)))
if 'r' in self.mode:
if self.size is None:
meta = self.conn.head_object(self.container, self.name)
try:
self.size = int(meta["content-length"])
except ValueError:
raise IOSError(EPERM, "Invalid file size")
if not whence:
offs = offset
elif whence == 1:
offs = self.total_size + offset
elif whence == 2:
offs = self.size - offset
else:
raise IOSError(EPERM, "Invalid file offset")
if offs < 0 or offs > self.size:
raise IOSError(EPERM, "Invalid file offset")
# we need to start over after a seek call
if self.obj is not None:
del self.obj # GC the generator
self.obj = None
self.total_size = offs
else:
raise IOSError(EPERM, "Seek not available for write operations")
class CloudFilesFD(object):
'''Acts like a file() object, but attached to a cloud files object'''
def __init__(self, cffs, container, obj, mode):
self.cffs = cffs
self.container = container
self.name = obj
self.mode = mode
self.closed = False
self.total_size = 0
self.stream = None
if not all([container, obj]):
self.closed = True
raise IOSError(EPERM, 'Container and object required')
self.container = self.cffs._get_container(self.container)
if 'r' in self.mode:
self.obj = self.container.get_object(self.name)
logging.debug("read fd obj.name=%r obj.size=%r" %
(self.obj.name, self.obj.size))
else: #write
self.obj = ChunkObject(self.container, obj)
self.obj.content_type = mimetypes.guess_type(obj)[0]
self.obj.prepare_chunk()
def write(self, data):
'''Write data to the object'''
if 'r' in self.mode:
raise IOSError(EPERM, "Can't write to stream opened for read")
self.obj.send_chunk(data)
def close(self):
'''Close the object and finish the data transfer'''
if 'r' in self.mode:
return
self.obj.finish_chunk()
def read(self, size=65536):
'''Read data from the object.
We can use just one request because 'seek' is not supported.
NB: It uses the size passed into the first call for all subsequent calls'''
if not self.stream:
self.stream = self.obj.stream(size)
logging.debug("read size=%r, total_size=%r, obj.size=%r" %
(size, self.total_size, self.obj.size))
try:
buff = self.stream.next()
self.total_size += len(buff)
except StopIteration:
return ""
else:
return buff
def seek(self, *kargs, **kwargs):
'''Seek in the object: FIXME doesn't work and raises an error'''
logging.debug("seek args=%s, kargs=%s" % (str(kargs), str(kwargs)))
raise IOSError(EPERM, "Seek not implemented")
class CloudFilesFD(object):
'''Acts like a file() object, but attached to a cloud files object'''
def __init__(self, cffs, container, obj, mode):
self.cffs = cffs
self.container = container
self.name = obj
self.mode = mode
self.closed = False
self.total_size = 0
self.stream = None
self.headers = dict()
if not all([container, obj]):
self.closed = True
raise IOSError(EPERM, 'Container and object required')
self.container = self.cffs._get_container(self.container)
if 'r' in self.mode:
self.obj = self.container.get_object(self.name)
logging.debug("read fd obj.name=%r obj.size=%r" % (self.obj.name, self.obj.size))
else: #write
self.obj = ChunkObject(self.container, obj)
self.obj.content_type = mimetypes.guess_type(obj)[0]
self.obj.prepare_chunk()
def write(self, data):
'''Write data to the object'''
if 'r' in self.mode:
raise IOSError(EPERM, "Can't write to stream opened for read")
self.obj.send_chunk(data)
def close(self):
'''Close the object and finish the data transfer'''
if 'r' in self.mode:
return
self.obj.finish_chunk()
def read(self, size=65536):
'''Read data from the object.
We can use just one request because 'seek' is not fully supported.
NB: It uses the size passed into the first call for all subsequent calls'''
if not self.stream:
self.stream = self.obj.stream(size, hdrs=self.headers)
logging.debug("read size=%r, total_size=%r, obj.size=%r" % (size, self.total_size, self.obj.size))
try:
buff = self.stream.next()
self.total_size += len(buff)
except StopIteration:
return ""
else:
return buff
def seek(self, offset, whence=None):
'''Seek in the object.
It's supported only for read operations because of the object storage limitations.'''
logging.debug("seek offset=%s, whence=%s" % (str(offset), str(whence)))
if 'r' in self.mode:
if not whence:
offs = offset
elif whence == 1:
offs = self.total_size + offset
elif whence == 2:
offs = self.obj.size - offset
else:
raise IOSError(EPERM, "Invalid file offset")
if offs < 0 or offs > self.obj.size:
raise IOSError(EPERM, "Invalid file offset")
# we need to start over after a seek call
if self.stream:
self.stream = None
self.obj = self.container.get_object(self.name)
self.headers['Range'] = "bytes=%s-" % offs
self.total_size = offs
else:
raise IOSError(EPERM, "Seek not available for write operations")