def __init__(self, root, device, *args, **kwargs):
kwargs['mount_check'] = False
super(DiskFile, self).__init__(root, device, *args, **kwargs)
host, port = device.split(':')
self.conn = KineticSwiftClient(host, int(port))
self.hashpath = os.path.basename(self.datadir.rstrip(os.path.sep))
self._buffer = ''
# this is the first "disk_chunk_size" + metadata
self._headbuffer = None
self._nounce = None
self.upload_size = 0
self.last_sync = 0
# configurables
self.write_depth = DEFAULT_DEPTH
try:
self._connect()
except socket.error:
self.logger.exception('unable to connect to %s:%s' %
(self.conn.hostname, self.conn.port))
self.conn.close()
raise diskfile.DiskFileDeviceUnavailable()
def setUp(self):
self.logger = debug_logger()
self.test_dir = tempfile.mkdtemp()
self.ports = self.PORTS
self._sim_map = {}
try:
self._sim_map = start_simulators(self.test_dir, *self.ports)
self.client_map = {}
for port in self.ports:
self.client_map[port] = KineticSwiftClient(
self.logger, 'localhost', port)
except Exception:
e, v, t = sys.exc_info()
self.tearDown()
raise e, v, t
def _new_connection(self, host, port, **kwargs):
kwargs.setdefault('connect_timeout', self.connect_timeout)
kwargs.setdefault('response_timeout', self.response_timeout)
for i in range(1, self.connect_retry + 1):
try:
return KineticSwiftClient(self.logger, host, int(port),
**kwargs)
except Timeout:
self.logger.warning('Drive %s:%s connect timeout #%d (%ds)' % (
host, port, i, self.connect_timeout))
except Exception:
self.logger.exception('Drive %s:%s connection error #%d' % (
host, port, i))
if i < self.connect_retry:
sleep(1)
msg = 'Unable to connect to drive %s:%s after %s attempts' % (
host, port, i)
self.logger.error(msg)
raise diskfile.DiskFileDeviceUnavailable()
def __init__(self, root, device, *args, **kwargs):
kwargs['mount_check'] = False
super(DiskFile, self).__init__(root, device, *args, **kwargs)
host, port = device.split(':')
self.conn = KineticSwiftClient(host, int(port))
self.hashpath = os.path.basename(self.datadir.rstrip(os.path.sep))
self._buffer = ''
# this is the first "disk_chunk_size" + metadata
self._headbuffer = None
self._nounce = None
self.upload_size = 0
self.last_sync = 0
# configurables
self.write_depth = DEFAULT_DEPTH
try:
self._connect()
except socket.error:
self.logger.exception(
'unable to connect to %s:%s' % (
self.conn.hostname, self.conn.port))
self.conn.close()
raise diskfile.DiskFileDeviceUnavailable()
class DiskFile(diskfile.DiskFile):
def __init__(self, root, device, *args, **kwargs):
kwargs['mount_check'] = False
super(DiskFile, self).__init__(root, device, *args, **kwargs)
host, port = device.split(':')
self.conn = KineticSwiftClient(host, int(port))
self.hashpath = os.path.basename(self.datadir.rstrip(os.path.sep))
self._buffer = ''
# this is the first "disk_chunk_size" + metadata
self._headbuffer = None
self._nounce = None
self.upload_size = 0
self.last_sync = 0
# configurables
self.write_depth = DEFAULT_DEPTH
try:
self._connect()
except socket.error:
self.logger.exception(
'unable to connect to %s:%s' % (
self.conn.hostname, self.conn.port))
self.conn.close()
raise diskfile.DiskFileDeviceUnavailable()
def _connect(self):
if not self.conn.isConnected:
self.conn.connect()
def open(self, **kwargs):
self._connect()
self._read()
return self
def close(self, **kwargs):
real_sock = None
green_sock = self.conn._socket
if hasattr(green_sock, 'fd'):
real_sock = getattr(green_sock.fd, '_sock', None)
self.conn.close()
if real_sock:
real_sock.close()
self._metadata = None
def _read(self):
key = object_key(self.hashpath)
entry = self.conn.getPrevious(key).wait()
if not entry or not entry.key.startswith(key[:-1]):
self._metadata = {} # mark object as "open"
return
self.data_file = '.ts.' not in entry.key
blob = entry.value
self._nounce = get_nounce(entry.key)
payload = msgpack.unpackb(blob)
self._metadata = payload['metadata']
self._headbuffer = payload['buffer']
def __iter__(self):
if not self._metadata:
return
yield self._headbuffer
keys = [chunk_key(self.hashpath, self._nounce, i + 1) for i in
range(int(self._metadata['X-Kinetic-Chunk-Count']))]
for entry in self.conn.get_keys(keys):
yield entry.value
@contextmanager
def create(self, size=None):
self._headbuffer = None
self._nounce = str(uuid4())
try:
self._connect()
self._pending_write = deque()
yield self
finally:
self.close()
def write(self, chunk):
self._buffer += chunk
self.upload_size += len(chunk)
diff = self.upload_size - self.last_sync
if diff >= self.disk_chunk_size:
self._sync_buffer()
self.last_sync = self.upload_size
def _submit_write(self, key, blob):
if len(self._pending_write) >= self.write_depth:
self._pending_write.popleft().wait()
pending_resp = self.conn.put(key, blob, force=True)
self._pending_write.append(pending_resp)
def _sync_buffer(self):
if not self._headbuffer:
# save the headbuffer
self._headbuffer = self._buffer
self._chunk_id = 0
elif self._buffer:
# write out the chunk buffer!
self._chunk_id += 1
key = chunk_key(self.hashpath, self._nounce, self._chunk_id)
self._submit_write(key, self._buffer)
self._buffer = ''
def _wait_write(self):
for resp in self._pending_write:
resp.wait()
def put(self, metadata, extension='.data'):
if extension == '.ts':
metadata['deleted'] = True
self._sync_buffer()
# zero index, chunk-count is len
metadata['X-Kinetic-Chunk-Count'] = self._chunk_id
metadata['X-Kinetic-Chunk-Nounce'] = self._nounce
metadata['name'] = self.name
self._metadata = metadata
payload = {'metadata': metadata, 'buffer': self._headbuffer}
blob = msgpack.packb(payload)
timestamp = diskfile.normalize_timestamp(metadata['X-Timestamp'])
key = object_key(self.hashpath, timestamp, extension, self._nounce)
self._submit_write(key, blob)
self._wait_write()
self._unlink_old(timestamp)
def _unlink_old(self, req_timestamp):
start_key = object_key(self.hashpath)[:-1]
end_key = object_key(self.hashpath, timestamp=req_timestamp)
resp = self.conn.getKeyRange(start_key, end_key, endKeyInclusive=False)
head_keys = resp.wait()
for key in head_keys:
nounce = get_nounce(key)
def key_gen():
yield key
i = 1
while True:
missing = yield chunk_key(self.hashpath, nounce, i)
i += 1
if missing:
break
self.conn.delete_keys(key_gen(), depth=4)
def quarantine(self):
pass
def get_data_file_size(self):
return self._metadata['Content-Length']
def get_conn(self, device):
host, port = device.split(':')
conn = KineticSwiftClient(host, int(port))
return conn
class DiskFile(diskfile.DiskFile):
def __init__(self, root, device, *args, **kwargs):
kwargs['mount_check'] = False
super(DiskFile, self).__init__(root, device, *args, **kwargs)
host, port = device.split(':')
self.conn = KineticSwiftClient(host, int(port))
self.hashpath = os.path.basename(self.datadir.rstrip(os.path.sep))
self._buffer = ''
# this is the first "disk_chunk_size" + metadata
self._headbuffer = None
self._nounce = None
self.upload_size = 0
self.last_sync = 0
# configurables
self.write_depth = DEFAULT_DEPTH
try:
self._connect()
except socket.error:
self.logger.exception(
'unable to connect to %s:%s' % (
self.conn.hostname, self.conn.port))
self.conn.close()
raise diskfile.DiskFileDeviceUnavailable()
def _connect(self):
if not self.conn.isConnected:
self.conn.connect()
def open(self, **kwargs):
self._connect()
self._read()
return self
def close(self, **kwargs):
real_sock = None
green_sock = self.conn._socket
if hasattr(green_sock, 'fd'):
real_sock = getattr(green_sock.fd, '_sock', None)
self.conn.close()
if real_sock:
real_sock.close()
self._metadata = None
def _read(self):
key = object_key(self.hashpath)
entry = self.conn.getPrevious(key).wait()
if not entry or not entry.key.startswith(key[:-1]):
self._metadata = {} # mark object as "open"
return
self.data_file = '.ts.' not in entry.key
blob = entry.value
self._nounce = get_nounce(entry.key)
payload = msgpack.unpackb(blob)
self._metadata = payload['metadata']
self._headbuffer = payload['buffer']
def __iter__(self):
if not self._metadata:
return
yield self._headbuffer
keys = [chunk_key(self.hashpath, self._nounce, i + 1) for i in
range(int(self._metadata['X-Kinetic-Chunk-Count']))]
for entry in self.conn.get_keys(keys):
yield entry.value
@contextmanager
def create(self, size=None):
self._headbuffer = None
self._nounce = str(uuid4())
try:
self._connect()
self._pending_write = deque()
yield self
finally:
self.close()
def write(self, chunk):
self._buffer += chunk
self.upload_size += len(chunk)
diff = self.upload_size - self.last_sync
if diff >= self.disk_chunk_size:
self._sync_buffer()
self.last_sync = self.upload_size
def _submit_write(self, key, blob):
if len(self._pending_write) >= self.write_depth:
self._pending_write.popleft().wait()
pending_resp = self.conn.put(key, blob, force=True)
self._pending_write.append(pending_resp)
def _sync_buffer(self):
if not self._headbuffer:
# save the headbuffer
self._headbuffer = self._buffer[:self.disk_chunk_size]
self._chunk_id = 0
elif self._buffer:
# write out the chunk buffer!
self._chunk_id += 1
key = chunk_key(self.hashpath, self._nounce, self._chunk_id)
self._submit_write(key, self._buffer[:self.disk_chunk_size])
self._buffer = self._buffer[self.disk_chunk_size:]
def _wait_write(self):
for resp in self._pending_write:
resp.wait()
def put(self, metadata, extension='.data'):
if extension == '.ts':
metadata['deleted'] = True
self._sync_buffer()
while self._buffer:
self._sync_buffer()
# zero index, chunk-count is len
metadata['X-Kinetic-Chunk-Count'] = self._chunk_id
metadata['X-Kinetic-Chunk-Nounce'] = self._nounce
metadata['name'] = self.name
self._metadata = metadata
payload = {'metadata': metadata, 'buffer': self._headbuffer}
blob = msgpack.packb(payload)
timestamp = diskfile.normalize_timestamp(metadata['X-Timestamp'])
key = object_key(self.hashpath, timestamp, extension, self._nounce)
self._submit_write(key, blob)
self._wait_write()
self._unlink_old(timestamp)
def _unlink_old(self, req_timestamp):
start_key = object_key(self.hashpath)[:-1]
end_key = object_key(self.hashpath, timestamp=req_timestamp)
resp = self.conn.getKeyRange(start_key, end_key, endKeyInclusive=False)
head_keys = resp.wait()
for key in head_keys:
nounce = get_nounce(key)
def key_gen():
yield key
i = 1
while True:
missing = yield chunk_key(self.hashpath, nounce, i)
i += 1
if missing:
break
self.conn.delete_keys(key_gen(), depth=4)
def quarantine(self):
pass
def get_data_file_size(self):
return self._metadata['Content-Length']
