def _stream(self, source, size, writers):
bytes_transferred = 0
# create EC encoding generator
ec_stream = ec_encode(self.storage_method, len(writers))
# init generator
ec_stream.send(None)
def send(data):
self.checksum.update(data)
# get the encoded fragments
fragments = ec_stream.send(data)
if fragments is None:
# not enough data given
return
current_writers = list(writers)
for writer in current_writers:
fragment = fragments[chunk_index[writer]]
if not writer.failed:
writer.checksum.update(fragment)
writer.send(fragment)
else:
current_writers.remove(writer)
self._check_quorum(current_writers)
# TODO handle no quorum
try:
# we use eventlet GreenPool to manage writers
with utils.ContextPool(len(writers)) as pool:
# convenient index to figure out which writer
# handles the resulting fragments
chunk_index = self._build_index(writers)
# init writers in pool
for writer in writers:
writer.start(pool)
# the main write loop
while True:
remaining_bytes = size - bytes_transferred
if io.WRITE_CHUNK_SIZE < remaining_bytes:
read_size = io.WRITE_CHUNK_SIZE
else:
read_size = remaining_bytes
with SourceReadTimeout(io.CLIENT_TIMEOUT):
try:
data = source.read(read_size)
except (ValueError, IOError) as e:
raise SourceReadError(str(e))
if len(data) == 0:
break
bytes_transferred += len(data)
send(data)
# flush out buffered data
send('')
# wait for all data to be processed
for writer in writers:
writer.wait()
# trailer headers
# metachunk size
# metachunk hash
metachunk_size = bytes_transferred
metachunk_hash = self.checksum.hexdigest()
for writer in writers:
writer.finish(metachunk_size, metachunk_hash)
return bytes_transferred
except SourceReadTimeout:
logger.warn('Source read timeout')
raise
except SourceReadError:
logger.warn('Source read error')
raise
except Timeout:
logger.exception('Timeout writing data')
raise
except Exception:
logger.exception('Exception writing data')
raise
def stream(self, source, size):
bytes_transferred = 0
meta_chunk = self.meta_chunk
if self.chunk_checksum_algo:
meta_checksum = hashlib.new(self.chunk_checksum_algo)
else:
meta_checksum = None
pile = GreenPile(len(meta_chunk))
failed_chunks = []
current_conns = []
for chunk in meta_chunk:
pile.spawn(self._connect_put, chunk)
for conn, chunk in pile:
if not conn:
failed_chunks.append(chunk)
else:
current_conns.append(conn)
self.quorum_or_fail([co.chunk for co in current_conns], failed_chunks)
bytes_transferred = 0
try:
with green.ContextPool(len(meta_chunk)) as pool:
for conn in current_conns:
conn.failed = False
conn.queue = LightQueue(io.PUT_QUEUE_DEPTH)
pool.spawn(self._send_data, conn)
while True:
buffer_size = self.buffer_size()
if size is not None:
remaining_bytes = size - bytes_transferred
if buffer_size < remaining_bytes:
read_size = buffer_size
else:
read_size = remaining_bytes
else:
read_size = buffer_size
with green.SourceReadTimeout(self.read_timeout):
try:
data = source.read(read_size)
except (ValueError, IOError) as err:
raise SourceReadError(str(err))
if len(data) == 0:
for conn in current_conns:
if not conn.failed:
conn.queue.put('')
break
self.checksum.update(data)
if meta_checksum:
meta_checksum.update(data)
bytes_transferred += len(data)
# copy current_conns to be able to remove a failed conn
for conn in current_conns[:]:
if not conn.failed:
conn.queue.put(data)
else:
current_conns.remove(conn)
failed_chunks.append(conn.chunk)
self.quorum_or_fail([co.chunk for co in current_conns],
failed_chunks)
for conn in current_conns:
while conn.queue.qsize():
sleep(0)
except green.SourceReadTimeout as err:
logger.warn('Source read timeout (reqid=%s): %s', self.reqid, err)
raise SourceReadTimeout(err)
except SourceReadError as err:
logger.warn('Source read error (reqid=%s): %s', self.reqid, err)
raise
except Timeout as to:
logger.error('Timeout writing data (reqid=%s): %s', self.reqid, to)
raise OioTimeout(to)
except Exception:
logger.exception('Exception writing data (reqid=%s)', self.reqid)
raise
success_chunks = []
for conn in current_conns:
if conn.failed:
failed_chunks.append(conn.chunk)
continue
pile.spawn(self._get_response, conn)
for (conn, resp) in pile:
if resp:
self._handle_resp(
conn, resp,
meta_checksum.hexdigest() if meta_checksum else None,
success_chunks, failed_chunks)
self.quorum_or_fail(success_chunks, failed_chunks)
for chunk in success_chunks:
chunk["size"] = bytes_transferred
return bytes_transferred, success_chunks[0]['hash'], success_chunks
def stream(self, source, size):
bytes_transferred = 0
def _connect_put(chunk):
raw_url = chunk["url"]
parsed = urlparse(raw_url)
try:
chunk_path = parsed.path.split('/')[-1]
h = {}
h["transfer-encoding"] = "chunked"
h[chunk_headers["content_id"]] = self.sysmeta['id']
h[chunk_headers["content_version"]] = self.sysmeta['version']
h[chunk_headers["content_path"]] = \
utils.quote(self.sysmeta['content_path'])
h[chunk_headers["content_chunkmethod"]] = \
self.sysmeta['chunk_method']
h[chunk_headers["content_policy"]] = self.sysmeta['policy']
h[chunk_headers["container_id"]] = self.sysmeta['container_id']
h[chunk_headers["chunk_pos"]] = chunk["pos"]
h[chunk_headers["chunk_id"]] = chunk_path
with ConnectionTimeout(io.CONNECTION_TIMEOUT):
conn = io.http_connect(parsed.netloc, 'PUT', parsed.path,
h)
conn.chunk = chunk
return conn, chunk
except (Exception, Timeout) as e:
msg = str(e)
logger.error("Failed to connect to %s (%s)", chunk, msg)
chunk['error'] = msg
return None, chunk
meta_chunk = self.meta_chunk
pile = GreenPile(len(meta_chunk))
failed_chunks = []
current_conns = []
for chunk in meta_chunk:
pile.spawn(_connect_put, chunk)
results = [d for d in pile]
for conn, chunk in results:
if not conn:
failed_chunks.append(chunk)
else:
current_conns.append(conn)
quorum = False
quorum = self._check_quorum(current_conns)
if not quorum:
raise exc.OioException("RAWX write failure")
bytes_transferred = 0
try:
with utils.ContextPool(len(meta_chunk)) as pool:
for conn in current_conns:
conn.failed = False
conn.queue = Queue(io.PUT_QUEUE_DEPTH)
pool.spawn(self._send_data, conn)
while True:
remaining_bytes = size - bytes_transferred
if io.WRITE_CHUNK_SIZE < remaining_bytes:
read_size = io.WRITE_CHUNK_SIZE
else:
read_size = remaining_bytes
with SourceReadTimeout(io.CLIENT_TIMEOUT):
try:
data = source.read(read_size)
except (ValueError, IOError) as e:
raise SourceReadError(str(e))
if len(data) == 0:
for conn in current_conns:
conn.queue.put('0\r\n\r\n')
break
self.checksum.update(data)
bytes_transferred += len(data)
for conn in current_conns:
if not conn.failed:
conn.queue.put('%x\r\n%s\r\n' % (len(data), data))
else:
current_conns.remove(conn)
quorum = self._check_quorum(current_conns)
if not quorum:
raise exc.OioException("RAWX write failure")
for conn in current_conns:
if conn.queue.unfinished_tasks:
conn.queue.join()
except SourceReadTimeout:
logger.warn('Source read timeout')
raise
except SourceReadError:
logger.warn('Source read error')
raise
except Timeout:
logger.exception('Timeout writing data')
raise
except Exception:
logger.exception('Exception writing data')
raise
success_chunks = []
for conn in current_conns:
if conn.failed:
failed_chunks.append(conn.chunk)
continue
pile.spawn(self._get_response, conn)
def _handle_resp(conn, resp):
if resp:
if resp.status == 201:
success_chunks.append(conn.chunk)
else:
conn.failed = True
conn.chunk['error'] = 'HTTP %s' % resp.status
failed_chunks.append(conn.chunk)
logger.error("Wrong status code from %s (%s)", conn.chunk,
resp.status)
conn.close()
for (conn, resp) in pile:
if resp:
_handle_resp(conn, resp)
quorum = self._check_quorum(success_chunks)
if not quorum:
raise exc.OioException("RAWX write failure")
meta_checksum = self.checksum.hexdigest()
for chunk in success_chunks:
chunk["size"] = bytes_transferred
chunk["hash"] = meta_checksum
return bytes_transferred, meta_checksum, success_chunks + failed_chunks