def __init__(self,
namespace,
concurrency=50,
error_file=None,
rebuild_file=None,
check_xattr=True,
limit_listings=0,
request_attempts=1,
logger=None,
verbose=False,
check_hash=False,
**_kwargs):
self.pool = GreenPool(concurrency)
self.error_file = error_file
self.check_xattr = bool(check_xattr)
self.check_hash = bool(check_hash)
self.logger = logger or get_logger(
{'namespace': namespace}, name='integrity', verbose=verbose)
# Optimisation for when we are only checking one object
# or one container.
# 0 -> do not limit
# 1 -> limit account listings (list of containers)
# 2 -> limit container listings (list of objects)
self.limit_listings = limit_listings
if self.error_file:
outfile = open(self.error_file, 'a')
self.error_writer = csv.writer(outfile, delimiter=' ')
self.rebuild_file = rebuild_file
if self.rebuild_file:
self.fd = open(self.rebuild_file, 'a')
self.rebuild_writer = csv.writer(self.fd, delimiter='|')
self.api = ObjectStorageApi(namespace,
logger=self.logger,
max_retries=request_attempts - 1,
request_attempts=request_attempts)
self.rdir_client = RdirClient({"namespace": namespace},
logger=self.logger)
self.accounts_checked = 0
self.containers_checked = 0
self.objects_checked = 0
self.chunks_checked = 0
self.account_not_found = 0
self.container_not_found = 0
self.object_not_found = 0
self.chunk_not_found = 0
self.account_exceptions = 0
self.container_exceptions = 0
self.object_exceptions = 0
self.chunk_exceptions = 0
self.list_cache = {}
self.running = {}
self.running_lock = Semaphore(1)
self.result_queue = Queue(concurrency)
self.run_time = 0
def __init__(self, chunk, conn, write_timeout=None,
chunk_checksum_algo='md5', **_kwargs):
self._chunk = chunk
self._conn = conn
self.failed = False
self.bytes_transferred = 0
if chunk_checksum_algo:
self.checksum = hashlib.new(chunk_checksum_algo)
else:
self.checksum = None
self.write_timeout = write_timeout or io.CHUNK_TIMEOUT
# we use eventlet Queue to pass data to the send coroutine
self.queue = Queue(io.PUT_QUEUE_DEPTH)
def take_action(self, parsed_args):
self.log.debug('take_action(%s)', parsed_args)
digits = self.app.client_manager.meta1_digits
concurrency = parsed_args.concurrency
conf = {'namespace': self.app.client_manager.namespace}
if parsed_args.proxy:
conf.update({'proxyd_url': parsed_args.proxy})
else:
ns_conf = self.app.client_manager.sds_conf
proxy = ns_conf.get('proxy')
conf.update({'proxyd_url': proxy})
workers = list()
with green.ContextPool(concurrency) as pool:
pile = GreenPile(pool)
prefix_queue = Queue(16)
# Prepare some workers
for _ in range(concurrency):
worker = WarmupWorker(self.app.client_manager.client_conf,
self.log)
workers.append(worker)
pile.spawn(worker.run, prefix_queue)
# Feed the queue
trace_increment = 0.01
trace_next = trace_increment
sent, total = 0, float(count_prefixes(digits))
for prefix in generate_prefixes(digits):
sent += 1
prefix_queue.put(prefix)
# Display the progression
ratio = float(sent) / total
if ratio >= trace_next:
self.log.info("... %d%%", int(ratio * 100.0))
trace_next += trace_increment
self.log.debug("Send the termination marker")
prefix_queue.join()
self.log.info("All the workers are done")
class EcChunkWriter(object):
"""
Writes an EC chunk
"""
def __init__(self, chunk, conn, write_timeout=None,
chunk_checksum_algo='md5', **_kwargs):
self._chunk = chunk
self._conn = conn
self.failed = False
self.bytes_transferred = 0
if chunk_checksum_algo:
self.checksum = hashlib.new(chunk_checksum_algo)
else:
self.checksum = None
self.write_timeout = write_timeout or io.CHUNK_TIMEOUT
# we use eventlet Queue to pass data to the send coroutine
self.queue = Queue(io.PUT_QUEUE_DEPTH)
@property
def chunk(self):
return self._chunk
@property
def conn(self):
return self._conn
@classmethod
def connect(cls, chunk, sysmeta, reqid=None,
connection_timeout=None, write_timeout=None, **kwargs):
raw_url = chunk.get("real_url", chunk["url"])
parsed = urlparse(raw_url)
chunk_path = parsed.path.split('/')[-1]
hdrs = headers_from_object_metadata(sysmeta)
if reqid:
hdrs['X-oio-req-id'] = reqid
hdrs[CHUNK_HEADERS["chunk_pos"]] = chunk["pos"]
hdrs[CHUNK_HEADERS["chunk_id"]] = chunk_path
# in the trailer
# metachunk_size & metachunk_hash
trailers = (CHUNK_HEADERS["metachunk_size"],
CHUNK_HEADERS["metachunk_hash"])
if kwargs.get('chunk_checksum_algo'):
trailers = trailers + (CHUNK_HEADERS["chunk_hash"], )
hdrs["Trailer"] = ', '.join(trailers)
with green.ConnectionTimeout(
connection_timeout or io.CONNECTION_TIMEOUT):
conn = io.http_connect(
parsed.netloc, 'PUT', parsed.path, hdrs)
conn.chunk = chunk
return cls(chunk, conn, write_timeout=write_timeout, **kwargs)
def start(self, pool):
"""Spawn the send coroutine"""
pool.spawn(self._send)
def _send(self):
"""Send coroutine loop"""
while True:
# fetch input data from the queue
data = self.queue.get()
# use HTTP transfer encoding chunked
# to write data to RAWX
if not self.failed:
# format the chunk
to_send = "%x\r\n%s\r\n" % (len(data), data)
try:
with green.ChunkWriteTimeout(self.write_timeout):
self.conn.send(to_send)
self.bytes_transferred += len(data)
except (Exception, green.ChunkWriteTimeout) as exc:
self.failed = True
msg = str(exc)
logger.warn("Failed to write to %s (%s)", self.chunk, msg)
self.chunk['error'] = 'write: %s' % msg
self.queue.task_done()
def wait(self):
"""
Wait until all data in the queue
has been processed by the send coroutine
"""
if self.queue.unfinished_tasks:
self.queue.join()
def send(self, data):
# do not send empty data because
# this will end the chunked body
if not data:
return
# put the data to send into the queue
# it will be processed by the send coroutine
self.queue.put(data)
def finish(self, metachunk_size, metachunk_hash):
"""Send metachunk_size and metachunk_hash as trailers"""
parts = [
'0\r\n',
'%s: %s\r\n' % (CHUNK_HEADERS['metachunk_size'],
metachunk_size),
'%s: %s\r\n' % (CHUNK_HEADERS['metachunk_hash'],
metachunk_hash)
]
if self.checksum:
parts.append('%s: %s\r\n' % (CHUNK_HEADERS['chunk_hash'],
self.checksum.hexdigest()))
parts.append('\r\n')
to_send = "".join(parts)
self.conn.send(to_send)
def getresponse(self):
"""Read the HTTP response from the connection"""
# As the server may buffer data before writing it to non-volatile
# storage, we don't know if we have to wait while sending data or
# while reading response, thus we apply the same timeout to both.
with Timeout(self.write_timeout):
return self.conn.getresponse()
def _decode_segments(self, fragment_iterators):
"""
Reads from fragments and yield full segments
"""
# we use eventlet Queue to read fragments
queues = []
# each iterators has its queue
for _j in range(len(fragment_iterators)):
queues.append(Queue(1))
def put_in_queue(fragment_iterator, queue):
"""
Coroutine to read the fragments from the iterator
"""
try:
for fragment in fragment_iterator:
# put the read fragment in the queue
queue.put(fragment)
# the queues are of size 1 so this coroutine blocks
# until we decode a full segment
except GreenletExit:
# ignore
pass
except green.ChunkReadTimeout as err:
logger.error('%s', err)
except Exception:
logger.exception("Exception on reading")
finally:
queue.resize(2)
# put None to indicate the decoding loop
# this is over
queue.put(None)
# close the iterator
fragment_iterator.close()
# we use eventlet GreenPool to manage the read of fragments
with green.ContextPool(len(fragment_iterators)) as pool:
# spawn coroutines to read the fragments
for fragment_iterator, queue in zip(fragment_iterators, queues):
pool.spawn(put_in_queue, fragment_iterator, queue)
# main decoding loop
while True:
data = []
# get the fragments from the queues
for queue in queues:
fragment = queue.get()
queue.task_done()
data.append(fragment)
if not all(data):
# one of the readers returned None
# impossible to read segment
break
# actually decode the fragments into a segment
try:
segment = self.storage_method.driver.decode(data)
except exceptions.ECError:
# something terrible happened
logger.exception("ERROR decoding fragments")
raise
yield segment
class EcChunkWriter(object):
"""
Writes an EC chunk
"""
def __init__(self, chunk, conn, write_timeout=None,
chunk_checksum_algo='md5', perfdata=None, **kwargs):
self._chunk = chunk
self._conn = conn
self.failed = False
self.bytes_transferred = 0
if chunk_checksum_algo:
self.checksum = hashlib.new(chunk_checksum_algo)
else:
self.checksum = None
self.write_timeout = write_timeout or io.CHUNK_TIMEOUT
# we use eventlet Queue to pass data to the send coroutine
self.queue = Queue(io.PUT_QUEUE_DEPTH)
self.reqid = kwargs.get('reqid')
self.perfdata = perfdata
self.logger = kwargs.get('logger', LOGGER)
@property
def chunk(self):
return self._chunk
@property
def conn(self):
return self._conn
@classmethod
def connect(cls, chunk, sysmeta, reqid=None,
connection_timeout=None, write_timeout=None, **kwargs):
raw_url = chunk.get("real_url", chunk["url"])
parsed = urlparse(raw_url)
chunk_path = parsed.path.split('/')[-1]
hdrs = headers_from_object_metadata(sysmeta)
if reqid:
hdrs[REQID_HEADER] = reqid
hdrs[CHUNK_HEADERS["chunk_pos"]] = chunk["pos"]
hdrs[CHUNK_HEADERS["chunk_id"]] = chunk_path
# in the trailer
# metachunk_size & metachunk_hash
trailers = (CHUNK_HEADERS["metachunk_size"],
CHUNK_HEADERS["metachunk_hash"])
if kwargs.get('chunk_checksum_algo'):
trailers = trailers + (CHUNK_HEADERS["chunk_hash"], )
hdrs["Trailer"] = ', '.join(trailers)
with ConnectionTimeout(
connection_timeout or io.CONNECTION_TIMEOUT):
perfdata = kwargs.get('perfdata', None)
if perfdata is not None:
connect_start = monotonic_time()
conn = io.http_connect(
parsed.netloc, 'PUT', parsed.path, hdrs, scheme=parsed.scheme)
conn.set_cork(True)
if perfdata is not None:
connect_end = monotonic_time()
perfdata_rawx = perfdata.setdefault('rawx', dict())
perfdata_rawx['connect.' + chunk['url']] = \
connect_end - connect_start
conn.chunk = chunk
return cls(chunk, conn, write_timeout=write_timeout,
reqid=reqid, **kwargs)
def start(self, pool):
"""Spawn the send coroutine"""
pool.spawn(self._send)
def _send(self):
"""Send coroutine loop"""
self.conn.upload_start = None
while not self.failed:
# fetch input data from the queue
data = self.queue.get()
# use HTTP transfer encoding chunked
# to write data to RAWX
try:
with ChunkWriteTimeout(self.write_timeout):
if self.perfdata is not None \
and self.conn.upload_start is None:
self.conn.upload_start = monotonic_time()
self.conn.send(b"%x\r\n" % len(data))
self.conn.send(data)
self.conn.send(b"\r\n")
self.bytes_transferred += len(data)
eventlet_yield()
except (Exception, ChunkWriteTimeout) as exc:
self.failed = True
msg = text_type(exc)
self.logger.warn("Failed to write to %s (%s, reqid=%s)",
self.chunk, msg, self.reqid)
self.chunk['error'] = 'write: %s' % msg
# Indicate that the data is completely sent
self.queue.task_done()
# Drain the queue before quitting
while True:
try:
self.queue.get_nowait()
self.queue.task_done()
except Empty:
break
def wait(self):
"""
Wait until all data in the queue
has been processed by the send coroutine
"""
self.logger.debug("Waiting for %s to receive data", self.chunk['url'])
# Wait until the data is completely sent to continue
self.queue.join()
def send(self, data):
# do not send empty data because
# this will end the chunked body
if not data:
return
# put the data to send into the queue
# it will be processed by the send coroutine
self.queue.put(data)
def finish(self, metachunk_size, metachunk_hash):
"""
Send metachunk_size and metachunk_hash as trailers.
:returns: the chunk object if the upload has failed, else None
"""
self.wait()
if self.failed:
self.logger.debug("NOT sending end marker and trailers to %s, "
"because upload has failed", self.chunk['url'])
return self.chunk
self.logger.debug("Sending end marker and trailers to %s",
self.chunk['url'])
parts = [
'0\r\n',
'%s: %s\r\n' % (CHUNK_HEADERS['metachunk_size'],
metachunk_size),
'%s: %s\r\n' % (CHUNK_HEADERS['metachunk_hash'],
metachunk_hash)
]
if self.checksum:
parts.append('%s: %s\r\n' % (CHUNK_HEADERS['chunk_hash'],
self.checksum.hexdigest()))
parts.append('\r\n')
to_send = ''.join(parts).encode('utf-8')
if self.perfdata is not None:
fin_start = monotonic_time()
try:
with ChunkWriteTimeout(self.write_timeout):
self.conn.send(to_send)
# Last segment sent, disable TCP_CORK to flush buffers
self.conn.set_cork(False)
except (Exception, ChunkWriteTimeout) as exc:
self.failed = True
msg = text_type(exc)
self.logger.warn("Failed to finish %s (%s, reqid=%s)",
self.chunk, msg, self.reqid)
self.chunk['error'] = 'finish: %s' % msg
return self.chunk
finally:
if self.perfdata is not None:
fin_end = monotonic_time()
rawx_perfdata = self.perfdata.setdefault('rawx', dict())
chunk_url = self.conn.chunk['url']
rawx_perfdata['upload_finish.' + chunk_url] = \
fin_end - fin_start
return None
def getresponse(self):
"""Read the HTTP response from the connection"""
try:
# As the server may buffer data before writing it to non-volatile
# storage, we don't know if we have to wait while sending data or
# while reading response, thus we apply the same timeout to both.
with ChunkWriteTimeout(self.write_timeout):
resp = self.conn.getresponse()
return resp
finally:
if self.perfdata is not None:
perfdata_rawx = self.perfdata.setdefault('rawx', dict())
chunk_url = self.conn.chunk['url']
upload_end = monotonic_time()
perfdata_rawx['upload.' + chunk_url] = \
upload_end - self.conn.upload_start
class Checker(object):
def __init__(self,
namespace,
concurrency=50,
error_file=None,
rebuild_file=None,
full=True,
limit_listings=0,
request_attempts=1,
logger=None,
verbose=False,
integrity=False):
self.pool = GreenPool(concurrency)
self.error_file = error_file
self.full = bool(full)
self.integrity = bool(integrity)
# Optimisation for when we are only checking one object
# or one container.
# 0 -> do not limit
# 1 -> limit account listings (list of containers)
# 2 -> limit container listings (list of objects)
self.limit_listings = limit_listings
if self.error_file:
outfile = open(self.error_file, 'a')
self.error_writer = csv.writer(outfile, delimiter=' ')
self.rebuild_file = rebuild_file
if self.rebuild_file:
fd = open(self.rebuild_file, 'a')
self.rebuild_writer = csv.writer(fd, delimiter='|')
self.logger = logger or get_logger(
{'namespace': namespace}, name='integrity', verbose=verbose)
self.api = ObjectStorageApi(namespace,
logger=self.logger,
max_retries=request_attempts - 1,
request_attempts=request_attempts)
self.accounts_checked = 0
self.containers_checked = 0
self.objects_checked = 0
self.chunks_checked = 0
self.account_not_found = 0
self.container_not_found = 0
self.object_not_found = 0
self.chunk_not_found = 0
self.account_exceptions = 0
self.container_exceptions = 0
self.object_exceptions = 0
self.chunk_exceptions = 0
self.list_cache = {}
self.running = {}
self.result_queue = Queue()
def complete_target_from_chunk_metadata(self, target, xattr_meta):
"""
Complete a Target object from metadata found in chunk's extended
attributes. In case the "fullpath" is not available, try to read
legacy metadata, and maybe ask meta1 to resolve the CID into
account and container names.
"""
# pylint: disable=unbalanced-tuple-unpacking
try:
acct, ct, path, vers, content_id = \
decode_fullpath(xattr_meta['full_path'])
target.account = acct
target.container = ct
target.obj = path
target.content_id = content_id
target.version = vers
except KeyError:
# No fullpath header, try legacy headers
if 'content_path' in xattr_meta:
target.obj = xattr_meta['content_path']
if 'content_id' in xattr_meta:
target.content_id = xattr_meta['content_id']
if 'content_version' in xattr_meta:
target.version = xattr_meta['content_version']
cid = xattr_meta.get('container_id')
if cid:
try:
md = self.api.directory.show(cid=cid)
acct = md.get('account')
ct = md.get('name')
if acct:
target.account = acct
if ct:
target.container = ct
except Exception as err:
self.logger.warn(
"Failed to resolve CID %s into account "
"and container names: %s", cid, err)
def send_result(self, target, errors=None):
"""
Put an item in the result queue.
"""
# TODO(FVE): send to an external queue.
self.result_queue.put(ItemResult(target, errors))
def write_error(self, target, irreparable=False):
if not self.error_file:
return
error = list()
if irreparable:
error.append('#IRREPARABLE')
error.append(target.account)
if target.container:
error.append(target.container)
if target.obj:
error.append(target.obj)
if target.chunk:
error.append(target.chunk)
self.error_writer.writerow(error)
def write_rebuilder_input(self, target, irreparable=False):
# FIXME(FVE): cid can be computed from account and container names
ct_meta = self.list_cache[(target.account, target.container)][1]
try:
cid = ct_meta['system']['sys.name'].split('.', 1)[0]
except KeyError:
cid = ct_meta['properties']['sys.name'].split('.', 1)[0]
error = list()
if irreparable:
error.append('#IRREPARABLE')
error.append(cid)
# FIXME(FVE): ensure we always resolve content_id,
# or pass object version along with object name.
error.append(target.content_id or target.obj)
error.append(target.chunk)
self.rebuild_writer.writerow(error)
def write_chunk_error(self, target, chunk=None, irreparable=False):
if chunk is not None:
target = target.copy()
target.chunk = chunk
self.write_error(target, irreparable=irreparable)
if self.rebuild_file:
self.write_rebuilder_input(target, irreparable=irreparable)
def _check_chunk_xattr(self, target, obj_meta, xattr_meta):
"""
Check coherency of chunk extended attributes with object metadata.
:returns: a list of errors
"""
errors = list()
# Composed position -> erasure coding
attr_prefix = 'meta' if '.' in obj_meta['pos'] else ''
attr_key = attr_prefix + 'chunk_size'
if str(obj_meta['size']) != xattr_meta.get(attr_key):
errors.append(
"'%s' xattr (%s) differs from size in meta2 (%s)" %
(attr_key, xattr_meta.get(attr_key), obj_meta['size']))
attr_key = attr_prefix + 'chunk_hash'
if obj_meta['hash'] != xattr_meta.get(attr_key):
errors.append(
"'%s' xattr (%s) differs from hash in meta2 (%s)" %
(attr_key, xattr_meta.get(attr_key), obj_meta['hash']))
return errors
def _check_chunk(self, target):
"""
Execute various checks on a chunk:
- does it appear in object's chunk list?
- is it reachable?
- are its extended attributes coherent?
:returns: the list of errors encountered,
and the chunk's owner object metadata.
"""
chunk = target.chunk
errors = list()
obj_meta = None
xattr_meta = None
try:
xattr_meta = self.api.blob_client.chunk_head(
chunk, xattr=self.full, check_hash=self.integrity)
except exc.NotFound as err:
self.chunk_not_found += 1
errors.append('Not found: %s' % (err, ))
except exc.FaultyChunk as err:
self.chunk_exceptions += 1
errors.append('Faulty: %r' % (err, ))
except Exception as err:
self.chunk_exceptions += 1
errors.append('Check failed: %s' % (err, ))
if not target.obj and xattr_meta:
self.complete_target_from_chunk_metadata(target, xattr_meta)
if target.obj:
obj_listing, obj_meta = self.check_obj(target.copy_object())
if chunk not in obj_listing:
errors.append('Missing from object listing')
db_meta = dict()
else:
db_meta = obj_listing[chunk]
if db_meta and xattr_meta and self.full:
errors.extend(
self._check_chunk_xattr(target, db_meta, xattr_meta))
self.send_result(target, errors)
self.chunks_checked += 1
return errors, obj_meta
def check_chunk(self, target):
errors, _obj_meta = self._check_chunk(target)
return errors
def _check_metachunk(self, target, stg_met, pos, chunks, recurse=0):
"""
Check that a metachunk has the right number of chunks.
:returns: the list of errors
"""
required = stg_met.expected_chunks
errors = list()
if len(chunks) < required:
missing_chunks = required - len(chunks)
if stg_met.ec:
subs = {x['num'] for x in chunks}
for sub in range(required):
if sub not in subs:
errors.append("Missing chunk at position %d.%d" %
(pos, sub))
else:
for _ in range(missing_chunks):
errors.append("Missing chunk at position %d" % pos)
if recurse > 0:
for chunk in chunks:
tcopy = target.copy()
tcopy.chunk = chunk['url']
chunk_errors, _ = self._check_chunk(tcopy)
if chunk_errors:
# The errors have already been reported by _check_chunk,
# but we must count this chunk among the unusable chunks
# of the current metachunk.
errors.append("Unusable chunk %s at position %s" %
(chunk['url'], chunk['pos']))
irreparable = required - len(errors) < stg_met.min_chunks_to_read
if irreparable:
errors.append(
"Unavailable metachunk at position %s (%d/%d chunks)" %
(pos, required - len(errors), stg_met.expected_chunks))
# Since the "metachunk" is not an official item type,
# this method does not report errors itself. Errors will
# be reported as object errors.
return errors
def _check_obj_policy(self, target, obj_meta, chunks, recurse=0):
"""
Check that the list of chunks of an object matches
the object's storage policy.
:returns: the list of errors encountered
"""
stg_met = STORAGE_METHODS.load(obj_meta['chunk_method'])
chunks_by_pos = _sort_chunks(chunks, stg_met.ec)
tasks = list()
for pos, chunks in chunks_by_pos.iteritems():
tasks.append((pos,
self.pool.spawn(self._check_metachunk,
target.copy(),
stg_met,
pos,
chunks,
recurse=recurse)))
errors = list()
for pos, task in tasks:
try:
errors.extend(task.wait())
except Exception as err:
errors.append("Check failed: pos %d: %s" % (pos, err))
return errors
def check_obj_versions(self, target, versions, recurse=0):
"""
Run checks of all versions of the targeted object in parallel.
"""
tasks = list()
for ov in versions:
tcopy = target.copy_object()
tcopy.content_id = ov['id']
tcopy.version = str(ov['version'])
tasks.append((tcopy.version,
self.pool.spawn(self.check_obj,
tcopy,
recurse=recurse)))
errors = list()
for version, task in tasks:
try:
task.wait()
except Exception as err:
errors.append("Check failed: version %s: %s" % (version, err))
if errors:
# Send a result with the target without version to tell
# we were not able to check all versions of the object.
self.send_result(target, errors)
def _load_obj_meta(self, target, errors):
"""
Load object metadata and chunks.
:param target: which object to check.
:param errors: list of errors that will be appended
in case any error occurs.
:returns: a tuple with object metadata and a list of chunks.
"""
try:
return self.api.object_locate(target.account,
target.container,
target.obj,
version=target.version,
properties=False)
except exc.NoSuchObject as err:
self.object_not_found += 1
errors.append('Not found: %s' % (err, ))
except Exception as err:
self.object_exceptions += 1
errors.append('Check failed: %s' % (err, ))
return None, []
def check_obj(self, target, recurse=0):
"""
Check one object version.
If no version is specified, all versions of the object will be checked.
:returns: the result of the check of the most recent version,
or the one that is explicitly targeted.
"""
account = target.account
container = target.container
obj = target.obj
vers = target.version # can be None
if (account, container, obj, vers) in self.running:
self.running[(account, container, obj, vers)].wait()
if (account, container, obj, vers) in self.list_cache:
return self.list_cache[(account, container, obj, vers)]
self.running[(account, container, obj, vers)] = Event()
self.logger.info('Checking object "%s"', target)
container_listing, _ = self.check_container(target.copy_container())
errors = list()
if obj not in container_listing:
errors.append('Missing from container listing')
# checksum = None
else:
versions = container_listing[obj]
if vers is None:
if target.content_id is None:
# No version specified, check all versions
self.check_obj_versions(target.copy_object(),
versions,
recurse=recurse)
# Now return the cached result of the most recent version
target.content_id = versions[0]['id']
target.version = str(versions[0]['version'])
res = self.check_obj(target, recurse=0)
self.running[(account, container, obj, vers)].send(True)
del self.running[(account, container, obj, vers)]
return res
else:
for ov in versions:
if ov['id'] == target.content_id:
vers = str(ov['version'])
target.version = vers
break
else:
errors.append('Missing from container listing')
# TODO check checksum match
# checksum = container_listing[obj]['hash']
pass
meta, chunks = self._load_obj_meta(target, errors)
chunk_listing = {c['url']: c for c in chunks}
if meta:
self.list_cache[(account, container, obj, vers)] = \
(chunk_listing, meta)
self.objects_checked += 1
self.running[(account, container, obj, vers)].send(True)
del self.running[(account, container, obj, vers)]
# Skip the check if we could not locate the object
if meta:
errors.extend(
self._check_obj_policy(target, meta, chunks, recurse=recurse))
self.send_result(target, errors)
return chunk_listing, meta
def check_container(self, target, recurse=0):
account = target.account
container = target.container
if (account, container) in self.running:
self.running[(account, container)].wait()
if (account, container) in self.list_cache:
return self.list_cache[(account, container)]
self.running[(account, container)] = Event()
self.logger.info('Checking container "%s"', target)
account_listing = self.check_account(target.copy_account())
errors = list()
if container not in account_listing:
errors.append('Missing from account listing')
marker = None
results = []
ct_meta = dict()
extra_args = dict()
if self.limit_listings > 1 and target.obj:
# When we are explicitly checking one object, start the listing
# where this object is supposed to be. Do not use a limit,
# but an end marker, in order to fetch all versions of the object.
extra_args['prefix'] = target.obj
extra_args['end_marker'] = target.obj + '\x00' # HACK
while True:
try:
resp = self.api.object_list(account,
container,
marker=marker,
versions=True,
**extra_args)
except exc.NoSuchContainer as err:
self.container_not_found += 1
errors.append('Not found: %s' % (err, ))
break
except Exception as err:
self.container_exceptions += 1
errors.append('Check failed: %s' % (err, ))
break
if resp.get('truncated', False):
marker = resp['next_marker']
if resp['objects']:
# safeguard, probably useless
if not marker:
marker = resp['objects'][-1]['name']
results.extend(resp['objects'])
if self.limit_listings > 1:
break
else:
ct_meta = resp
ct_meta.pop('objects')
break
container_listing = dict()
# Save all object versions, with the most recent first
for obj in results:
container_listing.setdefault(obj['name'], list()).append(obj)
for versions in container_listing.values():
versions.sort(key=lambda o: o['version'], reverse=True)
if self.limit_listings <= 1:
# We just listed the whole container, keep the result in a cache
self.containers_checked += 1
self.list_cache[(account, container)] = container_listing, ct_meta
self.running[(account, container)].send(True)
del self.running[(account, container)]
if recurse > 0:
for obj_vers in container_listing.values():
for obj in obj_vers:
tcopy = target.copy_object()
tcopy.obj = obj['name']
tcopy.content_id = obj['id']
tcopy.version = str(obj['version'])
self.pool.spawn_n(self.check_obj, tcopy, recurse - 1)
self.send_result(target, errors)
return container_listing, ct_meta
def check_account(self, target, recurse=0):
account = target.account
if account in self.running:
self.running[account].wait()
if account in self.list_cache:
return self.list_cache[account]
self.running[account] = Event()
self.logger.info('Checking account "%s"', target)
errors = list()
marker = None
results = []
extra_args = dict()
if self.limit_listings > 0 and target.container:
# When we are explicitly checking one container, start the listing
# where this container is supposed to be, and list only one
# container.
extra_args['prefix'] = target.container
extra_args['limit'] = 1
while True:
try:
resp = self.api.container_list(account,
marker=marker,
**extra_args)
except Exception as err:
self.account_exceptions += 1
errors.append('Check failed: %s' % (err, ))
break
if resp:
marker = resp[-1][0]
results.extend(resp)
if self.limit_listings > 0:
break
else:
break
containers = dict()
for container in results:
# Name, number of objects, number of bytes
containers[container[0]] = (container[1], container[2])
if self.limit_listings <= 0:
# We just listed the whole account, keep the result in a cache
self.accounts_checked += 1
self.list_cache[account] = containers
self.running[account].send(True)
del self.running[account]
if recurse > 0:
for container in containers:
tcopy = target.copy_account()
tcopy.container = container
self.pool.spawn_n(self.check_container, tcopy, recurse - 1)
self.send_result(target, errors)
return containers
def check(self, target, recurse=0):
if target.type == 'chunk':
self.pool.spawn_n(self.check_chunk, target)
elif target.type == 'object':
self.pool.spawn_n(self.check_obj, target, recurse)
elif target.type == 'container':
self.pool.spawn_n(self.check_container, target, recurse)
else:
self.pool.spawn_n(self.check_account, target, recurse)
def fetch_results(self):
while not self.result_queue.empty():
res = self.result_queue.get(True)
yield res
def log_result(self, result):
if result.errors:
if result.target.type == 'chunk':
# FIXME(FVE): check error criticity
# and set the irreparable flag.
self.write_chunk_error(result.target)
else:
self.write_error(result.target)
self.logger.warn('%s:\n%s', result.target,
result.errors_to_str(err_format=' %s'))
def run(self):
"""
Fetch results and write logs until all jobs have finished.
:returns: a generator yielding check results.
"""
while self.pool.running() + self.pool.waiting():
for result in self.fetch_results():
self.log_result(result)
yield result
sleep(0.1)
self.pool.waitall()
for result in self.fetch_results():
self.log_result(result)
yield result
def report(self):
success = True
def _report_stat(name, stat):
print("{0:18}: {1}".format(name, stat))
print()
print('Report')
_report_stat("Accounts checked", self.accounts_checked)
if self.account_not_found:
success = False
_report_stat("Missing accounts", self.account_not_found)
if self.account_exceptions:
success = False
_report_stat("Exceptions", self.account_exceptions)
print()
_report_stat("Containers checked", self.containers_checked)
if self.container_not_found:
success = False
_report_stat("Missing containers", self.container_not_found)
if self.container_exceptions:
success = False
_report_stat("Exceptions", self.container_exceptions)
print()
_report_stat("Objects checked", self.objects_checked)
if self.object_not_found:
success = False
_report_stat("Missing objects", self.object_not_found)
if self.object_exceptions:
success = False
_report_stat("Exceptions", self.object_exceptions)
print()
_report_stat("Chunks checked", self.chunks_checked)
if self.chunk_not_found:
success = False
_report_stat("Missing chunks", self.chunk_not_found)
if self.chunk_exceptions:
success = False
_report_stat("Exceptions", self.chunk_exceptions)
return success
def stream(self, source, size=None):
bytes_transferred = 0
meta_chunk = self.meta_chunk
if self.chunk_checksum_algo:
meta_checksum = hashlib.new(self.chunk_checksum_algo)
else:
meta_checksum = None
pile = GreenPile(len(meta_chunk))
failed_chunks = []
current_conns = []
for chunk in meta_chunk:
pile.spawn(self._connect_put, chunk)
for conn, chunk in [d for d in pile]:
if not conn:
failed_chunks.append(chunk)
else:
current_conns.append(conn)
self.quorum_or_fail([co.chunk for co in current_conns], failed_chunks)
bytes_transferred = 0
try:
with green.ContextPool(len(meta_chunk)) as pool:
for conn in current_conns:
conn.failed = False
conn.queue = Queue(io.PUT_QUEUE_DEPTH)
pool.spawn(self._send_data, conn)
while True:
if size is not None:
remaining_bytes = size - bytes_transferred
if io.WRITE_CHUNK_SIZE < remaining_bytes:
read_size = io.WRITE_CHUNK_SIZE
else:
read_size = remaining_bytes
else:
read_size = io.WRITE_CHUNK_SIZE
with green.SourceReadTimeout(self.read_timeout):
try:
data = source.read(read_size)
except (ValueError, IOError) as e:
raise SourceReadError(str(e))
if len(data) == 0:
for conn in current_conns:
if not conn.failed:
conn.queue.put('0\r\n\r\n')
break
self.checksum.update(data)
if meta_checksum:
meta_checksum.update(data)
bytes_transferred += len(data)
# copy current_conns to be able to remove a failed conn
for conn in current_conns[:]:
if not conn.failed:
conn.queue.put('%x\r\n%s\r\n' % (len(data), data))
else:
current_conns.remove(conn)
failed_chunks.append(conn.chunk)
self.quorum_or_fail([co.chunk for co in current_conns],
failed_chunks)
for conn in current_conns:
if conn.queue.unfinished_tasks:
conn.queue.join()
except green.SourceReadTimeout as err:
logger.warn('Source read timeout (reqid=%s): %s', self.reqid, err)
raise exc.SourceReadTimeout(err)
except SourceReadError as err:
logger.warn('Source read error (reqid=%s): %s', self.reqid, err)
raise
except Timeout as to:
logger.error('Timeout writing data (reqid=%s): %s', self.reqid, to)
raise exc.OioTimeout(to)
except Exception:
logger.exception('Exception writing data (reqid=%s)', self.reqid)
raise
success_chunks = []
for conn in current_conns:
if conn.failed:
failed_chunks.append(conn.chunk)
continue
pile.spawn(self._get_response, conn)
for (conn, resp) in pile:
if resp:
self._handle_resp(
conn, resp,
meta_checksum.hexdigest() if meta_checksum else None,
success_chunks, failed_chunks)
self.quorum_or_fail(success_chunks, failed_chunks)
for chunk in success_chunks:
chunk["size"] = bytes_transferred
return bytes_transferred, success_chunks[0]['hash'], success_chunks
