def inspect_partitions(bucket):
"""Discover the partitions on a bucket via introspection.
For large buckets which lack s3 inventories, salactus will attempt
to process objects in parallel on the bucket by breaking the bucket
into a separate keyspace partitions. It does this with a heurestic
that attempts to sample the keyspace and determine appropriate subparts.
This command provides additional visibility into the partitioning of
a bucket by showing how salactus would partition a given bucket.
"""
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s: %(name)s:%(levelname)s %(message)s")
logging.getLogger('botocore').setLevel(level=logging.WARNING)
state = db.db()
# add db.bucket accessor
found = None
for b in state.buckets():
if b.name == bucket:
found = b
break
if not found:
click.echo("no bucket named: %s" % bucket)
return
keyset = []
partitions = []
def process_keyset(bid, page):
keyset.append(len(page))
def process_bucket_iterator(bid, prefix, delimiter="", **continuation):
partitions.append(prefix)
# synchronous execution
def invoke(f, *args, **kw):
return f(*args, **kw)
# unleash the monkies ;-)
worker.connection.hincrby = lambda x, y, z: True
worker.invoke = invoke
worker.process_keyset = process_keyset
worker.process_bucket_iterator = process_bucket_iterator
# kick it off
worker.process_bucket_partitions(b.bucket_id)
keys_scanned = sum(keyset)
click.echo(
"Found %d partitions %s keys scanned during partitioning" % (
len(partitions), keys_scanned))
click.echo("\n".join(partitions))
def inspect_partitions(bucket):
"""Discover the partitions on a bucket via introspection.
For large buckets which lack s3 inventories, salactus will attempt
to process objects in parallel on the bucket by breaking the bucket
into a separate keyspace partitions. It does this with a heurestic
that attempts to sample the keyspace and determine appropriate subparts.
This command provides additional visibility into the partitioning of
a bucket by showing how salactus would partition a given bucket.
"""
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s: %(name)s:%(levelname)s %(message)s")
logging.getLogger('botocore').setLevel(level=logging.WARNING)
state = db.db()
# add db.bucket accessor
found = None
for b in state.buckets():
if b.name == bucket:
found = b
break
if not found:
click.echo("no bucket named: %s" % bucket)
return
keyset = []
partitions = []
def process_keyset(bid, page):
keyset.append(len(page))
def process_bucket_iterator(bid, prefix, delimiter="", **continuation):
partitions.append(prefix)
# synchronous execution
def invoke(f, *args, **kw):
return f(*args, **kw)
# unleash the monkies ;-)
worker.connection.hincrby = lambda x, y, z: True
worker.invoke = invoke
worker.process_keyset = process_keyset
worker.process_bucket_iterator = process_bucket_iterator
# kick it off
worker.process_bucket_partitions(b.bucket_id)
keys_scanned = sum(keyset)
click.echo(
"Found %d partitions %s keys scanned during partitioning" % (
len(partitions), keys_scanned))
click.echo("\n".join(partitions))