def test_default_logging(self, capsys):
cfg = ConfigStruct('/home/mycfg', 'options')
cfg.configure_basic_logging('me')
main_logger = logging.getLogger('me')
child_logger = main_logger.getChild('runt')
other_logger = logging.getLogger('stranger')
main_logger.info('main info')
child_logger.info('child info')
other_logger.info('other info')
other_logger.warn('other warn')
out, err = capsys.readouterr()
assert('main info' in err and
'child info' in err and
'other warn' in err and
not 'other info' in err)
def main(config_file, region, log_level, log_file, concurrency,
selection_string, accumulation_string, reduction_string,
command, s3_uri):
'''Perform simple listing, collating, or deleting of many S3 objects at the same time.
Examples:
\b
List empty objects:
s3workers list --select 'size == 0' s3://mybucket/myprefix
\b
Report total of all non-empty objects:
s3workers list --select 'size > 0' --reduce 'accumulator += size' s3://mybucket/myprefix
\b
Total size group by MD5:
s3workers list --accumulator '{}' --reduce 'v=accumulator.get(md5,0)+size; accumulator[md5]=v' s3://mybucket/myprefix
''' # noqa: E501
config = ConfigStruct(config_file, 'options', options=DEFAULTS)
opts = config.options
# let command line options have temporary precedence if provided values
opts.might_prefer(region=region, log_level=log_level, log_file=log_file,
concurrency=concurrency)
config.configure_basic_logging(__name__)
logger = logging.getLogger(__name__)
s3_uri = re.sub(r'^(s3:)?/+', '', s3_uri)
items = s3_uri.split('/', 1)
bucket_name = items[0]
prefix = items[1] if len(items) > 1 else ''
conn = s3.connect_to_region(opts.region) if opts.region else connect_s3()
bucket = conn.get_bucket(bucket_name)
progress = S3KeyProgress()
reducer = None
if reduction_string:
reducer = Reducer(reduction_string, accumulation_string)
def key_dumper(key):
accumulator = reducer.reduce(key.name, key.size, key.md5, key.last_modified)
progress.write('%s %10d %s %s => %s',
key.last_modified, key.size, key.md5, key.name, accumulator)
else:
def key_dumper(key):
progress.write('%s %10d %s %s', key.last_modified, key.size, key.md5, key.name)
def key_deleter(key):
progress.write('DELETING: %s %10d %s %s', key.last_modified, key.size, key.md5, key.name)
key.delete()
selector = compile(selection_string, '<select>', 'eval') if selection_string else None
handler = key_deleter if command == 'delete' else key_dumper
manager = Manager(opts.concurrency)
manager.start_workers()
logger.info('Preparing %d jobs for %d workers', len(SHARDS), manager.worker_count)
# break up jobs into single char prefix jobs
for shard in SHARDS:
manager.add_work(S3ListJob(bucket, prefix + shard, selector, handler, progress.report))
manager.wait_for_workers()
progress.finish()
if reducer:
click.echo('Final accumulator value: ' + str(reducer.accumulator))