def cli_empty_container(context, path, until_empty=False):
"""
Deletes all objects in the container.
By default, this will perform one pass at deleting all objects in
the container; so if objects revert to previous versions or if new
objects otherwise arise during the process, the container may not be
empty once done.
Set `until_empty` to True if you want multiple passes to keep trying
to fully empty the container. Note until_empty=True could run
forever if something else is making new objects faster than they're
being deleted.
See :py:mod:`swiftly.cli.delete` for context usage information.
See :py:class:`CLIDelete` for more information.
"""
path = path.rstrip('/').decode('utf8')
conc = Concurrency(context.concurrency)
def check_conc():
for (exc_type, exc_value, exc_tb, result) in \
conc.get_results().itervalues():
if exc_value:
with context.io_manager.with_stderr() as fp:
fp.write(str(exc_value))
fp.write('\n')
fp.flush()
marker = None
while True:
with context.client_manager.with_client() as client:
status, reason, headers, contents = client.get_container(
path,
marker=marker,
headers=context.headers,
query=context.query,
cdn=context.cdn)
if status // 100 != 2:
if status == 404 and context.ignore_404:
return
raise ReturnCode('listing container %r: %s %s' %
(path, status, reason))
if not contents:
if until_empty and marker:
marker = None
continue
break
for item in contents:
newpath = '%s/%s' % (path, item['name'])
new_context = context.copy()
new_context.ignore_404 = True
check_conc()
conc.spawn(newpath, cli_delete, new_context, newpath)
marker = item['name']
conc.join()
check_conc()
def put_stream(self, key, stream):
(container, obj) = key.split('/', limit=1)
size = None
func_size = getattr(stream, 'size', None)
if func_size:
size = func_size()
if not size:
return self.client.client.put_object(container, obj, stream)
if size <= self.segment_size:
headers = {'Content-Length': size}
(status, reason, headers, contents) = \
self.client.put_object(container, obj, stream,
headers=headers)
if status != 201:
self.logger.error('Failed to put object: %s/%s' %
(container, obj))
return False
cont_prefix = '%s_segments' % container
prefix = '%s/%s/' % (obj, self.segment_size)
conc = Concurrency(self.concurrency)
start = 0
segment = 0
while start < size:
obj_path = '%s%08d' % (prefix, segment)
conc.spawn(
segment,
self._put_recursive,
cont_prefix,
obj_path,
stream.copy(),
start,
)
for rv in conc.get_results().values():
if not rv:
conc.join()
self.logger.error(
'Failed to create segments for object: %s/%s' %
(container, obj))
return rv
segment += 1
start += self.client.segment_size
conc.join()
for rv in conc.get_results().values():
if not rv:
conc.join()
self.logger.error(
'Failed to create segments for object: %s/%s' %
(container, obj))
return rv
headers = {'Content-Length': 0, 'X-Object-Manifest': '%s/%s' \
% (cont_prefix, prefix)}
(status, reason, headers, contents) = \
self.client.put_object(container, obj, '', headers=headers)
if status != 201:
self.logger.error('Failed put manifest object: %s/%s' %
(container, obj))
return False
return True
def cli_ping(context, prefix):
"""
Performs a ping test.
See :py:mod:`swiftly.cli.ping` for context usage information.
See :py:class:`CLIPing` for more information.
:param context: The :py:class:`swiftly.cli.context.CLIContext` to
use.
:param prefix: The container name prefix to use. Default:
swiftly-ping
"""
if not prefix:
prefix = 'swiftly-ping'
ping_ring_object_puts = collections.defaultdict(lambda: [])
ping_ring_object_gets = collections.defaultdict(lambda: [])
ping_ring_object_deletes = collections.defaultdict(lambda: [])
context.ping_begin = context.ping_begin_last = time.time()
container = prefix + '-' + uuid.uuid4().hex
objects = [uuid.uuid4().hex for x in xrange(context.ping_count)]
conc = Concurrency(context.concurrency)
with context.client_manager.with_client() as client:
client.auth()
_cli_ping_status(context, 'auth', '-', None, None, None, None)
_cli_ping_status(context, 'account head', '-', *client.head_account())
_cli_ping_status(context, 'container put', '-',
*client.put_container(container))
if _cli_ping_objects(context, 'put', conc, container, objects,
_cli_ping_object_put, ping_ring_object_puts):
with context.io_manager.with_stderr() as fp:
fp.write('ERROR put objects did not complete successfully due to '
'previous error; but continuing\n')
fp.flush()
if _cli_ping_objects(context, 'get', conc, container, objects,
_cli_ping_object_get, ping_ring_object_gets):
with context.io_manager.with_stderr() as fp:
fp.write('ERROR get objects did not complete successfully due to '
'previous error; but continuing\n')
fp.flush()
if _cli_ping_objects(context, 'delete', conc, container, objects,
_cli_ping_object_delete, ping_ring_object_deletes):
with context.io_manager.with_stderr() as fp:
fp.write(
'ERROR delete objects did not complete successfully due to '
'previous error; but continuing\n')
fp.flush()
for attempt in xrange(5):
if attempt:
sleep(2**attempt)
with context.client_manager.with_client() as client:
try:
_cli_ping_status(context, 'container delete', '-',
*client.delete_container(container))
break
except ReturnCode as err:
with context.io_manager.with_stderr() as fp:
fp.write(str(err))
fp.write('\n')
fp.flush()
else:
with context.io_manager.with_stderr() as fp:
fp.write('ERROR could not confirm deletion of container due to '
'previous error; but continuing\n')
fp.flush()
end = time.time()
with context.io_manager.with_stdout() as fp:
if context.graphite:
fp.write('%s.ping_overall %.02f %d\n' %
(context.graphite, end - context.ping_begin, time.time()))
if context.ping_verbose:
fp.write('% 6.02fs total\n' % (end - context.ping_begin))
elif not context.graphite:
fp.write('%.02fs\n' % (end - context.ping_begin))
fp.flush()
ping_ring_overall = collections.defaultdict(lambda: [])
_cli_ping_ring_report(context, ping_ring_object_puts, 'PUT')
for ip, timings in ping_ring_object_puts.iteritems():
ping_ring_overall[ip].extend(timings)
_cli_ping_ring_report(context, ping_ring_object_gets, 'GET')
for ip, timings in ping_ring_object_gets.iteritems():
ping_ring_overall[ip].extend(timings)
_cli_ping_ring_report(context, ping_ring_object_deletes, 'DELETE')
for ip, timings in ping_ring_object_deletes.iteritems():
ping_ring_overall[ip].extend(timings)
_cli_ping_ring_report(context, ping_ring_overall, 'overall')
def cli_put_directory_structure(context, path):
"""
Performs PUTs rooted at the path using a directory structure
pointed to by context.input\_.
See :py:mod:`swiftly.cli.put` for context usage information.
See :py:class:`CLIPut` for more information.
"""
if not context.input_:
raise ReturnCode(
'called cli_put_directory_structure without context.input_ set')
if not os.path.isdir(context.input_):
raise ReturnCode('%r is not a directory' % context.input_)
if not path:
raise ReturnCode(
'uploading a directory structure requires at least a container '
'name')
new_context = context.copy()
new_context.input_ = None
container = path.split('/', 1)[0]
cli_put_container(new_context, container)
ilen = len(context.input_)
if not context.input_.endswith(os.sep):
ilen += 1
conc = Concurrency(context.concurrency)
for (dirpath, dirnames, filenames) in os.walk(context.input_):
if not dirnames and not filenames:
new_context = context.copy()
new_context.headers = dict(context.headers)
new_context.headers['content-type'] = 'text/directory'
new_context.headers['x-object-meta-mtime'] = \
'%f' % os.path.getmtime(context.input_)
new_context.input_ = None
new_context.empty = True
new_path = path
if path[-1] != '/':
new_path += '/'
new_path += dirpath[ilen:]
for (exc_type, exc_value, exc_tb, result) in \
six.itervalues(conc.get_results()):
if exc_value:
conc.join()
raise exc_value
conc.spawn(new_path, cli_put_object, new_context, new_path)
else:
for fname in filenames:
new_context = context.copy()
new_context.input_ = os.path.join(dirpath, fname)
new_path = path
if path[-1] != '/':
new_path += '/'
if dirpath[ilen:]:
new_path += dirpath[ilen:] + '/'
new_path += fname
for (exc_type, exc_value, exc_tb, result) in \
six.itervalues(conc.get_results()):
if exc_value:
conc.join()
raise exc_value
conc.spawn(new_path, cli_put_object, new_context, new_path)
conc.join()
for (exc_type, exc_value, exc_tb, result) in \
six.itervalues(conc.get_results()):
if exc_value:
raise exc_value
def cli_put_object(context, path):
"""
Performs a PUT on the object.
See :py:mod:`swiftly.cli.put` for context usage information.
See :py:class:`CLIPut` for more information.
"""
if context.different and context.encrypt:
raise ReturnCode(
'context.different will not work properly with context.encrypt '
'since encryption may change the object size')
put_headers = dict(context.headers)
if context.empty:
body = ''
put_headers['content-length'] = '0'
elif not context.input_ or context.input_ == '-':
stdin = context.io_manager.get_stdin()
if context.stdin_segmentation:
def reader():
while True:
chunk = stdin.read(65536)
if chunk:
yield chunk
else:
return
segment_body = FileLikeIter(reader(), context.segment_size)
prefix = _create_container(context, path, time.time(), 0)
new_context = context.copy()
new_context.stdin_segmentation = False
new_context.stdin = segment_body
new_context.headers = dict(context.headers)
segment_n = 0
path2info = {}
while not segment_body.is_empty():
segment_path = _get_segment_path(prefix, segment_n)
etag = cli_put_object(new_context, segment_path)
size = segment_body.limit - segment_body.left
path2info[segment_path] = (size, etag)
segment_body.reset_limit()
segment_n += 1
body = _get_manifest_body(context, prefix, path2info, put_headers)
else:
if hasattr(context, 'stdin'):
body = context.stdin
else:
body = stdin
elif context.seek is not None:
if context.encrypt:
raise ReturnCode(
'putting object %r: Cannot use encryption and context.seek' %
path)
body = open(context.input_, 'rb')
body.seek(context.seek)
else:
l_mtime = os.path.getmtime(context.input_)
l_size = os.path.getsize(context.input_)
put_headers['content-length'] = str(l_size)
if context.newer or context.different:
r_mtime = None
r_size = None
with context.client_manager.with_client() as client:
status, reason, headers, contents = client.head_object(
*path.split('/', 1),
headers=context.headers,
query=context.query,
cdn=context.cdn)
if hasattr(contents, 'read'):
contents.read()
if status // 100 == 2:
r_mtime = headers.get('x-object-meta-mtime')
if r_mtime:
try:
r_mtime = float(r_mtime)
except ValueError:
r_mtime = None
r_size = headers.get('content-length')
if r_size:
try:
r_size = int(r_size)
except ValueError:
r_size = None
elif status != 404:
raise ReturnCode(
'could not head %r for conditional check; skipping put: '
'%s %s' % (path, status, reason))
if context.newer and r_mtime is not None or l_mtime <= r_mtime:
return
if context.different and r_mtime is not None and \
l_mtime == r_mtime and r_size is not None and \
l_size == r_size:
return
put_headers['x-object-meta-mtime'] = '%f' % l_mtime
size = os.path.getsize(context.input_)
if size > context.segment_size:
if context.encrypt:
raise ReturnCode(
'putting object %r: Cannot use encryption for objects '
'greater than the segment size' % path)
prefix = _create_container(context, path, l_mtime, size)
conc = Concurrency(context.concurrency)
start = 0
segment = 0
path2info = {}
while start < size:
new_context = context.copy()
new_context.headers = dict(context.headers)
new_context.headers['content-length'] = str(
min(size - start, context.segment_size))
new_context.seek = start
new_path = _get_segment_path(prefix, segment)
for (ident, (exc_type, exc_value, exc_tb, result)) in \
six.iteritems(conc.get_results()):
if exc_value:
conc.join()
raise exc_value
path2info[ident] = result
conc.spawn(new_path, cli_put_object, new_context, new_path)
segment += 1
start += context.segment_size
conc.join()
for (ident, (exc_type, exc_value, exc_tb, result)) in \
six.iteritems(conc.get_results()):
if exc_value:
raise exc_value
path2info[ident] = result
body = _get_manifest_body(context, prefix, path2info, put_headers)
else:
body = open(context.input_, 'rb')
with context.client_manager.with_client() as client:
if context.encrypt:
content_length = put_headers.get('content-length')
if content_length:
content_length = int(content_length)
if hasattr(body, 'read'):
body = FileLikeIter(
aes_encrypt(context.encrypt,
body,
preamble=AES256CBC,
chunk_size=getattr(client, 'chunk_size',
65536),
content_length=content_length))
else:
body = FileLikeIter(
aes_encrypt(context.encrypt,
FileLikeIter([body]),
preamble=AES256CBC,
chunk_size=getattr(client, 'chunk_size',
65536),
content_length=content_length))
if 'content-length' in put_headers:
del put_headers['content-length']
container, obj = path.split('/', 1)
status, reason, headers, contents = client.put_object(
container,
obj,
body,
headers=put_headers,
query=context.query,
cdn=context.cdn)
if hasattr(contents, 'read'):
contents = contents.read()
if status // 100 != 2:
raise ReturnCode('putting object %r: %s %s %r' %
(path, status, reason, contents))
if context.seek is not None:
content_length = put_headers.get('content-length')
etag = headers.get('etag')
if content_length and etag:
content_length = int(content_length)
else:
with context.client_manager.with_client() as client:
container, obj = path.split('/', 1)
status, reason, headers, contents = client.head_object(
container, obj, cdn=context.cdn)
if hasattr(contents, 'read'):
contents = contents.read()
if status // 100 != 2:
raise ReturnCode('heading object %r: %s %s %r' %
(path, status, reason, contents))
content_length = headers.get('content-length')
etag = headers.get('etag')
if content_length:
content_length = int(content_length)
return content_length, etag
if context.stdin is not None:
return headers.get('etag')
def cli_get_container_listing(context, path=None):
"""
Performs a GET on the container as a listing request.
See :py:mod:`swiftly.cli.get` for context usage information.
See :py:class:`CLIGet` for more information.
"""
path = path.strip('/') if path else None
if not path or '/' in path:
raise ReturnCode(
'tried to get a container listing for non-container path %r' %
path)
context.suppress_container_name = True
limit = context.query.get('limit')
delimiter = context.query.get('delimiter')
prefix = context.query.get('prefix')
marker = context.query.get('marker')
end_marker = context.query.get('end_marker')
if context.raw:
with context.client_manager.with_client() as client:
status, reason, headers, contents = client.get_container(
path, decode_json=False, headers=context.headers, limit=limit,
marker=marker, end_marker=end_marker, query=context.query,
cdn=context.cdn)
if hasattr(contents, 'read'):
contents = contents.read()
if status // 100 != 2:
if status == 404 and context.ignore_404:
return
raise ReturnCode(
'listing container %r: %s %s' % (path, status, reason))
with context.io_manager.with_stdout() as fp:
if context.output_headers:
context.write_headers(
fp, headers, context.muted_container_headers)
fp.write(contents)
fp.flush()
return
with context.client_manager.with_client() as client:
status, reason, headers, contents = client.get_container(
path, headers=context.headers, limit=limit, delimiter=delimiter,
prefix=prefix, marker=marker, end_marker=end_marker,
query=context.query, cdn=context.cdn)
if status // 100 != 2:
if status == 404 and context.ignore_404:
return
if hasattr(contents, 'read'):
contents.read()
raise ReturnCode(
'listing container %r: %s %s' % (path, status, reason))
if context.output_headers and not context.all_objects:
with context.io_manager.with_stdout() as fp:
context.write_headers(
fp, headers, context.muted_container_headers)
conc = Concurrency(context.concurrency)
while contents:
if context.all_objects:
new_context = context.copy()
new_context.query = dict(new_context.query)
for remove in (
'limit', 'delimiter', 'prefix', 'marker', 'end_marker'):
if remove in new_context.query:
del new_context.query[remove]
for item in contents:
if 'name' in item:
for (exc_type, exc_value, exc_tb, result) in \
six.itervalues(conc.get_results()):
if exc_value:
conc.join()
raise exc_value
new_path = path + '/' + item['name'].encode('utf8')
conc.spawn(new_path, cli_get, new_context, new_path)
else:
with context.io_manager.with_stdout() as fp:
for item in contents:
if context.full:
fp.write('%13s %22s %32s %25s ' % (
item.get('bytes', '-'),
item.get('last_modified', '-')[:22].replace(
'T', ' '),
item.get('hash', '-'),
item.get('content_type', '-')))
fp.write(item.get(
'name', item.get('subdir')))
fp.write('\n')
fp.flush()
if limit:
break
marker = contents[-1].get('name', contents[-1].get('subdir', ''))
with context.client_manager.with_client() as client:
status, reason, headers, contents = client.get_container(
path, headers=context.headers, limit=limit,
delimiter=delimiter, prefix=prefix, end_marker=end_marker,
marker=marker, query=context.query, cdn=context.cdn)
if status // 100 != 2:
if status == 404 and context.ignore_404:
return
if hasattr(contents, 'read'):
contents.read()
raise ReturnCode(
'listing container %r: %s %s' % (path, status, reason))
conc.join()
for (exc_type, exc_value, exc_tb, result) in \
six.itervalues(conc.get_results()):
if exc_value:
raise exc_value
def cli_fordo(context, path=None):
"""
Issues commands for each item in an account or container listing.
See :py:mod:`swiftly.cli.fordo` for context usage information.
See :py:class:`CLIForDo` for more information.
"""
path = path.lstrip('/') if path else None
if path and '/' in path:
raise ReturnCode(
'path must be an empty string or a container name; was %r' % path)
limit = context.query.get('limit')
delimiter = context.query.get('delimiter')
prefix = context.query.get('prefix')
marker = context.query.get('marker')
end_marker = context.query.get('end_marker')
conc = Concurrency(context.concurrency)
while True:
with context.client_manager.with_client() as client:
if not path:
status, reason, headers, contents = client.get_account(
headers=context.headers, prefix=prefix,
delimiter=delimiter, marker=marker, end_marker=end_marker,
limit=limit, query=context.query, cdn=context.cdn)
else:
status, reason, headers, contents = client.get_container(
path, headers=context.headers, prefix=prefix,
delimiter=delimiter, marker=marker, end_marker=end_marker,
limit=limit, query=context.query, cdn=context.cdn)
if status // 100 != 2:
if status == 404 and context.ignore_404:
return
if hasattr(contents, 'read'):
contents.read()
if not path:
raise ReturnCode(
'listing account: %s %s' % (status, reason))
else:
raise ReturnCode(
'listing container %r: %s %s' % (path, status, reason))
if not contents:
break
for item in contents:
name = (path + '/' if path else '') + item.get(
'name', item.get('subdir'))
args = list(context.remaining_args)
try:
index = args.index('<item>')
except ValueError:
raise ReturnCode(
'No "<item>" designation found in the "do" clause.')
args[index] = name
for (exc_type, exc_value, exc_tb, result) in \
six.itervalues(conc.get_results()):
if exc_value:
conc.join()
raise exc_value
conc.spawn(name, _cli_call, context, name, args)
marker = contents[-1]['name']
if limit:
break
conc.join()
for (exc_type, exc_value, exc_tb, result) in \
six.itervalues(conc.get_results()):
if exc_value:
conc.join()
raise exc_value
def cli_put_object(context, path):
"""
Performs a PUT on the object.
See :py:mod:`swiftly.cli.put` for context usage information.
See :py:class:`CLIPut` for more information.
"""
if context.different and context.encrypt:
raise ReturnCode(
'context.different will not work properly with context.encrypt '
'since encryption may change the object size')
put_headers = dict(context.headers)
if context.empty:
body = ''
put_headers['content-length'] = '0'
elif not context.input_ or context.input_ == '-':
body = context.io_manager.get_stdin()
elif context.seek is not None:
if context.encrypt:
raise ReturnCode(
'putting object %r: Cannot use encryption and context.seek' %
path)
body = open(context.input_, 'rb')
body.seek(context.seek)
else:
l_mtime = os.path.getmtime(context.input_)
l_size = os.path.getsize(context.input_)
put_headers['content-length'] = str(l_size)
if context.newer or context.different:
r_mtime = None
r_size = None
with context.client_manager.with_client() as client:
status, reason, headers, contents = client.head_object(
*path.split('/', 1),
headers=context.headers,
query=context.query,
cdn=context.cdn)
if hasattr(contents, 'read'):
contents.read()
if status // 100 == 2:
r_mtime = headers.get('x-object-meta-mtime')
if r_mtime:
try:
r_mtime = float(r_mtime)
except ValueError:
r_mtime = None
r_size = headers.get('content-length')
if r_size:
try:
r_size = int(r_size)
except ValueError:
r_size = None
elif status != 404:
raise ReturnCode(
'could not head %r for conditional check; skipping put: '
'%s %s' % (path, status, reason))
if context.newer and r_mtime is not None or l_mtime <= r_mtime:
return
if context.different and r_mtime is not None and \
l_mtime == r_mtime and r_size is not None and \
l_size == r_size:
return
put_headers['x-object-meta-mtime'] = '%f' % l_mtime
size = os.path.getsize(context.input_)
if size > context.segment_size:
if context.encrypt:
raise ReturnCode(
'putting object %r: Cannot use encryption for objects '
'greater than the segment size' % path)
new_context = context.copy()
new_context.input_ = None
new_context.headers = None
new_context.query = None
container = path.split('/', 1)[0] + '_segments'
cli_put_container(new_context, container)
prefix = container + '/' + path.split('/', 1)[1]
prefix = '%s/%s/%s/' % (prefix, l_mtime, size)
conc = Concurrency(context.concurrency)
start = 0
segment = 0
path2info = {}
while start < size:
new_context = context.copy()
new_context.headers = dict(context.headers)
new_context.headers['content-length'] = str(
min(size - start, context.segment_size))
new_context.seek = start
new_path = '%s%08d' % (prefix, segment)
for (ident, (exc_type, exc_value, exc_tb, result)) in \
conc.get_results().iteritems():
if exc_value:
conc.join()
raise exc_value
path2info[ident] = result
conc.spawn(new_path, cli_put_object, new_context, new_path)
segment += 1
start += context.segment_size
conc.join()
for (ident, (exc_type, exc_value, exc_tb, result)) in \
conc.get_results().iteritems():
if exc_value:
raise exc_value
path2info[ident] = result
if context.static_segments:
body = json.dumps([{
'path': '/' + p,
'size_bytes': s,
'etag': e
} for p, (s, e) in sorted(path2info.iteritems())])
put_headers['content-length'] = str(len(body))
context.query['multipart-manifest'] = 'put'
else:
body = ''
put_headers['content-length'] = '0'
put_headers['x-object-manifest'] = prefix
else:
body = open(context.input_, 'rb')
with context.client_manager.with_client() as client:
if context.encrypt:
content_length = put_headers.get('content-length')
if content_length:
content_length = int(content_length)
if hasattr(body, 'read'):
body = FileLikeIter(
aes_encrypt(context.encrypt,
body,
preamble=AES256CBC,
chunk_size=getattr(client, 'chunk_size',
65536),
content_length=content_length))
else:
body = FileLikeIter(
aes_encrypt(context.encrypt,
FileLikeIter([body]),
preamble=AES256CBC,
chunk_size=getattr(client, 'chunk_size',
65536),
content_length=content_length))
if 'content-length' in put_headers:
del put_headers['content-length']
container, obj = path.split('/', 1)
status, reason, headers, contents = client.put_object(
container,
obj,
body,
headers=put_headers,
query=context.query,
cdn=context.cdn)
if hasattr(contents, 'read'):
contents = contents.read()
if status // 100 != 2:
raise ReturnCode('putting object %r: %s %s %r' %
(path, status, reason, contents))
if context.seek is not None:
content_length = put_headers.get('content-length')
etag = headers.get('etag')
if content_length and etag:
content_length = int(content_length)
else:
with context.client_manager.with_client() as client:
container, obj = path.split('/', 1)
status, reason, headers, contents = client.head_object(
container, obj, cdn=context.cdn)
if hasattr(contents, 'read'):
contents = contents.read()
if status // 100 != 2:
raise ReturnCode('heading object %r: %s %s %r' %
(path, status, reason, contents))
content_length = headers.get('content-length')
etag = headers.get('etag')
if content_length:
content_length = int(content_length)
return content_length, etag