def cli_empty_container(context, path, until_empty=False): """ Deletes all objects in the container. By default, this will perform one pass at deleting all objects in the container; so if objects revert to previous versions or if new objects otherwise arise during the process, the container may not be empty once done. Set `until_empty` to True if you want multiple passes to keep trying to fully empty the container. Note until_empty=True could run forever if something else is making new objects faster than they're being deleted. See :py:mod:`swiftly.cli.delete` for context usage information. See :py:class:`CLIDelete` for more information. """ path = path.rstrip('/').decode('utf8') conc = Concurrency(context.concurrency) def check_conc(): for (exc_type, exc_value, exc_tb, result) in \ six.itervalues(conc.get_results()): if exc_value: with context.io_manager.with_stderr() as fp: fp.write(str(exc_value)) fp.write('\n') fp.flush() marker = None while True: with context.client_manager.with_client() as client: status, reason, headers, contents = client.get_container( path, marker=marker, headers=context.headers, query=context.query, cdn=context.cdn) if status // 100 != 2: if status == 404 and context.ignore_404: return raise ReturnCode( 'listing container %r: %s %s' % (path, status, reason)) if not contents: if until_empty and marker: marker = None continue break for item in contents: newpath = '%s/%s' % (path, item['name']) new_context = context.copy() new_context.ignore_404 = True check_conc() conc.spawn(newpath, cli_delete, new_context, newpath) marker = item['name'] conc.join() check_conc()
def cli_empty_container(context, path): """ Empties the container at the path of all objects. See :py:mod:`swiftly.cli.delete` for context usage information. See :py:class:`CLIDelete` for more information. """ path = path.rstrip('/') conc = Concurrency(context.concurrency) def check_conc(): for (exc_type, exc_value, exc_tb, result) in \ conc.get_results().itervalues(): if exc_value: with context.io_manager.with_stderr() as fp: fp.write(str(exc_value)) fp.write('\n') fp.flush() marker = None while True: with context.client_manager.with_client() as client: status, reason, headers, contents = client.get_container( path, marker=marker, headers=context.headers, query=context.query, cdn=context.cdn) if status // 100 != 2: if status == 404 and context.ignore_404: return raise ReturnCode( 'listing container %r: %s %s' % (path, status, reason)) if not contents: break for item in contents: newpath = '%s/%s' % (path, item['name']) new_context = context.copy() new_context.ignore_404 = True check_conc() conc.spawn(newpath, cli_delete, new_context, newpath) marker = item['name'] conc.join() check_conc()
def cli_empty_container(context, path, until_empty=False): """ Deletes all objects in the container. By default, this will perform one pass at deleting all objects in the container; so if objects revert to previous versions or if new objects otherwise arise during the process, the container may not be empty once done. Set `until_empty` to True if you want multiple passes to keep trying to fully empty the container. Note until_empty=True could run forever if something else is making new objects faster than they're being deleted. See :py:mod:`swiftly.cli.delete` for context usage information. See :py:class:`CLIDelete` for more information. """ path = path.rstrip('/').decode('utf8') conc = Concurrency(context.concurrency) def check_conc(): for (exc_type, exc_value, exc_tb, result) in \ conc.get_results().itervalues(): if exc_value: with context.io_manager.with_stderr() as fp: fp.write(str(exc_value)) fp.write('\n') fp.flush() marker = None while True: with context.client_manager.with_client() as client: status, reason, headers, contents = client.get_container( path, marker=marker, headers=context.headers, query=context.query, cdn=context.cdn) if status // 100 != 2: if status == 404 and context.ignore_404: return raise ReturnCode('listing container %r: %s %s' % (path, status, reason)) if not contents: if until_empty and marker: marker = None continue break for item in contents: newpath = '%s/%s' % (path, item['name']) new_context = context.copy() new_context.ignore_404 = True check_conc() conc.spawn(newpath, cli_delete, new_context, newpath) marker = item['name'] conc.join() check_conc()
def cli_put_directory_structure(context, path): """ Performs PUTs rooted at the path using a directory structure pointed to by context.input\_. See :py:mod:`swiftly.cli.put` for context usage information. See :py:class:`CLIPut` for more information. """ if not context.input_: raise ReturnCode( 'called cli_put_directory_structure without context.input_ set') if not os.path.isdir(context.input_): raise ReturnCode( '%r is not a directory' % context.input_) if not path: raise ReturnCode( 'uploading a directory structure requires at least a container ' 'name') new_context = context.copy() new_context.input_ = None container = path.split('/', 1)[0] cli_put_container(new_context, container) ilen = len(context.input_) if not context.input_.endswith(os.sep): ilen += 1 conc = Concurrency(context.concurrency) for (dirpath, dirnames, filenames) in os.walk(context.input_): if not dirnames and not filenames: new_context = context.copy() new_context.headers = dict(context.headers) new_context.headers['content-type'] = 'text/directory' new_context.headers['x-object-meta-mtime'] = \ '%f' % os.path.getmtime(context.input_) new_context.input_ = None new_context.empty = True new_path = path if path[-1] != '/': new_path += '/' new_path += dirpath[ilen:] for (exc_type, exc_value, exc_tb, result) in \ six.itervalues(conc.get_results()): if exc_value: conc.join() raise exc_value conc.spawn(new_path, cli_put_object, new_context, new_path) else: for fname in filenames: new_context = context.copy() new_context.input_ = os.path.join(dirpath, fname) new_path = path if path[-1] != '/': new_path += '/' if dirpath[ilen:]: new_path += dirpath[ilen:] + '/' new_path += fname for (exc_type, exc_value, exc_tb, result) in \ six.itervalues(conc.get_results()): if exc_value: conc.join() raise exc_value conc.spawn(new_path, cli_put_object, new_context, new_path) conc.join() for (exc_type, exc_value, exc_tb, result) in \ six.itervalues(conc.get_results()): if exc_value: raise exc_value
def cli_put_object(context, path): """ Performs a PUT on the object. See :py:mod:`swiftly.cli.put` for context usage information. See :py:class:`CLIPut` for more information. """ if context.different and context.encrypt: raise ReturnCode( 'context.different will not work properly with context.encrypt ' 'since encryption may change the object size') put_headers = dict(context.headers) if context.empty: body = '' put_headers['content-length'] = '0' elif not context.input_ or context.input_ == '-': stdin = context.io_manager.get_stdin() if context.stdin_segmentation: def reader(): while True: chunk = stdin.read(65536) if chunk: yield chunk else: return segment_body = FileLikeIter(reader(), context.segment_size) prefix = _create_container(context, path, time.time(), 0) new_context = context.copy() new_context.stdin_segmentation = False new_context.stdin = segment_body new_context.headers = dict(context.headers) segment_n = 0 path2info = {} while not segment_body.is_empty(): segment_path = _get_segment_path(prefix, segment_n) etag = cli_put_object(new_context, segment_path) size = segment_body.limit - segment_body.left path2info[segment_path] = (size, etag) segment_body.reset_limit() segment_n += 1 body = _get_manifest_body(context, prefix, path2info, put_headers) else: if hasattr(context, 'stdin'): body = context.stdin else: body = stdin elif context.seek is not None: if context.encrypt: raise ReturnCode( 'putting object %r: Cannot use encryption and context.seek' % path) body = open(context.input_, 'rb') body.seek(context.seek) else: l_mtime = os.path.getmtime(context.input_) l_size = os.path.getsize(context.input_) put_headers['content-length'] = str(l_size) if context.newer or context.different: r_mtime = None r_size = None with context.client_manager.with_client() as client: status, reason, headers, contents = client.head_object( *path.split('/', 1), headers=context.headers, query=context.query, cdn=context.cdn) if hasattr(contents, 'read'): contents.read() if status // 100 == 2: r_mtime = headers.get('x-object-meta-mtime') if r_mtime: try: r_mtime = float(r_mtime) except ValueError: r_mtime = None r_size = headers.get('content-length') if r_size: try: r_size = int(r_size) except ValueError: r_size = None elif status != 404: raise ReturnCode( 'could not head %r for conditional check; skipping put: ' '%s %s' % (path, status, reason)) if context.newer and r_mtime is not None or l_mtime <= r_mtime: return if context.different and r_mtime is not None and \ l_mtime == r_mtime and r_size is not None and \ l_size == r_size: return put_headers['x-object-meta-mtime'] = '%f' % l_mtime size = os.path.getsize(context.input_) if size > context.segment_size: if context.encrypt: raise ReturnCode( 'putting object %r: Cannot use encryption for objects ' 'greater than the segment size' % path) prefix = _create_container(context, path, l_mtime, size) conc = Concurrency(context.concurrency) start = 0 segment = 0 path2info = {} while start < size: new_context = context.copy() new_context.headers = dict(context.headers) new_context.headers['content-length'] = str(min( size - start, context.segment_size)) new_context.seek = start new_path = _get_segment_path(prefix, segment) for (ident, (exc_type, exc_value, exc_tb, result)) in \ six.iteritems(conc.get_results()): if exc_value: conc.join() raise exc_value path2info[ident] = result conc.spawn( new_path, cli_put_object, new_context, new_path) segment += 1 start += context.segment_size conc.join() for (ident, (exc_type, exc_value, exc_tb, result)) in \ six.iteritems(conc.get_results()): if exc_value: raise exc_value path2info[ident] = result body = _get_manifest_body(context, prefix, path2info, put_headers) else: body = open(context.input_, 'rb') with context.client_manager.with_client() as client: if context.encrypt: content_length = put_headers.get('content-length') if content_length: content_length = int(content_length) if hasattr(body, 'read'): body = FileLikeIter(aes_encrypt( context.encrypt, body, preamble=AES256CBC, chunk_size=getattr(client, 'chunk_size', 65536), content_length=content_length)) else: body = FileLikeIter(aes_encrypt( context.encrypt, FileLikeIter([body]), preamble=AES256CBC, chunk_size=getattr(client, 'chunk_size', 65536), content_length=content_length)) if 'content-length' in put_headers: del put_headers['content-length'] container, obj = path.split('/', 1) status, reason, headers, contents = client.put_object( container, obj, body, headers=put_headers, query=context.query, cdn=context.cdn) if hasattr(contents, 'read'): contents = contents.read() if status // 100 != 2: raise ReturnCode( 'putting object %r: %s %s %r' % (path, status, reason, contents)) if context.seek is not None: content_length = put_headers.get('content-length') etag = headers.get('etag') if content_length and etag: content_length = int(content_length) else: with context.client_manager.with_client() as client: container, obj = path.split('/', 1) status, reason, headers, contents = client.head_object( container, obj, cdn=context.cdn) if hasattr(contents, 'read'): contents = contents.read() if status // 100 != 2: raise ReturnCode( 'heading object %r: %s %s %r' % (path, status, reason, contents)) content_length = headers.get('content-length') etag = headers.get('etag') if content_length: content_length = int(content_length) return content_length, etag if context.stdin is not None: return headers.get('etag')
def cli_get_container_listing(context, path=None): """ Performs a GET on the container as a listing request. See :py:mod:`swiftly.cli.get` for context usage information. See :py:class:`CLIGet` for more information. """ path = path.strip('/') if path else None if not path or '/' in path: raise ReturnCode( 'tried to get a container listing for non-container path %r' % path) context.suppress_container_name = True limit = context.query.get('limit') delimiter = context.query.get('delimiter') prefix = context.query.get('prefix') marker = context.query.get('marker') end_marker = context.query.get('end_marker') if context.raw: with context.client_manager.with_client() as client: status, reason, headers, contents = client.get_container( path, decode_json=False, headers=context.headers, limit=limit, marker=marker, end_marker=end_marker, query=context.query, cdn=context.cdn) if hasattr(contents, 'read'): contents = contents.read() if status // 100 != 2: if status == 404 and context.ignore_404: return raise ReturnCode( 'listing container %r: %s %s' % (path, status, reason)) with context.io_manager.with_stdout() as fp: if context.output_headers: context.write_headers( fp, headers, context.muted_container_headers) fp.write(contents) fp.flush() return with context.client_manager.with_client() as client: status, reason, headers, contents = client.get_container( path, headers=context.headers, limit=limit, delimiter=delimiter, prefix=prefix, marker=marker, end_marker=end_marker, query=context.query, cdn=context.cdn) if status // 100 != 2: if status == 404 and context.ignore_404: return if hasattr(contents, 'read'): contents.read() raise ReturnCode( 'listing container %r: %s %s' % (path, status, reason)) if context.output_headers and not context.all_objects: with context.io_manager.with_stdout() as fp: context.write_headers( fp, headers, context.muted_container_headers) conc = Concurrency(context.concurrency) while contents: if context.all_objects: new_context = context.copy() new_context.query = dict(new_context.query) for remove in ( 'limit', 'delimiter', 'prefix', 'marker', 'end_marker'): if remove in new_context.query: del new_context.query[remove] for item in contents: if 'name' in item: for (exc_type, exc_value, exc_tb, result) in \ conc.get_results().itervalues(): if exc_value: conc.join() raise exc_value new_path = path + '/' + item['name'].encode('utf8') conc.spawn(new_path, cli_get, new_context, new_path) else: with context.io_manager.with_stdout() as fp: for item in contents: if context.full: fp.write('%13s %22s %32s %25s ' % ( item.get('bytes', '-'), item.get('last_modified', '-')[:22].replace( 'T', ' '), item.get('hash', '-'), item.get('content_type', '-'))) fp.write(item.get( 'name', item.get('subdir')).encode('utf8')) fp.write('\n') fp.flush() if limit: break marker = contents[-1].get('name', contents[-1].get('subdir', '')) with context.client_manager.with_client() as client: status, reason, headers, contents = client.get_container( path, headers=context.headers, limit=limit, delimiter=delimiter, prefix=prefix, end_marker=end_marker, marker=marker, query=context.query, cdn=context.cdn) if status // 100 != 2: if status == 404 and context.ignore_404: return if hasattr(contents, 'read'): contents.read() raise ReturnCode( 'listing container %r: %s %s' % (path, status, reason)) conc.join() for (exc_type, exc_value, exc_tb, result) in \ conc.get_results().itervalues(): if exc_value: raise exc_value
def cli_ping(context, prefix): """ Performs a ping test. See :py:mod:`swiftly.cli.ping` for context usage information. See :py:class:`CLIPing` for more information. :param context: The :py:class:`swiftly.cli.context.CLIContext` to use. :param prefix: The container name prefix to use. Default: swiftly-ping """ if not prefix: prefix = 'swiftly-ping' ping_ring_object_puts = collections.defaultdict(lambda: []) ping_ring_object_gets = collections.defaultdict(lambda: []) ping_ring_object_deletes = collections.defaultdict(lambda: []) context.ping_begin = context.ping_begin_last = time.time() container = prefix + '-' + uuid.uuid4().hex objects = [uuid.uuid4().hex for x in xrange(context.ping_count)] conc = Concurrency(context.concurrency) with context.client_manager.with_client() as client: client.auth() _cli_ping_status(context, 'auth', '-', None, None, None, None) _cli_ping_status(context, 'account head', '-', *client.head_account()) _cli_ping_status(context, 'container put', '-', *client.put_container(container)) if _cli_ping_objects(context, 'put', conc, container, objects, _cli_ping_object_put, ping_ring_object_puts): with context.io_manager.with_stderr() as fp: fp.write('ERROR put objects did not complete successfully due to ' 'previous error; but continuing\n') fp.flush() if _cli_ping_objects(context, 'get', conc, container, objects, _cli_ping_object_get, ping_ring_object_gets): with context.io_manager.with_stderr() as fp: fp.write('ERROR get objects did not complete successfully due to ' 'previous error; but continuing\n') fp.flush() if _cli_ping_objects(context, 'delete', conc, container, objects, _cli_ping_object_delete, ping_ring_object_deletes): with context.io_manager.with_stderr() as fp: fp.write( 'ERROR delete objects did not complete successfully due to ' 'previous error; but continuing\n') fp.flush() for attempt in xrange(5): if attempt: sleep(2**attempt) with context.client_manager.with_client() as client: try: _cli_ping_status(context, 'container delete', '-', *client.delete_container(container)) break except ReturnCode as err: with context.io_manager.with_stderr() as fp: fp.write(str(err)) fp.write('\n') fp.flush() else: with context.io_manager.with_stderr() as fp: fp.write('ERROR could not confirm deletion of container due to ' 'previous error; but continuing\n') fp.flush() end = time.time() with context.io_manager.with_stdout() as fp: if context.graphite: fp.write('%s.ping_overall %.02f %d\n' % (context.graphite, end - context.ping_begin, time.time())) if context.ping_verbose: fp.write('% 6.02fs total\n' % (end - context.ping_begin)) elif not context.graphite: fp.write('%.02fs\n' % (end - context.ping_begin)) fp.flush() ping_ring_overall = collections.defaultdict(lambda: []) _cli_ping_ring_report(context, ping_ring_object_puts, 'PUT') for ip, timings in ping_ring_object_puts.iteritems(): ping_ring_overall[ip].extend(timings) _cli_ping_ring_report(context, ping_ring_object_gets, 'GET') for ip, timings in ping_ring_object_gets.iteritems(): ping_ring_overall[ip].extend(timings) _cli_ping_ring_report(context, ping_ring_object_deletes, 'DELETE') for ip, timings in ping_ring_object_deletes.iteritems(): ping_ring_overall[ip].extend(timings) _cli_ping_ring_report(context, ping_ring_overall, 'overall')
def cli_put_object(context, path): """ Performs a PUT on the object. See :py:mod:`swiftly.cli.put` for context usage information. See :py:class:`CLIPut` for more information. """ if context.different and context.encrypt: raise ReturnCode( 'context.different will not work properly with context.encrypt ' 'since encryption may change the object size') put_headers = dict(context.headers) if context.empty: body = '' put_headers['content-length'] = '0' elif not context.input_ or context.input_ == '-': body = context.io_manager.get_stdin() elif context.seek is not None: if context.encrypt: raise ReturnCode( 'putting object %r: Cannot use encryption and context.seek' % path) body = open(context.input_, 'rb') body.seek(context.seek) else: l_mtime = os.path.getmtime(context.input_) l_size = os.path.getsize(context.input_) put_headers['content-length'] = str(l_size) if context.newer or context.different: r_mtime = None r_size = None with context.client_manager.with_client() as client: status, reason, headers, contents = client.head_object( *path.split('/', 1), headers=context.headers, query=context.query, cdn=context.cdn) if hasattr(contents, 'read'): contents.read() if status // 100 == 2: r_mtime = headers.get('x-object-meta-mtime') if r_mtime: try: r_mtime = float(r_mtime) except ValueError: r_mtime = None r_size = headers.get('content-length') if r_size: try: r_size = int(r_size) except ValueError: r_size = None elif status != 404: raise ReturnCode( 'could not head %r for conditional check; skipping put: ' '%s %s' % (path, status, reason)) if context.newer and r_mtime is not None or l_mtime <= r_mtime: return if context.different and r_mtime is not None and \ l_mtime == r_mtime and r_size is not None and \ l_size == r_size: return put_headers['x-object-meta-mtime'] = '%f' % l_mtime size = os.path.getsize(context.input_) if size > context.segment_size: if context.encrypt: raise ReturnCode( 'putting object %r: Cannot use encryption for objects ' 'greater than the segment size' % path) new_context = context.copy() new_context.input_ = None new_context.headers = None new_context.query = None container = path.split('/', 1)[0] + '_segments' cli_put_container(new_context, container) prefix = container + '/' + path.split('/', 1)[1] prefix = '%s/%s/%s/' % (prefix, l_mtime, size) conc = Concurrency(context.concurrency) start = 0 segment = 0 path2info = {} while start < size: new_context = context.copy() new_context.headers = dict(context.headers) new_context.headers['content-length'] = str(min( size - start, context.segment_size)) new_context.seek = start new_path = '%s%08d' % (prefix, segment) for (ident, (exc_type, exc_value, exc_tb, result)) in \ six.iteritems(conc.get_results()): if exc_value: conc.join() raise exc_value path2info[ident] = result conc.spawn( new_path, cli_put_object, new_context, new_path) segment += 1 start += context.segment_size conc.join() for (ident, (exc_type, exc_value, exc_tb, result)) in \ six.iteritems(conc.get_results()): if exc_value: raise exc_value path2info[ident] = result if context.static_segments: body = json.dumps([ {'path': '/' + p, 'size_bytes': s, 'etag': e} for p, (s, e) in sorted(six.iteritems(path2info))]) put_headers['content-length'] = str(len(body)) context.query['multipart-manifest'] = 'put' else: body = '' put_headers['content-length'] = '0' put_headers['x-object-manifest'] = prefix else: body = open(context.input_, 'rb') with context.client_manager.with_client() as client: if context.encrypt: content_length = put_headers.get('content-length') if content_length: content_length = int(content_length) if hasattr(body, 'read'): body = FileLikeIter(aes_encrypt( context.encrypt, body, preamble=AES256CBC, chunk_size=getattr(client, 'chunk_size', 65536), content_length=content_length)) else: body = FileLikeIter(aes_encrypt( context.encrypt, FileLikeIter([body]), preamble=AES256CBC, chunk_size=getattr(client, 'chunk_size', 65536), content_length=content_length)) if 'content-length' in put_headers: del put_headers['content-length'] container, obj = path.split('/', 1) status, reason, headers, contents = client.put_object( container, obj, body, headers=put_headers, query=context.query, cdn=context.cdn) if hasattr(contents, 'read'): contents = contents.read() if status // 100 != 2: raise ReturnCode( 'putting object %r: %s %s %r' % (path, status, reason, contents)) if context.seek is not None: content_length = put_headers.get('content-length') etag = headers.get('etag') if content_length and etag: content_length = int(content_length) else: with context.client_manager.with_client() as client: container, obj = path.split('/', 1) status, reason, headers, contents = client.head_object( container, obj, cdn=context.cdn) if hasattr(contents, 'read'): contents = contents.read() if status // 100 != 2: raise ReturnCode( 'heading object %r: %s %s %r' % (path, status, reason, contents)) content_length = headers.get('content-length') etag = headers.get('etag') if content_length: content_length = int(content_length) return content_length, etag
def cli_put_directory_structure(context, path): """ Performs PUTs rooted at the path using a directory structure pointed to by context.input\_. See :py:mod:`swiftly.cli.put` for context usage information. See :py:class:`CLIPut` for more information. """ if not context.input_: raise ReturnCode( 'called cli_put_directory_structure without context.input_ set') if not os.path.isdir(context.input_): raise ReturnCode('%r is not a directory' % context.input_) if not path: raise ReturnCode( 'uploading a directory structure requires at least a container ' 'name') new_context = context.copy() new_context.input_ = None container = path.split('/', 1)[0] cli_put_container(new_context, container) ilen = len(context.input_) if not context.input_.endswith(os.sep): ilen += 1 conc = Concurrency(context.concurrency) for (dirpath, dirnames, filenames) in os.walk(context.input_): if not dirnames and not filenames: new_context = context.copy() new_context.headers = dict(context.headers) new_context.headers['content-type'] = 'text/directory' new_context.headers['x-object-meta-mtime'] = \ '%f' % os.path.getmtime(context.input_) new_context.input_ = None new_context.empty = True new_path = path if path[-1] != '/': new_path += '/' new_path += dirpath[ilen:] for (exc_type, exc_value, exc_tb, result) in \ six.itervalues(conc.get_results()): if exc_value: conc.join() raise exc_value conc.spawn(new_path, cli_put_object, new_context, new_path) else: for fname in filenames: new_context = context.copy() new_context.input_ = os.path.join(dirpath, fname) new_path = path if path[-1] != '/': new_path += '/' if dirpath[ilen:]: new_path += dirpath[ilen:] + '/' new_path += fname for (exc_type, exc_value, exc_tb, result) in \ six.itervalues(conc.get_results()): if exc_value: conc.join() raise exc_value conc.spawn(new_path, cli_put_object, new_context, new_path) conc.join() for (exc_type, exc_value, exc_tb, result) in \ six.itervalues(conc.get_results()): if exc_value: raise exc_value
def cli_put_object(context, path): """ Performs a PUT on the object. See :py:mod:`swiftly.cli.put` for context usage information. See :py:class:`CLIPut` for more information. """ if context.different and context.encrypt: raise ReturnCode( 'context.different will not work properly with context.encrypt ' 'since encryption may change the object size') put_headers = dict(context.headers) if context.empty: body = '' put_headers['content-length'] = '0' elif not context.input_ or context.input_ == '-': stdin = context.io_manager.get_stdin() if context.stdin_segmentation: def reader(): while True: chunk = stdin.read(65536) if chunk: yield chunk else: return segment_body = FileLikeIter(reader(), context.segment_size) prefix = _create_container(context, path, time.time(), 0) new_context = context.copy() new_context.stdin_segmentation = False new_context.stdin = segment_body new_context.headers = dict(context.headers) segment_n = 0 path2info = {} while not segment_body.is_empty(): segment_path = _get_segment_path(prefix, segment_n) etag = cli_put_object(new_context, segment_path) size = segment_body.limit - segment_body.left path2info[segment_path] = (size, etag) segment_body.reset_limit() segment_n += 1 body = _get_manifest_body(context, prefix, path2info, put_headers) else: if hasattr(context, 'stdin'): body = context.stdin else: body = stdin elif context.seek is not None: if context.encrypt: raise ReturnCode( 'putting object %r: Cannot use encryption and context.seek' % path) body = open(context.input_, 'rb') body.seek(context.seek) else: l_mtime = os.path.getmtime(context.input_) l_size = os.path.getsize(context.input_) put_headers['content-length'] = str(l_size) if context.newer or context.different: r_mtime = None r_size = None with context.client_manager.with_client() as client: status, reason, headers, contents = client.head_object( *path.split('/', 1), headers=context.headers, query=context.query, cdn=context.cdn) if hasattr(contents, 'read'): contents.read() if status // 100 == 2: r_mtime = headers.get('x-object-meta-mtime') if r_mtime: try: r_mtime = float(r_mtime) except ValueError: r_mtime = None r_size = headers.get('content-length') if r_size: try: r_size = int(r_size) except ValueError: r_size = None elif status != 404: raise ReturnCode( 'could not head %r for conditional check; skipping put: ' '%s %s' % (path, status, reason)) if context.newer and r_mtime is not None or l_mtime <= r_mtime: return if context.different and r_mtime is not None and \ l_mtime == r_mtime and r_size is not None and \ l_size == r_size: return put_headers['x-object-meta-mtime'] = '%f' % l_mtime size = os.path.getsize(context.input_) if size > context.segment_size: if context.encrypt: raise ReturnCode( 'putting object %r: Cannot use encryption for objects ' 'greater than the segment size' % path) prefix = _create_container(context, path, l_mtime, size) conc = Concurrency(context.concurrency) start = 0 segment = 0 path2info = {} while start < size: new_context = context.copy() new_context.headers = dict(context.headers) new_context.headers['content-length'] = str( min(size - start, context.segment_size)) new_context.seek = start new_path = _get_segment_path(prefix, segment) for (ident, (exc_type, exc_value, exc_tb, result)) in \ six.iteritems(conc.get_results()): if exc_value: conc.join() raise exc_value path2info[ident] = result conc.spawn(new_path, cli_put_object, new_context, new_path) segment += 1 start += context.segment_size conc.join() for (ident, (exc_type, exc_value, exc_tb, result)) in \ six.iteritems(conc.get_results()): if exc_value: raise exc_value path2info[ident] = result body = _get_manifest_body(context, prefix, path2info, put_headers) else: body = open(context.input_, 'rb') with context.client_manager.with_client() as client: if context.encrypt: content_length = put_headers.get('content-length') if content_length: content_length = int(content_length) if hasattr(body, 'read'): body = FileLikeIter( aes_encrypt(context.encrypt, body, preamble=AES256CBC, chunk_size=getattr(client, 'chunk_size', 65536), content_length=content_length)) else: body = FileLikeIter( aes_encrypt(context.encrypt, FileLikeIter([body]), preamble=AES256CBC, chunk_size=getattr(client, 'chunk_size', 65536), content_length=content_length)) if 'content-length' in put_headers: del put_headers['content-length'] container, obj = path.split('/', 1) status, reason, headers, contents = client.put_object( container, obj, body, headers=put_headers, query=context.query, cdn=context.cdn) if hasattr(contents, 'read'): contents = contents.read() if status // 100 != 2: raise ReturnCode('putting object %r: %s %s %r' % (path, status, reason, contents)) if context.seek is not None: content_length = put_headers.get('content-length') etag = headers.get('etag') if content_length and etag: content_length = int(content_length) else: with context.client_manager.with_client() as client: container, obj = path.split('/', 1) status, reason, headers, contents = client.head_object( container, obj, cdn=context.cdn) if hasattr(contents, 'read'): contents = contents.read() if status // 100 != 2: raise ReturnCode('heading object %r: %s %s %r' % (path, status, reason, contents)) content_length = headers.get('content-length') etag = headers.get('etag') if content_length: content_length = int(content_length) return content_length, etag if context.stdin is not None: return headers.get('etag')
def cli_get_container_listing(context, path=None): """ Performs a GET on the container as a listing request. See :py:mod:`swiftly.cli.get` for context usage information. See :py:class:`CLIGet` for more information. """ path = path.strip('/') if path else None if not path or '/' in path: raise ReturnCode( 'tried to get a container listing for non-container path %r' % path) context.suppress_container_name = True limit = context.query.get('limit') delimiter = context.query.get('delimiter') prefix = context.query.get('prefix') marker = context.query.get('marker') end_marker = context.query.get('end_marker') if context.raw: with context.client_manager.with_client() as client: status, reason, headers, contents = client.get_container( path, decode_json=False, headers=context.headers, limit=limit, marker=marker, end_marker=end_marker, query=context.query, cdn=context.cdn) if hasattr(contents, 'read'): contents = contents.read() if status // 100 != 2: if status == 404 and context.ignore_404: return raise ReturnCode( 'listing container %r: %s %s' % (path, status, reason)) with context.io_manager.with_stdout() as fp: if context.output_headers: context.write_headers( fp, headers, context.muted_container_headers) fp.write(contents) fp.flush() return with context.client_manager.with_client() as client: status, reason, headers, contents = client.get_container( path, headers=context.headers, limit=limit, delimiter=delimiter, prefix=prefix, marker=marker, end_marker=end_marker, query=context.query, cdn=context.cdn) if status // 100 != 2: if status == 404 and context.ignore_404: return if hasattr(contents, 'read'): contents.read() raise ReturnCode( 'listing container %r: %s %s' % (path, status, reason)) if context.output_headers and not context.all_objects: with context.io_manager.with_stdout() as fp: context.write_headers( fp, headers, context.muted_container_headers) conc = Concurrency(context.concurrency) while contents: if context.all_objects: new_context = context.copy() new_context.query = dict(new_context.query) for remove in ( 'limit', 'delimiter', 'prefix', 'marker', 'end_marker'): if remove in new_context.query: del new_context.query[remove] for item in contents: if 'name' in item: for (exc_type, exc_value, exc_tb, result) in \ six.itervalues(conc.get_results()): if exc_value: conc.join() raise exc_value new_path = path + '/' + item['name'].encode('utf8') conc.spawn(new_path, cli_get, new_context, new_path) else: with context.io_manager.with_stdout() as fp: for item in contents: if context.full: fp.write('%13s %22s %32s %25s ' % ( item.get('bytes', '-'), item.get('last_modified', '-')[:22].replace( 'T', ' '), item.get('hash', '-'), item.get('content_type', '-'))) fp.write(item.get( 'name', item.get('subdir'))) fp.write('\n') fp.flush() if limit: break marker = contents[-1].get('name', contents[-1].get('subdir', '')) with context.client_manager.with_client() as client: status, reason, headers, contents = client.get_container( path, headers=context.headers, limit=limit, delimiter=delimiter, prefix=prefix, end_marker=end_marker, marker=marker, query=context.query, cdn=context.cdn) if status // 100 != 2: if status == 404 and context.ignore_404: return if hasattr(contents, 'read'): contents.read() raise ReturnCode( 'listing container %r: %s %s' % (path, status, reason)) conc.join() for (exc_type, exc_value, exc_tb, result) in \ six.itervalues(conc.get_results()): if exc_value: raise exc_value
def cli_fordo(context, path=None): """ Issues commands for each item in an account or container listing. See :py:mod:`swiftly.cli.fordo` for context usage information. See :py:class:`CLIForDo` for more information. """ path = path.lstrip('/') if path else None if path and '/' in path: raise ReturnCode( 'path must be an empty string or a container name; was %r' % path) limit = context.query.get('limit') delimiter = context.query.get('delimiter') prefix = context.query.get('prefix') marker = context.query.get('marker') end_marker = context.query.get('end_marker') conc = Concurrency(context.concurrency) while True: with context.client_manager.with_client() as client: if not path: status, reason, headers, contents = client.get_account( headers=context.headers, prefix=prefix, delimiter=delimiter, marker=marker, end_marker=end_marker, limit=limit, query=context.query, cdn=context.cdn) else: status, reason, headers, contents = client.get_container( path, headers=context.headers, prefix=prefix, delimiter=delimiter, marker=marker, end_marker=end_marker, limit=limit, query=context.query, cdn=context.cdn) if status // 100 != 2: if status == 404 and context.ignore_404: return if hasattr(contents, 'read'): contents.read() if not path: raise ReturnCode( 'listing account: %s %s' % (status, reason)) else: raise ReturnCode( 'listing container %r: %s %s' % (path, status, reason)) if not contents: break for item in contents: name = (path + '/' if path else '') + item.get( 'name', item.get('subdir')) args = list(context.remaining_args) try: index = args.index('<item>') except ValueError: raise ReturnCode( 'No "<item>" designation found in the "do" clause.') args[index] = name for (exc_type, exc_value, exc_tb, result) in \ six.itervalues(conc.get_results()): if exc_value: conc.join() raise exc_value conc.spawn(name, _cli_call, context, name, args) marker = contents[-1]['name'] if limit: break conc.join() for (exc_type, exc_value, exc_tb, result) in \ six.itervalues(conc.get_results()): if exc_value: conc.join() raise exc_value
def cli_put_object(context, path): """ Performs a PUT on the object. See :py:mod:`swiftly.cli.put` for context usage information. See :py:class:`CLIPut` for more information. """ if context.different and context.encrypt: raise ReturnCode( 'context.different will not work properly with context.encrypt ' 'since encryption may change the object size') put_headers = dict(context.headers) if context.empty: body = '' put_headers['content-length'] = '0' elif not context.input_ or context.input_ == '-': body = context.io_manager.get_stdin() elif context.seek is not None: if context.encrypt: raise ReturnCode( 'putting object %r: Cannot use encryption and context.seek' % path) body = open(context.input_, 'rb') body.seek(context.seek) else: l_mtime = os.path.getmtime(context.input_) l_size = os.path.getsize(context.input_) put_headers['content-length'] = str(l_size) if context.newer or context.different: r_mtime = None r_size = None with context.client_manager.with_client() as client: status, reason, headers, contents = client.head_object( *path.split('/', 1), headers=context.headers, query=context.query, cdn=context.cdn) if hasattr(contents, 'read'): contents.read() if status // 100 == 2: r_mtime = headers.get('x-object-meta-mtime') if r_mtime: try: r_mtime = float(r_mtime) except ValueError: r_mtime = None r_size = headers.get('content-length') if r_size: try: r_size = int(r_size) except ValueError: r_size = None elif status != 404: raise ReturnCode( 'could not head %r for conditional check; skipping put: ' '%s %s' % (path, status, reason)) if context.newer and r_mtime is not None or l_mtime <= r_mtime: return if context.different and r_mtime is not None and \ l_mtime == r_mtime and r_size is not None and \ l_size == r_size: return put_headers['x-object-meta-mtime'] = '%f' % l_mtime size = os.path.getsize(context.input_) if size > context.segment_size: if context.encrypt: raise ReturnCode( 'putting object %r: Cannot use encryption for objects ' 'greater than the segment size' % path) new_context = context.copy() new_context.input_ = None new_context.headers = None new_context.query = None container = path.split('/', 1)[0] + '_segments' cli_put_container(new_context, container) prefix = container + '/' + path.split('/', 1)[1] prefix = '%s/%s/%s/' % (prefix, l_mtime, size) conc = Concurrency(context.concurrency) start = 0 segment = 0 path2info = {} while start < size: new_context = context.copy() new_context.headers = dict(context.headers) new_context.headers['content-length'] = str( min(size - start, context.segment_size)) new_context.seek = start new_path = '%s%08d' % (prefix, segment) for (ident, (exc_type, exc_value, exc_tb, result)) in \ conc.get_results().iteritems(): if exc_value: conc.join() raise exc_value path2info[ident] = result conc.spawn(new_path, cli_put_object, new_context, new_path) segment += 1 start += context.segment_size conc.join() for (ident, (exc_type, exc_value, exc_tb, result)) in \ conc.get_results().iteritems(): if exc_value: raise exc_value path2info[ident] = result if context.static_segments: body = json.dumps([{ 'path': '/' + p, 'size_bytes': s, 'etag': e } for p, (s, e) in sorted(path2info.iteritems())]) put_headers['content-length'] = str(len(body)) context.query['multipart-manifest'] = 'put' else: body = '' put_headers['content-length'] = '0' put_headers['x-object-manifest'] = prefix else: body = open(context.input_, 'rb') with context.client_manager.with_client() as client: if context.encrypt: content_length = put_headers.get('content-length') if content_length: content_length = int(content_length) if hasattr(body, 'read'): body = FileLikeIter( aes_encrypt(context.encrypt, body, preamble=AES256CBC, chunk_size=getattr(client, 'chunk_size', 65536), content_length=content_length)) else: body = FileLikeIter( aes_encrypt(context.encrypt, FileLikeIter([body]), preamble=AES256CBC, chunk_size=getattr(client, 'chunk_size', 65536), content_length=content_length)) if 'content-length' in put_headers: del put_headers['content-length'] container, obj = path.split('/', 1) status, reason, headers, contents = client.put_object( container, obj, body, headers=put_headers, query=context.query, cdn=context.cdn) if hasattr(contents, 'read'): contents = contents.read() if status // 100 != 2: raise ReturnCode('putting object %r: %s %s %r' % (path, status, reason, contents)) if context.seek is not None: content_length = put_headers.get('content-length') etag = headers.get('etag') if content_length and etag: content_length = int(content_length) else: with context.client_manager.with_client() as client: container, obj = path.split('/', 1) status, reason, headers, contents = client.head_object( container, obj, cdn=context.cdn) if hasattr(contents, 'read'): contents = contents.read() if status // 100 != 2: raise ReturnCode('heading object %r: %s %s %r' % (path, status, reason, contents)) content_length = headers.get('content-length') etag = headers.get('etag') if content_length: content_length = int(content_length) return content_length, etag
def put_stream(self, key, stream): (container, obj) = key.split('/', limit=1) size = None func_size = getattr(stream, 'size', None) if func_size: size = func_size() if not size: return self.client.client.put_object(container, obj, stream) if size <= self.segment_size: headers = {'Content-Length': size} (status, reason, headers, contents) = \ self.client.put_object(container, obj, stream, headers=headers) if status != 201: self.logger.error('Failed to put object: %s/%s' % (container, obj)) return False cont_prefix = '%s_segments' % container prefix = '%s/%s/' % (obj, self.segment_size) conc = Concurrency(self.concurrency) start = 0 segment = 0 while start < size: obj_path = '%s%08d' % (prefix, segment) conc.spawn( segment, self._put_recursive, cont_prefix, obj_path, stream.copy(), start, ) for rv in conc.get_results().values(): if not rv: conc.join() self.logger.error( 'Failed to create segments for object: %s/%s' % (container, obj)) return rv segment += 1 start += self.client.segment_size conc.join() for rv in conc.get_results().values(): if not rv: conc.join() self.logger.error( 'Failed to create segments for object: %s/%s' % (container, obj)) return rv headers = {'Content-Length': 0, 'X-Object-Manifest': '%s/%s' \ % (cont_prefix, prefix)} (status, reason, headers, contents) = \ self.client.put_object(container, obj, '', headers=headers) if status != 201: self.logger.error('Failed put manifest object: %s/%s' % (container, obj)) return False return True
def put_stream(self, key, stream): (container, obj) = key.split('/', limit=1) size = None func_size = getattr(stream, 'size', None) if func_size: size = func_size() if not size: return self.client.client.put_object(container, obj, stream) if size <= self.segment_size: headers = {'Content-Length': size} (status, reason, headers, contents) = \ self.client.put_object(container, obj, stream, headers=headers) if status != 201: self.logger.error('Failed to put object: %s/%s' % (container, obj)) return False cont_prefix = '%s_segments' % container prefix = '%s/%s/' % (obj, self.segment_size) conc = Concurrency(self.concurrency) start = 0 segment = 0 while start < size: obj_path = '%s%08d' % (prefix, segment) conc.spawn( segment, self._put_recursive, cont_prefix, obj_path, stream.copy(), start, ) for rv in conc.get_results().values(): if not rv: conc.join() self.logger.error('Failed to create segments for object: %s/%s' % (container, obj)) return rv segment += 1 start += self.client.segment_size conc.join() for rv in conc.get_results().values(): if not rv: conc.join() self.logger.error('Failed to create segments for object: %s/%s' % (container, obj)) return rv headers = {'Content-Length': 0, 'X-Object-Manifest': '%s/%s' \ % (cont_prefix, prefix)} (status, reason, headers, contents) = \ self.client.put_object(container, obj, '', headers=headers) if status != 201: self.logger.error('Failed put manifest object: %s/%s' % (container, obj)) return False return True