def cli_put_object(context, path): """ Performs a PUT on the object. See :py:mod:`swiftly.cli.put` for context usage information. See :py:class:`CLIPut` for more information. """ if context.different and context.encrypt: raise ReturnCode( 'context.different will not work properly with context.encrypt ' 'since encryption may change the object size') put_headers = dict(context.headers) if context.empty: body = '' put_headers['content-length'] = '0' elif not context.input_ or context.input_ == '-': stdin = context.io_manager.get_stdin() if context.stdin_segmentation: def reader(): while True: chunk = stdin.read(65536) if chunk: yield chunk else: return segment_body = FileLikeIter(reader(), context.segment_size) prefix = _create_container(context, path, time.time(), 0) new_context = context.copy() new_context.stdin_segmentation = False new_context.stdin = segment_body new_context.headers = dict(context.headers) segment_n = 0 path2info = {} while not segment_body.is_empty(): segment_path = _get_segment_path(prefix, segment_n) etag = cli_put_object(new_context, segment_path) size = segment_body.limit - segment_body.left path2info[segment_path] = (size, etag) segment_body.reset_limit() segment_n += 1 body = _get_manifest_body(context, prefix, path2info, put_headers) else: if hasattr(context, 'stdin'): body = context.stdin else: body = stdin elif context.seek is not None: if context.encrypt: raise ReturnCode( 'putting object %r: Cannot use encryption and context.seek' % path) body = open(context.input_, 'rb') body.seek(context.seek) else: l_mtime = os.path.getmtime(context.input_) l_size = os.path.getsize(context.input_) put_headers['content-length'] = str(l_size) if context.newer or context.different: r_mtime = None r_size = None with context.client_manager.with_client() as client: status, reason, headers, contents = client.head_object( *path.split('/', 1), headers=context.headers, query=context.query, cdn=context.cdn) if hasattr(contents, 'read'): contents.read() if status // 100 == 2: r_mtime = headers.get('x-object-meta-mtime') if r_mtime: try: r_mtime = float(r_mtime) except ValueError: r_mtime = None r_size = headers.get('content-length') if r_size: try: r_size = int(r_size) except ValueError: r_size = None elif status != 404: raise ReturnCode( 'could not head %r for conditional check; skipping put: ' '%s %s' % (path, status, reason)) if context.newer and r_mtime is not None or l_mtime <= r_mtime: return if context.different and r_mtime is not None and \ l_mtime == r_mtime and r_size is not None and \ l_size == r_size: return put_headers['x-object-meta-mtime'] = '%f' % l_mtime size = os.path.getsize(context.input_) if size > context.segment_size: if context.encrypt: raise ReturnCode( 'putting object %r: Cannot use encryption for objects ' 'greater than the segment size' % path) prefix = _create_container(context, path, l_mtime, size) conc = Concurrency(context.concurrency) start = 0 segment = 0 path2info = {} while start < size: new_context = context.copy() new_context.headers = dict(context.headers) new_context.headers['content-length'] = str( min(size - start, context.segment_size)) new_context.seek = start new_path = _get_segment_path(prefix, segment) for (ident, (exc_type, exc_value, exc_tb, result)) in \ six.iteritems(conc.get_results()): if exc_value: conc.join() raise exc_value path2info[ident] = result conc.spawn(new_path, cli_put_object, new_context, new_path) segment += 1 start += context.segment_size conc.join() for (ident, (exc_type, exc_value, exc_tb, result)) in \ six.iteritems(conc.get_results()): if exc_value: raise exc_value path2info[ident] = result body = _get_manifest_body(context, prefix, path2info, put_headers) else: body = open(context.input_, 'rb') with context.client_manager.with_client() as client: if context.encrypt: content_length = put_headers.get('content-length') if content_length: content_length = int(content_length) if hasattr(body, 'read'): body = FileLikeIter( aes_encrypt(context.encrypt, body, preamble=AES256CBC, chunk_size=getattr(client, 'chunk_size', 65536), content_length=content_length)) else: body = FileLikeIter( aes_encrypt(context.encrypt, FileLikeIter([body]), preamble=AES256CBC, chunk_size=getattr(client, 'chunk_size', 65536), content_length=content_length)) if 'content-length' in put_headers: del put_headers['content-length'] container, obj = path.split('/', 1) status, reason, headers, contents = client.put_object( container, obj, body, headers=put_headers, query=context.query, cdn=context.cdn) if hasattr(contents, 'read'): contents = contents.read() if status // 100 != 2: raise ReturnCode('putting object %r: %s %s %r' % (path, status, reason, contents)) if context.seek is not None: content_length = put_headers.get('content-length') etag = headers.get('etag') if content_length and etag: content_length = int(content_length) else: with context.client_manager.with_client() as client: container, obj = path.split('/', 1) status, reason, headers, contents = client.head_object( container, obj, cdn=context.cdn) if hasattr(contents, 'read'): contents = contents.read() if status // 100 != 2: raise ReturnCode('heading object %r: %s %s %r' % (path, status, reason, contents)) content_length = headers.get('content-length') etag = headers.get('etag') if content_length: content_length = int(content_length) return content_length, etag if context.stdin is not None: return headers.get('etag')
def cli_get_account_listing(context): """ Performs a GET on the account as a listing request. See :py:mod:`swiftly.cli.get` for context usage information. See :py:class:`CLIGet` for more information. """ limit = context.query.get('limit') delimiter = context.query.get('delimiter') prefix = context.query.get('prefix') marker = context.query.get('marker') end_marker = context.query.get('end_marker') if context.raw: with context.client_manager.with_client() as client: status, reason, headers, contents = client.get_account( decode_json=False, headers=context.headers, limit=limit, marker=marker, end_marker=end_marker, query=context.query, cdn=context.cdn) if hasattr(contents, 'read'): contents = contents.read() if status // 100 != 2: if status == 404 and context.ignore_404: return raise ReturnCode('listing account: %s %s' % (status, reason)) with context.io_manager.with_stdout() as fp: if context.output_headers: context.write_headers( fp, headers, context.muted_account_headers) fp.write(contents) fp.flush() return with context.client_manager.with_client() as client: status, reason, headers, contents = client.get_account( headers=context.headers, limit=limit, marker=marker, end_marker=end_marker, query=context.query, cdn=context.cdn) if status // 100 != 2: if status == 404 and context.ignore_404: return if hasattr(contents, 'read'): contents.read() raise ReturnCode('listing account: %s %s' % (status, reason)) if context.output_headers and not context.all_objects: with context.io_manager.with_stdout() as fp: context.write_headers( fp, headers, context.muted_account_headers) while contents: if context.all_objects: new_context = context.copy() new_context.query = dict(new_context.query) for remove in ( 'limit', 'delimiter', 'prefix', 'marker', 'end_marker'): if remove in new_context.query: del new_context.query[remove] for item in contents: if 'name' in item: new_path = item['name'].encode('utf8') cli_get_container_listing(new_context, new_path) else: with context.io_manager.with_stdout() as fp: for item in contents: if context.full: fp.write('%13s %13s ' % ( item.get('bytes', '-'), item.get('count', '-'))) fp.write(item.get( 'name', item.get('subdir'))) fp.write('\n') fp.flush() if limit: break marker = contents[-1].get('name', contents[-1].get('subdir', '')) with context.client_manager.with_client() as client: status, reason, headers, contents = client.get_account( headers=context.headers, limit=limit, delimiter=delimiter, prefix=prefix, end_marker=end_marker, marker=marker, query=context.query, cdn=context.cdn) if status // 100 != 2: if status == 404 and context.ignore_404: return if hasattr(contents, 'read'): contents.read() raise ReturnCode('listing account: %s %s' % (status, reason))
def cli_delete(context, path, body=None, recursive=False, yes_empty_account=False, yes_delete_account=False, until_empty=False): """ Deletes the item (account, container, or object) at the path. See :py:mod:`swiftly.cli.delete` for context usage information. See :py:class:`CLIDelete` for more information. :param context: The :py:class:`swiftly.cli.context.CLIContext` to use. :param path: The path of the item (acount, container, or object) to delete. :param body: The body to send with the DELETE request. Bodies are not normally sent with DELETE requests, but this can be useful with bulk deletes for instance. :param recursive: If True and the item is an account or container, deletes will be issued for any containing items as well. This does one pass at the deletion; so if objects revert to previous versions or if new objects otherwise arise during the process, the container(s) may not be empty once done. Set `until_empty` to True if you want multiple passes to keep trying to fully empty the containers. :param until_empty: If True and recursive is True, this will cause Swiftly to keep looping through the deletes until the containers are completely empty. Useful if you have object versioning turned on or otherwise have objects that seemingly reappear after being deleted. It could also run forever if you have something that's uploading objects at a faster rate than they are deleted. :param yes_empty_account: This must be set to True for verification when the item is an account and recursive is True. :param yes_delete_account: This must be set to True for verification when the item is an account and you really wish a delete to be issued for the account itself. """ path = path.lstrip('/') if path else '' if not path: if yes_empty_account: cli_empty_account(context, yes_empty_account=yes_empty_account, until_empty=until_empty) if yes_delete_account: with context.client_manager.with_client() as client: status, reason, headers, contents = client.delete_account( headers=context.headers, query=context.query, cdn=context.cdn, body=body, yes_i_mean_delete_the_account=yes_delete_account) if status // 100 != 2: if status == 404 and context.ignore_404: return raise ReturnCode('deleting account: %s %s' % (status, reason)) elif '/' not in path.rstrip('/'): path = path.rstrip('/') if recursive: cli_empty_container(context, path, until_empty=until_empty) with context.client_manager.with_client() as client: status, reason, headers, contents = client.delete_container( path, headers=context.headers, query=context.query, cdn=context.cdn, body=body) if status // 100 != 2: if status == 404 and context.ignore_404: return raise ReturnCode('deleting container %r: %s %s' % (path, status, reason)) else: with context.client_manager.with_client() as client: status, reason, headers, contents = client.delete_object( *path.split('/', 1), headers=context.headers, query=context.query, cdn=context.cdn, body=body) if status // 100 != 2: if status == 404 and context.ignore_404: return raise ReturnCode('deleting object %r: %s %s' % (path, status, reason))
def cli_get(context, path=None): """ Performs a GET on the item (account, container, or object). See :py:mod:`swiftly.cli.get` for context usage information. See :py:class:`CLIGet` for more information. """ path = path.lstrip('/') if path else None if not path: return cli_get_account_listing(context) elif '/' not in path.rstrip('/'): return cli_get_container_listing(context, path) status, reason, headers, contents = 0, 'Unknown', {}, '' with context.client_manager.with_client() as client: status, reason, headers, contents = client.get_object( *path.split('/', 1), headers=context.headers, query=context.query, cdn=context.cdn) if status // 100 != 2: if status == 404 and context.ignore_404: return if hasattr(contents, 'read'): contents.read() raise ReturnCode( 'getting object %r: %s %s' % (path, status, reason)) if context.decrypt: crypt_type = contents.read(1) if crypt_type == AES256CBC: contents = FileLikeIter(aes_decrypt( context.decrypt, contents, chunk_size=getattr(client, 'chunk_size', 65536))) else: raise ReturnCode( 'getting object %r: contents encrypted with unsupported ' 'type %r' % (path, crypt_type)) def disk_closed_callback(disk_path): if context.remove_empty_files and not os.path.getsize(disk_path): os.unlink(disk_path) if context.io_manager.stdout_root: dirname = os.path.dirname(disk_path) while dirname and dirname.startswith( context.io_manager.stdout_root): try: os.rmdir(dirname) except OSError: pass dirname = os.path.dirname(dirname) return if (headers.get('content-type') in ['text/directory', 'application/directory'] and headers.get('content-length') == '0'): os.unlink(disk_path) os.makedirs(disk_path) mtime = 0 if 'x-object-meta-mtime' in headers: mtime = float(headers['x-object-meta-mtime']) elif 'last-modified' in headers: mtime = time.mktime(time.strptime( headers['last-modified'], '%a, %d %b %Y %H:%M:%S %Z')) if mtime: os.utime(disk_path, (mtime, mtime)) out_path = path if context.suppress_container_name: out_path = out_path.split('/', 1)[1] out_path = context.io_manager.client_path_to_os_path(out_path) with context.io_manager.with_stdout( out_path, disk_closed_callback=disk_closed_callback) as fp: if context.output_headers: context.write_headers( fp, headers, context.muted_object_headers) fp.write('\n') chunk = contents.read(65536) while chunk: fp.write(chunk) chunk = contents.read(65536) fp.flush()
def cli_get_container_listing(context, path=None): """ Performs a GET on the container as a listing request. See :py:mod:`swiftly.cli.get` for context usage information. See :py:class:`CLIGet` for more information. """ path = path.strip('/') if path else None if not path or '/' in path: raise ReturnCode( 'tried to get a container listing for non-container path %r' % path) context.suppress_container_name = True limit = context.query.get('limit') delimiter = context.query.get('delimiter') prefix = context.query.get('prefix') marker = context.query.get('marker') end_marker = context.query.get('end_marker') if context.raw: with context.client_manager.with_client() as client: status, reason, headers, contents = client.get_container( path, decode_json=False, headers=context.headers, limit=limit, marker=marker, end_marker=end_marker, query=context.query, cdn=context.cdn) if hasattr(contents, 'read'): contents = contents.read() if status // 100 != 2: if status == 404 and context.ignore_404: return raise ReturnCode( 'listing container %r: %s %s' % (path, status, reason)) with context.io_manager.with_stdout() as fp: if context.output_headers: context.write_headers( fp, headers, context.muted_container_headers) fp.write(contents) fp.flush() return with context.client_manager.with_client() as client: status, reason, headers, contents = client.get_container( path, headers=context.headers, limit=limit, delimiter=delimiter, prefix=prefix, marker=marker, end_marker=end_marker, query=context.query, cdn=context.cdn) if status // 100 != 2: if status == 404 and context.ignore_404: return if hasattr(contents, 'read'): contents.read() raise ReturnCode( 'listing container %r: %s %s' % (path, status, reason)) if context.output_headers and not context.all_objects: with context.io_manager.with_stdout() as fp: context.write_headers( fp, headers, context.muted_container_headers) conc = Concurrency(context.concurrency) while contents: if context.all_objects: new_context = context.copy() new_context.query = dict(new_context.query) for remove in ( 'limit', 'delimiter', 'prefix', 'marker', 'end_marker'): if remove in new_context.query: del new_context.query[remove] for item in contents: if 'name' in item: for (exc_type, exc_value, exc_tb, result) in \ six.itervalues(conc.get_results()): if exc_value: conc.join() raise exc_value new_path = path + '/' + item['name'].encode('utf8') conc.spawn(new_path, cli_get, new_context, new_path) else: with context.io_manager.with_stdout() as fp: for item in contents: if context.full: fp.write('%13s %22s %32s %25s ' % ( item.get('bytes', '-'), item.get('last_modified', '-')[:22].replace( 'T', ' '), item.get('hash', '-'), item.get('content_type', '-'))) fp.write(item.get( 'name', item.get('subdir'))) fp.write('\n') fp.flush() if limit: break marker = contents[-1].get('name', contents[-1].get('subdir', '')) with context.client_manager.with_client() as client: status, reason, headers, contents = client.get_container( path, headers=context.headers, limit=limit, delimiter=delimiter, prefix=prefix, end_marker=end_marker, marker=marker, query=context.query, cdn=context.cdn) if status // 100 != 2: if status == 404 and context.ignore_404: return if hasattr(contents, 'read'): contents.read() raise ReturnCode( 'listing container %r: %s %s' % (path, status, reason)) conc.join() for (exc_type, exc_value, exc_tb, result) in \ six.itervalues(conc.get_results()): if exc_value: raise exc_value
def cli_fordo(context, path=None): """ Issues commands for each item in an account or container listing. See :py:mod:`swiftly.cli.fordo` for context usage information. See :py:class:`CLIForDo` for more information. """ path = path.lstrip('/') if path else None if path and '/' in path: raise ReturnCode( 'path must be an empty string or a container name; was %r' % path) limit = context.query.get('limit') delimiter = context.query.get('delimiter') prefix = context.query.get('prefix') marker = context.query.get('marker') end_marker = context.query.get('end_marker') conc = Concurrency(context.concurrency) while True: with context.client_manager.with_client() as client: if not path: status, reason, headers, contents = client.get_account( headers=context.headers, prefix=prefix, delimiter=delimiter, marker=marker, end_marker=end_marker, limit=limit, query=context.query, cdn=context.cdn) else: status, reason, headers, contents = client.get_container( path, headers=context.headers, prefix=prefix, delimiter=delimiter, marker=marker, end_marker=end_marker, limit=limit, query=context.query, cdn=context.cdn) if status // 100 != 2: if status == 404 and context.ignore_404: return if hasattr(contents, 'read'): contents.read() if not path: raise ReturnCode( 'listing account: %s %s' % (status, reason)) else: raise ReturnCode( 'listing container %r: %s %s' % (path, status, reason)) if not contents: break for item in contents: name = (path + '/' if path else '') + item.get( 'name', item.get('subdir')) args = list(context.remaining_args) try: index = args.index('<item>') except ValueError: raise ReturnCode( 'No "<item>" designation found in the "do" clause.') args[index] = name for (exc_type, exc_value, exc_tb, result) in \ six.itervalues(conc.get_results()): if exc_value: conc.join() raise exc_value conc.spawn(name, _cli_call, context, name, args) marker = contents[-1]['name'] if limit: break conc.join() for (exc_type, exc_value, exc_tb, result) in \ six.itervalues(conc.get_results()): if exc_value: conc.join() raise exc_value
def cli_put_directory_structure(context, path): """ Performs PUTs rooted at the path using a directory structure pointed to by context.input\_. See :py:mod:`swiftly.cli.put` for context usage information. See :py:class:`CLIPut` for more information. """ if not context.input_: raise ReturnCode( 'called cli_put_directory_structure without context.input_ set') if not os.path.isdir(context.input_): raise ReturnCode('%r is not a directory' % context.input_) if not path: raise ReturnCode( 'uploading a directory structure requires at least a container ' 'name') new_context = context.copy() new_context.input_ = None container = path.split('/', 1)[0] cli_put_container(new_context, container) ilen = len(context.input_) if not context.input_.endswith(os.sep): ilen += 1 conc = Concurrency(context.concurrency) for (dirpath, dirnames, filenames) in os.walk(context.input_): if not dirnames and not filenames: new_context = context.copy() new_context.headers = dict(context.headers) new_context.headers['content-type'] = 'text/directory' new_context.headers['x-object-meta-mtime'] = \ '%f' % os.path.getmtime(context.input_) new_context.input_ = None new_context.empty = True new_path = path if path[-1] != '/': new_path += '/' new_path += dirpath[ilen:] for (exc_type, exc_value, exc_tb, result) in \ conc.get_results().itervalues(): if exc_value: conc.join() raise exc_value conc.spawn(new_path, cli_put_object, new_context, new_path) else: for fname in filenames: new_context = context.copy() new_context.input_ = os.path.join(dirpath, fname) new_path = path if path[-1] != '/': new_path += '/' if dirpath[ilen:]: new_path += dirpath[ilen:] + '/' new_path += fname for (exc_type, exc_value, exc_tb, result) in \ conc.get_results().itervalues(): if exc_value: conc.join() raise exc_value conc.spawn(new_path, cli_put_object, new_context, new_path) conc.join() for (exc_type, exc_value, exc_tb, result) in \ conc.get_results().itervalues(): if exc_value: raise exc_value
def cli_put_object(context, path): """ Performs a PUT on the object. See :py:mod:`swiftly.cli.put` for context usage information. See :py:class:`CLIPut` for more information. """ if context.different and context.encrypt: raise ReturnCode( 'context.different will not work properly with context.encrypt ' 'since encryption may change the object size') put_headers = dict(context.headers) if context.empty: body = '' put_headers['content-length'] = '0' elif not context.input_ or context.input_ == '-': body = context.io_manager.get_stdin() elif context.seek is not None: if context.encrypt: raise ReturnCode( 'putting object %r: Cannot use encryption and context.seek' % path) body = open(context.input_, 'rb') body.seek(context.seek) else: l_mtime = os.path.getmtime(context.input_) l_size = os.path.getsize(context.input_) put_headers['content-length'] = str(l_size) if context.newer or context.different: r_mtime = None r_size = None with context.client_manager.with_client() as client: status, reason, headers, contents = client.head_object( *path.split('/', 1), headers=context.headers, query=context.query, cdn=context.cdn) if hasattr(contents, 'read'): contents.read() if status // 100 == 2: r_mtime = headers.get('x-object-meta-mtime') if r_mtime: try: r_mtime = float(r_mtime) except ValueError: r_mtime = None r_size = headers.get('content-length') if r_size: try: r_size = int(r_size) except ValueError: r_size = None elif status != 404: raise ReturnCode( 'could not head %r for conditional check; skipping put: ' '%s %s' % (path, status, reason)) if context.newer and r_mtime is not None or l_mtime <= r_mtime: return if context.different and r_mtime is not None and \ l_mtime == r_mtime and r_size is not None and \ l_size == r_size: return put_headers['x-object-meta-mtime'] = '%f' % l_mtime size = os.path.getsize(context.input_) if size > context.segment_size: if context.encrypt: raise ReturnCode( 'putting object %r: Cannot use encryption for objects ' 'greater than the segment size' % path) new_context = context.copy() new_context.input_ = None new_context.headers = None new_context.query = None container = path.split('/', 1)[0] + '_segments' cli_put_container(new_context, container) prefix = container + '/' + path.split('/', 1)[1] prefix = '%s/%s/%s/' % (prefix, l_mtime, size) conc = Concurrency(context.concurrency) start = 0 segment = 0 path2info = {} while start < size: new_context = context.copy() new_context.headers = dict(context.headers) new_context.headers['content-length'] = str( min(size - start, context.segment_size)) new_context.seek = start new_path = '%s%08d' % (prefix, segment) for (ident, (exc_type, exc_value, exc_tb, result)) in \ conc.get_results().iteritems(): if exc_value: conc.join() raise exc_value path2info[ident] = result conc.spawn(new_path, cli_put_object, new_context, new_path) segment += 1 start += context.segment_size conc.join() for (ident, (exc_type, exc_value, exc_tb, result)) in \ conc.get_results().iteritems(): if exc_value: raise exc_value path2info[ident] = result if context.static_segments: body = json.dumps([{ 'path': '/' + p, 'size_bytes': s, 'etag': e } for p, (s, e) in sorted(path2info.iteritems())]) put_headers['content-length'] = str(len(body)) context.query['multipart-manifest'] = 'put' else: body = '' put_headers['content-length'] = '0' put_headers['x-object-manifest'] = prefix else: body = open(context.input_, 'rb') with context.client_manager.with_client() as client: if context.encrypt: content_length = put_headers.get('content-length') if content_length: content_length = int(content_length) if hasattr(body, 'read'): body = FileLikeIter( aes_encrypt(context.encrypt, body, preamble=AES256CBC, chunk_size=getattr(client, 'chunk_size', 65536), content_length=content_length)) else: body = FileLikeIter( aes_encrypt(context.encrypt, FileLikeIter([body]), preamble=AES256CBC, chunk_size=getattr(client, 'chunk_size', 65536), content_length=content_length)) if 'content-length' in put_headers: del put_headers['content-length'] container, obj = path.split('/', 1) status, reason, headers, contents = client.put_object( container, obj, body, headers=put_headers, query=context.query, cdn=context.cdn) if hasattr(contents, 'read'): contents = contents.read() if status // 100 != 2: raise ReturnCode('putting object %r: %s %s %r' % (path, status, reason, contents)) if context.seek is not None: content_length = put_headers.get('content-length') etag = headers.get('etag') if content_length and etag: content_length = int(content_length) else: with context.client_manager.with_client() as client: container, obj = path.split('/', 1) status, reason, headers, contents = client.head_object( container, obj, cdn=context.cdn) if hasattr(contents, 'read'): contents = contents.read() if status // 100 != 2: raise ReturnCode('heading object %r: %s %s %r' % (path, status, reason, contents)) content_length = headers.get('content-length') etag = headers.get('etag') if content_length: content_length = int(content_length) return content_length, etag