def test_config_true_value(self):
orig_trues = u.TRUE_VALUES
try:
u.TRUE_VALUES = 'hello world'.split()
for val in 'hello world HELLO WORLD'.split():
self.assertTrue(u.config_true_value(val) is True)
self.assertTrue(u.config_true_value(True) is True)
self.assertTrue(u.config_true_value('foo') is False)
self.assertTrue(u.config_true_value(False) is False)
finally:
u.TRUE_VALUES = orig_trues
def test_config_true_value(self):
orig_trues = u.TRUE_VALUES
try:
u.TRUE_VALUES = 'hello world'.split()
for val in 'hello world HELLO WORLD'.split():
self.assertTrue(u.config_true_value(val) is True)
self.assertTrue(u.config_true_value(True) is True)
self.assertTrue(u.config_true_value('foo') is False)
self.assertTrue(u.config_true_value(False) is False)
finally:
u.TRUE_VALUES = orig_trues
def main(arguments=None):
if arguments:
argv = arguments
else:
argv = sys_argv
version = client_version
parser = OptionParser(version='%%prog %s' % version,
usage='''
usage: %%prog [--version] [--help] [--snet] [--verbose]
[--debug] [--info] [--quiet] [--auth <auth_url>]
[--auth-version <auth_version>] [--user <username>]
[--key <api_key>] [--retries <num_retries>]
[--os-username <auth-user-name>] [--os-password <auth-password>]
[--os-tenant-id <auth-tenant-id>]
[--os-tenant-name <auth-tenant-name>]
[--os-auth-url <auth-url>] [--os-auth-token <auth-token>]
[--os-storage-url <storage-url>] [--os-region-name <region-name>]
[--os-service-type <service-type>]
[--os-endpoint-type <endpoint-type>]
[--os-cacert <ca-certificate>] [--insecure]
[--no-ssl-compression]
<subcommand> ...
Command-line interface to the OpenStack Swift API.
Positional arguments:
<subcommand>
delete Delete a container or objects within a container.
download Download objects from containers.
list Lists the containers for the account or the objects
for a container.
post Updates meta information for the account, container,
or object; creates containers if not present.
stat Displays information for the account, container,
or object.
upload Uploads files or directories to the given container
capabilities List cluster capabilities.
exec Execute ZeroCloud job
Examples:
%%prog -A https://auth.api.rackspacecloud.com/v1.0 -U user -K api_key stat -v
%%prog --os-auth-url https://api.example.com/v2.0 --os-tenant-name tenant \\
--os-username user --os-password password list
%%prog --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\
--os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \\
list
%%prog list --lh
'''.strip('\n') % globals())
parser.add_option('-s', '--snet', action='store_true', dest='snet',
default=False, help='Use SERVICENET internal network.')
parser.add_option('-v', '--verbose', action='count', dest='verbose',
default=1, help='Print more info.')
parser.add_option('--debug', action='store_true', dest='debug',
default=False, help='Show the curl commands and results '
'of all http queries regardless of result status.')
parser.add_option('--info', action='store_true', dest='info',
default=False, help='Show the curl commands and results '
' of all http queries which return an error.')
parser.add_option('-q', '--quiet', action='store_const', dest='verbose',
const=0, default=1, help='Suppress status output.')
parser.add_option('-A', '--auth', dest='auth',
default=environ.get('ST_AUTH'),
help='URL for obtaining an auth token.')
parser.add_option('-V', '--auth-version',
dest='auth_version',
default=environ.get('ST_AUTH_VERSION', '1.0'),
type=str,
help='Specify a version for authentication. '
'Defaults to 1.0.')
parser.add_option('-U', '--user', dest='user',
default=environ.get('ST_USER'),
help='User name for obtaining an auth token.')
parser.add_option('-K', '--key', dest='key',
default=environ.get('ST_KEY'),
help='Key for obtaining an auth token.')
parser.add_option('-R', '--retries', type=int, default=5, dest='retries',
help='The number of times to retry a failed connection.')
parser.add_option('--os-username',
metavar='<auth-user-name>',
default=environ.get('OS_USERNAME'),
help='OpenStack username. Defaults to env[OS_USERNAME].')
parser.add_option('--os_username',
help=SUPPRESS_HELP)
parser.add_option('--os-password',
metavar='<auth-password>',
default=environ.get('OS_PASSWORD'),
help='OpenStack password. Defaults to env[OS_PASSWORD].')
parser.add_option('--os_password',
help=SUPPRESS_HELP)
parser.add_option('--os-tenant-id',
metavar='<auth-tenant-id>',
default=environ.get('OS_TENANT_ID'),
help='OpenStack tenant ID. '
'Defaults to env[OS_TENANT_ID].')
parser.add_option('--os_tenant_id',
help=SUPPRESS_HELP)
parser.add_option('--os-tenant-name',
metavar='<auth-tenant-name>',
default=environ.get('OS_TENANT_NAME'),
help='OpenStack tenant name. '
'Defaults to env[OS_TENANT_NAME].')
parser.add_option('--os_tenant_name',
help=SUPPRESS_HELP)
parser.add_option('--os-auth-url',
metavar='<auth-url>',
default=environ.get('OS_AUTH_URL'),
help='OpenStack auth URL. Defaults to env[OS_AUTH_URL].')
parser.add_option('--os_auth_url',
help=SUPPRESS_HELP)
parser.add_option('--os-auth-token',
metavar='<auth-token>',
default=environ.get('OS_AUTH_TOKEN'),
help='OpenStack token. Defaults to env[OS_AUTH_TOKEN]. '
'Used with --os-storage-url to bypass the '
'usual username/password authentication.')
parser.add_option('--os_auth_token',
help=SUPPRESS_HELP)
parser.add_option('--os-storage-url',
metavar='<storage-url>',
default=environ.get('OS_STORAGE_URL'),
help='OpenStack storage URL. '
'Defaults to env[OS_STORAGE_URL]. '
'Overrides the storage url returned during auth. '
'Will bypass authentication when used with '
'--os-auth-token.')
parser.add_option('--os_storage_url',
help=SUPPRESS_HELP)
parser.add_option('--os-region-name',
metavar='<region-name>',
default=environ.get('OS_REGION_NAME'),
help='OpenStack region name. '
'Defaults to env[OS_REGION_NAME].')
parser.add_option('--os_region_name',
help=SUPPRESS_HELP)
parser.add_option('--os-service-type',
metavar='<service-type>',
default=environ.get('OS_SERVICE_TYPE'),
help='OpenStack Service type. '
'Defaults to env[OS_SERVICE_TYPE].')
parser.add_option('--os_service_type',
help=SUPPRESS_HELP)
parser.add_option('--os-endpoint-type',
metavar='<endpoint-type>',
default=environ.get('OS_ENDPOINT_TYPE'),
help='OpenStack Endpoint type. '
'Defaults to env[OS_ENDPOINT_TYPE].')
parser.add_option('--os-cacert',
metavar='<ca-certificate>',
default=environ.get('OS_CACERT'),
help='Specify a CA bundle file to use in verifying a '
'TLS (https) server certificate. '
'Defaults to env[OS_CACERT].')
default_val = config_true_value(environ.get('SWIFTCLIENT_INSECURE'))
parser.add_option('--insecure',
action="store_true", dest="insecure",
default=default_val,
help='Allow swiftclient to access servers without '
'having to verify the SSL certificate. '
'Defaults to env[SWIFTCLIENT_INSECURE] '
'(set to \'true\' to enable).')
parser.add_option('--no-ssl-compression',
action='store_false', dest='ssl_compression',
default=True,
help='This option is deprecated and not used anymore. '
'SSL compression should be disabled by default '
'by the system SSL library.')
parser.disable_interspersed_args()
(options, args) = parse_args(parser, argv[1:], enforce_requires=False)
parser.enable_interspersed_args()
commands = ('delete', 'download', 'list', 'post',
'stat', 'upload', 'capabilities', 'info', 'exec')
if not args or args[0] not in commands:
parser.print_usage()
if args:
exit('no such command: %s' % args[0])
exit()
signal.signal(signal.SIGINT, immediate_exit)
if options.debug or options.info:
logging.getLogger("zwiftclient")
if options.debug:
logging.basicConfig(level=logging.DEBUG)
elif options.info:
logging.basicConfig(level=logging.INFO)
had_error = False
with MultiThreadingManager() as thread_manager:
parser.usage = globals()['st_%s_help' % args[0]]
try:
globals()['st_%s' % args[0]](parser, argv[1:], thread_manager)
except (ClientException, RequestException, socket.error) as err:
thread_manager.error(str(err))
had_error = thread_manager.error_count
if had_error:
exit(1)
def test_config_true_value(self):
for val in 'hello world HELLO WORLD'.split():
self.assertIs(u.config_true_value(val), True)
self.assertIs(u.config_true_value(True), True)
self.assertIs(u.config_true_value('foo'), False)
self.assertIs(u.config_true_value(False), False)
def _upload_object_job(self,
conn,
container,
source,
obj,
options,
results_queue=None):
if obj.startswith('./') or obj.startswith('.\\'):
obj = obj[2:]
if obj.startswith('/'):
obj = obj[1:]
res = {'action': 'upload_object', 'container': container, 'object': obj}
if hasattr(source, 'read'):
stream = source
path = None
else:
path = source
res['path'] = path
try:
if path is not None:
put_headers = {'x-object-meta-mtime': "%f" % getmtime(path)}
else:
put_headers = {'x-object-meta-mtime': "%f" % round(time())}
res['headers'] = put_headers
# We need to HEAD all objects now in case we're overwriting a
# manifest object and need to delete the old segments
# ourselves.
old_manifest = None
old_slo_manifest_paths = []
new_slo_manifest_paths = set()
segment_size = int(
0 if options['segment_size'] is None else options['segment_size'])
if (options['changed'] or options['skip_identical']
or not options['leave_segments']):
try:
headers = conn.head_object(container, obj)
is_slo = config_true_value(
headers.get('x-static-large-object'))
if options['skip_identical'] or (
is_slo and not options['leave_segments']):
chunk_data = self._get_chunk_data(conn, container, obj,
headers)
if options['skip_identical'] and self._is_identical(
chunk_data, path):
res.update({
'success': True,
'status': 'skipped-identical'
})
return res
cl = int(headers.get('content-length'))
mt = headers.get('x-object-meta-mtime')
if (path is not None and options['changed']
and cl == getsize(path)
and mt == put_headers['x-object-meta-mtime']):
res.update({'success': True, 'status': 'skipped-changed'})
return res
if not options['leave_segments']:
old_manifest = headers.get('x-object-manifest')
if is_slo:
for old_seg in chunk_data:
seg_path = old_seg['name'].lstrip('/')
if isinstance(seg_path, text_type):
seg_path = seg_path.encode('utf-8')
old_slo_manifest_paths.append(seg_path)
except ClientException as err:
if err.http_status != 404:
traceback, err_time = report_traceback()
logger.exception(err)
res.update({
'success': False,
'error': err,
'traceback': traceback,
'error_timestamp': err_time
})
return res
# Merge the command line header options to the put_headers
put_headers.update(split_headers(options['meta'], 'X-Object-Meta-'))
put_headers.update(split_headers(options['header'], ''))
# Don't do segment job if object is not big enough, and never do
# a segment job if we're reading from a stream - we may fail if we
# go over the single object limit, but this gives us a nice way
# to create objects from memory
if (path is not None and segment_size
and (getsize(path) > segment_size)):
res['large_object'] = True
seg_container = container + '_segments'
if options['segment_container']:
seg_container = options['segment_container']
full_size = getsize(path)
segment_futures = []
segment_pool = self.thread_manager.segment_pool
segment = 0
segment_start = 0
while segment_start < full_size:
if segment_start + segment_size > full_size:
segment_size = full_size - segment_start
if options['use_slo']:
segment_name = '%s/slo/%s/%s/%s/%08d' % (
obj, put_headers['x-object-meta-mtime'], full_size,
options['segment_size'], segment)
else:
segment_name = '%s/%s/%s/%s/%08d' % (
obj, put_headers['x-object-meta-mtime'], full_size,
options['segment_size'], segment)
seg = segment_pool.submit(self._upload_segment_job,
path,
container,
segment_name,
segment_start,
segment_size,
segment,
obj,
options,
results_queue=results_queue)
segment_futures.append(seg)
segment += 1
segment_start += segment_size
segment_results = []
errors = False
exceptions = []
for f in interruptable_as_completed(segment_futures):
try:
r = f.result()
if not r['success']:
errors = True
segment_results.append(r)
except Exception as err:
traceback, err_time = report_traceback()
logger.exception(err)
errors = True
exceptions.append((err, traceback, err_time))
if errors:
err = ClientException(
'Aborting manifest creation '
'because not all segments could be uploaded. %s/%s' %
(container, obj))
res.update({
'success': False,
'error': err,
'exceptions': exceptions,
'segment_results': segment_results
})
return res
res['segment_results'] = segment_results
if options['use_slo']:
response = self._upload_slo_manifest(conn, segment_results,
container, obj,
put_headers)
res['manifest_response_dict'] = response
new_slo_manifest_paths = {
seg['segment_location']
for seg in segment_results
}
else:
new_object_manifest = '%s/%s/%s/%s/%s/' % (
quote(seg_container.encode('utf8')),
quote(obj.encode('utf8')),
put_headers['x-object-meta-mtime'], full_size,
options['segment_size'])
if old_manifest and old_manifest.rstrip('/') == \
new_object_manifest.rstrip('/'):
old_manifest = None
put_headers['x-object-manifest'] = new_object_manifest
mr = {}
conn.put_object(container,
obj,
'',
content_length=0,
headers=put_headers,
response_dict=mr)
res['manifest_response_dict'] = mr
elif options['use_slo'] and segment_size and not path:
segment = 0
results = []
while True:
segment_name = '%s/slo/%s/%s/%08d' % (
obj, put_headers['x-object-meta-mtime'], segment_size,
segment)
seg_container = container + '_segments'
if options['segment_container']:
seg_container = options['segment_container']
ret = self._upload_stream_segment(conn, container, obj,
seg_container, segment_name,
segment_size, segment,
put_headers, stream)
if not ret['success']:
return ret
if (ret['complete'] and segment == 0) or\
ret['segment_size'] > 0:
results.append(ret)
if results_queue is not None:
# Don't insert the 0-sized segments or objects
# themselves
if ret['segment_location'] != '/%s/%s' % (
container, obj) and ret['segment_size'] > 0:
results_queue.put(ret)
if ret['complete']:
break
segment += 1
if results[0]['segment_location'] != '/%s/%s' % (container, obj):
response = self._upload_slo_manifest(conn, results, container,
obj, put_headers)
res['manifest_response_dict'] = response
new_slo_manifest_paths = {
r['segment_location']
for r in results
}
res['large_object'] = True
else:
res['response_dict'] = ret
res['large_object'] = False
else:
res['large_object'] = False
obr = {}
fp = None
try:
if path is not None:
content_length = getsize(path)
fp = open(path, 'rb', DISK_BUFFER)
contents = LengthWrapper(fp,
content_length,
md5=options['checksum'])
# TODO: patch here ... check if stream is already a LengthWrapper,
# and use it.
elif isinstance(stream, LengthWrapper):
content_length = stream._length
contents = stream
# TODO: patch end
else:
content_length = None
contents = ReadableToIterable(stream,
md5=options['checksum'])
etag = conn.put_object(container,
obj,
contents,
content_length=content_length,
headers=put_headers,
response_dict=obr)
res['response_dict'] = obr
if (options['checksum'] and etag
and etag != contents.get_md5sum()):
raise SwiftError(
'Object upload verification failed: '
'md5 mismatch, local {0} != remote {1} '
'(remote object has not been removed)'.format(
contents.get_md5sum(), etag))
finally:
if fp is not None:
fp.close()
if old_manifest or old_slo_manifest_paths:
drs = []
delobjsmap = {}
if old_manifest:
scontainer, sprefix = old_manifest.split('/', 1)
sprefix = sprefix.rstrip('/') + '/'
delobjsmap[scontainer] = []
for part in self.list(scontainer, {'prefix': sprefix}):
if not part["success"]:
raise part["error"]
delobjsmap[scontainer].extend(seg['name']
for seg in part['listing'])
if old_slo_manifest_paths:
for seg_to_delete in old_slo_manifest_paths:
if seg_to_delete in new_slo_manifest_paths:
continue
scont, sobj = \
seg_to_delete.split(b'/', 1)
delobjs_cont = delobjsmap.get(scont, [])
delobjs_cont.append(sobj)
delobjsmap[scont] = delobjs_cont
del_segs = []
for dscont, dsobjs in delobjsmap.items():
for dsobj in dsobjs:
del_seg = self.thread_manager.segment_pool.submit(
self._delete_segment,
dscont,
dsobj,
results_queue=results_queue)
del_segs.append(del_seg)
for del_seg in interruptable_as_completed(del_segs):
drs.append(del_seg.result())
res['segment_delete_results'] = drs
# return dict for printing
res.update({
'success': True,
'status': 'uploaded',
'attempts': conn.attempts
})
return res
except OSError as err:
traceback, err_time = report_traceback()
logger.exception(err)
if err.errno == ENOENT:
error = SwiftError('Local file %r not found' % path, exc=err)
else:
error = err
res.update({
'success': False,
'error': error,
'traceback': traceback,
'error_timestamp': err_time
})
except Exception as err:
traceback, err_time = report_traceback()
logger.exception(err)
res.update({
'success': False,
'error': err,
'traceback': traceback,
'error_timestamp': err_time
})
return res
