def swift_file_commit(_opts, container, obj_name, obj_source=None):
with SwiftService(options=_opts) as swift, OutputManager() as out_manager:
try:
if obj_source is None:
obj_source = obj_name # file path
#check if file exists
if (not os.path.exists(obj_source)):
print('File %s does not exist' % obj_source)
exit()
objs = [SwiftUploadObject(obj_source, obj_name)
] # (source, object name) --> (value,key)
for r in swift.upload(container, objs):
if r['success']:
if 'object' in r:
print(r['object'])
elif 'for_object' in r:
print('%s segment %s' %
(r['for_object'], r['segment_index']))
else:
error = r['error']
if r['action'] == "create_container":
logger.warning(
'Warning: failed to create container '
"'%s'%s", container, error)
elif r['action'] == "upload_object":
logger.error(
"Failed to upload object %s to container %s: %s" %
(container, r['object'], error))
else:
logger.error("%s" % error)
except SwiftError as e:
logger.error(e.value)
def stat(*args):
args = ('', ) + args
with OutputManager() as output:
parser = options()
try:
shell.st_stat(parser, list(args), output)
except (ClientException, RequestException, socket.error) as err:
output.error(str(err))
def sw_shell(sw_fun, *args):
args = ('', ) + args
with OutputManager() as output:
parser = shell_minimal_options()
try:
sw_fun(parser, list(args), output)
except (ClientException, RequestException, socket.error) as err:
output.error(str(err))
def handle(self, *args, **options):
_opts = {'object_uu_threads': 20}
dir = 'media' # settings.MEDIA_ROOT
container = settings.SWIFT_CONTAINER_NAME
with SwiftService(
options=_opts) as swift, OutputManager() as out_manager:
try:
# Collect all the files and folders in the given directory
objs = []
dir_markers = []
# change to directory so it isn't uploaded as part of all the
# file object names
os.chdir(dir)
for (_dir, _ds, _fs) in os.walk('.'):
if not (_ds + _fs):
dir_markers.append(_dir)
else:
objs.extend([os.path.join(_dir, _f) for _f in _fs])
# Now that we've collected all the required files and dir markers
# build the ``SwiftUploadObject``s for the call to upload
objs = [
SwiftUploadObject(o, object_name=o.replace(dir, dir, 1))
for o in objs
]
dir_markers = [
SwiftUploadObject(None,
object_name=d.replace(dir, dir, 1),
options={'dir_marker': True})
for d in dir_markers
]
# Schedule uploads on the SwiftService thread pool and iterate
# over the results
for r in swift.upload(container, objs + dir_markers):
if r['success']:
if 'object' in r:
print(r['object'])
elif 'for_object' in r:
print('%s segment %s' %
(r['for_object'], r['segment_index']))
else:
error = r['error']
if r['action'] == "create_container":
logger.warning(
'Warning: failed to create container '
"'%s'%s", container, error)
elif r['action'] == "upload_object":
logger.error(
"Failed to upload object %s to container %s: %s"
% (container, r['object'], error))
else:
logger.error("%s" % error)
os.chdir('..')
except SwiftError as e:
logger.error(e.value)
def upload_results(results_folder, destination_container):
SETTINGS_LOCATION = os.environ.get('SWIFT_SETTINGS')
if not SETTINGS_LOCATION:
raise RuntimeError("No OpenStack settings provided for the upload")
with open(SETTINGS_LOCATION, 'r') as sf:
settings = json.load(sf)
# generate filelist from results folder
files = [(os.path.join(dp, f)) for dp, dn, fn in os.walk(results_folder)
for f in fn]
labels = convert_filenames_to_labels(results_folder, files)
options = {
'object_uu_threads': 20,
'os_auth_token': get_keystone_token(settings)
}
# Remove password from the OpenStack settings, otherwise the SwiftService
# connection will be built ignoring the token and failing authentication
del (settings['os_password'])
options.update(**settings)
def generate_options(label):
if label in ['/info', '/info_fullres.json', '/transform.json']:
return {}
else:
return {'header': ['Content-Encoding:gzip']}
with SwiftService(
options=options) as swift, OutputManager() as out_manager:
try:
logger.info(F'Making public container {destination_container}')
swift.post(container=destination_container,
options={
'read_acl':
'.r:*,.rlistings',
'header':
['X-Container-Meta-Access-Control-Allow-Origin: *']
})
logger.info(
f'Uploading {len(files)} objects to {destination_container}')
start = perf_counter()
objects = [
SwiftUploadObject(file,
object_name=labels[index],
options=generate_options(labels[index]))
for index, file in enumerate(files)
]
for result in swift.upload(destination_container, objects):
if not result['success']:
logger.error(f"Failed to upload object")
raise RuntimeError("Failed to upload object")
finish = perf_counter()
logger.info(f'Completed in {timedelta(seconds=finish-start)}')
except SwiftError as e:
logger.exception(e.value)
raise RuntimeError("Failed to upload objects")
def setUp(self):
super(TestStatHelpers, self).setUp()
conn_attrs = {
'url': 'http://storage/v1/a',
'token': 'tk12345',
}
self.conn = mock.MagicMock(**conn_attrs)
self.options = {'human': False, 'verbose': 1}
self.stdout = StringIO()
self.stderr = StringIO()
self.output_manager = OutputManager(self.stdout, self.stderr)
def sw_shell(sw_fun,*args):
if _default_global_options['os_auth_token'] and _default_global_options['os_storage_url']:
args=args+("--os_auth_token",_default_global_options['os_auth_token'],
"--os_storage_url",_default_global_options['os_storage_url'])
args = ('',) + args
with OutputManager() as output:
parser = shell_minimal_options()
try:
sw_fun(parser, list(args), output)
except (ClientException, RequestException, socket.error) as err:
output.error(str(err))
def sw_shell(sw_fun, *args):
global swift_auth_token, storage_url
if swift_auth_token and storage_url:
args = args + ("--os_auth_token", swift_auth_token, "--os_storage_url",
storage_url)
args = ('', ) + args
with OutputManager() as output:
parser = shell_minimal_options()
try:
sw_fun(parser, list(args), output)
except (ClientException, RequestException, socket.error) as err:
output.error(str(err))
import logging
from os.path import join, walk
from swiftclient.multithreading import OutputManager
from swiftclient.service import SwiftError, SwiftService, SwiftUploadObject
from sys import argv
logging.basicConfig(level=logging.ERROR)
logging.getLogger("requests").setLevel(logging.CRITICAL)
logging.getLogger("swiftclient").setLevel(logging.CRITICAL)
logger = logging.getLogger(__name__)
_opts = {'object_uu_threads': 20}
dir = argv[1]
container = argv[2]
with SwiftService(options=_opts) as swift, OutputManager() as out_manager:
try:
# Collect all the files and folders in the given directory
objs = []
dir_markers = []
for (_dir, _ds, _fs) in walk(dir):
if not (_ds + _fs):
dir_markers.append(_dir)
else:
objs.extend([join(_dir, _f) for _f in _fs])
# Now that we've collected all the required files and dir markers
# build the ``SwiftUploadObject``s for the call to upload
objs = [
SwiftUploadObject(
o, object_name=o.replace(