class SwiftClient(object): """Client for Swift object/blob store of Openstack See http://swift.openstack.org Swift requires environment variables (OS_*) for the authentication and configuration""" def __init__(self, container, prefix=''): self.container = container self.prefix = prefix self.client = SwiftService() def download(self, source, target): objects = [self.prefix + '/' + source] options = {'out_file': target} return list(self.client.download(self.container, objects, options)) def upload(self, source, target): object_name = self.prefix + '/' + target objects = [SwiftUploadObject(source, object_name=object_name)] return list(self.client.upload(self.container, objects)) def ls(self, path): fpath = self.prefix + '/' + path + '/' clisting = self.client.list(self.container, {'prefix': fpath}) listing = list(clisting)[0]['listing'] result = [d['name'].replace(fpath, '') for d in listing] return result def url(self, path=''): return self.container + '/' + self.prefix + '/' + path
def test_delete_object_dlo_support(self): mock_q = Queue() s = SwiftService() mock_conn = self._get_mock_connection() expected_r = self._get_expected({ 'action': 'delete_object', 'success': True, 'dlo_segments_deleted': True }) # A DLO object is determined in _delete_object by heading the object # and checking for the existence of a x-object-manifest header. # Mock that here. mock_conn.head_object = Mock( return_value={'x-object-manifest': 'manifest_c/manifest_p'} ) mock_conn.get_container = Mock( side_effect=[(None, [{'name': 'test_seg_1'}, {'name': 'test_seg_2'}]), (None, {})] ) def get_mock_list_conn(options): return mock_conn with mock.patch('swiftclient.service.get_conn', get_mock_list_conn): r = s._delete_object( mock_conn, 'test_c', 'test_o', self.opts, mock_q ) self._assertDictEqual(expected_r, r) expected = [ mock.call('test_c', 'test_o', query_string=None, response_dict={}), mock.call('manifest_c', 'test_seg_1', response_dict={}), mock.call('manifest_c', 'test_seg_2', response_dict={})] mock_conn.delete_object.assert_has_calls(expected, any_order=True)
def saveObjsBackend(objs, backend, config): if(backend == 'hdfs'): for obj in objs: try: # obj[0] is hdfs path and obj[1] is local filesystem path if '.py' in obj[1]: # If you are uploading a module do not allow it to be overwritten subprocess.check_call(['hdfs', 'dfs', '-copyFromLocal', obj[1], obj[0]]) else: # If it is the metadata then it has to be overwritten everytime subprocess.check_call(['hdfs', 'dfs', '-copyFromLocal', '-f', obj[1], obj[0]]) except Exception as e: shutil.copyfile(config['BACKUP_METADATA_LOCAL_PATH'], config['METADATA_LOCAL_PATH']) raise RuntimeError(e) elif(backend == 'swift'): options = {'os_auth_url': os.environ['OS_AUTH_URL'], 'os_username': os.environ['OS_USERNAME'], 'os_password': os.environ['OS_PASSWORD'], 'os_tenant_id': os.environ['OS_TENANT_ID'], 'os_tenant_name': os .environ['OS_TENANT_NAME']} swiftService = SwiftService(options=options) objects = [] for obj in objs: objects.append(SwiftUploadObject(obj[1], object_name=obj[0])) swiftUpload = swiftService.upload(container='containerModules', objects=objects) for uploaded in swiftUpload: if("error" in uploaded.keys()): shutil.copyfile(config['BACKUP_METADATA_LOCAL_PATH'], config['METADATA_LOCAL_PATH']) raise RuntimeError(uploaded['error']) elif(backend == 'nfs'): for obj in objs: shutil.copyfile(obj[1], config['MODULES_DIR'] + obj[0]) print('Metadata/Module changed and uploaded')
def test_delete_object_dlo_support(self): mock_q = Queue() s = SwiftService() mock_conn = self._get_mock_connection() expected_r = self._get_expected({ 'action': 'delete_object', 'success': True, 'dlo_segments_deleted': True }) # A DLO object is determined in _delete_object by heading the object # and checking for the existence of a x-object-manifest header. # Mock that here. mock_conn.head_object = Mock( return_value={'x-object-manifest': 'manifest_c/manifest_p'}) mock_conn.get_container = Mock(side_effect=[(None, [{ 'name': 'test_seg_1' }, { 'name': 'test_seg_2' }]), (None, {})]) def get_mock_list_conn(options): return mock_conn with mock.patch('swiftclient.service.get_conn', get_mock_list_conn): r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) self._assertDictEqual(expected_r, r) expected = [ mock.call('test_c', 'test_o', query_string=None, response_dict={}), mock.call('manifest_c', 'test_seg_1', response_dict={}), mock.call('manifest_c', 'test_seg_2', response_dict={}) ] mock_conn.delete_object.assert_has_calls(expected, any_order=True)
def test_upload_object_job_identical_etag(self): with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 30) f.flush() mock_conn = mock.Mock() mock_conn.head_object.return_value = { 'content-length': 30, 'etag': md5(b'a' * 30).hexdigest()} type(mock_conn).attempts = mock.PropertyMock(return_value=2) s = SwiftService() r = s._upload_object_job(conn=mock_conn, container='test_c', source=f.name, obj='test_o', options={'changed': False, 'skip_identical': True, 'leave_segments': True, 'header': '', 'segment_size': 0}) self.assertTrue(r['success']) self.assertIn('status', r) self.assertEqual(r['status'], 'skipped-identical') self.assertEqual(mock_conn.put_object.call_count, 0) self.assertEqual(mock_conn.head_object.call_count, 1) mock_conn.head_object.assert_called_with('test_c', 'test_o')
def test_upload_object_job_file(self): # Uploading a file results in the file object being wrapped in a # LengthWrapper. This test sets the options in such a way that much # of _upload_object_job is skipped bringing the critical path down # to around 60 lines to ease testing. with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 30) f.flush() expected_r = { 'action': 'upload_object', 'attempts': 2, 'container': 'test_c', 'headers': {}, 'large_object': False, 'object': 'test_o', 'response_dict': {}, 'status': 'uploaded', 'success': True, } expected_mtime = float(os.path.getmtime(f.name)) mock_conn = mock.Mock() mock_conn.put_object.return_value = '' type(mock_conn).attempts = mock.PropertyMock(return_value=2) s = SwiftService() r = s._upload_object_job(conn=mock_conn, container='test_c', source=f.name, obj='test_o', options={'changed': False, 'skip_identical': False, 'leave_segments': True, 'header': '', 'segment_size': 0, 'checksum': True}) mtime = float(r['headers']['x-object-meta-mtime']) self.assertAlmostEqual(mtime, expected_mtime, delta=0.5) del r['headers']['x-object-meta-mtime'] self.assertEqual(r['path'], f.name) del r['path'] self._assertDictEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c', 'test_o', mock.ANY, content_length=30, headers={}, response_dict={}) contents = mock_conn.put_object.call_args[0][2] self.assertIsInstance(contents, utils.LengthWrapper) self.assertEqual(len(contents), 30) # This read forces the LengthWrapper to calculate the md5 # for the read content. This also checks that LengthWrapper was # initialized with md5=True self.assertEqual(contents.read(), b'a' * 30) self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())
def __init__(self, settings): options = get_os_settings(settings) self.swift = SwiftService(options) options = get_swift_settings(settings) self.temp_url_key = options['temp_url_key'] # TODO: hard coded template self.name_template = options['container'] + '_{name}'
def upload(source, dest): """ Upload file to a remote SWIFT store @param source: List of local source path to upload from. @type source : Dicrionary @param dest: The destination information such as destination url to upload the file. @type dest: Dictionary @return: True if upload is successful. Otherwise False. """ url = urlsplit(dest['url']) _, ver, account, container, object_name = url.path.split('/', 4) swift_opts = { 'os_storage_url': '{scheme}://{netloc}/{ver}/{account}'.format( scheme=re.sub(r'^swift\+', '', url.scheme), netloc=url.netloc, ver=ver, account=account) } # SwiftService knows about environment variables for opt in ('os_auth_url', 'os_username', 'os_password', 'os_tenant_name', 'os_storage_url'): if opt in dest: swift_opts[opt] = dest[opt] try: swift = SwiftService(swift_opts) headers = [] if 'content_type' in source: headers.append('Content-Type: {}'.format(source['content_type'])) retries = 5 # number of retries left backoff = 30 # wait time between retries backoff_inc = 30 # increase in wait time per retry while retries: retries -= 1 try: for result in swift.upload(container, [SwiftUploadObject(source['url'], object_name=object_name, options={'header': headers})]): # TODO: we may get result['action'] = 'create_container' # self.assertNotIn(member, container)d result['action'] = 'upload_object'; result['path'] = # source['url'] if not result['success']: raise Exception( 'Upload to Swift {container}/{object_name} failed with {error}'.format(object_name=object_name, **result)) # no exception we can continue retries = 0 except Exception as e: if not retries: # reraise if no retries left raise LOG.warn('Upload to Swift failed: %s - %d retries left', e, retries) time.sleep(backoff) backoff += backoff_inc backoff_inc += 30 except Exception as e: LOG.error("Upload to swift failed: %s", e, exc_info=True) raise
def test_upload_object_job_stream(self): # Streams are wrapped as ReadableToIterable with tempfile.TemporaryFile() as f: f.write(b'a' * 30) f.flush() f.seek(0) expected_r = { 'action': 'upload_object', 'attempts': 2, 'container': 'test_c', 'headers': {}, 'large_object': False, 'object': 'test_o', 'response_dict': {}, 'status': 'uploaded', 'success': True, 'path': None, } expected_mtime = float(time.time()) mock_conn = mock.Mock() mock_conn.put_object.return_value = '' type(mock_conn).attempts = mock.PropertyMock(return_value=2) s = SwiftService() r = s._upload_object_job(conn=mock_conn, container='test_c', source=f, obj='test_o', options={ 'changed': False, 'skip_identical': False, 'leave_segments': True, 'header': '', 'segment_size': 0, 'checksum': True }) mtime = float(r['headers']['x-object-meta-mtime']) self.assertAlmostEqual(mtime, expected_mtime, delta=0.5) del r['headers']['x-object-meta-mtime'] self._assertDictEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c', 'test_o', mock.ANY, content_length=None, headers={}, response_dict={}) contents = mock_conn.put_object.call_args[0][2] self.assertIsInstance(contents, utils.ReadableToIterable) self.assertEqual(contents.chunk_size, 65536) # next retrieves the first chunk of the stream or len(chunk_size) # or less, it also forces the md5 to be calculated. self.assertEqual(next(contents), b'a' * 30) self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())
def __init__(self, storage_id, container_name, auth_config=None, transfer_config=None): super(SwiftStorage, self).__init__(storage_id) opts = transfer_config or {} opts["auth_version"] = "2.0" if auth_config: for k, v in six.iteritems(auth_config): opts[k] = v self._client = SwiftService(opts) self._container = container_name
def test_upload_with_bad_segment_size(self): for bad in ('ten', '1234X', '100.3'): options = {'segment_size': bad} try: service = SwiftService(options) next(service.upload('c', 'o')) self.fail('Expected SwiftError when segment_size=%s' % bad) except SwiftError as exc: self.assertEqual('Segment size should be an integer value', exc.value)
class DiracStore(object): def __init__(self, container, topic, kafka): self.container = container self.swift = SwiftService() self.out = OutputManager() self.kafka = kafka auth = get_auth() self.url = auth[0] self.token = auth[1] self.topic = topic self.producer = KafkaProducer(bootstrap_servers=kafka) def send(self, resource): json = message_to_json(resource) print("sending " + json) self.producer.send(self.topic, json.encode('utf-8')) def make_resource(self, stat): resource = schema.Resource() resource.type = 'file' resource.name = stat['Object'] resource.location = self.url + '/' + self.container + '/' + stat['Object'] resource.mimeType = stat['Content Type'] resource.size = long(stat['Content Length']) resource.created = stat['Last Modified'] return resource def stat(self, paths): stat = {} for response in self.swift.stat(container=self.container, objects=paths): if response['success']: stat[response['object']] = {item[0]: item[1] for item in response['items']} return stat def store(self, path, source): isdir = os.path.isdir(source) base = source if isdir else os.path.dirname(source) sources = os.listdir(source) if isdir else [source] locations = [os.path.join(path, os.path.basename(file)) for file in sources] print(str(len(locations)) + " locations!") stats = self.stat(locations) objs = [SwiftUploadObject(os.path.join(base, os.path.basename(location)), object_name=location) for location in locations if not location in stats] print(str(len(objs)) + " previously unseen!") for response in self.swift.upload(self.container, objs): if response['success']: if 'object' in response: print('uploading ' + response['object']) stat = self.stat([response['object']]) resource = self.make_resource(stat.values()[0]) self.send(resource)
def test_upload_segment_job(self): with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 10) f.write(b'b' * 10) f.write(b'c' * 10) f.flush() # Mock the connection to return an empty etag. This # skips etag validation which would fail as the LengthWrapper # isnt read from. mock_conn = mock.Mock() mock_conn.put_object.return_value = '' type(mock_conn).attempts = mock.PropertyMock(return_value=2) expected_r = { 'action': 'upload_segment', 'for_object': 'test_o', 'segment_index': 2, 'segment_size': 10, 'segment_location': '/test_c_segments/test_s_1', 'log_line': 'test_o segment 2', 'success': True, 'response_dict': {}, 'segment_etag': '', 'attempts': 2, } s = SwiftService() r = s._upload_segment_job(conn=mock_conn, path=f.name, container='test_c', segment_name='test_s_1', segment_start=10, segment_size=10, segment_index=2, obj_name='test_o', options={ 'segment_container': None, 'checksum': True }) self._assertDictEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c_segments', 'test_s_1', mock.ANY, content_length=10, response_dict={}) contents = mock_conn.put_object.call_args[0][2] self.assertIsInstance(contents, utils.LengthWrapper) self.assertEqual(len(contents), 10) # This read forces the LengthWrapper to calculate the md5 # for the read content. self.assertEqual(contents.read(), b'b' * 10) self.assertEqual(contents.get_md5sum(), md5(b'b' * 10).hexdigest())
def test_upload_object_job_stream(self): # Streams are wrapped as ReadableToIterable with tempfile.TemporaryFile() as f: f.write(b'a' * 30) f.flush() f.seek(0) expected_r = { 'action': 'upload_object', 'attempts': 2, 'container': 'test_c', 'headers': {}, 'large_object': False, 'object': 'test_o', 'response_dict': {}, 'status': 'uploaded', 'success': True, 'path': None, } expected_mtime = float(time.time()) mock_conn = mock.Mock() mock_conn.put_object.return_value = '' type(mock_conn).attempts = mock.PropertyMock(return_value=2) s = SwiftService() r = s._upload_object_job(conn=mock_conn, container='test_c', source=f, obj='test_o', options={'changed': False, 'skip_identical': False, 'leave_segments': True, 'header': '', 'segment_size': 0, 'checksum': True}) mtime = float(r['headers']['x-object-meta-mtime']) self.assertAlmostEqual(mtime, expected_mtime, delta=0.5) del r['headers']['x-object-meta-mtime'] self._assertDictEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c', 'test_o', mock.ANY, content_length=None, headers={}, response_dict={}) contents = mock_conn.put_object.call_args[0][2] self.assertIsInstance(contents, utils.ReadableToIterable) self.assertEqual(contents.chunk_size, 65536) # next retrieves the first chunk of the stream or len(chunk_size) # or less, it also forces the md5 to be calculated. self.assertEqual(next(contents), b'a' * 30) self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())
def test_upload_segment_job(self): with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 10) f.write(b'b' * 10) f.write(b'c' * 10) f.flush() # Mock the connection to return an empty etag. This # skips etag validation which would fail as the LengthWrapper # isnt read from. mock_conn = mock.Mock() mock_conn.put_object.return_value = '' type(mock_conn).attempts = mock.PropertyMock(return_value=2) expected_r = { 'action': 'upload_segment', 'for_object': 'test_o', 'segment_index': 2, 'segment_size': 10, 'segment_location': '/test_c_segments/test_s_1', 'log_line': 'test_o segment 2', 'success': True, 'response_dict': {}, 'segment_etag': '', 'attempts': 2, } s = SwiftService() r = s._upload_segment_job(conn=mock_conn, path=f.name, container='test_c', segment_name='test_s_1', segment_start=10, segment_size=10, segment_index=2, obj_name='test_o', options={'segment_container': None, 'checksum': True}) self._assertDictEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c_segments', 'test_s_1', mock.ANY, content_length=10, response_dict={}) contents = mock_conn.put_object.call_args[0][2] self.assertIsInstance(contents, utils.LengthWrapper) self.assertEqual(len(contents), 10) # This read forces the LengthWrapper to calculate the md5 # for the read content. self.assertEqual(contents.read(), b'b' * 10) self.assertEqual(contents.get_md5sum(), md5(b'b' * 10).hexdigest())
def list_container(container, prefix): obj_list = [] myoptions={'prefix': prefix} stats=SwiftService(options=_default_global_options).stat() if not stats["success"] == True: print('not authenticated') return False with SwiftService(options=_default_global_options) as swift: listing=swift.list(container=container, options=myoptions) for o in listing: for i in o['listing']: obj_list.append(i['name']) return obj_list
def test_upload_object_job_identical_dlo(self): with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 30) f.flush() segment_etag = md5(b'a' * 10).hexdigest() mock_conn = mock.Mock() mock_conn.head_object.return_value = { 'x-object-manifest': 'test_c_segments/test_o/prefix', 'content-length': 30, 'etag': md5(segment_etag.encode('ascii') * 3).hexdigest()} mock_conn.get_container.side_effect = [ (None, [{"bytes": 10, "hash": segment_etag, "name": "test_o/prefix/00"}, {"bytes": 10, "hash": segment_etag, "name": "test_o/prefix/01"}]), (None, [{"bytes": 10, "hash": segment_etag, "name": "test_o/prefix/02"}]), (None, {})] type(mock_conn).attempts = mock.PropertyMock(return_value=2) s = SwiftService() with mock.patch('swiftclient.service.get_conn', return_value=mock_conn): r = s._upload_object_job(conn=mock_conn, container='test_c', source=f.name, obj='test_o', options={'changed': False, 'skip_identical': True, 'leave_segments': True, 'header': '', 'segment_size': 10}) self.assertIsNone(r.get('error')) self.assertTrue(r['success']) self.assertEqual('skipped-identical', r.get('status')) self.assertEqual(0, mock_conn.put_object.call_count) self.assertEqual(1, mock_conn.head_object.call_count) self.assertEqual(3, mock_conn.get_container.call_count) mock_conn.head_object.assert_called_with('test_c', 'test_o') expected = [ mock.call('test_c_segments', prefix='test_o/prefix', marker='', delimiter=None), mock.call('test_c_segments', prefix='test_o/prefix', marker="test_o/prefix/01", delimiter=None), mock.call('test_c_segments', prefix='test_o/prefix', marker="test_o/prefix/02", delimiter=None), ] mock_conn.get_container.assert_has_calls(expected)
async def get_object_metadata( conn: SwiftService, meta_cont: str, meta_obj: typing.Union[typing.List[str], None] ) -> typing.List[dict]: """Get object metadata.""" try: res = list(conn.stat(meta_cont, meta_obj)) # Fail if an object wasn't usable if False in [i["success"] for i in res]: raise aiohttp.web.HTTPNotFound() # Filter for metadata not already served with the list request res = [ [ i["object"], dict(filter(lambda j: "x-object-meta" in j[0], i["headers"].items())), ] for i in res ] # Strip unnecessary specifcations from header names and split open s3 # information so that it doesn't have to be done in the browser for i in res: i[1] = {k.replace("x-object-meta-", ""): v for k, v in i[1].items()} if "s3cmd-attrs" in i[1].keys(): i[1]["s3cmd-attrs"] = { k: v for k, v in [j.split(":") for j in i[1]["s3cmd-attrs"].split("/")] } return res except SwiftError: # Fail if container wasn't found raise aiohttp.web.HTTPNotFound()
def upload(self, input_filename, path_to_file, **kwargs): # fp = open(filename, "rb") # self.con.put_object(self.container, keyname, fp) import swiftclient.service from swiftclient.service import SwiftService from swiftclient.service import SwiftUploadObject swiftclient.service._default_global_options['os_auth_url'] = self.conf[ "auth_url"] swiftclient.service._default_global_options['os_username'] = self.conf[ "access_key"] swiftclient.service._default_global_options['os_password'] = self.conf[ "secret_key"] swiftclient.service._default_global_options[ 'os_tenant_name'] = self.conf["tenant_name"] swiftclient.service._default_global_options[ 'auth_version'] = self.conf["auth_version"] swiftclient.service._default_global_options['insecure'] = self.conf[ "insecure"] if 'segment_size' not in self.conf: self.conf["segment_size"] = '1073741824' if 'chunk_size' not in self.conf: self.conf["chunk_size"] = '20480' with SwiftService() as ss: for result in ss.upload( self.conf["s3_bucket"], [SwiftUploadObject(path_to_file, input_filename)], {'segment_size': self.conf["segment_size"]}): if not str(result['success']): self.print_upload_failure(path_to_file, result) raise Exception('Upload Failure of File: ' + path_to_file)
def __init__( self, os_tenant_id, os_tenant_name, os_username, os_password, os_region_name, os_storage_url, os_user_domain_name=SWIFT_OS_USER_DOMAIN_NAME, os_project_domain_name=SWIFT_OS_PROJECT_DOMAIN_NAME, os_auth_url=SWIFT_OS_AUTH_URL, os_identity_api_version=SWIFT_OS_IDENTITY_API_VERSION, ): """Setup options for the SwiftService""" self.os_tenant_id = os_tenant_id self.os_tenant_name = os_tenant_name self.os_username = os_username self.os_password = os_password self.os_region_name = os_region_name self.os_user_domain_name = os_user_domain_name self.os_project_domain_name = os_project_domain_name self.os_auth_url = os_auth_url self.os_identity_api_version = os_identity_api_version self.container = urlparse(os_storage_url).path.rpartition("/")[-1] self.os_storage_url = os_storage_url.removesuffix(f"/{self.container}") with SwiftService(self.options) as swift: stats = swift.stat() if not stats["success"]: msg = "Unable to connect to the requested container: %s" logger.error(msg, stats["error"]) raise BackendParameterException(msg % stats["error"])
def uploadfiles(container, options, objs): # first argument is the container to upload file to # second argument is a dictionary of the options to supply to SwiftUploadObject # third argument is a list of the files to upload with SwiftService() as swift: try: # create the SwiftUploadObject list of objects to upload #objs = [SwiftUploadObject(obj, options=options) for obj in objs] objs = [SwiftUploadObject(obj) for obj in objs] # Schedule uploads on the SwiftService thread pool and iterate over the results #for result in swift.upload(container, objs): for result in swift.upload(container, objs): if result['success']: if 'object' in result: print("Successfully uploaded %s." % result['object']) elif 'for_object' in result: print('%s segment %s' % (result['for_object'], result['segment_index'])) else: error = result['error'] if result['action'] == "create_container": print('Warning: failed to create container ' "'%s'%s", container, error) elif result['action'] == "upload_object": print( "Failed to upload object %s to container %s: %s" % (container, result['object'], error)) else: print("%s" % error) except SwiftError as e: print("SwiftError: %s" % e)
def upload_from_file(self, path): """ Stores the contents of the file pointed by the ``path`` variable. If the file is bigger than 5 Gig, it will be broken into segments. :type path: ``str`` :param path: Absolute path to the file to be uploaded to Swift. :rtype: ``bool`` :return: ``True`` if successful, ``False`` if not. .. note:: * The size of the segments chosen (or any of the other upload options) is not under user control. * If called this method will remap the ``swiftclient.service.get_conn`` factory method to ``self._provider._connect_swift`` .. seealso:: https://github.com/gvlproject/cloudbridge/issues/35#issuecomment-297629661 # noqa """ upload_options = {} if 'segment_size' not in upload_options: if os.path.getsize(path) >= FIVE_GIG: upload_options['segment_size'] = FIVE_GIG # remap the swift service's connection factory method swiftclient.service.get_conn = self._provider._connect_swift result = True with SwiftService() as swift: upload_object = SwiftUploadObject(path, object_name=self.name) for up_res in swift.upload(self.cbcontainer.name, [upload_object, ], options=upload_options): result = result and up_res['success'] return result
def execute(self, context): Path(self.output_path).parents[0].mkdir(parents=True, exist_ok=True) download_options = { "out_file": self.output_path, } options = None if self.conn_id is not None: options = {} connection = BaseHook.get_connection(self.conn_id) options["os_username"] = connection.login options["os_password"] = connection.password options["os_tenant_name"] = connection.host with SwiftService(options=options) as swift: try: for down_res in swift.download( container=self.container, objects=[self.object_id], options=download_options, ): if down_res["success"]: self.log.info("downloaded: %s", down_res["object"]) else: self.log.error("download failed: %s", down_res["object"]) except SwiftError as e: self.log.error(e.value)
def swift_metadata_create(container, sciobj_name, sciobj_metadata, options=None): """ Post metadata to existing object, by first GET existing metadata then concatenating two meta Input: Container name Object name Metadata object Output: None TODO: GET existing metadata """ #first retrieve existing metadata, otherwise, meta post will destroy existing one with SwiftService(options=options) as swift: try: post_options = {"meta": sciobj_metadata} r = swift.post(container=container, objects=[sciobj_name], options=post_options) try: for ri in r: if not ri['success']: print('object upload error') print('error:%s', r['error']) except Exception as e: print('ri[' '] failed, ', e) pass except Exception as e: print('stat error: ', e)
def mock_import_analysis(self, name='', description='', details='', filepath='', params='', inputs='', outputs=''): filename = filepath created = 'mm-dd-yyyy 00:00:00.000' user = '******' sessionQuery = self.session.query(Analysis).from_statement(text("SELECT * FROM analysis where name=:name")).\ params(name=name).first().side_effect = mod_se checkMod = sessionQuery(name=name) if(checkMod is None): analysisMod = Mock(spec=Analysis(name=name, filepath=filename, description=description, details=details, created=created, user=user, parameters=params, inputs=inputs, outputs=outputs)) options = {'os_auth_url': self.config['SWIFT_AUTH_URL'], 'os_username': self.config['SWIFT_USERNAME'], 'os_password': self.config['SWIFT_PASSWORD'], 'os_tenant_id': self.config['SWIFT_TENANT_ID'], 'os_tenant_name': self.config['SWIFT_TENANT_NAME']} swiftService = Mock(spec=SwiftService(options=options)) objects = [] objects.append(SwiftUploadObject(self.config['DB_LOCATION'], object_name='sqlite.db')) objects.append(SwiftUploadObject(filepath, object_name=filename)) swiftUpload = swiftService.upload(container='containerModules', objects=objects).return_value = ('Metadata', 'Module') for uploaded in swiftUpload: print(uploaded) self.session.add(analysisMod) self.session.commit() return "import_success" else: raise RuntimeError("Analysis " + name + " already exists")
def swift_delete(project_name): obj = 0 cont = 0 err = 0 success = True sw_opt = swift_options sw_opt['os_project_name'] = project_name with SwiftService(sw_opt) as swift: for delete in swift.delete(options={ 'yes_all': True, }): if delete['success']: if delete['action'] == 'delete_object': obj = obj + 1 if delete['action'] == 'delete_container': cont = cont + 1 else: err = err + 1 success = False print('Warning! Delete error: {}'.delete) result = 'Deleted objects: {}, containers: {}, errors: {}'.format( obj, cont, err) if success: return result else: return False
def swift_update_quota(project_name, new_quota_bytes): sw_opt = swift_options sw_opt['os_project_name'] = project_name with SwiftService(sw_opt) as swift: post = swift.post( options={'meta': ['quota-bytes:{}'.format(new_quota_bytes)]}) return post['success']
def create_swift_service(self, storage_url): try: return SwiftService( options={ 'user': self.__keystoneCredentials['username'], 'key': self.__keystoneCredentials['password'], 'os_project_id': self.__keystoneCredentials['project_id'], 'os_project_name': self.__keystoneCredentials['project_name'], 'os_project_domain_name': self.__keystoneCredentials['user_domain_name'], 'os_tenant_id': self.__keystoneCredentials['project_id'], 'os_tenant_name': self.__keystoneCredentials['project_name'], 'os_user_domain_name': self.__keystoneCredentials['user_domain_name'], 'os_auth_url': self.__keystoneCredentials['auth_url'], 'os_cacert': self.__certificatesPath, 'auth_version': '3', 'os_storage_url': storage_url }) except Exception as error: raise Exception('Connection to Swift failed: {}'.format(error))
def list_view(request): # lowercase important localconfig = { 'st_auth_version': 3, 'os_user_domain_name': 'admin_domain', 'os_username': '******', 'os_password': '******', 'os_auth_url': 'http://172.31.37.171:5000/v3'} container = 'bsm1' minimum_size = 10 * 1024 ** 2 items = [] with SwiftService(options=localconfig) as swift: try: list_parts_gen = swift.list(container=container) for page in list_parts_gen: if page["success"]: for item in page["listing"]: i_size = int(item["bytes"]) if i_size > minimum_size: i_name = item["name"] i_etag = item["hash"] print( "%s [size: %s] [etag: %s]" % (i_name, i_size, i_etag) ) items.append((i_name, i_size, i_etag)) else: raise page["error"] except SwiftError as e: logger.error(e.value) print(items) return JsonResponse(items, safe=False)
def read(self, name, chunk_size=None): """Read `name` object and stream its content in chunks of (max) 2 ** 16 Why chunks of (max) 2 ** 16 ? Because SwiftService opens a file to stream the object into: See swiftclient.service.py:2082 open(filename, 'rb', DISK_BUFFER) Where filename = "/dev/stdout" and DISK_BUFFER = 2 ** 16 """ logger.debug("Getting archive: %s", name) with SwiftService(self.options) as swift: download = next( swift.download(self.container, [name], {"out_file": "-"})) if "contents" not in download: msg = "Failed to download %s: %s" logger.error(msg, download["object"], download["error"]) raise BackendException(msg % (download["object"], download["error"])) size = 0 for chunk in download["contents"]: logger.debug("Chunk %s", len(chunk)) size += len(chunk) sys.stdout.buffer.write(chunk) # Archive fetched, add a new entry to the history self.append_to_history({ "backend": self.name, "command": "fetch", "id": name, "size": size, "fetched_at": now(), })
def swift_file_commit(_opts, container, obj_name, obj_source=None): with SwiftService(options=_opts) as swift, OutputManager() as out_manager: try: if obj_source is None: obj_source = obj_name # file path #check if file exists if (not os.path.exists(obj_source)): print('File %s does not exist' % obj_source) exit() objs = [SwiftUploadObject(obj_source, obj_name) ] # (source, object name) --> (value,key) for r in swift.upload(container, objs): if r['success']: if 'object' in r: print(r['object']) elif 'for_object' in r: print('%s segment %s' % (r['for_object'], r['segment_index'])) else: error = r['error'] if r['action'] == "create_container": logger.warning( 'Warning: failed to create container ' "'%s'%s", container, error) elif r['action'] == "upload_object": logger.error( "Failed to upload object %s to container %s: %s" % (container, r['object'], error)) else: logger.error("%s" % error) except SwiftError as e: logger.error(e.value)
def test_delete_empty_container_excpetion(self): mock_conn = self._get_mock_connection() mock_conn.delete_container = Mock(side_effect=self.exc) expected_r = self._get_expected({ 'action': 'delete_container', 'success': False, 'object': None, 'error': self.exc }) s = SwiftService() r = s._delete_empty_container(mock_conn, 'test_c') mock_conn.delete_container.assert_called_once_with('test_c', response_dict={}) self._assertDictEqual(expected_r, r)
def download_swiftclient(oid, container, name, delay=0, request_tries=5): try: print 'Prefetching object with SwiftClient: ' + oid + ' after ' + str( delay.total_seconds()) + ' seconds of delay.' time.sleep(delay.total_seconds()) _opts = { "auth_version": '2.0', # Should be '2.0' "os_username": environ.get('OS_USERNAME'), "os_password": environ.get('OS_PASSWORD'), "os_tenant_name": environ.get('OS_TENANT_NAME'), "os_auth_url": environ.get('OS_AUTH_URL'), "retries": request_tries, } headers = {"header": ["X-No-Prefetch:True"]} with SwiftService(options=_opts) as swift: for down_res in swift.download(container=container, objects=[name], options=headers): if down_res['success']: print("'%s' downloaded" % down_res['object']) return (oid, down_res['object'], down_res['response_dict']['headers']) else: print("'%s' download failed" % down_res['error']) return (oid, [], down_res['response_dict']['headers']) except Error as e: pass
def test_upload_object_job_identical_slo_with_nesting(self): with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 30) f.flush() seg_etag = md5(b'a' * 10).hexdigest() submanifest = "[%s]" % ",".join( ['{"bytes":10,"hash":"%s"}' % seg_etag] * 2) submanifest_etag = md5(seg_etag.encode('ascii') * 2).hexdigest() manifest = "[%s]" % ",".join([ '{"sub_slo":true,"name":"/test_c_segments/test_sub_slo",' '"bytes":20,"hash":"%s"}' % submanifest_etag, '{"bytes":10,"hash":"%s"}' % seg_etag]) mock_conn = mock.Mock() mock_conn.head_object.return_value = { 'x-static-large-object': True, 'content-length': 30, 'etag': md5(submanifest_etag.encode('ascii') + seg_etag.encode('ascii')).hexdigest()} mock_conn.get_object.side_effect = [ ({}, manifest.encode('ascii')), ({}, submanifest.encode('ascii'))] type(mock_conn).attempts = mock.PropertyMock(return_value=2) s = SwiftService() r = s._upload_object_job(conn=mock_conn, container='test_c', source=f.name, obj='test_o', options={'changed': False, 'skip_identical': True, 'leave_segments': True, 'header': '', 'segment_size': 10}) self.assertIsNone(r.get('error')) self.assertTrue(r['success']) self.assertEqual('skipped-identical', r.get('status')) self.assertEqual(0, mock_conn.put_object.call_count) self.assertEqual([mock.call('test_c', 'test_o')], mock_conn.head_object.mock_calls) self.assertEqual([ mock.call('test_c', 'test_o', query_string='multipart-manifest=get'), mock.call('test_c_segments', 'test_sub_slo', query_string='multipart-manifest=get'), ], mock_conn.get_object.mock_calls)
def test_delete_object(self): mock_q = Queue() mock_conn = self._get_mock_connection() mock_conn.head_object = Mock(return_value={}) expected_r = self._get_expected({ 'action': 'delete_object', 'success': True }) s = SwiftService() r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) mock_conn.head_object.assert_called_once_with('test_c', 'test_o') mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={} ) self._assertDictEqual(expected_r, r)
def test_delete_empty_container_excpetion(self): mock_conn = self._get_mock_connection() mock_conn.delete_container = Mock(side_effect=self.exc) expected_r = self._get_expected({ 'action': 'delete_container', 'success': False, 'object': None, 'error': self.exc }) s = SwiftService() r = s._delete_empty_container(mock_conn, 'test_c') mock_conn.delete_container.assert_called_once_with( 'test_c', response_dict={} ) self._assertDictEqual(expected_r, r)
def test_upload_object_job_etag_mismatch(self): # The etag test for both streams and files use the same code # so only one test should be needed. def _consuming_conn(*a, **kw): contents = a[2] contents.read() # Force md5 calculation return 'badresponseetag' with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 30) f.flush() mock_conn = mock.Mock() mock_conn.put_object.side_effect = _consuming_conn type(mock_conn).attempts = mock.PropertyMock(return_value=2) s = SwiftService() r = s._upload_object_job(conn=mock_conn, container='test_c', source=f.name, obj='test_o', options={ 'changed': False, 'skip_identical': False, 'leave_segments': True, 'header': '', 'segment_size': 0, 'checksum': True }) self.assertEqual(r['success'], False) self.assertIn('error', r) self.assertIn('md5 mismatch', str(r['error'])) self.assertEqual(mock_conn.put_object.call_count, 1) expected_headers = {'x-object-meta-mtime': mock.ANY} mock_conn.put_object.assert_called_with('test_c', 'test_o', mock.ANY, content_length=30, headers=expected_headers, response_dict={}) contents = mock_conn.put_object.call_args[0][2] self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())
def connection(self): options = None if self.swift_conn_id != "swift_default": options = {} connection = BaseHook.get_connection(self.swift_conn_id) options["os_username"] = connection.login options["os_password"] = connection.password options["os_tenant_name"] = connection.host yield SwiftService(options=options)
def handle(self, *args, **options): _opts = {'object_uu_threads': 20} dir = 'media' # settings.MEDIA_ROOT container = settings.SWIFT_CONTAINER_NAME with SwiftService( options=_opts) as swift, OutputManager() as out_manager: try: # Collect all the files and folders in the given directory objs = [] dir_markers = [] # change to directory so it isn't uploaded as part of all the # file object names os.chdir(dir) for (_dir, _ds, _fs) in os.walk('.'): if not (_ds + _fs): dir_markers.append(_dir) else: objs.extend([os.path.join(_dir, _f) for _f in _fs]) # Now that we've collected all the required files and dir markers # build the ``SwiftUploadObject``s for the call to upload objs = [ SwiftUploadObject(o, object_name=o.replace(dir, dir, 1)) for o in objs ] dir_markers = [ SwiftUploadObject(None, object_name=d.replace(dir, dir, 1), options={'dir_marker': True}) for d in dir_markers ] # Schedule uploads on the SwiftService thread pool and iterate # over the results for r in swift.upload(container, objs + dir_markers): if r['success']: if 'object' in r: print(r['object']) elif 'for_object' in r: print('%s segment %s' % (r['for_object'], r['segment_index'])) else: error = r['error'] if r['action'] == "create_container": logger.warning( 'Warning: failed to create container ' "'%s'%s", container, error) elif r['action'] == "upload_object": logger.error( "Failed to upload object %s to container %s: %s" % (container, r['object'], error)) else: logger.error("%s" % error) os.chdir('..') except SwiftError as e: logger.error(e.value)
def get_service_client_v1(tenant_name, user_name, key, auth_url, options={}): user = tenant_name + ':' + user_name return SwiftService(options=dict( { "auth_version": 1, "user": user, "key": key, "auth": auth_url, }, **options))
def test_delete_object(self): mock_q = Queue() mock_conn = self._get_mock_connection() mock_conn.head_object = Mock(return_value={}) expected_r = self._get_expected({ 'action': 'delete_object', 'success': True }) s = SwiftService() r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) mock_conn.head_object.assert_called_once_with('test_c', 'test_o') mock_conn.delete_object.assert_called_once_with('test_c', 'test_o', query_string=None, response_dict={}) self._assertDictEqual(expected_r, r)
def upload_results(results_folder, destination_container): SETTINGS_LOCATION = os.environ.get('SWIFT_SETTINGS') if not SETTINGS_LOCATION: raise RuntimeError("No OpenStack settings provided for the upload") with open(SETTINGS_LOCATION, 'r') as sf: settings = json.load(sf) # generate filelist from results folder files = [(os.path.join(dp, f)) for dp, dn, fn in os.walk(results_folder) for f in fn] labels = convert_filenames_to_labels(results_folder, files) options = { 'object_uu_threads': 20, 'os_auth_token': get_keystone_token(settings) } # Remove password from the OpenStack settings, otherwise the SwiftService # connection will be built ignoring the token and failing authentication del (settings['os_password']) options.update(**settings) def generate_options(label): if label in ['/info', '/info_fullres.json', '/transform.json']: return {} else: return {'header': ['Content-Encoding:gzip']} with SwiftService( options=options) as swift, OutputManager() as out_manager: try: logger.info(F'Making public container {destination_container}') swift.post(container=destination_container, options={ 'read_acl': '.r:*,.rlistings', 'header': ['X-Container-Meta-Access-Control-Allow-Origin: *'] }) logger.info( f'Uploading {len(files)} objects to {destination_container}') start = perf_counter() objects = [ SwiftUploadObject(file, object_name=labels[index], options=generate_options(labels[index])) for index, file in enumerate(files) ] for result in swift.upload(destination_container, objects): if not result['success']: logger.error(f"Failed to upload object") raise RuntimeError("Failed to upload object") finish = perf_counter() logger.info(f'Completed in {timedelta(seconds=finish-start)}') except SwiftError as e: logger.exception(e.value) raise RuntimeError("Failed to upload objects")
def test_upload_object_job_etag_mismatch(self): # The etag test for both streams and files use the same code # so only one test should be needed. def _consuming_conn(*a, **kw): contents = a[2] contents.read() # Force md5 calculation return 'badresponseetag' with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 30) f.flush() mock_conn = mock.Mock() mock_conn.put_object.side_effect = _consuming_conn type(mock_conn).attempts = mock.PropertyMock(return_value=2) s = SwiftService() r = s._upload_object_job(conn=mock_conn, container='test_c', source=f.name, obj='test_o', options={'changed': False, 'skip_identical': False, 'leave_segments': True, 'header': '', 'segment_size': 0, 'checksum': True}) self.assertEqual(r['success'], False) self.assertIn('error', r) self.assertIn('md5 mismatch', str(r['error'])) self.assertEqual(mock_conn.put_object.call_count, 1) expected_headers = {'x-object-meta-mtime': mock.ANY} mock_conn.put_object.assert_called_with('test_c', 'test_o', mock.ANY, content_length=30, headers=expected_headers, response_dict={}) contents = mock_conn.put_object.call_args[0][2] self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())
def __init__(self, container, topic, kafka): self.container = container self.swift = SwiftService() self.out = OutputManager() self.kafka = kafka auth = get_auth() self.url = auth[0] self.token = auth[1] self.topic = topic self.producer = KafkaProducer(bootstrap_servers=kafka)
def test_delete_object_exception(self): mock_q = Queue() mock_conn = self._get_mock_connection() mock_conn.delete_object = Mock(side_effect=self.exc) expected_r = self._get_expected({ 'action': 'delete_object', 'success': False, 'error': self.exc }) # _delete_object doesnt populate attempts or response dict if it hits # an error. This may not be the correct behaviour. del expected_r['response_dict'], expected_r['attempts'] s = SwiftService() r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) mock_conn.head_object.assert_called_once_with('test_c', 'test_o') mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={} ) self._assertDictEqual(expected_r, r)
def test_upload_segment_job_etag_mismatch(self): def _consuming_conn(*a, **kw): contents = a[2] contents.read() # Force md5 calculation return 'badresponseetag' with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 10) f.write(b'b' * 10) f.write(b'c' * 10) f.flush() mock_conn = mock.Mock() mock_conn.put_object.side_effect = _consuming_conn type(mock_conn).attempts = mock.PropertyMock(return_value=2) s = SwiftService() r = s._upload_segment_job(conn=mock_conn, path=f.name, container='test_c', segment_name='test_s_1', segment_start=10, segment_size=10, segment_index=2, obj_name='test_o', options={'segment_container': None, 'checksum': True}) self.assertIn('error', r) self.assertIn('md5 mismatch', str(r['error'])) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c_segments', 'test_s_1', mock.ANY, content_length=10, response_dict={}) contents = mock_conn.put_object.call_args[0][2] self.assertEqual(contents.get_md5sum(), md5(b'b' * 10).hexdigest())
def ck_cdn(self, secrets): if self.client.rax_id: rax_id = self.client.rax_id p_okg("client.rax_id: {}".format(rax_id)) else: p_warn("client.rax_id not set.") return if self.client.bucket_id: bucket_id = self.client.bucket_id p_okg("client.bucket_id: {}".format(bucket_id)) else: p_fail("client.bucket_id not set.") print("logging in...") uploader = swift_uploader.Uploader() # pprint.pprint(pw.swift[rax_id]) uploader.user=rax_id cf = uploader.auth() print("checking for valid bucket...") from swiftclient.service import SwiftService swift = SwiftService() stats_it = swift.stat(container=bucket_id) """ containers = cf.get_all_containers() container_names = [container.name for container in containers] print("container_names", container_names) if bucket_id in container_names: p_okg('"{}" found.'.format(bucket_id)) else: p_fail('"{}" not found.'.format(bucket_id)) """ # not sure what to do with this... # container = cf.get_container(bucket_id) return
def test_delete_empty_container(self): mock_conn = self._get_mock_connection() expected_r = self._get_expected({ 'action': 'delete_container', 'success': True, 'object': None }) r = SwiftService._delete_empty_container(mock_conn, 'test_c') mock_conn.delete_container.assert_called_once_with( 'test_c', response_dict={} ) self._assertDictEqual(expected_r, r)
def test_delete_object_slo_support(self): # If SLO headers are present the delete call should include an # additional query string to cause the right delete server side mock_q = Queue() mock_conn = self._get_mock_connection() mock_conn.head_object = Mock( return_value={'x-static-large-object': True} ) expected_r = self._get_expected({ 'action': 'delete_object', 'success': True }) s = SwiftService() r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) mock_conn.head_object.assert_called_once_with('test_c', 'test_o') mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string='multipart-manifest=delete', response_dict={} ) self._assertDictEqual(expected_r, r)
def test_delete_segment(self): mock_q = Queue() mock_conn = self._get_mock_connection() expected_r = self._get_expected({ 'action': 'delete_segment', 'object': 'test_s', 'success': True, }) r = SwiftService._delete_segment(mock_conn, 'test_c', 'test_s', mock_q) mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_s', response_dict={} ) self._assertDictEqual(expected_r, r) self._assertDictEqual(expected_r, self._get_queue(mock_q))
def getObjsBackend(objs, backend, config): if(backend == 'hdfs'): client = Client(socket.gethostname(), config['HADOOP_RPC_PORT'], use_trash=False) for obj in objs: try: copy_gen = client.copyToLocal([obj[0]], obj[1]) for copy_item in copy_gen: pass except Exception as e: print(e) elif(backend == 'swift'): options = {'os_auth_url': os.environ['OS_AUTH_URL'], 'os_username': os.environ['OS_USERNAME'], 'os_password': os.environ['OS_PASSWORD'], 'os_tenant_id': os.environ['OS_TENANT_ID'], 'os_tenant_name': os.environ['OS_TENANT_NAME']} swiftService = SwiftService(options=options) for obj in objs: # Create the containers which are used in this application for Object Storage if(obj[0] == 'sqlite.db'): swiftService.post(container='containerFiles') swiftService.post(container='containerFeatures') swiftService.post(container='containerModules') out_file = obj[1] # Get the output file location from runner localoptions = {'out_file': out_file} objects = [] objects.append(obj[0]) swiftDownload = swiftService.download(container='containerModules', objects=objects, options=localoptions) for downloaded in swiftDownload: if("error" in downloaded.keys()): raise RuntimeError(downloaded['error']) # print(downloaded) elif(backend == 'nfs'): # Every file is already in respective local dirs pass
def __init__(self, container, prefix=''): self.container = container self.prefix = prefix self.client = SwiftService()
def download(source, dest=None): """ Download files from a SWIFT object store @param source_info: Source information such as the source to download from. @type source_info: a dictionary @param local_dest_dir: The local directory to store file @type local_dest_dir: str @return: True and a list of file downloaded if successful. Otherwise False. """ if not dest: dest = tempfile.mkstemp() url = urlsplit(source['url']) _, ver, account, container, object_name = url.path.split('/', 4) swift_opts = { 'os_storage_url': '{scheme}://{netloc}/{ver}/{account}'.format( scheme=re.sub(r'^swift\+', '', url.scheme), netloc=url.netloc, ver=ver, account=account) } # SwiftService knows about environment variables for opt in ('os_auth_url', 'os_username', 'os_password', 'os_tenant_name', 'os_storage_url'): if opt in source: swift_opts[opt] = source[opt] try: swift = SwiftService(swift_opts) filelist = [] if os.path.exists(dest) and os.path.isdir(dest): outfilename = os.path.join(dest, os.path.basename(object_name)) else: outfilename = dest retries = 5 # number of retries left backoff = 30 # wait time between retries backoff_inc = 30 # increase in wait time per retry while retries: filelist = [] retries -= 1 try: for result in swift.download(container, [object_name], {'out_file': outfilename}): # result dict: success # action: 'download_object' # success: True # container: ... # object: ... # path: .... # pseudodir: ... # start_time, finish_time, headers_receipt, auth_end_time, # read_length, attempts, response_dict # result dict: error # action: 'download_object' # success: False # error: ... # traceback: ... # container, object, error_timestamp, response_dict, path # psudodir, attempts if not result['success']: raise Exception( 'Download from selfelfwift {container}/{object} to {out_file} failed with {error}'.format(out_file=outfilename, **result)) outfile = {'url': outfilename, 'name': os.path.basename(object_name), 'content_type': result['response_dict']['headers'].get('content-type', 'application/octet-stream')} filelist.append(outfile) # no exception we can continue retries = 0 except Exception as e: if not retries: # reraise if no retries left raise LOG.warn("Download from Swift failed: %s - %d retries left", e, retries) time.sleep(backoff) backoff += backoff_inc backoff_inc += 30 return filelist except Exception as e: LOG.error("Download from Swift failed: %s", e, exc_info=True) raise