class SwiftClient(object):
"""Client for Swift object/blob store of Openstack
See http://swift.openstack.org
Swift requires environment variables (OS_*) for the authentication and configuration"""
def __init__(self, container, prefix=''):
self.container = container
self.prefix = prefix
self.client = SwiftService()
def download(self, source, target):
objects = [self.prefix + '/' + source]
options = {'out_file': target}
return list(self.client.download(self.container, objects, options))
def upload(self, source, target):
object_name = self.prefix + '/' + target
objects = [SwiftUploadObject(source, object_name=object_name)]
return list(self.client.upload(self.container, objects))
def ls(self, path):
fpath = self.prefix + '/' + path + '/'
clisting = self.client.list(self.container, {'prefix': fpath})
listing = list(clisting)[0]['listing']
result = [d['name'].replace(fpath, '') for d in listing]
return result
def url(self, path=''):
return self.container + '/' + self.prefix + '/' + path
def test_delete_object_dlo_support(self):
mock_q = Queue()
s = SwiftService()
mock_conn = self._get_mock_connection()
expected_r = self._get_expected({
'action': 'delete_object',
'success': True,
'dlo_segments_deleted': True
})
# A DLO object is determined in _delete_object by heading the object
# and checking for the existence of a x-object-manifest header.
# Mock that here.
mock_conn.head_object = Mock(
return_value={'x-object-manifest': 'manifest_c/manifest_p'}
)
mock_conn.get_container = Mock(
side_effect=[(None, [{'name': 'test_seg_1'},
{'name': 'test_seg_2'}]),
(None, {})]
)
def get_mock_list_conn(options):
return mock_conn
with mock.patch('swiftclient.service.get_conn', get_mock_list_conn):
r = s._delete_object(
mock_conn, 'test_c', 'test_o', self.opts, mock_q
)
self._assertDictEqual(expected_r, r)
expected = [
mock.call('test_c', 'test_o', query_string=None, response_dict={}),
mock.call('manifest_c', 'test_seg_1', response_dict={}),
mock.call('manifest_c', 'test_seg_2', response_dict={})]
mock_conn.delete_object.assert_has_calls(expected, any_order=True)
def saveObjsBackend(objs, backend, config):
if(backend == 'hdfs'):
for obj in objs:
try:
# obj[0] is hdfs path and obj[1] is local filesystem path
if '.py' in obj[1]: # If you are uploading a module do not allow it to be overwritten
subprocess.check_call(['hdfs', 'dfs', '-copyFromLocal', obj[1], obj[0]])
else: # If it is the metadata then it has to be overwritten everytime
subprocess.check_call(['hdfs', 'dfs', '-copyFromLocal', '-f', obj[1], obj[0]])
except Exception as e:
shutil.copyfile(config['BACKUP_METADATA_LOCAL_PATH'], config['METADATA_LOCAL_PATH'])
raise RuntimeError(e)
elif(backend == 'swift'):
options = {'os_auth_url': os.environ['OS_AUTH_URL'], 'os_username': os.environ['OS_USERNAME'], 'os_password': os.environ['OS_PASSWORD'], 'os_tenant_id': os.environ['OS_TENANT_ID'], 'os_tenant_name': os .environ['OS_TENANT_NAME']}
swiftService = SwiftService(options=options)
objects = []
for obj in objs:
objects.append(SwiftUploadObject(obj[1], object_name=obj[0]))
swiftUpload = swiftService.upload(container='containerModules', objects=objects)
for uploaded in swiftUpload:
if("error" in uploaded.keys()):
shutil.copyfile(config['BACKUP_METADATA_LOCAL_PATH'], config['METADATA_LOCAL_PATH'])
raise RuntimeError(uploaded['error'])
elif(backend == 'nfs'):
for obj in objs:
shutil.copyfile(obj[1], config['MODULES_DIR'] + obj[0])
print('Metadata/Module changed and uploaded')
def test_delete_object_dlo_support(self):
mock_q = Queue()
s = SwiftService()
mock_conn = self._get_mock_connection()
expected_r = self._get_expected({
'action': 'delete_object',
'success': True,
'dlo_segments_deleted': True
})
# A DLO object is determined in _delete_object by heading the object
# and checking for the existence of a x-object-manifest header.
# Mock that here.
mock_conn.head_object = Mock(
return_value={'x-object-manifest': 'manifest_c/manifest_p'})
mock_conn.get_container = Mock(side_effect=[(None,
[{
'name': 'test_seg_1'
}, {
'name': 'test_seg_2'
}]), (None, {})])
def get_mock_list_conn(options):
return mock_conn
with mock.patch('swiftclient.service.get_conn', get_mock_list_conn):
r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts,
mock_q)
self._assertDictEqual(expected_r, r)
expected = [
mock.call('test_c', 'test_o', query_string=None, response_dict={}),
mock.call('manifest_c', 'test_seg_1', response_dict={}),
mock.call('manifest_c', 'test_seg_2', response_dict={})
]
mock_conn.delete_object.assert_has_calls(expected, any_order=True)
def test_upload_object_job_identical_etag(self):
with tempfile.NamedTemporaryFile() as f:
f.write(b'a' * 30)
f.flush()
mock_conn = mock.Mock()
mock_conn.head_object.return_value = {
'content-length': 30,
'etag': md5(b'a' * 30).hexdigest()}
type(mock_conn).attempts = mock.PropertyMock(return_value=2)
s = SwiftService()
r = s._upload_object_job(conn=mock_conn,
container='test_c',
source=f.name,
obj='test_o',
options={'changed': False,
'skip_identical': True,
'leave_segments': True,
'header': '',
'segment_size': 0})
self.assertTrue(r['success'])
self.assertIn('status', r)
self.assertEqual(r['status'], 'skipped-identical')
self.assertEqual(mock_conn.put_object.call_count, 0)
self.assertEqual(mock_conn.head_object.call_count, 1)
mock_conn.head_object.assert_called_with('test_c', 'test_o')
def test_upload_object_job_file(self):
# Uploading a file results in the file object being wrapped in a
# LengthWrapper. This test sets the options in such a way that much
# of _upload_object_job is skipped bringing the critical path down
# to around 60 lines to ease testing.
with tempfile.NamedTemporaryFile() as f:
f.write(b'a' * 30)
f.flush()
expected_r = {
'action': 'upload_object',
'attempts': 2,
'container': 'test_c',
'headers': {},
'large_object': False,
'object': 'test_o',
'response_dict': {},
'status': 'uploaded',
'success': True,
}
expected_mtime = float(os.path.getmtime(f.name))
mock_conn = mock.Mock()
mock_conn.put_object.return_value = ''
type(mock_conn).attempts = mock.PropertyMock(return_value=2)
s = SwiftService()
r = s._upload_object_job(conn=mock_conn,
container='test_c',
source=f.name,
obj='test_o',
options={'changed': False,
'skip_identical': False,
'leave_segments': True,
'header': '',
'segment_size': 0,
'checksum': True})
mtime = float(r['headers']['x-object-meta-mtime'])
self.assertAlmostEqual(mtime, expected_mtime, delta=0.5)
del r['headers']['x-object-meta-mtime']
self.assertEqual(r['path'], f.name)
del r['path']
self._assertDictEqual(r, expected_r)
self.assertEqual(mock_conn.put_object.call_count, 1)
mock_conn.put_object.assert_called_with('test_c', 'test_o',
mock.ANY,
content_length=30,
headers={},
response_dict={})
contents = mock_conn.put_object.call_args[0][2]
self.assertIsInstance(contents, utils.LengthWrapper)
self.assertEqual(len(contents), 30)
# This read forces the LengthWrapper to calculate the md5
# for the read content. This also checks that LengthWrapper was
# initialized with md5=True
self.assertEqual(contents.read(), b'a' * 30)
self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())
def __init__(self, settings):
options = get_os_settings(settings)
self.swift = SwiftService(options)
options = get_swift_settings(settings)
self.temp_url_key = options['temp_url_key']
# TODO: hard coded template
self.name_template = options['container'] + '_{name}'
def upload(source, dest):
"""
Upload file to a remote SWIFT store
@param source: List of local source path to upload from.
@type source : Dicrionary
@param dest: The destination information such as destination url to upload the file.
@type dest: Dictionary
@return: True if upload is successful. Otherwise False.
"""
url = urlsplit(dest['url'])
_, ver, account, container, object_name = url.path.split('/', 4)
swift_opts = {
'os_storage_url': '{scheme}://{netloc}/{ver}/{account}'.format(
scheme=re.sub(r'^swift\+', '', url.scheme),
netloc=url.netloc,
ver=ver,
account=account)
}
# SwiftService knows about environment variables
for opt in ('os_auth_url', 'os_username', 'os_password', 'os_tenant_name', 'os_storage_url'):
if opt in dest:
swift_opts[opt] = dest[opt]
try:
swift = SwiftService(swift_opts)
headers = []
if 'content_type' in source:
headers.append('Content-Type: {}'.format(source['content_type']))
retries = 5 # number of retries left
backoff = 30 # wait time between retries
backoff_inc = 30 # increase in wait time per retry
while retries:
retries -= 1
try:
for result in swift.upload(container, [SwiftUploadObject(source['url'], object_name=object_name, options={'header': headers})]):
# TODO: we may get result['action'] = 'create_container'
# self.assertNotIn(member, container)d result['action'] = 'upload_object'; result['path'] =
# source['url']
if not result['success']:
raise Exception(
'Upload to Swift {container}/{object_name} failed with {error}'.format(object_name=object_name, **result))
# no exception we can continue
retries = 0
except Exception as e:
if not retries:
# reraise if no retries left
raise
LOG.warn('Upload to Swift failed: %s - %d retries left', e, retries)
time.sleep(backoff)
backoff += backoff_inc
backoff_inc += 30
except Exception as e:
LOG.error("Upload to swift failed: %s", e, exc_info=True)
raise
def test_upload_object_job_stream(self):
# Streams are wrapped as ReadableToIterable
with tempfile.TemporaryFile() as f:
f.write(b'a' * 30)
f.flush()
f.seek(0)
expected_r = {
'action': 'upload_object',
'attempts': 2,
'container': 'test_c',
'headers': {},
'large_object': False,
'object': 'test_o',
'response_dict': {},
'status': 'uploaded',
'success': True,
'path': None,
}
expected_mtime = float(time.time())
mock_conn = mock.Mock()
mock_conn.put_object.return_value = ''
type(mock_conn).attempts = mock.PropertyMock(return_value=2)
s = SwiftService()
r = s._upload_object_job(conn=mock_conn,
container='test_c',
source=f,
obj='test_o',
options={
'changed': False,
'skip_identical': False,
'leave_segments': True,
'header': '',
'segment_size': 0,
'checksum': True
})
mtime = float(r['headers']['x-object-meta-mtime'])
self.assertAlmostEqual(mtime, expected_mtime, delta=0.5)
del r['headers']['x-object-meta-mtime']
self._assertDictEqual(r, expected_r)
self.assertEqual(mock_conn.put_object.call_count, 1)
mock_conn.put_object.assert_called_with('test_c',
'test_o',
mock.ANY,
content_length=None,
headers={},
response_dict={})
contents = mock_conn.put_object.call_args[0][2]
self.assertIsInstance(contents, utils.ReadableToIterable)
self.assertEqual(contents.chunk_size, 65536)
# next retrieves the first chunk of the stream or len(chunk_size)
# or less, it also forces the md5 to be calculated.
self.assertEqual(next(contents), b'a' * 30)
self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())
def __init__(self, storage_id, container_name, auth_config=None, transfer_config=None):
super(SwiftStorage, self).__init__(storage_id)
opts = transfer_config or {}
opts["auth_version"] = "2.0"
if auth_config:
for k, v in six.iteritems(auth_config):
opts[k] = v
self._client = SwiftService(opts)
self._container = container_name
def test_upload_with_bad_segment_size(self):
for bad in ('ten', '1234X', '100.3'):
options = {'segment_size': bad}
try:
service = SwiftService(options)
next(service.upload('c', 'o'))
self.fail('Expected SwiftError when segment_size=%s' % bad)
except SwiftError as exc:
self.assertEqual('Segment size should be an integer value',
exc.value)
def test_upload_with_bad_segment_size(self):
for bad in ('ten', '1234X', '100.3'):
options = {'segment_size': bad}
try:
service = SwiftService(options)
next(service.upload('c', 'o'))
self.fail('Expected SwiftError when segment_size=%s' % bad)
except SwiftError as exc:
self.assertEqual('Segment size should be an integer value',
exc.value)
class DiracStore(object):
def __init__(self, container, topic, kafka):
self.container = container
self.swift = SwiftService()
self.out = OutputManager()
self.kafka = kafka
auth = get_auth()
self.url = auth[0]
self.token = auth[1]
self.topic = topic
self.producer = KafkaProducer(bootstrap_servers=kafka)
def send(self, resource):
json = message_to_json(resource)
print("sending " + json)
self.producer.send(self.topic, json.encode('utf-8'))
def make_resource(self, stat):
resource = schema.Resource()
resource.type = 'file'
resource.name = stat['Object']
resource.location = self.url + '/' + self.container + '/' + stat['Object']
resource.mimeType = stat['Content Type']
resource.size = long(stat['Content Length'])
resource.created = stat['Last Modified']
return resource
def stat(self, paths):
stat = {}
for response in self.swift.stat(container=self.container, objects=paths):
if response['success']:
stat[response['object']] = {item[0]: item[1] for item in response['items']}
return stat
def store(self, path, source):
isdir = os.path.isdir(source)
base = source if isdir else os.path.dirname(source)
sources = os.listdir(source) if isdir else [source]
locations = [os.path.join(path, os.path.basename(file)) for file in sources]
print(str(len(locations)) + " locations!")
stats = self.stat(locations)
objs = [SwiftUploadObject(os.path.join(base, os.path.basename(location)), object_name=location) for location in locations if not location in stats]
print(str(len(objs)) + " previously unseen!")
for response in self.swift.upload(self.container, objs):
if response['success']:
if 'object' in response:
print('uploading ' + response['object'])
stat = self.stat([response['object']])
resource = self.make_resource(stat.values()[0])
self.send(resource)
def test_upload_segment_job(self):
with tempfile.NamedTemporaryFile() as f:
f.write(b'a' * 10)
f.write(b'b' * 10)
f.write(b'c' * 10)
f.flush()
# Mock the connection to return an empty etag. This
# skips etag validation which would fail as the LengthWrapper
# isnt read from.
mock_conn = mock.Mock()
mock_conn.put_object.return_value = ''
type(mock_conn).attempts = mock.PropertyMock(return_value=2)
expected_r = {
'action': 'upload_segment',
'for_object': 'test_o',
'segment_index': 2,
'segment_size': 10,
'segment_location': '/test_c_segments/test_s_1',
'log_line': 'test_o segment 2',
'success': True,
'response_dict': {},
'segment_etag': '',
'attempts': 2,
}
s = SwiftService()
r = s._upload_segment_job(conn=mock_conn,
path=f.name,
container='test_c',
segment_name='test_s_1',
segment_start=10,
segment_size=10,
segment_index=2,
obj_name='test_o',
options={
'segment_container': None,
'checksum': True
})
self._assertDictEqual(r, expected_r)
self.assertEqual(mock_conn.put_object.call_count, 1)
mock_conn.put_object.assert_called_with('test_c_segments',
'test_s_1',
mock.ANY,
content_length=10,
response_dict={})
contents = mock_conn.put_object.call_args[0][2]
self.assertIsInstance(contents, utils.LengthWrapper)
self.assertEqual(len(contents), 10)
# This read forces the LengthWrapper to calculate the md5
# for the read content.
self.assertEqual(contents.read(), b'b' * 10)
self.assertEqual(contents.get_md5sum(), md5(b'b' * 10).hexdigest())
def test_upload_object_job_stream(self):
# Streams are wrapped as ReadableToIterable
with tempfile.TemporaryFile() as f:
f.write(b'a' * 30)
f.flush()
f.seek(0)
expected_r = {
'action': 'upload_object',
'attempts': 2,
'container': 'test_c',
'headers': {},
'large_object': False,
'object': 'test_o',
'response_dict': {},
'status': 'uploaded',
'success': True,
'path': None,
}
expected_mtime = float(time.time())
mock_conn = mock.Mock()
mock_conn.put_object.return_value = ''
type(mock_conn).attempts = mock.PropertyMock(return_value=2)
s = SwiftService()
r = s._upload_object_job(conn=mock_conn,
container='test_c',
source=f,
obj='test_o',
options={'changed': False,
'skip_identical': False,
'leave_segments': True,
'header': '',
'segment_size': 0,
'checksum': True})
mtime = float(r['headers']['x-object-meta-mtime'])
self.assertAlmostEqual(mtime, expected_mtime, delta=0.5)
del r['headers']['x-object-meta-mtime']
self._assertDictEqual(r, expected_r)
self.assertEqual(mock_conn.put_object.call_count, 1)
mock_conn.put_object.assert_called_with('test_c', 'test_o',
mock.ANY,
content_length=None,
headers={},
response_dict={})
contents = mock_conn.put_object.call_args[0][2]
self.assertIsInstance(contents, utils.ReadableToIterable)
self.assertEqual(contents.chunk_size, 65536)
# next retrieves the first chunk of the stream or len(chunk_size)
# or less, it also forces the md5 to be calculated.
self.assertEqual(next(contents), b'a' * 30)
self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())
def test_upload_segment_job(self):
with tempfile.NamedTemporaryFile() as f:
f.write(b'a' * 10)
f.write(b'b' * 10)
f.write(b'c' * 10)
f.flush()
# Mock the connection to return an empty etag. This
# skips etag validation which would fail as the LengthWrapper
# isnt read from.
mock_conn = mock.Mock()
mock_conn.put_object.return_value = ''
type(mock_conn).attempts = mock.PropertyMock(return_value=2)
expected_r = {
'action': 'upload_segment',
'for_object': 'test_o',
'segment_index': 2,
'segment_size': 10,
'segment_location': '/test_c_segments/test_s_1',
'log_line': 'test_o segment 2',
'success': True,
'response_dict': {},
'segment_etag': '',
'attempts': 2,
}
s = SwiftService()
r = s._upload_segment_job(conn=mock_conn,
path=f.name,
container='test_c',
segment_name='test_s_1',
segment_start=10,
segment_size=10,
segment_index=2,
obj_name='test_o',
options={'segment_container': None,
'checksum': True})
self._assertDictEqual(r, expected_r)
self.assertEqual(mock_conn.put_object.call_count, 1)
mock_conn.put_object.assert_called_with('test_c_segments',
'test_s_1',
mock.ANY,
content_length=10,
response_dict={})
contents = mock_conn.put_object.call_args[0][2]
self.assertIsInstance(contents, utils.LengthWrapper)
self.assertEqual(len(contents), 10)
# This read forces the LengthWrapper to calculate the md5
# for the read content.
self.assertEqual(contents.read(), b'b' * 10)
self.assertEqual(contents.get_md5sum(), md5(b'b' * 10).hexdigest())
def list_container(container, prefix):
obj_list = []
myoptions={'prefix': prefix}
stats=SwiftService(options=_default_global_options).stat()
if not stats["success"] == True:
print('not authenticated')
return False
with SwiftService(options=_default_global_options) as swift:
listing=swift.list(container=container, options=myoptions)
for o in listing:
for i in o['listing']:
obj_list.append(i['name'])
return obj_list
def test_upload_object_job_identical_dlo(self):
with tempfile.NamedTemporaryFile() as f:
f.write(b'a' * 30)
f.flush()
segment_etag = md5(b'a' * 10).hexdigest()
mock_conn = mock.Mock()
mock_conn.head_object.return_value = {
'x-object-manifest': 'test_c_segments/test_o/prefix',
'content-length': 30,
'etag': md5(segment_etag.encode('ascii') * 3).hexdigest()}
mock_conn.get_container.side_effect = [
(None, [{"bytes": 10, "hash": segment_etag,
"name": "test_o/prefix/00"},
{"bytes": 10, "hash": segment_etag,
"name": "test_o/prefix/01"}]),
(None, [{"bytes": 10, "hash": segment_etag,
"name": "test_o/prefix/02"}]),
(None, {})]
type(mock_conn).attempts = mock.PropertyMock(return_value=2)
s = SwiftService()
with mock.patch('swiftclient.service.get_conn',
return_value=mock_conn):
r = s._upload_object_job(conn=mock_conn,
container='test_c',
source=f.name,
obj='test_o',
options={'changed': False,
'skip_identical': True,
'leave_segments': True,
'header': '',
'segment_size': 10})
self.assertIsNone(r.get('error'))
self.assertTrue(r['success'])
self.assertEqual('skipped-identical', r.get('status'))
self.assertEqual(0, mock_conn.put_object.call_count)
self.assertEqual(1, mock_conn.head_object.call_count)
self.assertEqual(3, mock_conn.get_container.call_count)
mock_conn.head_object.assert_called_with('test_c', 'test_o')
expected = [
mock.call('test_c_segments', prefix='test_o/prefix',
marker='', delimiter=None),
mock.call('test_c_segments', prefix='test_o/prefix',
marker="test_o/prefix/01", delimiter=None),
mock.call('test_c_segments', prefix='test_o/prefix',
marker="test_o/prefix/02", delimiter=None),
]
mock_conn.get_container.assert_has_calls(expected)
async def get_object_metadata(
conn: SwiftService, meta_cont: str, meta_obj: typing.Union[typing.List[str], None]
) -> typing.List[dict]:
"""Get object metadata."""
try:
res = list(conn.stat(meta_cont, meta_obj))
# Fail if an object wasn't usable
if False in [i["success"] for i in res]:
raise aiohttp.web.HTTPNotFound()
# Filter for metadata not already served with the list request
res = [
[
i["object"],
dict(filter(lambda j: "x-object-meta" in j[0], i["headers"].items())),
]
for i in res
]
# Strip unnecessary specifcations from header names and split open s3
# information so that it doesn't have to be done in the browser
for i in res:
i[1] = {k.replace("x-object-meta-", ""): v for k, v in i[1].items()}
if "s3cmd-attrs" in i[1].keys():
i[1]["s3cmd-attrs"] = {
k: v
for k, v in [j.split(":") for j in i[1]["s3cmd-attrs"].split("/")]
}
return res
except SwiftError:
# Fail if container wasn't found
raise aiohttp.web.HTTPNotFound()
def upload(self, input_filename, path_to_file, **kwargs):
# fp = open(filename, "rb")
# self.con.put_object(self.container, keyname, fp)
import swiftclient.service
from swiftclient.service import SwiftService
from swiftclient.service import SwiftUploadObject
swiftclient.service._default_global_options['os_auth_url'] = self.conf[
"auth_url"]
swiftclient.service._default_global_options['os_username'] = self.conf[
"access_key"]
swiftclient.service._default_global_options['os_password'] = self.conf[
"secret_key"]
swiftclient.service._default_global_options[
'os_tenant_name'] = self.conf["tenant_name"]
swiftclient.service._default_global_options[
'auth_version'] = self.conf["auth_version"]
swiftclient.service._default_global_options['insecure'] = self.conf[
"insecure"]
if 'segment_size' not in self.conf:
self.conf["segment_size"] = '1073741824'
if 'chunk_size' not in self.conf:
self.conf["chunk_size"] = '20480'
with SwiftService() as ss:
for result in ss.upload(
self.conf["s3_bucket"],
[SwiftUploadObject(path_to_file, input_filename)],
{'segment_size': self.conf["segment_size"]}):
if not str(result['success']):
self.print_upload_failure(path_to_file, result)
raise Exception('Upload Failure of File: ' + path_to_file)
def __init__(
self,
os_tenant_id,
os_tenant_name,
os_username,
os_password,
os_region_name,
os_storage_url,
os_user_domain_name=SWIFT_OS_USER_DOMAIN_NAME,
os_project_domain_name=SWIFT_OS_PROJECT_DOMAIN_NAME,
os_auth_url=SWIFT_OS_AUTH_URL,
os_identity_api_version=SWIFT_OS_IDENTITY_API_VERSION,
):
"""Setup options for the SwiftService"""
self.os_tenant_id = os_tenant_id
self.os_tenant_name = os_tenant_name
self.os_username = os_username
self.os_password = os_password
self.os_region_name = os_region_name
self.os_user_domain_name = os_user_domain_name
self.os_project_domain_name = os_project_domain_name
self.os_auth_url = os_auth_url
self.os_identity_api_version = os_identity_api_version
self.container = urlparse(os_storage_url).path.rpartition("/")[-1]
self.os_storage_url = os_storage_url.removesuffix(f"/{self.container}")
with SwiftService(self.options) as swift:
stats = swift.stat()
if not stats["success"]:
msg = "Unable to connect to the requested container: %s"
logger.error(msg, stats["error"])
raise BackendParameterException(msg % stats["error"])
def uploadfiles(container, options, objs):
# first argument is the container to upload file to
# second argument is a dictionary of the options to supply to SwiftUploadObject
# third argument is a list of the files to upload
with SwiftService() as swift:
try:
# create the SwiftUploadObject list of objects to upload
#objs = [SwiftUploadObject(obj, options=options) for obj in objs]
objs = [SwiftUploadObject(obj) for obj in objs]
# Schedule uploads on the SwiftService thread pool and iterate over the results
#for result in swift.upload(container, objs):
for result in swift.upload(container, objs):
if result['success']:
if 'object' in result:
print("Successfully uploaded %s." % result['object'])
elif 'for_object' in result:
print('%s segment %s' %
(result['for_object'], result['segment_index']))
else:
error = result['error']
if result['action'] == "create_container":
print('Warning: failed to create container '
"'%s'%s", container, error)
elif result['action'] == "upload_object":
print(
"Failed to upload object %s to container %s: %s" %
(container, result['object'], error))
else:
print("%s" % error)
except SwiftError as e:
print("SwiftError: %s" % e)
def upload_from_file(self, path):
"""
Stores the contents of the file pointed by the ``path`` variable.
If the file is bigger than 5 Gig, it will be broken into segments.
:type path: ``str``
:param path: Absolute path to the file to be uploaded to Swift.
:rtype: ``bool``
:return: ``True`` if successful, ``False`` if not.
.. note::
* The size of the segments chosen (or any of the other upload
options) is not under user control.
* If called this method will remap the
``swiftclient.service.get_conn`` factory method to
``self._provider._connect_swift``
.. seealso:: https://github.com/gvlproject/cloudbridge/issues/35#issuecomment-297629661 # noqa
"""
upload_options = {}
if 'segment_size' not in upload_options:
if os.path.getsize(path) >= FIVE_GIG:
upload_options['segment_size'] = FIVE_GIG
# remap the swift service's connection factory method
swiftclient.service.get_conn = self._provider._connect_swift
result = True
with SwiftService() as swift:
upload_object = SwiftUploadObject(path, object_name=self.name)
for up_res in swift.upload(self.cbcontainer.name,
[upload_object, ],
options=upload_options):
result = result and up_res['success']
return result
def execute(self, context):
Path(self.output_path).parents[0].mkdir(parents=True, exist_ok=True)
download_options = {
"out_file": self.output_path,
}
options = None
if self.conn_id is not None:
options = {}
connection = BaseHook.get_connection(self.conn_id)
options["os_username"] = connection.login
options["os_password"] = connection.password
options["os_tenant_name"] = connection.host
with SwiftService(options=options) as swift:
try:
for down_res in swift.download(
container=self.container,
objects=[self.object_id],
options=download_options,
):
if down_res["success"]:
self.log.info("downloaded: %s", down_res["object"])
else:
self.log.error("download failed: %s",
down_res["object"])
except SwiftError as e:
self.log.error(e.value)
def swift_metadata_create(container,
sciobj_name,
sciobj_metadata,
options=None):
"""
Post metadata to existing object, by first GET existing metadata then concatenating two meta
Input:
Container name
Object name
Metadata object
Output:
None
TODO: GET existing metadata
"""
#first retrieve existing metadata, otherwise, meta post will destroy existing one
with SwiftService(options=options) as swift:
try:
post_options = {"meta": sciobj_metadata}
r = swift.post(container=container,
objects=[sciobj_name],
options=post_options)
try:
for ri in r:
if not ri['success']:
print('object upload error')
print('error:%s', r['error'])
except Exception as e:
print('ri[' '] failed, ', e)
pass
except Exception as e:
print('stat error: ', e)
def mock_import_analysis(self, name='', description='', details='', filepath='', params='', inputs='', outputs=''):
filename = filepath
created = 'mm-dd-yyyy 00:00:00.000'
user = '******'
sessionQuery = self.session.query(Analysis).from_statement(text("SELECT * FROM analysis where name=:name")).\
params(name=name).first().side_effect = mod_se
checkMod = sessionQuery(name=name)
if(checkMod is None):
analysisMod = Mock(spec=Analysis(name=name, filepath=filename, description=description, details=details, created=created, user=user, parameters=params, inputs=inputs, outputs=outputs))
options = {'os_auth_url': self.config['SWIFT_AUTH_URL'], 'os_username': self.config['SWIFT_USERNAME'], 'os_password': self.config['SWIFT_PASSWORD'], 'os_tenant_id': self.config['SWIFT_TENANT_ID'], 'os_tenant_name': self.config['SWIFT_TENANT_NAME']}
swiftService = Mock(spec=SwiftService(options=options))
objects = []
objects.append(SwiftUploadObject(self.config['DB_LOCATION'], object_name='sqlite.db'))
objects.append(SwiftUploadObject(filepath, object_name=filename))
swiftUpload = swiftService.upload(container='containerModules', objects=objects).return_value = ('Metadata', 'Module')
for uploaded in swiftUpload:
print(uploaded)
self.session.add(analysisMod)
self.session.commit()
return "import_success"
else:
raise RuntimeError("Analysis " + name + " already exists")
def swift_delete(project_name):
obj = 0
cont = 0
err = 0
success = True
sw_opt = swift_options
sw_opt['os_project_name'] = project_name
with SwiftService(sw_opt) as swift:
for delete in swift.delete(options={
'yes_all': True,
}):
if delete['success']:
if delete['action'] == 'delete_object':
obj = obj + 1
if delete['action'] == 'delete_container':
cont = cont + 1
else:
err = err + 1
success = False
print('Warning! Delete error: {}'.delete)
result = 'Deleted objects: {}, containers: {}, errors: {}'.format(
obj, cont, err)
if success:
return result
else:
return False
def swift_update_quota(project_name, new_quota_bytes):
sw_opt = swift_options
sw_opt['os_project_name'] = project_name
with SwiftService(sw_opt) as swift:
post = swift.post(
options={'meta': ['quota-bytes:{}'.format(new_quota_bytes)]})
return post['success']
def create_swift_service(self, storage_url):
try:
return SwiftService(
options={
'user':
self.__keystoneCredentials['username'],
'key':
self.__keystoneCredentials['password'],
'os_project_id':
self.__keystoneCredentials['project_id'],
'os_project_name':
self.__keystoneCredentials['project_name'],
'os_project_domain_name':
self.__keystoneCredentials['user_domain_name'],
'os_tenant_id':
self.__keystoneCredentials['project_id'],
'os_tenant_name':
self.__keystoneCredentials['project_name'],
'os_user_domain_name':
self.__keystoneCredentials['user_domain_name'],
'os_auth_url':
self.__keystoneCredentials['auth_url'],
'os_cacert':
self.__certificatesPath,
'auth_version':
'3',
'os_storage_url':
storage_url
})
except Exception as error:
raise Exception('Connection to Swift failed: {}'.format(error))
def list_view(request):
# lowercase important
localconfig = { 'st_auth_version': 3,
'os_user_domain_name': 'admin_domain',
'os_username': '******',
'os_password': '******',
'os_auth_url': 'http://172.31.37.171:5000/v3'}
container = 'bsm1'
minimum_size = 10 * 1024 ** 2
items = []
with SwiftService(options=localconfig) as swift:
try:
list_parts_gen = swift.list(container=container)
for page in list_parts_gen:
if page["success"]:
for item in page["listing"]:
i_size = int(item["bytes"])
if i_size > minimum_size:
i_name = item["name"]
i_etag = item["hash"]
print(
"%s [size: %s] [etag: %s]" %
(i_name, i_size, i_etag)
)
items.append((i_name, i_size, i_etag))
else:
raise page["error"]
except SwiftError as e:
logger.error(e.value)
print(items)
return JsonResponse(items, safe=False)
def read(self, name, chunk_size=None):
"""Read `name` object and stream its content in chunks of (max) 2 ** 16
Why chunks of (max) 2 ** 16 ?
Because SwiftService opens a file to stream the object into:
See swiftclient.service.py:2082 open(filename, 'rb', DISK_BUFFER)
Where filename = "/dev/stdout" and DISK_BUFFER = 2 ** 16
"""
logger.debug("Getting archive: %s", name)
with SwiftService(self.options) as swift:
download = next(
swift.download(self.container, [name], {"out_file": "-"}))
if "contents" not in download:
msg = "Failed to download %s: %s"
logger.error(msg, download["object"], download["error"])
raise BackendException(msg %
(download["object"], download["error"]))
size = 0
for chunk in download["contents"]:
logger.debug("Chunk %s", len(chunk))
size += len(chunk)
sys.stdout.buffer.write(chunk)
# Archive fetched, add a new entry to the history
self.append_to_history({
"backend": self.name,
"command": "fetch",
"id": name,
"size": size,
"fetched_at": now(),
})
def swift_file_commit(_opts, container, obj_name, obj_source=None):
with SwiftService(options=_opts) as swift, OutputManager() as out_manager:
try:
if obj_source is None:
obj_source = obj_name # file path
#check if file exists
if (not os.path.exists(obj_source)):
print('File %s does not exist' % obj_source)
exit()
objs = [SwiftUploadObject(obj_source, obj_name)
] # (source, object name) --> (value,key)
for r in swift.upload(container, objs):
if r['success']:
if 'object' in r:
print(r['object'])
elif 'for_object' in r:
print('%s segment %s' %
(r['for_object'], r['segment_index']))
else:
error = r['error']
if r['action'] == "create_container":
logger.warning(
'Warning: failed to create container '
"'%s'%s", container, error)
elif r['action'] == "upload_object":
logger.error(
"Failed to upload object %s to container %s: %s" %
(container, r['object'], error))
else:
logger.error("%s" % error)
except SwiftError as e:
logger.error(e.value)
def test_delete_empty_container_excpetion(self):
mock_conn = self._get_mock_connection()
mock_conn.delete_container = Mock(side_effect=self.exc)
expected_r = self._get_expected({
'action': 'delete_container',
'success': False,
'object': None,
'error': self.exc
})
s = SwiftService()
r = s._delete_empty_container(mock_conn, 'test_c')
mock_conn.delete_container.assert_called_once_with('test_c',
response_dict={})
self._assertDictEqual(expected_r, r)
def download_swiftclient(oid, container, name, delay=0, request_tries=5):
try:
print 'Prefetching object with SwiftClient: ' + oid + ' after ' + str(
delay.total_seconds()) + ' seconds of delay.'
time.sleep(delay.total_seconds())
_opts = {
"auth_version": '2.0', # Should be '2.0'
"os_username": environ.get('OS_USERNAME'),
"os_password": environ.get('OS_PASSWORD'),
"os_tenant_name": environ.get('OS_TENANT_NAME'),
"os_auth_url": environ.get('OS_AUTH_URL'),
"retries": request_tries,
}
headers = {"header": ["X-No-Prefetch:True"]}
with SwiftService(options=_opts) as swift:
for down_res in swift.download(container=container,
objects=[name],
options=headers):
if down_res['success']:
print("'%s' downloaded" % down_res['object'])
return (oid, down_res['object'],
down_res['response_dict']['headers'])
else:
print("'%s' download failed" % down_res['error'])
return (oid, [], down_res['response_dict']['headers'])
except Error as e:
pass
def test_upload_object_job_identical_slo_with_nesting(self):
with tempfile.NamedTemporaryFile() as f:
f.write(b'a' * 30)
f.flush()
seg_etag = md5(b'a' * 10).hexdigest()
submanifest = "[%s]" % ",".join(
['{"bytes":10,"hash":"%s"}' % seg_etag] * 2)
submanifest_etag = md5(seg_etag.encode('ascii') * 2).hexdigest()
manifest = "[%s]" % ",".join([
'{"sub_slo":true,"name":"/test_c_segments/test_sub_slo",'
'"bytes":20,"hash":"%s"}' % submanifest_etag,
'{"bytes":10,"hash":"%s"}' % seg_etag])
mock_conn = mock.Mock()
mock_conn.head_object.return_value = {
'x-static-large-object': True,
'content-length': 30,
'etag': md5(submanifest_etag.encode('ascii') +
seg_etag.encode('ascii')).hexdigest()}
mock_conn.get_object.side_effect = [
({}, manifest.encode('ascii')),
({}, submanifest.encode('ascii'))]
type(mock_conn).attempts = mock.PropertyMock(return_value=2)
s = SwiftService()
r = s._upload_object_job(conn=mock_conn,
container='test_c',
source=f.name,
obj='test_o',
options={'changed': False,
'skip_identical': True,
'leave_segments': True,
'header': '',
'segment_size': 10})
self.assertIsNone(r.get('error'))
self.assertTrue(r['success'])
self.assertEqual('skipped-identical', r.get('status'))
self.assertEqual(0, mock_conn.put_object.call_count)
self.assertEqual([mock.call('test_c', 'test_o')],
mock_conn.head_object.mock_calls)
self.assertEqual([
mock.call('test_c', 'test_o',
query_string='multipart-manifest=get'),
mock.call('test_c_segments', 'test_sub_slo',
query_string='multipart-manifest=get'),
], mock_conn.get_object.mock_calls)
def test_delete_object(self):
mock_q = Queue()
mock_conn = self._get_mock_connection()
mock_conn.head_object = Mock(return_value={})
expected_r = self._get_expected({
'action': 'delete_object',
'success': True
})
s = SwiftService()
r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q)
mock_conn.head_object.assert_called_once_with('test_c', 'test_o')
mock_conn.delete_object.assert_called_once_with(
'test_c', 'test_o', query_string=None, response_dict={}
)
self._assertDictEqual(expected_r, r)
def test_delete_empty_container_excpetion(self):
mock_conn = self._get_mock_connection()
mock_conn.delete_container = Mock(side_effect=self.exc)
expected_r = self._get_expected({
'action': 'delete_container',
'success': False,
'object': None,
'error': self.exc
})
s = SwiftService()
r = s._delete_empty_container(mock_conn, 'test_c')
mock_conn.delete_container.assert_called_once_with(
'test_c', response_dict={}
)
self._assertDictEqual(expected_r, r)
def test_upload_object_job_etag_mismatch(self):
# The etag test for both streams and files use the same code
# so only one test should be needed.
def _consuming_conn(*a, **kw):
contents = a[2]
contents.read() # Force md5 calculation
return 'badresponseetag'
with tempfile.NamedTemporaryFile() as f:
f.write(b'a' * 30)
f.flush()
mock_conn = mock.Mock()
mock_conn.put_object.side_effect = _consuming_conn
type(mock_conn).attempts = mock.PropertyMock(return_value=2)
s = SwiftService()
r = s._upload_object_job(conn=mock_conn,
container='test_c',
source=f.name,
obj='test_o',
options={
'changed': False,
'skip_identical': False,
'leave_segments': True,
'header': '',
'segment_size': 0,
'checksum': True
})
self.assertEqual(r['success'], False)
self.assertIn('error', r)
self.assertIn('md5 mismatch', str(r['error']))
self.assertEqual(mock_conn.put_object.call_count, 1)
expected_headers = {'x-object-meta-mtime': mock.ANY}
mock_conn.put_object.assert_called_with('test_c',
'test_o',
mock.ANY,
content_length=30,
headers=expected_headers,
response_dict={})
contents = mock_conn.put_object.call_args[0][2]
self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())
def connection(self):
options = None
if self.swift_conn_id != "swift_default":
options = {}
connection = BaseHook.get_connection(self.swift_conn_id)
options["os_username"] = connection.login
options["os_password"] = connection.password
options["os_tenant_name"] = connection.host
yield SwiftService(options=options)
def handle(self, *args, **options):
_opts = {'object_uu_threads': 20}
dir = 'media' # settings.MEDIA_ROOT
container = settings.SWIFT_CONTAINER_NAME
with SwiftService(
options=_opts) as swift, OutputManager() as out_manager:
try:
# Collect all the files and folders in the given directory
objs = []
dir_markers = []
# change to directory so it isn't uploaded as part of all the
# file object names
os.chdir(dir)
for (_dir, _ds, _fs) in os.walk('.'):
if not (_ds + _fs):
dir_markers.append(_dir)
else:
objs.extend([os.path.join(_dir, _f) for _f in _fs])
# Now that we've collected all the required files and dir markers
# build the ``SwiftUploadObject``s for the call to upload
objs = [
SwiftUploadObject(o, object_name=o.replace(dir, dir, 1))
for o in objs
]
dir_markers = [
SwiftUploadObject(None,
object_name=d.replace(dir, dir, 1),
options={'dir_marker': True})
for d in dir_markers
]
# Schedule uploads on the SwiftService thread pool and iterate
# over the results
for r in swift.upload(container, objs + dir_markers):
if r['success']:
if 'object' in r:
print(r['object'])
elif 'for_object' in r:
print('%s segment %s' %
(r['for_object'], r['segment_index']))
else:
error = r['error']
if r['action'] == "create_container":
logger.warning(
'Warning: failed to create container '
"'%s'%s", container, error)
elif r['action'] == "upload_object":
logger.error(
"Failed to upload object %s to container %s: %s"
% (container, r['object'], error))
else:
logger.error("%s" % error)
os.chdir('..')
except SwiftError as e:
logger.error(e.value)
def get_service_client_v1(tenant_name, user_name, key, auth_url, options={}):
user = tenant_name + ':' + user_name
return SwiftService(options=dict(
{
"auth_version": 1,
"user": user,
"key": key,
"auth": auth_url,
}, **options))
def test_delete_object(self):
mock_q = Queue()
mock_conn = self._get_mock_connection()
mock_conn.head_object = Mock(return_value={})
expected_r = self._get_expected({
'action': 'delete_object',
'success': True
})
s = SwiftService()
r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q)
mock_conn.head_object.assert_called_once_with('test_c', 'test_o')
mock_conn.delete_object.assert_called_once_with('test_c',
'test_o',
query_string=None,
response_dict={})
self._assertDictEqual(expected_r, r)
def upload_results(results_folder, destination_container):
SETTINGS_LOCATION = os.environ.get('SWIFT_SETTINGS')
if not SETTINGS_LOCATION:
raise RuntimeError("No OpenStack settings provided for the upload")
with open(SETTINGS_LOCATION, 'r') as sf:
settings = json.load(sf)
# generate filelist from results folder
files = [(os.path.join(dp, f)) for dp, dn, fn in os.walk(results_folder)
for f in fn]
labels = convert_filenames_to_labels(results_folder, files)
options = {
'object_uu_threads': 20,
'os_auth_token': get_keystone_token(settings)
}
# Remove password from the OpenStack settings, otherwise the SwiftService
# connection will be built ignoring the token and failing authentication
del (settings['os_password'])
options.update(**settings)
def generate_options(label):
if label in ['/info', '/info_fullres.json', '/transform.json']:
return {}
else:
return {'header': ['Content-Encoding:gzip']}
with SwiftService(
options=options) as swift, OutputManager() as out_manager:
try:
logger.info(F'Making public container {destination_container}')
swift.post(container=destination_container,
options={
'read_acl':
'.r:*,.rlistings',
'header':
['X-Container-Meta-Access-Control-Allow-Origin: *']
})
logger.info(
f'Uploading {len(files)} objects to {destination_container}')
start = perf_counter()
objects = [
SwiftUploadObject(file,
object_name=labels[index],
options=generate_options(labels[index]))
for index, file in enumerate(files)
]
for result in swift.upload(destination_container, objects):
if not result['success']:
logger.error(f"Failed to upload object")
raise RuntimeError("Failed to upload object")
finish = perf_counter()
logger.info(f'Completed in {timedelta(seconds=finish-start)}')
except SwiftError as e:
logger.exception(e.value)
raise RuntimeError("Failed to upload objects")
def test_upload_object_job_etag_mismatch(self):
# The etag test for both streams and files use the same code
# so only one test should be needed.
def _consuming_conn(*a, **kw):
contents = a[2]
contents.read() # Force md5 calculation
return 'badresponseetag'
with tempfile.NamedTemporaryFile() as f:
f.write(b'a' * 30)
f.flush()
mock_conn = mock.Mock()
mock_conn.put_object.side_effect = _consuming_conn
type(mock_conn).attempts = mock.PropertyMock(return_value=2)
s = SwiftService()
r = s._upload_object_job(conn=mock_conn,
container='test_c',
source=f.name,
obj='test_o',
options={'changed': False,
'skip_identical': False,
'leave_segments': True,
'header': '',
'segment_size': 0,
'checksum': True})
self.assertEqual(r['success'], False)
self.assertIn('error', r)
self.assertIn('md5 mismatch', str(r['error']))
self.assertEqual(mock_conn.put_object.call_count, 1)
expected_headers = {'x-object-meta-mtime': mock.ANY}
mock_conn.put_object.assert_called_with('test_c', 'test_o',
mock.ANY,
content_length=30,
headers=expected_headers,
response_dict={})
contents = mock_conn.put_object.call_args[0][2]
self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())
def __init__(self, container, topic, kafka):
self.container = container
self.swift = SwiftService()
self.out = OutputManager()
self.kafka = kafka
auth = get_auth()
self.url = auth[0]
self.token = auth[1]
self.topic = topic
self.producer = KafkaProducer(bootstrap_servers=kafka)
def test_delete_object_exception(self):
mock_q = Queue()
mock_conn = self._get_mock_connection()
mock_conn.delete_object = Mock(side_effect=self.exc)
expected_r = self._get_expected({
'action': 'delete_object',
'success': False,
'error': self.exc
})
# _delete_object doesnt populate attempts or response dict if it hits
# an error. This may not be the correct behaviour.
del expected_r['response_dict'], expected_r['attempts']
s = SwiftService()
r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q)
mock_conn.head_object.assert_called_once_with('test_c', 'test_o')
mock_conn.delete_object.assert_called_once_with(
'test_c', 'test_o', query_string=None, response_dict={}
)
self._assertDictEqual(expected_r, r)
def test_upload_segment_job_etag_mismatch(self):
def _consuming_conn(*a, **kw):
contents = a[2]
contents.read() # Force md5 calculation
return 'badresponseetag'
with tempfile.NamedTemporaryFile() as f:
f.write(b'a' * 10)
f.write(b'b' * 10)
f.write(b'c' * 10)
f.flush()
mock_conn = mock.Mock()
mock_conn.put_object.side_effect = _consuming_conn
type(mock_conn).attempts = mock.PropertyMock(return_value=2)
s = SwiftService()
r = s._upload_segment_job(conn=mock_conn,
path=f.name,
container='test_c',
segment_name='test_s_1',
segment_start=10,
segment_size=10,
segment_index=2,
obj_name='test_o',
options={'segment_container': None,
'checksum': True})
self.assertIn('error', r)
self.assertIn('md5 mismatch', str(r['error']))
self.assertEqual(mock_conn.put_object.call_count, 1)
mock_conn.put_object.assert_called_with('test_c_segments',
'test_s_1',
mock.ANY,
content_length=10,
response_dict={})
contents = mock_conn.put_object.call_args[0][2]
self.assertEqual(contents.get_md5sum(), md5(b'b' * 10).hexdigest())
def ck_cdn(self, secrets):
if self.client.rax_id:
rax_id = self.client.rax_id
p_okg("client.rax_id: {}".format(rax_id))
else:
p_warn("client.rax_id not set.")
return
if self.client.bucket_id:
bucket_id = self.client.bucket_id
p_okg("client.bucket_id: {}".format(bucket_id))
else:
p_fail("client.bucket_id not set.")
print("logging in...")
uploader = swift_uploader.Uploader()
# pprint.pprint(pw.swift[rax_id])
uploader.user=rax_id
cf = uploader.auth()
print("checking for valid bucket...")
from swiftclient.service import SwiftService
swift = SwiftService()
stats_it = swift.stat(container=bucket_id)
"""
containers = cf.get_all_containers()
container_names = [container.name for container in containers]
print("container_names", container_names)
if bucket_id in container_names:
p_okg('"{}" found.'.format(bucket_id))
else:
p_fail('"{}" not found.'.format(bucket_id))
"""
# not sure what to do with this...
# container = cf.get_container(bucket_id)
return
def test_delete_empty_container(self):
mock_conn = self._get_mock_connection()
expected_r = self._get_expected({
'action': 'delete_container',
'success': True,
'object': None
})
r = SwiftService._delete_empty_container(mock_conn, 'test_c')
mock_conn.delete_container.assert_called_once_with(
'test_c', response_dict={}
)
self._assertDictEqual(expected_r, r)
def test_delete_object_slo_support(self):
# If SLO headers are present the delete call should include an
# additional query string to cause the right delete server side
mock_q = Queue()
mock_conn = self._get_mock_connection()
mock_conn.head_object = Mock(
return_value={'x-static-large-object': True}
)
expected_r = self._get_expected({
'action': 'delete_object',
'success': True
})
s = SwiftService()
r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q)
mock_conn.head_object.assert_called_once_with('test_c', 'test_o')
mock_conn.delete_object.assert_called_once_with(
'test_c', 'test_o',
query_string='multipart-manifest=delete',
response_dict={}
)
self._assertDictEqual(expected_r, r)
def test_delete_segment(self):
mock_q = Queue()
mock_conn = self._get_mock_connection()
expected_r = self._get_expected({
'action': 'delete_segment',
'object': 'test_s',
'success': True,
})
r = SwiftService._delete_segment(mock_conn, 'test_c', 'test_s', mock_q)
mock_conn.delete_object.assert_called_once_with(
'test_c', 'test_s', response_dict={}
)
self._assertDictEqual(expected_r, r)
self._assertDictEqual(expected_r, self._get_queue(mock_q))
def getObjsBackend(objs, backend, config):
if(backend == 'hdfs'):
client = Client(socket.gethostname(), config['HADOOP_RPC_PORT'], use_trash=False)
for obj in objs:
try:
copy_gen = client.copyToLocal([obj[0]], obj[1])
for copy_item in copy_gen:
pass
except Exception as e:
print(e)
elif(backend == 'swift'):
options = {'os_auth_url': os.environ['OS_AUTH_URL'], 'os_username': os.environ['OS_USERNAME'], 'os_password': os.environ['OS_PASSWORD'], 'os_tenant_id': os.environ['OS_TENANT_ID'], 'os_tenant_name': os.environ['OS_TENANT_NAME']}
swiftService = SwiftService(options=options)
for obj in objs:
# Create the containers which are used in this application for Object Storage
if(obj[0] == 'sqlite.db'):
swiftService.post(container='containerFiles')
swiftService.post(container='containerFeatures')
swiftService.post(container='containerModules')
out_file = obj[1] # Get the output file location from runner
localoptions = {'out_file': out_file}
objects = []
objects.append(obj[0])
swiftDownload = swiftService.download(container='containerModules', objects=objects, options=localoptions)
for downloaded in swiftDownload:
if("error" in downloaded.keys()):
raise RuntimeError(downloaded['error'])
# print(downloaded)
elif(backend == 'nfs'): # Every file is already in respective local dirs
pass
def __init__(self, container, prefix=''):
self.container = container
self.prefix = prefix
self.client = SwiftService()
def download(source, dest=None):
"""
Download files from a SWIFT object store
@param source_info: Source information such as the source to download from.
@type source_info: a dictionary
@param local_dest_dir: The local directory to store file
@type local_dest_dir: str
@return: True and a list of file downloaded if successful. Otherwise False.
"""
if not dest:
dest = tempfile.mkstemp()
url = urlsplit(source['url'])
_, ver, account, container, object_name = url.path.split('/', 4)
swift_opts = {
'os_storage_url': '{scheme}://{netloc}/{ver}/{account}'.format(
scheme=re.sub(r'^swift\+', '', url.scheme),
netloc=url.netloc,
ver=ver,
account=account)
}
# SwiftService knows about environment variables
for opt in ('os_auth_url', 'os_username', 'os_password', 'os_tenant_name', 'os_storage_url'):
if opt in source:
swift_opts[opt] = source[opt]
try:
swift = SwiftService(swift_opts)
filelist = []
if os.path.exists(dest) and os.path.isdir(dest):
outfilename = os.path.join(dest, os.path.basename(object_name))
else:
outfilename = dest
retries = 5 # number of retries left
backoff = 30 # wait time between retries
backoff_inc = 30 # increase in wait time per retry
while retries:
filelist = []
retries -= 1
try:
for result in swift.download(container, [object_name], {'out_file': outfilename}):
# result dict: success
# action: 'download_object'
# success: True
# container: ...
# object: ...
# path: ....
# pseudodir: ...
# start_time, finish_time, headers_receipt, auth_end_time,
# read_length, attempts, response_dict
# result dict: error
# action: 'download_object'
# success: False
# error: ...
# traceback: ...
# container, object, error_timestamp, response_dict, path
# psudodir, attempts
if not result['success']:
raise Exception(
'Download from selfelfwift {container}/{object} to {out_file} failed with {error}'.format(out_file=outfilename, **result))
outfile = {'url': outfilename,
'name': os.path.basename(object_name),
'content_type': result['response_dict']['headers'].get('content-type', 'application/octet-stream')}
filelist.append(outfile)
# no exception we can continue
retries = 0
except Exception as e:
if not retries:
# reraise if no retries left
raise
LOG.warn("Download from Swift failed: %s - %d retries left", e, retries)
time.sleep(backoff)
backoff += backoff_inc
backoff_inc += 30
return filelist
except Exception as e:
LOG.error("Download from Swift failed: %s", e, exc_info=True)
raise
