def create_sw_conn(swift_auth_token,storage_url):
if swift_auth_token and storage_url:
return swiftclient.Connection(preauthtoken=swift_auth_token,
preauthurl=storage_url)
swift_auth=os.environ.get("ST_AUTH")
swift_user=os.environ.get("ST_USER")
swift_key=os.environ.get("ST_KEY")
if swift_auth and swift_user and swift_key:
return swiftclient.Connection(authurl=swift_auth,user=swift_user,
key=swift_key)
print("Error: Swift environment not configured!")
sys.exit()
def setup_feature(feat):
# prevent ini parser from lowercasing params
_RESTORE.append((ConfigParser, "optionxform",
ConfigParser.optionxform))
ConfigParser.optionxform = lambda self, x: x
# make connections work
if STORAGE == "s3":
_RESTORE.append((s3.S3FileSystem, "_get_connection",
s3.S3FileSystem._get_connection))
s3.S3FileSystem._get_connection = lambda self: connect_s3()
s3.S3FileSystem._bucket_location = lambda self: ''
elif STORAGE == "gcs":
def get_pk(f="gcs_pk.p12"): # TODO:
with open(f, "rb") as fd:
pk = fd.read()
return base64.b64encode(pk)
ACCESS_DATA = {
"service_account_name": '*****@*****.**',
"key": get_pk(),
}
_RESTORE.append((gcs, "bus", gcs.bus))
gcs.bus = mock.MagicMock()
gcs.bus.platform.get_access_data = lambda k: ACCESS_DATA[k]
gsm = GoogleServiceManager(gcs.bus.platform,
"storage", "v1beta2", *STORAGE_FULL_SCOPE)
gcs.bus.platform.get_numeric_project_id.return_value = '876103924605'
gcs.bus.platform.new_storage_client = lambda: gsm.get_service()
elif STORAGE == "swift":
_RESTORE.append((swift.SwiftFileSystem, "_get_connection",
swift.SwiftFileSystem._get_connection))
swift.SwiftFileSystem._get_connection = lambda self: swiftclient.Connection(
"https://identity.api.rackspacecloud.com/v1.0",
os.environ["RS_USERNAME"], os.environ["RS_API_KEY"])
elif STORAGE == "swift-enter-it":
_RESTORE.append((swift.SwiftFileSystem, "_get_connection",
swift.SwiftFileSystem._get_connection))
swift.SwiftFileSystem._get_connection = lambda self: swiftclient.Connection(
"http://folsom.enter.it:5000/v2.0",
os.environ["ENTER_IT_USERNAME"], os.environ["ENTER_IT_API_KEY"], auth_version="2")
def test_integration_pacsfile_create_success(self):
chris_username = '******'
chris_password = '******'
User.objects.create_user(username=chris_username,
password=chris_password)
path = 'SERVICES/PACS/MyPACS/123456-crazy/brain_crazy_study/SAG_T1_MPRAGE/file2.dcm'
# initiate a Swift service connection
conn = swiftclient.Connection(
user=settings.SWIFT_USERNAME,
key=settings.SWIFT_KEY,
authurl=settings.SWIFT_AUTH_URL,
)
# create container in case it doesn't already exist
conn.put_container(settings.SWIFT_CONTAINER_NAME)
# upload file to Swift storage
with io.StringIO("test file") as file1:
conn.put_object(settings.SWIFT_CONTAINER_NAME,
path,
contents=file1.read(),
content_type='text/plain')
# make the POST request using the chris user
self.client.login(username=chris_username, password=chris_password)
response = self.client.post(self.create_read_url,
data=self.post,
content_type=self.content_type)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
# delete file from Swift storage
conn.delete_object(settings.SWIFT_CONTAINER_NAME, path)
def test_integration_fileresource_download_success(self):
# create a test file
test_file_path = self.test_dir
self.test_file = test_file_path + '/file1.txt'
file = open(self.test_file, "w")
file.write("test file")
file.close()
# initiate a Swift service connection
conn = swiftclient.Connection(
user=settings.SWIFT_USERNAME,
key=settings.SWIFT_KEY,
authurl=settings.SWIFT_AUTH_URL,
)
# create container in case it doesn't already exist
conn.put_container(settings.SWIFT_CONTAINER_NAME)
# upload file to Swift storage
with open(self.test_file, 'r') as file1:
conn.put_object(settings.SWIFT_CONTAINER_NAME, '/tests/file1.txt',
contents=file1.read(),
content_type='text/plain')
self.client.login(username=self.username, password=self.password)
response = self.client.get(self.download_url)
self.assertEquals(response.status_code, 200)
self.assertEquals(str(response.content,'utf-8'), "test file")
# delete file from Swift storage
conn.delete_object(settings.SWIFT_CONTAINER_NAME, '/tests/file1.txt')
def client(username, password, trust_id=None):
'''return a Swift client
This will return a Swift client for the specified username scoped to the
current context project, unless a trust identifier is specified.
If a trust identifier is present then the Swift client will be created
based on a preauthorized token generated by the username scoped to the
trust identifier.
:param username: The username for the Swift client
:param password: The password associated with the username
:param trust_id: A trust identifier for scoping the username (optional)
:returns: A Swift client object
'''
if trust_id:
proxyclient = k.client_for_proxy_user(username, password, trust_id)
return client_from_token(proxyclient.auth_token)
else:
return swiftclient.Connection(auth_version='2.0',
cacert=CONF.swift.ca_file,
insecure=CONF.swift.api_insecure,
authurl=su.retrieve_auth_url(),
user=username,
key=password,
tenant_name=sh.retrieve_tenant())
def get_swift_conn(user, key, authurl):
conn = swiftclient.Connection(
user=user,
key=key,
authurl=authurl,
)
return conn
def connect(self, *args, **kwargs) -> dict:
"""
Connect to swift storage and return the connection object,
as well an optional "prepend" string to fully qualify
object location in swift storage.
The 'prependBucketPath' is somewhat 'legacy' to a similar
method in charm.py and included here with the idea
to eventually converge on a single swift-based intermediary
library for both pfcon and CUBE.
"""
d_ret = {
'status': True,
'conn': None,
'user': self.state('/swift/username'),
'key': self.state('/swift/key'),
'authurl': self.state('/swift/auth_url'),
'container_name': self.state('/swift/container_name')
}
# initiate a swift service connection, based on internal
# settings already available in the django variable space.
try:
d_ret['conn'] = swiftclient.Connection(
user = d_ret['user'],
key = d_ret['key'],
authurl = d_ret['authurl']
)
except:
d_ret['status'] = False
return d_ret
def GetObjStoContainerInfo(container):
try:
ibmobjectstoreconn = swiftclient.Connection(key=password,
authurl=authurl,
auth_version='3',
os_options={
"project_id":
projectId,
"user_id": userId,
"region_name": region
})
objs = []
for data in ibmobjectstoreconn.get_container(container)[1]:
del data['hash']
data[
'downloadURL'] = thehost + "/" + container + "/" + data['name']
objs.append(data)
jobjs = '{"objects":' + json.dumps(objs) + '}'
return Response(jobjs, mimetype='application/json', status=200)
except ClientException as ce:
return MakeJSONMsgResponse(
{
"message": ce.msg + ". Bad container name?",
"containername": container
}, 404)
def GetObjectStorage(container, filename):
if 'VCAP_SERVICES' not in os.environ:
return MakeJSONMsgResponse({"message": "cannot authenticate"}, 500)
try:
ibmobjectstoreconn = swiftclient.Connection(key=password,
authurl=authurl,
auth_version='3',
os_options={
"project_id":
projectId,
"user_id": userId,
"region_name": region
})
obj = ibmobjectstoreconn.get_object(container, filename)
if filename.endswith('.txt'):
return Response(obj[1], mimetype='text/plain', status=200)
elif filename.endswith('.csv'):
return Response(obj[1], mimetype='text/csv', status=200)
elif filename.endswith('.json'):
return Response(obj[1], mimetype='application/json', status=200)
else:
return Response(obj[1], mimetype='application/binary', status=200)
except ClientException as ce:
return MakeJSONMsgResponse(
{
"message": ce.msg,
"containername": container,
"filename": filename
}, 404)
def setup(self):
self.conn = swiftclient.Connection(
user=USER,
key=KEY,
authurl=AUTHURL,
)
self.container = 'test'
self.config = Config({
'user': USER,
'key': KEY,
'authurl': AUTHURL,
})
self.backend = SwiftBackend(self.container, self.config)
yield
try:
headers, items = self.conn.get_container(self.backend.name)
for i in items:
self.conn.delete_object(self.backend.name, i['name'])
self.conn.delete_container(self.backend.name)
except swiftclient.ClientException as e:
assert False, "Failed to delete container ->" + str(e)
def swift_con(dest_container_name):
# TODO: This probably could be simplified. Why do we have several identifiers ?
if dest_container_name == "storage-pc-dev":
user = settings.SWIFT_TESTING_USER
key = settings.SWIFT_TESTING_KEY
tenant_name = settings.SWIFT_TESTING_TENANT_NAME
region_name = settings.SWIFT_TESTING_REGION_NAME
elif dest_container_name == "storage-pc-staging":
user = settings.SWIFT_STAGING_USER
key = settings.SWIFT_STAGING_KEY
tenant_name = settings.SWIFT_STAGING_TENANT_NAME
region_name = settings.SWIFT_STAGING_REGION_NAME
else:
print("Ce conteneur ne semble pas exister")
return 1
options = {"region_name": region_name}
return swiftclient.Connection(
user=user,
key=key,
authurl=settings.SWIFT_AUTH_URL,
os_options=options,
tenant_name=tenant_name,
auth_version="3",
)
def _get_connection(self):
"""
Subclasses may override to use different connection setup
"""
return swiftclient.Connection(
self.auth_url, self.account_username, self.password,
auth_version=self.auth_version)
def swiftauth():
conn = swiftclient.Connection(set.SWIFT_AUTH_URL,
set.SWIFT_USERNAME,
set.SWIFT_KEY,
auth_version=set.SWIFT_AUTH_VERSION,
tenant_name=set.SWIFT_TENANT_NAME)
return conn
def test_integration_mananger_can_check_plugin_app_exec_status(self):
"""
Test whether the manager can check a plugin's app execution status.
NB: Note the directory overrides on input and output dirs! This
is file system space in the plugin container, and thus by hardcoding
this here we are relying on totally out-of-band knowledge!
This must be fixed in later versions!
"""
user = User.objects.get(username=self.username)
plugin = Plugin.objects.get(name=self.plugin_fs_name)
pl_inst = PluginInstance.objects.create(
plugin=plugin,
owner=user,
compute_resource=plugin.compute_resource)
parameter_dict = {'dir': './'}
manager.PluginAppManager.run_plugin_app(
pl_inst,
parameter_dict,
service='pfcon',
inputDirOverride='/share/incoming',
outputDirOverride='/share/outgoing')
manager.PluginAppManager.check_plugin_app_exec_status(pl_inst)
self.assertEqual(pl_inst.status, 'started')
# In the following we keep checking the status until the job ends with
# 'finishedSuccessfully'. The code runs in a lazy loop poll with a
# max number of attempts at 2 second intervals.
maxLoopTries = 20
currentLoop = 1
b_checkAgain = True
while b_checkAgain:
str_responseStatus = manager.PluginAppManager.check_plugin_app_exec_status(
pl_inst)
if str_responseStatus == 'finishedSuccessfully':
b_checkAgain = False
else:
time.sleep(2)
currentLoop += 1
if currentLoop == maxLoopTries:
b_checkAgain = False
self.assertEqual(pl_inst.status, 'finishedSuccessfully')
str_fileCreatedByPlugin = os.path.join(pl_inst.get_output_path(),
'out.txt')
# initiate a Swift service connection
conn = swiftclient.Connection(
user=settings.SWIFT_USERNAME,
key=settings.SWIFT_KEY,
authurl=settings.SWIFT_AUTH_URL,
)
# make sure str_fileCreatedByPlugin file was created in Swift storage
object_list = conn.get_container(settings.SWIFT_CONTAINER_NAME,
prefix=str_fileCreatedByPlugin,
full_listing=True)[1]
self.assertEqual(object_list[0]['name'], str_fileCreatedByPlugin)
def validate(self, data):
"""
Overriden to check whether the provided path is under SERVICES/<service_name>/
path and already registered.
"""
# remove service_name as it is not part of the model
service_name = data.pop('service_name')
# verify that the file path is correct
path = data.get('path')
path = path.strip(' ').strip('/')
prefix = 'SERVICES/%s/' % service_name
if not path.startswith(prefix):
error_msg = "File path must start with '%s'." % prefix
raise serializers.ValidationError([error_msg])
# verify that the file is indeed already in Swift
conn = swiftclient.Connection(user=settings.SWIFT_USERNAME,
key=settings.SWIFT_KEY,
authurl=settings.SWIFT_AUTH_URL)
object_list = conn.get_container(settings.SWIFT_CONTAINER_NAME,
prefix=path)[1]
if not object_list:
raise serializers.ValidationError(
{'path': ["Could not find this path."]})
# verify that the file has not already been registered
try:
ServiceFile.objects.get(fname=path)
except ServiceFile.DoesNotExist:
pass
else:
error_msg = "File has already been registered."
raise serializers.ValidationError({'path': [error_msg]})
# update validated data
(service, tf) = Service.objects.get_or_create(identifier=service_name)
data.update({'path': path, 'service': service})
return data
def get_connection(self, location):
if not location.user:
reason = (_("Location is missing user:password information."))
LOG.debug(reason)
raise exception.BadStoreUri(message=reason)
auth_url = location.swift_url
if not auth_url.endswith('/'):
auth_url += '/'
if self.auth_version == '2':
try:
tenant_name, user = location.user.split(':')
except ValueError:
reason = (_("Badly formed tenant:user '%(user)s' in "
"Swift URI") % {'user': location.user})
LOG.debug(reason)
raise exception.BadStoreUri()
else:
tenant_name = None
user = location.user
os_options = {}
if self.region:
os_options['region_name'] = self.region
os_options['endpoint_type'] = self.endpoint_type
os_options['service_type'] = self.service_type
return swiftclient.Connection(
auth_url, user, location.key, insecure=self.insecure,
tenant_name=tenant_name, snet=self.snet,
auth_version=self.auth_version, os_options=os_options)
def get_connection(self, location):
return swiftclient.Connection(
None, self.context.user, None,
preauthurl=location.swift_url,
preauthtoken=self.context.auth_tok,
tenant_name=self.context.tenant,
auth_version='2', snet=self.snet, insecure=self.insecure)
def swiftstorage_connect(self, *args, **kwargs):
"""
Connect to swift storage and return the connection object,
as well an optional "prepend" string to fully qualify
object location in swift storage.
"""
b_status = True
b_prependBucketPath = False
for k, v in kwargs.items():
if k == 'prependBucketPath': b_prependBucketPath = v
d_ret = {'status': b_status, 'conn': None, 'prependBucketPath': ""}
# initiate a swift service connection, based on internal
# settings already available in the django variable space.
try:
d_ret['conn'] = swiftclient.Connection(
user=settings.SWIFT_USERNAME,
key=settings.SWIFT_KEY,
authurl=settings.SWIFT_AUTH_URL,
)
except:
d_ret['status'] = False
if b_prependBucketPath:
d_ret[
'prependBucketPath'] = self.c_pluginInst.owner.username + '/uploads'
return d_ret
def GetObjStoreInfo():
try:
ibmobjectstoreconn = swiftclient.Connection(key=password,
authurl=authurl,
auth_version='3',
os_options={
"project_id":
projectId,
"user_id": userId,
"region_name": region
})
conns = []
for container in ibmobjectstoreconn.get_account()[1]:
container['accessURL'] = thehost + "/" + container['name']
container['objects'] = container['count']
del container['count']
conns.append(container)
jconns = '{"containers":' + json.dumps(conns) + '}'
return Response(jconns, mimetype='application/json', status=200)
except ClientException as ce:
return MakeJSONMsgResponse(
{
"message": ce.msg,
"containername": container,
"filename": filename
}, 404)
def swift_con(config):
"""
Function to connect to Open Stack Swift Object Storage
:param config: JSON configuration
:return: swiftclient.Connection
"""
return swiftclient.Connection(user=config['swift_storage']['user'],key=config['swift_storage']['key'], authurl=config['swift_storage']['auth_url'],os_options=config['swift_storage']['options'],tenant_name=config['swift_storage']['tenant_name'], auth_version=config['swift_storage']['auth_version'])
def register_output_files(self):
"""
Custom method to register files generated by the plugin instance object
with the REST API.
"""
# initiate a Swift service connection
conn = swiftclient.Connection(
user=settings.SWIFT_USERNAME,
key=settings.SWIFT_KEY,
authurl=settings.SWIFT_AUTH_URL,
)
output_path = self.get_output_path()
# get the full list of objects with prefix output_path in Swift storage
object_list = conn.get_container(settings.SWIFT_CONTAINER_NAME,
prefix=output_path,
full_listing=True)[1]
root_instance = self.get_root_instance()
feed = root_instance.feed
fileCount = 0
for object in object_list:
feedfile = FeedFile(plugin_inst=self, feed=feed)
feedfile.fname.name = object['name']
feedfile.save()
fileCount += 1
return fileCount
def __init__(self, session):
self._session = session
self._img_cli = glanceclient.Client(session=self._session, version=2)
self._obj_cli = swiftclient.Connection(session=self._session,
cacert=SYSTEM_CA_BUNDLE)
self._ks = keystoneclient.v3.Client(session=session)
self._stores = None
def __init__(self, auth_url, tenant, container, user, password):
self.swift_client = swiftclient.Connection(authurl=auth_url,
user="******".join(
[user, tenant]),
key=password)
self._container = container
self.swift_client.head_container(container)
def _get_connection(self):
account = username = password = project_domain = user_domain = None
if self.auth_version != '3':
self.skipTest('SKIPPING KEYSTONE-V3-SPECIFIC FUNCTIONAL TESTS')
try:
account = TEST_CONFIG['account4']
username = TEST_CONFIG['username4']
user_domain = TEST_CONFIG['domain4']
project_domain = TEST_CONFIG['domain4']
password = TEST_CONFIG['password4']
except KeyError:
self.skipTest('SKIPPING KEYSTONE-V3-SPECIFIC FUNCTIONAL TESTS' +
' - NO CONFIG')
os_options = {
'project_name': account,
'project_domain_name': project_domain,
'user_domain_name': user_domain
}
return swiftclient.Connection(self.auth_url,
username,
password,
auth_version=self.auth_version,
cacert=self.cacert,
os_options=os_options)
def _get_object_storage_client(self, credentials):
if not CONF.service_available.swift:
return None
import swiftclient
auth_url = CONF.identity.uri
# add current tenant to swift operator role group.
admin_credentials = auth.get_default_credentials('identity_admin')
keystone_admin = self._get_identity_client(admin_credentials)
# enable test user to operate swift by adding operator role to him.
roles = keystone_admin.roles.list()
operator_role = CONF.object_storage.operator_role
member_role = [role for role in roles if role.name == operator_role][0]
# NOTE(maurosr): This is surrounded in the try-except block cause
# neutron tests doesn't have tenant isolation.
try:
keystone_admin.roles.add_user_role(self.identity_client.user_id,
member_role.id,
self.identity_client.tenant_id)
except keystoneclient.exceptions.Conflict:
pass
endpoint_type = CONF.object_storage.endpoint_type
os_options = {'endpoint_type': endpoint_type}
return swiftclient.Connection(auth_url,
credentials.username,
credentials.password,
tenant_name=credentials.tenant_name,
auth_version='2',
os_options=os_options)
def __connect(self):
return swiftclient.Connection(
user=self.user,
key=self.key,
authurl=self.ceph_radosgw_url + "/auth",
insecure=True,
)
def client_from_token(token):
'''return a Swift client authenticated from a token.'''
return swiftclient.Connection(auth_version='2.0',
cacert=CONF.swift.ca_file,
insecure=CONF.swift.api_insecure,
preauthurl=su.retrieve_preauth_url(),
preauthtoken=token)
def retrieve_from_openstack(self, container, object):
"""Retrieves file from OpenStack Swift.
Args:
container: Container to retrieve file from.
object: File object to retrieve.
Returns:
A byte string containing the file content.
"""
global swift_client
if swift_client is None:
swift_client = swiftclient.Connection(
auth_version=conf.remote_cfg["st_auth_version"],
authurl=conf.remote_cfg["os_auth_url"],
user=conf.remote_cfg["os_username"],
key=conf.remote_cfg["os_password"],
cert=conf.remote_cfg["os_cert"],
cacert=conf.remote_cfg["os_cacert"],
retries=conf.remote_cfg["remote_retries"],
timeout=conf.remote_cfg["remote_timeout"])
os_options = {
"user_domain_name": conf.remote_cfg["os_user_domain_name"],
"project_domain_name":
conf.remote_cfg["os_project_domain_name"],
"project_name": conf.remote_cfg["os_project_name"]
}
if not all(value is None for value in os_options.values()):
swift_client.os_options = os_options
(response_headers,
object_data) = swift_client.get_object(container, object)
return object_data
def connect(creds):
"""
Construct a connection value from a container
"""
return swiftclient.Connection(
authurl=creds.authurl,
user=creds.user,
key=creds.password,
auth_version=creds.auth_version,
tenant_name=creds.tenant_name,
os_options={
"region_name": creds.region,
"endpoint_type": creds.endpoint_type,
"domain_id": creds.domain_id,
"domain_name": creds.domain_name,
"tenant_id": creds.tenant_id,
"user_id": creds.user_id,
"user_domain_id": creds.user_domain_id,
"user_domain_name": creds.user_domain_name,
"project_id": creds.project_id,
"project_name": creds.project_name,
"project_domain_id": creds.project_domain_id,
"project_domain_name": creds.project_domain_name,
}
)
def client(username, password, trust_id=None):
'''return a Swift client
This will return a Swift client for the specified username scoped to the
current context project, unless a trust identifier is specified.
If a trust identifier is present then the Swift client will be created
based on a preauthorized token generated by the username scoped to the
trust identifier.
:param username: The username for the Swift client
:param password: The password associated with the username
:param trust_id: A trust identifier for scoping the username (optional)
:returns: A Swift client object
'''
if trust_id:
proxyauth = k.auth_for_proxy(username, password, trust_id)
return client_from_token(k.token_from_auth(proxyauth))
else:
return swiftclient.Connection(
auth_version='2.0',
cacert=CONF.swift.ca_file,
insecure=CONF.swift.api_insecure,
authurl=su.retrieve_auth_url(CONF.keystone.endpoint_type),
user=username,
key=password,
tenant_name=sh.retrieve_tenant(),
retries=CONF.retries.retries_number,
retry_on_ratelimit=True,
starting_backoff=CONF.retries.retry_after,
max_backoff=CONF.retries.retry_after)