def create_swift_client(self):
try:
swift = SwiftClient(auth_version='3',
os_options=self.__keystoneCredentials,
authurl=self.__keystoneCredentials['auth_url'],
user=self.__keystoneCredentials['username'],
key=self.__keystoneCredentials['password'],
cacert=self.__certificatesPath)
swift.get_auth()
return swift
except Exception as error:
raise Exception('Connection to Swift failed: {}'.format(error))
def get_auth():
conn = Connection(
authurl=os.environ['OS_AUTH_URL'],
user=os.environ['OS_USERNAME'],
key=os.environ['OS_PASSWORD'],
tenant_name=os.environ['OS_TENANT_NAME'],
auth_version=2.0)
return conn.get_auth()
def swift_conn(authurl, user, project, password):
'''
authurl : http://keystone:5000
user: aaa
project: aaa
password: bbb
'''
# by default add http schema
if not authurl.startswith("http"):
authurl = "http://" + authurl
os_options = {"tenant_name": project}
conn = SwiftConnection(authurl=authurl,
user=user,
key=password,
os_options=os_options,
auth_version=3)
conn.get_auth()
yield conn
conn.close()
class SwiftSession(Session):
"""Configures access to secured resources stored in OpenStack Swift Object Storage.
"""
def __init__(self,
session=None,
swift_storage_url=None,
swift_auth_token=None,
swift_auth_v1_url=None,
swift_user=None,
swift_key=None):
"""Create new OpenStack Swift Object Storage Session.
Three methods are possible:
1. Create session by the swiftclient library.
2. The SWIFT_STORAGE_URL and SWIFT_AUTH_TOKEN (this method is recommended by GDAL docs).
3. The SWIFT_AUTH_V1_URL, SWIFT_USER and SWIFT_KEY (This depends on the swiftclient library).
Parameters
----------
session: optional
A swiftclient connection object
swift_storage_url:
the storage URL
swift_auth_token:
the value of the x-auth-token authorization token
swift_storage_url: string, optional
authentication URL
swift_user: string, optional
user name to authenticate as
swift_key: string, optional
key/password to authenticate with
Examples
--------
>>> import rasterio
>>> from rasterio.session import SwiftSession
>>> fp = '/vsiswift/bucket/key.tif'
>>> conn = Connection(authurl='http://127.0.0.1:7777/auth/v1.0', user='******', key='testing')
>>> session = SwiftSession(conn)
>>> with rasterio.Env(session):
>>> with rasterio.open(fp) as src:
>>> print(src.profile)
"""
if swift_storage_url and swift_auth_token:
self._creds = {
"swift_storage_url": swift_storage_url,
"swift_auth_token": swift_auth_token
}
else:
from swiftclient.client import Connection
if session:
self._session = session
else:
self._session = Connection(authurl=swift_auth_v1_url,
user=swift_user,
key=swift_key)
self._creds = {
"swift_storage_url": self._session.get_auth()[0],
"swift_auth_token": self._session.get_auth()[1]
}
@classmethod
def hascreds(cls, config):
"""Determine if the given configuration has proper credentials
Parameters
----------
cls : class
A Session class.
config : dict
GDAL configuration as a dict.
Returns
-------
bool
"""
return 'SWIFT_STORAGE_URL' in config and 'SWIFT_AUTH_TOKEN' in config
@property
def credentials(self):
"""The session credentials as a dict"""
return self._creds
def get_credential_options(self):
"""Get credentials as GDAL configuration options
Returns
-------
dict
"""
return {k.upper(): v for k, v in self.credentials.items()}
class SwiftFileServer(object):
""" Swift File server """
def __init__(self, region_name):
# self.cfg = cfg.CONF
self.region_name = region_name
# self.authurl = self.cfg.FileSvc.authurl
# self.auth_version = self.cfg.FileSvc.auth_version
# self.user = self.cfg.FileSvc.user
# self.key = self.cfg.FileSvc.key
# self.tenant_name = self.cfg.FileSvc.tenant_name
# self.container_name = self.cfg.FileSvc.container_name
# self.temp_url_key = self.cfg.FileSvc.temp_url_key
# self.temp_url_key_2 = self.cfg.FileSvc.temp_url_key_2
# self.chosen_temp_url_key = self.cfg.FileSvc.chosen_temp_url_key
# self.authurl='http://10.204.248.50:35357/v2.0'
self.authurl = 'http://10.204.248.228:35357/v2.0'
self.auth_version = '2.0'
self.user = '******'
self.key = 'passw0rd'
self.tenant_name = 'admin'
self.temp_url_key = 'mykey'
self.temp_url_key_2 = 'mykey2'
self.chosen_temp_url_key = 'temp_url_key'
self.container_name = 'mycontainer'
self.storageurl = None
self.swift_conn = None
def connect_to_swift(self):
""" return connect to swift fileserver """
for i in range(RETRY_CNT):
i += 1
try:
self.__init_swift_fileserver()
break
except Exception as err:
print(
'[Try %d/%d ]: Connecting swift fileserver failed. %s %s',
i, RETRY_CNT, err, traceback.format_exc())
if i == RETRY_CNT:
raise
def __init_swift_fileserver(self):
try:
options = {
'authurl': self.authurl,
'user': self.user,
'key': self.key,
'auth_version': self.auth_version,
'tenant_name': self.tenant_name,
'insecure': True,
'timeout': CONN_TIMEOUT,
'os_options': {
'region_name': self.region_name
}
}
headers = {
'Temp-URL-Key': self.temp_url_key,
'Temp-URL-Key-2': self.temp_url_key_2
}
print_opts = copy.deepcopy(options)
print_opts['key'] = '<password stripped>'
self.swift_conn = Connection(**options)
self.swift_conn.post_account(headers)
self.swift_conn.put_container(self.container_name)
self.storageurl = self.swift_conn.get_auth()[0]
print('swift-file-server: Connected. options %s storageurl %s',
print_opts, self.storageurl)
except Exception as err:
print('swift-file-server: Connect FAILED %s options %s', err,
print_opts)
raise
def upload_file(self, fpath, fname=None, expires=3600):
""" upload the file in 'filepath' on to the file server and return a
temporary url for the users to download """
if not self.swift_conn:
self.connect_to_swift()
if fname is None:
fname = os.path.basename(fpath)
finp = open(fpath, 'rb')
try:
self.swift_conn.put_object(container=self.container_name,
obj=fname,
contents=finp)
print('swift-file-server: Uploading file %s ... [OK]', fname)
except Exception as err:
logging.error(
'swift-file-server: Unable to upload the file %s: %s', fname,
err)
raise
# return self.get_temp_download_url(fname,expires)
def get_temp_download_url(self, fname, expires):
""" return the temporary download url """
file_uri = '%s/%s/%s' % (self.storageurl, self.container_name, fname)
file_path = urlparse(file_uri).path
key = getattr(self, self.chosen_temp_url_key)
try:
temp_url = generate_temp_url(file_path, expires, key, 'GET')
except Exception as err:
logging.error(
'swift-file-server: Generating temp url for %s failed %s',
fname, err)
raise
download_url = self.storageurl.replace(
urlparse(self.storageurl).path, temp_url)
print('swift-file-server: Temporary download URL for file %s: %s',
fname, download_url)
return download_url
def delete_file(self, fpath):
""" Delete the file from the file server """
fname = os.path.basename(fpath)
try:
self.swift_conn.delete_object(container=self.container_name,
obj=fname)
print('swift-file-server: Deleting file %s ... [OK]', fname)
except Exception as err:
print(
'swift-file-server: Deleting file %s ... [FAIL]: %s %s (IGNORED) ',
fname, err, traceback.format_exc())
def getObjectContent(self, filename):
return self.swift_conn.get_object(container=self.container_name,
obj=filename)
class SwiftFileServer(object):
""" Swift File server """
def __init__(self, region_name):
# self.cfg = cfg.CONF
self.region_name = region_name
# self.authurl = self.cfg.FileSvc.authurl
# self.auth_version = self.cfg.FileSvc.auth_version
# self.user = self.cfg.FileSvc.user
# self.key = self.cfg.FileSvc.key
# self.tenant_name = self.cfg.FileSvc.tenant_name
# self.container_name = self.cfg.FileSvc.container_name
# self.temp_url_key = self.cfg.FileSvc.temp_url_key
# self.temp_url_key_2 = self.cfg.FileSvc.temp_url_key_2
# self.chosen_temp_url_key = self.cfg.FileSvc.chosen_temp_url_key
# self.authurl='http://10.204.248.50:35357/v2.0'
self.authurl = 'http://10.204.248.228:35357/v2.0'
self.auth_version='2.0'
self.user='******'
self.key='passw0rd'
self.tenant_name='admin'
self.temp_url_key= 'mykey'
self.temp_url_key_2='mykey2'
self.chosen_temp_url_key= 'temp_url_key'
self.container_name= 'mycontainer'
self.storageurl = None
self.swift_conn = None
def connect_to_swift(self):
""" return connect to swift fileserver """
for i in range(RETRY_CNT):
i += 1
try:
self.__init_swift_fileserver()
break
except Exception as err:
print('[Try %d/%d ]: Connecting swift fileserver failed. %s %s',
i, RETRY_CNT, err, traceback.format_exc())
if i == RETRY_CNT:
raise
def __init_swift_fileserver(self):
try:
options = {
'authurl': self.authurl,
'user': self.user,
'key': self.key,
'auth_version': self.auth_version,
'tenant_name': self.tenant_name,
'insecure': True,
'timeout': CONN_TIMEOUT,
'os_options': {'region_name': self.region_name}}
headers = {'Temp-URL-Key': self.temp_url_key,
'Temp-URL-Key-2': self.temp_url_key_2
}
print_opts = copy.deepcopy(options)
print_opts['key'] = '<password stripped>'
self.swift_conn = Connection(**options)
self.swift_conn.post_account(headers)
self.swift_conn.put_container(self.container_name)
self.storageurl = self.swift_conn.get_auth()[0]
print ('swift-file-server: Connected. options %s storageurl %s',
print_opts, self.storageurl)
except Exception as err:
print('swift-file-server: Connect FAILED %s options %s',
err, print_opts)
raise
def upload_file(self, fpath, fname=None,expires=3600):
""" upload the file in 'filepath' on to the file server and return a
temporary url for the users to download """
if not self.swift_conn:
self.connect_to_swift()
if fname is None:
fname = os.path.basename(fpath)
finp = open(fpath, 'rb')
try:
self.swift_conn.put_object(
container=self.container_name, obj=fname, contents=finp)
print ('swift-file-server: Uploading file %s ... [OK]', fname)
except Exception as err:
logging.error(
'swift-file-server: Unable to upload the file %s: %s', fname, err)
raise
# return self.get_temp_download_url(fname,expires)
def get_temp_download_url(self, fname, expires):
""" return the temporary download url """
file_uri = '%s/%s/%s' % (self.storageurl, self.container_name, fname)
file_path = urlparse(file_uri).path
key = getattr(self, self.chosen_temp_url_key)
try:
temp_url = generate_temp_url(file_path, expires, key, 'GET')
except Exception as err:
logging.error(
'swift-file-server: Generating temp url for %s failed %s', fname, err)
raise
download_url = self.storageurl.replace(
urlparse(self.storageurl).path, temp_url)
print(
'swift-file-server: Temporary download URL for file %s: %s', fname, download_url)
return download_url
def delete_file(self, fpath):
""" Delete the file from the file server """
fname = os.path.basename(fpath)
try:
self.swift_conn.delete_object(
container=self.container_name, obj=fname)
print('swift-file-server: Deleting file %s ... [OK]', fname)
except Exception as err:
print(
'swift-file-server: Deleting file %s ... [FAIL]: %s %s (IGNORED) ', fname, err, traceback.format_exc())
def getObjectContent(self, filename):
return self.swift_conn.get_object(container=self.container_name, obj=filename)
class SwiftSession(Session):
"""Configures access to secured resources stored in OpenStack Swift Object Storage.
"""
def __init__(self, session=None,
swift_storage_url=None, swift_auth_token=None,
swift_auth_v1_url=None, swift_user=None, swift_key=None):
"""Create new OpenStack Swift Object Storage Session.
Three methods are possible:
1. Create session by the swiftclient library.
2. The SWIFT_STORAGE_URL and SWIFT_AUTH_TOKEN (this method is recommended by GDAL docs).
3. The SWIFT_AUTH_V1_URL, SWIFT_USER and SWIFT_KEY (This depends on the swiftclient library).
Parameters
----------
session: optional
A swiftclient connection object
swift_storage_url:
the storage URL
swift_auth_token:
the value of the x-auth-token authorization token
swift_storage_url: string, optional
authentication URL
swift_user: string, optional
user name to authenticate as
swift_key: string, optional
key/password to authenticate with
Examples
--------
>>> import rasterio
>>> from rasterio.session import SwiftSession
>>> fp = '/vsiswift/bucket/key.tif'
>>> conn = Connection(authurl='http://127.0.0.1:7777/auth/v1.0', user='******', key='testing')
>>> session = SwiftSession(conn)
>>> with rasterio.Env(session):
>>> with rasterio.open(fp) as src:
>>> print(src.profile)
"""
if swift_storage_url and swift_auth_token:
self._creds = {
"swift_storage_url": swift_storage_url,
"swift_auth_token": swift_auth_token
}
else:
from swiftclient.client import Connection
if session:
self._session = session
else:
self._session = Connection(
authurl=swift_auth_v1_url,
user=swift_user,
key=swift_key
)
self._creds = {
"swift_storage_url": self._session.get_auth()[0],
"swift_auth_token": self._session.get_auth()[1]
}
@classmethod
def hascreds(cls, config):
"""Determine if the given configuration has proper credentials
Parameters
----------
cls : class
A Session class.
config : dict
GDAL configuration as a dict.
Returns
-------
bool
"""
return 'SWIFT_STORAGE_URL' in config and 'SWIFT_AUTH_TOKEN' in config
@property
def credentials(self):
"""The session credentials as a dict"""
return self._creds
def get_credential_options(self):
"""Get credentials as GDAL configuration options
Returns
-------
dict
"""
return {k.upper(): v for k, v in self.credentials.items()}
