def test_generate_temp_url_bad_seconds(self):
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, 'not_an_int', self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'seconds must be a whole number')
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, -1, self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'seconds must be a whole number')
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, 1.1, self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'seconds must be a whole number')
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, '-1', self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'seconds must be a whole number')
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, '1.1', self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'seconds must be a whole number')
def test_generate_temp_url_bad_seconds(self):
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, 'not_an_int', self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'seconds must be a whole number')
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, -1, self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'seconds must be a whole number')
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, 1.1, self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'seconds must be a whole number')
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, '-1', self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'seconds must be a whole number')
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, '1.1', self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'seconds must be a whole number')
def test_generate_temp_url_bad_path(self):
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url('/v1/a/c', 60, self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'path must be full path to an object e.g. /v1/a/c/o')
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url('v1/a/c/o', 60, self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'path must be full path to an object e.g. /v1/a/c/o')
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url('blah/v1/a/c/o', 60, self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'path must be full path to an object e.g. /v1/a/c/o')
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url('/v1//c/o', 60, self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'path must be full path to an object e.g. /v1/a/c/o')
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url('/v1/a/c/', 60, self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'path must be full path to an object e.g. /v1/a/c/o')
def test_generate_temp_url_bad_path(self):
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url('/v1/a/c', 60, self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'path must be full path to an object e.g. /v1/a/c/o')
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url('v1/a/c/o', 60, self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'path must be full path to an object e.g. /v1/a/c/o')
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url('blah/v1/a/c/o', 60, self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'path must be full path to an object e.g. /v1/a/c/o')
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url('/v1//c/o', 60, self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'path must be full path to an object e.g. /v1/a/c/o')
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url('/v1/a/c/', 60, self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'path must be full path to an object e.g. /v1/a/c/o')
def test_generate_temp_url_bad_seconds(self):
with self.assertRaises(TypeError) as exc_manager:
u.generate_temp_url(self.url, 'not_an_int', self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'seconds must be an integer')
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, -1, self.key, self.method)
self.assertEqual(exc_manager.exception.args[0],
'seconds must be a positive integer')
def get_temp_url(self, container, obj, timeout):
"""Returns the temp url for the given Swift object.
:param container: The name of the container in which Swift object
is placed.
:param obj: The name of the Swift object.
:param timeout: The timeout in seconds after which the generated url
should expire.
:returns: The temp url for the object.
:raises: SwiftOperationError, if any operation with Swift fails.
"""
try:
account_info = self.connection.head_account()
except swift_exceptions.ClientException as e:
operation = _("head account")
raise exception.SwiftOperationError(operation=operation, error=e)
parse_result = urlparse.urlparse(self.connection.url)
swift_object_path = '/'.join((parse_result.path, container, obj))
temp_url_key = account_info.get('x-account-meta-temp-url-key')
if not temp_url_key:
raise exception.MissingParameterValue(
_('Swift temporary URLs require a shared secret to be '
'created. You must provide pre-generate the key on '
'the project used to access Swift.'))
url_path = swift_utils.generate_temp_url(swift_object_path, timeout,
temp_url_key, 'GET')
return urlparse.urlunparse((parse_result.scheme, parse_result.netloc,
url_path, None, None, None))
def get_temp_url(self, container_name, obj_name, timeout=None,
method='PUT'):
'''
Return a Swift TempURL.
'''
def tenant_uuid():
access = self.context.auth_token_info['access']
for role in access['user']['roles']:
if role['name'] == 'object-store:default':
return role['tenantId']
key_header = 'x-account-meta-temp-url-key'
if key_header in self.client().head_account():
key = self.client().head_account()[key_header]
else:
key = hashlib.sha224(str(random.getrandbits(256))).hexdigest()[:32]
self.client().post_account({key_header: key})
path = '/v1/%s/%s/%s' % (tenant_uuid(), container_name, obj_name)
if timeout is None:
timeout = swift.MAX_EPOCH - 60 - time.time()
tempurl = swiftclient_utils.generate_temp_url(path, timeout, key,
method)
sw_url = parse.urlparse(self.client().url)
return '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl)
def test_generate_temp_url(self, time_mock, hmac_mock):
expected_url = ('/v1/AUTH_account/c/o?'
'temp_url_sig=temp_url_signature&'
'temp_url_expires=1400003600')
url = u.generate_temp_url(self.url, self.seconds, self.key,
self.method)
self.assertEqual(url, expected_url)
def get_temp_url(self, container, object, timeout):
"""Returns the temp url for the given Swift object.
:param container: The name of the container in which Swift object
is placed.
:param object: The name of the Swift object.
:param timeout: The timeout in seconds after which the generated url
should expire.
:returns: The temp url for the object.
:raises: SwiftOperationError, if any operation with Swift fails.
"""
try:
account_info = self.connection.head_account()
except swift_exceptions.ClientException as e:
operation = _("head account")
raise exception.SwiftOperationError(operation=operation,
error=e)
storage_url, token = self.connection.get_auth()
parse_result = parse.urlparse(storage_url)
swift_object_path = '/'.join((parse_result.path, container, object))
temp_url_key = account_info['x-account-meta-temp-url-key']
url_path = swift_utils.generate_temp_url(swift_object_path, timeout,
temp_url_key, 'GET')
return parse.urlunparse((parse_result.scheme,
parse_result.netloc,
url_path,
None,
None,
None))
def generate_presigned_url(self, key):
"""
Generates temporary public URL from given full key
to swift object
:param key: Full path to the object.
:return: Full URL to the object for unauthenticated used to
being able to download object.
"""
full_path = "/v1/{}/{}/{}".format(
self.account,
self.config.bucket,
key
)
url = generate_temp_url(
full_path,
self.config.expires,
self.temporary_url_key,
'GET'
).split(self.config.bucket)[1]
if not self.config.swift_url_prefix.endswith('/'):
if url.startswith('/'):
return "{}{}".format(self.config.swift_url_prefix, url)
else:
return "{}/{}".format(self.config.swift_url_prefix, url)
return self.config.swift_url_prefix[:-1] + url
async def swift_download_object(
request: aiohttp.web.Request) -> aiohttp.web.Response:
"""Point a user to a temporary pre-signed download URL."""
session = api_check(request)
request.app['Log'].info(
'API call for download object from {0}, sess: {1} :: {2}'.format(
request.remote,
session,
time.ctime(),
))
serv = request.app['Creds'][session]['ST_conn']
sess = request.app['Creds'][session]['OS_sess']
temp_url_key = await get_tempurl_key(serv)
request.app['Log'].debug("Using %s as temporary URL key", temp_url_key)
# Generate temporary URL
host = sess.get_endpoint(service_type="object-store").split('/v1')[0]
path_begin = sess.get_endpoint(service_type="object-store").replace(
host, "")
request.app['Log'].debug("Using %s as host and %s as path start.", host,
path_begin)
container = request.query['bucket']
object_key = request.query['objkey']
lifetime = 60 * 15
# In the path creation, the stats['items'][0][1] is the tenant id from
# server statistics, the order should be significant, so this shouldn't
# be a problem
path = f'{path_begin}/{container}/{object_key}'
dloadurl = (host + generate_temp_url(path, lifetime, temp_url_key, 'GET'))
response = aiohttp.web.Response(status=302, )
response.headers['Location'] = dloadurl
return response
def get_temp_url(self,
container_name,
obj_name,
timeout=None,
method='PUT'):
"""Return a Swift TempURL."""
def tenant_uuid():
access = self.context.auth_token_info['access']
for role in access['user']['roles']:
if role['name'] == 'object-store:default':
return role['tenantId']
key_header = 'x-account-meta-temp-url-key'
if key_header in self.client().head_account():
key = self.client().head_account()[key_header]
else:
key = hashlib.sha224(str(random.getrandbits(256))).hexdigest()[:32]
self.client().post_account({key_header: key})
path = '/v1/%s/%s/%s' % (tenant_uuid(), container_name, obj_name)
if timeout is None:
timeout = swift.MAX_EPOCH - 60 - time.time()
tempurl = swiftclient_utils.generate_temp_url(path, timeout, key,
method)
sw_url = parse.urlparse(self.client().url)
return '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl)
def test_generate_temp_url_prefix(self, time_mock, hmac_mock):
hmac_mock().hexdigest.return_value = 'temp_url_signature'
prefixes = ['', 'o', 'p0/p1/']
for p in prefixes:
hmac_mock.reset_mock()
path = '/v1/AUTH_account/c/' + p
expected_url = path + ('?temp_url_sig=temp_url_signature'
'&temp_url_expires=1400003600'
'&temp_url_prefix=' + p)
expected_body = '\n'.join([
self.method,
'1400003600',
'prefix:' + path,
]).encode('utf-8')
url = u.generate_temp_url(path, self.seconds,
self.key, self.method, prefix=True)
key = self.key
if not isinstance(key, six.binary_type):
key = key.encode('utf-8')
self.assertEqual(url, expected_url)
self.assertEqual(hmac_mock.mock_calls, [
mock.call(key, expected_body, sha1),
mock.call().hexdigest(),
])
self.assertIsInstance(url, type(path))
def publish_template(self, name, contents):
oc = self.object_client
# post the object
oc.put_object(self.object_container_name, name, contents)
# TODO(asalkeld) see if this is causing problems.
# self.addCleanup(self.object_client.delete_object,
# self.object_container_name, name)
# make the tempurl
key_header = 'x-account-meta-temp-url-key'
if key_header not in oc.head_account():
swift_key = hashlib.sha224(str(
random.getrandbits(256))).hexdigest()[:32]
LOG.warn('setting swift key to %s' % swift_key)
oc.post_account({key_header: swift_key})
key = oc.head_account()[key_header]
path = '/v1/AUTH_%s/%s/%s' % (self.project_id,
self.object_container_name, name)
timeout = self.conf.build_timeout * 10
tempurl = swiftclient_utils.generate_temp_url(path, timeout, key,
'GET')
sw_url = parse.urlparse(oc.url)
full_url = '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl)
def download():
r = requests.get(full_url)
LOG.info('GET: %s -> %s' % (full_url, r.status_code))
return r.status_code == requests.codes.ok
# make sure that the object is available.
test.call_until_true(self.conf.build_timeout, self.conf.build_interval,
download)
return full_url
def test_generate_temp_url_prefix(self, time_mock, hmac_mock):
hmac_mock().hexdigest.return_value = 'temp_url_signature'
prefixes = ['', 'o', 'p0/p1/']
for p in prefixes:
hmac_mock.reset_mock()
path = '/v1/AUTH_account/c/' + p
expected_url = path + ('?temp_url_sig=temp_url_signature'
'&temp_url_expires=1400003600'
'&temp_url_prefix=' + p)
expected_body = '\n'.join([
self.method,
'1400003600',
'prefix:' + path,
]).encode('utf-8')
url = u.generate_temp_url(path,
self.seconds,
self.key,
self.method,
prefix=True)
key = self.key
if not isinstance(key, six.binary_type):
key = key.encode('utf-8')
self.assertEqual(url, expected_url)
self.assertEqual(hmac_mock.mock_calls, [
mock.call(key, expected_body, sha1),
mock.call().hexdigest(),
])
self.assertIsInstance(url, type(path))
async def swift_download_object(request: aiohttp.web.Request) -> aiohttp.web.Response:
"""Point a user to a temporary pre-signed download URL."""
session = api_check(request)
request.app["Log"].info(
"API call for download object from "
f"{request.remote}, sess: {session} :: {time.ctime()}"
)
serv = request.app["Sessions"][session]["ST_conn"]
sess = request.app["Sessions"][session]["OS_sess"]
temp_url_key = await get_tempurl_key(serv)
request.app["Log"].debug(f"Using {temp_url_key} as temporary URL key")
# Generate temporary URL
host = sess.get_endpoint(service_type="object-store").split("/v1")[0]
path_begin = sess.get_endpoint(service_type="object-store").replace(host, "")
request.app["Log"].debug(f"Using {host} as host and {path_begin} as path start.")
container = request.query["bucket"]
object_key = request.query["objkey"]
lifetime = 60 * 15
# In the path creation, the stats['items'][0][1] is the tenant id from
# server statistics, the order should be significant, so this shouldn't
# be a problem
path = f"{path_begin}/{container}/{object_key}"
dloadurl = host + generate_temp_url(path, lifetime, temp_url_key, "GET")
response = aiohttp.web.Response(
status=302,
)
response.headers["Location"] = dloadurl
return response
def test_generate_absolute_expiry_temp_url(self, hmac_mock):
expected_url = ('/v1/AUTH_account/c/o?'
'temp_url_sig=temp_url_signature&'
'temp_url_expires=2146636800')
url = u.generate_temp_url(self.url, 2146636800, self.key, self.method,
absolute=True)
self.assertEqual(url, expected_url)
def get_temp_url(self,
container_name,
obj_name,
timeout=None,
method='PUT'):
'''
Return a Swift TempURL.
'''
key_header = 'x-account-meta-temp-url-key'
if key_header not in self.client().head_account():
self.client().post_account({
key_header:
hashlib.sha224(str(random.getrandbits(256))).hexdigest()[:32]
})
key = self.client().head_account()[key_header]
path = '/v1/AUTH_%s/%s/%s' % (self.context.tenant_id, container_name,
obj_name)
if timeout is None:
timeout = MAX_EPOCH - 60 - time.time()
tempurl = swiftclient_utils.generate_temp_url(path, timeout, key,
method)
sw_url = urlparse.urlparse(self.client().url)
return '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl)
def publish_template(self, name, contents):
oc = self.object_client
# post the object
oc.put_object(self.object_container_name, name, contents)
# TODO(asalkeld) see if this is causing problems.
# self.addCleanup(self.object_client.delete_object,
# self.object_container_name, name)
# make the tempurl
key_header = 'x-account-meta-temp-url-key'
if key_header not in oc.head_account():
swift_key = hashlib.sha224(
str(random.getrandbits(256))).hexdigest()[:32]
LOG.warn('setting swift key to %s' % swift_key)
oc.post_account({key_header: swift_key})
key = oc.head_account()[key_header]
path = '/v1/AUTH_%s/%s/%s' % (self.project_id,
self.object_container_name, name)
timeout = self.conf.build_timeout * 10
tempurl = swiftclient_utils.generate_temp_url(path, timeout,
key, 'GET')
sw_url = parse.urlparse(oc.url)
full_url = '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl)
def download():
r = requests.get(full_url)
LOG.info('GET: %s -> %s' % (full_url, r.status_code))
return r.status_code == requests.codes.ok
# make sure that the object is available.
test.call_until_true(self.conf.build_timeout,
self.conf.build_interval, download)
return full_url
def test_generate_temp_url(self, time_mock, hmac_mock):
expected_url = (
'/v1/AUTH_account/c/o?'
'temp_url_sig=temp_url_signature&'
'temp_url_expires=1400003600')
url = u.generate_temp_url(self.url, self.seconds, self.key,
self.method)
self.assertEqual(url, expected_url)
async def swift_download_object(
request: aiohttp.web.Request) -> aiohttp.web.Response:
"""Point a user to a temporary pre-signed download URL."""
session = api_check(request)
request.app['Log'].info(
'API call for download object from {0}, sess: {1} :: {2}'.format(
request.remote,
session,
time.ctime(),
))
serv = request.app['Creds'][session]['ST_conn']
sess = request.app['Creds'][session]['OS_sess']
stats = serv.stat()
# Check for the existence of the key headers
acc_meta_hdr = stats['headers']
if 'x-account-meta-temp-url-key' in acc_meta_hdr.keys():
temp_url_key = acc_meta_hdr['x-account-meta-temp-url-key']
elif 'x-acccount-meta-temp-url-key-2' in acc_meta_hdr.keys():
temp_url_key = acc_meta_hdr['x-account-meta-temp-url-key-2']
# If the key headers don't exist, assume that the key has to be created by
# the service
else:
# The hash only provides random data for the key, it doesn't have to
# be cryptographically secure.
temp_url_key = hashlib.md5(os.urandom(128)).hexdigest() # nosec
# This service will use the X-Account-Meta-Temp-URL-Key-2 header for
# its own key storage, if no existing keys are provided.
meta_options = {"meta": ["Temp-URL-Key-2:{0}".format(temp_url_key)]}
retval = serv.post(options=meta_options)
if not retval['success']:
raise aiohttp.web.HTTPServerError()
request.app['Log'].info(
"Created a temp url key for account {0} Key:{1} :: {2}".format(
stats['items'][0][1], temp_url_key, time.ctime()))
request.app['Log'].debug("Using {0} as temporary URL key :: {1}".format(
temp_url_key, time.ctime()))
# Generate temporary URL
host = sess.get_endpoint(service_type="object-store").split('/v1')[0]
path_begin = sess.get_endpoint(service_type="object-store").replace(
host, "")
request.app['Log'].debug("Using %s as host and %s as path start.", host,
path_begin)
container = request.query['bucket']
object_key = request.query['objkey']
lifetime = 60 * 15
# In the path creation, the stats['items'][0][1] is the tenant id from
# server statistics, the order should be significant, so this shouldn't
# be a problem
path = '%s/%s/%s' % (path_begin, container, object_key)
dloadurl = (host + generate_temp_url(path, lifetime, temp_url_key, 'GET'))
response = aiohttp.web.Response(status=302, )
response.headers['Location'] = dloadurl
return response
def generate_temp_report_url(account,
container,
key,
temp_url_key,
expires=60 * 60 * 48):
full_path = "/v1/{}/{}/{}".format(account, container, key)
url = generate_temp_url(full_path, int(time() + int(expires)),
temp_url_key, 'GET')
return url
def test_generate_absolute_expiry_temp_url(self, hmac_mock):
if isinstance(self.expected_url, six.binary_type):
expected_url = self.expected_url.replace(
b'1400003600', b'2146636800')
else:
expected_url = self.expected_url.replace(
u'1400003600', u'2146636800')
url = u.generate_temp_url(self.url, 2146636800, self.key, self.method,
absolute=True)
self.assertEqual(url, expected_url)
def test_generate_absolute_expiry_temp_url(self, hmac_mock):
if isinstance(self.expected_url, six.binary_type):
expected_url = self.expected_url.replace(
b'1400003600', b'2146636800')
else:
expected_url = self.expected_url.replace(
u'1400003600', u'2146636800')
url = u.generate_temp_url(self.url, 2146636800, self.key, self.method,
absolute=True)
self.assertEqual(url, expected_url)
def generate_url(self, expires_in):
# Set a temp url key on the object (http://bit.ly/2NBiXGD)
temp_url_key = "cloudbridge-tmp-url-key"
self._provider.swift.post_account(
headers={"x-account-meta-temp-url-key": temp_url_key})
base_url = urlparse(self._provider.swift.get_service_auth()[0])
access_point = "{0}://{1}".format(base_url.scheme, base_url.netloc)
url_path = "/".join([base_url.path, self.cbcontainer.name, self.name])
return urljoin(access_point, generate_temp_url(url_path, expires_in,
temp_url_key, 'GET'))
def get_temp_url(self, container_name, object_name):
tenant_id = self.auth['tenant_id']
url = f'/v1/nc_{tenant_id}/{container_name}/{object_name}'
key = self.auth['tempurl_key'].encode('utf-8')
method = 'GET'
seconds = 60 * 60 * 24
temp_url = generate_temp_url(path=url,
seconds=seconds,
key=key,
method=method)
return 'https://object-storage.tyo2.conoha.io' + temp_url
def getTempUrl(filename):
# f = open("mykey.txt")
# mykey = f.read()
ConnUtil.con.post_account(
headers={"X-Account-Meta-Temp-Url-Key": "mykey"}) # temp account
return ConnUtil.end_point + generate_temp_url(
"/v1/AUTH_test/user_uploads/" + filename,
seconds=3600,
key="mykey",
method='GET')
f.close()
def _path(self, name):
url = urlparse.urljoin(self.base_url, urlparse.quote(name))
# Are we building a temporary url?
if self.use_temp_urls:
expires = int(time() + int(self.temp_url_duration))
path = urlparse.unquote(urlparse.urlsplit(url).path)
tmp_path = generate_temp_url(path, expires, self.temp_url_key, 'GET', absolute=True)
url = urlparse.urljoin(self.base_url, tmp_path)
return url
def test_generate_temp_url_ip_range(self, time_mock, hmac_mock):
hmac_mock().hexdigest.return_value = 'temp_url_signature'
ip_ranges = [
'1.2.3.4',
'1.2.3.4/24',
'2001:db8::',
b'1.2.3.4',
b'1.2.3.4/24',
b'2001:db8::',
]
path = '/v1/AUTH_account/c/o/'
expected_url = path + ('?temp_url_sig=temp_url_signature'
'&temp_url_expires=1400003600'
'&temp_url_ip_range=')
for ip_range in ip_ranges:
hmac_mock.reset_mock()
url = u.generate_temp_url(path,
self.seconds,
self.key,
self.method,
ip_range=ip_range)
key = self.key
if not isinstance(key, six.binary_type):
key = key.encode('utf-8')
if isinstance(ip_range, six.binary_type):
ip_range_expected_url = (expected_url +
ip_range.decode('utf-8'))
expected_body = '\n'.join([
'ip=' + ip_range.decode('utf-8'),
self.method,
'1400003600',
path,
]).encode('utf-8')
else:
ip_range_expected_url = expected_url + ip_range
expected_body = '\n'.join([
'ip=' + ip_range,
self.method,
'1400003600',
path,
]).encode('utf-8')
self.assertEqual(url, ip_range_expected_url)
self.assertEqual(hmac_mock.mock_calls, [
mock.call(key, expected_body, sha1),
mock.call().hexdigest(),
])
self.assertIsInstance(url, type(path))
def test_generate_temp_url(self, time_mock, hmac_mock):
hmac_mock().hexdigest.return_value = 'temp_url_signature'
url = u.generate_temp_url(self.url, self.seconds, self.key,
self.method)
key = self.key
if not isinstance(key, six.binary_type):
key = key.encode('utf-8')
self.assertEqual(url, self.expected_url)
self.assertEqual(hmac_mock.mock_calls, [
mock.call(),
mock.call(key, self.expected_body, sha1),
mock.call().hexdigest(),
])
self.assertIsInstance(url, type(self.url))
def generate_temp_url(self, user_id, path, name):
container, object_name = self.build_object_name(user_id, path, name)
# discover swift endpoint urls
conn = get_conn(self.swift._options)
url, token = conn.get_auth()
urlparts = urlparse(url)
# generate swift path /v1/<account>/<container>/<userid>/path
path = '/'.join((urlparts.path, container, object_name))
# TODO: valid for 5 minutes
temp_url = generate_temp_url(path,
300,
self.temp_url_key,
method='GET')
return urljoin(url, temp_url)
def swift_temp_url(self, image_info):
"""Generate a no-auth Swift temporary URL.
This function will generate the temporary Swift URL using the image
id from Glance and the config options: 'swift_endpoint_url',
'swift_api_version', 'swift_account' and 'swift_container'.
The temporary URL will be valid for 'swift_temp_url_duration' seconds.
This allows Ironic to download a Glance image without passing around
an auth_token.
:param image_info: The return from a GET request to Glance for a
certain image_id. Should be a dictionary, with keys like 'name' and
'checksum'. See
http://docs.openstack.org/developer/glance/glanceapi.html for
examples.
:returns: A signed Swift URL from which an image can be downloaded,
without authentication.
:raises: InvalidParameterValue if Swift config options are not set
correctly.
:raises: MissingParameterValue if a required parameter is not set.
:raises: ImageUnacceptable if the image info from Glance does not
have a image ID.
"""
self._validate_temp_url_config()
if ('id' not in image_info or not
utils.is_uuid_like(image_info['id'])):
raise exc.ImageUnacceptable(_(
'The given image info does not have a valid image id: %s')
% image_info)
url_fragments = {
'endpoint_url': CONF.glance.swift_endpoint_url,
'api_version': CONF.glance.swift_api_version,
'account': CONF.glance.swift_account,
'container': self._get_swift_container(image_info['id']),
'object_id': image_info['id']
}
template = '/{api_version}/{account}/{container}/{object_id}'
url_path = template.format(**url_fragments)
path = swift_utils.generate_temp_url(
path=url_path,
seconds=CONF.glance.swift_temp_url_duration,
key=CONF.glance.swift_temp_url_key,
method='GET')
return '{endpoint_url}{url_path}'.format(
endpoint_url=url_fragments['endpoint_url'], url_path=path)
def test_generate_temp_url(self, time_mock, hmac_mock):
hmac_mock().hexdigest.return_value = 'temp_url_signature'
url = u.generate_temp_url(self.url, self.seconds,
self.key, self.method)
key = self.key
if not isinstance(key, six.binary_type):
key = key.encode('utf-8')
self.assertEqual(url, self.expected_url)
self.assertEqual(hmac_mock.mock_calls, [
mock.call(),
mock.call(key, self.expected_body, sha1),
mock.call().hexdigest(),
])
self.assertIsInstance(url, type(self.url))
def swift_temp_url(self, image_info):
"""Generate a no-auth Swift temporary URL.
This function will generate the temporary Swift URL using the image
id from Glance and the config options: 'swift_endpoint_url',
'swift_api_version', 'swift_account' and 'swift_container'.
The temporary URL will be valid for 'swift_temp_url_duration' seconds.
This allows Ironic to download a Glance image without passing around
an auth_token.
:param image_info: The return from a GET request to Glance for a
certain image_id. Should be a dictionary, with keys like 'name' and
'checksum'. See
http://docs.openstack.org/developer/glance/glanceapi.html for
examples.
:returns: A signed Swift URL from which an image can be downloaded,
without authentication.
:raises: InvalidParameterValue if Swift config options are not set
correctly.
:raises: MissingParameterValue if a required parameter is not set.
:raises: ImageUnacceptable if the image info from Glance does not
have a image ID.
"""
self._validate_temp_url_config()
if ('id' not in image_info or not
utils.is_uuid_like(image_info['id'])):
raise exc.ImageUnacceptable(_(
'The given image info does not have a valid image id: %s')
% image_info)
url_fragments = {
'endpoint_url': CONF.glance.swift_endpoint_url,
'api_version': CONF.glance.swift_api_version,
'account': CONF.glance.swift_account,
'container': self._get_swift_container(image_info['id']),
'object_id': image_info['id']
}
template = '/{api_version}/{account}/{container}/{object_id}'
url_path = template.format(**url_fragments)
path = swift_utils.generate_temp_url(
path=url_path,
seconds=CONF.glance.swift_temp_url_duration,
key=CONF.glance.swift_temp_url_key,
method='GET')
return '{endpoint_url}{url_path}'.format(
endpoint_url=url_fragments['endpoint_url'], url_path=path)
def test_generate_temp_url_iso8601_argument(self, hmac_mock):
hmac_mock().hexdigest.return_value = 'temp_url_signature'
url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z', self.key,
self.method)
self.assertEqual(url, self.expected_url)
# Don't care about absolute arg.
url = u.generate_temp_url(self.url,
'2014-05-13T17:53:20Z',
self.key,
self.method,
absolute=True)
self.assertEqual(url, self.expected_url)
lt = localtime()
expires = strftime(u.EXPIRES_ISO8601_FORMAT[:-1], lt)
if not isinstance(self.expected_url, six.string_types):
expected_url = self.expected_url.replace(
b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii'))
else:
expected_url = self.expected_url.replace('1400003600',
str(int(mktime(lt))))
url = u.generate_temp_url(self.url, expires, self.key, self.method)
self.assertEqual(url, expected_url)
expires = strftime(u.SHORT_EXPIRES_ISO8601_FORMAT, lt)
lt = strptime(expires, u.SHORT_EXPIRES_ISO8601_FORMAT)
if not isinstance(self.expected_url, six.string_types):
expected_url = self.expected_url.replace(
b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii'))
else:
expected_url = self.expected_url.replace('1400003600',
str(int(mktime(lt))))
url = u.generate_temp_url(self.url, expires, self.key, self.method)
self.assertEqual(url, expected_url)
def test_generate_temp_url_iso8601_argument(self, hmac_mock):
hmac_mock().hexdigest.return_value = 'temp_url_signature'
url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z',
self.key, self.method)
self.assertEqual(url, self.expected_url)
# Don't care about absolute arg.
url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z',
self.key, self.method, absolute=True)
self.assertEqual(url, self.expected_url)
lt = localtime()
expires = strftime(u.EXPIRES_ISO8601_FORMAT[:-1], lt)
if not isinstance(self.expected_url, six.string_types):
expected_url = self.expected_url.replace(
b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii'))
else:
expected_url = self.expected_url.replace(
'1400003600', str(int(mktime(lt))))
url = u.generate_temp_url(self.url, expires,
self.key, self.method)
self.assertEqual(url, expected_url)
expires = strftime(u.SHORT_EXPIRES_ISO8601_FORMAT, lt)
lt = strptime(expires, u.SHORT_EXPIRES_ISO8601_FORMAT)
if not isinstance(self.expected_url, six.string_types):
expected_url = self.expected_url.replace(
b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii'))
else:
expected_url = self.expected_url.replace(
'1400003600', str(int(mktime(lt))))
url = u.generate_temp_url(self.url, expires,
self.key, self.method)
self.assertEqual(url, expected_url)
def publish_template(self, contents, cleanup=True):
oc = self.object_client
# post the object
oc.put_object(self.object_container_name, 'template.yaml', contents)
if cleanup:
self.addCleanup(oc.delete_object, self.object_container_name,
'template.yaml')
path = '/v1/AUTH_%s/%s/%s' % (
self.project_id, self.object_container_name, 'template.yaml')
timeout = self.conf.build_timeout * 10
tempurl = swiftclient_utils.generate_temp_url(path, timeout,
self.swift_key, 'GET')
sw_url = parse.urlparse(oc.url)
return '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl)
def _path(self, name):
try:
name = name.encode('utf-8')
except UnicodeDecodeError:
pass
url = urlparse.urljoin(self.base_url, urlparse.quote(name))
# Are we building a temporary url?
if self.use_temp_urls:
expires = int(time() + int(self.temp_url_duration))
path = urlparse.unquote(urlparse.urlsplit(url).path)
tmp_path = generate_temp_url(path, expires, self.temp_url_key, 'GET', absolute=True)
url = urlparse.urljoin(self.base_url, tmp_path)
return url
def publish_template(self, contents, cleanup=True):
oc = self.object_client
# post the object
oc.put_object(self.object_container_name, 'template.yaml', contents)
if cleanup:
self.addCleanup(oc.delete_object,
self.object_container_name,
'template.yaml')
path = '/v1/AUTH_%s/%s/%s' % (self.project_id,
self.object_container_name,
'template.yaml')
timeout = self.conf.build_timeout * 10
tempurl = swiftclient_utils.generate_temp_url(path, timeout,
self.swift_key, 'GET')
sw_url = parse.urlparse(oc.url)
return '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl)
def get_temp_download_url(self, fname, expires):
""" return the temporary download url """
file_uri = '%s/%s/%s' % (self.storageurl, self.container_name, fname)
file_path = urlparse(file_uri).path
key = getattr(self, self.chosen_temp_url_key)
try:
temp_url = generate_temp_url(file_path, expires, key, 'GET')
except Exception as err:
logging.error(
'swift-file-server: Generating temp url for %s failed %s', fname, err)
raise
download_url = self.storageurl.replace(
urlparse(self.storageurl).path, temp_url)
print(
'swift-file-server: Temporary download URL for file %s: %s', fname, download_url)
return download_url
def run(self, context):
swift_client = self.get_object_client(context)
try:
cont_stat = swift_client.head_container(self.container)
except swiftexceptions.ClientException:
cont_stat = {}
key = cont_stat.get('x-container-meta-temp-url-key')
if not key:
key = str(uuid.uuid4())
cont_stat = swift_client.put_container(
self.container, {'X-Container-Meta-Temp-Url-Key': key})
parsed = urllib.parse.urlparse(swift_client.url)
path = "%s/%s/%s" % (parsed.path, self.container, self.obj)
temp_path = generate_temp_url(path, self.valid, key, self.method)
return "%s://%s%s" % (parsed.scheme, parsed.netloc, temp_path)
def run(self, context):
swift_client = self.get_object_client(context)
try:
cont_stat = swift_client.head_container(self.container)
except swiftexceptions.ClientException:
cont_stat = {}
key = cont_stat.get('x-container-meta-temp-url-key')
if not key:
key = str(uuid.uuid4())
cont_stat = swift_client.put_container(
self.container, {'X-Container-Meta-Temp-Url-Key': key})
parsed = urllib.parse.urlparse(swift_client.url)
path = "%s/%s/%s" % (parsed.path, self.container, self.obj)
temp_path = generate_temp_url(path, self.valid, key, self.method)
return "%s://%s%s" % (parsed.scheme, parsed.netloc, temp_path)
def test_generate_temp_url_ip_range(self, time_mock, hmac_mock):
hmac_mock().hexdigest.return_value = 'temp_url_signature'
ip_ranges = [
'1.2.3.4', '1.2.3.4/24', '2001:db8::',
b'1.2.3.4', b'1.2.3.4/24', b'2001:db8::',
]
path = '/v1/AUTH_account/c/o/'
expected_url = path + ('?temp_url_sig=temp_url_signature'
'&temp_url_expires=1400003600'
'&temp_url_ip_range=')
for ip_range in ip_ranges:
hmac_mock.reset_mock()
url = u.generate_temp_url(path, self.seconds,
self.key, self.method,
ip_range=ip_range)
key = self.key
if not isinstance(key, six.binary_type):
key = key.encode('utf-8')
if isinstance(ip_range, six.binary_type):
ip_range_expected_url = (
expected_url + ip_range.decode('utf-8')
)
expected_body = '\n'.join([
'ip=' + ip_range.decode('utf-8'),
self.method,
'1400003600',
path,
]).encode('utf-8')
else:
ip_range_expected_url = expected_url + ip_range
expected_body = '\n'.join([
'ip=' + ip_range,
self.method,
'1400003600',
path,
]).encode('utf-8')
self.assertEqual(url, ip_range_expected_url)
self.assertEqual(hmac_mock.mock_calls, [
mock.call(key, expected_body, sha1),
mock.call().hexdigest(),
])
self.assertIsInstance(url, type(path))
def outputs(request):
# TODO: maybe use server:outputpath here?
container = config.get_config_value('SwiftStorage', 'container')
# TODO: TEM_PURL_KEY not needed in pywps
temp_url_key = get_temp_url_key()
path_prefix = '/v1/AUTH_{}/{}/'.format(os.environ['OS_PROJECT_ID'],
container)
temp_url = generate_temp_url(
path=path_prefix + '/'.join(request.matchdict['filename']),
seconds=60 * 60 * 24, # temp url valid for 24hrs
key=temp_url_key,
method='GET',
# prefix=True,
# iso8601=True,
# ip_range=???
)
return HTTPFound(location='https://swift.rc.nectar.org.au' + temp_url)
def get_temp_download_url(self, fname, expires):
""" return the temporary download url """
file_uri = '%s/%s/%s' % (self.storageurl, self.container_name, fname)
file_path = urlparse(file_uri).path
key = getattr(self, self.chosen_temp_url_key)
try:
temp_url = generate_temp_url(file_path, expires, key, 'GET')
except Exception as err:
logging.error(
'swift-file-server: Generating temp url for %s failed %s',
fname, err)
raise
download_url = self.storageurl.replace(
urlparse(self.storageurl).path, temp_url)
print('swift-file-server: Temporary download URL for file %s: %s',
fname, download_url)
return download_url
def swift_tempurl(path, date):
swift_endpoint = environ.get('SWIFT_ENDPOINT')
swift_path = environ.get('SWIFT_PATH')
swift_tempurl_key = environ.get('SWIFT_TEMPURL_KEY')
assert (swift_endpoint)
assert (swift_path)
assert (swift_tempurl_key)
path = "{}{}".format(swift_path, path)
timestamp = int(date.strftime("%s"))
temp_url = generate_temp_url(path,
timestamp,
swift_tempurl_key,
'GET',
absolute=True)
return "{}{}".format(swift_endpoint, temp_url)
def test_generate_temp_url_iso8601_output(self, time_mock, hmac_mock):
hmac_mock().hexdigest.return_value = 'temp_url_signature'
url = u.generate_temp_url(self.url, self.seconds,
self.key, self.method,
iso8601=True)
key = self.key
if not isinstance(key, six.binary_type):
key = key.encode('utf-8')
expires = strftime(u.EXPIRES_ISO8601_FORMAT, gmtime(1400003600))
if not isinstance(self.url, six.string_types):
self.assertTrue(url.endswith(bytes(expires, 'utf-8')))
else:
self.assertTrue(url.endswith(expires))
self.assertEqual(hmac_mock.mock_calls, [
mock.call(),
mock.call(key, self.expected_body, sha1),
mock.call().hexdigest(),
])
self.assertIsInstance(url, type(self.url))
def get_temp_url(self, container_name, obj_name, timeout=None,
method='PUT'):
'''
Return a Swift TempURL.
'''
key_header = 'x-account-meta-temp-url-key'
if key_header in self.client().head_account():
key = self.client().head_account()[key_header]
else:
key = hashlib.sha224(str(random.getrandbits(256))).hexdigest()[:32]
self.client().post_account({key_header: key})
path = '/v1/AUTH_%s/%s/%s' % (self.context.tenant_id, container_name,
obj_name)
if timeout is None:
timeout = MAX_EPOCH - 60 - time.time()
tempurl = swiftclient_utils.generate_temp_url(path, timeout, key,
method)
sw_url = urlparse.urlparse(self.client().url)
return '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl)
def create_temp_url(swift_client, name, timeout, container=None):
container = container or '%(name)s-%(uuid)s' % {
'name': name, 'uuid': uuid.uuid4()}
object_name = str(uuid.uuid4())
swift_client.put_container(container)
key_header = 'x-account-meta-temp-url-key'
if key_header not in swift_client.head_account():
swift_client.post_account({
key_header: six.text_type(uuid.uuid4())[:32]})
key = swift_client.head_account()[key_header]
project_path = swift_client.url.split('/')[-1]
path = '/v1/%s/%s/%s' % (project_path, container, object_name)
timeout_secs = timeout * 60
tempurl = swiftclient_utils.generate_temp_url(path, timeout_secs, key,
'PUT')
sw_url = urlparse.urlparse(swift_client.url)
put_url = '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl)
swift_client.put_object(container, object_name, '')
return put_url
def _generate_temp_url(self, path, seconds, key, method, endpoint,
image_id):
"""Get Swift temporary URL.
Generates (or returns the cached one if caching is enabled) a
temporary URL that gives unauthenticated access to the Swift object.
:param path: The full path to the Swift object. Example:
/v1/AUTH_account/c/o.
:param seconds: The amount of time in seconds the temporary URL will
be valid for.
:param key: The secret temporary URL key set on the Swift cluster.
:param method: A HTTP method, typically either GET or PUT, to allow for
this temporary URL.
:param endpoint: Endpoint URL of Swift service.
:param image_id: UUID of a Glance image.
:returns: temporary URL
"""
if CONF.glance.swift_temp_url_cache_enabled:
self._remove_expired_items_from_cache()
if image_id in self._cache:
return self._cache[image_id].url
path = swift_utils.generate_temp_url(
path=path, seconds=seconds, key=key, method=method)
temp_url = '{endpoint_url}{url_path}'.format(
endpoint_url=endpoint, url_path=path)
if CONF.glance.swift_temp_url_cache_enabled:
query = urlparse.urlparse(temp_url).query
exp_time_str = dict(urlparse.parse_qsl(query))['temp_url_expires']
self._cache[image_id] = TempUrlCacheElement(
url=temp_url, url_expires_at=int(exp_time_str)
)
return temp_url
def publish_template(self, name, contents):
oc = self.object_client
# post the object
oc.put_object(self.object_container_name, name, contents)
# TODO(asalkeld) see if this is causing problems.
# self.addCleanup(self.object_client.delete_object,
# self.object_container_name, name)
# make the tempurl
key_header = 'x-account-meta-temp-url-key'
if key_header not in oc.head_account():
swift_key = hashlib.sha224(
str(random.getrandbits(256))).hexdigest()[:32]
LOG.warn('setting swift key to %s' % swift_key)
oc.post_account({key_header: swift_key})
key = oc.head_account()[key_header]
path = '/v1/AUTH_%s/%s/%s' % (self.project_id,
self.object_container_name, name)
timeout = self.conf.build_timeout * 10
tempurl = swiftclient_utils.generate_temp_url(path, timeout,
key, 'GET')
sw_url = parse.urlparse(oc.url)
return '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl)
def test_generate_temp_url_bad_time(self):
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, 'not_an_int', self.key, self.method)
self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG)
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, -1, self.key, self.method)
self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG)
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, 1.1, self.key, self.method)
self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG)
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, '-1', self.key, self.method)
self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG)
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, '1.1', self.key, self.method)
self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG)
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, '2015-05', self.key, self.method)
self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG)
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(
self.url, '2015-05-01T01:00', self.key, self.method)
self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG)
def test_generate_temp_url_invalid_path(self):
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(b'/v1/a/c/\xff', self.seconds, self.key,
self.method)
self.assertEqual(exc_manager.exception.args[0],
'path must be representable as UTF-8')
