def test_generate_temp_url_bad_seconds(self): with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(self.url, 'not_an_int', self.key, self.method) self.assertEqual(exc_manager.exception.args[0], 'seconds must be a whole number') with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(self.url, -1, self.key, self.method) self.assertEqual(exc_manager.exception.args[0], 'seconds must be a whole number') with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(self.url, 1.1, self.key, self.method) self.assertEqual(exc_manager.exception.args[0], 'seconds must be a whole number') with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(self.url, '-1', self.key, self.method) self.assertEqual(exc_manager.exception.args[0], 'seconds must be a whole number') with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(self.url, '1.1', self.key, self.method) self.assertEqual(exc_manager.exception.args[0], 'seconds must be a whole number')
def test_generate_temp_url_bad_path(self): with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url('/v1/a/c', 60, self.key, self.method) self.assertEqual(exc_manager.exception.args[0], 'path must be full path to an object e.g. /v1/a/c/o') with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url('v1/a/c/o', 60, self.key, self.method) self.assertEqual(exc_manager.exception.args[0], 'path must be full path to an object e.g. /v1/a/c/o') with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url('blah/v1/a/c/o', 60, self.key, self.method) self.assertEqual(exc_manager.exception.args[0], 'path must be full path to an object e.g. /v1/a/c/o') with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url('/v1//c/o', 60, self.key, self.method) self.assertEqual(exc_manager.exception.args[0], 'path must be full path to an object e.g. /v1/a/c/o') with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url('/v1/a/c/', 60, self.key, self.method) self.assertEqual(exc_manager.exception.args[0], 'path must be full path to an object e.g. /v1/a/c/o')
def test_generate_temp_url_bad_seconds(self): with self.assertRaises(TypeError) as exc_manager: u.generate_temp_url(self.url, 'not_an_int', self.key, self.method) self.assertEqual(exc_manager.exception.args[0], 'seconds must be an integer') with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(self.url, -1, self.key, self.method) self.assertEqual(exc_manager.exception.args[0], 'seconds must be a positive integer')
def get_temp_url(self, container, obj, timeout): """Returns the temp url for the given Swift object. :param container: The name of the container in which Swift object is placed. :param obj: The name of the Swift object. :param timeout: The timeout in seconds after which the generated url should expire. :returns: The temp url for the object. :raises: SwiftOperationError, if any operation with Swift fails. """ try: account_info = self.connection.head_account() except swift_exceptions.ClientException as e: operation = _("head account") raise exception.SwiftOperationError(operation=operation, error=e) parse_result = urlparse.urlparse(self.connection.url) swift_object_path = '/'.join((parse_result.path, container, obj)) temp_url_key = account_info.get('x-account-meta-temp-url-key') if not temp_url_key: raise exception.MissingParameterValue( _('Swift temporary URLs require a shared secret to be ' 'created. You must provide pre-generate the key on ' 'the project used to access Swift.')) url_path = swift_utils.generate_temp_url(swift_object_path, timeout, temp_url_key, 'GET') return urlparse.urlunparse((parse_result.scheme, parse_result.netloc, url_path, None, None, None))
def get_temp_url(self, container_name, obj_name, timeout=None, method='PUT'): ''' Return a Swift TempURL. ''' def tenant_uuid(): access = self.context.auth_token_info['access'] for role in access['user']['roles']: if role['name'] == 'object-store:default': return role['tenantId'] key_header = 'x-account-meta-temp-url-key' if key_header in self.client().head_account(): key = self.client().head_account()[key_header] else: key = hashlib.sha224(str(random.getrandbits(256))).hexdigest()[:32] self.client().post_account({key_header: key}) path = '/v1/%s/%s/%s' % (tenant_uuid(), container_name, obj_name) if timeout is None: timeout = swift.MAX_EPOCH - 60 - time.time() tempurl = swiftclient_utils.generate_temp_url(path, timeout, key, method) sw_url = parse.urlparse(self.client().url) return '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl)
def test_generate_temp_url(self, time_mock, hmac_mock): expected_url = ('/v1/AUTH_account/c/o?' 'temp_url_sig=temp_url_signature&' 'temp_url_expires=1400003600') url = u.generate_temp_url(self.url, self.seconds, self.key, self.method) self.assertEqual(url, expected_url)
def get_temp_url(self, container, object, timeout): """Returns the temp url for the given Swift object. :param container: The name of the container in which Swift object is placed. :param object: The name of the Swift object. :param timeout: The timeout in seconds after which the generated url should expire. :returns: The temp url for the object. :raises: SwiftOperationError, if any operation with Swift fails. """ try: account_info = self.connection.head_account() except swift_exceptions.ClientException as e: operation = _("head account") raise exception.SwiftOperationError(operation=operation, error=e) storage_url, token = self.connection.get_auth() parse_result = parse.urlparse(storage_url) swift_object_path = '/'.join((parse_result.path, container, object)) temp_url_key = account_info['x-account-meta-temp-url-key'] url_path = swift_utils.generate_temp_url(swift_object_path, timeout, temp_url_key, 'GET') return parse.urlunparse((parse_result.scheme, parse_result.netloc, url_path, None, None, None))
def generate_presigned_url(self, key): """ Generates temporary public URL from given full key to swift object :param key: Full path to the object. :return: Full URL to the object for unauthenticated used to being able to download object. """ full_path = "/v1/{}/{}/{}".format( self.account, self.config.bucket, key ) url = generate_temp_url( full_path, self.config.expires, self.temporary_url_key, 'GET' ).split(self.config.bucket)[1] if not self.config.swift_url_prefix.endswith('/'): if url.startswith('/'): return "{}{}".format(self.config.swift_url_prefix, url) else: return "{}/{}".format(self.config.swift_url_prefix, url) return self.config.swift_url_prefix[:-1] + url
async def swift_download_object( request: aiohttp.web.Request) -> aiohttp.web.Response: """Point a user to a temporary pre-signed download URL.""" session = api_check(request) request.app['Log'].info( 'API call for download object from {0}, sess: {1} :: {2}'.format( request.remote, session, time.ctime(), )) serv = request.app['Creds'][session]['ST_conn'] sess = request.app['Creds'][session]['OS_sess'] temp_url_key = await get_tempurl_key(serv) request.app['Log'].debug("Using %s as temporary URL key", temp_url_key) # Generate temporary URL host = sess.get_endpoint(service_type="object-store").split('/v1')[0] path_begin = sess.get_endpoint(service_type="object-store").replace( host, "") request.app['Log'].debug("Using %s as host and %s as path start.", host, path_begin) container = request.query['bucket'] object_key = request.query['objkey'] lifetime = 60 * 15 # In the path creation, the stats['items'][0][1] is the tenant id from # server statistics, the order should be significant, so this shouldn't # be a problem path = f'{path_begin}/{container}/{object_key}' dloadurl = (host + generate_temp_url(path, lifetime, temp_url_key, 'GET')) response = aiohttp.web.Response(status=302, ) response.headers['Location'] = dloadurl return response
def get_temp_url(self, container_name, obj_name, timeout=None, method='PUT'): """Return a Swift TempURL.""" def tenant_uuid(): access = self.context.auth_token_info['access'] for role in access['user']['roles']: if role['name'] == 'object-store:default': return role['tenantId'] key_header = 'x-account-meta-temp-url-key' if key_header in self.client().head_account(): key = self.client().head_account()[key_header] else: key = hashlib.sha224(str(random.getrandbits(256))).hexdigest()[:32] self.client().post_account({key_header: key}) path = '/v1/%s/%s/%s' % (tenant_uuid(), container_name, obj_name) if timeout is None: timeout = swift.MAX_EPOCH - 60 - time.time() tempurl = swiftclient_utils.generate_temp_url(path, timeout, key, method) sw_url = parse.urlparse(self.client().url) return '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl)
def test_generate_temp_url_prefix(self, time_mock, hmac_mock): hmac_mock().hexdigest.return_value = 'temp_url_signature' prefixes = ['', 'o', 'p0/p1/'] for p in prefixes: hmac_mock.reset_mock() path = '/v1/AUTH_account/c/' + p expected_url = path + ('?temp_url_sig=temp_url_signature' '&temp_url_expires=1400003600' '&temp_url_prefix=' + p) expected_body = '\n'.join([ self.method, '1400003600', 'prefix:' + path, ]).encode('utf-8') url = u.generate_temp_url(path, self.seconds, self.key, self.method, prefix=True) key = self.key if not isinstance(key, six.binary_type): key = key.encode('utf-8') self.assertEqual(url, expected_url) self.assertEqual(hmac_mock.mock_calls, [ mock.call(key, expected_body, sha1), mock.call().hexdigest(), ]) self.assertIsInstance(url, type(path))
def publish_template(self, name, contents): oc = self.object_client # post the object oc.put_object(self.object_container_name, name, contents) # TODO(asalkeld) see if this is causing problems. # self.addCleanup(self.object_client.delete_object, # self.object_container_name, name) # make the tempurl key_header = 'x-account-meta-temp-url-key' if key_header not in oc.head_account(): swift_key = hashlib.sha224(str( random.getrandbits(256))).hexdigest()[:32] LOG.warn('setting swift key to %s' % swift_key) oc.post_account({key_header: swift_key}) key = oc.head_account()[key_header] path = '/v1/AUTH_%s/%s/%s' % (self.project_id, self.object_container_name, name) timeout = self.conf.build_timeout * 10 tempurl = swiftclient_utils.generate_temp_url(path, timeout, key, 'GET') sw_url = parse.urlparse(oc.url) full_url = '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl) def download(): r = requests.get(full_url) LOG.info('GET: %s -> %s' % (full_url, r.status_code)) return r.status_code == requests.codes.ok # make sure that the object is available. test.call_until_true(self.conf.build_timeout, self.conf.build_interval, download) return full_url
async def swift_download_object(request: aiohttp.web.Request) -> aiohttp.web.Response: """Point a user to a temporary pre-signed download URL.""" session = api_check(request) request.app["Log"].info( "API call for download object from " f"{request.remote}, sess: {session} :: {time.ctime()}" ) serv = request.app["Sessions"][session]["ST_conn"] sess = request.app["Sessions"][session]["OS_sess"] temp_url_key = await get_tempurl_key(serv) request.app["Log"].debug(f"Using {temp_url_key} as temporary URL key") # Generate temporary URL host = sess.get_endpoint(service_type="object-store").split("/v1")[0] path_begin = sess.get_endpoint(service_type="object-store").replace(host, "") request.app["Log"].debug(f"Using {host} as host and {path_begin} as path start.") container = request.query["bucket"] object_key = request.query["objkey"] lifetime = 60 * 15 # In the path creation, the stats['items'][0][1] is the tenant id from # server statistics, the order should be significant, so this shouldn't # be a problem path = f"{path_begin}/{container}/{object_key}" dloadurl = host + generate_temp_url(path, lifetime, temp_url_key, "GET") response = aiohttp.web.Response( status=302, ) response.headers["Location"] = dloadurl return response
def test_generate_absolute_expiry_temp_url(self, hmac_mock): expected_url = ('/v1/AUTH_account/c/o?' 'temp_url_sig=temp_url_signature&' 'temp_url_expires=2146636800') url = u.generate_temp_url(self.url, 2146636800, self.key, self.method, absolute=True) self.assertEqual(url, expected_url)
def get_temp_url(self, container_name, obj_name, timeout=None, method='PUT'): ''' Return a Swift TempURL. ''' key_header = 'x-account-meta-temp-url-key' if key_header not in self.client().head_account(): self.client().post_account({ key_header: hashlib.sha224(str(random.getrandbits(256))).hexdigest()[:32] }) key = self.client().head_account()[key_header] path = '/v1/AUTH_%s/%s/%s' % (self.context.tenant_id, container_name, obj_name) if timeout is None: timeout = MAX_EPOCH - 60 - time.time() tempurl = swiftclient_utils.generate_temp_url(path, timeout, key, method) sw_url = urlparse.urlparse(self.client().url) return '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl)
def publish_template(self, name, contents): oc = self.object_client # post the object oc.put_object(self.object_container_name, name, contents) # TODO(asalkeld) see if this is causing problems. # self.addCleanup(self.object_client.delete_object, # self.object_container_name, name) # make the tempurl key_header = 'x-account-meta-temp-url-key' if key_header not in oc.head_account(): swift_key = hashlib.sha224( str(random.getrandbits(256))).hexdigest()[:32] LOG.warn('setting swift key to %s' % swift_key) oc.post_account({key_header: swift_key}) key = oc.head_account()[key_header] path = '/v1/AUTH_%s/%s/%s' % (self.project_id, self.object_container_name, name) timeout = self.conf.build_timeout * 10 tempurl = swiftclient_utils.generate_temp_url(path, timeout, key, 'GET') sw_url = parse.urlparse(oc.url) full_url = '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl) def download(): r = requests.get(full_url) LOG.info('GET: %s -> %s' % (full_url, r.status_code)) return r.status_code == requests.codes.ok # make sure that the object is available. test.call_until_true(self.conf.build_timeout, self.conf.build_interval, download) return full_url
def test_generate_temp_url(self, time_mock, hmac_mock): expected_url = ( '/v1/AUTH_account/c/o?' 'temp_url_sig=temp_url_signature&' 'temp_url_expires=1400003600') url = u.generate_temp_url(self.url, self.seconds, self.key, self.method) self.assertEqual(url, expected_url)
async def swift_download_object( request: aiohttp.web.Request) -> aiohttp.web.Response: """Point a user to a temporary pre-signed download URL.""" session = api_check(request) request.app['Log'].info( 'API call for download object from {0}, sess: {1} :: {2}'.format( request.remote, session, time.ctime(), )) serv = request.app['Creds'][session]['ST_conn'] sess = request.app['Creds'][session]['OS_sess'] stats = serv.stat() # Check for the existence of the key headers acc_meta_hdr = stats['headers'] if 'x-account-meta-temp-url-key' in acc_meta_hdr.keys(): temp_url_key = acc_meta_hdr['x-account-meta-temp-url-key'] elif 'x-acccount-meta-temp-url-key-2' in acc_meta_hdr.keys(): temp_url_key = acc_meta_hdr['x-account-meta-temp-url-key-2'] # If the key headers don't exist, assume that the key has to be created by # the service else: # The hash only provides random data for the key, it doesn't have to # be cryptographically secure. temp_url_key = hashlib.md5(os.urandom(128)).hexdigest() # nosec # This service will use the X-Account-Meta-Temp-URL-Key-2 header for # its own key storage, if no existing keys are provided. meta_options = {"meta": ["Temp-URL-Key-2:{0}".format(temp_url_key)]} retval = serv.post(options=meta_options) if not retval['success']: raise aiohttp.web.HTTPServerError() request.app['Log'].info( "Created a temp url key for account {0} Key:{1} :: {2}".format( stats['items'][0][1], temp_url_key, time.ctime())) request.app['Log'].debug("Using {0} as temporary URL key :: {1}".format( temp_url_key, time.ctime())) # Generate temporary URL host = sess.get_endpoint(service_type="object-store").split('/v1')[0] path_begin = sess.get_endpoint(service_type="object-store").replace( host, "") request.app['Log'].debug("Using %s as host and %s as path start.", host, path_begin) container = request.query['bucket'] object_key = request.query['objkey'] lifetime = 60 * 15 # In the path creation, the stats['items'][0][1] is the tenant id from # server statistics, the order should be significant, so this shouldn't # be a problem path = '%s/%s/%s' % (path_begin, container, object_key) dloadurl = (host + generate_temp_url(path, lifetime, temp_url_key, 'GET')) response = aiohttp.web.Response(status=302, ) response.headers['Location'] = dloadurl return response
def generate_temp_report_url(account, container, key, temp_url_key, expires=60 * 60 * 48): full_path = "/v1/{}/{}/{}".format(account, container, key) url = generate_temp_url(full_path, int(time() + int(expires)), temp_url_key, 'GET') return url
def test_generate_absolute_expiry_temp_url(self, hmac_mock): if isinstance(self.expected_url, six.binary_type): expected_url = self.expected_url.replace( b'1400003600', b'2146636800') else: expected_url = self.expected_url.replace( u'1400003600', u'2146636800') url = u.generate_temp_url(self.url, 2146636800, self.key, self.method, absolute=True) self.assertEqual(url, expected_url)
def generate_url(self, expires_in): # Set a temp url key on the object (http://bit.ly/2NBiXGD) temp_url_key = "cloudbridge-tmp-url-key" self._provider.swift.post_account( headers={"x-account-meta-temp-url-key": temp_url_key}) base_url = urlparse(self._provider.swift.get_service_auth()[0]) access_point = "{0}://{1}".format(base_url.scheme, base_url.netloc) url_path = "/".join([base_url.path, self.cbcontainer.name, self.name]) return urljoin(access_point, generate_temp_url(url_path, expires_in, temp_url_key, 'GET'))
def get_temp_url(self, container_name, object_name): tenant_id = self.auth['tenant_id'] url = f'/v1/nc_{tenant_id}/{container_name}/{object_name}' key = self.auth['tempurl_key'].encode('utf-8') method = 'GET' seconds = 60 * 60 * 24 temp_url = generate_temp_url(path=url, seconds=seconds, key=key, method=method) return 'https://object-storage.tyo2.conoha.io' + temp_url
def getTempUrl(filename): # f = open("mykey.txt") # mykey = f.read() ConnUtil.con.post_account( headers={"X-Account-Meta-Temp-Url-Key": "mykey"}) # temp account return ConnUtil.end_point + generate_temp_url( "/v1/AUTH_test/user_uploads/" + filename, seconds=3600, key="mykey", method='GET') f.close()
def _path(self, name): url = urlparse.urljoin(self.base_url, urlparse.quote(name)) # Are we building a temporary url? if self.use_temp_urls: expires = int(time() + int(self.temp_url_duration)) path = urlparse.unquote(urlparse.urlsplit(url).path) tmp_path = generate_temp_url(path, expires, self.temp_url_key, 'GET', absolute=True) url = urlparse.urljoin(self.base_url, tmp_path) return url
def test_generate_temp_url_ip_range(self, time_mock, hmac_mock): hmac_mock().hexdigest.return_value = 'temp_url_signature' ip_ranges = [ '1.2.3.4', '1.2.3.4/24', '2001:db8::', b'1.2.3.4', b'1.2.3.4/24', b'2001:db8::', ] path = '/v1/AUTH_account/c/o/' expected_url = path + ('?temp_url_sig=temp_url_signature' '&temp_url_expires=1400003600' '&temp_url_ip_range=') for ip_range in ip_ranges: hmac_mock.reset_mock() url = u.generate_temp_url(path, self.seconds, self.key, self.method, ip_range=ip_range) key = self.key if not isinstance(key, six.binary_type): key = key.encode('utf-8') if isinstance(ip_range, six.binary_type): ip_range_expected_url = (expected_url + ip_range.decode('utf-8')) expected_body = '\n'.join([ 'ip=' + ip_range.decode('utf-8'), self.method, '1400003600', path, ]).encode('utf-8') else: ip_range_expected_url = expected_url + ip_range expected_body = '\n'.join([ 'ip=' + ip_range, self.method, '1400003600', path, ]).encode('utf-8') self.assertEqual(url, ip_range_expected_url) self.assertEqual(hmac_mock.mock_calls, [ mock.call(key, expected_body, sha1), mock.call().hexdigest(), ]) self.assertIsInstance(url, type(path))
def test_generate_temp_url(self, time_mock, hmac_mock): hmac_mock().hexdigest.return_value = 'temp_url_signature' url = u.generate_temp_url(self.url, self.seconds, self.key, self.method) key = self.key if not isinstance(key, six.binary_type): key = key.encode('utf-8') self.assertEqual(url, self.expected_url) self.assertEqual(hmac_mock.mock_calls, [ mock.call(), mock.call(key, self.expected_body, sha1), mock.call().hexdigest(), ]) self.assertIsInstance(url, type(self.url))
def generate_temp_url(self, user_id, path, name): container, object_name = self.build_object_name(user_id, path, name) # discover swift endpoint urls conn = get_conn(self.swift._options) url, token = conn.get_auth() urlparts = urlparse(url) # generate swift path /v1/<account>/<container>/<userid>/path path = '/'.join((urlparts.path, container, object_name)) # TODO: valid for 5 minutes temp_url = generate_temp_url(path, 300, self.temp_url_key, method='GET') return urljoin(url, temp_url)
def swift_temp_url(self, image_info): """Generate a no-auth Swift temporary URL. This function will generate the temporary Swift URL using the image id from Glance and the config options: 'swift_endpoint_url', 'swift_api_version', 'swift_account' and 'swift_container'. The temporary URL will be valid for 'swift_temp_url_duration' seconds. This allows Ironic to download a Glance image without passing around an auth_token. :param image_info: The return from a GET request to Glance for a certain image_id. Should be a dictionary, with keys like 'name' and 'checksum'. See http://docs.openstack.org/developer/glance/glanceapi.html for examples. :returns: A signed Swift URL from which an image can be downloaded, without authentication. :raises: InvalidParameterValue if Swift config options are not set correctly. :raises: MissingParameterValue if a required parameter is not set. :raises: ImageUnacceptable if the image info from Glance does not have a image ID. """ self._validate_temp_url_config() if ('id' not in image_info or not utils.is_uuid_like(image_info['id'])): raise exc.ImageUnacceptable(_( 'The given image info does not have a valid image id: %s') % image_info) url_fragments = { 'endpoint_url': CONF.glance.swift_endpoint_url, 'api_version': CONF.glance.swift_api_version, 'account': CONF.glance.swift_account, 'container': self._get_swift_container(image_info['id']), 'object_id': image_info['id'] } template = '/{api_version}/{account}/{container}/{object_id}' url_path = template.format(**url_fragments) path = swift_utils.generate_temp_url( path=url_path, seconds=CONF.glance.swift_temp_url_duration, key=CONF.glance.swift_temp_url_key, method='GET') return '{endpoint_url}{url_path}'.format( endpoint_url=url_fragments['endpoint_url'], url_path=path)
def test_generate_temp_url_iso8601_argument(self, hmac_mock): hmac_mock().hexdigest.return_value = 'temp_url_signature' url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z', self.key, self.method) self.assertEqual(url, self.expected_url) # Don't care about absolute arg. url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z', self.key, self.method, absolute=True) self.assertEqual(url, self.expected_url) lt = localtime() expires = strftime(u.EXPIRES_ISO8601_FORMAT[:-1], lt) if not isinstance(self.expected_url, six.string_types): expected_url = self.expected_url.replace( b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii')) else: expected_url = self.expected_url.replace('1400003600', str(int(mktime(lt)))) url = u.generate_temp_url(self.url, expires, self.key, self.method) self.assertEqual(url, expected_url) expires = strftime(u.SHORT_EXPIRES_ISO8601_FORMAT, lt) lt = strptime(expires, u.SHORT_EXPIRES_ISO8601_FORMAT) if not isinstance(self.expected_url, six.string_types): expected_url = self.expected_url.replace( b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii')) else: expected_url = self.expected_url.replace('1400003600', str(int(mktime(lt)))) url = u.generate_temp_url(self.url, expires, self.key, self.method) self.assertEqual(url, expected_url)
def test_generate_temp_url_iso8601_argument(self, hmac_mock): hmac_mock().hexdigest.return_value = 'temp_url_signature' url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z', self.key, self.method) self.assertEqual(url, self.expected_url) # Don't care about absolute arg. url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z', self.key, self.method, absolute=True) self.assertEqual(url, self.expected_url) lt = localtime() expires = strftime(u.EXPIRES_ISO8601_FORMAT[:-1], lt) if not isinstance(self.expected_url, six.string_types): expected_url = self.expected_url.replace( b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii')) else: expected_url = self.expected_url.replace( '1400003600', str(int(mktime(lt)))) url = u.generate_temp_url(self.url, expires, self.key, self.method) self.assertEqual(url, expected_url) expires = strftime(u.SHORT_EXPIRES_ISO8601_FORMAT, lt) lt = strptime(expires, u.SHORT_EXPIRES_ISO8601_FORMAT) if not isinstance(self.expected_url, six.string_types): expected_url = self.expected_url.replace( b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii')) else: expected_url = self.expected_url.replace( '1400003600', str(int(mktime(lt)))) url = u.generate_temp_url(self.url, expires, self.key, self.method) self.assertEqual(url, expected_url)
def publish_template(self, contents, cleanup=True): oc = self.object_client # post the object oc.put_object(self.object_container_name, 'template.yaml', contents) if cleanup: self.addCleanup(oc.delete_object, self.object_container_name, 'template.yaml') path = '/v1/AUTH_%s/%s/%s' % ( self.project_id, self.object_container_name, 'template.yaml') timeout = self.conf.build_timeout * 10 tempurl = swiftclient_utils.generate_temp_url(path, timeout, self.swift_key, 'GET') sw_url = parse.urlparse(oc.url) return '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl)
def _path(self, name): try: name = name.encode('utf-8') except UnicodeDecodeError: pass url = urlparse.urljoin(self.base_url, urlparse.quote(name)) # Are we building a temporary url? if self.use_temp_urls: expires = int(time() + int(self.temp_url_duration)) path = urlparse.unquote(urlparse.urlsplit(url).path) tmp_path = generate_temp_url(path, expires, self.temp_url_key, 'GET', absolute=True) url = urlparse.urljoin(self.base_url, tmp_path) return url
def publish_template(self, contents, cleanup=True): oc = self.object_client # post the object oc.put_object(self.object_container_name, 'template.yaml', contents) if cleanup: self.addCleanup(oc.delete_object, self.object_container_name, 'template.yaml') path = '/v1/AUTH_%s/%s/%s' % (self.project_id, self.object_container_name, 'template.yaml') timeout = self.conf.build_timeout * 10 tempurl = swiftclient_utils.generate_temp_url(path, timeout, self.swift_key, 'GET') sw_url = parse.urlparse(oc.url) return '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl)
def get_temp_download_url(self, fname, expires): """ return the temporary download url """ file_uri = '%s/%s/%s' % (self.storageurl, self.container_name, fname) file_path = urlparse(file_uri).path key = getattr(self, self.chosen_temp_url_key) try: temp_url = generate_temp_url(file_path, expires, key, 'GET') except Exception as err: logging.error( 'swift-file-server: Generating temp url for %s failed %s', fname, err) raise download_url = self.storageurl.replace( urlparse(self.storageurl).path, temp_url) print( 'swift-file-server: Temporary download URL for file %s: %s', fname, download_url) return download_url
def run(self, context): swift_client = self.get_object_client(context) try: cont_stat = swift_client.head_container(self.container) except swiftexceptions.ClientException: cont_stat = {} key = cont_stat.get('x-container-meta-temp-url-key') if not key: key = str(uuid.uuid4()) cont_stat = swift_client.put_container( self.container, {'X-Container-Meta-Temp-Url-Key': key}) parsed = urllib.parse.urlparse(swift_client.url) path = "%s/%s/%s" % (parsed.path, self.container, self.obj) temp_path = generate_temp_url(path, self.valid, key, self.method) return "%s://%s%s" % (parsed.scheme, parsed.netloc, temp_path)
def test_generate_temp_url_ip_range(self, time_mock, hmac_mock): hmac_mock().hexdigest.return_value = 'temp_url_signature' ip_ranges = [ '1.2.3.4', '1.2.3.4/24', '2001:db8::', b'1.2.3.4', b'1.2.3.4/24', b'2001:db8::', ] path = '/v1/AUTH_account/c/o/' expected_url = path + ('?temp_url_sig=temp_url_signature' '&temp_url_expires=1400003600' '&temp_url_ip_range=') for ip_range in ip_ranges: hmac_mock.reset_mock() url = u.generate_temp_url(path, self.seconds, self.key, self.method, ip_range=ip_range) key = self.key if not isinstance(key, six.binary_type): key = key.encode('utf-8') if isinstance(ip_range, six.binary_type): ip_range_expected_url = ( expected_url + ip_range.decode('utf-8') ) expected_body = '\n'.join([ 'ip=' + ip_range.decode('utf-8'), self.method, '1400003600', path, ]).encode('utf-8') else: ip_range_expected_url = expected_url + ip_range expected_body = '\n'.join([ 'ip=' + ip_range, self.method, '1400003600', path, ]).encode('utf-8') self.assertEqual(url, ip_range_expected_url) self.assertEqual(hmac_mock.mock_calls, [ mock.call(key, expected_body, sha1), mock.call().hexdigest(), ]) self.assertIsInstance(url, type(path))
def outputs(request): # TODO: maybe use server:outputpath here? container = config.get_config_value('SwiftStorage', 'container') # TODO: TEM_PURL_KEY not needed in pywps temp_url_key = get_temp_url_key() path_prefix = '/v1/AUTH_{}/{}/'.format(os.environ['OS_PROJECT_ID'], container) temp_url = generate_temp_url( path=path_prefix + '/'.join(request.matchdict['filename']), seconds=60 * 60 * 24, # temp url valid for 24hrs key=temp_url_key, method='GET', # prefix=True, # iso8601=True, # ip_range=??? ) return HTTPFound(location='https://swift.rc.nectar.org.au' + temp_url)
def get_temp_download_url(self, fname, expires): """ return the temporary download url """ file_uri = '%s/%s/%s' % (self.storageurl, self.container_name, fname) file_path = urlparse(file_uri).path key = getattr(self, self.chosen_temp_url_key) try: temp_url = generate_temp_url(file_path, expires, key, 'GET') except Exception as err: logging.error( 'swift-file-server: Generating temp url for %s failed %s', fname, err) raise download_url = self.storageurl.replace( urlparse(self.storageurl).path, temp_url) print('swift-file-server: Temporary download URL for file %s: %s', fname, download_url) return download_url
def swift_tempurl(path, date): swift_endpoint = environ.get('SWIFT_ENDPOINT') swift_path = environ.get('SWIFT_PATH') swift_tempurl_key = environ.get('SWIFT_TEMPURL_KEY') assert (swift_endpoint) assert (swift_path) assert (swift_tempurl_key) path = "{}{}".format(swift_path, path) timestamp = int(date.strftime("%s")) temp_url = generate_temp_url(path, timestamp, swift_tempurl_key, 'GET', absolute=True) return "{}{}".format(swift_endpoint, temp_url)
def test_generate_temp_url_iso8601_output(self, time_mock, hmac_mock): hmac_mock().hexdigest.return_value = 'temp_url_signature' url = u.generate_temp_url(self.url, self.seconds, self.key, self.method, iso8601=True) key = self.key if not isinstance(key, six.binary_type): key = key.encode('utf-8') expires = strftime(u.EXPIRES_ISO8601_FORMAT, gmtime(1400003600)) if not isinstance(self.url, six.string_types): self.assertTrue(url.endswith(bytes(expires, 'utf-8'))) else: self.assertTrue(url.endswith(expires)) self.assertEqual(hmac_mock.mock_calls, [ mock.call(), mock.call(key, self.expected_body, sha1), mock.call().hexdigest(), ]) self.assertIsInstance(url, type(self.url))
def get_temp_url(self, container_name, obj_name, timeout=None, method='PUT'): ''' Return a Swift TempURL. ''' key_header = 'x-account-meta-temp-url-key' if key_header in self.client().head_account(): key = self.client().head_account()[key_header] else: key = hashlib.sha224(str(random.getrandbits(256))).hexdigest()[:32] self.client().post_account({key_header: key}) path = '/v1/AUTH_%s/%s/%s' % (self.context.tenant_id, container_name, obj_name) if timeout is None: timeout = MAX_EPOCH - 60 - time.time() tempurl = swiftclient_utils.generate_temp_url(path, timeout, key, method) sw_url = urlparse.urlparse(self.client().url) return '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl)
def create_temp_url(swift_client, name, timeout, container=None): container = container or '%(name)s-%(uuid)s' % { 'name': name, 'uuid': uuid.uuid4()} object_name = str(uuid.uuid4()) swift_client.put_container(container) key_header = 'x-account-meta-temp-url-key' if key_header not in swift_client.head_account(): swift_client.post_account({ key_header: six.text_type(uuid.uuid4())[:32]}) key = swift_client.head_account()[key_header] project_path = swift_client.url.split('/')[-1] path = '/v1/%s/%s/%s' % (project_path, container, object_name) timeout_secs = timeout * 60 tempurl = swiftclient_utils.generate_temp_url(path, timeout_secs, key, 'PUT') sw_url = urlparse.urlparse(swift_client.url) put_url = '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl) swift_client.put_object(container, object_name, '') return put_url
def _generate_temp_url(self, path, seconds, key, method, endpoint, image_id): """Get Swift temporary URL. Generates (or returns the cached one if caching is enabled) a temporary URL that gives unauthenticated access to the Swift object. :param path: The full path to the Swift object. Example: /v1/AUTH_account/c/o. :param seconds: The amount of time in seconds the temporary URL will be valid for. :param key: The secret temporary URL key set on the Swift cluster. :param method: A HTTP method, typically either GET or PUT, to allow for this temporary URL. :param endpoint: Endpoint URL of Swift service. :param image_id: UUID of a Glance image. :returns: temporary URL """ if CONF.glance.swift_temp_url_cache_enabled: self._remove_expired_items_from_cache() if image_id in self._cache: return self._cache[image_id].url path = swift_utils.generate_temp_url( path=path, seconds=seconds, key=key, method=method) temp_url = '{endpoint_url}{url_path}'.format( endpoint_url=endpoint, url_path=path) if CONF.glance.swift_temp_url_cache_enabled: query = urlparse.urlparse(temp_url).query exp_time_str = dict(urlparse.parse_qsl(query))['temp_url_expires'] self._cache[image_id] = TempUrlCacheElement( url=temp_url, url_expires_at=int(exp_time_str) ) return temp_url
def publish_template(self, name, contents): oc = self.object_client # post the object oc.put_object(self.object_container_name, name, contents) # TODO(asalkeld) see if this is causing problems. # self.addCleanup(self.object_client.delete_object, # self.object_container_name, name) # make the tempurl key_header = 'x-account-meta-temp-url-key' if key_header not in oc.head_account(): swift_key = hashlib.sha224( str(random.getrandbits(256))).hexdigest()[:32] LOG.warn('setting swift key to %s' % swift_key) oc.post_account({key_header: swift_key}) key = oc.head_account()[key_header] path = '/v1/AUTH_%s/%s/%s' % (self.project_id, self.object_container_name, name) timeout = self.conf.build_timeout * 10 tempurl = swiftclient_utils.generate_temp_url(path, timeout, key, 'GET') sw_url = parse.urlparse(oc.url) return '%s://%s%s' % (sw_url.scheme, sw_url.netloc, tempurl)
def test_generate_temp_url_bad_time(self): with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(self.url, 'not_an_int', self.key, self.method) self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(self.url, -1, self.key, self.method) self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(self.url, 1.1, self.key, self.method) self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(self.url, '-1', self.key, self.method) self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(self.url, '1.1', self.key, self.method) self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(self.url, '2015-05', self.key, self.method) self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url( self.url, '2015-05-01T01:00', self.key, self.method) self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG)
def test_generate_temp_url_invalid_path(self): with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(b'/v1/a/c/\xff', self.seconds, self.key, self.method) self.assertEqual(exc_manager.exception.args[0], 'path must be representable as UTF-8')