def sync_account(self): """Sync a single account with url/tok to dest_url/dest_tok.""" orig_storage_cnx = self.get_cloudfiles_auth_orig() dest_auth_url = get_config('auth', 'keystone_dest') cfg = get_config('auth', 'keystone_dest_credentials') (orig_user, orig_password, orig_tenant) = cfg.split(':') dest_storage_url = get_config('auth', 'dest_storage_url') # we assume orig and dest passwd are the same obv synchronized. dst_st_url, dest_token = self.get_swift_auth( dest_auth_url, orig_tenant, orig_user, orig_password) dest_storage_cnx = swiftclient.http_connection(dest_storage_url) try: orig_containers = orig_storage_cnx.cloudfiles.get_all_containers() dest_account_headers, dest_containers = ( swiftclient.get_account(None, dest_token, http_conn=dest_storage_cnx, full_listing=True)) except(swiftclient.client.ClientException), e: logging.info("error getting account: %s" % ( e.http_reason)) return
def send_request(self, method, url, token=None, headers=None, service_token=None): headers = {} if headers is None else headers.copy() headers.update({"Content-Type": "application/json", "Accept": "application/json"}) if token: headers["X-Auth-Token"] = token if service_token: headers["X-Service-Token"] = service_token if self.insecure: parsed, conn = http_connection(url, insecure=self.insecure) else: parsed, conn = http_connection(url) conn.request(method, parsed.path, headers=headers) resp = conn.getresponse() return resp
def sync_account(self, orig_storage_url, orig_token, dest_storage_url, dest_token): """Sync a single account with url/tok to dest_url/dest_tok.""" orig_storage_cnx = swiftclient.http_connection(orig_storage_url) dest_storage_cnx = swiftclient.http_connection(dest_storage_url) account_id = os.path.basename(orig_storage_url.replace("AUTH_", "")) try: orig_account_headers, orig_containers = swiftclient.get_account( None, orig_token, http_conn=orig_storage_cnx, full_listing=True ) dest_account_headers, dest_containers = swiftclient.get_account( None, dest_token, http_conn=dest_storage_cnx, full_listing=True ) except (swiftclient.client.ClientException), e: logging.info("error getting account: %s, %s" % (account_id, e.http_reason)) return
def connect_swift(): http_conn = swiftclient.http_connection(PRE_AUTH_URL) try: swiftclient.head_container(PRE_AUTH_URL, PRE_AUTH_TOKEN, CONTAINER_NAME, http_conn=http_conn) except swiftclient.ClientException: swiftclient.put_container(PRE_AUTH_URL, PRE_AUTH_TOKEN, CONTAINER_NAME, http_conn=http_conn) return http_conn
def __init__(self): self.last_headers_name = None self.last_headers_value = None # Get authentication token self.storage_url, self.token = swiftclient.get_auth( self.api_auth_url, self.api_username, self.api_key, auth_version=self.auth_version, os_options=dict( list({ "tenant_id": self.tenant_id, "tenant_name": self.tenant_name }.items()) + list(self.os_extra_options.items())), ) self.http_conn = swiftclient.http_connection(self.storage_url) # Check container try: swiftclient.head_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn) except swiftclient.ClientException: headers = {} if self.auto_create_container: if self.auto_create_container_public: headers['X-Container-Read'] = '.r:*' swiftclient.put_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn, headers=headers) else: raise ImproperlyConfigured("Container %s does not exist." % self.container_name) if self.auto_base_url: # Derive a base URL based on the authentication information from # the server, optionally overriding the protocol, host/port and # potentially adding a path fragment before the auth information. self.base_url = self.storage_url + '/' if self.override_base_url is not None: # override the protocol and host, append any path fragments split_derived = urllib.parse.urlsplit(self.base_url) split_override = urllib.parse.urlsplit(self.override_base_url) split_result = [''] * 5 split_result[0:2] = split_override[0:2] split_result[2] = (split_override[2] + split_derived[2]).replace('//', '/') self.base_url = urllib.parse.urlunsplit(split_result) self.base_url = urllib.parse.urljoin(self.base_url, self.container_name) self.base_url += '/' else: self.base_url = self.override_base_url
def send_request(self, method, url, token=None, headers=None, service_token=None): headers = {} if headers is None else headers.copy() headers.update({'Content-Type': 'application/json', 'Accept': 'application/json'}) if token: headers['X-Auth-Token'] = token if service_token: headers['X-Service-Token'] = service_token if self.insecure: parsed, conn = http_connection(url, insecure=self.insecure) else: parsed, conn = http_connection(url) conn.request(method, parsed.path, headers=headers) resp = conn.getresponse() return resp
def __init__(self, **settings): # check if some of the settings provided as class attributes # should be overwritten for name, value in settings.items(): if hasattr(self, name): setattr(self, name, value) self.last_headers_name = None self.last_headers_value = None # Get authentication token self.storage_url, self.token = swiftclient.get_auth( self.api_auth_url, self.api_username, self.api_key, auth_version=self.auth_version, os_options={"tenant_name": self.tenant_name}, ) self.http_conn = swiftclient.http_connection(self.storage_url) # Check container try: swiftclient.head_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn) except swiftclient.ClientException: if self.auto_create_container: swiftclient.put_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn) else: raise ImproperlyConfigured("Container %s does not exist." % self.container_name) if self.auto_base_url: # Derive a base URL based on the authentication information from # the server, optionally overriding the protocol, host/port and # potentially adding a path fragment before the auth information. self.base_url = self.storage_url + '/' if self.override_base_url is not None: # override the protocol and host, append any path fragments split_derived = urlparse.urlsplit(self.base_url) split_override = urlparse.urlsplit(self.override_base_url) split_result = [''] * 5 split_result[0:2] = split_override[0:2] split_result[2] = (split_override[2] + split_derived[2]).replace('//', '/') self.base_url = urlparse.urlunsplit(split_result) self.base_url = urlparse.urljoin(self.base_url, self.container_name) self.base_url += '/' else: self.base_url = self.override_base_url
def connection(url): if has_insecure: parsed_url, http_conn = http_connection(url, insecure=insecure) else: parsed_url, http_conn = http_connection(url) orig_request = http_conn.request # Add the policy header if policy_specified is set def request_with_policy(method, url, body=None, headers={}): version, account, container, obj = split_path(url, 1, 4, True) if policy_specified and method == "PUT" and container and not obj and "X-Storage-Policy" not in headers: headers["X-Storage-Policy"] = policy_specified return orig_request(method, url, body, headers) http_conn.request = request_with_policy return parsed_url, http_conn
def __init__(self): self.last_headers_name = None self.last_headers_value = None # Get authentication token self.storage_url, self.token = swiftclient.get_auth( self.api_auth_url, self.api_username, self.api_key, auth_version=self.auth_version, os_options=dict({"tenant_name": self.tenant_name}.items() + self.os_extra_options.items()), cacert=self.custom_ca, insecure=self.ssl_insecure ) self.http_conn = swiftclient.http_connection(self.storage_url) # Check container try: swiftclient.head_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn) except swiftclient.ClientException: headers = {} if self.auto_create_container: if self.auto_create_container_public: headers['X-Container-Read'] = '.r:*' swiftclient.put_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn, headers=headers) else: raise ImproperlyConfigured( "Container %s does not exist." % self.container_name) if self.auto_base_url: # Derive a base URL based on the authentication information from # the server, optionally overriding the protocol, host/port and # potentially adding a path fragment before the auth information. self.base_url = self.storage_url + '/' if self.override_base_url is not None: # override the protocol and host, append any path fragments split_derived = urlparse.urlsplit(self.base_url) split_override = urlparse.urlsplit(self.override_base_url) split_result = [''] * 5 split_result[0:2] = split_override[0:2] split_result[2] = (split_override[2] + split_derived[2]).replace('//', '/') self.base_url = urlparse.urlunsplit(split_result) self.base_url = urlparse.urljoin(self.base_url, self.container_name) self.base_url += '/' else: self.base_url = self.override_base_url
def retry(func, *args, **kwargs): """ You can use the kwargs to override: 'retries' (default: 5) 'use_account' (default: 1) - which user's token to pass 'url_account' (default: matches 'use_account') - which user's storage URL 'resource' (default: url[url_account] - URL to connect to; retry() will interpolate the variable :storage_url: if present """ global url, token, parsed, conn retries = kwargs.get('retries', 5) attempts, backoff = 0, 1 # use account #1 by default; turn user's 1-indexed account into 0-indexed use_account = kwargs.pop('use_account', 1) - 1 # access our own account by default url_account = kwargs.pop('url_account', use_account + 1) - 1 while attempts <= retries: attempts += 1 try: if not url[use_account] or not token[use_account]: url[use_account], token[use_account] = \ get_auth(swift_test_auth, swift_test_user[use_account], swift_test_key[use_account], snet=False, tenant_name=swift_test_tenant[use_account], auth_version=swift_test_auth_version, os_options={}) parsed[use_account] = conn[use_account] = None if not parsed[use_account] or not conn[use_account]: parsed[use_account], conn[use_account] = \ http_connection(url[use_account]) # default resource is the account url[url_account] resource = kwargs.pop('resource', '%(storage_url)s') template_vars = {'storage_url': url[url_account]} parsed_result = urlparse(resource % template_vars) return func(url[url_account], token[use_account], parsed_result, conn[url_account], *args, **kwargs) except (socket.error, HTTPException): if attempts > retries: raise parsed[use_account] = conn[use_account] = None except AuthError: url[use_account] = token[use_account] = None continue except InternalServerError: pass if attempts <= retries: sleep(backoff) backoff *= 2 raise Exception('No result after %s retries.' % retries)
def connection(url): if has_insecure: parsed_url, http_conn = http_connection(url, insecure=insecure) else: parsed_url, http_conn = http_connection(url) orig_request = http_conn.request # Add the policy header if policy_specified is set def request_with_policy(method, url, body=None, headers={}): version, account, container, obj = split_path(url, 1, 4, True) if policy_specified and method == 'PUT' and container and not obj \ and 'X-Storage-Policy' not in headers: headers['X-Storage-Policy'] = policy_specified return orig_request(method, url, body, headers) http_conn.request = request_with_policy return parsed_url, http_conn
def __init__(self, **settings): # check if some of the settings provided as class attributes # should be overwritten for name, value in settings.items(): if hasattr(self, name): setattr(self, name, value) self.last_headers_name = None self.last_headers_value = None # Get authentication token self.storage_url, self.token = swiftclient.get_auth( self.api_auth_url, self.api_username, self.api_key, auth_version=self.auth_version, os_options={"tenant_name": self.tenant_name}, ) self.http_conn = swiftclient.http_connection(self.storage_url) # Check container try: swiftclient.head_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn) except swiftclient.ClientException: if self.auto_create_container: swiftclient.put_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn) else: raise ImproperlyConfigured( "Container %s does not exist." % self.container_name) if self.auto_base_url: # Derive a base URL based on the authentication information from # the server, optionally overriding the protocol, host/port and # potentially adding a path fragment before the auth information. self.base_url = self.storage_url + '/' if self.override_base_url is not None: # override the protocol and host, append any path fragments split_derived = urlparse.urlsplit(self.base_url) split_override = urlparse.urlsplit(self.override_base_url) split_result = [''] * 5 split_result[0:2] = split_override[0:2] split_result[2] = (split_override[2] + split_derived[2]).replace('//', '/') self.base_url = urlparse.urlunsplit(split_result) self.base_url = urlparse.urljoin(self.base_url, self.container_name) self.base_url += '/' else: self.base_url = self.override_base_url
def sync_account(self, orig_storage_url, orig_token, dest_storage_url, dest_token): """Sync a single account with url/tok to dest_url/dest_tok.""" orig_storage_cnx = swiftclient.http_connection(orig_storage_url) dest_storage_cnx = swiftclient.http_connection(dest_storage_url) account_id = os.path.basename(orig_storage_url.replace("AUTH_", '')) try: orig_account_headers, orig_containers = ( swiftclient.get_account(None, orig_token, http_conn=orig_storage_cnx, full_listing=True)) dest_account_headers, dest_containers = ( swiftclient.get_account(None, dest_token, http_conn=dest_storage_cnx, full_listing=True)) except(swiftclient.client.ClientException), e: logging.info("error getting account: %s, %s" % ( account_id, e.http_reason)) return
def _post_job(url, token, json_data, http_conn=None, response_dict=None): # Modelled after swiftclient.client.post_account. headers = {'X-Auth-Token': token, 'Accept': 'application/json', 'X-Zerovm-Execute': '1.0', 'Content-Type': 'application/json'} if http_conn: parsed, conn = http_conn else: parsed, conn = swiftclient.http_connection(url) return conn.request('POST', parsed.path, json_data, headers)
def retry(func, *args, **kwargs): """ You can use the kwargs to override the 'retries' (default: 5) and 'use_account' (default: 1). """ global url, token, parsed, conn retries = kwargs.get('retries', 5) use_account = 1 if 'use_account' in kwargs: use_account = kwargs['use_account'] del kwargs['use_account'] use_account -= 1 attempts = 0 backoff = 1 while attempts <= retries: attempts += 1 try: if not url[use_account] or not token[use_account]: url[use_account], token[use_account] = \ get_auth(swift_test_auth, swift_test_user[use_account], swift_test_key[use_account], snet=False, tenant_name=swift_test_tenant[use_account], auth_version=swift_test_auth_version, os_options={}) parsed[use_account] = conn[use_account] = None if not parsed[use_account] or not conn[use_account]: parsed[use_account], conn[use_account] = \ http_connection(url[use_account]) return func(url[use_account], token[use_account], parsed[use_account], conn[use_account], *args, **kwargs) except (socket.error, HTTPException): if attempts > retries: raise parsed[use_account] = conn[use_account] = None except AuthError: url[use_account] = token[use_account] = None continue except InternalServerError: pass if attempts <= retries: sleep(backoff) backoff *= 2 raise Exception('No result after %s retries.' % retries)
def health_check(url, logger): scheme = urlparse.urlparse(url).scheme netloc = urlparse.urlparse(url).netloc url = scheme + '://' + netloc + '/healthcheck' parsed, conn = http_connection(url) logger.debug('GET %s' % url) conn.request('GET', parsed.path, '', {'X-Auth-Token': 'none-needed'}) resp = conn.getresponse() resp.read() if resp.status < 200 or resp.status >= 300: raise ClientException('GET /healthcheck failed', http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=parsed.path, http_status=resp.status, http_reason=resp.reason) resp_headers = {} for header, value in resp.getheaders(): resp_headers[header.lower()] = value return resp_headers
def _post_job(url, token, json_data, http_conn=None, response_dict=None): # Modelled after swiftclient.client.post_account. headers = {'X-Auth-Token': token, 'Accept': 'application/json', 'X-Zerovm-Execute': '1.0', 'Content-Type': 'application/json'} if http_conn: parsed, conn = http_conn else: parsed, conn = swiftclient.http_connection(url) conn.request('POST', parsed.path, json_data, headers) resp = conn.getresponse() body = resp.read() swiftclient.http_log((url, 'POST'), {'headers': headers}, resp, body) swiftclient.store_response(resp, response_dict) LOG.debug('response status: %s' % resp.status) print(body)
def _post_job(url, token, data, http_conn=None, response_dict=None, content_type='application/json', content_length=None): # Modelled after swiftclient.client.post_account. headers = {'X-Auth-Token': token, 'X-Zerovm-Execute': '1.0', 'Content-Type': content_type} if content_length: headers['Content-Length'] = str(content_length) if http_conn: parsed, conn = http_conn else: parsed, conn = swiftclient.http_connection(url) conn.request('POST', parsed.path, data, headers) resp = conn.getresponse() body = resp.read() swiftclient.http_log((url, 'POST'), {'headers': headers}, resp, body) swiftclient.store_response(resp, response_dict) print(body)
def connection(url): if has_insecure: return http_connection(url, insecure=insecure) return http_connection(url)
def __init__(self, **settings): # check if some of the settings provided as class attributes # should be overwritten for name, value in settings.items(): if hasattr(self, name): setattr(self, name, value) validate_settings(self) self.last_headers_name = None self.last_headers_value = None self.os_options = { 'tenant_id': self.tenant_id, 'tenant_name': self.tenant_name, 'user_domain_id': self.user_domain_id, 'user_domain_name': self.user_domain_name, 'project_domain_id': self.project_domain_id, 'project_domain_name': self.project_domain_name } self.os_options.update(self.os_extra_options) # Get authentication token self.storage_url, self.token = swiftclient.get_auth( self.api_auth_url, self.api_username, self.api_key, auth_version=self.auth_version, os_options=self.os_options) self.http_conn = swiftclient.http_connection(self.storage_url) # Check container try: swiftclient.head_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn) except swiftclient.ClientException: headers = {} if self.auto_create_container: if self.auto_create_container_public: headers['X-Container-Read'] = '.r:*' if self.auto_create_container_allow_orgin: headers['X-Container-Meta-Access-Control-Allow-Origin'] = \ self.auto_create_container_allow_orgin swiftclient.put_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn, headers=headers) else: raise ImproperlyConfigured( "Container %s does not exist." % self.container_name) if self.auto_base_url: # Derive a base URL based on the authentication information from # the server, optionally overriding the protocol, host/port and # potentially adding a path fragment before the auth information. self.base_url = self.storage_url + '/' if self.override_base_url is not None: # override the protocol and host, append any path fragments split_derived = urlparse.urlsplit(self.base_url) split_override = urlparse.urlsplit(self.override_base_url) split_result = [''] * 5 split_result[0:2] = split_override[0:2] split_result[2] = (split_override[2] + split_derived[2] ).replace('//', '/') self.base_url = urlparse.urlunsplit(split_result) self.base_url = urlparse.urljoin(self.base_url, self.container_name) self.base_url += '/' else: self.base_url = self.override_base_url
def create(self): return client.http_connection(self.url)
def exec_account(url, token=None, container=None, name=None, contents=None, content_length=None, etag=None, chunk_size=None, content_type=None, headers=None, http_conn=None, proxy=None, query_string=None, response_dict=None): """ Execute a job :param url: storage URL :param token: auth token; if None, no token will be sent :param container: container name that the object is in; if None, the container name is expected to be part of the url :param name: object name to put; if None, the object name is expected to be part of the url :param contents: a string or a file like object to read object data from; if None, a zero-byte put will be done :param content_length: value to send as content-length header; also limits the amount read from contents; if None, it will be computed via the contents or chunked transfer encoding will be used :param etag: etag of contents; if None, no etag will be sent :param chunk_size: chunk size of data to write; it defaults to 65536; used only if the the contents object has a 'read' method, eg. file-like objects, ignored otherwise :param content_type: value to send as content-type header; if None, no content-type will be set (remote end will likely try to auto-detect it) :param headers: additional headers to include in the request, if any :param http_conn: HTTP connection object (If None, it will create the conn object) :param proxy: proxy to connect through, if any; None by default; str of the format 'http://127.0.0.1:8888' to set one :param query_string: if set will be appended with '?' to generated path :param response_dict: an optional dictionary into which to place the response - status, reason and headers :returns: (headers, body) tuple :raises ClientException: HTTP POST request failed """ if http_conn: parsed, conn = http_conn else: parsed, conn = http_connection(url, proxy=proxy) path = parsed.path if container: path = '%s/%s' % (path.rstrip('/'), quote(container)) if name: path = '%s/%s' % (path.rstrip('/'), quote(name)) if query_string: path += '?' + query_string if headers: headers = dict(headers) else: headers = {} if token: headers['X-Auth-Token'] = token if etag: headers['ETag'] = etag.strip('"') if content_length is not None: headers['Content-Length'] = str(content_length) else: for n, v in headers.items(): if n.lower() == 'content-length': content_length = int(v) if content_type is not None: headers['Content-Type'] = content_type if not contents: headers['Content-Length'] = '0' headers['X-Zerovm-Execute'] = '1.0' if hasattr(contents, 'read'): if chunk_size is None: chunk_size = 65536 if content_length is None: def chunk_reader(): while True: data = contents.read(chunk_size) if not data: break yield data conn.request('POST', path, data=chunk_reader(), headers=headers) else: # Fixes https://github.com/kennethreitz/requests/issues/1648 data = LengthWrapper(contents, content_length) conn.request('POST', path, data=data, headers=headers) else: if chunk_size is not None: warn_msg = '%s object has no \"read\" method, ignoring chunk_size'\ % type(contents).__name__ warnings.warn(warn_msg, stacklevel=2) conn.request('POST', path, contents, headers) resp = conn.getresponse() body = resp.read() headers = {'X-Auth-Token': token} http_log(('%s%s' % (url.replace(parsed.path, ''), path), 'POST',), {'headers': headers}, resp, body) store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: raise ClientException('Object PUT failed', http_scheme=parsed.scheme, http_host=conn.host, http_path=path, http_status=resp.status, http_reason=resp.reason, http_response_content=body) return body, dict(resp.getheaders())