def _auth1(self): status = 0 reason = 'Unknown' attempt = 0 while attempt < self.attempts: attempt += 1 self.verbose('Attempting auth v1 with %s', self.auth_url) parsed, conn = self._connect(self.auth_url) self.verbose('> GET %s', parsed.path) conn.request( 'GET', parsed.path, '', { 'User-Agent': self.user_agent, 'X-Auth-User': quote(self.auth_user), 'X-Auth-Key': quote(self.auth_key) }) try: resp = conn.getresponse() status = resp.status reason = resp.reason self.verbose('< %s %s', status, reason) hdrs = headers_to_dict(resp.getheaders()) resp.read() resp.close() conn.close() except Exception as err: status = 0 reason = str(err) hdrs = {} if status == 401: break if status // 100 == 2: try: self.storage_url = hdrs['x-storage-url'] except KeyError: status = 0 reason = 'No x-storage-url header' break if self.snet: parsed = list(urlparse.urlparse(self.storage_url)) # Second item in the list is the netloc parsed[1] = 'snet-' + parsed[1] self.storage_url = urlparse.urlunparse(parsed) self.cdn_url = hdrs.get('x-cdn-management-url') self.auth_token = hdrs.get('x-auth-token') if not self.auth_token: self.auth_token = hdrs.get('x-storage-token') if not self.auth_token: status = 500 reason = ( 'No x-auth-token or x-storage-token header in ' 'response') break self._auth_save_cache() break elif status // 100 != 5: break self.client.sleep(2**attempt) return status, reason
def _auth1(self): status = 0 reason = 'Unknown' attempt = 0 while attempt < self.attempts: attempt += 1 self.verbose('Attempting auth v1 with %s', self.auth_url) parsed, conn = self._connect(self.auth_url) self.verbose('> GET %s', parsed.path) conn.request( 'GET', parsed.path, '', {'User-Agent': self.user_agent, 'X-Auth-User': quote(self.auth_user), 'X-Auth-Key': quote(self.auth_key)}) try: resp = conn.getresponse() status = resp.status reason = resp.reason self.verbose('< %s %s', status, reason) hdrs = headers_to_dict(resp.getheaders()) resp.read() resp.close() conn.close() except Exception as err: status = 0 reason = str(err) hdrs = {} if status == 401: break if status // 100 == 2: try: self.storage_url = hdrs['x-storage-url'] except KeyError: status = 0 reason = 'No x-storage-url header' break if self.snet: parsed = list(urlparse.urlparse(self.storage_url)) # Second item in the list is the netloc parsed[1] = 'snet-' + parsed[1] self.storage_url = urlparse.urlunparse(parsed) self.cdn_url = hdrs.get('x-cdn-management-url') self.auth_token = hdrs.get('x-auth-token') if not self.auth_token: self.auth_token = hdrs.get('x-storage-token') if not self.auth_token: status = 500 reason = ( 'No x-auth-token or x-storage-token header in ' 'response') break self._auth_save_cache() break elif status // 100 != 5: break self.client.sleep(2 ** attempt) return status, reason
def request(self, method, path, contents, headers, decode_json=False, stream=False, query=None, cdn=False): """ See :py:func:`swiftly.client.client.Client.request` """ if query: path += '?' + '&'.join( ('%s=%s' % (quote(k), quote(v)) if v else quote(k)) for k, v in sorted(query.iteritems())) reset_func = self._default_reset_func if isinstance(contents, basestring): contents = StringIO.StringIO(contents) tell = getattr(contents, 'tell', None) seek = getattr(contents, 'seek', None) if tell and seek: try: orig_pos = tell() reset_func = lambda: seek(orig_pos) except Exception: tell = seek = None elif not contents: reset_func = lambda: None status = 0 reason = 'Unknown' attempt = 0 while attempt < self.attempts: attempt += 1 if time() >= self.conn_discard: self.storage_conn = None self.cdn_conn = None if cdn: conn = self.cdn_conn conn_path = self.cdn_path else: conn = self.storage_conn conn_path = self.storage_path if not conn: parsed, conn = self._connect(cdn=cdn) if conn: if cdn: self.cdn_conn = conn self.cdn_path = conn_path = parsed.path else: self.storage_conn = conn self.storage_path = conn_path = parsed.path else: raise self.HTTPException('%s %s failed: No connection' % (method, path)) self.conn_discard = time() + 4 titled_headers = dict((k.title(), v) for k, v in { 'User-Agent': self.user_agent, 'X-Auth-Token': self.auth_token }.iteritems()) if headers: titled_headers.update( (k.title(), v) for k, v in headers.iteritems()) try: if not hasattr(contents, 'read'): if method not in self.no_content_methods and contents and \ 'Content-Length' not in titled_headers and \ 'Transfer-Encoding' not in titled_headers: titled_headers['Content-Length'] = str( len(contents or '')) verbose_headers = ' '.join( '%s: %s' % (k, v) for k, v in sorted(titled_headers.iteritems())) self.verbose('> %s %s %s', method, conn_path + path, verbose_headers) conn.request(method, conn_path + path, contents, titled_headers) else: conn.putrequest(method, conn_path + path) content_length = None for h, v in sorted(titled_headers.iteritems()): if h == 'Content-Length': content_length = int(v) conn.putheader(h, v) if method not in self.no_content_methods and \ content_length is None: titled_headers['Transfer-Encoding'] = 'chunked' conn.putheader('Transfer-Encoding', 'chunked') conn.endheaders() verbose_headers = ' '.join( '%s: %s' % (k, v) for k, v in sorted(titled_headers.iteritems())) self.verbose('> %s %s %s', method, conn_path + path, verbose_headers) if method not in self.no_content_methods and \ content_length is None: chunk = contents.read(self.chunk_size) while chunk: conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) chunk = contents.read(self.chunk_size) conn.send('0\r\n\r\n') else: left = content_length while left > 0: size = self.chunk_size if size > left: size = left chunk = contents.read(size) if not chunk: raise IOError('Early EOF from input') conn.send(chunk) left -= len(chunk) resp = conn.getresponse() status = resp.status reason = resp.reason hdrs = headers_to_dict(resp.getheaders()) if stream: value = resp else: value = resp.read() resp.close() except Exception as err: status = 0 reason = '%s %s' % (type(err), str(err)) hdrs = {} value = None self.verbose('< %s %s', status or '-', reason) if status == 401: if stream: value.close() conn.close() self.auth() attempt -= 1 elif status and status // 100 != 5: if not stream and decode_json and status // 100 == 2: if value: value = json.loads(value) else: value = None self.conn_discard = time() + 4 return (status, reason, hdrs, value) else: if stream and value: value.close() conn.close() if reset_func: reset_func() self.sleep(2**attempt) raise self.HTTPException('%s %s failed: %s %s' % (method, path, status, reason))
def request(self, method, path, contents, headers, decode_json=False, stream=False, query=None, cdn=False): """ See :py:func:`swiftly.client.client.Client.request` """ if query: path += '?' + '&'.join( ('%s=%s' % (quote(k), quote(v)) if v else quote(k)) for k, v in sorted(six.iteritems(query))) reset_func = self._default_reset_func if isinstance(contents, six.string_types): contents = StringIO(contents) tell = getattr(contents, 'tell', None) seek = getattr(contents, 'seek', None) if tell and seek: try: orig_pos = tell() reset_func = lambda: seek(orig_pos) except Exception: tell = seek = None elif not contents: reset_func = lambda: None status = 0 reason = 'Unknown' attempt = 0 while attempt < self.attempts: attempt += 1 if time() >= self.conn_discard: self.storage_conn = None self.cdn_conn = None if cdn: conn = self.cdn_conn conn_path = self.cdn_path else: conn = self.storage_conn conn_path = self.storage_path if not conn: parsed, conn = self._connect(cdn=cdn) if conn: if cdn: self.cdn_conn = conn self.cdn_path = conn_path = parsed.path else: self.storage_conn = conn self.storage_path = conn_path = parsed.path else: raise self.HTTPException( '%s %s failed: No connection' % (method, path)) self.conn_discard = time() + 4 titled_headers = dict((k.title(), v) for k, v in six.iteritems({ 'User-Agent': self.user_agent, 'X-Auth-Token': self.auth_token})) if headers: titled_headers.update( (k.title(), v) for k, v in six.iteritems(headers)) try: if not hasattr(contents, 'read'): if method not in self.no_content_methods and contents and \ 'Content-Length' not in titled_headers and \ 'Transfer-Encoding' not in titled_headers: titled_headers['Content-Length'] = str( len(contents or '')) verbose_headers = ' '.join( '%s: %s' % (k, v) for k, v in sorted(six.iteritems(titled_headers))) self.verbose( '> %s %s %s', method, conn_path + path, verbose_headers) conn.request( method, conn_path + path, contents, titled_headers) else: conn.putrequest(method, conn_path + path) content_length = None for h, v in sorted(six.iteritems(titled_headers)): if h == 'Content-Length': content_length = int(v) conn.putheader(h, v) if method not in self.no_content_methods and \ content_length is None: titled_headers['Transfer-Encoding'] = 'chunked' conn.putheader('Transfer-Encoding', 'chunked') conn.endheaders() verbose_headers = ' '.join( '%s: %s' % (k, v) for k, v in sorted(six.iteritems(titled_headers))) self.verbose( '> %s %s %s', method, conn_path + path, verbose_headers) if method not in self.no_content_methods and \ content_length is None: chunk = contents.read(self.chunk_size) while chunk: conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) chunk = contents.read(self.chunk_size) conn.send('0\r\n\r\n') else: left = content_length or 0 while left > 0: size = self.chunk_size if size > left: size = left chunk = contents.read(size) if not chunk: raise IOError('Early EOF from input') conn.send(chunk) left -= len(chunk) resp = conn.getresponse() status = resp.status reason = resp.reason hdrs = headers_to_dict(resp.getheaders()) if stream: value = resp else: value = resp.read() resp.close() except Exception as err: status = 0 reason = '%s %s' % (type(err), str(err)) hdrs = {} value = None self.verbose('< %s %s', status or '-', reason) if status == 401: if stream: value.close() conn.close() self.auth() attempt -= 1 elif status and status // 100 != 5: if not stream and decode_json and status // 100 == 2: if value: value = json.loads(value.decode('utf-8')) else: value = None self.conn_discard = time() + 4 return (status, reason, hdrs, value) else: if stream and value: value.close() conn.close() if reset_func: reset_func() self.sleep(2 ** attempt) raise self.HTTPException( '%s %s failed: %s %s' % (method, path, status, reason))
def request(self, method, path, contents, headers, decode_json=False, stream=False, query=None, cdn=False): """ See :py:func:`swiftly.client.client.Client.request` """ if query: path += '?' + '&'.join( ('%s=%s' % (quote(k), quote(v)) if v else quote(k)) for k, v in sorted(six.iteritems(query))) reset_func = self._default_reset_func if isinstance(contents, six.string_types): contents = StringIO(contents) tell = getattr(contents, 'tell', None) seek = getattr(contents, 'seek', None) if tell and seek: try: orig_pos = tell() reset_func = lambda: seek(orig_pos) except Exception: tell = seek = None elif not contents: reset_func = lambda: None status = 0 reason = 'Unknown' attempt = 0 while attempt < self.attempts: attempt += 1 if cdn: conn_path = self.cdn_path else: conn_path = self.storage_path titled_headers = dict((k.title(), v) for k, v in six.iteritems({ 'User-Agent': self.user_agent})) if headers: titled_headers.update( (k.title(), v) for k, v in six.iteritems(headers)) resp = None if not hasattr(contents, 'read'): if method not in self.no_content_methods and contents and \ 'Content-Length' not in titled_headers and \ 'Transfer-Encoding' not in titled_headers: titled_headers['Content-Length'] = str( len(contents or '')) req = self.Request.blank( conn_path + path, environ={'REQUEST_METHOD': method, 'swift_owner': True}, headers=titled_headers, body=contents) verbose_headers = ' '.join( '%s: %s' % (k, v) for k, v in six.iteritems(titled_headers)) self.verbose( '> %s %s %s', method, conn_path + path, verbose_headers) resp = req.get_response(self.swift_proxy) else: req = self.Request.blank( conn_path + path, environ={'REQUEST_METHOD': method, 'swift_owner': True}, headers=titled_headers) content_length = None for h, v in six.iteritems(titled_headers): if h.lower() == 'content-length': content_length = int(v) req.headers[h] = v if method not in self.no_content_methods and \ content_length is None: titled_headers['Transfer-Encoding'] = 'chunked' req.headers['Transfer-Encoding'] = 'chunked' else: req.content_length = content_length req.body_file = contents verbose_headers = ' '.join( '%s: %s' % (k, v) for k, v in six.iteritems(titled_headers)) self.verbose( '> %s %s %s', method, conn_path + path, verbose_headers) resp = req.get_response(self.swift_proxy) status = resp.status_int reason = resp.status.split(' ', 1)[1] hdrs = headers_to_dict(resp.headers.items()) if stream: def iter_reader(size=-1): if size == -1: return ''.join(resp.app_iter) else: try: return next(resp.app_iter) except StopIteration: return '' iter_reader.read = iter_reader value = iter_reader else: value = resp.body self.verbose('< %s %s', status, reason) if status and status // 100 != 5: if not stream and decode_json and status // 100 == 2: if value: value = json.loads(value) else: value = None return (status, reason, hdrs, value) if reset_func: reset_func() self.sleep(2 ** attempt) raise Exception('%s %s failed: %s %s' % (method, path, status, reason))
def request(self, method, path, contents, headers, decode_json=False, stream=False, query=None, cdn=False): """ See :py:func:`swiftly.client.client.Client.request` """ if query: path += '?' + '&'.join( ('%s=%s' % (quote(k), quote(v)) if v else quote(k)) for k, v in sorted(six.iteritems(query))) reset_func = self._default_reset_func if isinstance(contents, six.string_types): contents = StringIO(contents) tell = getattr(contents, 'tell', None) seek = getattr(contents, 'seek', None) if tell and seek: try: orig_pos = tell() reset_func = lambda: seek(orig_pos) except Exception: tell = seek = None elif not contents: reset_func = lambda: None status = 0 reason = 'Unknown' attempt = 0 while attempt < self.attempts: attempt += 1 if cdn: conn_path = self.cdn_path else: conn_path = self.storage_path titled_headers = dict( (k.title(), v) for k, v in six.iteritems({'User-Agent': self.user_agent})) if headers: titled_headers.update( (k.title(), v) for k, v in six.iteritems(headers)) resp = None if not hasattr(contents, 'read'): if method not in self.no_content_methods and contents and \ 'Content-Length' not in titled_headers and \ 'Transfer-Encoding' not in titled_headers: titled_headers['Content-Length'] = str(len(contents or '')) req = self.Request.blank(conn_path + path, environ={ 'REQUEST_METHOD': method, 'swift_owner': True }, headers=titled_headers, body=contents) verbose_headers = ' '.join( '%s: %s' % (k, v) for k, v in six.iteritems(titled_headers)) self.verbose('> %s %s %s', method, conn_path + path, verbose_headers) resp = req.get_response(self.swift_proxy) else: req = self.Request.blank(conn_path + path, environ={ 'REQUEST_METHOD': method, 'swift_owner': True }, headers=titled_headers) content_length = None for h, v in six.iteritems(titled_headers): if h.lower() == 'content-length': content_length = int(v) req.headers[h] = v if method not in self.no_content_methods and \ content_length is None: titled_headers['Transfer-Encoding'] = 'chunked' req.headers['Transfer-Encoding'] = 'chunked' else: req.content_length = content_length req.body_file = contents verbose_headers = ' '.join( '%s: %s' % (k, v) for k, v in six.iteritems(titled_headers)) self.verbose('> %s %s %s', method, conn_path + path, verbose_headers) resp = req.get_response(self.swift_proxy) status = resp.status_int reason = resp.status.split(' ', 1)[1] hdrs = headers_to_dict(resp.headers.items()) if stream: def iter_reader(size=-1): if size == -1: return ''.join(resp.app_iter) else: try: return next(resp.app_iter) except StopIteration: return '' iter_reader.read = iter_reader value = iter_reader else: value = resp.body self.verbose('< %s %s', status, reason) if status and status // 100 != 5: if not stream and decode_json and status // 100 == 2: if value: value = json.loads(value) else: value = None return (status, reason, hdrs, value) if reset_func: reset_func() self.sleep(2**attempt) raise Exception('%s %s failed: %s %s' % (method, path, status, reason))