def _auth1(self):
status = 0
reason = 'Unknown'
attempt = 0
while attempt < self.attempts:
attempt += 1
self.verbose('Attempting auth v1 with %s', self.auth_url)
parsed, conn = self._connect(self.auth_url)
self.verbose('> GET %s', parsed.path)
conn.request(
'GET', parsed.path, '', {
'User-Agent': self.user_agent,
'X-Auth-User': quote(self.auth_user),
'X-Auth-Key': quote(self.auth_key)
})
try:
resp = conn.getresponse()
status = resp.status
reason = resp.reason
self.verbose('< %s %s', status, reason)
hdrs = headers_to_dict(resp.getheaders())
resp.read()
resp.close()
conn.close()
except Exception as err:
status = 0
reason = str(err)
hdrs = {}
if status == 401:
break
if status // 100 == 2:
try:
self.storage_url = hdrs['x-storage-url']
except KeyError:
status = 0
reason = 'No x-storage-url header'
break
if self.snet:
parsed = list(urlparse.urlparse(self.storage_url))
# Second item in the list is the netloc
parsed[1] = 'snet-' + parsed[1]
self.storage_url = urlparse.urlunparse(parsed)
self.cdn_url = hdrs.get('x-cdn-management-url')
self.auth_token = hdrs.get('x-auth-token')
if not self.auth_token:
self.auth_token = hdrs.get('x-storage-token')
if not self.auth_token:
status = 500
reason = (
'No x-auth-token or x-storage-token header in '
'response')
break
self._auth_save_cache()
break
elif status // 100 != 5:
break
self.client.sleep(2**attempt)
return status, reason
def _auth1(self):
status = 0
reason = 'Unknown'
attempt = 0
while attempt < self.attempts:
attempt += 1
self.verbose('Attempting auth v1 with %s', self.auth_url)
parsed, conn = self._connect(self.auth_url)
self.verbose('> GET %s', parsed.path)
conn.request(
'GET', parsed.path, '',
{'User-Agent': self.user_agent,
'X-Auth-User': quote(self.auth_user),
'X-Auth-Key': quote(self.auth_key)})
try:
resp = conn.getresponse()
status = resp.status
reason = resp.reason
self.verbose('< %s %s', status, reason)
hdrs = headers_to_dict(resp.getheaders())
resp.read()
resp.close()
conn.close()
except Exception as err:
status = 0
reason = str(err)
hdrs = {}
if status == 401:
break
if status // 100 == 2:
try:
self.storage_url = hdrs['x-storage-url']
except KeyError:
status = 0
reason = 'No x-storage-url header'
break
if self.snet:
parsed = list(urlparse.urlparse(self.storage_url))
# Second item in the list is the netloc
parsed[1] = 'snet-' + parsed[1]
self.storage_url = urlparse.urlunparse(parsed)
self.cdn_url = hdrs.get('x-cdn-management-url')
self.auth_token = hdrs.get('x-auth-token')
if not self.auth_token:
self.auth_token = hdrs.get('x-storage-token')
if not self.auth_token:
status = 500
reason = (
'No x-auth-token or x-storage-token header in '
'response')
break
self._auth_save_cache()
break
elif status // 100 != 5:
break
self.client.sleep(2 ** attempt)
return status, reason
def request(self,
method,
path,
contents,
headers,
decode_json=False,
stream=False,
query=None,
cdn=False):
"""
See :py:func:`swiftly.client.client.Client.request`
"""
if query:
path += '?' + '&'.join(
('%s=%s' % (quote(k), quote(v)) if v else quote(k))
for k, v in sorted(query.iteritems()))
reset_func = self._default_reset_func
if isinstance(contents, basestring):
contents = StringIO.StringIO(contents)
tell = getattr(contents, 'tell', None)
seek = getattr(contents, 'seek', None)
if tell and seek:
try:
orig_pos = tell()
reset_func = lambda: seek(orig_pos)
except Exception:
tell = seek = None
elif not contents:
reset_func = lambda: None
status = 0
reason = 'Unknown'
attempt = 0
while attempt < self.attempts:
attempt += 1
if time() >= self.conn_discard:
self.storage_conn = None
self.cdn_conn = None
if cdn:
conn = self.cdn_conn
conn_path = self.cdn_path
else:
conn = self.storage_conn
conn_path = self.storage_path
if not conn:
parsed, conn = self._connect(cdn=cdn)
if conn:
if cdn:
self.cdn_conn = conn
self.cdn_path = conn_path = parsed.path
else:
self.storage_conn = conn
self.storage_path = conn_path = parsed.path
else:
raise self.HTTPException('%s %s failed: No connection' %
(method, path))
self.conn_discard = time() + 4
titled_headers = dict((k.title(), v) for k, v in {
'User-Agent': self.user_agent,
'X-Auth-Token': self.auth_token
}.iteritems())
if headers:
titled_headers.update(
(k.title(), v) for k, v in headers.iteritems())
try:
if not hasattr(contents, 'read'):
if method not in self.no_content_methods and contents and \
'Content-Length' not in titled_headers and \
'Transfer-Encoding' not in titled_headers:
titled_headers['Content-Length'] = str(
len(contents or ''))
verbose_headers = ' '.join(
'%s: %s' % (k, v)
for k, v in sorted(titled_headers.iteritems()))
self.verbose('> %s %s %s', method, conn_path + path,
verbose_headers)
conn.request(method, conn_path + path, contents,
titled_headers)
else:
conn.putrequest(method, conn_path + path)
content_length = None
for h, v in sorted(titled_headers.iteritems()):
if h == 'Content-Length':
content_length = int(v)
conn.putheader(h, v)
if method not in self.no_content_methods and \
content_length is None:
titled_headers['Transfer-Encoding'] = 'chunked'
conn.putheader('Transfer-Encoding', 'chunked')
conn.endheaders()
verbose_headers = ' '.join(
'%s: %s' % (k, v)
for k, v in sorted(titled_headers.iteritems()))
self.verbose('> %s %s %s', method, conn_path + path,
verbose_headers)
if method not in self.no_content_methods and \
content_length is None:
chunk = contents.read(self.chunk_size)
while chunk:
conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
chunk = contents.read(self.chunk_size)
conn.send('0\r\n\r\n')
else:
left = content_length
while left > 0:
size = self.chunk_size
if size > left:
size = left
chunk = contents.read(size)
if not chunk:
raise IOError('Early EOF from input')
conn.send(chunk)
left -= len(chunk)
resp = conn.getresponse()
status = resp.status
reason = resp.reason
hdrs = headers_to_dict(resp.getheaders())
if stream:
value = resp
else:
value = resp.read()
resp.close()
except Exception as err:
status = 0
reason = '%s %s' % (type(err), str(err))
hdrs = {}
value = None
self.verbose('< %s %s', status or '-', reason)
if status == 401:
if stream:
value.close()
conn.close()
self.auth()
attempt -= 1
elif status and status // 100 != 5:
if not stream and decode_json and status // 100 == 2:
if value:
value = json.loads(value)
else:
value = None
self.conn_discard = time() + 4
return (status, reason, hdrs, value)
else:
if stream and value:
value.close()
conn.close()
if reset_func:
reset_func()
self.sleep(2**attempt)
raise self.HTTPException('%s %s failed: %s %s' %
(method, path, status, reason))
def request(self, method, path, contents, headers, decode_json=False,
stream=False, query=None, cdn=False):
"""
See :py:func:`swiftly.client.client.Client.request`
"""
if query:
path += '?' + '&'.join(
('%s=%s' % (quote(k), quote(v)) if v else quote(k))
for k, v in sorted(six.iteritems(query)))
reset_func = self._default_reset_func
if isinstance(contents, six.string_types):
contents = StringIO(contents)
tell = getattr(contents, 'tell', None)
seek = getattr(contents, 'seek', None)
if tell and seek:
try:
orig_pos = tell()
reset_func = lambda: seek(orig_pos)
except Exception:
tell = seek = None
elif not contents:
reset_func = lambda: None
status = 0
reason = 'Unknown'
attempt = 0
while attempt < self.attempts:
attempt += 1
if time() >= self.conn_discard:
self.storage_conn = None
self.cdn_conn = None
if cdn:
conn = self.cdn_conn
conn_path = self.cdn_path
else:
conn = self.storage_conn
conn_path = self.storage_path
if not conn:
parsed, conn = self._connect(cdn=cdn)
if conn:
if cdn:
self.cdn_conn = conn
self.cdn_path = conn_path = parsed.path
else:
self.storage_conn = conn
self.storage_path = conn_path = parsed.path
else:
raise self.HTTPException(
'%s %s failed: No connection' % (method, path))
self.conn_discard = time() + 4
titled_headers = dict((k.title(), v) for k, v in six.iteritems({
'User-Agent': self.user_agent,
'X-Auth-Token': self.auth_token}))
if headers:
titled_headers.update(
(k.title(), v) for k, v in six.iteritems(headers))
try:
if not hasattr(contents, 'read'):
if method not in self.no_content_methods and contents and \
'Content-Length' not in titled_headers and \
'Transfer-Encoding' not in titled_headers:
titled_headers['Content-Length'] = str(
len(contents or ''))
verbose_headers = ' '.join(
'%s: %s' % (k, v)
for k, v in sorted(six.iteritems(titled_headers)))
self.verbose(
'> %s %s %s', method, conn_path + path,
verbose_headers)
conn.request(
method, conn_path + path, contents, titled_headers)
else:
conn.putrequest(method, conn_path + path)
content_length = None
for h, v in sorted(six.iteritems(titled_headers)):
if h == 'Content-Length':
content_length = int(v)
conn.putheader(h, v)
if method not in self.no_content_methods and \
content_length is None:
titled_headers['Transfer-Encoding'] = 'chunked'
conn.putheader('Transfer-Encoding', 'chunked')
conn.endheaders()
verbose_headers = ' '.join(
'%s: %s' % (k, v)
for k, v in sorted(six.iteritems(titled_headers)))
self.verbose(
'> %s %s %s', method, conn_path + path,
verbose_headers)
if method not in self.no_content_methods and \
content_length is None:
chunk = contents.read(self.chunk_size)
while chunk:
conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
chunk = contents.read(self.chunk_size)
conn.send('0\r\n\r\n')
else:
left = content_length or 0
while left > 0:
size = self.chunk_size
if size > left:
size = left
chunk = contents.read(size)
if not chunk:
raise IOError('Early EOF from input')
conn.send(chunk)
left -= len(chunk)
resp = conn.getresponse()
status = resp.status
reason = resp.reason
hdrs = headers_to_dict(resp.getheaders())
if stream:
value = resp
else:
value = resp.read()
resp.close()
except Exception as err:
status = 0
reason = '%s %s' % (type(err), str(err))
hdrs = {}
value = None
self.verbose('< %s %s', status or '-', reason)
if status == 401:
if stream:
value.close()
conn.close()
self.auth()
attempt -= 1
elif status and status // 100 != 5:
if not stream and decode_json and status // 100 == 2:
if value:
value = json.loads(value.decode('utf-8'))
else:
value = None
self.conn_discard = time() + 4
return (status, reason, hdrs, value)
else:
if stream and value:
value.close()
conn.close()
if reset_func:
reset_func()
self.sleep(2 ** attempt)
raise self.HTTPException(
'%s %s failed: %s %s' % (method, path, status, reason))
def request(self, method, path, contents, headers, decode_json=False,
stream=False, query=None, cdn=False):
"""
See :py:func:`swiftly.client.client.Client.request`
"""
if query:
path += '?' + '&'.join(
('%s=%s' % (quote(k), quote(v)) if v else quote(k))
for k, v in sorted(six.iteritems(query)))
reset_func = self._default_reset_func
if isinstance(contents, six.string_types):
contents = StringIO(contents)
tell = getattr(contents, 'tell', None)
seek = getattr(contents, 'seek', None)
if tell and seek:
try:
orig_pos = tell()
reset_func = lambda: seek(orig_pos)
except Exception:
tell = seek = None
elif not contents:
reset_func = lambda: None
status = 0
reason = 'Unknown'
attempt = 0
while attempt < self.attempts:
attempt += 1
if cdn:
conn_path = self.cdn_path
else:
conn_path = self.storage_path
titled_headers = dict((k.title(), v) for k, v in six.iteritems({
'User-Agent': self.user_agent}))
if headers:
titled_headers.update(
(k.title(), v) for k, v in six.iteritems(headers))
resp = None
if not hasattr(contents, 'read'):
if method not in self.no_content_methods and contents and \
'Content-Length' not in titled_headers and \
'Transfer-Encoding' not in titled_headers:
titled_headers['Content-Length'] = str(
len(contents or ''))
req = self.Request.blank(
conn_path + path,
environ={'REQUEST_METHOD': method, 'swift_owner': True},
headers=titled_headers, body=contents)
verbose_headers = ' '.join(
'%s: %s' % (k, v) for k, v in six.iteritems(titled_headers))
self.verbose(
'> %s %s %s', method, conn_path + path, verbose_headers)
resp = req.get_response(self.swift_proxy)
else:
req = self.Request.blank(
conn_path + path,
environ={'REQUEST_METHOD': method, 'swift_owner': True},
headers=titled_headers)
content_length = None
for h, v in six.iteritems(titled_headers):
if h.lower() == 'content-length':
content_length = int(v)
req.headers[h] = v
if method not in self.no_content_methods and \
content_length is None:
titled_headers['Transfer-Encoding'] = 'chunked'
req.headers['Transfer-Encoding'] = 'chunked'
else:
req.content_length = content_length
req.body_file = contents
verbose_headers = ' '.join(
'%s: %s' % (k, v) for k, v in six.iteritems(titled_headers))
self.verbose(
'> %s %s %s', method, conn_path + path, verbose_headers)
resp = req.get_response(self.swift_proxy)
status = resp.status_int
reason = resp.status.split(' ', 1)[1]
hdrs = headers_to_dict(resp.headers.items())
if stream:
def iter_reader(size=-1):
if size == -1:
return ''.join(resp.app_iter)
else:
try:
return next(resp.app_iter)
except StopIteration:
return ''
iter_reader.read = iter_reader
value = iter_reader
else:
value = resp.body
self.verbose('< %s %s', status, reason)
if status and status // 100 != 5:
if not stream and decode_json and status // 100 == 2:
if value:
value = json.loads(value)
else:
value = None
return (status, reason, hdrs, value)
if reset_func:
reset_func()
self.sleep(2 ** attempt)
raise Exception('%s %s failed: %s %s' % (method, path, status, reason))
def _object_path(self, container, obj):
container = self._container_path(container)
# Leading/trailing slashes are allowed in object names, so don't strip
# them.
return container + '/' + quote(obj)
def request(self,
method,
path,
contents,
headers,
decode_json=False,
stream=False,
query=None,
cdn=False):
"""
See :py:func:`swiftly.client.client.Client.request`
"""
if query:
path += '?' + '&'.join(
('%s=%s' % (quote(k), quote(v)) if v else quote(k))
for k, v in sorted(six.iteritems(query)))
reset_func = self._default_reset_func
if isinstance(contents, six.string_types):
contents = StringIO(contents)
tell = getattr(contents, 'tell', None)
seek = getattr(contents, 'seek', None)
if tell and seek:
try:
orig_pos = tell()
reset_func = lambda: seek(orig_pos)
except Exception:
tell = seek = None
elif not contents:
reset_func = lambda: None
status = 0
reason = 'Unknown'
attempt = 0
while attempt < self.attempts:
attempt += 1
if cdn:
conn_path = self.cdn_path
else:
conn_path = self.storage_path
titled_headers = dict(
(k.title(), v)
for k, v in six.iteritems({'User-Agent': self.user_agent}))
if headers:
titled_headers.update(
(k.title(), v) for k, v in six.iteritems(headers))
resp = None
if not hasattr(contents, 'read'):
if method not in self.no_content_methods and contents and \
'Content-Length' not in titled_headers and \
'Transfer-Encoding' not in titled_headers:
titled_headers['Content-Length'] = str(len(contents or ''))
req = self.Request.blank(conn_path + path,
environ={
'REQUEST_METHOD': method,
'swift_owner': True
},
headers=titled_headers,
body=contents)
verbose_headers = ' '.join(
'%s: %s' % (k, v)
for k, v in six.iteritems(titled_headers))
self.verbose('> %s %s %s', method, conn_path + path,
verbose_headers)
resp = req.get_response(self.swift_proxy)
else:
req = self.Request.blank(conn_path + path,
environ={
'REQUEST_METHOD': method,
'swift_owner': True
},
headers=titled_headers)
content_length = None
for h, v in six.iteritems(titled_headers):
if h.lower() == 'content-length':
content_length = int(v)
req.headers[h] = v
if method not in self.no_content_methods and \
content_length is None:
titled_headers['Transfer-Encoding'] = 'chunked'
req.headers['Transfer-Encoding'] = 'chunked'
else:
req.content_length = content_length
req.body_file = contents
verbose_headers = ' '.join(
'%s: %s' % (k, v)
for k, v in six.iteritems(titled_headers))
self.verbose('> %s %s %s', method, conn_path + path,
verbose_headers)
resp = req.get_response(self.swift_proxy)
status = resp.status_int
reason = resp.status.split(' ', 1)[1]
hdrs = headers_to_dict(resp.headers.items())
if stream:
def iter_reader(size=-1):
if size == -1:
return ''.join(resp.app_iter)
else:
try:
return next(resp.app_iter)
except StopIteration:
return ''
iter_reader.read = iter_reader
value = iter_reader
else:
value = resp.body
self.verbose('< %s %s', status, reason)
if status and status // 100 != 5:
if not stream and decode_json and status // 100 == 2:
if value:
value = json.loads(value)
else:
value = None
return (status, reason, hdrs, value)
if reset_func:
reset_func()
self.sleep(2**attempt)
raise Exception('%s %s failed: %s %s' % (method, path, status, reason))
def _container_path(self, container):
container = container.rstrip('/')
if container.startswith('/'):
return quote(container)
else:
return '/' + quote(container)
def _object_path(self, container, obj):
container = self._container_path(container)
# Leading/trailing slashes are allowed in object names, so don't strip
# them.
return container + '/' + quote(obj)
def _container_path(self, container):
container = container.rstrip('/')
if container.startswith('/'):
return quote(container)
else:
return '/' + quote(container)
def get_account_hash(self):
"""
See :py:func:`swiftly.client.client.Client.get_account_hash`
"""
return quote(self.local_path, safe='')
def get_account_hash(self):
"""
See :py:func:`swiftly.client.client.Client.get_account_hash`
"""
return quote(self.local_path, safe='')
