def make_request(self, method, path=[], data='', hdrs=None, parms=None):
"""
Given a method (i.e. GET, PUT, POST, etc), a path, data, header and
metadata dicts, and an optional dictionary of query parameters,
performs an http request.
@type method: str
@param method: http method
@type path: list
@param path: the url's path, include [container_name], [obj_name]
@type hdrs: dict
@param hdrs: http headers
@type parms: dict
@param parms: query args
"""
if self._share_request and self._share_user_uri:
path = '/%s/%s' % \
(self._share_user_uri.rstrip('/'), '/'.join([unicode_quote(i) for i in path]))
else:
path = '/%s/%s' % \
(self.uri.rstrip('/'), '/'.join([unicode_quote(i) for i in path]))
if isinstance(parms, dict) and parms:
# 查询参数中的变量是固定的，为limits等字符，都为英文字符
# 但查询参数中的值有可能是unicode值，
# 因此，对于对于查询参数中的值需要进行unicode处理，使用unicode_quote()
# 这应该算一个bug，可以提交给作者
query_args = \
['%s=%s' % (unicode_quote(x),
unicode_quote(y)) for (x, y) in parms.items()]
path = '%s?%s' % (path, '&'.join(query_args))
headers = {
# 设置了Content-Length，这样上传或下载文件时需要优化一下
'Content-Length': str(len(data)),
'User-Agent': self.user_agent,
'X-Auth-Token': self.token
}
isinstance(hdrs, dict) and headers.update(hdrs)
def retry_request():
'''Re-connect and re-try a failed request once'''
self.http_connect()
self.connection.request(method, path, data, headers)
return self.connection.getresponse()
try:
self.connection.request(method, path, data, headers)
response = self.connection.getresponse()
except (socket.error, IOError, HTTPException):
response = retry_request()
if response.status == 401:
self._authenticate()
headers['X-Auth-Token'] = self.token
response = retry_request()
return response
def make_request(self, method, path=[], data='', hdrs=None, parms=None):
"""
Given a method (i.e. GET, PUT, POST, etc), a path, data, header and
metadata dicts, and an optional dictionary of query parameters,
performs an http request.
@type method: str
@param method: http method
@type path: list
@param path: the url's path, include [container_name], [obj_name]
@type hdrs: dict
@param hdrs: http headers
@type parms: dict
@param parms: query args
"""
if self._share_request and self._share_user_uri:
path = '/%s/%s' % \
(self._share_user_uri.rstrip('/'), '/'.join([unicode_quote(i) for i in path]))
else:
path = '/%s/%s' % \
(self.uri.rstrip('/'), '/'.join([unicode_quote(i) for i in path]))
if isinstance(parms, dict) and parms:
# 查询参数中的变量是固定的，为limits等字符，都为英文字符
# 但查询参数中的值有可能是unicode值，
# 因此，对于对于查询参数中的值需要进行unicode处理，使用unicode_quote()
# 这应该算一个bug，可以提交给作者
query_args = \
['%s=%s' % (unicode_quote(x),
unicode_quote(y)) for (x, y) in parms.items()]
path = '%s?%s' % (path, '&'.join(query_args))
headers = {
# 设置了Content-Length，这样上传或下载文件时需要优化一下
'Content-Length': str(len(data)),
'User-Agent': self.user_agent,
'X-Auth-Token': self.token
}
isinstance(hdrs, dict) and headers.update(hdrs)
def retry_request():
'''Re-connect and re-try a failed request once'''
self.http_connect()
self.connection.request(method, path, data, headers)
return self.connection.getresponse()
try:
self.connection.request(method, path, data, headers)
response = self.connection.getresponse()
except (socket.error, IOError, HTTPException):
response = retry_request()
if response.status == 401:
self._authenticate()
headers['X-Auth-Token'] = self.token
response = retry_request()
return response
def make_request(self, method, path=[], data='', hdrs=None, parms=None):
"""
Given a method (i.e. GET, PUT, POST, etc), a path, data, header and
metadata dicts, and an optional dictionary of query parameters,
performs an http request.
"""
path = '/%s/%s' % \
(self.uri.rstrip('/'), '/'.join([unicode_quote(i) for i in path]))
if isinstance(parms, dict) and parms:
path = '%s?%s' % (path, urlencode(parms))
headers = {'Content-Length': str(len(data)),
'User-Agent': self.user_agent,
'X-Auth-Token': self.token}
isinstance(hdrs, dict) and headers.update(hdrs)
def retry_request():
'''Re-connect and re-try a failed request once'''
self.http_connect()
self.connection.request(method, path, data, headers)
return self.connection.getresponse()
try:
self.connection.request(method, path, data, headers)
response = self.connection.getresponse()
except (socket.error, IOError, HTTPException):
response = retry_request()
if response.status == 401:
self._authenticate()
headers['X-Auth-Token'] = self.token
response = retry_request()
return response
def copy_from(self, container_name, name):
"""
Copy another object's contents to this object.
"""
self._name_check()
self._name_check(name)
# This method implicitly disables verification.
if not self._etag_override:
self._etag = None
headers = self._make_headers()
headers['X-Copy-From'] = unicode_quote("%s/%s" % (container_name, name))
headers['Content-Length'] = 0
response = self.container.conn.make_request(
'PUT', [self.container.name, self.name], hdrs=headers, data='')
buff = response.read()
if response.status < 200 or response.status > 299:
raise ResponseError(response.status, response.reason)
# Reset the etag to what the server returns.
for hdr in response.getheaders():
if hdr[0].lower() == 'etag':
self._etag = hdr[1]
def cdn_request(self, method, path=[], data='', hdrs=None):
"""
Given a method (i.e. GET, PUT, POST, etc), a path, data, header and
metadata dicts, performs an http request against the CDN service.
"""
if not self.cdn_enabled:
raise CDNNotEnabled()
path = '/%s/%s' % \
(self.uri.rstrip('/'), '/'.join([unicode_quote(i) for i in path]))
headers = {
'Content-Length': str(len(data)),
'User-Agent': self.user_agent,
'X-Auth-Token': self.token
}
if isinstance(hdrs, dict):
headers.update(hdrs)
def retry_request():
'''Re-connect and re-try a failed request once'''
self.cdn_connect()
self.cdn_connection.request(method, path, data, headers)
return self.cdn_connection.getresponse()
try:
self.cdn_connection.request(method, path, data, headers)
response = self.cdn_connection.getresponse()
except (socket.error, IOError, HTTPException):
response = retry_request()
if response.status == 401:
self._authenticate()
headers['X-Auth-Token'] = self.token
response = retry_request()
return response
def cdn_request(self, method, path=[], data='', hdrs=None):
"""
Given a method (i.e. GET, PUT, POST, etc), a path, data, header and
metadata dicts, performs an http request against the CDN service.
"""
if not self.cdn_enabled:
raise CDNNotEnabled()
path = '/%s/%s' % \
(self.uri.rstrip('/'), '/'.join([unicode_quote(i) for i in path]))
headers = {'Content-Length': str(len(data)),
'User-Agent': self.user_agent,
'X-Auth-Token': self.token}
if isinstance(hdrs, dict):
headers.update(hdrs)
def retry_request():
'''Re-connect and re-try a failed request once'''
self.cdn_connect()
self.cdn_connection.request(method, path, data, headers)
return self.cdn_connection.getresponse()
try:
self.cdn_connection.request(method, path, data, headers)
response = self.cdn_connection.getresponse()
except (socket.error, IOError, HTTPException):
response = retry_request()
if response.status == 401:
self._authenticate()
headers['X-Auth-Token'] = self.token
response = retry_request()
return response
def copy_from(self, container_name, name):
"""
Copy another object's contents to this object.
"""
self._name_check()
self._name_check(name)
# This method implicitly disables verification.
if not self._etag_override:
self._etag = None
headers = self._make_headers()
headers['X-Copy-From'] = unicode_quote("%s/%s" % (container_name, name))
headers['Content-Length'] = 0
response = self.container.conn.make_request(
'PUT', [self.container.name, self.name], hdrs=headers, data='')
buff = response.read()
if response.status < 200 or response.status > 299:
raise ResponseError(response.status, response.reason)
# Reset the etag to what the server returns.
for hdr in response.getheaders():
if hdr[0].lower() == 'etag':
self._etag = hdr[1]
def __get_conn_for_write(self):
headers = self._make_headers()
headers['X-Auth-Token'] = self.container.conn.token
path = "/%s/%s/%s" % (self.container.conn.uri.rstrip('/'), \
unicode_quote(self.container.name), unicode_quote(self.name))
# Requests are handled a little differently for writes ...
http = self.container.conn.connection
# TODO: more/better exception handling please
http.putrequest('PUT', path)
for hdr in headers:
http.putheader(hdr, headers[hdr])
http.putheader('User-Agent', self.container.conn.user_agent)
http.endheaders()
return http
def __get_conn_for_write(self):
headers = self._make_headers()
headers['X-Auth-Token'] = self.container.conn.token
path = "/%s/%s/%s" % (self.container.conn.uri.rstrip('/'), \
unicode_quote(self.container.name), unicode_quote(self.name))
# Requests are handled a little differently for writes ...
http = self.container.conn.connection
# TODO: more/better exception handling please
http.putrequest('PUT', path)
for hdr in headers:
http.putheader(hdr, headers[hdr])
http.putheader('User-Agent', self.container.conn.user_agent)
http.endheaders()
return http
def _construct_path(self, path, parms=None):
"""
Construct path for http request.
"""
path = '/%s/%s' % \
(self.uri.rstrip('/'), '/'.join([unicode_quote(i) for i in path]))
if isinstance(parms, dict) and parms:
path = '%s?%s' % (path, urlencode(parms))
return path
def make_request(self, method, path=[], data='', hdrs=None, parms=None):
"""
Given a method (i.e. GET, PUT, POST, etc), a path, data, header and
metadata dicts, and an optional dictionary of query parameters,
performs an http request.
"""
query_args = ""
path = '/%s/%s' % \
(self.uri.rstrip('/'), '/'.join(
[unicode_quote(i) for i in path]))
if isinstance(parms, dict) and parms:
query_args = \
['%s=%s' % (quote(x),
quote(str(y))) for (x, y) in parms.items()]
elif isinstance(parms, list) and parms:
query_args = \
["%s" % x for x in parms]
path = '%s?%s' % (path, '&'.join(query_args))
headers = {
'Content-Length': str(len(data)),
'User-Agent': self.user_agent,
'X-Auth-Token': self.token,
'Content-Type': 'application/xml'
}
isinstance(hdrs, dict) and headers.update(hdrs)
def retry_request():
'''Re-connect and re-try a failed request once'''
self.http_connect()
self.connection.request(method, path, data, headers)
return self.connection.getresponse()
try:
if 'PYTHON_CLOUDDNS_DEBUG' in os.environ and \
os.environ['PYTHON_CLOUDDNS_DEBUG'].strip():
import sys
url = "https://%s%s\n" % \
(self.connection_args[0],
path)
sys.stderr.write("METHOD: %s\n" % (str(method)))
sys.stderr.write("URL: %s" % (url))
sys.stderr.write("HEADERS: %s\n" % (str(headers)))
sys.stderr.write("DATA: %s\n" % (str(data)))
sys.stderr.write("curl -X '%s' -H 'X-Auth-Token: %s' %s %s" % \
(method, self.token, url, str(data)))
self.connection.request(method, path, data, headers)
response = self.connection.getresponse()
except (socket.error, IOError, HTTPException):
response = retry_request()
if response.status == 401:
self._authenticate()
headers['X-Auth-Token'] = self.token
response = retry_request()
return response
def make_request(self, method, path=[], data='', hdrs=None, parms=None):
"""
Given a method (i.e. GET, PUT, POST, etc), a path, data, header and
metadata dicts, and an optional dictionary of query parameters,
performs an http request.
"""
query_args = ""
path = '/%s/%s' % \
(self.uri.rstrip('/'), '/'.join(
[unicode_quote(i) for i in path]))
if isinstance(parms, dict) and parms:
query_args = \
['%s=%s' % (quote(x),
quote(str(y))) for (x, y) in parms.items()]
elif isinstance(parms, list) and parms:
query_args = \
["%s" % x for x in parms]
path = '%s?%s' % (path, '&'.join(query_args))
headers = {'Content-Length': str(len(data)),
'User-Agent': self.user_agent,
'X-Auth-Token': self.token,
'Content-Type': 'application/xml'}
isinstance(hdrs, dict) and headers.update(hdrs)
def retry_request():
'''Re-connect and re-try a failed request once'''
self.http_connect()
self.connection.request(method, path, data, headers)
return self.connection.getresponse()
try:
if 'PYTHON_CLOUDDNS_DEBUG' in os.environ and \
os.environ['PYTHON_CLOUDDNS_DEBUG'].strip():
import sys
url = "https://%s%s\n" % \
(self.connection_args[0],
path)
sys.stderr.write("METHOD: %s\n" % (str(method)))
sys.stderr.write("URL: %s" % (url))
sys.stderr.write("HEADERS: %s\n" % (str(headers)))
sys.stderr.write("DATA: %s\n" % (str(data)))
sys.stderr.write("curl -X '%s' -H 'X-Auth-Token: %s' %s %s" % \
(method, self.token, url, str(data)))
self.connection.request(method, path, data, headers)
response = self.connection.getresponse()
except (socket.error, IOError, HTTPException):
response = retry_request()
if response.status == 401:
self._authenticate()
headers['X-Auth-Token'] = self.token
response = retry_request()
return response
def public_streaming_uri(self):
"""
Retrieve the streaming URI for this object, if its container is public.
>>> container1 = connection['container1']
>>> container1.make_public()
>>> container1.create_object('file.txt').write('testing')
>>> container1['file.txt'].public_streaming_uri()
'https://c61.stream.rackcdn.com/file.txt'
@return: the public Streaming URI for this object
@rtype: str
"""
return "%s/%s" % (self.container.public_streaming_uri().rstrip('/'),
unicode_quote(self.name))
def public_streaming_uri(self):
"""
Retrieve the streaming URI for this object, if its container is public.
>>> container1 = connection['container1']
>>> container1.make_public()
>>> container1.create_object('file.txt').write('testing')
>>> container1['file.txt'].public_streaming_uri()
'https://c61.stream.rackcdn.com/file.txt'
@return: the public Streaming URI for this object
@rtype: str
"""
return "%s/%s" % (self.container.public_streaming_uri().rstrip('/'),
unicode_quote(self.name))
def make_request(self, method, path=[], data='', hdrs=None, parms=None):
"""
Given a method (i.e. GET, PUT, POST, etc), a path, data, header and
metadata dicts, and an optional dictionary of query parameters,
performs an http request.
"""
path = '/%s/%s' % \
(self.uri.rstrip('/'), '/'.join([unicode_quote(i) for i in path]))
if isinstance(parms, dict) and parms:
query_args = \
['%s=%s' % (quote(x),
quote(str(y))) for (x, y) in parms.items()]
path = '%s?%s' % (path, '&'.join(query_args))
headers = {
'Content-Length': str(len(data)),
'User-Agent': self.user_agent,
'X-Auth-Token': self.token
}
isinstance(hdrs, dict) and headers.update(hdrs)
def retry_request():
'''Re-connect and re-try a failed request once'''
self.http_connect()
self.connection.request(method, path, data, headers)
return self.connection.getresponse()
try:
self.connection.request(method, path, data, headers)
response = self.connection.getresponse()
except (socket.error, IOError, HTTPException):
response = retry_request()
if response.status == 401:
self._authenticate()
headers['X-Auth-Token'] = self.token
response = retry_request()
return response
def send(self, iterable):
"""
Write potentially transient data to the remote storage system using a
generator or stream.
If the object's size is not set, chunked transfer encoding will be
used to upload the file.
If the object's size attribute is set, it will be used as the
Content-Length. If the generator raises StopIteration prior to
yielding the right number of bytes, an IncompleteSend exception is
raised.
If the content_type attribute is not set then a value of
application/octet-stream will be used.
Server-side verification will be performed if an md5 checksum is
assigned to the etag property before calling this method,
otherwise no verification will be performed, (verification
can be performed afterward though by using the etag attribute
which is set to the value returned by the server).
>>> test_object = container.create_object('backup.tar.gz')
>>> pfd = os.popen('tar -czvf - ./data/', 'r')
>>> test_object.send(pfd)
@param iterable: stream or generator which yields the content to upload
@type iterable: generator or stream
"""
self._name_check()
if isinstance(iterable, basestring):
# use write to buffer the string and avoid sending it 1 byte at a time
self.write(iterable)
if hasattr(iterable, "read"):
def file_iterator(file):
chunk = file.read(4095)
while chunk:
yield chunk
chunk = file.read(4095)
raise StopIteration()
iterable = file_iterator(iterable)
# This method implicitly disables verification.
if not self._etag_override:
self._etag = None
if not self.content_type:
self.content_type = "application/octet-stream"
path = "/%s/%s/%s" % (
self.container.conn.uri.rstrip("/"),
unicode_quote(self.container.name),
unicode_quote(self.name),
)
headers = self._make_headers()
if self.size is None:
del headers["Content-Length"]
headers["Transfer-Encoding"] = "chunked"
headers["X-Auth-Token"] = self.container.conn.token
headers["User-Agent"] = self.container.conn.user_agent
http = self.container.conn.connection
http.putrequest("PUT", path)
for key, value in headers.iteritems():
http.putheader(key, value)
http.endheaders()
response = None
transferred = 0
try:
for chunk in iterable:
if self.size is None:
http.send("%X\r\n" % len(chunk))
http.send(chunk)
http.send("\r\n")
else:
http.send(chunk)
transferred += len(chunk)
if self.size is None:
http.send("0\r\n\r\n")
# If the generator didn't yield enough data, stop, drop, and roll.
elif transferred < self.size:
raise IncompleteSend()
response = http.getresponse()
buff = response.read()
except timeout, err:
if response:
# pylint: disable-msg=E1101
response.read()
raise err
def send(self, iterable):
"""
Write potentially transient data to the remote storage system using a
generator or stream.
If the object's size is not set, chunked transfer encoding will be
used to upload the file.
If the object's size attribute is set, it will be used as the
Content-Length. If the generator raises StopIteration prior to
yielding the right number of bytes, an IncompleteSend exception is
raised.
If the content_type attribute is not set then a value of
application/octet-stream will be used.
Server-side verification will be performed if an md5 checksum is
assigned to the etag property before calling this method,
otherwise no verification will be performed, (verification
can be performed afterward though by using the etag attribute
which is set to the value returned by the server).
>>> test_object = container.create_object('backup.tar.gz')
>>> pfd = os.popen('tar -czvf - ./data/', 'r')
>>> test_object.send(pfd)
@param iterable: stream or generator which yields the content to upload
@type iterable: generator or stream
"""
self._name_check()
if isinstance(iterable, basestring):
# use write to buffer the string and avoid sending it 1 byte at a time
self.write(iterable)
if hasattr(iterable, 'read'):
def file_iterator(file):
chunk = file.read(4095)
while chunk:
yield chunk
chunk = file.read(4095)
raise StopIteration()
iterable = file_iterator(iterable)
# This method implicitly disables verification.
if not self._etag_override:
self._etag = None
if not self.content_type:
self.content_type = 'application/octet-stream'
path = "/%s/%s/%s" % (self.container.conn.uri.rstrip('/'), \
unicode_quote(self.container.name), unicode_quote(self.name))
headers = self._make_headers()
if self.size is None:
del headers['Content-Length']
headers['Transfer-Encoding'] = 'chunked'
headers['X-Auth-Token'] = self.container.conn.token
headers['User-Agent'] = self.container.conn.user_agent
http = self.container.conn.connection
http.putrequest('PUT', path)
for key, value in headers.iteritems():
http.putheader(key, value)
http.endheaders()
response = None
transferred = 0
try:
for chunk in iterable:
if self.size is None:
http.send("%X\r\n" % len(chunk))
http.send(chunk)
http.send("\r\n")
else:
http.send(chunk)
transferred += len(chunk)
if self.size is None:
http.send("0\r\n\r\n")
# If the generator didn't yield enough data, stop, drop, and roll.
elif transferred < self.size:
raise IncompleteSend()
response = http.getresponse()
buff = response.read()
except timeout, err:
if response:
# pylint: disable-msg=E1101
response.read()
raise err
def send(self, base_iterable, retry=True):
"""
Write potentially transient data to the remote storage system using a
generator or stream.
If the object's size is not set, chunked transfer encoding will be
used to upload the file.
If the object's size attribute is set, it will be used as the
Content-Length. If the generator raises StopIteration prior to
yielding the right number of bytes, an IncompleteSend exception is
raised.
If the content_type attribute is not set then a value of
application/octet-stream will be used.
Server-side verification will be performed if an md5 checksum is
assigned to the etag property before calling this method,
otherwise no verification will be performed, (verification
can be performed afterward though by using the etag attribute
which is set to the value returned by the server).
>>> test_object = container.create_object('backup.tar.gz')
>>> pfd = os.popen('tar -czvf - ./data/', 'r')
>>> test_object.send(pfd)
@param base_iterable: stream or generator which yields the content to upload
@type iterable: generator or stream
@param retry: control if there should be a request retry on fails
@type retry: boolean
"""
self._name_check()
# to retry a request, we must get the iterable copied
iterable, iterable_backup = itertools.tee(base_iterable)
if isinstance(iterable, basestring):
# use write to buffer the string and avoid sending it 1 byte at a time
self.write(iterable)
if hasattr(iterable, 'read'):
def file_iterator(file):
chunk = file.read(4095)
while chunk:
yield chunk
chunk = file.read(4095)
raise StopIteration()
iterable = file_iterator(iterable)
# This method implicitly disables verification.
if not self._etag_override:
self._etag = None
if not self.content_type:
self.content_type = 'application/octet-stream'
path = "/%s/%s/%s" % (self.container.conn.uri.rstrip('/'), \
unicode_quote(self.container.name), unicode_quote(self.name))
headers = self._make_headers()
if self.size is None:
del headers['Content-Length']
headers['Transfer-Encoding'] = 'chunked'
headers['X-Auth-Token'] = self.container.conn.token
headers['User-Agent'] = self.container.conn.user_agent
http = self.container.conn.connection
http.putrequest('PUT', path)
for key, value in headers.iteritems():
http.putheader(key, value)
http.endheaders()
response = None
transferred = 0
try:
for chunk in iterable:
try:
if self.size is None:
http.send("%X\r\n" % len(chunk))
http.send(chunk)
http.send("\r\n")
else:
http.send(chunk)
except (socket.error, IOError), e:
# the http connection was lost and must be restarted. As
# the send can't be rewinded, retry the send once using the
# iterable backup.
if retry:
return self.send(iterable_backup, retry=False)
else:
raise e
transferred += len(chunk)
if self.size is None:
http.send("0\r\n\r\n")
# If the generator didn't yield enough data, stop, drop, and roll.
elif transferred < self.size:
raise IncompleteSend()
try:
response = http.getresponse()
except ssl.SSLError, e:
# the send reached the timeout limit, this could be raised by a
# too large file or by a temporary network oscilation so,
# retry the send once using the iterable backup. In this case
# the connection must be reseted before retry
if retry:
self.container.conn.http_connect()
return self.send(iterable_backup, retry=False)
else:
raise e
