def generate_pre_signed_url(self,
bucket_name,
key,
timestamp=0,
expiration_in_seconds=1800,
headers=None,
params=None,
headers_to_sign=None,
protocol=None,
config=None):
"""
Get an authorization url with expire time
:type timestamp: int
:param timestamp: None
:type expiration_in_seconds: int
:param expiration_in_seconds: None
:type options: dict
:param options: None
:return:
**URL string**
"""
config = self._merge_config(config)
headers = headers or {}
params = params or {}
# specified protocol > protocol in endpoint > default protocol
endpoint_protocol, endpoint_host, endpoint_port = \
utils.parse_host_port(config.endpoint, config.protocol)
protocol = protocol or endpoint_protocol
full_host = endpoint_host
if endpoint_port != config.protocol.default_port:
full_host += ':' + str(endpoint_port)
headers[http_headers.HOST] = full_host
path = self._get_path(config, bucket_name, key)
params[http_headers.AUTHORIZATION.lower()] = bce_v1_signer.sign(
config.credentials,
http_methods.GET,
path,
headers,
params,
timestamp,
expiration_in_seconds,
headers_to_sign)
return "%s://%s%s?%s" % (protocol.name,
full_host,
path,
utils.get_canonical_querystring(params, False))
def generate_pre_signed_url(self,
bucket_name,
key,
timestamp=0,
expiration_in_seconds=1800,
headers=None,
params=None,
headers_to_sign=None,
protocol=None,
config=None):
"""
Get an authorization url with expire time
:type timestamp: int
:param timestamp: None
:type expiration_in_seconds: int
:param expiration_in_seconds: None
:type options: dict
:param options: None
:return:
**URL string**
"""
config = self._merge_config(config)
headers = headers or {}
params = params or {}
# specified protocol > protocol in endpoint > default protocol
endpoint_protocol, endpoint_host, endpoint_port = \
utils.parse_host_port(config.endpoint, config.protocol)
protocol = protocol or endpoint_protocol
full_host = endpoint_host
if endpoint_port != config.protocol.default_port:
full_host += ':' + str(endpoint_port)
headers[http_headers.HOST] = full_host
path = self._get_path(config, bucket_name, key)
params[http_headers.AUTHORIZATION.lower()] = bce_v1_signer.sign(
config.credentials,
http_methods.GET,
path,
headers,
params,
timestamp,
expiration_in_seconds,
headers_to_sign)
return "%s://%s%s?%s" % (protocol.name,
full_host,
path,
utils.get_canonical_querystring(params, False))
def sign(credentials,
http_method,
path,
headers,
params,
timestamp=0,
expiration_in_seconds=1800,
headers_to_sign=None):
"""
Create the authorization
"""
_logger.debug('Sign params: %s %s %s %s %d %d %s' %
(http_method, path, headers, params, timestamp,
expiration_in_seconds, headers_to_sign))
headers = headers or {}
params = params or {}
sign_key_info = 'bce-auth-v1/%s/%s/%d' % (
credentials.access_key_id, utils.get_canonical_time(timestamp),
expiration_in_seconds)
sign_key = hmac.new(
credentials.secret_access_key.encode(baidubce.DEFAULT_ENCODING),
sign_key_info.encode(baidubce.DEFAULT_ENCODING),
hashlib.sha256).hexdigest()
canonical_uri = path
canonical_querystring = utils.get_canonical_querystring(params, True)
canonical_headers = _get_canonical_headers(headers, headers_to_sign)
string_to_sign = '\n'.join(
[http_method, canonical_uri, canonical_querystring, canonical_headers])
sign_result = hmac.new(sign_key.encode(baidubce.DEFAULT_ENCODING),
string_to_sign.encode(baidubce.DEFAULT_ENCODING),
hashlib.sha256).hexdigest()
if headers_to_sign:
result = '%s/%s/%s' % (sign_key_info, ';'.join(headers_to_sign),
sign_result)
else:
result = '%s//%s' % (sign_key_info, sign_result)
_logger.debug('sign_key=[%s] sign_string=[%d bytes][ %s ]' %
(sign_key, len(string_to_sign), string_to_sign))
_logger.debug('result=%s' % result)
return result
def generate_pre_signed_url(self,
bucket_name,
key,
timestamp=0,
expiration_in_seconds=1800,
headers=None,
params=None,
headers_to_sign=None,
protocol=None,
config=None):
"""
Get an authorization url with expire time
:type timestamp: int
:param timestamp: None
:type expiration_in_seconds: int
:param expiration_in_seconds: None
:type options: dict
:param options: None
:return:
**URL string**
"""
config = self._merge_config(config)
headers = headers or {}
params = params or {}
protocol = protocol or bce_client_configuration.DEFAULT_PROTOCOL.name
headers[http_headers.HOST] = config.endpoint
path = self._get_path(config, bucket_name, key)
params[http_headers.AUTHORIZATION.lower()] = bce_v1_signer.sign(
config.credentials,
http_methods.GET,
self._get_path(config, bucket_name, key),
headers,
params,
timestamp,
expiration_in_seconds,
headers_to_sign)
return "%s://%s%s?%s" % (protocol,
config.endpoint,
path,
utils.get_canonical_querystring(params, False))
def sign(credentials, http_method, path, headers, params,
timestamp=0, expiration_in_seconds=1800, headers_to_sign=None):
"""
Create the authorization
"""
_logger.debug('Sign params: %s %s %s %s %d %d %s' % (
http_method, path, headers, params, timestamp, expiration_in_seconds, headers_to_sign))
headers = headers or {}
params = params or {}
sign_key_info = 'bce-auth-v1/%s/%s/%d' % (
credentials.access_key_id,
utils.get_canonical_time(timestamp),
expiration_in_seconds)
sign_key = hmac.new(
credentials.secret_access_key,
sign_key_info,
hashlib.sha256).hexdigest()
canonical_uri = path
canonical_querystring = utils.get_canonical_querystring(params, True)
canonical_headers = _get_canonical_headers(headers, headers_to_sign)
string_to_sign = '\n'.join(
[http_method, canonical_uri, canonical_querystring, canonical_headers])
sign_result = hmac.new(sign_key, string_to_sign, hashlib.sha256).hexdigest()
if headers_to_sign:
result = '%s/%s/%s' % (sign_key_info, ';'.join(headers_to_sign), sign_result)
else:
result = '%s//%s' % (sign_key_info, sign_result)
_logger.debug('sign_key=[%s] sign_string=[%d bytes][ %s ]' %
(sign_key, len(string_to_sign), string_to_sign))
_logger.debug('result=%s' % result)
return result
def send_request(config, sign_function, response_handler_functions,
http_method, path, body, headers, params):
"""
Send request to BCE services.
:param config
:type config: baidubce.BceClientConfiguration
:param sign_function:
:param response_handler_functions:
:type response_handler_functions: list
:param request:
:type request: baidubce.internal.InternalRequest
:return:
:rtype: baidubce.BceResponse
"""
_logger.debug(b'%s request start: %s %s, %s, %s', http_method, path,
headers, params, body)
headers = headers or {}
user_agent = 'bce-sdk-python/%s/%s/%s' % (compat.convert_to_string(
baidubce.SDK_VERSION), sys.version, sys.platform)
user_agent = user_agent.replace('\n', '')
user_agent = compat.convert_to_bytes(user_agent)
headers[http_headers.USER_AGENT] = user_agent
should_get_new_date = False
if http_headers.BCE_DATE not in headers:
should_get_new_date = True
headers[http_headers.HOST] = config.endpoint
if isinstance(body, str):
body = body.encode(baidubce.DEFAULT_ENCODING)
if not body:
headers[http_headers.CONTENT_LENGTH] = 0
elif isinstance(body, bytes):
headers[http_headers.CONTENT_LENGTH] = len(body)
elif http_headers.CONTENT_LENGTH not in headers:
raise ValueError(b'No %s is specified.' % http_headers.CONTENT_LENGTH)
# store the offset of fp body
offset = None
if hasattr(body, "tell") and hasattr(body, "seek"):
offset = body.tell()
protocol, host, port = utils.parse_host_port(config.endpoint,
config.protocol)
headers[http_headers.HOST] = host
if port != config.protocol.default_port:
headers[http_headers.HOST] += b':' + compat.convert_to_bytes(port)
headers[http_headers.AUTHORIZATION] = sign_function(
config.credentials, http_method, path, headers, params)
encoded_params = utils.get_canonical_querystring(params, False)
if len(encoded_params) > 0:
uri = path + b'?' + encoded_params
else:
uri = path
check_headers(headers)
retries_attempted = 0
errors = []
while True:
conn = None
try:
# restore the offset of fp body when retrying
if should_get_new_date is True:
headers[http_headers.BCE_DATE] = utils.get_canonical_time()
headers[http_headers.AUTHORIZATION] = sign_function(
config.credentials, http_method, path, headers, params)
if retries_attempted > 0 and offset is not None:
body.seek(offset)
conn = _get_connection(protocol, host, port,
config.connection_timeout_in_mills)
_logger.debug(
'request args:method=%s, uri=%s, headers=%s,patams=%s, body=%s',
http_method, uri, headers, params, body)
http_response = _send_http_request(conn, http_method, uri, headers,
body, config.send_buf_size)
headers_list = http_response.getheaders()
# on py3 ,values of headers_list is decoded with ios-8859-1 from
# utf-8 binary bytes
# headers_list[*][0] is lowercase on py2
# headers_list[*][0] is raw value py3
if compat.PY3 and isinstance(headers_list, list):
temp_heads = []
for k, v in headers_list:
k = k.encode('latin-1').decode('utf-8')
v = v.encode('latin-1').decode('utf-8')
k = k.lower()
temp_heads.append((k, v))
headers_list = temp_heads
_logger.debug('request return: status=%d, headers=%s' %
(http_response.status, headers_list))
response = BceResponse()
response.set_metadata_from_headers(dict(headers_list))
for handler_function in response_handler_functions:
if handler_function(http_response, response):
break
return response
except Exception as e:
if conn is not None:
conn.close()
# insert ">>>>" before all trace back lines and then save it
errors.append('\n'.join(
'>>>>' + line for line in traceback.format_exc().splitlines()))
if config.retry_policy.should_retry(e, retries_attempted):
delay_in_millis = config.retry_policy.get_delay_before_next_retry_in_millis(
e, retries_attempted)
time.sleep(delay_in_millis / 1000.0)
else:
raise BceHttpClientError(
'Unable to execute HTTP request. Retried %d times. '
'All trace backs:\n%s' %
(retries_attempted, '\n'.join(errors)), e)
retries_attempted += 1
def send_request(
self,
config,
sign_function,
response_handler_functions,
http_method, path, body, headers, params, special=False):
"""
Send request to BCE services.
:param config
:type config: baidubce.BceClientConfiguration
:param sign_function:
:param response_handler_functions:
:type response_handler_functions: list
:param request:
:type request: baidubce.internal.InternalRequest
:return:
:rtype: baidubce.BceResponse
"""
t = int(time.time())
_logger.debug('%s request start: %s %s, %s, %s, %d',
http_method, path, headers, params, body, t)
headers = headers or {}
user_agent = 'bce-sdk-python/%s/%s/%s' % (
baidubce.SDK_VERSION, sys.version, sys.platform)
user_agent = user_agent.replace('\n', '')
headers[http_headers.USER_AGENT] = user_agent
should_get_new_date = False
headers[http_headers.HOST] = config.endpoint
if isinstance(body, str):
body = body.encode(baidubce.DEFAULT_ENCODING)
if not body:
headers[http_headers.CONTENT_LENGTH] = 0
elif isinstance(body, str):
headers[http_headers.CONTENT_LENGTH] = len(body)
elif http_headers.CONTENT_LENGTH not in headers:
raise ValueError('No %s is specified.' % http_headers.CONTENT_LENGTH)
offset = None
if hasattr(body, "tell") and hasattr(body, "seek"):
offset = body.tell()
protocol, host, port = utils.parse_host_port(config.endpoint, config.protocol)
path = quote(path)
headers[http_headers.HOST] = host
if port != config.protocol.default_port:
headers[http_headers.HOST] += ':' + str(port)
headers[http_headers.AUTHORIZATION] = sign_function(
config.credentials, http_method, path, headers, params,
headers_to_sign=["host", "content-type"])
encoded_params = utils.get_canonical_querystring(params, False)
if len(encoded_params) > 0:
uri = path + '?' + encoded_params
else:
uri = path
bce_http_client.check_headers(headers)
retries_attempted = 0
errors = []
while True:
conn = None
try:
if should_get_new_date is True:
headers[http_headers.BCE_DATE] = utils.get_canonical_time()
headers[http_headers.AUTHORIZATION] = sign_function(
config.credentials, http_method, path, headers, params,
headers_to_sign=["host", "content-type"])
if retries_attempted > 0 and offset is not None:
body.seek(offset)
conn = bce_http_client._get_connection(protocol, host,
port, config.connection_timeout_in_mills)
http_response = bce_http_client._send_http_request(
conn, http_method, uri, headers, body, config.send_buf_size)
headers_list = http_response.getheaders()
_logger.debug(
'request return: status=%d, headers=%s' % (http_response.status, headers_list))
if special:
return http_response
response = bce_http_client.BceResponse()
response.set_metadata_from_headers(dict(headers_list))
for handler_function in response_handler_functions:
if handler_function(http_response, response):
break
return response
except Exception as e:
if conn is not None:
conn.close()
# insert ">>>>" before all trace back lines and then save it
errors.append('\n'.join('>>>>' + line for line in traceback.format_exc().
splitlines()))
if config.retry_policy.should_retry(e, retries_attempted):
delay_in_millis = config.retry_policy.get_delay_before_next_retry_in_millis(
e, retries_attempted)
time.sleep(delay_in_millis / 1000.0)
else:
raise bce_http_client.BceHttpClientError('Unable to execute HTTP request. '
'Retried %d times. All trace backs:\n'
'%s' % (retries_attempted,
'\n'.join(errors)), e)
def send_request(
config,
sign_function,
response_handler_functions,
http_method, path, body, headers, params):
"""
Send request to BCE services.
:param config
:type config: baidubce.BceClientConfiguration
:param sign_function:
:param response_handler_functions:
:type response_handler_functions: list
:param request:
:type request: baidubce.internal.InternalRequest
:return:
:rtype: baidubce.BceResponse
"""
_logger.debug('%s request start: %s %s, %s, %s',
http_method, path, headers, params, body)
headers = headers or {}
user_agent = 'bce-sdk-python/%s/%s/%s' % (
baidubce.SDK_VERSION, sys.version, sys.platform)
user_agent = user_agent.replace('\n', '')
headers[http_headers.USER_AGENT] = user_agent
should_get_new_date = False
if http_headers.BCE_DATE not in headers:
should_get_new_date = True
headers[http_headers.HOST] = config.endpoint
if isinstance(body, unicode):
body = body.encode(baidubce.DEFAULT_ENCODING)
if not body:
headers[http_headers.CONTENT_LENGTH] = 0
elif isinstance(body, str):
headers[http_headers.CONTENT_LENGTH] = len(body)
elif http_headers.CONTENT_LENGTH not in headers:
raise ValueError('No %s is specified.' % http_headers.CONTENT_LENGTH)
# store the offset of fp body
offset = None
if hasattr(body, "tell") and hasattr(body, "seek"):
offset = body.tell()
protocol, host, port = utils.parse_host_port(config.endpoint, config.protocol)
headers[http_headers.HOST] = host
if port != config.protocol.default_port:
headers[http_headers.HOST] += ':' + str(port)
headers[http_headers.AUTHORIZATION] = sign_function(
config.credentials, http_method, path, headers, params)
encoded_params = utils.get_canonical_querystring(params, False)
if len(encoded_params) > 0:
uri = path + '?' + encoded_params
else:
uri = path
check_headers(headers)
retries_attempted = 0
errors = []
while True:
conn = None
try:
# restore the offset of fp body when retrying
if should_get_new_date is True:
headers[http_headers.BCE_DATE] = utils.get_canonical_time()
headers[http_headers.AUTHORIZATION] = sign_function(
config.credentials, http_method, path, headers, params)
if retries_attempted > 0 and offset is not None:
body.seek(offset)
conn = _get_connection(protocol, host, port, config.connection_timeout_in_mills)
http_response = _send_http_request(
conn, http_method, uri, headers, body, config.send_buf_size)
headers_list = http_response.getheaders()
_logger.debug(
'request return: status=%d, headers=%s' % (http_response.status, headers_list))
response = BceResponse()
response.set_metadata_from_headers(dict(headers_list))
for handler_function in response_handler_functions:
if handler_function(http_response, response):
break
return response
except Exception as e:
if conn is not None:
conn.close()
# insert ">>>>" before all trace back lines and then save it
errors.append('\n'.join('>>>>' + line for line in traceback.format_exc().splitlines()))
if config.retry_policy.should_retry(e, retries_attempted):
delay_in_millis = config.retry_policy.get_delay_before_next_retry_in_millis(
e, retries_attempted)
time.sleep(delay_in_millis / 1000.0)
else:
raise BceHttpClientError('Unable to execute HTTP request. Retried %d times. '
'All trace backs:\n%s' % (retries_attempted,
'\n'.join(errors)), e)
retries_attempted += 1
