def generate_pre_signed_url(self, bucket_name, key, timestamp=0, expiration_in_seconds=1800, headers=None, params=None, headers_to_sign=None, protocol=None, config=None): """ Get an authorization url with expire time :type timestamp: int :param timestamp: None :type expiration_in_seconds: int :param expiration_in_seconds: None :type options: dict :param options: None :return: **URL string** """ config = self._merge_config(config) headers = headers or {} params = params or {} # specified protocol > protocol in endpoint > default protocol endpoint_protocol, endpoint_host, endpoint_port = \ utils.parse_host_port(config.endpoint, config.protocol) protocol = protocol or endpoint_protocol full_host = endpoint_host if endpoint_port != config.protocol.default_port: full_host += ':' + str(endpoint_port) headers[http_headers.HOST] = full_host path = self._get_path(config, bucket_name, key) params[http_headers.AUTHORIZATION.lower()] = bce_v1_signer.sign( config.credentials, http_methods.GET, path, headers, params, timestamp, expiration_in_seconds, headers_to_sign) return "%s://%s%s?%s" % (protocol.name, full_host, path, utils.get_canonical_querystring(params, False))
def sign(credentials, http_method, path, headers, params, timestamp=0, expiration_in_seconds=1800, headers_to_sign=None): """ Create the authorization """ _logger.debug('Sign params: %s %s %s %s %d %d %s' % (http_method, path, headers, params, timestamp, expiration_in_seconds, headers_to_sign)) headers = headers or {} params = params or {} sign_key_info = 'bce-auth-v1/%s/%s/%d' % ( credentials.access_key_id, utils.get_canonical_time(timestamp), expiration_in_seconds) sign_key = hmac.new( credentials.secret_access_key.encode(baidubce.DEFAULT_ENCODING), sign_key_info.encode(baidubce.DEFAULT_ENCODING), hashlib.sha256).hexdigest() canonical_uri = path canonical_querystring = utils.get_canonical_querystring(params, True) canonical_headers = _get_canonical_headers(headers, headers_to_sign) string_to_sign = '\n'.join( [http_method, canonical_uri, canonical_querystring, canonical_headers]) sign_result = hmac.new(sign_key.encode(baidubce.DEFAULT_ENCODING), string_to_sign.encode(baidubce.DEFAULT_ENCODING), hashlib.sha256).hexdigest() if headers_to_sign: result = '%s/%s/%s' % (sign_key_info, ';'.join(headers_to_sign), sign_result) else: result = '%s//%s' % (sign_key_info, sign_result) _logger.debug('sign_key=[%s] sign_string=[%d bytes][ %s ]' % (sign_key, len(string_to_sign), string_to_sign)) _logger.debug('result=%s' % result) return result
def generate_pre_signed_url(self, bucket_name, key, timestamp=0, expiration_in_seconds=1800, headers=None, params=None, headers_to_sign=None, protocol=None, config=None): """ Get an authorization url with expire time :type timestamp: int :param timestamp: None :type expiration_in_seconds: int :param expiration_in_seconds: None :type options: dict :param options: None :return: **URL string** """ config = self._merge_config(config) headers = headers or {} params = params or {} protocol = protocol or bce_client_configuration.DEFAULT_PROTOCOL.name headers[http_headers.HOST] = config.endpoint path = self._get_path(config, bucket_name, key) params[http_headers.AUTHORIZATION.lower()] = bce_v1_signer.sign( config.credentials, http_methods.GET, self._get_path(config, bucket_name, key), headers, params, timestamp, expiration_in_seconds, headers_to_sign) return "%s://%s%s?%s" % (protocol, config.endpoint, path, utils.get_canonical_querystring(params, False))
def sign(credentials, http_method, path, headers, params, timestamp=0, expiration_in_seconds=1800, headers_to_sign=None): """ Create the authorization """ _logger.debug('Sign params: %s %s %s %s %d %d %s' % ( http_method, path, headers, params, timestamp, expiration_in_seconds, headers_to_sign)) headers = headers or {} params = params or {} sign_key_info = 'bce-auth-v1/%s/%s/%d' % ( credentials.access_key_id, utils.get_canonical_time(timestamp), expiration_in_seconds) sign_key = hmac.new( credentials.secret_access_key, sign_key_info, hashlib.sha256).hexdigest() canonical_uri = path canonical_querystring = utils.get_canonical_querystring(params, True) canonical_headers = _get_canonical_headers(headers, headers_to_sign) string_to_sign = '\n'.join( [http_method, canonical_uri, canonical_querystring, canonical_headers]) sign_result = hmac.new(sign_key, string_to_sign, hashlib.sha256).hexdigest() if headers_to_sign: result = '%s/%s/%s' % (sign_key_info, ';'.join(headers_to_sign), sign_result) else: result = '%s//%s' % (sign_key_info, sign_result) _logger.debug('sign_key=[%s] sign_string=[%d bytes][ %s ]' % (sign_key, len(string_to_sign), string_to_sign)) _logger.debug('result=%s' % result) return result
def send_request(config, sign_function, response_handler_functions, http_method, path, body, headers, params): """ Send request to BCE services. :param config :type config: baidubce.BceClientConfiguration :param sign_function: :param response_handler_functions: :type response_handler_functions: list :param request: :type request: baidubce.internal.InternalRequest :return: :rtype: baidubce.BceResponse """ _logger.debug(b'%s request start: %s %s, %s, %s', http_method, path, headers, params, body) headers = headers or {} user_agent = 'bce-sdk-python/%s/%s/%s' % (compat.convert_to_string( baidubce.SDK_VERSION), sys.version, sys.platform) user_agent = user_agent.replace('\n', '') user_agent = compat.convert_to_bytes(user_agent) headers[http_headers.USER_AGENT] = user_agent should_get_new_date = False if http_headers.BCE_DATE not in headers: should_get_new_date = True headers[http_headers.HOST] = config.endpoint if isinstance(body, str): body = body.encode(baidubce.DEFAULT_ENCODING) if not body: headers[http_headers.CONTENT_LENGTH] = 0 elif isinstance(body, bytes): headers[http_headers.CONTENT_LENGTH] = len(body) elif http_headers.CONTENT_LENGTH not in headers: raise ValueError(b'No %s is specified.' % http_headers.CONTENT_LENGTH) # store the offset of fp body offset = None if hasattr(body, "tell") and hasattr(body, "seek"): offset = body.tell() protocol, host, port = utils.parse_host_port(config.endpoint, config.protocol) headers[http_headers.HOST] = host if port != config.protocol.default_port: headers[http_headers.HOST] += b':' + compat.convert_to_bytes(port) headers[http_headers.AUTHORIZATION] = sign_function( config.credentials, http_method, path, headers, params) encoded_params = utils.get_canonical_querystring(params, False) if len(encoded_params) > 0: uri = path + b'?' + encoded_params else: uri = path check_headers(headers) retries_attempted = 0 errors = [] while True: conn = None try: # restore the offset of fp body when retrying if should_get_new_date is True: headers[http_headers.BCE_DATE] = utils.get_canonical_time() headers[http_headers.AUTHORIZATION] = sign_function( config.credentials, http_method, path, headers, params) if retries_attempted > 0 and offset is not None: body.seek(offset) conn = _get_connection(protocol, host, port, config.connection_timeout_in_mills) _logger.debug( 'request args:method=%s, uri=%s, headers=%s,patams=%s, body=%s', http_method, uri, headers, params, body) http_response = _send_http_request(conn, http_method, uri, headers, body, config.send_buf_size) headers_list = http_response.getheaders() # on py3 ,values of headers_list is decoded with ios-8859-1 from # utf-8 binary bytes # headers_list[*][0] is lowercase on py2 # headers_list[*][0] is raw value py3 if compat.PY3 and isinstance(headers_list, list): temp_heads = [] for k, v in headers_list: k = k.encode('latin-1').decode('utf-8') v = v.encode('latin-1').decode('utf-8') k = k.lower() temp_heads.append((k, v)) headers_list = temp_heads _logger.debug('request return: status=%d, headers=%s' % (http_response.status, headers_list)) response = BceResponse() response.set_metadata_from_headers(dict(headers_list)) for handler_function in response_handler_functions: if handler_function(http_response, response): break return response except Exception as e: if conn is not None: conn.close() # insert ">>>>" before all trace back lines and then save it errors.append('\n'.join( '>>>>' + line for line in traceback.format_exc().splitlines())) if config.retry_policy.should_retry(e, retries_attempted): delay_in_millis = config.retry_policy.get_delay_before_next_retry_in_millis( e, retries_attempted) time.sleep(delay_in_millis / 1000.0) else: raise BceHttpClientError( 'Unable to execute HTTP request. Retried %d times. ' 'All trace backs:\n%s' % (retries_attempted, '\n'.join(errors)), e) retries_attempted += 1
def send_request( self, config, sign_function, response_handler_functions, http_method, path, body, headers, params, special=False): """ Send request to BCE services. :param config :type config: baidubce.BceClientConfiguration :param sign_function: :param response_handler_functions: :type response_handler_functions: list :param request: :type request: baidubce.internal.InternalRequest :return: :rtype: baidubce.BceResponse """ t = int(time.time()) _logger.debug('%s request start: %s %s, %s, %s, %d', http_method, path, headers, params, body, t) headers = headers or {} user_agent = 'bce-sdk-python/%s/%s/%s' % ( baidubce.SDK_VERSION, sys.version, sys.platform) user_agent = user_agent.replace('\n', '') headers[http_headers.USER_AGENT] = user_agent should_get_new_date = False headers[http_headers.HOST] = config.endpoint if isinstance(body, str): body = body.encode(baidubce.DEFAULT_ENCODING) if not body: headers[http_headers.CONTENT_LENGTH] = 0 elif isinstance(body, str): headers[http_headers.CONTENT_LENGTH] = len(body) elif http_headers.CONTENT_LENGTH not in headers: raise ValueError('No %s is specified.' % http_headers.CONTENT_LENGTH) offset = None if hasattr(body, "tell") and hasattr(body, "seek"): offset = body.tell() protocol, host, port = utils.parse_host_port(config.endpoint, config.protocol) path = quote(path) headers[http_headers.HOST] = host if port != config.protocol.default_port: headers[http_headers.HOST] += ':' + str(port) headers[http_headers.AUTHORIZATION] = sign_function( config.credentials, http_method, path, headers, params, headers_to_sign=["host", "content-type"]) encoded_params = utils.get_canonical_querystring(params, False) if len(encoded_params) > 0: uri = path + '?' + encoded_params else: uri = path bce_http_client.check_headers(headers) retries_attempted = 0 errors = [] while True: conn = None try: if should_get_new_date is True: headers[http_headers.BCE_DATE] = utils.get_canonical_time() headers[http_headers.AUTHORIZATION] = sign_function( config.credentials, http_method, path, headers, params, headers_to_sign=["host", "content-type"]) if retries_attempted > 0 and offset is not None: body.seek(offset) conn = bce_http_client._get_connection(protocol, host, port, config.connection_timeout_in_mills) http_response = bce_http_client._send_http_request( conn, http_method, uri, headers, body, config.send_buf_size) headers_list = http_response.getheaders() _logger.debug( 'request return: status=%d, headers=%s' % (http_response.status, headers_list)) if special: return http_response response = bce_http_client.BceResponse() response.set_metadata_from_headers(dict(headers_list)) for handler_function in response_handler_functions: if handler_function(http_response, response): break return response except Exception as e: if conn is not None: conn.close() # insert ">>>>" before all trace back lines and then save it errors.append('\n'.join('>>>>' + line for line in traceback.format_exc(). splitlines())) if config.retry_policy.should_retry(e, retries_attempted): delay_in_millis = config.retry_policy.get_delay_before_next_retry_in_millis( e, retries_attempted) time.sleep(delay_in_millis / 1000.0) else: raise bce_http_client.BceHttpClientError('Unable to execute HTTP request. ' 'Retried %d times. All trace backs:\n' '%s' % (retries_attempted, '\n'.join(errors)), e)
def send_request( config, sign_function, response_handler_functions, http_method, path, body, headers, params): """ Send request to BCE services. :param config :type config: baidubce.BceClientConfiguration :param sign_function: :param response_handler_functions: :type response_handler_functions: list :param request: :type request: baidubce.internal.InternalRequest :return: :rtype: baidubce.BceResponse """ _logger.debug('%s request start: %s %s, %s, %s', http_method, path, headers, params, body) headers = headers or {} user_agent = 'bce-sdk-python/%s/%s/%s' % ( baidubce.SDK_VERSION, sys.version, sys.platform) user_agent = user_agent.replace('\n', '') headers[http_headers.USER_AGENT] = user_agent should_get_new_date = False if http_headers.BCE_DATE not in headers: should_get_new_date = True headers[http_headers.HOST] = config.endpoint if isinstance(body, unicode): body = body.encode(baidubce.DEFAULT_ENCODING) if not body: headers[http_headers.CONTENT_LENGTH] = 0 elif isinstance(body, str): headers[http_headers.CONTENT_LENGTH] = len(body) elif http_headers.CONTENT_LENGTH not in headers: raise ValueError('No %s is specified.' % http_headers.CONTENT_LENGTH) # store the offset of fp body offset = None if hasattr(body, "tell") and hasattr(body, "seek"): offset = body.tell() protocol, host, port = utils.parse_host_port(config.endpoint, config.protocol) headers[http_headers.HOST] = host if port != config.protocol.default_port: headers[http_headers.HOST] += ':' + str(port) headers[http_headers.AUTHORIZATION] = sign_function( config.credentials, http_method, path, headers, params) encoded_params = utils.get_canonical_querystring(params, False) if len(encoded_params) > 0: uri = path + '?' + encoded_params else: uri = path check_headers(headers) retries_attempted = 0 errors = [] while True: conn = None try: # restore the offset of fp body when retrying if should_get_new_date is True: headers[http_headers.BCE_DATE] = utils.get_canonical_time() headers[http_headers.AUTHORIZATION] = sign_function( config.credentials, http_method, path, headers, params) if retries_attempted > 0 and offset is not None: body.seek(offset) conn = _get_connection(protocol, host, port, config.connection_timeout_in_mills) http_response = _send_http_request( conn, http_method, uri, headers, body, config.send_buf_size) headers_list = http_response.getheaders() _logger.debug( 'request return: status=%d, headers=%s' % (http_response.status, headers_list)) response = BceResponse() response.set_metadata_from_headers(dict(headers_list)) for handler_function in response_handler_functions: if handler_function(http_response, response): break return response except Exception as e: if conn is not None: conn.close() # insert ">>>>" before all trace back lines and then save it errors.append('\n'.join('>>>>' + line for line in traceback.format_exc().splitlines())) if config.retry_policy.should_retry(e, retries_attempted): delay_in_millis = config.retry_policy.get_delay_before_next_retry_in_millis( e, retries_attempted) time.sleep(delay_in_millis / 1000.0) else: raise BceHttpClientError('Unable to execute HTTP request. Retried %d times. ' 'All trace backs:\n%s' % (retries_attempted, '\n'.join(errors)), e) retries_attempted += 1