def save(self, directory_path): """ Save the material (the unencrypted PEM encoded RSA private key) of a newly created KeyPair to a local file. :type directory_path: string :param directory_path: The fully qualified path to the directory in which the keypair will be saved. The keypair file will be named using the name of the keypair as the base name and .pem for the file extension. If a file of that name already exists in the directory, an exception will be raised and the old file will not be overwritten. :rtype: bool :return: True if successful. """ if self.material: directory_path = os.path.expanduser(directory_path) file_path = os.path.join(directory_path, '%s.pem' % self.name) if os.path.exists(file_path): raise BotoClientError( '%s already exists, it will not be overwritten' % file_path) fp = open(file_path, 'wb') fp.write(self.material) fp.close() os.chmod(file_path, 0o600) return True else: raise BotoClientError('KeyPair contains no material')
def create(self, bucket_name, key, data, headers=None, acl=None): """ Creates a file on Amazon S3. :param bucket_name: Name of bucket to use :param key: Key to use :param data: File contents :param headers: File headers :param acl: File permissions. :type bucket_name: string :type key: string :type data: anything :type headers: dict :type acl: string. Any string of: ('private', 'public-read', 'public-read-write', 'authenticated-read', 'bucket-owner-read', 'bucket-owner-full-control', 'log-delivery-write') :raises: BotoClientError """ # Returns bucket connection bucket = self._connect_and_get_bucket(bucket_name) # Create a Key instance for this bucket k = Key(bucket) k.key = key # Create the file in S3 try: k.set_contents_from_string( data, headers=headers, ) if acl: k.set_acl(acl) except: return BotoClientError("Error uploading file to Amazon S3")
def copy_to_region(self, region, name=None): """ Create a copy of this security group in another region. Note that the new security group will be a separate entity and will not stay in sync automatically after the copy operation. :type region: :class:`boto.ec2.regioninfo.RegionInfo` :param region: The region to which this security group will be copied. :type name: string :param name: The name of the copy. If not supplied, the copy will have the same name as this security group. :rtype: :class:`boto.ec2.securitygroup.SecurityGroup` :return: The new security group. """ if region.name == self.region: raise BotoClientError('Unable to copy to the same Region') conn_params = self.connection.get_params() rconn = region.connect(**conn_params) sg = rconn.create_security_group(name or self.name, self.description) source_groups = [] for rule in self.rules: for grant in rule.grants: if grant.name: if grant.name not in source_groups: source_groups.append(grant.name) sg.authorize(None, None, None, None, grant) else: sg.authorize(rule.ip_protocol, rule.from_port, rule.to_port, grant.cidr_ip) return sg
def open_write(self): """ Open this key for writing. Not yet implemented """ raise BotoClientError('Not Implemented')
def check_lowercase_bucketname(n): if not (n + 'a').islower(): raise BotoClientError( "Bucket names cannot contain upper-case " "characters when using either the sub-domain or virtual " "hosting calling format.") return True
def change_storage_class(self, new_storage_class, dst_bucket=None): """ Change the storage class of an existing key. Depending on whether a different destination bucket is supplied or not, this will either move the item within the bucket, preserving all metadata and ACL info bucket changing the storage class or it will copy the item to the provided destination bucket, also preserving metadata and ACL info. :type new_storage_class: string :param new_storage_class: The new storage class for the Key. Possible values are: * STANDARD * REDUCED_REDUNDANCY :type dst_bucket: string :param dst_bucket: The name of a destination bucket. If not provided the current bucket of the key will be used. """ if new_storage_class == 'STANDARD': return self.copy(self.bucket.name, self.name, reduced_redundancy=False, preserve_acl=True) elif new_storage_class == 'REDUCED_REDUNDANCY': return self.copy(self.bucket.name, self.name, reduced_redundancy=True, preserve_acl=True) else: raise BotoClientError('Invalid storage class: %s' % new_storage_class)
def open_write(self, headers=None): """ Open this key for writing. Not yet implemented @type headers: dict @param headers: Headers to pass in the write request """ raise BotoClientError('Not Implemented')
def open(self, mode='r', headers=None, query_args=None): if mode == 'r': self.mode = 'r' self.open_read(headers=headers, query_args=query_args) elif mode == 'w': self.mode = 'w' self.open_write(headers=headers) else: raise BotoClientError('Invalid mode: %s' % mode)
def test_delete_cert_pdf_raise_BotoClientError(self): botoexception = BotoClientError(reason="reason") with patch('pdfgen.views.CertificateHonor.delete', spec=True, side_effect=botoexception) as moc1: contents = delete_cert_pdf(self.username, self.course_id, self.key) self.assertEqual(contents, json.dumps({"error": "BotoClientError: reason"})) moc1.assert_called_once_with()
def get_signature(self, params, verb, path): if self.SignatureVersion == '0': t = self.calc_signature_0(params) elif self.SignatureVersion == '1': t = self.calc_signature_1(params) elif self.SignatureVersion == '2': t = self.calc_signature_2(params, verb, path) else: raise BotoClientError('Unknown Signature Version: %s' % self.SignatureVersion) return t
def next(self): """ By providing a next method, the key object supports use as an iterator. For example, you can now say: for bytes in key: write bytes to a file or whatever All of the HTTP connection stuff is handled for you. """ raise BotoClientError('Not Implemented')
def get_regions(service_name, region_cls=None, connection_cls=None, provider=None): """ Given a service name (like ``ec2``), returns a list of ``RegionInfo`` objects for that service. This leverages the ``endpoints.json`` file (+ optional user overrides) to configure/construct all the objects. :param service_name: The name of the service to construct the ``RegionInfo`` objects for. Ex: ``ec2``, ``s3``, ``sns``, etc. :type service_name: string :param region_cls: (Optional) The class to use when constructing. By default, this is ``RegionInfo``. :type region_cls: class :param connection_cls: (Optional) The connection class for the ``RegionInfo`` object. Providing this allows the ``connect`` method on the ``RegionInfo`` to work. Default is ``None`` (no connection). :type connection_cls: class :param provider: (Optional) The provider object for the ``RegionInfo`` object. Providing this allows the ``connect`` method on the ``RegionInfo`` to use this provider instead of the default. Default is ``None`` (use default provider 'aws'). :type provider: string or Provider :returns: A list of configured ``RegionInfo`` objects :rtype: list """ endpoints = load_regions() if service_name not in endpoints: raise BotoClientError( "Service '%s' not found in endpoints." % service_name ) if region_cls is None: region_cls = RegionInfo region_objs = [] for region_name, endpoint in endpoints.get(service_name, {}).items(): region_objs.append( region_cls( name=region_name, endpoint=endpoint, connection_cls=connection_cls, provider=provider ) ) return region_objs
def open(self, mode='r', headers=None, query_args=None, override_num_retries=None, callback=None): if mode == 'r': self.mode = 'r' self.open_read(headers=headers, query_args=query_args, override_num_retries=override_num_retries, callback=callback) elif mode == 'w': self.mode = 'w' self.open_write(headers=headers, override_num_retries=override_num_retries, callback=callback) else: raise BotoClientError('Invalid mode: %s' % mode)
def open_write(self, headers=None, override_num_retries=None): """ Open this key for writing. Not yet implemented :type headers: dict :param headers: Headers to pass in the write request :type override_num_retries: int :param override_num_retries: If not None will override configured num_retries parameter for underlying PUT. """ raise BotoClientError('Not Implemented')
def delete(self, bucket_name, key): """ Deletes a file from Amazon S3. :param bucket_name: Name of the bucket :param key: Key to delete :type bucket_name: string :type key: string :raises: BotoClientError """ bucket = self._connect_and_get_bucket(bucket_name) key = bucket.get_key(key) if not key: raise BotoClientError('Invalid Amazon S3 Key') try: key.delete() except: raise BotoClientError("Could not delete Amazon S3 file")
def create_bucket(self, bucket_name, headers=None, location='', policy=None): """ Creates a new located bucket. By default it's in the USA. You can pass Location.EU to create an European bucket. :type bucket_name: string :param bucket_name: The name of the new bucket :type headers: dict :param headers: Additional headers to pass along with the request to AWS. :type location: :class:`boto.s3.connection.Location` :param location: The location of the new bucket :type policy: :class:`boto.s3.acl.CannedACLStrings` :param policy: A canned ACL policy that will be applied to the new key in S3. """ if not bucket_name.islower(): raise BotoClientError("Bucket names must be lower case.") if policy: if headers: headers['x-amz-acl'] = policy else: headers = {'x-amz-acl': policy} if location == '': data = '' else: data = '<CreateBucketConstraint><LocationConstraint>' + \ location + '</LocationConstraint></CreateBucketConstraint>' response = self.make_request('PUT', bucket_name, headers=headers, data=data) body = response.read() if response.status == 409: raise S3CreateError(response.status, response.reason, body) if response.status == 200: return Bucket(self, bucket_name) else: raise self.provider.storage_response_error(response.status, response.reason, body)
def compose(self, components, content_type=None, headers=None): """Create a new object from a sequence of existing objects. The content of the object representing this Key will be the concatenation of the given object sequence. For more detail, visit https://developers.google.com/storage/docs/composite-objects :type components list of Keys :param components List of gs.Keys representing the component objects :type content_type (optional) string :param content_type Content type for the new composite object. """ compose_req = [] for key in components: if key.bucket.name != self.bucket.name: raise BotoClientError( 'GCS does not support inter-bucket composing') generation_tag = '' if key.generation: generation_tag = ('<Generation>%s</Generation>' % str(key.generation)) compose_req.append('<Component><Name>%s</Name>%s</Component>' % (key.name, generation_tag)) compose_req_xml = ('<ComposeRequest>%s</ComposeRequest>' % ''.join(compose_req)) headers = headers or {} if content_type: headers['Content-Type'] = content_type resp = self.bucket.connection.make_request( 'PUT', get_utf8_value(self.bucket.name), get_utf8_value(self.name), headers=headers, query_args='compose', data=get_utf8_value(compose_req_xml)) if resp.status < 200 or resp.status > 299: raise self.bucket.connection.provider.storage_response_error( resp.status, resp.reason, resp.read()) # Return the generation so that the result URI can be built with this # for automatic parallel uploads. return resp.getheader('x-goog-generation')
def copy_to_region_vpc(self, region=None, vpc=None, name=None, dry_run=False): if region.name == self.region: raise BotoClientError('Unable to copy to the same Region') conn_params = self.connection.get_params() rconn = region.connect(**conn_params) conn = region.connect(**conn_params) sg = rconn.create_security_group(name or self.name, self.description, vpc, dry_run=dry_run) source_groups = [] for rule in self.rules: for grant in rule.grants: grant_nom = grant.name or grant.group_id if grant_nom: if grant_nom not in source_groups: source_groups.append(grant_nom) sg.authorize(None, None, None, None, grant, dry_run=dry_run) else: sg.authorize(rule.ip_protocol, rule.from_port, rule.to_port, grant.cidr_ip, dry_run=dry_run) for rule in self.rules_egress: for grant in rule.grants: try: conn.authorize_security_group_egress(sg.id, rule.ip_protocol, rule.from_port, rule.to_port, src_group_id=None, cidr_ip=grant.cidr_ip) except EC2ResponseError as e: if not e.error_code == "InvalidPermission.Duplicate": print(str(e.message)) return sg
def copy_to_region(self, region, dry_run=False): """ Create a new key pair of the same new in another region. Note that the new key pair will use a different ssh cert than the this key pair. After doing the copy, you will need to save the material associated with the new key pair (use the save method) to a local file. :type region: :class:`boto.ec2.regioninfo.RegionInfo` :param region: The region to which this security group will be copied. :rtype: :class:`boto.ec2.keypair.KeyPair` :return: The new key pair """ if region.name == self.region: raise BotoClientError('Unable to copy to the same Region') conn_params = self.connection.get_params() rconn = region.connect(**conn_params) kp = rconn.create_key_pair(self.name, dry_run=dry_run) return kp
def _connect_and_get_bucket(self, bucket_name): """ Connects to Amazon S3, and returns a Bucket instance. :param bucket_name: Name of bucket :type bucket_name: string :return: Bucket :rtype: Bucket :raises: BotoClienterror """ try: conn = S3Connection( self.AWS_access, self.AWS_secret, ) bucket = conn.get_bucket(bucket_name) except: raise BotoClientError('Error connecting to Amazon S3') return bucket
def check_lowercase_bucketname(n): """ Bucket names must not contain uppercase characters. We check for this by appending a lowercase character and testing with islower(). Note this also covers cases like numeric bucket names with dashes. >>> check_lowercase_bucketname("Aaaa") Traceback (most recent call last): ... BotoClientError: S3Error: Bucket names cannot contain upper-case characters when using either the sub-domain or virtual hosting calling format. >>> check_lowercase_bucketname("1234-5678-9123") True >>> check_lowercase_bucketname("abcdefg1234") True """ if not (n + 'a').islower(): raise BotoClientError("Bucket names cannot contain upper-case " \ "characters when using either the sub-domain or virtual " \ "hosting calling format.") return True
def set_contents_from_file(self, fp, headers=None, replace=True, cb=None, num_cb=10, policy=None, md5=None, res_upload_handler=None, size=None, rewind=False): """ Store an object in GS using the name of the Key object as the key in GS and the contents of the file pointed to by 'fp' as the contents. :type fp: file :param fp: the file whose contents are to be uploaded :type headers: dict :param headers: additional HTTP headers to be sent with the PUT request. :type replace: bool :param replace: If this parameter is False, the method will first check to see if an object exists in the bucket with the same key. If it does, it won't overwrite it. The default value is True which will overwrite the object. :type cb: function :param cb: a callback function that will be called to report progress on the upload. The callback should accept two integer parameters, the first representing the number of bytes that have been successfully transmitted to GS and the second representing the total number of bytes that need to be transmitted. :type num_cb: int :param num_cb: (optional) If a callback is specified with the cb parameter, this parameter determines the granularity of the callback by defining the maximum number of times the callback will be called during the file transfer. :type policy: :class:`boto.gs.acl.CannedACLStrings` :param policy: A canned ACL policy that will be applied to the new key in GS. :type md5: A tuple containing the hexdigest version of the MD5 checksum of the file as the first element and the Base64-encoded version of the plain checksum as the second element. This is the same format returned by the compute_md5 method. :param md5: If you need to compute the MD5 for any reason prior to upload, it's silly to have to do it twice so this param, if present, will be used as the MD5 values of the file. Otherwise, the checksum will be computed. :type res_upload_handler: ResumableUploadHandler :param res_upload_handler: If provided, this handler will perform the upload. :type size: int :param size: (optional) The Maximum number of bytes to read from the file pointer (fp). This is useful when uploading a file in multiple parts where you are splitting the file up into different ranges to be uploaded. If not specified, the default behaviour is to read all bytes from the file pointer. Less bytes may be available. Notes: 1. The "size" parameter currently cannot be used when a resumable upload handler is given but is still useful for uploading part of a file as implemented by the parent class. 2. At present Google Cloud Storage does not support multipart uploads. :type rewind: bool :param rewind: (optional) If True, the file pointer (fp) will be rewound to the start before any bytes are read from it. The default behaviour is False which reads from the current position of the file pointer (fp). :rtype: int :return: The number of bytes written to the key. TODO: At some point we should refactor the Bucket and Key classes, to move functionality common to all providers into a parent class, and provider-specific functionality into subclasses (rather than just overriding/sharing code the way it currently works). """ provider = self.bucket.connection.provider if res_upload_handler and size: # could use size instead of file_length if provided but... raise BotoClientError('"size" param not supported for resumable uploads.') headers = headers or {} if policy: headers[provider.acl_header] = policy if rewind: # caller requests reading from beginning of fp. fp.seek(0, os.SEEK_SET) else: spos = fp.tell() fp.seek(0, os.SEEK_END) if fp.tell() == spos: fp.seek(0, os.SEEK_SET) if fp.tell() != spos: # Raise an exception as this is likely a programming error # whereby there is data before the fp but nothing after it. fp.seek(spos) raise AttributeError( 'fp is at EOF. Use rewind option or seek() to data start.') # seek back to the correct position. fp.seek(spos) if hasattr(fp, 'name'): self.path = fp.name if self.bucket != None: if not md5: # compute_md5() and also set self.size to actual # size of the bytes read computing the md5. md5 = self.compute_md5(fp, size) # adjust size if required size = self.size elif size: self.size = size else: # If md5 is provided, still need to size so # calculate based on bytes to end of content spos = fp.tell() fp.seek(0, os.SEEK_END) self.size = fp.tell() - spos fp.seek(spos) size = self.size self.md5 = md5[0] self.base64md5 = md5[1] if self.name == None: self.name = self.md5 if not replace: if self.bucket.lookup(self.name): return if res_upload_handler: res_upload_handler.send_file(self, fp, headers, cb, num_cb) else: # Not a resumable transfer so use basic send_file mechanism. self.send_file(fp, headers, cb, num_cb, size=size)
def _mexe(self, request, sender=None, override_num_retries=None, retry_handler=None): """ mexe - Multi-execute inside a loop, retrying multiple times to handle transient Internet errors by simply trying again. Also handles redirects. This code was inspired by the S3Utils classes posted to the boto-users Google group by Larry Bates. Thanks! """ boto.log.debug('Method: %s' % request.method) boto.log.debug('Path: %s' % request.path) boto.log.debug('Data: %s' % request.body) boto.log.debug('Headers: %s' % request.headers) boto.log.debug('Host: %s' % request.host) boto.log.debug('Port: %s' % request.port) boto.log.debug('Params: %s' % request.params) response = None body = None e = None if override_num_retries is None: num_retries = config.getint('Boto', 'num_retries', self.num_retries) else: num_retries = override_num_retries i = 0 connection = self.get_http_connection(request.host, request.port, self.is_secure) # Convert body to bytes if needed if not isinstance(request.body, bytes) and hasattr(request.body, 'encode'): request.body = request.body.encode('utf-8') while i <= num_retries: # Use binary exponential backoff to desynchronize client requests. next_sleep = min(random.random() * (2 ** i), boto.config.get('Boto', 'max_retry_delay', 60)) try: # we now re-sign each request before it is retried boto.log.debug('Token: %s' % self.provider.security_token) request.authorize(connection=self) # Only force header for non-s3 connections, because s3 uses # an older signing method + bucket resource URLs that include # the port info. All others should be now be up to date and # not include the port. if 's3' not in self._required_auth_capability(): if not getattr(self, 'anon', False): self.set_host_header(request) boto.log.debug('Final headers: %s' % request.headers) request.start_time = datetime.now() if callable(sender): response = sender(connection, request.method, request.path, request.body, request.headers) else: connection.request(request.method, request.path, request.body, request.headers) response = connection.getresponse() boto.log.debug('Response headers: %s' % response.getheaders()) location = response.getheader('location') # -- gross hack -- # http_client gets confused with chunked responses to HEAD requests # so I have to fake it out if request.method == 'HEAD' and getattr(response, 'chunked', False): response.chunked = 0 if callable(retry_handler): status = retry_handler(response, i, next_sleep) if status: msg, i, next_sleep = status if msg: boto.log.debug(msg) time.sleep(next_sleep) continue if response.status in [500, 502, 503, 504]: msg = 'Received %d response. ' % response.status msg += 'Retrying in %3.1f seconds' % next_sleep boto.log.debug(msg) body = response.read() if isinstance(body, bytes): body = body.decode('utf-8') elif response.status < 300 or response.status >= 400 or \ not location: # don't return connection to the pool if response contains # Connection:close header, because the connection has been # closed and default reconnect behavior may do something # different than new_http_connection. Also, it's probably # less efficient to try to reuse a closed connection. conn_header_value = response.getheader('connection') if conn_header_value == 'close': connection.close() else: self.put_http_connection(request.host, request.port, self.is_secure, connection) if self.request_hook is not None: self.request_hook.handle_request_data(request, response) return response else: scheme, request.host, request.path, \ params, query, fragment = urlparse(location) if query: request.path += '?' + query # urlparse can return both host and port in netloc, so if # that's the case we need to split them up properly if ':' in request.host: request.host, request.port = request.host.split(':', 1) msg = 'Redirecting: %s' % scheme + '://' msg += request.host + request.path boto.log.debug(msg) connection = self.get_http_connection(request.host, request.port, scheme == 'https') response = None continue except PleaseRetryException as e: boto.log.debug('encountered a retry exception: %s' % e) connection = self.new_http_connection(request.host, request.port, self.is_secure) response = e.response except self.http_exceptions as e: for unretryable in self.http_unretryable_exceptions: if isinstance(e, unretryable): boto.log.debug( 'encountered unretryable %s exception, re-raising' % e.__class__.__name__) raise boto.log.debug('encountered %s exception, reconnecting' % \ e.__class__.__name__) connection = self.new_http_connection(request.host, request.port, self.is_secure) time.sleep(next_sleep) i += 1 # If we made it here, it's because we have exhausted our retries # and stil haven't succeeded. So, if we have a response object, # use it to raise an exception. # Otherwise, raise the exception that must have already happened. if self.request_hook is not None: self.request_hook.handle_request_data(request, response, error=True) if response: raise BotoServerError(response.status, response.reason, body) elif e: raise else: msg = 'Please report this exception as a Boto Issue!' raise BotoClientError(msg)
def wrapper(*args, **kwargs): if len(args) == 3 and not (args[2].islower() or args[2].isalnum()): raise BotoClientError("Bucket names cannot contain upper-case " \ "characters when using either the sub-domain or virtual " \ "hosting calling format.") return f(*args, **kwargs)
def __init__(self): """Uncallable constructor on abstract base StorageUri class. """ raise BotoClientError('Attempt to instantiate abstract StorageUri ' 'class')
def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, path='/', provider='aws'): """ :type host: str :param host: The host to make the connection to :keyword str aws_access_key_id: Your AWS Access Key ID (provided by Amazon). If none is specified, the value in your ``AWS_ACCESS_KEY_ID`` environmental variable is used. :keyword str aws_secret_access_key: Your AWS Secret Access Key (provided by Amazon). If none is specified, the value in your ``AWS_SECRET_ACCESS_KEY`` environmental variable is used. :type is_secure: boolean :param is_secure: Whether the connection is over SSL :type https_connection_factory: list or tuple :param https_connection_factory: A pair of an HTTP connection factory and the exceptions to catch. The factory should have a similar interface to L{httplib.HTTPSConnection}. :param str proxy: Address/hostname for a proxy server :type proxy_port: int :param proxy_port: The port to use when connecting over a proxy :type proxy_user: str :param proxy_user: The username to connect with on the proxy :type proxy_pass: str :param proxy_pass: The password to use when connection over a proxy. :type port: int :param port: The port to use to connect """ self.num_retries = 5 # Override passed-in is_secure setting if value was defined in config. if config.has_option('Boto', 'is_secure'): is_secure = config.getboolean('Boto', 'is_secure') self.is_secure = is_secure # Whether or not to validate server certificates. At some point in the # future, the default should be flipped to true. self.https_validate_certificates = config.getbool( 'Boto', 'https_validate_certificates', False) if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION: raise BotoClientError( "SSL server certificate validation is enabled in boto " "configuration, but Python dependencies required to " "support this feature are not available. Certificate " "validation is only supported when running under Python " "2.6 or later.") self.ca_certificates_file = config.get_value('Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE) self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) # define exceptions from httplib that we want to catch and retry self.http_exceptions = (httplib.HTTPException, socket.error, socket.gaierror) # define subclasses of the above that are not retryable. self.http_unretryable_exceptions = [] if HAVE_HTTPS_CONNECTION: self.http_unretryable_exceptions.append(ssl.SSLError) self.http_unretryable_exceptions.append( https_connection.InvalidCertificateException) # define values in socket exceptions we don't want to catch self.socket_exception_values = (errno.EINTR, ) if https_connection_factory is not None: self.https_connection_factory = https_connection_factory[0] self.http_exceptions += https_connection_factory[1] else: self.https_connection_factory = None if (is_secure): self.protocol = 'https' else: self.protocol = 'http' self.host = host self.path = path if debug: self.debug = debug else: self.debug = config.getint('Boto', 'debug', debug) if port: self.port = port else: self.port = PORTS_BY_SECURITY[is_secure] # Timeout used to tell httplib how long to wait for socket timeouts. # Default is to leave timeout unchanged, which will in turn result in # the socket's default global timeout being used. To specify a # timeout, set http_socket_timeout in Boto config. Regardless, # timeouts will only be applied if Python is 2.6 or greater. self.http_connection_kwargs = {} if (sys.version_info[0], sys.version_info[1]) >= (2, 6): if config.has_option('Boto', 'http_socket_timeout'): timeout = config.getint('Boto', 'http_socket_timeout') self.http_connection_kwargs['timeout'] = timeout self.provider = Provider(provider, aws_access_key_id, aws_secret_access_key) # allow config file to override default host if self.provider.host: self.host = self.provider.host # cache up to 20 connections per host, up to 20 hosts self._pool = ConnectionPool(20, 20) self._connection = (self.server_name(), self.is_secure) self._last_rs = None self._auth_handler = auth.get_auth_handler( host, config, self.provider, self._required_auth_capability())
class AWSAuthConnection(object): def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, path='/', provider='aws'): """ :type host: str :param host: The host to make the connection to :keyword str aws_access_key_id: Your AWS Access Key ID (provided by Amazon). If none is specified, the value in your ``AWS_ACCESS_KEY_ID`` environmental variable is used. :keyword str aws_secret_access_key: Your AWS Secret Access Key (provided by Amazon). If none is specified, the value in your ``AWS_SECRET_ACCESS_KEY`` environmental variable is used. :type is_secure: boolean :param is_secure: Whether the connection is over SSL :type https_connection_factory: list or tuple :param https_connection_factory: A pair of an HTTP connection factory and the exceptions to catch. The factory should have a similar interface to L{httplib.HTTPSConnection}. :param str proxy: Address/hostname for a proxy server :type proxy_port: int :param proxy_port: The port to use when connecting over a proxy :type proxy_user: str :param proxy_user: The username to connect with on the proxy :type proxy_pass: str :param proxy_pass: The password to use when connection over a proxy. :type port: int :param port: The port to use to connect """ self.num_retries = 5 # Override passed-in is_secure setting if value was defined in config. if config.has_option('Boto', 'is_secure'): is_secure = config.getboolean('Boto', 'is_secure') self.is_secure = is_secure # Whether or not to validate server certificates. At some point in the # future, the default should be flipped to true. self.https_validate_certificates = config.getbool( 'Boto', 'https_validate_certificates', False) if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION: raise BotoClientError( "SSL server certificate validation is enabled in boto " "configuration, but Python dependencies required to " "support this feature are not available. Certificate " "validation is only supported when running under Python " "2.6 or later.") self.ca_certificates_file = config.get_value('Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE) self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) # define exceptions from httplib that we want to catch and retry self.http_exceptions = (httplib.HTTPException, socket.error, socket.gaierror) # define subclasses of the above that are not retryable. self.http_unretryable_exceptions = [] if HAVE_HTTPS_CONNECTION: self.http_unretryable_exceptions.append(ssl.SSLError) self.http_unretryable_exceptions.append( https_connection.InvalidCertificateException) # define values in socket exceptions we don't want to catch self.socket_exception_values = (errno.EINTR, ) if https_connection_factory is not None: self.https_connection_factory = https_connection_factory[0] self.http_exceptions += https_connection_factory[1] else: self.https_connection_factory = None if (is_secure): self.protocol = 'https' else: self.protocol = 'http' self.host = host self.path = path if debug: self.debug = debug else: self.debug = config.getint('Boto', 'debug', debug) if port: self.port = port else: self.port = PORTS_BY_SECURITY[is_secure] # Timeout used to tell httplib how long to wait for socket timeouts. # Default is to leave timeout unchanged, which will in turn result in # the socket's default global timeout being used. To specify a # timeout, set http_socket_timeout in Boto config. Regardless, # timeouts will only be applied if Python is 2.6 or greater. self.http_connection_kwargs = {} if (sys.version_info[0], sys.version_info[1]) >= (2, 6): if config.has_option('Boto', 'http_socket_timeout'): timeout = config.getint('Boto', 'http_socket_timeout') self.http_connection_kwargs['timeout'] = timeout self.provider = Provider(provider, aws_access_key_id, aws_secret_access_key) # allow config file to override default host if self.provider.host: self.host = self.provider.host # cache up to 20 connections per host, up to 20 hosts self._pool = ConnectionPool(20, 20) self._connection = (self.server_name(), self.is_secure) self._last_rs = None self._auth_handler = auth.get_auth_handler( host, config, self.provider, self._required_auth_capability()) def __repr__(self): return '%s:%s' % (self.__class__.__name__, self.host) def _required_auth_capability(self): return [] def _cached_name(self, host, is_secure): if host is None: host = self.server_name() cached_name = is_secure and 'https://' or 'http://' cached_name += host return cached_name def connection(self): return self.get_http_connection(*self._connection) connection = property(connection) def aws_access_key_id(self): return self.provider.access_key aws_access_key_id = property(aws_access_key_id) gs_access_key_id = aws_access_key_id access_key = aws_access_key_id def aws_secret_access_key(self): return self.provider.secret_key aws_secret_access_key = property(aws_secret_access_key) gs_secret_access_key = aws_secret_access_key secret_key = aws_secret_access_key def get_path(self, path='/'): pos = path.find('?') if pos >= 0: params = path[pos:] path = path[:pos] else: params = None if path[-1] == '/': need_trailing = True else: need_trailing = False path_elements = self.path.split('/') path_elements.extend(path.split('/')) path_elements = [p for p in path_elements if p] path = '/' + '/'.join(path_elements) if path[-1] != '/' and need_trailing: path += '/' if params: path = path + params return path def server_name(self, port=None): if not port: port = self.port if port == 80: signature_host = self.host else: # This unfortunate little hack can be attributed to # a difference in the 2.6 version of httplib. In old # versions, it would append ":443" to the hostname sent # in the Host header and so we needed to make sure we # did the same when calculating the V2 signature. In 2.6 # (and higher!) # it no longer does that. Hence, this kludge. if sys.version[:3] in ('2.6', '2.7') and port == 443: signature_host = self.host else: signature_host = '%s:%d' % (self.host, port) return signature_host def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass): self.proxy = proxy self.proxy_port = proxy_port self.proxy_user = proxy_user self.proxy_pass = proxy_pass if os.environ.has_key('http_proxy') and not self.proxy: pattern = re.compile( '(?:http://)?' \ '(?:(?P<user>\w+):(?P<pass>.*)@)?' \ '(?P<host>[\w\-\.]+)' \ '(?::(?P<port>\d+))?' ) match = pattern.match(os.environ['http_proxy']) if match: self.proxy = match.group('host') self.proxy_port = match.group('port') self.proxy_user = match.group('user') self.proxy_pass = match.group('pass') else: if not self.proxy: self.proxy = config.get_value('Boto', 'proxy', None) if not self.proxy_port: self.proxy_port = config.get_value('Boto', 'proxy_port', None) if not self.proxy_user: self.proxy_user = config.get_value('Boto', 'proxy_user', None) if not self.proxy_pass: self.proxy_pass = config.get_value('Boto', 'proxy_pass', None) if not self.proxy_port and self.proxy: print "http_proxy environment variable does not specify " \ "a port, using default" self.proxy_port = self.port self.use_proxy = (self.proxy != None) def get_http_connection(self, host, is_secure): queue = self._pool[self._cached_name(host, is_secure)] try: return queue.get_nowait() except Queue.Empty: return self.new_http_connection(host, is_secure) def new_http_connection(self, host, is_secure): if self.use_proxy: host = '%s:%d' % (self.proxy, int(self.proxy_port)) if host is None: host = self.server_name() if is_secure: boto.log.debug('establishing HTTPS connection: host=%s, kwargs=%s', host, self.http_connection_kwargs) if self.use_proxy: connection = self.proxy_ssl() elif self.https_connection_factory: connection = self.https_connection_factory(host) elif self.https_validate_certificates and HAVE_HTTPS_CONNECTION: connection = https_connection.CertValidatingHTTPSConnection( host, ca_certs=self.ca_certificates_file, **self.http_connection_kwargs) else: connection = httplib.HTTPSConnection( host, **self.http_connection_kwargs) else: boto.log.debug('establishing HTTP connection: kwargs=%s' % self.http_connection_kwargs) connection = httplib.HTTPConnection(host, **self.http_connection_kwargs) if self.debug > 1: connection.set_debuglevel(self.debug) # self.connection must be maintained for backwards-compatibility # however, it must be dynamically pulled from the connection pool # set a private variable which will enable that if host.split(':')[0] == self.host and is_secure == self.is_secure: self._connection = (host, is_secure) return connection def put_http_connection(self, host, is_secure, connection): try: self._pool[self._cached_name(host, is_secure)].put_nowait(connection) except Queue.Full: # gracefully fail in case of pool overflow connection.close() def proxy_ssl(self): host = '%s:%d' % (self.host, self.port) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: sock.connect((self.proxy, int(self.proxy_port))) except: raise boto.log.debug("Proxy connection: CONNECT %s HTTP/1.0\r\n", host) sock.sendall("CONNECT %s HTTP/1.0\r\n" % host) sock.sendall("User-Agent: %s\r\n" % UserAgent) if self.proxy_user and self.proxy_pass: for k, v in self.get_proxy_auth_header().items(): sock.sendall("%s: %s\r\n" % (k, v)) sock.sendall("\r\n") resp = httplib.HTTPResponse(sock, strict=True, debuglevel=self.debug) resp.begin() if resp.status != 200: # Fake a socket error, use a code that make it obvious it hasn't # been generated by the socket library raise socket.error( -71, "Error talking to HTTP proxy %s:%s: %s (%s)" % (self.proxy, self.proxy_port, resp.status, resp.reason)) # We can safely close the response, it duped the original socket resp.close() h = httplib.HTTPConnection(host) if self.https_validate_certificates and HAVE_HTTPS_CONNECTION: boto.log.debug( "wrapping ssl socket for proxied connection; " "CA certificate file=%s", self.ca_certificates_file) key_file = self.http_connection_kwargs.get('key_file', None) cert_file = self.http_connection_kwargs.get('cert_file', None) sslSock = ssl.wrap_socket(sock, keyfile=key_file, certfile=cert_file, cert_reqs=ssl.CERT_REQUIRED, ca_certs=self.ca_certificates_file) cert = sslSock.getpeercert() hostname = self.host.split(':', 0)[0] if not https_connection.ValidateCertificateHostname( cert, hostname): raise https_connection.InvalidCertificateException( hostname, cert, 'hostname mismatch') else: # Fallback for old Python without ssl.wrap_socket if hasattr(httplib, 'ssl'): sslSock = httplib.ssl.SSLSocket(sock) else: sslSock = socket.ssl(sock, None, None) sslSock = httplib.FakeSocket(sock, sslSock) # This is a bit unclean h.sock = sslSock return h def prefix_proxy_to_path(self, path, host=None): path = self.protocol + '://' + (host or self.server_name()) + path return path def get_proxy_auth_header(self): auth = base64.encodestring(self.proxy_user + ':' + self.proxy_pass) return {'Proxy-Authorization': 'Basic %s' % auth} def _mexe(self, method, path, data, headers, host=None, sender=None, override_num_retries=None): """ mexe - Multi-execute inside a loop, retrying multiple times to handle transient Internet errors by simply trying again. Also handles redirects. This code was inspired by the S3Utils classes posted to the boto-users Google group by Larry Bates. Thanks! """ boto.log.debug('Method: %s' % method) boto.log.debug('Path: %s' % path) boto.log.debug('Data: %s' % data) boto.log.debug('Headers: %s' % headers) boto.log.debug('Host: %s' % host) response = None body = None e = None if override_num_retries is None: num_retries = config.getint('Boto', 'num_retries', self.num_retries) else: num_retries = override_num_retries i = 0 connection = self.get_http_connection(host, self.is_secure) while i <= num_retries: try: if callable(sender): response = sender(connection, method, path, data, headers) else: connection.request(method, path, data, headers) response = connection.getresponse() location = response.getheader('location') # -- gross hack -- # httplib gets confused with chunked responses to HEAD requests # so I have to fake it out if method == 'HEAD' and getattr(response, 'chunked', False): response.chunked = 0 if response.status == 500 or response.status == 503: boto.log.debug( 'received %d response, retrying in %d seconds' % (response.status, 2**i)) body = response.read() elif response.status == 408: body = response.read() print '-------------------------' print ' 4 0 8 ' print 'path=%s' % path print body print '-------------------------' elif response.status < 300 or response.status >= 400 or \ not location: self.put_http_connection(host, self.is_secure, connection) return response else: scheme, host, path, params, query, fragment = \ urlparse.urlparse(location) if query: path += '?' + query boto.log.debug('Redirecting: %s' % scheme + '://' + host + path) connection = self.get_http_connection( host, scheme == 'https') continue except KeyboardInterrupt: sys.exit('Keyboard Interrupt') except self.http_exceptions, e: for unretryable in self.http_unretryable_exceptions: if isinstance(e, unretryable): boto.log.debug( 'encountered unretryable %s exception, re-raising' % e.__class__.__name__) raise e boto.log.debug('encountered %s exception, reconnecting' % \ e.__class__.__name__) connection = self.new_http_connection(host, self.is_secure) time.sleep(2**i) i += 1 # If we made it here, it's because we have exhausted our retries and stil haven't # succeeded. So, if we have a response object, use it to raise an exception. # Otherwise, raise the exception that must have already happened. if response: raise BotoServerError(response.status, response.reason, body) elif e: raise e else: raise BotoClientError( 'Please report this exception as a Boto Issue!')
def set_contents_from_file(self, fp, headers=None, replace=True, cb=None, num_cb=10, policy=None, md5=None, res_upload_handler=None, size=None, rewind=False, if_generation=None): """ Store an object in GS using the name of the Key object as the key in GS and the contents of the file pointed to by 'fp' as the contents. :type fp: file :param fp: the file whose contents are to be uploaded :type headers: dict :param headers: additional HTTP headers to be sent with the PUT request. :type replace: bool :param replace: If this parameter is False, the method will first check to see if an object exists in the bucket with the same key. If it does, it won't overwrite it. The default value is True which will overwrite the object. :type cb: function :param cb: a callback function that will be called to report progress on the upload. The callback should accept two integer parameters, the first representing the number of bytes that have been successfully transmitted to GS and the second representing the total number of bytes that need to be transmitted. :type num_cb: int :param num_cb: (optional) If a callback is specified with the cb parameter, this parameter determines the granularity of the callback by defining the maximum number of times the callback will be called during the file transfer. :type policy: :class:`boto.gs.acl.CannedACLStrings` :param policy: A canned ACL policy that will be applied to the new key in GS. :type md5: A tuple containing the hexdigest version of the MD5 checksum of the file as the first element and the Base64-encoded version of the plain checksum as the second element. This is the same format returned by the compute_md5 method. :param md5: If you need to compute the MD5 for any reason prior to upload, it's silly to have to do it twice so this param, if present, will be used as the MD5 values of the file. Otherwise, the checksum will be computed. :type res_upload_handler: ResumableUploadHandler :param res_upload_handler: If provided, this handler will perform the upload. :type size: int :param size: (optional) The Maximum number of bytes to read from the file pointer (fp). This is useful when uploading a file in multiple parts where you are splitting the file up into different ranges to be uploaded. If not specified, the default behaviour is to read all bytes from the file pointer. Less bytes may be available. Notes: 1. The "size" parameter currently cannot be used when a resumable upload handler is given but is still useful for uploading part of a file as implemented by the parent class. 2. At present Google Cloud Storage does not support multipart uploads. :type rewind: bool :param rewind: (optional) If True, the file pointer (fp) will be rewound to the start before any bytes are read from it. The default behaviour is False which reads from the current position of the file pointer (fp). :type if_generation: int :param if_generation: (optional) If set to a generation number, the object will only be written to if its current generation number is this value. If set to the value 0, the object will only be written if it doesn't already exist. :rtype: int :return: The number of bytes written to the key. TODO: At some point we should refactor the Bucket and Key classes, to move functionality common to all providers into a parent class, and provider-specific functionality into subclasses (rather than just overriding/sharing code the way it currently works). """ provider = self.bucket.connection.provider if res_upload_handler and size: # could use size instead of file_length if provided but... raise BotoClientError( '"size" param not supported for resumable uploads.') headers = headers or {} if policy: headers[provider.acl_header] = policy if rewind: # caller requests reading from beginning of fp. fp.seek(0, os.SEEK_SET) else: # The following seek/tell/seek logic is intended # to detect applications using the older interface to # set_contents_from_file(), which automatically rewound the # file each time the Key was reused. This changed with commit # 14ee2d03f4665fe20d19a85286f78d39d924237e, to support uploads # split into multiple parts and uploaded in parallel, and at # the time of that commit this check was added because otherwise # older programs would get a success status and upload an empty # object. Unfortuantely, it's very inefficient for fp's implemented # by KeyFile (used, for example, by gsutil when copying between # providers). So, we skip the check for the KeyFile case. # TODO: At some point consider removing this seek/tell/seek # logic, after enough time has passed that it's unlikely any # programs remain that assume the older auto-rewind interface. if not isinstance(fp, KeyFile): spos = fp.tell() fp.seek(0, os.SEEK_END) if fp.tell() == spos: fp.seek(0, os.SEEK_SET) if fp.tell() != spos: # Raise an exception as this is likely a programming # error whereby there is data before the fp but nothing # after it. fp.seek(spos) raise AttributeError('fp is at EOF. Use rewind option ' 'or seek() to data start.') # seek back to the correct position. fp.seek(spos) if hasattr(fp, 'name'): self.path = fp.name if self.bucket is not None: if isinstance(fp, KeyFile): # Avoid EOF seek for KeyFile case as it's very inefficient. key = fp.getkey() size = key.size - fp.tell() self.size = size # At present both GCS and S3 use MD5 for the etag for # non-multipart-uploaded objects. If the etag is 32 hex # chars use it as an MD5, to avoid having to read the file # twice while transferring. if (re.match('^"[a-fA-F0-9]{32}"$', key.etag)): etag = key.etag.strip('"') md5 = (etag, base64.b64encode(binascii.unhexlify(etag))) if size: self.size = size else: # If md5 is provided, still need to size so # calculate based on bytes to end of content spos = fp.tell() fp.seek(0, os.SEEK_END) self.size = fp.tell() - spos fp.seek(spos) size = self.size if md5 is None: md5 = self.compute_md5(fp, size) self.md5 = md5[0] self.base64md5 = md5[1] if self.name is None: self.name = self.md5 if not replace: if self.bucket.lookup(self.name): return if if_generation is not None: headers['x-goog-if-generation-match'] = str(if_generation) if res_upload_handler: res_upload_handler.send_file(self, fp, headers, cb, num_cb) else: # Not a resumable transfer so use basic send_file mechanism. self.send_file(fp, headers, cb, num_cb, size=size)
def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, path='/', provider='aws', security_token=None, suppress_consec_slashes=True, validate_certs=True, profile_name=None): """ :type host: str :param host: The host to make the connection to :keyword str aws_access_key_id: Your AWS Access Key ID (provided by Amazon). If none is specified, the value in your ``AWS_ACCESS_KEY_ID`` environmental variable is used. :keyword str aws_secret_access_key: Your AWS Secret Access Key (provided by Amazon). If none is specified, the value in your ``AWS_SECRET_ACCESS_KEY`` environmental variable is used. :keyword str security_token: The security token associated with temporary credentials issued by STS. Optional unless using temporary credentials. If none is specified, the environment variable ``AWS_SECURITY_TOKEN`` is used if defined. :type is_secure: boolean :param is_secure: Whether the connection is over SSL :type https_connection_factory: list or tuple :param https_connection_factory: A pair of an HTTP connection factory and the exceptions to catch. The factory should have a similar interface to L{http_client.HTTPSConnection}. :param str proxy: Address/hostname for a proxy server :type proxy_port: int :param proxy_port: The port to use when connecting over a proxy :type proxy_user: str :param proxy_user: The username to connect with on the proxy :type proxy_pass: str :param proxy_pass: The password to use when connection over a proxy. :type port: int :param port: The port to use to connect :type suppress_consec_slashes: bool :param suppress_consec_slashes: If provided, controls whether consecutive slashes will be suppressed in key paths. :type validate_certs: bool :param validate_certs: Controls whether SSL certificates will be validated or not. Defaults to True. :type profile_name: str :param profile_name: Override usual Credentials section in config file to use a named set of keys instead. """ self.suppress_consec_slashes = suppress_consec_slashes self.num_retries = 6 # Override passed-in is_secure setting if value was defined in config. if config.has_option('Boto', 'is_secure'): is_secure = config.getboolean('Boto', 'is_secure') self.is_secure = is_secure # Whether or not to validate server certificates. # The default is now to validate certificates. This can be # overridden in the boto config file are by passing an # explicit validate_certs parameter to the class constructor. self.https_validate_certificates = config.getbool( 'Boto', 'https_validate_certificates', validate_certs) if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION: raise BotoClientError( "SSL server certificate validation is enabled in boto " "configuration, but Python dependencies required to " "support this feature are not available. Certificate " "validation is only supported when running under Python " "2.6 or later.") certs_file = config.get_value( 'Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE) if certs_file == 'system': certs_file = None self.ca_certificates_file = certs_file if port: self.port = port else: self.port = PORTS_BY_SECURITY[is_secure] self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) # define exceptions from http_client that we want to catch and retry self.http_exceptions = (http_client.HTTPException, socket.error, socket.gaierror, http_client.BadStatusLine) # define subclasses of the above that are not retryable. self.http_unretryable_exceptions = [] if HAVE_HTTPS_CONNECTION: self.http_unretryable_exceptions.append( https_connection.InvalidCertificateException) # define values in socket exceptions we don't want to catch self.socket_exception_values = (errno.EINTR,) if https_connection_factory is not None: self.https_connection_factory = https_connection_factory[0] self.http_exceptions += https_connection_factory[1] else: self.https_connection_factory = None if (is_secure): self.protocol = 'https' else: self.protocol = 'http' self.host = host self.path = path # if the value passed in for debug if not isinstance(debug, six.integer_types): debug = 0 self.debug = config.getint('Boto', 'debug', debug) self.host_header = None # Timeout used to tell http_client how long to wait for socket timeouts. # Default is to leave timeout unchanged, which will in turn result in # the socket's default global timeout being used. To specify a # timeout, set http_socket_timeout in Boto config. Regardless, # timeouts will only be applied if Python is 2.6 or greater. self.http_connection_kwargs = {} if (sys.version_info[0], sys.version_info[1]) >= (2, 6): # If timeout isn't defined in boto config file, use 70 second # default as recommended by # http://docs.aws.amazon.com/amazonswf/latest/apireference/API_PollForActivityTask.html self.http_connection_kwargs['timeout'] = config.getint( 'Boto', 'http_socket_timeout', 70) if isinstance(provider, Provider): # Allow overriding Provider self.provider = provider else: self._provider_type = provider self.provider = Provider(self._provider_type, aws_access_key_id, aws_secret_access_key, security_token, profile_name) # Allow config file to override default host, port, and host header. if self.provider.host: self.host = self.provider.host if self.provider.port: self.port = self.provider.port if self.provider.host_header: self.host_header = self.provider.host_header self._pool = ConnectionPool() self._connection = (self.host, self.port, self.is_secure) self._last_rs = None self._auth_handler = auth.get_auth_handler( host, config, self.provider, self._required_auth_capability()) if getattr(self, 'AuthServiceName', None) is not None: self.auth_service_name = self.AuthServiceName self.request_hook = None
e.__class__.__name__) connection = self.new_http_connection(request.host, request.port, self.is_secure) time.sleep(next_sleep) i += 1 # If we made it here, it's because we have exhausted our retries # and stil haven't succeeded. So, if we have a response object, # use it to raise an exception. # Otherwise, raise the exception that must have already happened. if response: raise BotoServerError(response.status, response.reason, body) elif e: raise e else: msg = 'Please report this exception as a Boto Issue!' raise BotoClientError(msg) def build_base_http_request(self, method, path, auth_path, params=None, headers=None, data='', host=None): path = self.get_path(path) if auth_path is not None: auth_path = self.get_path(auth_path) if params == None: params = {} else: params = params.copy() if headers == None: headers = {} else: headers = headers.copy() if (self.host_header and