def is_walrus(s3_url): """ Return True if it's Walrus endpoint, not S3 We assume anything other than *.amazonaws.com is Walrus""" if s3_url is not None: o = urlparse.urlparse(s3_url) return not o.hostname.endswith('amazonaws.com') else: return False
def main(): argument_spec = ec2_argument_spec() argument_spec.update(dict( bucket = dict(required=True), dest = dict(default=None), encrypt = dict(default=True, type='bool'), expiry = dict(default=600, aliases=['expiration']), headers = dict(type='dict'), marker = dict(default=None), max_keys = dict(default=1000), metadata = dict(type='dict'), mode = dict(choices=['get', 'put', 'delete', 'create', 'geturl', 'getstr', 'delobj', 'list'], required=True), object = dict(), permission = dict(type='list', default=['private']), version = dict(default=None), overwrite = dict(aliases=['force'], default='always'), prefix = dict(default=None), retries = dict(aliases=['retry'], type='int', default=0), s3_url = dict(aliases=['S3_URL']), rgw = dict(default='no', type='bool'), src = dict(), ), ) module = AnsibleModule(argument_spec=argument_spec) if not HAS_BOTO: module.fail_json(msg='boto required for this module') bucket = module.params.get('bucket') encrypt = module.params.get('encrypt') expiry = int(module.params['expiry']) if module.params.get('dest'): dest = os.path.expanduser(module.params.get('dest')) headers = module.params.get('headers') marker = module.params.get('marker') max_keys = module.params.get('max_keys') metadata = module.params.get('metadata') mode = module.params.get('mode') obj = module.params.get('object') version = module.params.get('version') overwrite = module.params.get('overwrite') prefix = module.params.get('prefix') retries = module.params.get('retries') s3_url = module.params.get('s3_url') rgw = module.params.get('rgw') src = module.params.get('src') for acl in module.params.get('permission'): if acl not in CannedACLStrings: module.fail_json(msg='Unknown permission specified: %s' % str(acl)) if overwrite not in ['always', 'never', 'different']: if module.boolean(overwrite): overwrite = 'always' else: overwrite = 'never' region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module) if region in ('us-east-1', '', None): # S3ism for the US Standard region location = Location.DEFAULT else: # Boto uses symbolic names for locations but region strings will # actually work fine for everything except us-east-1 (US Standard) location = region if module.params.get('object'): obj = os.path.expanduser(module.params['object']) # allow eucarc environment variables to be used if ansible vars aren't set if not s3_url and 'S3_URL' in os.environ: s3_url = os.environ['S3_URL'] # rgw requires an explicit url if rgw and not s3_url: module.fail_json(msg='rgw flavour requires s3_url') # bucket names with .'s in them need to use the calling_format option, # otherwise the connection will fail. See https://github.com/boto/boto/issues/2836 # for more details. if '.' in bucket: aws_connect_kwargs['calling_format'] = OrdinaryCallingFormat() # Look at s3_url and tweak connection settings # if connecting to RGW, Walrus or fakes3 try: if s3_url and rgw: rgw = urlparse.urlparse(s3_url) s3 = boto.connect_s3( is_secure=rgw.scheme == 'https', host=rgw.hostname, port=rgw.port, calling_format=OrdinaryCallingFormat(), **aws_connect_kwargs ) elif is_fakes3(s3_url): fakes3 = urlparse.urlparse(s3_url) s3 = S3Connection( is_secure=fakes3.scheme == 'fakes3s', host=fakes3.hostname, port=fakes3.port, calling_format=OrdinaryCallingFormat(), **aws_connect_kwargs ) elif is_walrus(s3_url): walrus = urlparse.urlparse(s3_url).hostname s3 = boto.connect_walrus(walrus, **aws_connect_kwargs) else: aws_connect_kwargs['is_secure'] = True try: s3 = connect_to_aws(boto.s3, location, **aws_connect_kwargs) except AnsibleAWSError: # use this as fallback because connect_to_region seems to fail in boto + non 'classic' aws accounts in some cases s3 = boto.connect_s3(**aws_connect_kwargs) except boto.exception.NoAuthHandlerFound as e: module.fail_json(msg='No Authentication Handler found: %s ' % str(e)) except Exception as e: module.fail_json(msg='Failed to connect to S3: %s' % str(e)) if s3 is None: # this should never happen module.fail_json(msg ='Unknown error, failed to create s3 connection, no information from boto.') # If our mode is a GET operation (download), go through the procedure as appropriate ... if mode == 'get': # First, we check to see if the bucket exists, we get "bucket" returned. bucketrtn = bucket_check(module, s3, bucket) if bucketrtn is False: module.fail_json(msg="Source bucket cannot be found", failed=True) # Next, we check to see if the key in the bucket exists. If it exists, it also returns key_matches md5sum check. keyrtn = key_check(module, s3, bucket, obj, version=version) if keyrtn is False: if version is not None: module.fail_json(msg="Key %s with version id %s does not exist."% (obj, version), failed=True) else: module.fail_json(msg="Key %s does not exist."%obj, failed=True) # If the destination path doesn't exist or overwrite is True, no need to do the md5um etag check, so just download. pathrtn = path_check(dest) if pathrtn is False or overwrite == 'always': download_s3file(module, s3, bucket, obj, dest, retries, version=version) # Compare the remote MD5 sum of the object with the local dest md5sum, if it already exists. if pathrtn is True: md5_remote = keysum(module, s3, bucket, obj, version=version) md5_local = module.md5(dest) if md5_local == md5_remote: sum_matches = True if overwrite == 'always': download_s3file(module, s3, bucket, obj, dest, retries, version=version) else: module.exit_json(msg="Local and remote object are identical, ignoring. Use overwrite=always parameter to force.", changed=False) else: sum_matches = False if overwrite in ('always', 'different'): download_s3file(module, s3, bucket, obj, dest, retries, version=version) else: module.exit_json(msg="WARNING: Checksums do not match. Use overwrite parameter to force download.") # Firstly, if key_matches is TRUE and overwrite is not enabled, we EXIT with a helpful message. if sum_matches is True and overwrite == 'never': module.exit_json(msg="Local and remote object are identical, ignoring. Use overwrite parameter to force.", changed=False) # if our mode is a PUT operation (upload), go through the procedure as appropriate ... if mode == 'put': # Use this snippet to debug through conditionals: # module.exit_json(msg="Bucket return %s"%bucketrtn) # Lets check the src path. pathrtn = path_check(src) if pathrtn is False: module.fail_json(msg="Local object for PUT does not exist", failed=True) # Lets check to see if bucket exists to get ground truth. bucketrtn = bucket_check(module, s3, bucket) if bucketrtn is True: keyrtn = key_check(module, s3, bucket, obj) # Lets check key state. Does it exist and if it does, compute the etag md5sum. if bucketrtn is True and keyrtn is True: md5_remote = keysum(module, s3, bucket, obj) md5_local = module.md5(src) if md5_local == md5_remote: sum_matches = True if overwrite == 'always': upload_s3file(module, s3, bucket, obj, src, expiry, metadata, encrypt, headers) else: get_download_url(module, s3, bucket, obj, expiry, changed=False) else: sum_matches = False if overwrite in ('always', 'different'): upload_s3file(module, s3, bucket, obj, src, expiry, metadata, encrypt, headers) else: module.exit_json(msg="WARNING: Checksums do not match. Use overwrite parameter to force upload.") # If neither exist (based on bucket existence), we can create both. if bucketrtn is False and pathrtn is True: create_bucket(module, s3, bucket, location) upload_s3file(module, s3, bucket, obj, src, expiry, metadata, encrypt, headers) # If bucket exists but key doesn't, just upload. if bucketrtn is True and pathrtn is True and keyrtn is False: upload_s3file(module, s3, bucket, obj, src, expiry, metadata, encrypt, headers) # Delete an object from a bucket, not the entire bucket if mode == 'delobj': if obj is None: module.fail_json(msg="object parameter is required", failed=True); if bucket: bucketrtn = bucket_check(module, s3, bucket) if bucketrtn is True: deletertn = delete_key(module, s3, bucket, obj) if deletertn is True: module.exit_json(msg="Object %s deleted from bucket %s." % (obj, bucket), changed=True) else: module.fail_json(msg="Bucket does not exist.", changed=False) else: module.fail_json(msg="Bucket parameter is required.", failed=True) # Delete an entire bucket, including all objects in the bucket if mode == 'delete': if bucket: bucketrtn = bucket_check(module, s3, bucket) if bucketrtn is True: deletertn = delete_bucket(module, s3, bucket) if deletertn is True: module.exit_json(msg="Bucket %s and all keys have been deleted."%bucket, changed=True) else: module.fail_json(msg="Bucket does not exist.", changed=False) else: module.fail_json(msg="Bucket parameter is required.", failed=True) # Support for listing a set of keys if mode == 'list': bucket_object = get_bucket(module, s3, bucket) # If the bucket does not exist then bail out if bucket_object is None: module.fail_json(msg="Target bucket (%s) cannot be found"% bucket, failed=True) list_keys(module, bucket_object, prefix, marker, max_keys) # Need to research how to create directories without "populating" a key, so this should just do bucket creation for now. # WE SHOULD ENABLE SOME WAY OF CREATING AN EMPTY KEY TO CREATE "DIRECTORY" STRUCTURE, AWS CONSOLE DOES THIS. if mode == 'create': if bucket and not obj: bucketrtn = bucket_check(module, s3, bucket) if bucketrtn is True: module.exit_json(msg="Bucket already exists.", changed=False) else: module.exit_json(msg="Bucket created successfully", changed=create_bucket(module, s3, bucket, location)) if bucket and obj: bucketrtn = bucket_check(module, s3, bucket) if obj.endswith('/'): dirobj = obj else: dirobj = obj + "/" if bucketrtn is True: keyrtn = key_check(module, s3, bucket, dirobj) if keyrtn is True: module.exit_json(msg="Bucket %s and key %s already exists."% (bucket, obj), changed=False) else: create_dirkey(module, s3, bucket, dirobj) if bucketrtn is False: created = create_bucket(module, s3, bucket, location) create_dirkey(module, s3, bucket, dirobj) # Support for grabbing the time-expired URL for an object in S3/Walrus. if mode == 'geturl': if bucket and obj: bucketrtn = bucket_check(module, s3, bucket) if bucketrtn is False: module.fail_json(msg="Bucket %s does not exist."%bucket, failed=True) else: keyrtn = key_check(module, s3, bucket, obj) if keyrtn is True: get_download_url(module, s3, bucket, obj, expiry) else: module.fail_json(msg="Key %s does not exist."%obj, failed=True) else: module.fail_json(msg="Bucket and Object parameters must be set", failed=True) if mode == 'getstr': if bucket and obj: bucketrtn = bucket_check(module, s3, bucket) if bucketrtn is False: module.fail_json(msg="Bucket %s does not exist."%bucket, failed=True) else: keyrtn = key_check(module, s3, bucket, obj, version=version) if keyrtn is True: download_s3str(module, s3, bucket, obj, version=version) else: if version is not None: module.fail_json(msg="Key %s with version id %s does not exist."% (obj, version), failed=True) else: module.fail_json(msg="Key %s does not exist."%obj, failed=True) module.exit_json(failed=False)
def is_fakes3(s3_url): """ Return True if s3_url has scheme fakes3:// """ if s3_url is not None: return urlparse.urlparse(s3_url).scheme in ('fakes3', 'fakes3s') else: return False
def main(): argument_spec = ec2_argument_spec() argument_spec.update( dict(bucket=dict(required=True), object=dict(), version=dict(default=None), s3_url=dict(aliases=['S3_URL']), retries=dict(aliases=['retry'], type='int', default=0), rgw=dict(default='no', type='bool')), ) module = AnsibleModule(argument_spec=argument_spec, supports_check_mode=False) if not HAS_BOTO: module.fail_json(msg='boto required for this module') bucket = module.params.get('bucket') s3_url = module.params.get('s3_url') rgw = module.params.get('rgw') version = module.params.get('version') retries = module.params.get('retries') region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module) if region in ('eu-east-1', '', None): location = Location.DEFAULT else: location = region if module.params.get('object'): obj = os.path.expanduser(module.params['object']) if not s3_url and 'S3_URL' in os.environ: s3_url = os.environ['S3_URL'] if rgw and not s3_url: module.fail_json(msg='rgw flavour requires s3_url') if '.' in bucket: aws_connect_kwargs['calling_format'] = OrdinaryCallingFormat() try: if s3_url and rgw: rgw = urlparse.urlparse(s3_url) s3 = boto.connect_s3(is_secure=rgw.scheme == 'https', host=rgw.hostname, port=rgw.port, calling_format=OrdinaryCallingFormat(), **aws_connect_kwargs) elif is_fakes3(s3_url): fakes3 = urlparse.urlparse(s3_url) s3 = S3Connection(is_secure=fakes3.scheme == 'fakes3s', host=fakes3.hostname, port=fakes3.port, calling_format=OrdinaryCallingFormat(), **aws_connect_kwargs) elif is_walrus(s3_url): walrus = urlparse.urlparse(s3_url).hostname s3 = boto.connect_walrus(walrus, **aws_connect_kwargs) else: aws_connect_kwargs['is_secure'] = True try: s3 = connect_to_aws(boto.s3, location, **aws_connect_kwargs) except AnsibleAWSError: s3 = boto.connect_s3(**aws_connect_kwargs) except boto.exception.NoAuthHandlerFound as e: module.fail_json(msg='No Authentication Handler found: %s ' % str(e)) except Exception as e: module.fail_json(msg='Failed to connect to S3: %s' % str(e)) if s3 is None: module.fail_json( msg= 'Unknown error, failed to create s3 connection, no information from boto.' ) bucketrtn = bucket_check(module, s3, bucket) if bucketrtn is False: module.fail_json(msg="Source bucket cannot be found", failed=True) keyrtn = key_check(module, s3, bucket, obj, version=version) if keyrtn is False: if version is not None: module.fail_json(msg="Key %s with version id %s does not exist." % (obj, version), failed=True) else: module.fail_json(msg="Key %s does not exist." % obj, failed=True) read_s3file(module, s3, bucket, obj, retries, version=version) module.exit_json(failed=False)
def is_walrus(s3_url): if s3_url is not None: o = urlparse.urlparse(s3_url) return not o.hostname.endswith('amazonaws.com') else: return False
def is_fakes3(s3_url): if s3_url is not None: return urlparse.urlparse(s3_url).scheme in ('fakes3', 'fakes3s') else: return False