def images(self):
"""Return a list of Docker images on the current machine."""
output = SystemCommand(self.cmd.images).output
output.pop(0)
return [
row.split(' ', 1)[0] + ':' +
row.split(' ', 1)[1].strip().split(' ', 1)[0] for row in output
]
def push(self):
"""Push a docker image to a DockerHub repo."""
print('Pushing Docker image ({0})'.format(self.cmd.docker_image))
sc = SystemCommand(self.cmd.push, decode_output=False)
self.add_command(sc.command)
self.add_task('Pushed Docker image {0} to DockerHub repo'.format(
self.cmd.docker_image))
def eb_deploy_multi(values):
# Get Dockerize parameters
docker = dockerize()[0]
# Remote directory relative to Project root
remote_dir = os.path.join(docker.source, REMOTE_DIRECTORY)
# Copy the Dockerrun.aws.json file to '-remote' directory
if not os.path.exists(remote_dir):
os.mkdir(remote_dir)
shutil.copyfile(os.path.join(docker.source, 'Dockerrun.aws.json'),
os.path.join(remote_dir, 'Dockerrun.aws.json'))
# Change directory to source
os.chdir(remote_dir)
print(os.getcwd())
print('Using root directory: {0}'.format(remote_dir))
with Timer('Deployed to AWS EB'):
SystemCommand(
'eb deploy {env} --label {version} --message "{message}"'.format(
env=values['aws_eb_env'],
version=docker.tag,
message=values['desc']), False)
return
def move(self,
src_path,
dst_path,
dst_bucket=None,
recursive=False,
include=None,
exclude=None):
"""
Move an S3 file or folder to another
:param src_path: Path to source file or folder in S3 bucket
:param dst_path: Path to destination file or folder
:param dst_bucket: Bucket to copy to, defaults to same bucket
:param recursive: Recursively copy all files within the directory
:param include: Don't exclude files or objects in the command that match the specified pattern
:param exclude: Exclude all files or objects from the command that matches the specified pattern
More on inclusion and exclusion parameters...
http://docs.aws.amazon.com/cli/latest/reference/s3/index.html#use-of-exclude-and-include-filters
"""
uri1 = '{uri}/{src}'.format(uri=self.bucket_uri, src=src_path)
uri2 = '{uri}/{dst}'.format(
uri=bucket_uri(dst_bucket) if dst_bucket else self.bucket_uri,
dst=dst_path)
# Move recursively if both URI's are directories and NOT files
return SystemCommand(
self.cmd.move(object1=uri1,
object2=uri2,
recursive=is_recursive_needed(
uri1, uri2, recursive_default=recursive),
include=include,
exclude=exclude))
def list(self,
remote_path='',
recursive=False,
human_readable=False,
summarize=False):
"""
List files/folders in a S3 bucket path.
Optionally, return information on file size on each objects as
well as summary info.
:param remote_path: Path to object root in S3 bucket
:param recursive: Recursively list files/folders
:param human_readable: Displays file sizes in human readable format
:param summarize: Displays summary information (number of objects, total size)
:return:
"""
return [
out.rsplit(' ', 1)[-1] for out in SystemCommand(
self.cmd.list(uri='{0}/{1}'.format(
self.bucket_uri, remote_path_root(remote_path)),
recursive=recursive,
human_readable=human_readable,
summarize=summarize))
]
def create_bucket(self, region='us-east-1'):
"""
Create a new S3 bucket.
:param region: Bucket's hosting region
"""
# Validate that the bucket does not already exist
assert self.bucket_name not in self.buckets, 'ERROR: Bucket `{0}` already exists.'.format(
self.bucket_name)
# Create the bucket
create = SystemCommand(self.cmd.make_bucket(self.bucket_uri, region))
# Enable transfer acceleration
SystemCommand(self.cmd.enable_transfer_acceleration(self.bucket_name))
return create
def list(self, all_apps=False, verbose=False):
"""
Lists all environments in the current application or all environments in all applications.
:param all_apps: Lists all environments from all applications.
:param verbose: Provides more detailed information about all environments, including instances.
:return: List of Elastic Beanstalk applications
"""
def verbose_env(environment):
return environment.split(' : ')[0].replace('* ', '')
def verbose_ec2(environment):
return environment.split(' : ')[-1].replace("['",
'').replace("']", '')
self.initialize()
cmd = 'eb list'
cmd += ' --all' if all_apps else ''
if verbose:
output = SystemCommand(cmd + ' --verbose').output
response = {
'region': output.pop(0).replace('Region: ', ''),
'application': output.pop(0).replace('Application: ', '')
}
output.pop(0).replace('Environments: ', '')
response['environments'] = {
verbose_env(i): verbose_ec2(i)
for i in output
}
return response
else:
return set(
[i.replace('* ', '') for i in SystemCommand(cmd).output])
def buckets(self):
"""
List all available S3 buckets.
Execute the `aws s3 ls` command and decode the output
"""
return [
out.rsplit(' ', 1)[-1] for out in SystemCommand(self.cmd.list())
]
def is_acceleration_enabled(self):
"""Determine if transfer acceleration is enabled for an AWS S3 bucket."""
output = SystemCommand(
self.cmd.acceleration_enabled_status(self.bucket_name)).output
if len(output) > 0:
return output[0].strip('"').lower() == 'enabled'
else:
return False
def pull(self, resolve_tag=True):
"""Push a docker image to a DockerHub repo."""
if resolve_tag and not self.cmd.tag:
self.cmd.tag = self.image_tags[0]
print('Pulling Docker image ({0})'.format(self.cmd.docker_image))
sc = SystemCommand(self.cmd.pull, decode_output=False)
self.add_command(sc.command)
self.add_task('Pulled Docker image {0} from DockerHub repo'.format(
self.cmd.docker_image))
def exists(self, remote_path):
"""
Check to see if an S3 key (file or directory) exists
:return: Bool
"""
# Check to see if a result was returned, if not then key does not exist
return True if len(
SystemCommand(
self.cmd.list('{0}/{1}'.format(self.bucket_uri,
remote_path)))) > 0 else False
def delete_bucket(self, force=False):
"""
Deletes an empty S3 bucket. A bucket must be completely empty of objects and versioned
objects before it can be deleted. However, the force parameter can be used to delete
the non-versioned objects in the bucket before the bucket is deleted.
:param force: Deletes all objects in the bucket including the bucket itself
"""
# Validate that the bucket does exist
assert self.bucket_name in self.buckets, 'ERROR: Bucket `{0}` does not exists.'.format(
self.bucket_name)
return SystemCommand(self.cmd.remove_bucket(self.bucket_uri, force))
def run(self):
"""Push a docker image to a DockerHub repo."""
print('Locally running Docker image')
sc = SystemCommand(self.cmd.run, decode_output=False)
self.add_command(sc.command)
if sc.success:
self.add_task('Running Docker image ({0}) on local machine'.format(
self.cmd.docker_image))
else:
self.add_task(
'ERROR: Unable to running Docker image ({0}) on local machine'.
format(self.cmd.docker_image))
def environments(self):
"""Retrieve a list of environments in the current EB application."""
cmd = 'aws elasticbeanstalk describe-environments --application-name {0}'.format(
self.aws_application_name)
return {
d[3].split('.', 1)[0]: {
'running_version': d[-1],
'status': d[-2]
}
for d in [i.split('\t') for i in SystemCommand(cmd).output]
if d[0].lower() == 'environments'
}
def bootstrap(self):
"""Bootstrap docker-compose service development by pulling existing images then building services."""
print('Bootstrapping docker-compose services')
for index, cmd in enumerate(self.cmd.bootstrap):
sc = SystemCommand(cmd, decode_output=False)
self.add_command(sc.command)
if sc.success:
self.add_task('SUCCESS ({}/{}): {}'.format(
index + 1,
len(self.cmd.bootstrap) + 1, sc.command))
else:
self.add_task('ERROR ({}/{}): {}'.format(
index + 1,
len(self.cmd.bootstrap) + 1, sc.command))
def reboot(self):
"""Reboot docker-compose container services by rebuilding then restarting."""
print('Bootstrapping docker-compose services')
for index, cmd in enumerate(self.cmd.reboot):
sc = SystemCommand(cmd, decode_output=False)
self.add_command(sc.command)
if sc.success:
self.add_task('SUCCESS ({}/{}): {}'.format(
index + 1,
len(self.cmd.reboot) + 1, sc.command))
else:
self.add_task('ERROR ({}/{}): {}'.format(
index + 1,
len(self.cmd.reboot) + 1, sc.command))
def pre_sign(self, remote_path, expiration=3600):
"""
Generate a pre-signed URL for an Amazon S3 object.
This allows anyone who receives the pre-signed URL to retrieve the S3 object
with an HTTP GET request.
:param remote_path: Path to S3 object relative to bucket root
:param expiration: Number of seconds until the pre-signed URL expires
:return:
"""
return SystemCommand(
self.cmd.pre_sign(
'{uri}/{src}'.format(uri=self.bucket_uri, src=remote_path),
expiration))[0]
def initialize(self, source=None):
"""Initialize the docker application if it hasn't been previously initialized."""
# Path to .elasticbeanstalk directory
source = self.source if not source else source
# Initialize docker
os.chdir(source)
SystemCommand(
'eb init --region {0} --keyname {1} -p docker {2}'.format(
self.aws_region, self.aws_instance_key, self.docker_repo))
self.add_task("Initialized '{0}' as an EB application".format(
self.aws_application_name))
# Edit default region value in config.yaml
self.set_region(source)
def _deploy(self):
"""Use awsebcli command '$eb deploy' to deploy an updated Elastic Beanstalk environment."""
# Update a Dockerrun.aws.json file in -remote directory
self.Dockerrun.create()
# Initialize application in -remote directory
self.initialize(self.Dockerrun.remote_source)
os.chdir(self.Dockerrun.remote_source)
SystemCommand(
'eb deploy {env} --label {version} --message "{message}"'.format(
env=self.aws_environment_name,
version=self.aws_version,
message=self.aws_version_description))
self.add_task('Deployed Elastic Beanstalk environment {0}'.format(
self.aws_environment_name))
def delete(self, remote_path, recursive=False, include=None, exclude=None):
"""
Delete an S3 object from a bucket.
:param remote_path: Path to S3 object relative to bucket root
:param recursive: Recursively copy all files within the directory
:param include: Don't exclude files or objects in the command that match the specified pattern
:param exclude: Exclude all files or objects from the command that matches the specified pattern
:return: Command string
"""
# Delete recursively if both URI's are directories and NOT files
return SystemCommand(
self.cmd.remove(uri='{uri}/{src}'.format(uri=self.bucket_uri,
src=remote_path),
recursive=is_recursive_needed(
remote_path, recursive_default=recursive),
include=include,
exclude=exclude))
def download(self,
remote_path,
local_path=os.getcwd(),
recursive=False,
quiet=None):
"""
Download a file or folder from an S3 bucket.
:param remote_path: S3 key, aka remote path relative to S3 bucket's root
:param local_path: Path to file on local disk
:param recursive: Recursively download files/folders
:param quiet: When true, does not display the operations performed from the specified command
"""
return SystemCommand(
self.cmd.copy(object1='{0}/{1}'.format(self.bucket_uri,
remote_path),
object2=local_path,
recursive=recursive,
quiet=quiet if quiet else self.quiet))
def upload(self, local_path, remote_path=None, acl='private', quiet=None):
"""
Upload a local file to an S3 bucket.
:param local_path: Path to file on local disk
:param remote_path: S3 key, aka remote path relative to S3 bucket's root
:param acl: Access permissions, must be either 'private', 'public-read' or 'public-read-write'
:param quiet: When true, does not display the operations performed from the specified command
"""
# Recursively upload files if the local target is a folder
# Use local_path file/folder name as remote_path if none is specified
remote_path = os.path.basename(
local_path) if not remote_path else remote_path
assert_acl(acl)
return SystemCommand(
self.cmd.copy(
object1=local_path,
object2='{0}/{1}'.format(self.bucket_uri, remote_path),
recursive=True if os.path.isdir(local_path) else False,
acl=acl,
quiet=quiet if quiet else self.quiet))
def deploy(self):
"""Deploy a docker image from a DockerHub repo to a AWS elastic beanstalk environment instance."""
# Check to see if AWS EB Environment already exists or if it is 'Terminated'
try:
environments = self.environments.get(
self.aws_environment_name).get('status', None) == 'Terminated'
except AttributeError:
environments = True
if any(condition for condition in (
self.aws_environment_name not in self.environments,
environments)):
print('Creating Elastic Beanstalk environment')
self._create()
else:
print('Deploying Elastic Beanstalk environment')
self._deploy()
# Dump deployment data/results to JSON
self.update_history(self.json_path, self.parameters)
# Open Elastic Beanstalk in a browser
SystemCommand('eb open')
def copy(self,
src_path,
dst_path,
dst_bucket=None,
recursive=False,
include=None,
exclude=None,
acl='private',
quiet=None):
"""
Copy an S3 file or folder to another
:param src_path: Path to source file or folder in S3 bucket
:param dst_path: Path to destination file or folder
:param dst_bucket: Bucket to copy to, defaults to same bucket
:param recursive: Recursively copy all files within the directory
:param include: Don't exclude files or objects in the command that match the specified pattern
:param exclude: Exclude all files or objects from the command that matches the specified pattern
:param acl: Access permissions, must be either 'private', 'public-read' or 'public-read-write'
:param quiet: When true, does not display the operations performed from the specified command
More on inclusion and exclusion parameters...
http://docs.aws.amazon.com/cli/latest/reference/s3/index.html#use-of-exclude-and-include-filters
"""
uri1 = '{uri}/{src}'.format(uri=self.bucket_uri, src=src_path)
uri2 = '{uri}/{dst}'.format(
uri=bucket_uri(dst_bucket) if dst_bucket else self.bucket_uri,
dst=dst_path)
# Copy recursively if both URI's are directories and NOT files
return SystemCommand(
self.cmd.copy(object1=uri1,
object2=uri2,
recursive=is_recursive_needed(
uri1, uri2, recursive_default=recursive),
include=include,
exclude=exclude,
acl=acl,
quiet=quiet if quiet else self.quiet))
def _create(self):
"""Use awsebcli command `$ eb create` to create a new Elastic Beanstalk environment."""
# Create directory with '-remote' extension next to source
if not os.path.exists(self.Dockerrun.remote_source):
os.mkdir(self.Dockerrun.remote_source)
self.add_task(
"Created directory '{0}' for storing Dockerrun file".format(
self.Dockerrun.remote_source))
# Create a Dockerrun.aws.json file in -remote directory
self.Dockerrun.create()
# Initialize application in -remote directory
self.initialize(self.Dockerrun.remote_source)
# Create Elastic Beanstalk environment in current application
os.chdir(self.Dockerrun.remote_source)
cmd = 'eb create {env} --keyname {key}'.format(
env=self.aws_environment_name, key=self.aws_instance_key)
SystemCommand(cmd)
self.Dockerrun.destroy()
self.add_task('Created Elastic Beanstalk environment {0}'.format(
self.aws_environment_name))
def sync(self,
local_path,
remote_path=None,
delete=False,
acl='private',
quiet=None,
remote_source=False):
"""
Synchronize local files with an S3 bucket.
S3 sync only copies missing or outdated files or objects between
the source and target. However, you can also supply the --delete
option to remove files or objects from the target that are not
present in the source.
:param local_path: Local source directory
:param remote_path: Destination directory (relative to bucket root)
:param delete: Sync with deletion, disabled by default
:param acl: Access permissions, must be either 'private', 'public-read' or 'public-read-write'
:param quiet: When true, does not display the operations performed from the specified command
:param remote_source: When true, remote_path is used as the source instead of destination
"""
assert_acl(acl)
uri = '{0}/{1}'.format(
self.bucket_uri,
os.path.basename(local_path) if not remote_path else remote_path)
# Sync from the S3 bucket
destination, source = (local_path,
uri) if remote_source else (uri, local_path)
return SystemCommand(
self.cmd.sync(source=source,
destination=destination,
delete=delete,
acl=acl,
quiet=quiet if quiet else self.quiet))
def delete_images(self):
"""Delete all images on the current machine."""
if len(self.images) > 0:
return SystemCommand(self.cmd.delete_images)
def clean(self):
"""Remove stopped containers and intermediate images from the current machine."""
return SystemCommand(self.cmd.clean)
def build(self):
"""Build a docker image for distribution to DockerHub."""
print('Building Docker image ({0})'.format(self.cmd.docker_image))
sc = SystemCommand(self.cmd.build, decode_output=False)
self.add_command(sc.command)
self.add_task('Built Docker image ({0})'.format(self.cmd.docker_image))
def delete_volumes(self):
"""Delete all volumes on the current machine."""
return SystemCommand(self.cmd.delete_volumes)