def _serialize_(self, commands, tmp_dir, app_name, docker_links, config):
if not self.has_value():
return
for port in config.ports:
if port.container_port != port.host_port:
raise DMakeException(
"AWS Elastic Beanstalk only supports ports binding which are the same in the container and the host."
)
ports = [{
"ContainerPort": ports.container_port
} for ports in config.ports]
volumes = [
{
"HostDirectory": volume.host_volume,
"ContainerDirectory": volume.container_volume
} for volume in config.volumes if volume.host_volume !=
"/var/log/deepomatic" # Cannot specify a volume both in logging and mounting
]
if config.pre_deploy_script != "" or \
config.mid_deploy_script != "" or \
config.post_deploy_script != "":
raise DMakeException(
"Pre/Mid/Post-Deploy scripts for AWS is not supported yet.")
if len(config.docker_opts) > 0:
raise DMakeException(
"Docker options for AWS is not supported yet.")
# Generate Dockerrun.aws.json
data = {
"AWSEBDockerrunVersion": "1",
# "Authentication": {
# "Bucket": "my-bucket",
# "Key": "mydockercfg"
# },
# "Image": {
# "Name": "quay.io/johndoe/private-image",
# "Update": "true"
# },
"Ports": ports,
"Volumes": volumes,
"Logging": "/var/log/deepomatic"
}
with open(os.path.join(tmp_dir, "Dockerrun.aws.json"),
'w') as dockerrun:
json.dump(data, dockerrun)
common.run_shell_command(
'deepomake_replace_vars %s %s' %
(self.options, os.path.join(tmp_dir, 'options.txt')))
append_command(commands,
'sh',
shell='deepomake_deploy_aws_eb "%s" "%s" "%s" "%s"' %
(tmp_dir, app_name, self.region, self.stack))
def _generate_env_file_(self):
if self.env_file is None:
tmp_dir = common.run_shell_command('deepomake_make_tmp_dir')
env_file = os.path.join(tmp_dir, 'env.txt')
with open(env_file, 'w') as f:
for key, value in self.env.items():
value = common.eval_str_in_env(value)
f.write('%s=%s\n' % (key, value))
self.env_file = env_file
return self.env_file
def _serialize_(self, commands, tmp_dir, app_name, docker_links, config):
if not self.has_value():
return
opts = config.full_docker_opts(False)
launch_links = ""
for link in docker_links:
launch_links += 'if [ \\`docker ps -f name=%s | wc -l\\` = "1" ]; then set +e; docker rm -f %s 2> /dev/null ; set -e; docker run -d --name %s %s -i %s; fi\n' % (
link.link_name, link.link_name, link.link_name,
link.deployed_options, link.image_name)
opts += " --link %s" % link.link_name
common.run_shell_command(
'export APP_NAME="%s" && export DOCKER_OPTS="%s" && export LAUNCH_LINK="%s" && export PRE_DEPLOY_HOOKS="%s" && export MID_DEPLOY_HOOKS="%s" && export POST_DEPLOY_HOOKS="%s" && deepomake_copy_template deploy/deploy_ssh/start_app.sh %s && deepomake_copy_template deploy/deploy_ssh/start_cmd.sh %s'
%
(app_name + "-%s" % common.branch.lower(), opts, launch_links,
config.pre_deploy_script, config.mid_deploy_script,
config.post_deploy_script, os.path.join(tmp_dir, "start_app.sh"),
os.path.join(tmp_dir, "start_cmd.sh")))
cmd = 'deepomake_deploy_ssh "%s" "%s" "%s" "%s"' % (
tmp_dir, self.user, self.host, self.port)
append_command(commands, 'sh', shell=cmd)
def generate_build(self, commands, path_dir, app_name, service_name,
docker_base, env, config):
tmp_dir = common.run_shell_command('deepomake_make_tmp_dir')
dockerfile_template = os.path.join(tmp_dir, 'Dockerfile_template')
with open(dockerfile_template, 'w') as f:
f.write('FROM %s\n' % docker_base)
f.write('\n')
f.write('${ENV_VARS}\n')
f.write('\n')
f.write('WORKDIR %s\n' % self.workdir)
f.write('\n')
for port in config.ports:
f.write('EXPOSE %s\n' % port.container_port)
f.write('\n')
for target in self.install_targets:
f.write(target.docker_cmd(commands, path_dir, tmp_dir) + "\n")
f.write('\n')
if self.install_script is not None:
generate_copy_command(commands, path_dir, tmp_dir,
self.install_script)
f.write('COPY %s %s\n' %
(self.install_script,
os.path.join(self.workdir, self.install_script)))
f.write('RUN cd %s && ./%s\n' %
(self.workdir, self.install_script))
f.write('\n')
generate_copy_command(commands, path_dir, tmp_dir,
self.start_script)
f.write('COPY %s %s\n' %
(self.start_script,
os.path.join(self.workdir, self.start_script)))
if self.entrypoint is not None:
f.write('ENTRYPOINT ["./%s"]\n' % self.entrypoint)
f.write('CMD ["./%s"]\n' % self.start_script)
generate_dockerfile(commands, tmp_dir, env)
image_name = self.get_image_name(app_name, service_name)
append_command(commands,
'sh',
shell='deepomake_build_docker "%s" "%s"' %
(tmp_dir, image_name))
return tmp_dir
def look_for_changed_directories():
if common.target is None:
return None
common.logger.info("Looking for changes between HEAD and %s" %
common.target)
try:
output = common.run_shell_command(
"git diff --name-only origin/%s...HEAD" % common.target)
except common.ShellError as e:
common.logger.error("Error: " + str(e))
return None
#common.logger.info("Changed files:")
#common.logger.info(output)
changed_dirs = set()
for file in [file.strip() for file in output.split('\n')]:
if len(file) == 0:
continue
d = os.path.dirname(file)
if d in changed_dirs:
continue
# Only keep bottom changed directories
do_add = True
to_remove = []
for directory in changed_dirs:
if directory.startswith(d): # sub directory of d
do_add = False
elif d.startswith(directory):
to_remove = directory
changed_dirs.difference_update(to_remove)
if do_add:
changed_dirs.add(d)
return list(changed_dirs)
def load_deepomake_files_list():
build_files = filter(
lambda f: len(f.strip()) > 0,
common.run_shell_command("find . -name deepomake.yml").split("\n"))
build_files = [file[2:] for file in build_files]
return build_files
def make(root_dir, sub_dir, dmake_command, app, options):
if 'DMAKE_TMP_DIR' in os.environ:
del os.environ['DMAKE_TMP_DIR']
common.init(dmake_command, root_dir, options)
if dmake_command == "stop":
common.run_shell_command(
"docker rm -f `docker ps -q -f name=%s.%s.%s`" %
(app, common.branch, common.build_id))
return
if dmake_command == "deploy" and common.is_local:
r = common.read_input(
"Carefull ! Are you sure you want to do that ? [Y/n] ")
if r.lower() != 'y' and r != "":
print('Aborting')
sys.exit(0)
# Format args
auto_complete = False
auto_completed_app = None
if app is not None:
n = len(app.split('/'))
if n > 2:
raise DMakeException(
'Cannot have more than one slash in the app name')
auto_complete = n == 1
if auto_complete:
auto_completed_app = None
auto_complete_is_app = None
else:
auto_completed_app = app
# Load build files
build_files = load_deepomake_files_list()
if len(build_files) == 0:
raise DMakeException('No deepomake.yml file found !')
# Sort by increasing length to make sure we load parent files first
sorted(build_files, key=lambda file: len(file))
# Load all deepomake.yml files (except those blacklisted)
blacklist = []
loaded_files = {}
service_providers = {}
service_dependencies = {}
for file in build_files:
load_deepomake_file(loaded_files, blacklist, service_providers,
service_dependencies, file)
# Remove black listed files
for f in blacklist:
if f in loaded_files:
del loaded_files[f]
# Register all apps and services in the repo
docker_links = {}
services = {}
for file, deepomake in loaded_files.items():
app_name = deepomake.get_app_name()
if app_name not in docker_links:
docker_links[app_name] = {}
if app_name not in services:
services[app_name] = {}
if auto_complete and app_name == app:
if auto_completed_app is None:
auto_completed_app = app
app_services = services[app_name]
for service in deepomake.get_services():
needs = [
"%s/%s" % (app_name, sa) for sa in service.needed_services
]
full_service_name = "%s/%s" % (app_name, service.service_name)
if service.service_name in app_services:
raise DMakeException("Duplicated sub-app name: '%s'" %
full_service_name)
add_service_provider(service_providers, full_service_name, file,
needs)
app_services[service.service_name] = service
if auto_complete and service.service_name == app:
if auto_completed_app is None:
auto_completed_app = full_service_name
auto_complete_is_app = False
else:
raise DMakeException(
"Ambigous app name '%s' is matching sub-app '%s' and %sapp '%s'"
% (app, full_service_name, "" if auto_complete_is_app
else "sub-", auto_completed_app))
app_links = docker_links[app_name]
for link in deepomake.get_docker_links():
if link.link_name in app_links:
raise DMakeException(
"Duplicate link name '%s' for application '%s'. Link names must be unique inside each app."
% (link.link_name, app_name))
app_links[link.link_name] = link
if auto_complete and auto_completed_app is None:
raise DMakeException(
"Could not find any app or sub-app matching '%s'" % app)
# Filter base images which are not provided
for deps in service_dependencies.values():
to_delete = []
for i, dep in enumerate(deps):
if dep[1] not in service_providers:
to_delete.append(i)
to_delete.reverse()
for i in to_delete:
del deps[i]
is_app_only = auto_completed_app is None or auto_completed_app.find(
'/') < 0
if dmake_command == "run" and is_app_only:
common.options.dependencies = True
if auto_completed_app is None:
# Find file where changes have happened
find_active_files(loaded_files, service_providers,
service_dependencies, sub_dir, dmake_command)
else:
if is_app_only: # app only
if dmake_command == 'shell':
raise DMakeException("Could not find sub-app '%s'" % app)
active_file = set()
app_services = services[auto_completed_app]
for service in app_services.values():
full_service_name = "%s/%s" % (auto_completed_app,
service.service_name)
file, _ = service_providers[full_service_name]
active_file.add(file)
for file in active_file:
activate_file(loaded_files, service_providers,
service_dependencies, dmake_command, file)
else:
activate_service(loaded_files, service_providers,
service_dependencies, dmake_command,
auto_completed_app)
# check services circularity
sorted_leaves = check_no_circular_dependencies(service_dependencies)
sorted_leaves = filter(lambda a_b__c: a_b__c[0][0] == dmake_command,
sorted_leaves)
build_files_order = order_dependencies(service_dependencies, sorted_leaves)
# Sort by order
ordered_build_files = sorted(build_files_order.items(),
key=lambda file_order: file_order[1])
# Separate into base / build / tests / deploy
n = len(ordered_build_files)
base = list(
filter(lambda a_b__c: a_b__c[0][0] in ['base'], ordered_build_files))
build = list(
filter(
lambda a_b__c: a_b__c[0][0] in ['build_service', 'build_docker'],
ordered_build_files))
test = list(
filter(
lambda a_b__c: a_b__c[0][
0] in ['build_tests', 'test', 'shell', 'run_link', 'run'],
ordered_build_files))
deploy = list(
filter(lambda a_b__c: a_b__c[0][0] in ['deploy'], ordered_build_files))
if len(base) + len(build) + len(test) + len(deploy) != len(
ordered_build_files):
raise Exception(
'Something went wrong when reorganizing build steps. One of the commands is probably missing.'
)
ordered_build_files = [('Building Docker', base), ('Building App', build),
('Running and Testing App', test)]
if not common.is_pr:
ordered_build_files.append(
('Deploying', list(deploy))
) # build_service needs to be before others otherwise it may mess-up with tests
# Display commands
common.logger.info("Okay, here is the plan:")
for stage, commands in ordered_build_files:
if len(commands) > 0:
common.logger.info("## %s ##" % (stage))
for (command, service), order in commands:
# Sanity check
sub_task_orders = [
build_files_order[a]
for a in service_dependencies[(command, service)]
]
if any(map(lambda o: order <= o, sub_task_orders)):
raise DMakeException('Bad ordering')
common.logger.info("- %s @ %s" % (command, service))
# Generate the list of command to run
all_commands = []
append_command(all_commands, 'env', var="REPO", value=common.repo)
append_command(all_commands, 'env', var="COMMIT", value=common.commit_id)
append_command(all_commands, 'env', var="BUILD", value=common.build_id)
append_command(all_commands, 'env', var="BRANCH", value=common.branch)
append_command(all_commands, 'env', var="ENV_TYPE", value=common.env_type)
append_command(all_commands,
'env',
var="DMAKE_TMP_DIR",
value=common.tmp_dir)
for stage, commands in ordered_build_files:
if len(commands) > 0:
append_command(all_commands,
'stage',
name=stage,
concurrency=1 if stage == "Deploying" else None)
for (command, service), order in commands:
append_command(all_commands,
'sh',
shell='echo "Running %s @ %s"' % (command, service))
if command in ['build_tests', 'build_service']:
deepomake = loaded_files[service]
else:
file, _ = service_providers[service]
deepomake = loaded_files[file]
app_name = deepomake.get_app_name()
links = docker_links[app_name]
app_services = services[app_name]
#try:
if True: # HACK
if command == "base":
deepomake.generate_base(all_commands)
elif command == "shell":
deepomake.generate_shell(all_commands, service,
app_services, links)
elif command == "test":
deepomake.generate_test(all_commands, service,
app_services, links)
elif command == "run":
deepomake.generate_run(all_commands, service, links)
elif command == "run_link":
deepomake.generate_run_link(all_commands, service, links)
elif command == "build_tests":
deepomake.generate_build_tests(all_commands)
elif command == "build_service":
deepomake.generate_build_services(all_commands)
elif command == "build_docker":
deepomake.generate_build(all_commands, service)
elif command == "deploy":
deepomake.generate_deploy(all_commands, service, links)
else:
raise Exception("Unkown command '%s'" % command)
# except DMakeException as e:
# print(('ERROR in file %s:\n' % file) + str(e))
# sys.exit(1)
# Check stages do not appear twice (otherwise it may block Jenkins)
stage_names = set()
for cmd, kwargs in all_commands:
if cmd == "stage":
name = kwargs['name']
if name in stage_names:
raise DMakeException('Duplicate stage name: %s' % name)
else:
stage_names.add(name)
# If not on Pull Request, tag the commit as deployed
if dmake_command == "deploy" and not common.is_pr:
append_command(all_commands,
'git_tag',
tag='deployed_version_%s' % common.branch)
# Generate output
if common.is_local:
file_to_generate = os.path.join(common.tmp_dir, "DMakefile")
else:
file_to_generate = "DMakefile"
generate_command(file_to_generate, all_commands)
common.logger.info("Output has been written to %s" % file_to_generate)
# If on local, run the commands
if common.is_local:
result = os.system('bash %s' % file_to_generate)
if result != 0 or dmake_command != 'run':
os.system('deepomake_clean %s' % common.tmp_dir)
def _serialize_(self, commands, path_dir):
if self.base_image.has_value():
# Make the temporary directory
tmp_dir = common.run_shell_command('deepomake_make_tmp_dir')
# Copy file and compute their md5
files_to_copy = []
for file in self.base_image.copy_files + self.base_image.install_scripts:
files_to_copy.append(file)
if self.base_image.python_requirements:
files_to_copy.append(self.base_image.python_requirements)
if self.base_image.python3_requirements:
files_to_copy.append(self.base_image.python3_requirements)
# Copy file and keep their md5
md5s = {}
for file in files_to_copy:
md5s[file] = common.run_shell_command(
'deepomake_copy_file %s %s' % (os.path.join(
path_dir, file), os.path.join(tmp_dir, 'user', file)))
# Set RUN command
run_cmd = "cd user"
for file in self.base_image.install_scripts:
run_cmd += " && ./%s" % file
# Install pip if needed
if self.base_image.python_requirements:
run_cmd += " && bash ../install_pip.sh && pip install --process-dependency-links -r " + self.base_image.python_requirements
if self.base_image.python3_requirements:
run_cmd += " && bash ../install_pip3.sh && pip3 install --process-dependency-links -r " + self.base_image.python3_requirements
# Save the command in a bash file
file = 'run_cmd.sh'
with open(os.path.join(tmp_dir, file), 'w') as f:
f.write(run_cmd)
md5s[file] = common.run_shell_command('deepomake_md5 %s' %
os.path.join(tmp_dir, file))
# HACK: copy key while #493 is not closed: https://github.com/docker/for-mac/issues/483
if common.key_file is not None:
common.run_shell_command(
'cp %s %s' %
(common.key_file, os.path.join(tmp_dir, 'key')))
# Local environment for temmplates
local_env = []
local_env.append("export ROOT_IMAGE=%s" % self.root_image)
local_env = ' && '.join(local_env)
if len(local_env) > 0:
local_env += ' && '
# Copy templates
for file in [
"make_base.sh", "config.logrotate", "load_credentials.sh",
"install_pip.sh", "install_pip3.sh"
]:
md5s[file] = common.run_shell_command(
'%s deepomake_copy_template docker-base/%s %s' %
(local_env, file, os.path.join(tmp_dir, file)))
# Output md5s for comparison
with open(os.path.join(tmp_dir, 'md5s'), 'w') as f:
for md5 in md5s.items():
f.write('%s %s\n' % md5)
# Append Docker Base build command
append_command(
commands,
'sh',
shell='deepomake_build_base_docker "%s" "%s" "%s" "%s" "%s"' %
(tmp_dir, self.root_image, self.base_image.name,
self._get_tag_(), self.base_image.version))