def __init__(self, config):
self.config = config
if self.config.ENV:
env = '-' + self.config.ENV
else:
env = ''
self.compose_path = os.path.join(self.config.BUILD_PATH,
'docker-compose.yml')
if not os.path.exists(
self.compose_path
) and self.config.steps_key not in self.config.global_commands:
message(
'docker-compose.yml file is missing. To run docker-console you need to do one of this things:\n'
'go to project wrapper path, \n'
'specify absolute path to project wrapper by -p (--docker-run-path) option\n'
'create alias with your project wrapper path in ~/.docker_console/aliases (then docker-console @project_dev)',
'warning')
exit(0)
compose_template_path = os.path.join(
self.config.BUILD_PATH, 'docker', 'compose_templates',
'docker-compose' + env + '-template.yml')
if not os.path.isfile(compose_template_path):
compose_template_path = os.path.join(
self.config.BUILD_PATH,
'docker-compose' + env + '-template.yml')
self.compose_template_path = compose_template_path
self.base_alias = self.get_project_name(self.config.BUILD_PATH)
def docker_init(self):
self._copy_init_tpl_files('docker_init')
app_path = os.path.join(self.config.BUILD_PATH, 'app')
if not os.path.exists(app_path):
os.mkdir(app_path)
message("Docker has been correctly initialized in this project.", 'info')
message("If you want to automatically add config entry for this project to /etc/hosts, please run 'docker-console add-host-to-etc-hosts' command", 'info')
def wait_for_all_containers_to_finish(self, command_containers_ids):
parallel_tests_finished = False
while not parallel_tests_finished:
print "\n\nWaiting 10 sec for next test status checking."
time.sleep(10)
parallel_tests_finished = True
for container_id in command_containers_ids:
container_tests_info = ''
for info_name in ['group', 'suite']:
if info_name in command_containers_ids[container_id]:
container_tests_info = '(%s: %s)' % (
info_name,
command_containers_ids[container_id][info_name])
break
test_running = run_cmd('docker ps -q -f id=%s' % container_id,
return_output=True)
if command_containers_ids[container_id]['finished']:
print "Container %s finished %s" % (container_id,
container_tests_info)
elif not test_running:
command_containers_ids[container_id]['finished'] = True
print "Container %s finished right now %s" % (
container_id, container_tests_info)
else:
print "Container %s still running %s" % (
container_id, container_tests_info)
parallel_tests_finished = False
if parallel_tests_finished:
message('All containers finished', 'info')
def tests_init(self):
self.docker._copy_init_tpl_files('tests_init')
message("Tests has been correctly initialized in this project.", 'info')
message(
"If you wolud like to have source code of testing environment locally you need to run 'composer install' command in tests directory.",
'info')
def tests_init(self):
self.docker._copy_init_tpl_files('tests_init')
message("Tests has been correctly initialized in this project.", 'info')
message(
"If you wolud like to have source code of testing environment locally you need to run 'composer install' command in tests directory.",
'info')
def wait_for_all_containers_to_finish(self, command_containers_ids):
parallel_tests_finished = False
while not parallel_tests_finished:
print "\n\nWaiting 10 sec for next test status checking."
time.sleep(10)
parallel_tests_finished = True
for container_id in command_containers_ids:
container_tests_info = ''
for info_name in ['group', 'suite']:
if info_name in command_containers_ids[container_id]:
container_tests_info = '(%s: %s)' % (info_name, command_containers_ids[container_id][info_name])
break
test_running = run_cmd('docker ps -q -f id=%s' % container_id, return_output=True)
if command_containers_ids[container_id]['finished']:
print "Container %s finished %s" % (container_id, container_tests_info)
elif not test_running:
command_containers_ids[container_id]['finished'] = True
print "Container %s finished right now %s" % (container_id, container_tests_info)
else:
print "Container %s still running %s" % (container_id, container_tests_info)
parallel_tests_finished = False
if parallel_tests_finished:
message('All containers finished', 'info')
def get_container_ip(self):
web_container_alias = self._container_alias("web:web").split(':')[0]
if not run_cmd('docker ps -q -f name=%s' % web_container_alias, return_output=True):
message("Docker is not up for this project and it will be started. It is required to add config entry to /etc/hosts.", 'info')
self.docker_up()
web_container_ip_address = run_cmd('docker inspect --format "{{ .NetworkSettings.IPAddress }}" %s'
% web_container_alias,
return_output=True)
return web_container_ip_address
def test_run_parallel(self):
self.robo_init()
split_type = cmd_options.parallel_group_by
number_of_groups = int(cmd_options.parallel_test_groups)
suites = cmd_options.parallel_test_suites.split(',')
self.docker_codecept('build')
self.docker_codecept('clean')
command_containers_ids = {}
# Run parallel testing of suites.
if len(suites) > 1:
number_of_groups = len(suites)
for idx in range(len(suites)):
suite = suites[idx]
suite_number = idx + 1
command_container_id = self.docker_codecept(
'run tests/%s --html report_parallel_%s.html --xml report_parallel_%s.xml'
% (suite, suite_number, suite_number),
parallel=True)
command_containers_ids[command_container_id] = {
'finished': False,
'suite': suite
}
# Run parallel testing of groups.
else:
self.docker_robo('parallel:split-by-%s %s' %
(split_type, number_of_groups))
for group in xrange(1, number_of_groups + 1):
group_file_path = os.path.join(
self.config.BUILD_PATH, self.config.TESTS['LOCATION'],
'tests/_output/parallel_group_%s' % group)
if os.path.isfile(group_file_path):
command_container_id = self.docker_codecept(
'run --group parallel_group_%s --html report_parallel_%s.html --xml report_parallel_%s.xml -vvv'
% (group, group, group),
parallel=True)
command_containers_ids[command_container_id] = {
'finished': False,
'group': 'parallel_group_%s' % group
}
if len(command_containers_ids) > 0:
number_of_groups = len(command_containers_ids)
message(
'%s containers with parallel test groups has been started. '
'We need to wait until all of them will finish.' %
number_of_groups, 'info')
self.wait_for_all_containers_to_finish(command_containers_ids)
self.docker_robo('parallel:merge-htmlresults %s' %
(number_of_groups))
self.docker_robo('parallel:merge-xmlresults %s' %
(number_of_groups))
else:
message('Seems like there is no group files created.', 'info')
run_cmd('docker stop selenium-test-%s' % self.docker.base_alias)
run_cmd('docker rm selenium-test-%s' % self.docker.base_alias)
def docker_init(self):
self._copy_init_tpl_files('docker_init')
app_path = os.path.join(self.config.BUILD_PATH, 'app')
if not os.path.exists(app_path):
os.mkdir(app_path)
message("Docker has been correctly initialized in this project.",
'info')
message(
"If you want to automatically add config entry for this project to /etc/hosts, please run 'docker-console add-host-to-etc-hosts' command",
'info')
def unpack_private_files(self, remove_existing=False):
message('Unpacking private files')
src = os.path.join(self.config.BUILD_PATH, self.config.DRUPAL[self.config.drupal_site]['PRIVATE_FILES_ARCHIVE'])
dest = os.path.join(self.config.WEB['APP_ROOT'], self.config.DRUPAL[self.config.drupal_site]['PRIVATE_FILES_DST'])
if remove_existing and os.path.exists(os.path.join(dest, 'private')):
shutil.rmtree(os.path.join(dest, 'private'))
unpack_tar(src, dest)
os.system("chmod -Rf 777 %s" % os.path.join(dest, 'private'))
def get_container_ip(self):
web_container_alias = self._container_alias("web:web").split(':')[0]
if not run_cmd('docker ps -q -f name=%s' % web_container_alias,
return_output=True):
message(
"Docker is not up for this project and it will be started. It is required to add config entry to /etc/hosts.",
'info')
self.docker_up()
web_container_ip_address = run_cmd(
'docker inspect --format "{{ .NetworkSettings.IPAddress }}" %s' %
web_container_alias,
return_output=True)
return web_container_ip_address
def tests_run(self):
if ('selenium_image' not in self.config.TESTS['IMAGES']) or (
'codecept_image' not in self.config.TESTS['IMAGES']):
message('selenium_image or codecept_image is missing in TESTS config.', 'error')
exit(0)
if len(self.config.args) > 2:
args = 'tests/' + ' '.join(self.config.args[2:])
else:
args = ''
self.docker_codecept('build')
self.docker_codecept('clean')
self.docker_codecept('run %s --html --xml' % args)
run_cmd('docker stop selenium-test-%s' % self.docker.base_alias)
run_cmd('docker rm selenium-test-%s' % self.docker.base_alias)
def unpack_private_files(self, remove_existing=False):
message('Unpacking private files')
src = os.path.join(
self.config.BUILD_PATH, self.config.DRUPAL[self.config.drupal_site]
['PRIVATE_FILES_ARCHIVE'])
dest = os.path.join(
self.config.WEB['APP_ROOT'],
self.config.DRUPAL[self.config.drupal_site]['PRIVATE_FILES_DST'])
if remove_existing and os.path.exists(os.path.join(dest, 'private')):
shutil.rmtree(os.path.join(dest, 'private'))
unpack_tar(src, dest)
os.system("chmod -Rf 777 %s" % os.path.join(dest, 'private'))
def tests_run(self):
if ('selenium_image' not in self.config.TESTS['IMAGES']) or (
'codecept_image' not in self.config.TESTS['IMAGES']):
message('selenium_image or codecept_image is missing in TESTS config.', 'error')
exit(0)
if len(self.config.args) > 2:
args = 'tests/' + ' '.join(self.config.args[2:])
else:
args = ''
self.docker_codecept('build')
self.docker_codecept('clean')
self.docker_codecept('run %s --html --xml' % args)
run_cmd('docker stop selenium-test-%s' % self.docker.base_alias)
run_cmd('docker rm selenium-test-%s' % self.docker.base_alias)
def __init__(self, config):
super(Builder, self).__init__(config)
# check if executed command is global
if self.config.steps_key not in self.config.global_commands:
self.drupal_site = cmd_options.drupal_site
if self.drupal_site not in self.config.DRUPAL:
message("Site '%s' is not available in DRUPAL config in %s/docker_console/dc_settings.py" % (self.drupal_site, self.config.BUILD_PATH), 'error')
exit(0)
if os.environ.get("SITE_URI"):
self.config.DRUPAL[self.drupal_site]['SITE_URI'] = os.environ.get("SITE_URI")
self.config.drupal_site = self.drupal_site
self.docker = Docker(self.config)
self.tests = Tests(self.config)
self.drush = Drush(self.config)
self.drupal_settings = DrupalSettings(self.config)
self.archive = Archive(self.config)
def __init__(self, config):
self.config = config
if self.config.ENV:
env = '-' + self.config.ENV
else:
env = ''
self.compose_path = os.path.join(self.config.BUILD_PATH, 'docker-compose.yml')
if not os.path.exists(self.compose_path) and self.config.steps_key not in self.config.global_commands:
message('docker-compose.yml file is missing. To run docker-console you need to do one of this things:\n'
'go to project wrapper path, \n'
'specify absolute path to project wrapper by -p (--docker-run-path) option\n'
'create alias with your project wrapper path in ~/.docker_console/aliases (then docker-console @project_dev)', 'warning')
exit(0)
compose_template_path = os.path.join(self.config.BUILD_PATH, 'docker', 'compose_templates', 'docker-compose' + env + '-template.yml')
if not os.path.isfile(compose_template_path):
compose_template_path = os.path.join(self.config.BUILD_PATH, 'docker-compose' + env + '-template.yml')
self.compose_template_path = compose_template_path
self.base_alias = self.get_project_name(self.config.BUILD_PATH)
def __init__(self, config):
super(Builder, self).__init__(config)
# check if executed command is global
if self.config.steps_key not in self.config.global_commands:
self.drupal_site = cmd_options.drupal_site
if self.drupal_site not in self.config.DRUPAL:
message(
"Site '%s' is not available in DRUPAL config in %s/docker_console/dc_settings.py"
% (self.drupal_site, self.config.BUILD_PATH), 'error')
exit(0)
if os.environ.get("SITE_URI"):
self.config.DRUPAL[
self.drupal_site]['SITE_URI'] = os.environ.get("SITE_URI")
self.config.drupal_site = self.drupal_site
self.docker = Docker(self.config)
self.tests = Tests(self.config)
self.drush = Drush(self.config)
self.drupal_settings = DrupalSettings(self.config)
self.archive = Archive(self.config)
def test_run_parallel(self):
self.robo_init()
split_type = cmd_options.parallel_group_by
number_of_groups = int(cmd_options.parallel_test_groups)
suites = cmd_options.parallel_test_suites.split(',')
self.docker_codecept('build')
self.docker_codecept('clean')
command_containers_ids = {}
# Run parallel testing of suites.
if len(suites) > 1:
number_of_groups = len(suites)
for idx in range(len(suites)):
suite = suites[idx]
suite_number = idx + 1
command_container_id = self.docker_codecept(
'run tests/%s --html report_parallel_%s.html --xml report_parallel_%s.xml' % (suite, suite_number, suite_number), parallel=True)
command_containers_ids[command_container_id] = {'finished': False, 'suite': suite}
# Run parallel testing of groups.
else:
self.docker_robo('parallel:split-by-%s %s' % (split_type, number_of_groups))
for group in xrange(1, number_of_groups + 1):
group_file_path = os.path.join(self.config.BUILD_PATH, self.config.TESTS['LOCATION'], 'tests/_output/parallel_group_%s' % group)
if os.path.isfile(group_file_path):
command_container_id = self.docker_codecept(
'run --group parallel_group_%s --html report_parallel_%s.html --xml report_parallel_%s.xml -vvv' % (group, group, group),
parallel=True)
command_containers_ids[command_container_id] = {'finished': False, 'group': 'parallel_group_%s' % group}
if len(command_containers_ids) > 0:
number_of_groups = len(command_containers_ids)
message('%s containers with parallel test groups has been started. '
'We need to wait until all of them will finish.' % number_of_groups, 'info')
self.wait_for_all_containers_to_finish(command_containers_ids)
self.docker_robo('parallel:merge-htmlresults %s' % (number_of_groups))
self.docker_robo('parallel:merge-xmlresults %s' % (number_of_groups))
else:
message('Seems like there is no group files created.', 'info')
run_cmd('docker stop selenium-test-%s' % self.docker.base_alias)
run_cmd('docker rm selenium-test-%s' % self.docker.base_alias)
def add_entry_to_etc_hosts(self):
try:
with open('/etc/hosts', 'rt') as f:
host_names = self.docker_get_compose_option_value(
'web', 'VIRTUAL_HOST', 'environment').replace(',', ' ')
phpmyadmin_host = self.docker_get_compose_option_value(
'phpmyadmin', 'VIRTUAL_HOST',
'environment').replace(',', ' ')
if phpmyadmin_host:
host_names += ' ' + phpmyadmin_host
s = f.read()
if not host_names in s:
s += "\n%s\t\t%s\n" % ('127.0.0.1', host_names)
with open('/tmp/etc_hosts.tmp', 'wt') as outf:
outf.write(s)
run_cmd('sudo mv /tmp/etc_hosts.tmp /etc/hosts')
message(
"Config entry with hosts: >>>%s<<< has beed succesfully added to /etc/hosts file."
% host_names, 'info')
else:
message(
"Config entry with hosts: >>>%s<<< was not added to /etc/hosts file because it already contains entry with this hosts"
% host_names, 'info')
except:
message('Config entry was not added to /etc/hosts.', 'warning')
def add_entry_to_etc_hosts(self):
try:
nginx_proxy_ip = self.get_nginx_proxy_ip()
if not nginx_proxy_ip:
raise Exception
with open('/etc/hosts', 'rt') as f:
host_names = self.docker_get_compose_option_value('web', 'VIRTUAL_HOST', 'environment').replace(',', ' ')
phpmyadmin_host = self.docker_get_compose_option_value('phpmyadmin', 'VIRTUAL_HOST', 'environment').replace(',', ' ')
if phpmyadmin_host:
host_names += ' ' + phpmyadmin_host
s = f.read()
if not host_names in s:
s += "\n%s\t\t%s\n" % (nginx_proxy_ip, host_names)
with open('/tmp/etc_hosts.tmp', 'wt') as outf:
outf.write(s)
run_cmd('sudo mv /tmp/etc_hosts.tmp /etc/hosts')
message("Config entry with hosts: >>>%s<<< has beed succesfully added to /etc/hosts file." % host_names, 'info')
else:
message("Config entry with hosts: >>>%s<<< was not added to /etc/hosts file because it already contains entry with this hosts" % host_names, 'info')
except:
message('Config entry was not added to /etc/hosts.', 'warning')
def drush_drop_tables(self):
message('Drop Tables')
self.run('sql-drop -y')
def drush_drop_tables(self):
message('Drop Tables')
self.run('sql-drop -y')
if arg.startswith('@'):
try:
DOCKER_RUN_PATH = aliases[arg.lstrip('@')]['path'].rstrip('/')
except:
pass
del args[idx]
del sys.argv[idx]
if cmd_options.docker_run_path is not None:
DOCKER_RUN_PATH = cmd_options.docker_run_path.rstrip('/')
sys.path.append(os.path.join(DOCKER_RUN_PATH, 'docker_console'))
sys.path.append(os.path.join(OS_USER_HOME_PATH, '.docker_console'))
try:
from dc_overrides import *
except Exception as exception:
if "No module named dc_overrides" in str(exception):
if os.path.exists(
os.path.join(DOCKER_RUN_PATH, 'docker', 'docker_drupal',
'docker_drupal_overrides.py')):
if len(args) > 0 and args[0] not in global_commands:
message(
"You probably forgot to migrate file %s/docker/docker_drupal/docker_drupal_overrides.py to %s/docker_console/dc_overrides.py"
% (DOCKER_RUN_PATH, DOCKER_RUN_PATH), 'error')
message(
"Use command 'init' to create %s/docker_console/dc_overrides.py file, and then adjust overrides manually."
% DOCKER_RUN_PATH, 'error')
else:
print "Error during dc_overrides file import: ", exception
def _copy_init_tpl_files(self, type):
init_tpl = {
'docker_init': {
'default_init_tpl_dir': 'docker_init_templates',
'custom_init_tpl_dir': 'custom_docker_init_templates',
'info_msg':
'You have to specify docker init template using option --tpl',
'available_msg': 'Available %s docker init templates: %s',
'missing_msg': "Docker init template '%s' does not exists."
},
'tests_init': {
'default_init_tpl_dir': 'tests_init_templates',
'custom_init_tpl_dir': 'custom_tests_init_templates',
'info_msg':
'You have to specify tests init template using option --tpl',
'available_msg': 'Available %s tests init templates: %s',
'missing_msg': "Tests init template '%s' does not exists."
}
}
default_init_tpl_path = os.path.join(
os.path.dirname(sys.modules['docker_console'].__file__),
init_tpl[type]['default_init_tpl_dir'])
custom_init_tpl_path = os.path.join(
os.path.expanduser('~'), '.docker_console',
init_tpl[type]['custom_init_tpl_dir'])
all_default_templates = []
all_custom_templates = []
for item in os.listdir(default_init_tpl_path):
if os.path.isdir(os.path.join(default_init_tpl_path, item)):
all_default_templates.append(item)
if os.path.exists(custom_init_tpl_path):
for item in os.listdir(custom_init_tpl_path):
if os.path.isdir(os.path.join(custom_init_tpl_path, item)):
all_custom_templates.append(item)
if not cmd_options.init_template:
message(init_tpl[type]['info_msg'], 'error')
message(
init_tpl[type]['available_msg'] %
('default', ', '.join(all_default_templates)), 'error')
if len(all_custom_templates):
message(
init_tpl[type]['available_msg'] %
('custom', ', '.join(all_custom_templates)), 'error')
exit(0)
if cmd_options.init_template and cmd_options.init_template not in all_default_templates and cmd_options.init_template not in all_custom_templates:
message(init_tpl[type]['missing_msg'] % cmd_options.init_template,
'error')
message(
init_tpl[type]['available_msg'] %
('default', ', '.join(all_default_templates)), 'error')
if len(all_custom_templates):
message(
init_tpl[type]['available_msg'] %
('custom', ', '.join(all_custom_templates)), 'error')
exit(0)
if cmd_options.init_template in all_custom_templates:
temp_path = create_dir_copy(
os.path.join(custom_init_tpl_path, cmd_options.init_template))
else:
temp_path = create_dir_copy(
os.path.join(default_init_tpl_path, cmd_options.init_template))
for root, dirs, files in os.walk(temp_path):
for name in files:
if name.endswith('-tpl'):
src = os.path.join(root, name)
self._init_tpl_render(src)
create_dir_copy(temp_path, self.config.BUILD_PATH,
cmd_options.docker_init_replace_conf)
dir_util.remove_tree(temp_path)
def docker_update_images(self):
run_cmd('docker-compose stop', cwd=self.config.BUILD_PATH)
run_cmd('docker-compose rm -f', cwd=self.config.BUILD_PATH)
run_cmd('docker-compose pull', cwd=self.config.BUILD_PATH)
run_cmd('docker-compose build --pull', cwd=self.config.BUILD_PATH)
ALL_DEV_DOCKER_IMAGES = []
for image in self.config.DEV_DOCKER_IMAGES:
if isinstance(self.config.DEV_DOCKER_IMAGES[image], tuple):
ALL_DEV_DOCKER_IMAGES.append(self.config.DEV_DOCKER_IMAGES[image])
elif isinstance(self.config.DEV_DOCKER_IMAGES[image], list):
ALL_DEV_DOCKER_IMAGES += self.config.DEV_DOCKER_IMAGES[image]
for image in self.config.TESTS['IMAGES']:
if isinstance(self.config.TESTS['IMAGES'][image], tuple):
ALL_DEV_DOCKER_IMAGES.append(self.config.TESTS['IMAGES'][image])
elif isinstance(self.config.TESTS['IMAGES'][image], list):
ALL_DEV_DOCKER_IMAGES += self.config.TESTS['IMAGES'][image]
custom_images_hashes_path = os.path.join(self.config.BUILD_PATH, 'docker', 'custom_images', 'hashes.yml')
hashes_content = {}
if os.path.exists(custom_images_hashes_path):
custom_images_hashes = open(custom_images_hashes_path)
hashes_content = yaml.load(custom_images_hashes)
custom_images_hashes.close()
if type(hashes_content) is not dict:
hashes_content = {}
for DEV_DOCKER_IMAGE, DEV_DOCKER_IMAGE_DOCKERFILE_PATH in ALL_DEV_DOCKER_IMAGES:
if DEV_DOCKER_IMAGE_DOCKERFILE_PATH is not None:
needs_rebuild = False
dockerfile = open(DEV_DOCKER_IMAGE_DOCKERFILE_PATH + '/Dockerfile')
dockerfile_content = dockerfile.read()
# Pull image that custom images inherits from and get ID of that image.
base_image_match = re.search('FROM (.+)', dockerfile_content)
images_id = ''
if base_image_match is not None:
base_image = base_image_match.group(1)
message('Pulling image %s that is used as base image in %s custom image.' % (base_image, DEV_DOCKER_IMAGE), 'info')
run_cmd('docker pull %s' % base_image)
image_id = self.get_image_id(base_image)
# Create hash from base image ID and Dockerfile content and check if it has changed, if yes rebuild it.
# This also handles the situation when base image will be updated manually
# or from other project thanks to base image id checking.
dockerfile_hash = hashlib.md5(image_id + dockerfile_content).hexdigest()
if not needs_rebuild and DEV_DOCKER_IMAGE not in hashes_content:
message('Hash for image %s not found in %s. '
'Custom image will be rebuilt as we can\'t check whether it is up to date or not.'
% (DEV_DOCKER_IMAGE, custom_images_hashes_path)
, 'info')
needs_rebuild = True
if not needs_rebuild and hashes_content[DEV_DOCKER_IMAGE] != dockerfile_hash:
message('Docker file for image %s was changed or base image was updated so image will be rebuilt.' % DEV_DOCKER_IMAGE, 'info')
needs_rebuild = True
hashes_content[DEV_DOCKER_IMAGE] = dockerfile_hash
if needs_rebuild:
run_cmd('docker build --no-cache --pull -t %s %s' %
(DEV_DOCKER_IMAGE, os.path.join(self.config.BUILD_PATH, DEV_DOCKER_IMAGE_DOCKERFILE_PATH)),
cwd=self.config.BUILD_PATH)
else:
message('Image %s is up to date and will not be rebuilt.' % DEV_DOCKER_IMAGE, 'info')
else:
run_cmd('docker pull %s' % DEV_DOCKER_IMAGE)
if len(hashes_content.keys()) > 0:
custom_images_hashes = open(custom_images_hashes_path, 'w')
yaml.dump(hashes_content, custom_images_hashes, default_flow_style=False)
custom_images_hashes.close()
def docker_update_images(self):
run_cmd('docker-compose stop', cwd=self.config.BUILD_PATH)
run_cmd('docker-compose rm -f', cwd=self.config.BUILD_PATH)
run_cmd('docker-compose pull', cwd=self.config.BUILD_PATH)
run_cmd('docker-compose build --pull', cwd=self.config.BUILD_PATH)
ALL_DEV_DOCKER_IMAGES = []
for image in self.config.DEV_DOCKER_IMAGES:
if isinstance(self.config.DEV_DOCKER_IMAGES[image], tuple):
ALL_DEV_DOCKER_IMAGES.append(
self.config.DEV_DOCKER_IMAGES[image])
elif isinstance(self.config.DEV_DOCKER_IMAGES[image], list):
ALL_DEV_DOCKER_IMAGES += self.config.DEV_DOCKER_IMAGES[image]
for image in self.config.TESTS['IMAGES']:
if isinstance(self.config.TESTS['IMAGES'][image], tuple):
ALL_DEV_DOCKER_IMAGES.append(
self.config.TESTS['IMAGES'][image])
elif isinstance(self.config.TESTS['IMAGES'][image], list):
ALL_DEV_DOCKER_IMAGES += self.config.TESTS['IMAGES'][image]
custom_images_hashes_path = os.path.join(self.config.BUILD_PATH,
'docker', 'custom_images',
'hashes.yml')
hashes_content = {}
if os.path.exists(custom_images_hashes_path):
custom_images_hashes = open(custom_images_hashes_path)
hashes_content = yaml.load(custom_images_hashes)
custom_images_hashes.close()
if type(hashes_content) is not dict:
hashes_content = {}
for DEV_DOCKER_IMAGE, DEV_DOCKER_IMAGE_DOCKERFILE_PATH in ALL_DEV_DOCKER_IMAGES:
if DEV_DOCKER_IMAGE_DOCKERFILE_PATH is not None:
needs_rebuild = False
dockerfile = open(DEV_DOCKER_IMAGE_DOCKERFILE_PATH +
'/Dockerfile')
dockerfile_content = dockerfile.read()
# Pull image that custom images inherits from and get ID of that image.
base_image_match = re.search('FROM (.+)', dockerfile_content)
images_id = ''
if base_image_match is not None:
base_image = base_image_match.group(1)
message(
'Pulling image %s that is used as base image in %s custom image.'
% (base_image, DEV_DOCKER_IMAGE), 'info')
run_cmd('docker pull %s' % base_image)
image_id = self.get_image_id(base_image)
# Create hash from base image ID and Dockerfile content and check if it has changed, if yes rebuild it.
# This also handles the situation when base image will be updated manually
# or from other project thanks to base image id checking.
dockerfile_hash = hashlib.md5(image_id +
dockerfile_content).hexdigest()
if not needs_rebuild and DEV_DOCKER_IMAGE not in hashes_content:
message(
'Hash for image %s not found in %s. '
'Custom image will be rebuilt as we can\'t check whether it is up to date or not.'
% (DEV_DOCKER_IMAGE, custom_images_hashes_path),
'info')
needs_rebuild = True
if not needs_rebuild and hashes_content[
DEV_DOCKER_IMAGE] != dockerfile_hash:
message(
'Docker file for image %s was changed or base image was updated so image will be rebuilt.'
% DEV_DOCKER_IMAGE, 'info')
needs_rebuild = True
hashes_content[DEV_DOCKER_IMAGE] = dockerfile_hash
if needs_rebuild:
run_cmd('docker build --no-cache --pull -t %s %s' %
(DEV_DOCKER_IMAGE,
os.path.join(self.config.BUILD_PATH,
DEV_DOCKER_IMAGE_DOCKERFILE_PATH)),
cwd=self.config.BUILD_PATH)
else:
message(
'Image %s is up to date and will not be rebuilt.' %
DEV_DOCKER_IMAGE, 'info')
else:
run_cmd('docker pull %s' % DEV_DOCKER_IMAGE)
if len(hashes_content.keys()) > 0:
custom_images_hashes = open(custom_images_hashes_path, 'w')
yaml.dump(hashes_content,
custom_images_hashes,
default_flow_style=False)
custom_images_hashes.close()
def _copy_init_tpl_files(self, type):
init_tpl = {
'docker_init': {
'default_init_tpl_dir': 'docker_init_templates',
'custom_init_tpl_dir': 'custom_docker_init_templates',
'info_msg': 'You have to specify docker init template using option --tpl',
'available_msg': 'Available %s docker init templates: %s',
'missing_msg': "Docker init template '%s' does not exists."
},
'tests_init': {
'default_init_tpl_dir': 'tests_init_templates',
'custom_init_tpl_dir': 'custom_tests_init_templates',
'info_msg': 'You have to specify tests init template using option --tpl',
'available_msg': 'Available %s tests init templates: %s',
'missing_msg': "Tests init template '%s' does not exists."
}
}
default_init_tpl_path = os.path.join(os.path.dirname(sys.modules['docker_console'].__file__), init_tpl[type]['default_init_tpl_dir'])
custom_init_tpl_path = os.path.join(os.path.expanduser('~'), '.docker_console', init_tpl[type]['custom_init_tpl_dir'])
all_default_templates = []
all_custom_templates = []
for item in os.listdir(default_init_tpl_path):
if os.path.isdir(os.path.join(default_init_tpl_path, item)):
all_default_templates.append(item)
if os.path.exists(custom_init_tpl_path):
for item in os.listdir(custom_init_tpl_path):
if os.path.isdir(os.path.join(custom_init_tpl_path, item)):
all_custom_templates.append(item)
if not cmd_options.init_template:
message(init_tpl[type]['info_msg'], 'error')
message(init_tpl[type]['available_msg'] % ('default', ', '.join(all_default_templates)), 'error')
if len(all_custom_templates):
message(init_tpl[type]['available_msg'] % ('custom', ', '.join(all_custom_templates)), 'error')
exit(0)
if cmd_options.init_template and cmd_options.init_template not in all_default_templates and cmd_options.init_template not in all_custom_templates:
message(init_tpl[type]['missing_msg'] % cmd_options.init_template, 'error')
message(init_tpl[type]['available_msg'] % ('default', ', '.join(all_default_templates)), 'error')
if len(all_custom_templates):
message(init_tpl[type]['available_msg'] % ('custom', ', '.join(all_custom_templates)), 'error')
exit(0)
if cmd_options.init_template in all_custom_templates:
temp_path = create_dir_copy(os.path.join(custom_init_tpl_path, cmd_options.init_template))
else:
temp_path = create_dir_copy(os.path.join(default_init_tpl_path, cmd_options.init_template))
for root, dirs, files in os.walk(temp_path):
for name in files:
if name.endswith('-tpl'):
src = os.path.join(root, name)
self._init_tpl_render(src)
create_dir_copy(temp_path, self.config.BUILD_PATH, cmd_options.docker_init_replace_conf)
dir_util.remove_tree(temp_path)
config = None
try:
config = import_module('dc_settings')
config.global_commands = global_commands
config.WEB['APP_ROOT'] = os.path.realpath(
os.path.join(config.BUILD_PATH, config.WEB['APP_LOCATION']))
except Exception as exception:
if "No module named dc_settings" in str(exception):
if os.path.exists(
os.path.join(DOCKER_RUN_PATH, 'docker', 'docker_drupal',
'docker_drupal_config_overrides.py')):
if len(args) > 0 and args[0] not in global_commands:
message(
"You probably forgot to migrate file %s/docker/docker_drupal/docker_drupal_config_overrides.py to %s/docker_console/dc_settings.py"
% (DOCKER_RUN_PATH, DOCKER_RUN_PATH), 'error')
message(
"%s/docker_console/dc_settings.py file is required and must contain WEB, DB, DEV_DOCKER_IMAGES and engine specific settings."
% DOCKER_RUN_PATH, 'error')
message(
"Settings in docker_drupal_config_overrides.py and dc_settings.py are not compatibile, the best solution is to use command 'init' and then adjust default config manually.",
'error')
exit(0)
elif len(args) > 0 and args[0] not in global_commands:
message(
"Missing %s/docker_console/dc_settings.py file. This file is required and must contain WEB, DB, DEV_DOCKER_IMAGES and engine specific settings. Use command 'init' and then adjust default config manually."
% DOCKER_RUN_PATH, 'error')
exit(0)
else:
print "Error during dc_settings file import: ", exception
DOCKER_RUN_PATH = os.getcwd()
OS_USER_HOME_PATH = os.path.expanduser('~')
for idx, arg in enumerate(args):
if arg.startswith('@'):
try:
DOCKER_RUN_PATH = aliases[arg.lstrip('@')]['path'].rstrip('/')
except:
pass
del args[idx]
del sys.argv[idx]
if cmd_options.docker_run_path is not None:
DOCKER_RUN_PATH = cmd_options.docker_run_path.rstrip('/')
sys.path.append(os.path.join(DOCKER_RUN_PATH, 'docker_console'))
sys.path.append(os.path.join(OS_USER_HOME_PATH, '.docker_console'))
try:
from dc_overrides import *
except Exception as exception:
if "No module named dc_overrides" in str(exception):
if os.path.exists(os.path.join(DOCKER_RUN_PATH, 'docker', 'docker_drupal', 'docker_drupal_overrides.py')):
if len(args) > 0 and args[0] not in global_commands:
message("You probably forgot to migrate file %s/docker/docker_drupal/docker_drupal_overrides.py to %s/docker_console/dc_overrides.py" % (DOCKER_RUN_PATH, DOCKER_RUN_PATH), 'error')
message("Use command 'init' to create %s/docker_console/dc_overrides.py file, and then adjust overrides manually." % DOCKER_RUN_PATH, 'error')
else:
print "Error during dc_overrides file import: ", exception
try:
from docker_console import commands_overrides
except:
pass
config = None
try:
config = import_module('dc_settings')
config.global_commands = global_commands
config.WEB['APP_ROOT'] = os.path.realpath(os.path.join(config.BUILD_PATH, config.WEB['APP_LOCATION']))
except Exception as exception:
if "No module named dc_settings" in str(exception):
if os.path.exists(os.path.join(DOCKER_RUN_PATH, 'docker', 'docker_drupal', 'docker_drupal_config_overrides.py')):
if len(args) > 0 and args[0] not in global_commands:
message("You probably forgot to migrate file %s/docker/docker_drupal/docker_drupal_config_overrides.py to %s/docker_console/dc_settings.py" % (DOCKER_RUN_PATH, DOCKER_RUN_PATH), 'error')
message("%s/docker_console/dc_settings.py file is required and must contain WEB, DB, DEV_DOCKER_IMAGES and engine specific settings." % DOCKER_RUN_PATH, 'error')
message("Settings in docker_drupal_config_overrides.py and dc_settings.py are not compatibile, the best solution is to use command 'init' and then adjust default config manually.", 'error')
exit(0)
elif len(args) > 0 and args[0] not in global_commands:
message("Missing %s/docker_console/dc_settings.py file. This file is required and must contain WEB, DB, DEV_DOCKER_IMAGES and engine specific settings. Use command 'init' and then adjust default config manually." % DOCKER_RUN_PATH, 'error')
exit(0)
else:
print "Error during dc_settings file import: ", exception
exit(0)
engine_loaded = False
if config and hasattr(config, 'WEB') and 'USE_CUSTOM_ENGINE' in config.WEB and config.WEB['USE_CUSTOM_ENGINE']:
try:
# try to load custom web engine from user home dir
