def __init__(self, conf, prof):
self.col = Colours()
self.conf = conf
self.prof = prof
self.dcyaml = {}
self.profconf = {}
self.names = {}
self.volumes = []
def copy_custom_scripts(cs_conf, basedir, active_app):
col = Colours()
for typ in cs_conf:
for con in cs_conf[typ]:
dockdir = pj(basedir, 'docker', con)
if exists(dockdir) is True:
target_folder = pj(dockdir, 'rootfs', 'tmp', 'custom_scripts',
typ)
source_folder = expand(cs_conf[typ][con], active_app)
if isdir(source_folder) is True:
files = listfiles_only(source_folder)
if len(files) > 0:
print('\nAdd custom scripts to container ' +
col.gre(shortname(dockdir)))
for fil in files:
copy_file(fil, target_folder)
print('')
class DCompose():
def __init__(self, conf, prof):
self.col = Colours()
self.conf = conf
self.prof = prof
self.dcyaml = {}
self.profconf = {}
self.names = {}
self.volumes = []
def expand_vars_arr(self, arr, container_name=None):
for i, el in enumerate(arr):
arr[i] = self.expand_vars(arr[i], container_name)
return arr
def expand_vars(self, string, container_name=None):
if '<' not in string and '>' not in string:
return string
# expand variables set in config
string = string\
.replace('<HOME>', os.environ['HOME'])\
.replace('<ACTIVE_APP>', self.conf['conf']['active_app'])\
.replace('<CONTAINER_PGAPP>', self.nam_con('pgapp'))\
.replace('<CONTAINER_PGDATA>', self.nam_con('pgdata'))\
.replace('<CONTAINER_WPDB>', self.nam_con('wpdb'))\
.replace('<CONTAINER_RABBITMQ>', self.nam_con('rabbitmq'))\
.replace('<UID>', self.conf['user']['idstr'])\
.replace('<GID>', self.conf['user']['groupstr'])
# expand env var placeholders set in other env vars
if container_name is not None:
try:
env_vars = self.conf['conf']['env'][container_name]
except KeyError:
pass
else:
for k in env_vars:
key = str(k)
val = str(env_vars[key])
if key != '' and val != '':
string = string.replace('<' + key.upper() + '>', val)
# add additional packages
if container_name is not None:
try:
p = ' '.join(
self.conf['conf']['additional_packages'][container_name])
except KeyError:
pass
except TypeError:
print(self.col.err() + self.col.yel(
'Please check additional packages config entry for ') +
container_name +
self.col.yel(' and make sure it is a not empty list.'))
print('If you do not wish to use additional packages remove ' +
' or comment the whole list entry.')
sys.exit()
else:
var = '<ADDITIONAL_PACKAGES>'
if var in string:
string = string.replace('<ADDITIONAL_PACKAGES>', p)
string = uncomment_line(string)
return string
def iter_services(self):
return self.conf['conf']['env']
# service and container names
def make_names(self):
for service in self.iter_services():
self.names[service] = {}
self.names[service]['con'] =\
'dqdev' + '-' + service + '-' + self.conf['prof']['name']
self.names[service]['img'] =\
'dqdev' + '_' + service + '_' + self.conf['prof']['name']
def nam_img(self, service):
return self.names[service]['img']
def nam_con(self, service):
return self.names[service]['con']
def nam_daiq(self):
for service in self.names:
img = self.nam_img(service)
if 'daiquiri' in img:
return img
return None
def container_enabled(self, container_name):
try:
return self.conf['conf']['enable_containers'][container_name]
except KeyError:
return False
# template
def make_template(self):
self.dcyaml['version'] = '3.7'
self.dcyaml['services'] = {}
self.dcyaml['volumes'] = {}
for service in self.iter_services():
if self.container_enabled(service) is True:
c = self.nam_img(service)
self.dcyaml['services'][c] = {}
self.dcyaml['services'][c]['build'] = {}
self.dcyaml['services'][c]['build']['context'] =\
'../../../docker/' + service
self.dcyaml['services'][c]['container_name'] =\
self.nam_con(service)
self.dcyaml['services'][c]['restart'] = 'always'
# depends on
def add_depends_on(self):
self.dcyaml['services'][self.nam_daiq()]['depends_on'] = []
for service in self.dcyaml['services']:
if 'daiquiri' not in service:
self.dcyaml['services'][self.nam_daiq()]['depends_on']\
.append(service)
# env
def add_env(self):
for service in self.iter_services():
try:
env_arr = []
for k in self.conf['conf']['env'][service]:
key = str(k)
val = str(self.conf['conf']['env'][service][key])
env_arr.append(key.upper() + '=' + val)
env = self.expand_vars_arr(env_arr, service)
except KeyError:
pass
else:
try:
p = self.conf['conf']['portmap'][service]
except KeyError:
pass
else:
env.append('EXPOSED_PORT=' + str(p['exposed']))
for mp in self.conf['conf']['docker_volume_mountpoints']:
key = ''.join(re.findall('[A-Z0-9]', mp.upper()))
val = self.conf['conf']['docker_volume_mountpoints'][mp]
env.append(key + '=' + val)
# try because exception occurs when a container is disabled
try:
self.dcyaml['services'][self.nam_img(
service)]['environment'] = env
except KeyError:
pass
# network
def add_networks(self):
nn = self.prof.conf['prof']['network_name']
self.dcyaml['networks'] = {}
self.dcyaml['networks'][nn] = {}
self.dcyaml['networks'][nn]['external'] = {}
self.dcyaml['networks'][nn]['external']['name'] = nn
for service in self.dcyaml['services']:
self.dcyaml['services'][service]['networks'] = [nn]
# ports
def add_ports(self):
for service in self.conf['conf']['portmap']:
try:
self.dcyaml['services'][self.nam_img(service)]['ports'] =\
[self.conf['conf']['portmap'][service]['envstr']]
except KeyError:
pass
# volumes
def add_volumes(self):
for vol in self.volumes:
self.dcyaml['volumes'][vol['name']] = {}
self.dcyaml['volumes'][vol['name']]['driver_opts'] =\
vol['driver_opts']
for service in self.dcyaml['services']:
self.dcyaml['services'][service]['volumes'] = []
for vol in self.volumes:
if rxbool(vol['mount_inside'], service) is True:
self.dcyaml['services'][service]['volumes'].append(
vol['name'] + ':' + self.expand_vars(vol['mp']))
def make_volumes(self):
vols = []
for volname in self.conf['conf']['docker_volume_mountpoints']:
try:
fol = self.conf['conf']['folders_on_host'][volname]
except KeyError:
fol = self.conf['conf']['folders_on_host'][self.conf['conf']
['active_app']]
v = self.make_volume(
volname + '_' + self.profconf['name'],
self.conf['conf']['docker_volume_mountpoints']
[volname], # noqa: E501
fol,
volname.startswith('dq_'))
if self.valid_volume(v) is True:
vols.append(v)
for volname in self.conf['conf']['enable_database_volumes']:
if self.conf['conf']['enable_database_volumes'][
volname] is True: # noqa: E501
volfolder = pj(
self.prof.get_profile_folder_by_name(
self.profconf['name']), volname)
mkdir(volfolder)
mp = '/var/lib/mysql'
if volname.startswith('pg'):
mp = '/var/lib/postgresql/data'
vols.append(
self.make_volume(volname + '_' + self.profconf['name'],
mp,
volfolder,
mount_inside=volname))
self.volumes = vols
def make_volume(self,
volname,
mp,
folder_on_host,
required_git=False,
mount_inside='.*'):
vol = {}
vol['name'] = volname
vol['mp'] = mp
vol['required_git'] = required_git
vol['mount_inside'] = mount_inside
vol['driver_opts'] = {}
vol['driver_opts']['o'] = 'bind'
vol['driver_opts']['type'] = 'none'
vol['driver_opts']['device'] = self.expand_vars(folder_on_host)
return vol
def valid_volume(self, vol):
r = False
dev = vol['driver_opts']['device']
is_dir = os.path.isdir(dev)
if is_dir is False and vol['required_git'] is False:
print('Run without volume ' + self.col.yel(vol['name']) +
'. Path does not exist on host ' + self.col.yel(dev))
if is_dir is True:
r = True
if vol['required_git'] is True:
ig = is_git(dev)
if ig[0] is False:
print('\n' + self.col.err() + 'Folder ' + self.col.yel(dev) +
' does not look like a git repo. ' +
'\nPlease make sure that it contains the source of ' +
self.col.yel(vol['name']) + '\n')
x(1)
else:
r = True
return r
def write_yaml(self):
if self.conf['dry_run'] is True:
print(self.col.yel('\nDry run, dc yaml would look like this:'))
pprint(self.dcyaml)
else:
print('Write dc yaml to ' +
self.col.yel(self.conf['files']['dc_yaml']) + '\n')
write_yaml(self.dcyaml, self.conf['files']['dc_yaml'])
def render_dockerfile_templates(self):
arr = find(self.conf['basedir'], '.*/dockerfile.tpl', 'f')
for fn in arr:
print('Render dockerfile template ' + self.col.yel(fn))
self.render_template_file(fn)
def render_template_file(self, filename):
container_name = rxsearch(r'[a-z0-9A-Z-]+(?=/dockerfile.tpl$)',
filename)
new_filename = rxsearch(r'.*(?=\.)', filename)
arr = []
try:
filecontent = open(filename, 'r')
except Exception as e:
raise (e)
else:
for line in filecontent.read().splitlines():
arr.append(self.expand_vars(line, container_name))
write_array_to_file(arr, new_filename)
# main
def render_dc_yaml(self, profname=None):
self.profconf = self.prof.read_profile_config(profname)
self.make_names()
self.make_template()
self.make_volumes()
self.add_depends_on()
self.add_env()
self.add_ports()
self.add_networks()
self.add_volumes()
self.write_yaml()
'--display_profile',
type=str,
nargs='*',
default=None,
help='display currently active profile')
parser.add_argument(
'-n',
'--dry_run',
action='store_true',
default=False,
help=('do not run any docker-compose commands nor ' +
'save rendered docker-compose.yaml, just print them'))
args = parser.parse_args()
if __name__ == '__main__':
col = Colours()
conf = init(args)
prof = Profile(conf)
dco = DCompose(conf, prof)
if conf['args']['list'] is True:
prof.list()
x()
if args.create_profile is not None:
prof.create(args.create_profile)
if args.set_profile is not None:
prof.set(args.set_profile)
if args.display_profile is not None:
def init(args):
col = Colours()
conf = {}
n = os.path.realpath(__file__)
basedir = '/'.join(n.split('/')[:-2])
conf['basedir'] = basedir
conf['args'] = {}
conf['files'] = {}
conf['prof'] = {}
conf['prof']['basedir'] = pj(conf['basedir'], 'usr', 'profiles')
conf['files']['active_conf'] = pj(conf['prof']['basedir'], 'active.toml')
conf['files']['base_conf'] = pj(basedir, 'conf', 'baseconf.toml')
conf['files']['base_secrets'] = pj(basedir, 'conf', 'secrets.toml')
conf['args']['list'] = True
conf['args']['down'] = parse_nargs(args.down)
conf['args']['remove_network'] = parse_bool(args.remove_network)
conf['args']['remove_images'] = parse_bool(args.remove_images)
conf['args']['render'] = parse_nargs(args.render)
conf['args']['run'] = parse_nargs(args.run)
conf['args']['stop'] = parse_nargs(args.stop)
conf['args']['display_profile'] = parse_nargs(args.display_profile)
conf['args']['tail_logs'] = parse_nargs(args.tail_logs)
conf['args']['set'] = args.set_profile
conf['args']['create'] = args.create_profile
apc = read_toml(conf['files']['active_conf'])
conf['prof']['name'] = ''
if apc is not None:
conf['prof']['name'] = apc['active_profile_name']
for arg in conf['args']:
if arg != 'list':
val = conf['args'][arg]
if val is not None:
conf['args']['list'] = None
if isinstance(val, str):
conf['prof']['name'] = val
break
# read base configurations
print('Read base config ' + col.yel(conf['files']['base_conf']))
base_conf = read_toml(conf['files']['base_conf'])
print('Read base secrets ' + col.yel(conf['files']['base_secrets']))
base_secrets = read_toml(conf['files']['base_secrets'])
base_conf['env'] = merge_dictionaries(base_conf['env'], base_secrets)
conf['conf'] = base_conf
# stop when no active profile was detected
if conf['prof']['name'] == '':
print(
col.red('No profile active. ' +
'Please set one to be able to continue.'))
x()
# read profile configurations
conf['prof']['folder'] = pj(conf['prof']['basedir'], conf['prof']['name'])
conf['prof']['network_name'] = 'dqdevnet_' + conf['prof']['name']
conf['files']['dc_yaml'] = pj(conf['prof']['folder'],
'docker-compose.yaml')
conf['files']['prof_conf'] = pj(conf['prof']['folder'], 'conf.toml')
conf['files']['prof_secrets'] = pj(conf['prof']['folder'], 'secrets.toml')
if conf['args']['set'] is None:
print('\nUse profile ' + col.gre(conf['prof']['name']))
if isfile(conf['files']['prof_conf']) is True:
if conf['args']['set'] is None:
print('Read prof config ' + col.yel(conf['files']['prof_conf']))
prof_conf = read_toml(conf['files']['prof_conf'])
# merge the two
conf['conf'] = merge_dictionaries(conf['conf'], prof_conf)
else:
if args.set_profile is None and args.create_profile is None:
print(
col.red('\nWarning') +
'\n Profile config does not exist: ' +
col.yel(conf['files']['prof_conf']) +
'\n All base settings are going to be applied. ' +
'It is highly likely your setup ' +
'will turn out to be unusable.\n')
if isfile(conf['files']['prof_secrets']) is True:
print('Read prof secrets ' + col.yel(conf['files']['prof_conf']))
prof_secrets = read_toml(conf['files']['prof_secrets'])
conf['conf']['env'] =\
merge_dictionaries(conf['conf']['env'], prof_secrets)
# user settings
conf['user'] = {}
conf['user']['id'] = os.getuid()
conf['user']['idstr'] = str(conf['user']['id'])
conf['user']['group'] = get_group(conf['user']['id'])
conf['user']['groupstr'] = str(conf['user']['group'])
conf['dry_run'] = args.dry_run
mkdir(conf['prof']['basedir'])
clean_temp_files(conf['basedir'], conf['conf']['enable_containers'])
copy_custom_scripts(conf['conf']['custom_scripts'], conf['basedir'],
conf['conf']['active_app'])
create_rootfs_folders(conf['basedir'])
conf['conf']['portmap'] = parse_ports(conf)
del conf['conf']['exposed_ports']
return conf
def __init__(self, conf):
self.c = Colours()
self.conf = conf
self.need_sudo = self.need_sudo()
class Runner():
def __init__(self, conf):
self.c = Colours()
self.conf = conf
self.need_sudo = self.need_sudo()
def run_cmd_fg(self, cmd):
print(self.c.mag(' '.join(cmd)))
if self.conf['dry_run'] is False:
try:
subprocess.run(cmd)
except KeyboardInterrupt:
pass
def need_sudo(self):
g = run_cmd(['groups'])
if bool(re.search('docker', g)) is True:
return False
else:
return True
def file_arg_compose(self):
return ['-f', self.conf['files']['dc_yaml']]
def run_docker(self, args):
cmd_arr = []
if self.need_sudo is True:
cmd_arr.append('sudo')
cmd_arr.append('docker')
cmd_arr.extend(args)
self.run_cmd_fg(cmd_arr)
def run_compose(self, args):
cmd_arr = []
if self.need_sudo is True:
cmd_arr.append('sudo')
cmd_arr.append('docker-compose')
cmd_arr.extend(self.file_arg_compose())
cmd_arr.extend(args)
self.run_cmd_fg(cmd_arr)
# docker compose commands
def start(self):
self.run_compose(['up', '--build', '-d'])
self.tail_logs()
def stop(self):
self.run_compose(['stop'])
def down(self):
args = ['down', '--volumes', '--remove-orphans']
if self.conf['args']['remove_images'] is True:
args.append('--rmi all')
self.run_compose(args)
def tail_logs(self):
self.run_compose(['logs', '-f'])
def remove_images(self):
self.run_compose(['down', '--volume'])
def create_network(self):
self.run_docker([
'network', 'create', self.conf['prof']['network_name']
])
def remove_network(self):
self.run_docker([
'network', 'remove', self.conf['prof']['network_name']
])
def __init__(self, conf):
self.conf = conf
self.c = Colours()
class Profile():
def __init__(self, conf):
self.conf = conf
self.c = Colours()
def create(self, profname):
if self.profile_exists(profname) is True:
print('Please check ' + self.c.yel(self.conf['prof']['basedir']) +
'\nDid nothing. Profile ' + self.c.yel(profname) +
' seems to exist')
else:
mkdir(self.get_profile_folder_by_name(profname))
conf_yaml = pj(self.get_profile_folder_by_name(profname),
'conf.toml')
secrets_yaml = pj(self.get_profile_folder_by_name(profname),
'secrets.toml')
print('Fresh profile ' + self.c.yel(profname) +
' created inside folder ' +
self.c.yel(self.get_profile_folder_by_name(profname)) +
'\nPlease add your local settings to ' +
self.c.yel(conf_yaml) + '\nAnd don\'t forget your secrets.')
copyfile(self.conf['files']['base_conf'], conf_yaml)
copyfile(self.conf['files']['base_secrets'], secrets_yaml)
def set(self, profname):
if self.profile_exists(profname) is False:
print('Unable to set. Profile ' + self.c.yel(profname) +
' does not seem to exist')
else:
print('Set active profile ' + self.c.yel(profname))
p = {}
p['active_profile_name'] = profname
write_toml(p, self.conf['files']['active_conf'])
def read_profile_config(self, profname=None):
if profname is None or profname is True:
profname = self.conf['prof']['name']
if profname is None:
print('Unable to detect active profile. Either set one or use ' +
'the command line arg.')
x(1)
r = {}
f = find(self.conf['prof']['basedir'], profname + r'$', 'd')
if len(f) < 1:
print('Please check ' + self.c.yel(self.conf['prof']['basedir']) +
'\nProfile ' + self.c.yel(profname) +
' does not seem to exist. ')
x(1)
if len(f) > 1:
print('Please check ' + self.c.yel(self.conf['prof']['basedir']) +
'\nMultiple profiles matched: ')
for el in f:
print('\t' + el)
x(1)
r['name'] = profname
r['yaml'] = pj(f[0], 'conf.toml')
r['folder'] = path_up_to_last_slash(f[0])
r['dc_yaml'] = pj(r['folder'], 'docker-compose.yaml')
if isfile(r['yaml']) is True:
r['conf'] = read_toml(r['yaml'])
return r
def boolstr(self, bool):
if bool is True:
return '*'
else:
return ''
def list(self):
print(self.c.yel('\nThe following profiles are available\n'))
arr = find(self.conf['prof']['basedir'],
r'.*/profiles/[a-zA-Z0-9-_]+$', 'd')
head = ['profile', 'has conf', 'active', 'volumes']
tabledata = []
for el in arr:
shortname = rxsearch(r'[^/]+/[^/]+$', el)
profname = rxsearch(r'[^/]+$', shortname)
ap = self.conf['prof']['name']
has_conf = self.boolstr(isfile(pj(el, 'conf.toml')))
active = self.boolstr(profname == ap)
listdirs_only(self.get_profile_folder_by_name(el))
volumes = ' '.join(
listdirs_only(self.get_profile_folder_by_name(el)))
tabledata.append([profname, has_conf, active, volumes])
ptable(head, tabledata)
print()
def get_profile_folder_by_name(self, profname=None):
if profname is None:
profname = self.conf['prof']['name']
return pj(self.conf['prof']['basedir'], profname)
def profile_exists(self, profname):
return isdir(self.get_profile_folder_by_name(profname))