def _render_one_template(cls, env, template, result_filename, data, overwrite):
# Get a template instance
tpl = None
try:
logger.debug('Using template file: {0}'.format(template))
tpl = env.get_template(template)
except jinja2.TemplateNotFound as e:
raise exceptions.CommandException('Template {t} not found in path {p}.'.\
format(t=template, p=env.loader.searchpath))
except jinja2.TemplateError as e:
raise exceptions.CommandException('Template file failure: {0}'.format(e.message))
# Check if destination file exists, overwrite if needed
if os.path.exists(result_filename):
if overwrite:
logger.info('Overwriting the destination file {0}'.format(result_filename))
os.remove(result_filename)
else:
raise exceptions.CommandException('The destination file already exists: {0}'.\
format(result_filename))
# Generate an output file finally...
with open(result_filename, 'w') as out:
result = tpl.render(**data)
out.write(result)
return (True, 'success')
def _get_docker_run_args(cls, inp):
if not isinstance(inp, dict):
raise exceptions.CommandException('docker_r expects mapping as input.')
if not 'image' in inp:
raise exceptions.CommandException('docker_r requires "image" argument.')
return {'image': inp['image'], 'args': inp.get('args', '')}
def check_args(cls, c):
if c.comm_type == 'dda_w':
if not isinstance(c.input_res, list) or len(c.input_res) != 2:
msg = 'dda_w expects Yaml list with path to .devassistant and mapping to write.'
raise exceptions.CommandException(msg)
else:
if not isinstance(c.input_res, six.string_types):
msg = '{0} expects a string as an argument.'.format(c.comm_type)
raise exceptions.CommandException(msg)
def run(cls, c):
if c.input_res and not isinstance(c.input_res, dict):
raise exceptions.CommandException('{0} needs a mapping as input!'.format(c.comm_type))
if c.comm_type == 'ask_password':
res = DialogHelper.ask_for_password(**c.input_res)
elif c.comm_type == 'ask_confirm':
res = DialogHelper.ask_for_confirm_with_message(**c.input_res)
else:
raise exceptions.CommandException('Unknown command type {ct}.'.format(ct=c.comm_type))
return bool(res), res
def run(cls, c):
if c.comm_type in map(lambda x: 'log_{0}'.format(x), settings.LOG_LEVELS_MAP):
logger.log(settings.LOG_SHORT_TO_NUM_LEVEL[c.comm_type[-1]], c.input_res)
if c.comm_type[-1] in 'ce':
e = exceptions.CommandException(c.input_res)
e.already_logged = True
raise e
else:
raise exceptions.CommandException('Unknown command type {ct}.'.format(ct=c.comm_type))
return [True, c.input_res]
def run(cls, c):
"""Only users in "docker" group can use docker; there are three possible situations:
1) user is not added to docker group => we need to add him there and then go to 2)
2) user has been added to docker group, but would need to log out for it to
take effect => use "newgrp" (_cmd_for_newgrp) for all docker commands
3) user has been added to docker group in a previous login session => all ok
"""
if not cls._docker_group_active() and not cls._docker_group_added():
# situation 1
cls._docker_group_add()
# else situation 3
if not cls._docker_service_running():
cls._docker_service_enable_and_run()
if c.comm_type == 'docker_build':
# TODO: allow providing another argument - a repository name/tag for the built image
ret = cls._docker_build(c.input_res)
elif c.comm_type == 'docker_run':
ret = cls._docker_run(c.input_res)
elif c.comm_type == 'docker_attach':
ret = cls._docker_attach(c.input_res)
elif c.comm_type == 'docker_find_img':
ret = cls._docker_find_image(c.input_res)
elif c.comm_type == 'docker_container_ip':
ret = cls._docker_get_container_attr('{{.NetworkSettings.IPAddress}}', c.input_res)
elif c.comm_type == 'docker_container_name':
ret = cls._docker_get_container_attr('{{.Name}}', c.input_res)
else:
raise exceptions.CommandException('Unknown command type {ct}.'.format(ct=c.comm_type))
return ret
def run(cls, c):
assistant = c.kwargs['__assistant__']
kwargs = copy.deepcopy(c.kwargs)
try:
yaml_name, section_name = c.input_res.rsplit('.', 1)
except ValueError:
raise exceptions.CommandException('"use" command expects "use: what.which_section".')
# Modify kwargs based on command
if cls.is_snippet_call(c.input_res):
snip = cls.get_snippet(yaml_name)
section = cls.get_snippet_section(section_name, snip)
kwargs['__files__'].append(snip.get_files_section())
kwargs['__files_dir__'].append(snip.get_files_dir())
kwargs['__sourcefiles__'].append(snip.path)
else:
assistant = cls.get_assistant(yaml_name, section_name, assistant)
section = cls.get_assistant_section(section_name, assistant)
kwargs['__assistant__'] = assistant
# Get section with modified kwargs
if section_name.startswith('dependencies'):
result = lang.dependencies_section(section, kwargs, runner=assistant)
else:
result = lang.run_section(section, kwargs, runner=assistant)
return result
def run(cls, c):
"""Arguments given to 'github' command may be:
- Just a string (action), which implies that all the other arguments are deducted from
global context and local system.
- List containing a string (action) as a first item and rest of the args in a dict.
(args not specified in the dict are taken from global context.
Possible arguments:
- login - taken from 'github' or system username - represents Github login
- reponame - taken from 'name' (first applies os.path.basename) - repo to operate on
"""
comm, kwargs = cls.format_args(c)
if not cls._gh_module:
logger.warning('PyGithub not installed, cannot execute github command.')
return [False, '']
# we pass arguments as kwargs, so that the auth decorator can easily query them
# NOTE: these are not the variables from global context, but rather what
# cls.format_args returned
if comm == 'create_repo':
ret = cls._github_create_repo(**kwargs)
elif comm == 'push':
ret = cls._github_push()
elif comm == 'create_and_push':
ret = cls._github_create_and_push(**kwargs)
elif comm == 'add_remote_origin':
ret = cls._github_add_remote_origin(**kwargs)
elif comm == 'create_fork':
ret = cls._github_fork(**kwargs)
else:
raise exceptions.CommandException('Unknown command type {ct}.'.format(ct=c.comm_type))
return ret
def run(self):
for cr in type(self).load_command_runners().command_runners:
if cr.matches(self):
return cr.run(self)
raise exceptions.CommandException('No runner for command "{ct}: {c}".'.\
format(ct=self.comm_type, c=self.input_res))
def _get_github_user(cls, login):
if not cls._user:
try:
# try logging with token
token = cls._github_token(login)
gh = cls._gh_module.Github(login_or_token=token)
cls._user = gh.get_user()
# try if the authentication was successful
cls._user.login
except cls._gh_module.GithubException:
# if the token was set, it was wrong, so make sure it's reset
cls._token = None
# login with username/password
password = DialogHelper.ask_for_password(
prompt='Github Password for {username}:'.format(username=login))
gh = cls._gh_module.Github(login_or_token=login, password=password)
cls._user = gh.get_user()
try:
cls._user.login
cls._github_create_auth() # create auth for future use
except cls._gh_module.GithubException as e:
msg = 'Wrong username or password\nGitHub exception: {0}'.format(e)
# reset cls._user to None, so that we don't use it if calling this multiple times
cls._user = None
raise exceptions.CommandException(msg)
return cls._user
def _github_create_ssh_key(cls):
"""Creates a local ssh key, if it doesn't exist already, and uploads it to Github."""
try:
login = cls._user.login
pkey_path = '{home}/.ssh/{keyname}'.format(
home=os.path.expanduser('~'),
keyname=settings.GITHUB_SSH_KEYNAME.format(login=login))
# generate ssh key only if it doesn't exist
if not os.path.exists(pkey_path):
ClHelper.run_command('ssh-keygen -t rsa -f {pkey_path}\
-N \"\" -C \"DevAssistant\"'.format(
pkey_path=pkey_path))
try:
ClHelper.run_command(
'ssh-add {pkey_path}'.format(pkey_path=pkey_path))
except exceptions.ClException:
# ssh agent might not be running
env = cls._start_ssh_agent()
ClHelper.run_command(
'ssh-add {pkey_path}'.format(pkey_path=pkey_path), env=env)
public_key = ClHelper.run_command(
'cat {pkey_path}.pub'.format(pkey_path=pkey_path))
cls._user.create_key("DevAssistant", public_key)
except exceptions.ClException as e:
msg = 'Couldn\'t create a new ssh key: {0}'.format(e)
raise exceptions.CommandException(msg)
def docker_service_enable_and_run(cls):
# TODO: add some conditionals for various platforms
logger.info('Enabling and running docker service ...')
try:
cmd_str = 'bash -c "systemctl enable docker && systemctl start docker"'
ClHelper.run_command(cmd_str, as_user='******')
except exceptions.ClException:
raise exceptions.CommandException(
'Failed to enable and run docker service.')
# we need to wait until /var/run/docker.sock is created
# let's wait for 30 seconds
logger.info(
'Waiting for /var/run/docker.sock to be created (max 15 seconds) ...'
)
success = False
for i in range(0, 30):
time.sleep(i * 0.5)
try:
ClHelper.run_command('ls /var/run/docker.sock')
success = True
break
except exceptions.ClException:
pass
if not success:
logger.warning(
'/var/run/docker.sock doesn\'t exist, docker will likely not work!'
)
def run(cls, c):
if not isinstance(c.input_res, list):
msg = 'Dependencies for installation must be list, got {v}.'.format(v=c.input_res)
raise exceptions.CommandException(msg)
di = DependencyInstaller()
di.install(c.input_res)
return [True, c.input_res]
def __dot_devassistant_read_exact(cls, directory):
"""Helper for other methods that read .devassistant file."""
dda_path = os.path.join(os.path.abspath(os.path.expanduser(directory)), '.devassistant')
try:
with open(dda_path, 'r') as stream:
return yaml.load(stream)
except IOError as e:
msg = 'Couldn\'t find/open/read .devassistant file: {0}'.format(e)
raise exceptions.CommandException(msg)
def get_assistant(cls, assistant_name, section_name, origin_assistant):
if assistant_name == 'self':
if not hasattr(origin_assistant, '_' + section_name):
raise exceptions.CommandException('Assistant "{a}" has no section "{s}"'.\
format(a=origin_assistant.name,
s=section_name))
return origin_assistant
elif assistant_name == 'super':
a = origin_assistant.superassistant
while a:
if hasattr(a, 'assert_fully_loaded'):
a.assert_fully_loaded()
if hasattr(a, '_' + section_name):
return a
a = a.superassistant
raise exceptions.CommandException('No superassistant of {a} has section {s}'.\
format(a=origin_assistant.name,
s=section_name))
def add_user_to_docker_group(cls, username):
try:
logger.info('Adding {0} to group docker ...'.format(username))
ClHelper.run_command(
'bash -c "usermod -a -G docker {0}"'.format(username),
as_user='******')
except exceptions.ClException as e:
msg = 'Failed to add user to "docker" group: {0}'.format(e.output)
raise exceptions.CommandException(msg)
def _try_obtain_common_params(cls, comm):
""" Retrieve parameters common for all jinja_render* actions from Command instance.
These are mandatory:
- 'template' template descriptor from `files' section. it consist of
the only `source' key -- a name of template to use
- 'data' dict of parameters to use when rendering
- 'destination' path for output files
These are optional:
- 'overwrite' overwrite file(s) if it (they) exist(s)
"""
args = comm.input_res
ct = comm.comm_type
wrong_tpl_msg = '{0} requires a "template" argument which must point to a file'.format(ct)
wrong_tpl_msg += ' in "files" section. Got: {0}'.format(args.get('template', None))
if 'template' not in args or not isinstance(args['template'], dict):
raise exceptions.CommandException(wrong_tpl_msg)
template = args['template']
if 'source' not in template or not isinstance(template['source'], six.string_types):
raise exceptions.CommandException(wrong_tpl_msg)
template = template['source']
if 'destination' not in args or not isinstance(args['destination'], six.string_types):
msg = '{0} requires a string "destination" argument. Got: {1}'.\
format(ct, args.get('destination'))
raise exceptions.CommandException(msg)
destination = args['destination']
if not os.path.isdir(destination):
msg = '{0}: Specified "destination" directory "{1}" doesn\'t exist!'.\
format(ct, destination)
raise exceptions.CommandException(msg)
data = {}
if 'data' in args and isinstance(args['data'], dict):
data = args['data']
logger.debug('Template context data: {0}'.format(data))
overwrite = args.get('overwrite', False)
overwrite = True if str(overwrite).lower() in ['true', 'yes'] else False
return (template, destination, data, overwrite)
def get_snippet_section(cls, section_name, snip):
if section_name.startswith('run'):
section = snip.get_run_section(section_name) if snip else None
else:
section = snip.get_dependencies_section(section_name) if snip else None
if not section:
raise exceptions.CommandException('Couldn\'t find section "{t}" in snippet "{n}".'.\
format(t=section_name, n=snip.dotted_name))
return section
def run(cls, c):
args = cls._get_args(c.input_res, c.kwargs)
contdir, topdir = os.path.split(args['from'])
normalized_topdir = lang.Command('normalize', topdir).run()[1]
try: # ok, this is a bit ugly, but we need to check multiple calls for the exception
if contdir: # we need to create containing directory
if not args['accept_path']:
msg = 'Path is not accepted as project name by this assistant (got "{0}")'
raise exceptions.CommandException(msg.format(args['from']))
if not os.path.exists(contdir):
os.makedirs(contdir)
elif not os.path.isdir(contdir):
msg = 'Can\'t create subdirectory in "{0}", it\'s not a directory'.\
format(contdir)
raise exceptions.CommandException(msg)
actual_topdir = normalized_topdir if args['create_topdir'] == 'normalized' else topdir
topdir_fullpath = os.path.join(contdir, actual_topdir)
if args['create_topdir']:
if os.path.exists(topdir_fullpath):
if args['on_existing'] == 'fail':
msg = 'Directory "{0}" already exists, can\'t proceed'.format(topdir_fullpath)
raise exceptions.CommandException(msg)
elif not os.path.isdir(topdir_fullpath):
msg = 'Location "{0}" exists, but is not a directory, can\'t proceed'.\
format(topdir_fullpath)
raise exceptions.CommandException(msg)
else:
os.makedirs(topdir_fullpath)
except OSError as e:
msg = 'Failed to create directory {0}: {1}'.format(args['from'], e.message)
raise CommandException(msg)
# if contdir == '', then return current dir ('.')
c.kwargs[args['contdir_var']] = contdir or '.'
c.kwargs[args['topdir_var']] = topdir
c.kwargs[args['topdir_normalized_var']] = normalized_topdir
return (True, topdir_fullpath if args['create_topdir'] else contdir)
def run(self):
for crs_prefix, crs in type(
self).load_command_runners().command_runners.items():
if self.prefix == crs_prefix:
# traverse in reversed order, so that dynamically loaded user command runners
# can outrun (=> override) the builtin ones
for cr in reversed(crs):
if cr.matches(self):
return cr(self).run()
prefix_with_colon = self.prefix + '.' if self.prefix else self.prefix
raise exceptions.CommandException(
'No runner for command "{p}{ct}: {c}".'.format(p=prefix_with_colon,
ct=self.comm_type,
c=self.input_res))
def _get_args(cls, inp, ctxt):
args = {}
if not isinstance(inp, dict):
raise exceptions.CommandException('"setup_project_dir" expects mapping as input')
args['from'] = inp.get('from', None)
if args['from'] is None:
raise exceptions.CommandException('"setup_project_dir" requires "from" argument')
args['contdir_var'] = inp.get('contdir_var', 'contdir')
args['topdir_var'] = inp.get('topdir_var', 'topdir')
args['topdir_normalized_var'] = inp.get('topdir_normalized_var', 'topdir_normalized')
args['accept_path'] = bool(inp.get('accept_path', True))
args['create_topdir'] = inp.get('create_topdir', True)
if not args['create_topdir'] in [True, False, 'normalized']:
msg = '"setup_project_dir" expects "create_topdir" to be one of: ' +\
'True, False, normalized'
raise exceptions.CommandException(msg)
args['on_existing'] = inp.get('on_existing', 'fail')
if not args['on_existing'] in ['fail', 'pass']:
msg = '"setup_project_dir" expects "on_existing" to be one of: "fail", "pass"'
raise exceptions.CommandException(msg)
return args
def _github_create_ssh_key(cls):
try:
login = cls._user.login
pkey_path = '{home}/.ssh/{keyname}'.format(home=os.path.expanduser('~'),
keyname=settings.GITHUB_SSH_KEYNAME.format(login=login))
# TODO: handle situation where {pkey_path} exists, but it's not registered on GH
# generate ssh key
ClHelper.run_command('ssh-keygen -t rsa -f {pkey_path}\
-N \"\" -C \"DevAssistant\"'.\
format(pkey_path=pkey_path))
ClHelper.run_command('ssh-add {pkey_path}'.format(pkey_path=pkey_path))
public_key = ClHelper.run_command('cat {pkey_path}.pub'.format(pkey_path=pkey_path))
cls._user.create_key("devassistant", public_key)
except exceptions.ClException as e:
msg = 'Couldn\'t create a new ssh key: {e}'.format(e)
raise exceptions.CommandException(msg)
def run(cls, c):
cls.check_args(c)
if c.comm_type == 'dda_c':
cls._dot_devassistant_create(c.input_res, c.kwargs)
elif c.comm_type == 'dda_r':
cls._dot_devassistant_read(c.input_res, c.kwargs)
elif c.comm_type == 'dda_dependencies':
cls._dot_devassistant_dependencies(c.input_res, c.kwargs)
elif c.comm_type == 'dda_run':
cls._dot_devassistant_run(c.input_res, c.kwargs)
elif c.comm_type == 'dda_w':
cls._dot_devassistant_write(c.input_res)
else:
raise exceptions.CommandException('Unknown command type {ct}.'.format(ct=c.comm_type))
return [True, '']
def _guess_reponame(cls, explicit, ctxt):
"""Extract reponame, either from explicitly given string or from 'name' global variable,
which is possibly a path.
Args:
ctxt: global context
Returns:
guessed reponame
"""
name = explicit
if not name:
name = os.path.basename(ctxt.get('name', ''))
if not name:
raise exceptions.CommandException('Cannot guess Github reponame - no argument given'
'and there is no "name" variable.')
return name
def run(cls, c):
"""Normalizes c.input_res (string):
- removes digit from start
- replaces dashes and whitespaces with underscores
"""
to_norm = c.input_res
if not isinstance(to_norm, six.string_types):
raise exceptions.CommandException('"normalize" expects string input, got {0}'.\
format(to_norm))
normalized = to_norm.lstrip('0123456789')
badchars = '-+\\|()[]{}<>,./:\'" \t;`!@#$%^&*'
if sys.version_info[0] < 3:
tt = string.maketrans(badchars, '_' * len(badchars))
else:
tt = str.maketrans(badchars, '_' * len(badchars))
normalized = normalized.translate(tt)
return (True, normalized)
def _guess_repo_url(cls, explicit, ctxt):
"""Get repo to fork in form of '<login>/<reponame>' from explicitly given string or
global variable 'url'.
Args:
ctxt: global context
Returns:
guessed fork reponame
"""
url = explicit or ctxt.get('url')
if not url:
raise exceptions.CommandException('Cannot guess name of Github repo to fork - no'
'argument given and there is no "url" variable.')
url = url[:-4] if url.endswith('.git') else url
# if using git@github:username/reponame.git, strip the stuff before ":"
url = url.split(':')[-1]
return '/'.join(url.split('/')[-2:])
def run(cls, c):
# Transform list of dicts (where keys are unique) into a single dict
args = c.input_res
logger.debug('Jinja2Runner args={0}'.format(repr(args)))
# Create a jinja environment
logger.debug('Using templates dir: {0}'.format(c.files_dir))
env = jinja2.Environment(loader=jinja2.FileSystemLoader(c.files_dir))
env.trim_blocks = True
env.lstrip_blocks = True
template, destination, data, overwrite = cls._try_obtain_common_params(c)
if c.comm_type == 'jinja_render':
given_output = args.get('output', '')
if not isinstance(given_output, six.string_types):
raise exceptions.CommandException('Jinja2Runner: output must be string, got {0}'.\
format(given_output))
result_fn = cls._make_output_file_name(destination, template, given_output)
cls._render_one_template(env, template, result_fn, data, overwrite)
elif c.comm_type == 'jinja_render_dir':
cls._render_dir(env, template, destination, data, overwrite)
return (True, 'success')
def get_snippet(cls, yaml_name):
try:
return yaml_snippet_loader.YamlSnippetLoader.get_snippet_by_name(yaml_name)
except exceptions.SnippetNotFoundException as e:
raise exceptions.CommandException(e)
def get_assistant_section(cls, section_name, assistant):
if not hasattr(assistant, '_' + section_name):
raise exceptions.CommandException('Assistant {a} has no section {s}'.\
format(a=assistant.name,
s=section_name))
return getattr(assistant, '_' + section_name)
def get_user_from_comm_type(cls, comm_type):
split_type = comm_type.split()
if len(split_type) != 2:
raise exceptions.CommandException('"as" expects format "as <username>".')
user = split_type[1]
return user
