def run(cls, c):
"""Arguments given to 'github' command may be:
- Just a string (action), which implies that all the other arguments are deducted from
global context and local system.
- List containing a string (action) as a first item and rest of the args in a dict.
(args not specified in the dict are taken from global context.
Possible arguments:
- login - taken from 'github' or system username - represents Github login
- reponame - taken from 'name' (first applies os.path.basename) - repo to operate on
"""
comm, kwargs = cls.format_args(c)
if not cls._gh_module:
logger.warning('PyGithub not installed, cannot execute github command.')
return [False, '']
# we pass arguments as kwargs, so that the auth decorator can easily query them
# NOTE: these are not the variables from global context, but rather what
# cls.format_args returned
if comm == 'create_repo':
ret = cls._github_create_repo(**kwargs)
elif comm == 'push':
ret = cls._github_push()
elif comm == 'create_and_push':
ret = cls._github_create_and_push(**kwargs)
elif comm == 'add_remote_origin':
ret = cls._github_add_remote_origin(**kwargs)
elif comm == 'create_fork':
ret = cls._github_fork(**kwargs)
else:
raise exceptions.CommandException('Unknown command type {ct}.'.format(ct=c.comm_type))
return ret
def docker_service_enable_and_run(cls):
# TODO: add some conditionals for various platforms
logger.info('Enabling and running docker service ...')
try:
cmd_str = 'bash -c "systemctl enable docker && systemctl start docker"'
ClHelper.run_command(cmd_str, as_user='******')
except exceptions.ClException:
raise exceptions.CommandException(
'Failed to enable and run docker service.')
# we need to wait until /var/run/docker.sock is created
# let's wait for 30 seconds
logger.info(
'Waiting for /var/run/docker.sock to be created (max 15 seconds) ...'
)
success = False
for i in range(0, 30):
time.sleep(i * 0.5)
try:
ClHelper.run_command('ls /var/run/docker.sock')
success = True
break
except exceptions.ClException:
pass
if not success:
logger.warning(
'/var/run/docker.sock doesn\'t exist, docker will likely not work!'
)
def assistant_from_yaml(cls, source, y, superassistant, fully_loaded=True,
role=settings.DEFAULT_ASSISTANT_ROLE):
"""Constructs instance of YamlAssistant loaded from given structure y, loaded
from source file source.
Args:
source: path to assistant source file
y: loaded yaml structure
superassistant: superassistant of this assistant
Returns:
YamlAssistant instance constructed from y with source file source
"""
try:
name, attrs = y.popitem() # assume only one key and value
except AttributeError:
logger.warning('File {source} is not formatted properly - no top level mapping.'.\
format(source=source))
return None
assistant = yaml_assistant.YamlAssistant(name,
attrs,
source,
superassistant,
fully_loaded=fully_loaded,
role=role)
return assistant
def run(cls, comm_type, comm, **kwargs):
# from_struct is not callable from yaml assistants (yet)
if comm_type == 'dependencies_from_struct':
struct = comm
elif comm_type == 'dependencies_from_dda':
struct = []
dda_content = run_command('dda_r', comm, **kwargs)
original_assistant_path = dda_content.get('subassistant_path', [])
if original_assistant_path:
# if we have an original path, try to get original assistant
original_path_as_dict = {}
for i, subas in enumerate(original_assistant_path):
original_path_as_dict[settings.SUBASSISTANT_N_STRING.format(i)] = subas
from devassistant.bin import CreatorAssistant
from devassistant.assistants import yaml_assistant
try:
path = CreatorAssistant().get_selected_subassistant_path(**original_path_as_dict)
except exceptions.AssistantNotFoundException as e:
path = []
logger.warning(e)
for a in path:
#TODO: maybe figure out more DRY code (similar is in path_runner, too)
if 'dependencies' in vars(a.__class__) or isinstance(a, yaml_assistant.YamlAssistant):
struct.extend(a.dependencies(**dda_content.get('original_kwargs', {})))
#TODO: add possibility of installing arbitrary dependencies specified in .devassistant
cls._install_from_struct(struct)
def run(cls, comm_type, comm, **kwargs):
if comm_type in map(lambda x: 'log_{0}'.format(x), settings.LOG_LEVELS_MAP):
logger.log(logging._levelNames[settings.LOG_LEVELS_MAP[comm_type[-1]]], comm)
if comm_type[-1] in 'ce':
raise exceptions.RunException(comm)
else:
logger.warning('Unknown logging command {0} with message {1}'.format(comm_type, comm))
def run(cls, comm_type, comm, **kwargs):
if comm_type == 'dda_c':
return cls._dot_devassistant_create(comm, **kwargs)
elif comm_type == 'dda_r':
return cls._dot_devassistant_read(comm, **kwargs)
else:
logger.warning('Unknown .devassistant command {0}, skipping.'.format(comm_type))
def _github_create_twofactor_authorization(cls, ui):
"""Create an authorization for a GitHub user using two-factor
authentication. Unlike its non-two-factor counterpart, this method
does not traverse the available authentications as they are not
visible until the user logs in.
Please note: cls._user's attributes are not accessible until the
authorization is created due to the way (py)github works.
"""
try:
try: # This is necessary to trigger sending a 2FA key to the user
auth = cls._user.create_authorization()
except cls._gh_exceptions.GithubException:
onetime_pw = DialogHelper.ask_for_password(
ui, prompt='Your one time password:'******'repo', 'user', 'admin:public_key'],
note="DevAssistant",
onetime_password=onetime_pw)
cls._user = cls._gh_module.Github(
login_or_token=auth.token).get_user()
logger.debug(
'Two-factor authorization for user "{0}" created'.format(
cls._user.login))
cls._github_store_authorization(cls._user, auth)
logger.debug('Two-factor authorization token stored')
except cls._gh_exceptions.GithubException as e:
logger.warning(
'Creating two-factor authorization failed: {0}'.format(e))
def dependencies_section(section, kwargs, runner=None):
# "deps" is the same structure as gets returned by "dependencies" method
skip_else = False
deps = []
for i, dep in enumerate(section):
if getattr(runner, 'stop_flag', False):
break
for dep_type, dep_list in dep.items():
# rpm dependencies (can't handle anything else yet)
# we don't allow general commands, only "call"/"use" command here
if dep_type in ['call', 'use']:
deps.extend(Command(dep_type, dep_list, kwargs).run())
elif dep_type in package_managers.managers.keys(): # handle known types of deps the same, just by appending to "deps" list
fmtd = list(map(lambda dep: format_str(dep, kwargs), dep_list))
deps.append({dep_type: fmtd})
elif dep_type.startswith('if'):
possible_else = None
if len(section) > i + 1: # do we have "else" clause?
possible_else = list(section[i + 1].items())[0]
_, skip_else, to_run = get_section_from_condition((dep_type, dep_list), possible_else, kwargs)
if to_run:
deps.extend(dependencies_section(to_run, kwargs, runner=runner))
elif dep_type == 'else':
# else on its own means error, otherwise execute it
if not skip_else:
msg = 'Yaml error: encountered "else" with no associated "if", skipping.'
logger.error(msg)
raise exceptions.YamlSyntaxError(msg)
skip_else = False
else:
logger.warning('Unknown dependency type {0}, skipping.'.format(dep_type))
return deps
def _github_fork(cls, **kwargs):
"""Create a fork of repo from kwargs['fork_repo'].
Note: the kwargs are not the global context here, but what cls.format_args returns.
Raises:
devassistant.exceptions.CommandException on error
"""
fork_login, fork_reponame = kwargs['repo_url'].split('/')
logger.info('Forking {repo} for user {login} on Github ...'.\
format(login=kwargs['login'], repo=kwargs['repo_url']))
success = False
msg = ''
try:
repo = cls._gh_module.Github().get_user(fork_login).get_repo(fork_reponame)
fork = cls._user.create_fork(repo)
success = True
msg = fork.ssh_url
except cls._gh_module.GithubException as e:
msg = 'Failed to create Github fork with error: {err}'.format(err=e)
except BaseException as e:
msg = 'Exception while forking GH repo: {0}'.\
format(getattr(e, 'message', 'Unknown error'))
if success:
logger.info('Fork is ready at {url}.'.format(url=fork.html_url))
else:
logger.warning(msg)
return (success, msg)
def _construct_args(self, struct):
args = []
for arg_name, arg_params in struct.items():
use_snippet = arg_params.pop('use', None) or arg_params.pop('snippet', None)
if use_snippet:
# if snippet is used, take this parameter from snippet and update
# it with current arg_params, if any
try:
problem = None
snippet = yaml_snippet_loader.YamlSnippetLoader.get_snippet_by_name(use_snippet)
arg_params = dict(snippet.args.pop(arg_name), **arg_params)
except exceptions.SnippetNotFoundException as e:
problem = 'Couldn\'t expand argument {arg} in assistant {a}: ' + six.text_type(e)
except KeyError as e: # snippet doesn't have the requested argument
problem = 'Couldn\'t find argument {arg} in snippet {snip} wanted by assistant {a}.'
if problem:
logger.warning(problem.format(snip=use_snippet,
arg=arg_name,
a=self.name))
continue
# this works much like snippet.args.pop(arg_name).update(arg_params),
# but unlike it, this actually returns the updated dict
arg = argument.Argument(arg_name, *arg_params.pop('flags'), **arg_params)
args.append(arg)
return args
def signal_handler(signal, frame):
if package_managers.DependencyInstaller.install_lock:
logger.warning('Can\'t interrupt dependency installation!')
else:
logger.info('DevAssistant received SIGINT, exiting ...')
utils.run_exitfuncs()
sys.exit(0)
def _try_login_with_password_ntimes(cls, login, ntimes):
user = None
for i in range(0, ntimes):
password = DialogHelper.ask_for_password(
prompt='Github Password for {username}:'.format(username=login))
# user pressed Ctrl + D
if password is None:
break
gh = cls._gh_module.Github(login_or_token=login, password=password)
user = gh.get_user()
try:
user.login
break # if user.login doesn't raise, authentication has been successful
except cls._gh_module.GithubException as e:
user = None
msg = 'Wrong Github username or password; message from Github: {0}\n'.\
format(e.data.get('message', 'Unknown authentication error'))
msg += 'Try again or press {0} to abort.'
if current_run.UI == 'cli':
msg = msg.format('Ctrl + D')
else:
msg = msg.format('"Cancel"')
logger.warning(msg)
return user
def get_assistants_from_file_hierarchy(cls, file_hierarchy, superassistant,
role=settings.DEFAULT_ASSISTANT_ROLE):
"""Accepts file_hierarch as returned by cls.get_assistant_file_hierarchy and returns
instances of YamlAssistant for loaded files
Args:
file_hierarchy: structure as described in cls.get_assistants_file_hierarchy
role: role of all assistants in this hierarchy (we could find
this out dynamically but it's not worth the pain)
Returns:
list of top level assistants from given hierarchy; these assistants contain
references to instances of their subassistants (and their subassistants, ...)
"""
result = []
warn_msg = 'Failed to load assistant {source}, skipping subassistants.'
for name, attrs in file_hierarchy.items():
loaded_yaml = yaml_loader.YamlLoader.load_yaml_by_path(attrs['source'])
if loaded_yaml is None: # there was an error parsing yaml
logger.warning(warn_msg.format(source=attrs['source']))
continue
try:
ass = cls.assistant_from_yaml(attrs['source'],
loaded_yaml,
superassistant,
role=role)
except exceptions.YamlError as e:
logger.warning(e)
continue
ass._subassistants = cls.get_assistants_from_file_hierarchy(attrs['subhierarchy'],
ass,
role=role)
result.append(ass)
return result
def _dot_devassistant_dependencies(cls, comm, **kwargs):
struct = []
dda_content = cls._dot_devassistant_read(comm, **kwargs)
original_assistant_path = dda_content.get('subassistant_path', [])
if original_assistant_path:
# if we have an original path, try to get original assistant
original_path_as_dict = {}
for i, subas in enumerate(original_assistant_path):
original_path_as_dict[settings.SUBASSISTANT_N_STRING.format(
i)] = subas
from devassistant.bin import CreatorAssistant
from devassistant import yaml_assistant
try:
path = CreatorAssistant().get_selected_subassistant_path(
**original_path_as_dict)
except exceptions.AssistantNotFoundException as e:
path = []
logger.warning(str(e))
for a in path:
#TODO: maybe figure out more DRY code (similar is in path_runner, too)
if 'dependencies' in vars(a.__class__) or isinstance(
a, yaml_assistant.YamlAssistant):
struct.extend(
a.dependencies(
**dda_content.get('original_kwargs', {})))
struct.extend(kwargs['__assistant__']._dependencies_section(
dda_content.get('dependencies', []), **kwargs))
run_command('dependencies', struct, **kwargs)
def run(cls, c):
"""Arguments given to 'github' command may be:
- Just a string (action), which implies that all the other arguments are deducted from
global context and local system.
- List containing a string (action) as a first item and rest of the args in a dict.
(args not specified in the dict are taken from global context.
Possible arguments:
- login - taken from 'github' or system username - represents Github login
- reponame - taken from 'name' (first applies os.path.basename) - repo to operate on
"""
comm, kwargs = cls.format_args(c)
if not cls._gh_module:
logger.warning('PyGithub not installed, cannot execute github command.')
return [False, '']
# we pass arguments as kwargs, so that the auth decorator can easily query them
# NOTE: these are not the variables from global context, but rather what
# cls.format_args returned
if comm == 'create_repo':
ret = cls._github_create_repo(**kwargs)
elif comm == 'push':
ret = cls._github_push()
elif comm == 'create_and_push':
ret = cls._github_create_and_push(**kwargs)
elif comm == 'add_remote_origin':
ret = cls._github_add_remote_origin(**kwargs)
elif comm == 'create_fork':
ret = cls._github_fork(**kwargs)
else:
raise exceptions.CommandException('Unknown command type {ct}.'.format(ct=c.comm_type))
return ret
def _github_push(cls):
try:
ret = ClHelper.run_command("git push -u origin master")
logger.info('Source code was successfully pushed.')
return (True, ret)
except exceptions.ClException as e:
logger.warning('Problem pushing source code: {0}'.format(e.output))
return (False, e.output)
def _github_push(cls):
try:
ret = ClHelper.run_command("git push -u origin master")
logger.info('Source code was successfully pushed.')
return (True, ret)
except exceptions.ClException as e:
logger.warning('Problem pushing source code: {0}'.format(e.output))
return (False, e.output)
def run(cls, comm_type, comm, **kwargs):
if comm_type == 'dda_c':
return cls._dot_devassistant_create(comm, **kwargs)
elif comm_type == 'dda_r':
return cls._dot_devassistant_read(comm, **kwargs)
else:
logger.warning(
'Unknown .devassistant command {0}, skipping.'.format(
comm_type))
def run(cls, comm_type, comm, **kwargs):
if comm == 'create_repo':
cls._github_create_repo(**kwargs)
elif comm == 'push':
cls._github_push(**kwargs)
elif comm == 'create_and_push':
cls._github_create_and_push(**kwargs)
else:
logger.warning('Unknow github command {0}, skipping.'.format(comm))
def inner(func_cls, *args, **kwargs):
if not cls._gh_module:
logger.warning('PyGithub not installed, skipping github authentication procedures.')
elif not func_cls._user:
# authenticate user, possibly also creating authentication for future use
func_cls._user = cls._get_github_user(kwargs['login'])
# create ssh key for pushing
cls._github_create_ssh_key()
return func(func_cls, *args, **kwargs)
def run(cls, comm_type, comm, **kwargs):
if comm == 'create_repo':
cls._github_create_repo(**kwargs)
elif comm == 'push':
cls._github_push(**kwargs)
elif comm == 'create_and_push':
cls._github_create_and_push(**kwargs)
else:
logger.warning('Unknow github command {0}, skipping.'.format(comm))
def _construct_args(self, struct):
args = []
for arg_name, arg_params in struct.items():
try:
args.append(argument.Argument.construct_arg(arg_name, arg_params))
except exceptions.ExecutionException as e:
msg = 'Problem when constructing argument {arg} in assistant {a}: {e}'.\
format(arg=arg_name, a=self.name, e=six.text_type(e))
logger.warning(msg)
return args
def load_yaml_by_path(cls, path):
"""Load a yaml file that is at given path"""
try:
return yaml.load(open(path, 'r'), Loader=Loader)
except yaml.scanner.ScannerError as e:
logger.warning('Yaml error in {path} (line {ln}, column {col}): {err}'.\
format(path=path,
ln=e.problem_mark.line,
col=e.problem_mark.column,
err=e.problem))
return None
def load_yaml_by_path(cls, path):
"""Load a yaml file that is at given path"""
try:
return yaml.load(open(path, 'r'), Loader=Loader) or {}
except yaml.scanner.ScannerError as e:
logger.warning('Yaml error in {path} (line {ln}, column {col}): {err}'.\
format(path=path,
ln=e.problem_mark.line,
col=e.problem_mark.column,
err=e.problem))
return None
def _construct_args(self, struct):
args = []
for arg_name, arg_params in struct.items():
try:
args.append(
argument.Argument.construct_arg(arg_name, arg_params))
except exceptions.ExecutionException as e:
msg = 'Problem when constructing argument {arg} in assistant {a}: {e}'.\
format(arg=arg_name, a=self.name, e=six.text_type(e))
logger.warning(msg)
return args
def run(cls, comm_type, comm, **kwargs):
if comm_type in map(lambda x: 'log_{0}'.format(x),
settings.LOG_LEVELS_MAP):
logger.log(
logging._levelNames[settings.LOG_LEVELS_MAP[comm_type[-1]]],
comm)
if comm_type[-1] in 'ce':
raise exceptions.RunException(comm)
else:
logger.warning(
'Unknown logging command {0} with message {1}'.format(
comm_type, comm))
def run(cls, comm_type, comm, **kwargs):
if not cls._gh_module:
logger.warning('PyGithub not installed, cannot execute github command.')
return
if comm == 'create_repo':
cls._github_create_repo(**kwargs)
elif comm == 'push':
cls._github_push(**kwargs)
elif comm == 'create_and_push':
cls._github_create_and_push(**kwargs)
else:
logger.warning('Unknow github command {0}, skipping.'.format(comm))
def inner(func_cls, *args, **kwargs):
if not cls._gh_module:
logger.warning(
'PyGithub not installed, skipping github authentication procedures.'
)
elif not func_cls._user:
# authenticate user, possibly also creating authentication for future use
func_cls._user = cls._get_github_user(
cls._github_login(**kwargs), cls._github_token(**kwargs),
**kwargs)
# create ssh key for pushing
cls._github_create_ssh_key(**kwargs)
func(func_cls, *args, **kwargs)
def inner(func_cls, *args, **kwargs):
if not cls._gh_module:
logger.warning('PyGithub not installed, skipping github authentication procedures.')
elif not func_cls._user:
# authenticate user, possibly also creating authentication for future use
func_cls._user = cls._get_github_user(kwargs['login'])
# create an ssh key for pushing if we don't have one
if not cls._github_ssh_key_exists():
cls._github_create_ssh_key()
# next, create ~/.ssh/config entry for the key, if system username != GH login
if cls._ssh_key_needs_config_entry():
cls._create_ssh_config_entry()
return func(func_cls, *args, **kwargs)
def run(cls, comm_type, comm, **kwargs):
if not cls._gh_module:
logger.warning(
'PyGithub not installed, cannot execute github command.')
return
if comm == 'create_repo':
cls._github_create_repo(**kwargs)
elif comm == 'push':
cls._github_push(**kwargs)
elif comm == 'create_and_push':
cls._github_create_and_push(**kwargs)
else:
logger.warning('Unknow github command {0}, skipping.'.format(comm))
def _github_create_auth(cls, **kwargs):
""" Store token into ~/.gitconfig.
If token is not defined then store it into ~/.gitconfig file
"""
if not cls._token:
try:
auth = cls._user.create_authorization(scopes=['repo', 'user'], note="DeveloperAssistant")
ClHelper.run_command("git config --global github.token.{login} {token}".format(
login=cls._user.login,
token=auth.token))
ClHelper.run_command("git config --global github.user.{login} {login}".format(
login=cls._user.login))
except cls._gh_module.GithubException as e:
logger.warning('Creating authorization failed: {0}'.format(e))
def _github_create_auth(cls):
""" Store token into ~/.gitconfig.
If token is not defined then store it into ~/.gitconfig file
"""
if not cls._token:
try:
auth = cls._user.create_authorization(scopes=['repo', 'user'], note="DeveloperAssistant")
ClHelper.run_command("git config --global github.token.{login} {token}".format(
login=cls._user.login,
token=auth.token))
ClHelper.run_command("git config --global github.user.{login} {login}".format(
login=cls._user.login))
except cls._gh_module.GithubException as e:
logger.warning('Creating authorization failed: {0}'.format(e))
def _dependencies_section(self, section, **kwargs):
for dep in section:
dep_type, dep_list = dep.popitem()
# rpm dependencies (can't handle anything else yet)
if dep_type == 'rpm':
self._install_dependencies(*dep_list, **kwargs)
elif dep_type == 'snippet':
snippet, section_name = self._get_snippet_and_section_name(dep_list, **kwargs)
section = snippet.get_dependencies_section(section_name) if snippet else None
if section:
self._dependencies_section(section, **kwargs)
else:
logger.warning('Couldn\'t find dependencies section "{0}", in snippet {1}, skipping.'.format(section_name,
dep_list.split('(')[0]))
else:
logger.warning('Unknown dependency type {0}, skipping.'.format(dep_type))
def _github_create_simple_authorization(cls):
"""Create a GitHub authorization for the given user in case they don't
already have one.
"""
try:
auth = None
for a in cls._user.get_authorizations():
if a.note == 'DevAssistant':
auth = a
if not auth:
auth = cls._user.create_authorization(
scopes=['repo', 'user', 'admin:public_key'],
note="DevAssistant")
cls._github_store_authorization(cls._user, auth)
except cls._gh_exceptions.GithubException as e:
logger.warning('Creating authorization failed: {0}'.format(e))
def _github_create_simple_authorization(cls):
"""Create a GitHub authorization for the given user in case they don't
already have one.
"""
try:
auth = None
for a in cls._user.get_authorizations():
if a.note == 'DevAssistant':
auth = a
if not auth:
auth = cls._user.create_authorization(
scopes=['repo', 'user', 'admin:public_key'],
note="DevAssistant")
cls._github_store_authorization(cls._user, auth)
except cls._gh_exceptions.GithubException as e:
logger.warning('Creating authorization failed: {0}'.format(e))
def _run_one_section(self, section, kwargs):
execute_else = False
for i, command_dict in enumerate(section):
for comm_type, comm in command_dict.items():
if comm_type.startswith('run'):
s = self._get_section_to_run(section=comm, kwargs_override=False, **kwargs)
# use copy of kwargs, so that local kwargs don't get modified
self._run_one_section(s, copy.deepcopy(kwargs))
elif comm_type == 'snippet':
snippet, section_name = self._get_snippet_and_section_name(comm, **kwargs)
section = snippet.get_run_section(section_name) if snippet else None
if section:
# push and pop snippet's files into kwargs
if '__files__' in kwargs:
kwargs['__files__'].append(snippet.get_files_section())
else:
kwargs['__files__'] = [snippet.get_files_section()]
# use copy of kwargs, so that local kwargs don't get modified
self._run_one_section(section, copy.deepcopy(kwargs))
kwargs['__files__'].pop()
else:
logger.warning('Couldn\'t find run section "{0}", in snippet {1} skipping.'.format(section_name,
comm.split('(')[0]))
elif comm_type.startswith('$'):
# intentionally pass kwargs as dict, not as keywords
self._assign_variable(comm_type, comm, kwargs)
elif comm_type.startswith('if'):
if self._evaluate(comm_type[2:].strip(), **kwargs):
# run with original kwargs, so that they might be changed for code after this if
self._run_one_section(comm, kwargs)
elif len(section) > i + 1:
next_section_dict = section[i + 1]
next_section_comm_type, next_section_comm = list(next_section_dict.items())[0]
if next_section_comm_type == 'else':
execute_else = True
elif comm_type == 'else':
# else on its own means error, otherwise execute it
if not list(section[i - 1].items())[0][0].startswith('if'):
logger.warning('Yaml error: encountered "else" with no associated "if", skipping.')
elif execute_else:
execute_else = False
# run with original kwargs, so that they might be changed for code after this if
self._run_one_section(comm, kwargs)
else:
files = kwargs['__files__'][-1] if kwargs.get('__files__', None) else self._files
run_command(comm_type, CommandFormatter.format(comm, self.template_dir, files, **kwargs), **kwargs)
def _github_add_remote_origin(cls, **kwargs):
"""Note: the kwargs are not the global context here, but what cls.format_args returns."""
reponame = kwargs['reponame']
login = kwargs['login']
# if system username != GH login, we need to use [email protected]{login}:...
# else just [email protected]:...
dash_login = ''
if getpass.getuser() != login:
dash_login = '******' + login
try:
logger.info('Adding Github repo as git remote ...')
ret = ClHelper.run_command("git remote add origin [email protected]{dl}:{l}/{r}.git".\
format(dl=dash_login, l=login, r=reponame))
logger.info('Successfully added Github repo as git remote.')
return (True, ret)
except exceptions.ClException as e:
logger.warning('Problem adding Github repo as git remote: {0}.'.format(e.output))
return (False, e.output)
def _construct_args(self, struct):
args = []
# Construct properly the iterable args from either a dict, or a list
if isinstance(struct, dict):
temp_args = struct.items()
elif isinstance(struct, list):
temp_args = [list(argdict.items())[0] for argdict in struct]
else:
raise TypeError('Args struct should be dict or list, is {0}'.format(type(struct)))
for (arg_name, arg_params) in temp_args:
try:
args.append(argument.Argument.construct_arg(arg_name, arg_params))
except exceptions.ExecutionException as e:
msg = 'Problem when constructing argument {arg} in assistant {a}: {e}'.\
format(arg=arg_name, a=self.name, e=six.text_type(e))
logger.warning(msg)
return args
def _github_add_remote_origin(cls, **kwargs):
"""Note: the kwargs are not the global context here, but what cls.format_args returns."""
reponame = kwargs['reponame']
login = kwargs['login']
# if system username != GH login, we need to use [email protected]{login}:...
# else just [email protected]:...
dash_login = ''
if getpass.getuser() != login:
dash_login = '******' + login
try:
logger.info('Adding Github repo as git remote ...')
ret = ClHelper.run_command("git remote add origin [email protected]{dl}:{l}/{r}.git".\
format(dl=dash_login, l=login, r=reponame))
logger.info('Successfully added Github repo as git remote.')
return (True, ret)
except exceptions.ClException as e:
logger.warning('Problem adding Github repo as git remote: {0}.'.format(e.output))
return (False, e.output)
def inner(func_cls, *args, **kwargs):
if not cls._gh_module:
logger.warning('PyGithub not installed, skipping Github auth procedures.')
elif not func_cls._user:
# authenticate user, possibly also creating authentication for future use
login = kwargs['login'].encode(utils.defenc) if not six.PY3 else kwargs['login']
func_cls._user = cls._get_github_user(login, kwargs['ui'])
if func_cls._user is None:
msg = 'Github authentication failed, skipping Github command.'
logger.warning(msg)
return (False, msg)
# create an ssh key for pushing if we don't have one
if not cls._github_ssh_key_exists():
cls._github_create_ssh_key()
# next, create ~/.ssh/config entry for the key, if system username != GH login
if cls._ssh_key_needs_config_entry():
cls._create_ssh_config_entry()
return func(func_cls, *args, **kwargs)
def logging(self, **kwargs):
kwargs = self.proper_kwargs(**kwargs)
for l in self._logging:
handler_type, l_list = l.popitem()
if handler_type == 'file':
level, lfile = l_list
expanded_lfile = self._format(lfile, **kwargs)
# make dirs, create logger
if not os.path.exists(os.path.dirname(expanded_lfile)):
os.makedirs(os.path.dirname(expanded_lfile))
# add handler and formatter
handler = logging.FileHandler(expanded_lfile, 'a+')
formatter = logging.Formatter('%(asctime)-15s %(levelname)s - %(message)s')
handler.setFormatter(formatter)
handler.setLevel(getattr(logging, level.upper()))
# register handler with the global logger
logger.addHandler(handler)
else:
logger.warning('Unknown logger type {0}, ignoring.'.format(handler_type))
def logging(self, kwargs):
# TODO: this doesn't seem to work, fix it...
self.proper_kwargs('logging', kwargs)
for l in self._logging:
handler_type, l_list = l.popitem()
if handler_type == 'file':
level, lfile = l_list
expanded_lfile = self._format(lfile, kwargs)
# make dirs, create logger
if not os.path.exists(os.path.dirname(expanded_lfile)):
os.makedirs(os.path.dirname(expanded_lfile))
# add handler and formatter
handler = logging.FileHandler(expanded_lfile, 'a+')
formatter = logging.Formatter('%(asctime)-15s [%(event_type)] %(levelname)s - %(message)s')
handler.setFormatter(formatter)
handler.setLevel(getattr(logging, level.upper()))
# register handler with the global logger
logger.addHandler(handler)
else:
logger.warning('Unknown logger type {0}, ignoring.'.format(handler_type))
def inner(func_cls, *args, **kwargs):
if not cls._gh_module:
logger.warning(
'PyGithub not installed, skipping Github auth procedures.')
elif not func_cls._user:
# authenticate user, possibly also creating authentication for future use
login = kwargs['login'].encode(
utils.defenc) if not six.PY3 else kwargs['login']
func_cls._user = cls._get_github_user(login, kwargs['ui'])
if func_cls._user is None:
msg = 'Github authentication failed, skipping Github command.'
logger.warning(msg)
return (False, msg)
# create an ssh key for pushing if we don't have one
if not cls._github_ssh_key_exists():
cls._github_create_ssh_key()
# next, create ~/.ssh/config entry for the key, if system username != GH login
if cls._ssh_key_needs_config_entry():
cls._create_ssh_config_entry()
return func(func_cls, *args, **kwargs)
def _dependencies_section(self, section, **kwargs):
for dep in section:
dep_type, dep_list = dep.popitem()
# rpm dependencies (can't handle anything else yet)
if dep_type == 'rpm':
self._install_dependencies(*dep_list, **kwargs)
elif dep_type == 'snippet':
snippet, section_name = self._get_snippet_and_section_name(
dep_list, **kwargs)
section = snippet.get_dependencies_section(
section_name) if snippet else None
if section:
self._dependencies_section(section, **kwargs)
else:
logger.warning(
'Couldn\'t find dependencies section "{0}", in snippet {1}, skipping.'
.format(section_name,
dep_list.split('(')[0]))
else:
logger.warning(
'Unknown dependency type {0}, skipping.'.format(dep_type))
def _dependencies_section(self, section, **kwargs):
# "deps" is the same structure as gets returned by "dependencies" method
skip_else = False
deps = []
for i, dep in enumerate(section):
for dep_type, dep_list in dep.items():
# rpm dependencies (can't handle anything else yet)
if dep_type == 'call':
section = self._get_section_from_call(dep_list, 'dependencies', **kwargs)
if section is not None:
deps.extend(self._dependencies_section(section, **kwargs))
else:
logger.warning('Couldn\'t find dependencies section "{0}", in snippet {1}, skipping.'.format(dep_list.split('.')))
elif dep_type in package_managers.managers.keys(): # handle known types of deps the same, just by appending to "deps" list
deps.append({dep_type: dep_list})
elif dep_type.startswith('if'):
possible_else = None
if len(section) > i + 1: # do we have "else" clause?
possible_else = list(section[i + 1].items())[0]
_, skip_else, to_run = self._get_section_from_condition((dep_type, dep_list), possible_else, **kwargs)
if to_run:
deps.extend(self._dependencies_section(to_run, **kwargs))
elif dep_type == 'else':
# else on its own means error, otherwise execute it
if not skip_else:
logger.warning('Yaml error: encountered "else" with no associated "if", skipping.')
skip_else = False
else:
logger.warning('Unknown dependency type {0}, skipping.'.format(dep_type))
return deps
def _github_create_repo(cls, **kwargs):
"""Create repo on GitHub.
Note: the kwargs are not the global context here, but what cls.format_args returns.
If repository already exists then CommandException will be raised.
Raises:
devassistant.exceptions.CommandException on error
"""
reponame = kwargs['reponame']
if reponame in map(lambda x: x.name, cls._user.get_repos()):
msg = 'Failed to create Github repo: {0}/{1} alread exists.'.\
format(cls._user.login, reponame)
logger.warning(msg)
return (False, msg)
else:
msg = ''
success = False
try:
new_repo = cls._user.create_repo(reponame, private=kwargs['private'])
msg = new_repo.clone_url
success = True
except cls._gh_module.GithubException as e:
gh_errs = e.data.get('errors', [])
gh_errs = '; '.join(map(lambda err: err.get('message', ''), gh_errs))
msg = 'Failed to create GitHub repo. This sometime happens when you delete '
msg += 'a repo and then you want to create the same one immediately. If that\'s '
msg += 'the case, wait for few minutes and then try again.\n'
msg += 'Github errors: ' + gh_errs
except BaseException as e:
msg = 'Failed to create Github repo: {0}'.\
format(getattr(e, 'message', 'Unknown error'))
if success:
logger.info('Your new repository: {0}'.format(new_repo.html_url))
else:
logger.warning(msg)
return (success, msg)
def _github_create_repo(cls, **kwargs):
"""Create repo on GitHub.
Note: the kwargs are not the global context here, but what cls.format_args returns.
If repository already exists then CommandException will be raised.
Raises:
devassistant.exceptions.CommandException on error
"""
reponame = kwargs['reponame']
if reponame in map(lambda x: x.name, cls._user.get_repos()):
msg = 'Failed to create Github repo: {0}/{1} alread exists.'.\
format(cls._user.login, reponame)
logger.warning(msg)
return (False, msg)
else:
msg = ''
success = False
try:
new_repo = cls._user.create_repo(reponame, private=kwargs['private'])
msg = new_repo.clone_url
success = True
except cls._gh_module.GithubException as e:
gh_errs = e.data.get('errors', [])
gh_errs = '; '.join(map(lambda err: err.get('message', ''), gh_errs))
msg = 'Failed to create GitHub repo. This sometime happens when you delete '
msg += 'a repo and then you want to create the same one immediately. If that\'s '
msg += 'the case, wait for few minutes and then try again.\n'
msg += 'Github errors: ' + gh_errs
except BaseException as e:
msg = 'Failed to create Github repo: {0}'.\
format(getattr(e, 'message', 'Unknown error'))
if success:
logger.info('Your new repository: {0}'.format(new_repo.html_url))
else:
logger.warning(msg)
return (success, msg)
def _dot_devassistant_dependencies(cls, comm, **kwargs):
struct = []
dda_content = cls._dot_devassistant_read(comm, **kwargs)
original_assistant_path = dda_content.get('subassistant_path', [])
if original_assistant_path:
# if we have an original path, try to get original assistant
original_path_as_dict = {}
for i, subas in enumerate(original_assistant_path):
original_path_as_dict[settings.SUBASSISTANT_N_STRING.format(i)] = subas
from devassistant.bin import CreatorAssistant
from devassistant import yaml_assistant
try:
path = CreatorAssistant().get_selected_subassistant_path(**original_path_as_dict)
except exceptions.AssistantNotFoundException as e:
path = []
logger.warning(str(e))
for a in path:
#TODO: maybe figure out more DRY code (similar is in path_runner, too)
if 'dependencies' in vars(a.__class__) or isinstance(a, yaml_assistant.YamlAssistant):
struct.extend(a.dependencies(**dda_content.get('original_kwargs', {})))
struct.extend(kwargs['__assistant__']._dependencies_section(dda_content.get('dependencies', []), **kwargs))
run_command('dependencies', struct, **kwargs)
def _github_fork(cls, **kwargs):
"""Create a fork of repo from kwargs['fork_repo'].
Note: the kwargs are not the global context here, but what cls.format_args returns.
Raises:
devassistant.exceptions.CommandException on error
"""
timeout = 300 # 5 minutes
fork_login, fork_reponame = kwargs['repo_url'].split('/')
logger.info('Forking {repo} for user {login} on Github ...'.\
format(login=kwargs['login'], repo=kwargs['repo_url']))
success = False
msg = ''
try:
repo = cls._gh_module.Github().get_user(fork_login).get_repo(fork_reponame)
fork = cls._user.create_fork(repo)
while timeout > 0:
time.sleep(5)
timeout -= 5
try:
fork.get_contents('/') # This function doesn't throw an exception when clonable
success = True
break
except cls._gh_module.GithubException as e:
if 'is empty' not in str(e):
raise e
msg = fork.ssh_url
except cls._gh_module.GithubException as e:
msg = 'Failed to create Github fork with error: {err}'.format(err=e)
except BaseException as e:
msg = 'Exception while forking GH repo: {0}'.\
format(getattr(e, 'message', 'Unknown error'))
if success:
logger.info('Fork is ready at {url}.'.format(url=fork.html_url))
else:
logger.warning(msg)
return (success, msg)
def _github_fork(cls, **kwargs):
"""Create a fork of repo from kwargs['fork_repo'].
Note: the kwargs are not the global context here, but what cls.format_args returns.
Raises:
devassistant.exceptions.CommandException on error
"""
timeout = 300 # 5 minutes
fork_login, fork_reponame = kwargs['repo_url'].split('/')
logger.info('Forking {repo} for user {login} on Github ...'.\
format(login=kwargs['login'], repo=kwargs['repo_url']))
success = False
msg = ''
try:
repo = cls._gh_module.Github().get_user(fork_login).get_repo(fork_reponame)
fork = cls._user.create_fork(repo)
while timeout > 0:
time.sleep(5)
timeout -= 5
try:
fork.get_contents('/') # This function doesn't throw an exception when clonable
success = True
break
except cls._gh_module.GithubException as e:
if 'is empty' not in str(e):
raise e
msg = fork.ssh_url
except cls._gh_module.GithubException as e:
msg = 'Failed to create Github fork with error: {err}'.format(err=e)
except BaseException as e:
msg = 'Exception while forking GH repo: {0}'.\
format(getattr(e, 'message', 'Unknown error'))
if success:
logger.info('Fork is ready at {url}.'.format(url=fork.html_url))
else:
logger.warning(msg)
return (success, msg)
def dependencies_section(section, kwargs, runner=None):
# "deps" is the same structure as gets returned by "dependencies" method
skip_else = False
deps = []
for i, dep in enumerate(section):
if getattr(runner, 'stop_flag', False):
break
for dep_type, dep_list in dep.items():
# rpm dependencies (can't handle anything else yet)
# we don't allow general commands, only "call"/"use" command here
if dep_type in ['call', 'use']:
deps.extend(Command(dep_type, dep_list, kwargs).run())
# handle known types of deps the same, just by appending to "deps" list
elif dep_type in package_managers.managers.keys():
fmtd = list(map(lambda dep: format_str(dep, kwargs), dep_list))
deps.append({dep_type: fmtd})
elif dep_type.startswith('if'):
possible_else = None
if len(section) > i + 1: # do we have "else" clause?
possible_else = list(section[i + 1].items())[0]
_, skip_else, to_run = get_section_from_condition(
(dep_type, dep_list), possible_else, kwargs)
if to_run:
deps.extend(
dependencies_section(to_run, kwargs, runner=runner))
elif dep_type == 'else':
# else on its own means error, otherwise execute it
if not skip_else:
msg = 'Yaml error: encountered "else" with no associated "if", skipping.'
logger.error(msg)
raise exceptions.YamlSyntaxError(msg)
skip_else = False
else:
logger.warning(
'Unknown dependency type {0}, skipping.'.format(dep_type))
return deps
def get_assistants_from_file_hierarchy(
cls,
file_hierarchy,
superassistant,
role=settings.DEFAULT_ASSISTANT_ROLE):
"""Accepts file_hierarch as returned by cls.get_assistant_file_hierarchy and returns
instances of YamlAssistant for loaded files
Args:
file_hierarchy: structure as described in cls.get_assistants_file_hierarchy
role: role of all assistants in this hierarchy (we could find
this out dynamically but it's not worth the pain)
Returns:
list of top level assistants from given hierarchy; these assistants contain
references to instances of their subassistants (and their subassistants, ...)
"""
result = []
warn_msg = 'Failed to load assistant {source}, skipping subassistants.'
for name, attrs in file_hierarchy.items():
loaded_yaml = yaml_loader.YamlLoader.load_yaml_by_path(
attrs['source'])
if loaded_yaml is None: # there was an error parsing yaml
logger.warning(warn_msg.format(source=attrs['source']))
continue
try:
ass = cls.assistant_from_yaml(attrs['source'],
loaded_yaml,
superassistant,
role=role)
except exceptions.YamlError as e:
logger.warning(e)
continue
ass._subassistants = cls.get_assistants_from_file_hierarchy(
attrs['subhierarchy'], ass, role=role)
result.append(ass)
return result
def _github_create_auth(cls):
""" Store token into ~/.gitconfig.
Note: this uses cls._user.get_authorizations(), which only works if cls._user
was authorized by login/password, doesn't work for token auth (TODO: why?).
If token is not defined then store it into ~/.gitconfig file
"""
if not cls._token:
try:
auth = None
for a in cls._user.get_authorizations():
if a.note == 'DevAssistant':
auth = a
if not auth:
auth = cls._user.create_authorization(
scopes=['repo', 'user', 'admin:public_key'],
note="DevAssistant")
ClHelper.run_command("git config --global github.token.{login} {token}".format(
login=cls._user.login,
token=auth.token))
ClHelper.run_command("git config --global github.user.{login} {login}".format(
login=cls._user.login))
except cls._gh_module.GithubException as e:
logger.warning('Creating authorization failed: {0}'.format(e))
def _dependencies_section(self, section, **kwargs):
# "deps" is the same structure as gets returned by "dependencies" method
skip_else = False
deps = []
for i, dep in enumerate(section):
for dep_type, dep_list in dep.items():
# rpm dependencies (can't handle anything else yet)
if dep_type == 'call':
section = self._get_section_from_call(
dep_list, 'dependencies', **kwargs)
if section is not None:
deps.extend(
self._dependencies_section(section, **kwargs))
else:
logger.warning(
'Couldn\'t find dependencies section "{0}", in snippet {1}, skipping.'
.format(dep_list.split('.')))
elif dep_type in package_managers.managers.keys(
): # handle known types of deps the same, just by appending to "deps" list
deps.append({dep_type: dep_list})
elif dep_type.startswith('if'):
possible_else = None
if len(section) > i + 1: # do we have "else" clause?
possible_else = list(section[i + 1].items())[0]
_, skip_else, to_run = self._get_section_from_condition(
(dep_type, dep_list), possible_else, **kwargs)
if to_run:
deps.extend(
self._dependencies_section(to_run, **kwargs))
elif dep_type == 'else':
# else on its own means error, otherwise execute it
if not skip_else:
logger.warning(
'Yaml error: encountered "else" with no associated "if", skipping.'
)
skip_else = False
else:
logger.warning(
'Unknown dependency type {0}, skipping.'.format(
dep_type))
return deps
def _run_one_section(self, section, kwargs):
skip_else = False
for i, command_dict in enumerate(section):
if self.stop_flag:
break
for comm_type, comm in command_dict.items():
if comm_type.startswith('call'):
# calling workflow:
# 1) get proper run section (either from self or from snippet)
# 2) if running snippet, add its files to kwargs['__files__']
# 3) actually run
# 4) if running snippet, pop its files from kwargs['__files__']
sect = self._get_section_from_call(comm, 'run')
if sect is None:
logger.warning(
'Couldn\'t find section to run: {0}.'.format(comm))
continue
if self._is_snippet_call(comm, **kwargs):
# we're calling a snippet => add files and template_dir to kwargs
snippet = yaml_snippet_loader.YamlSnippetLoader.get_snippet_by_name(
comm.split('.')[0])
if '__files__' not in kwargs:
kwargs['__files__'] = []
kwargs['__template_dir__'] = []
kwargs['__files__'].append(snippet.get_files_section())
kwargs['__template_dir__'].append(
snippet.get_template_dir())
self._run_one_section(sect, copy.deepcopy(kwargs))
if self._is_snippet_call(comm, **kwargs):
kwargs['__files__'].pop()
kwargs['__template_dir__'].pop()
elif comm_type.startswith('$'):
# intentionally pass kwargs as dict, not as keywords
try:
self._assign_variable(comm_type, comm, kwargs)
except exceptions.YamlSyntaxError as e:
logger.error(e)
raise e
elif comm_type.startswith('if'):
possible_else = None
if len(section) > i + 1: # do we have "else" clause?
possible_else = list(section[i + 1].items())[0]
_, skip_else, to_run = self._get_section_from_condition(
(comm_type, comm), possible_else, **kwargs)
if to_run:
# run with original kwargs, so that they might be changed for code after this if
self._run_one_section(to_run, kwargs)
elif comm_type == 'else':
if not skip_else:
logger.warning(
'Yaml error: encountered "else" with no associated "if", skipping.'
)
skip_else = False
elif comm_type.startswith('for'):
# syntax: "for $i in $x: <section> or "for $i in cl_command: <section>"
try:
control_var, expression = self._parse_for(comm_type)
except exceptions.YamlSyntaxError as e:
logger.error(e)
raise e
try:
eval_expression = evaluate_expression()
except exceptions.YamlSyntaxError as e:
logger.log(e)
raise e
for i in eval_expression:
kwargs[control_var] = i
self._run_one_section(comm, kwargs)
elif comm_type.startswith('scl'):
if '__scls__' not in kwargs:
kwargs['__scls__'] = []
# list of lists of scl names
kwargs['__scls__'].append(comm_type.split()[1:])
self._run_one_section(comm, kwargs)
kwargs['__scls__'].pop()
else:
files = kwargs['__files__'][-1] if kwargs.get(
'__files__', None) else self._files
template_dir = kwargs['__template_dir__'][-1] if kwargs.get(
'__template_dir__', None) else self.template_dir
run_command(
comm_type,
CommandFormatter.format(comm_type, comm, template_dir,
files, **kwargs), **kwargs)
