def run(cls, c): """Arguments given to 'github' command may be: - Just a string (action), which implies that all the other arguments are deducted from global context and local system. - List containing a string (action) as a first item and rest of the args in a dict. (args not specified in the dict are taken from global context. Possible arguments: - login - taken from 'github' or system username - represents Github login - reponame - taken from 'name' (first applies os.path.basename) - repo to operate on """ comm, kwargs = cls.format_args(c) if not cls._gh_module: logger.warning('PyGithub not installed, cannot execute github command.') return [False, ''] # we pass arguments as kwargs, so that the auth decorator can easily query them # NOTE: these are not the variables from global context, but rather what # cls.format_args returned if comm == 'create_repo': ret = cls._github_create_repo(**kwargs) elif comm == 'push': ret = cls._github_push() elif comm == 'create_and_push': ret = cls._github_create_and_push(**kwargs) elif comm == 'add_remote_origin': ret = cls._github_add_remote_origin(**kwargs) elif comm == 'create_fork': ret = cls._github_fork(**kwargs) else: raise exceptions.CommandException('Unknown command type {ct}.'.format(ct=c.comm_type)) return ret
def docker_service_enable_and_run(cls): # TODO: add some conditionals for various platforms logger.info('Enabling and running docker service ...') try: cmd_str = 'bash -c "systemctl enable docker && systemctl start docker"' ClHelper.run_command(cmd_str, as_user='******') except exceptions.ClException: raise exceptions.CommandException( 'Failed to enable and run docker service.') # we need to wait until /var/run/docker.sock is created # let's wait for 30 seconds logger.info( 'Waiting for /var/run/docker.sock to be created (max 15 seconds) ...' ) success = False for i in range(0, 30): time.sleep(i * 0.5) try: ClHelper.run_command('ls /var/run/docker.sock') success = True break except exceptions.ClException: pass if not success: logger.warning( '/var/run/docker.sock doesn\'t exist, docker will likely not work!' )
def assistant_from_yaml(cls, source, y, superassistant, fully_loaded=True, role=settings.DEFAULT_ASSISTANT_ROLE): """Constructs instance of YamlAssistant loaded from given structure y, loaded from source file source. Args: source: path to assistant source file y: loaded yaml structure superassistant: superassistant of this assistant Returns: YamlAssistant instance constructed from y with source file source """ try: name, attrs = y.popitem() # assume only one key and value except AttributeError: logger.warning('File {source} is not formatted properly - no top level mapping.'.\ format(source=source)) return None assistant = yaml_assistant.YamlAssistant(name, attrs, source, superassistant, fully_loaded=fully_loaded, role=role) return assistant
def run(cls, comm_type, comm, **kwargs): # from_struct is not callable from yaml assistants (yet) if comm_type == 'dependencies_from_struct': struct = comm elif comm_type == 'dependencies_from_dda': struct = [] dda_content = run_command('dda_r', comm, **kwargs) original_assistant_path = dda_content.get('subassistant_path', []) if original_assistant_path: # if we have an original path, try to get original assistant original_path_as_dict = {} for i, subas in enumerate(original_assistant_path): original_path_as_dict[settings.SUBASSISTANT_N_STRING.format(i)] = subas from devassistant.bin import CreatorAssistant from devassistant.assistants import yaml_assistant try: path = CreatorAssistant().get_selected_subassistant_path(**original_path_as_dict) except exceptions.AssistantNotFoundException as e: path = [] logger.warning(e) for a in path: #TODO: maybe figure out more DRY code (similar is in path_runner, too) if 'dependencies' in vars(a.__class__) or isinstance(a, yaml_assistant.YamlAssistant): struct.extend(a.dependencies(**dda_content.get('original_kwargs', {}))) #TODO: add possibility of installing arbitrary dependencies specified in .devassistant cls._install_from_struct(struct)
def run(cls, comm_type, comm, **kwargs): if comm_type in map(lambda x: 'log_{0}'.format(x), settings.LOG_LEVELS_MAP): logger.log(logging._levelNames[settings.LOG_LEVELS_MAP[comm_type[-1]]], comm) if comm_type[-1] in 'ce': raise exceptions.RunException(comm) else: logger.warning('Unknown logging command {0} with message {1}'.format(comm_type, comm))
def run(cls, comm_type, comm, **kwargs): if comm_type == 'dda_c': return cls._dot_devassistant_create(comm, **kwargs) elif comm_type == 'dda_r': return cls._dot_devassistant_read(comm, **kwargs) else: logger.warning('Unknown .devassistant command {0}, skipping.'.format(comm_type))
def _github_create_twofactor_authorization(cls, ui): """Create an authorization for a GitHub user using two-factor authentication. Unlike its non-two-factor counterpart, this method does not traverse the available authentications as they are not visible until the user logs in. Please note: cls._user's attributes are not accessible until the authorization is created due to the way (py)github works. """ try: try: # This is necessary to trigger sending a 2FA key to the user auth = cls._user.create_authorization() except cls._gh_exceptions.GithubException: onetime_pw = DialogHelper.ask_for_password( ui, prompt='Your one time password:'******'repo', 'user', 'admin:public_key'], note="DevAssistant", onetime_password=onetime_pw) cls._user = cls._gh_module.Github( login_or_token=auth.token).get_user() logger.debug( 'Two-factor authorization for user "{0}" created'.format( cls._user.login)) cls._github_store_authorization(cls._user, auth) logger.debug('Two-factor authorization token stored') except cls._gh_exceptions.GithubException as e: logger.warning( 'Creating two-factor authorization failed: {0}'.format(e))
def dependencies_section(section, kwargs, runner=None): # "deps" is the same structure as gets returned by "dependencies" method skip_else = False deps = [] for i, dep in enumerate(section): if getattr(runner, 'stop_flag', False): break for dep_type, dep_list in dep.items(): # rpm dependencies (can't handle anything else yet) # we don't allow general commands, only "call"/"use" command here if dep_type in ['call', 'use']: deps.extend(Command(dep_type, dep_list, kwargs).run()) elif dep_type in package_managers.managers.keys(): # handle known types of deps the same, just by appending to "deps" list fmtd = list(map(lambda dep: format_str(dep, kwargs), dep_list)) deps.append({dep_type: fmtd}) elif dep_type.startswith('if'): possible_else = None if len(section) > i + 1: # do we have "else" clause? possible_else = list(section[i + 1].items())[0] _, skip_else, to_run = get_section_from_condition((dep_type, dep_list), possible_else, kwargs) if to_run: deps.extend(dependencies_section(to_run, kwargs, runner=runner)) elif dep_type == 'else': # else on its own means error, otherwise execute it if not skip_else: msg = 'Yaml error: encountered "else" with no associated "if", skipping.' logger.error(msg) raise exceptions.YamlSyntaxError(msg) skip_else = False else: logger.warning('Unknown dependency type {0}, skipping.'.format(dep_type)) return deps
def _github_fork(cls, **kwargs): """Create a fork of repo from kwargs['fork_repo']. Note: the kwargs are not the global context here, but what cls.format_args returns. Raises: devassistant.exceptions.CommandException on error """ fork_login, fork_reponame = kwargs['repo_url'].split('/') logger.info('Forking {repo} for user {login} on Github ...'.\ format(login=kwargs['login'], repo=kwargs['repo_url'])) success = False msg = '' try: repo = cls._gh_module.Github().get_user(fork_login).get_repo(fork_reponame) fork = cls._user.create_fork(repo) success = True msg = fork.ssh_url except cls._gh_module.GithubException as e: msg = 'Failed to create Github fork with error: {err}'.format(err=e) except BaseException as e: msg = 'Exception while forking GH repo: {0}'.\ format(getattr(e, 'message', 'Unknown error')) if success: logger.info('Fork is ready at {url}.'.format(url=fork.html_url)) else: logger.warning(msg) return (success, msg)
def _construct_args(self, struct): args = [] for arg_name, arg_params in struct.items(): use_snippet = arg_params.pop('use', None) or arg_params.pop('snippet', None) if use_snippet: # if snippet is used, take this parameter from snippet and update # it with current arg_params, if any try: problem = None snippet = yaml_snippet_loader.YamlSnippetLoader.get_snippet_by_name(use_snippet) arg_params = dict(snippet.args.pop(arg_name), **arg_params) except exceptions.SnippetNotFoundException as e: problem = 'Couldn\'t expand argument {arg} in assistant {a}: ' + six.text_type(e) except KeyError as e: # snippet doesn't have the requested argument problem = 'Couldn\'t find argument {arg} in snippet {snip} wanted by assistant {a}.' if problem: logger.warning(problem.format(snip=use_snippet, arg=arg_name, a=self.name)) continue # this works much like snippet.args.pop(arg_name).update(arg_params), # but unlike it, this actually returns the updated dict arg = argument.Argument(arg_name, *arg_params.pop('flags'), **arg_params) args.append(arg) return args
def signal_handler(signal, frame): if package_managers.DependencyInstaller.install_lock: logger.warning('Can\'t interrupt dependency installation!') else: logger.info('DevAssistant received SIGINT, exiting ...') utils.run_exitfuncs() sys.exit(0)
def _try_login_with_password_ntimes(cls, login, ntimes): user = None for i in range(0, ntimes): password = DialogHelper.ask_for_password( prompt='Github Password for {username}:'.format(username=login)) # user pressed Ctrl + D if password is None: break gh = cls._gh_module.Github(login_or_token=login, password=password) user = gh.get_user() try: user.login break # if user.login doesn't raise, authentication has been successful except cls._gh_module.GithubException as e: user = None msg = 'Wrong Github username or password; message from Github: {0}\n'.\ format(e.data.get('message', 'Unknown authentication error')) msg += 'Try again or press {0} to abort.' if current_run.UI == 'cli': msg = msg.format('Ctrl + D') else: msg = msg.format('"Cancel"') logger.warning(msg) return user
def get_assistants_from_file_hierarchy(cls, file_hierarchy, superassistant, role=settings.DEFAULT_ASSISTANT_ROLE): """Accepts file_hierarch as returned by cls.get_assistant_file_hierarchy and returns instances of YamlAssistant for loaded files Args: file_hierarchy: structure as described in cls.get_assistants_file_hierarchy role: role of all assistants in this hierarchy (we could find this out dynamically but it's not worth the pain) Returns: list of top level assistants from given hierarchy; these assistants contain references to instances of their subassistants (and their subassistants, ...) """ result = [] warn_msg = 'Failed to load assistant {source}, skipping subassistants.' for name, attrs in file_hierarchy.items(): loaded_yaml = yaml_loader.YamlLoader.load_yaml_by_path(attrs['source']) if loaded_yaml is None: # there was an error parsing yaml logger.warning(warn_msg.format(source=attrs['source'])) continue try: ass = cls.assistant_from_yaml(attrs['source'], loaded_yaml, superassistant, role=role) except exceptions.YamlError as e: logger.warning(e) continue ass._subassistants = cls.get_assistants_from_file_hierarchy(attrs['subhierarchy'], ass, role=role) result.append(ass) return result
def _dot_devassistant_dependencies(cls, comm, **kwargs): struct = [] dda_content = cls._dot_devassistant_read(comm, **kwargs) original_assistant_path = dda_content.get('subassistant_path', []) if original_assistant_path: # if we have an original path, try to get original assistant original_path_as_dict = {} for i, subas in enumerate(original_assistant_path): original_path_as_dict[settings.SUBASSISTANT_N_STRING.format( i)] = subas from devassistant.bin import CreatorAssistant from devassistant import yaml_assistant try: path = CreatorAssistant().get_selected_subassistant_path( **original_path_as_dict) except exceptions.AssistantNotFoundException as e: path = [] logger.warning(str(e)) for a in path: #TODO: maybe figure out more DRY code (similar is in path_runner, too) if 'dependencies' in vars(a.__class__) or isinstance( a, yaml_assistant.YamlAssistant): struct.extend( a.dependencies( **dda_content.get('original_kwargs', {}))) struct.extend(kwargs['__assistant__']._dependencies_section( dda_content.get('dependencies', []), **kwargs)) run_command('dependencies', struct, **kwargs)
def _github_push(cls): try: ret = ClHelper.run_command("git push -u origin master") logger.info('Source code was successfully pushed.') return (True, ret) except exceptions.ClException as e: logger.warning('Problem pushing source code: {0}'.format(e.output)) return (False, e.output)
def run(cls, comm_type, comm, **kwargs): if comm_type == 'dda_c': return cls._dot_devassistant_create(comm, **kwargs) elif comm_type == 'dda_r': return cls._dot_devassistant_read(comm, **kwargs) else: logger.warning( 'Unknown .devassistant command {0}, skipping.'.format( comm_type))
def run(cls, comm_type, comm, **kwargs): if comm == 'create_repo': cls._github_create_repo(**kwargs) elif comm == 'push': cls._github_push(**kwargs) elif comm == 'create_and_push': cls._github_create_and_push(**kwargs) else: logger.warning('Unknow github command {0}, skipping.'.format(comm))
def inner(func_cls, *args, **kwargs): if not cls._gh_module: logger.warning('PyGithub not installed, skipping github authentication procedures.') elif not func_cls._user: # authenticate user, possibly also creating authentication for future use func_cls._user = cls._get_github_user(kwargs['login']) # create ssh key for pushing cls._github_create_ssh_key() return func(func_cls, *args, **kwargs)
def _construct_args(self, struct): args = [] for arg_name, arg_params in struct.items(): try: args.append(argument.Argument.construct_arg(arg_name, arg_params)) except exceptions.ExecutionException as e: msg = 'Problem when constructing argument {arg} in assistant {a}: {e}'.\ format(arg=arg_name, a=self.name, e=six.text_type(e)) logger.warning(msg) return args
def load_yaml_by_path(cls, path): """Load a yaml file that is at given path""" try: return yaml.load(open(path, 'r'), Loader=Loader) except yaml.scanner.ScannerError as e: logger.warning('Yaml error in {path} (line {ln}, column {col}): {err}'.\ format(path=path, ln=e.problem_mark.line, col=e.problem_mark.column, err=e.problem)) return None
def load_yaml_by_path(cls, path): """Load a yaml file that is at given path""" try: return yaml.load(open(path, 'r'), Loader=Loader) or {} except yaml.scanner.ScannerError as e: logger.warning('Yaml error in {path} (line {ln}, column {col}): {err}'.\ format(path=path, ln=e.problem_mark.line, col=e.problem_mark.column, err=e.problem)) return None
def _construct_args(self, struct): args = [] for arg_name, arg_params in struct.items(): try: args.append( argument.Argument.construct_arg(arg_name, arg_params)) except exceptions.ExecutionException as e: msg = 'Problem when constructing argument {arg} in assistant {a}: {e}'.\ format(arg=arg_name, a=self.name, e=six.text_type(e)) logger.warning(msg) return args
def run(cls, comm_type, comm, **kwargs): if comm_type in map(lambda x: 'log_{0}'.format(x), settings.LOG_LEVELS_MAP): logger.log( logging._levelNames[settings.LOG_LEVELS_MAP[comm_type[-1]]], comm) if comm_type[-1] in 'ce': raise exceptions.RunException(comm) else: logger.warning( 'Unknown logging command {0} with message {1}'.format( comm_type, comm))
def run(cls, comm_type, comm, **kwargs): if not cls._gh_module: logger.warning('PyGithub not installed, cannot execute github command.') return if comm == 'create_repo': cls._github_create_repo(**kwargs) elif comm == 'push': cls._github_push(**kwargs) elif comm == 'create_and_push': cls._github_create_and_push(**kwargs) else: logger.warning('Unknow github command {0}, skipping.'.format(comm))
def inner(func_cls, *args, **kwargs): if not cls._gh_module: logger.warning( 'PyGithub not installed, skipping github authentication procedures.' ) elif not func_cls._user: # authenticate user, possibly also creating authentication for future use func_cls._user = cls._get_github_user( cls._github_login(**kwargs), cls._github_token(**kwargs), **kwargs) # create ssh key for pushing cls._github_create_ssh_key(**kwargs) func(func_cls, *args, **kwargs)
def inner(func_cls, *args, **kwargs): if not cls._gh_module: logger.warning('PyGithub not installed, skipping github authentication procedures.') elif not func_cls._user: # authenticate user, possibly also creating authentication for future use func_cls._user = cls._get_github_user(kwargs['login']) # create an ssh key for pushing if we don't have one if not cls._github_ssh_key_exists(): cls._github_create_ssh_key() # next, create ~/.ssh/config entry for the key, if system username != GH login if cls._ssh_key_needs_config_entry(): cls._create_ssh_config_entry() return func(func_cls, *args, **kwargs)
def run(cls, comm_type, comm, **kwargs): if not cls._gh_module: logger.warning( 'PyGithub not installed, cannot execute github command.') return if comm == 'create_repo': cls._github_create_repo(**kwargs) elif comm == 'push': cls._github_push(**kwargs) elif comm == 'create_and_push': cls._github_create_and_push(**kwargs) else: logger.warning('Unknow github command {0}, skipping.'.format(comm))
def _github_create_auth(cls, **kwargs): """ Store token into ~/.gitconfig. If token is not defined then store it into ~/.gitconfig file """ if not cls._token: try: auth = cls._user.create_authorization(scopes=['repo', 'user'], note="DeveloperAssistant") ClHelper.run_command("git config --global github.token.{login} {token}".format( login=cls._user.login, token=auth.token)) ClHelper.run_command("git config --global github.user.{login} {login}".format( login=cls._user.login)) except cls._gh_module.GithubException as e: logger.warning('Creating authorization failed: {0}'.format(e))
def _github_create_auth(cls): """ Store token into ~/.gitconfig. If token is not defined then store it into ~/.gitconfig file """ if not cls._token: try: auth = cls._user.create_authorization(scopes=['repo', 'user'], note="DeveloperAssistant") ClHelper.run_command("git config --global github.token.{login} {token}".format( login=cls._user.login, token=auth.token)) ClHelper.run_command("git config --global github.user.{login} {login}".format( login=cls._user.login)) except cls._gh_module.GithubException as e: logger.warning('Creating authorization failed: {0}'.format(e))
def _dependencies_section(self, section, **kwargs): for dep in section: dep_type, dep_list = dep.popitem() # rpm dependencies (can't handle anything else yet) if dep_type == 'rpm': self._install_dependencies(*dep_list, **kwargs) elif dep_type == 'snippet': snippet, section_name = self._get_snippet_and_section_name(dep_list, **kwargs) section = snippet.get_dependencies_section(section_name) if snippet else None if section: self._dependencies_section(section, **kwargs) else: logger.warning('Couldn\'t find dependencies section "{0}", in snippet {1}, skipping.'.format(section_name, dep_list.split('(')[0])) else: logger.warning('Unknown dependency type {0}, skipping.'.format(dep_type))
def _github_create_simple_authorization(cls): """Create a GitHub authorization for the given user in case they don't already have one. """ try: auth = None for a in cls._user.get_authorizations(): if a.note == 'DevAssistant': auth = a if not auth: auth = cls._user.create_authorization( scopes=['repo', 'user', 'admin:public_key'], note="DevAssistant") cls._github_store_authorization(cls._user, auth) except cls._gh_exceptions.GithubException as e: logger.warning('Creating authorization failed: {0}'.format(e))
def _run_one_section(self, section, kwargs): execute_else = False for i, command_dict in enumerate(section): for comm_type, comm in command_dict.items(): if comm_type.startswith('run'): s = self._get_section_to_run(section=comm, kwargs_override=False, **kwargs) # use copy of kwargs, so that local kwargs don't get modified self._run_one_section(s, copy.deepcopy(kwargs)) elif comm_type == 'snippet': snippet, section_name = self._get_snippet_and_section_name(comm, **kwargs) section = snippet.get_run_section(section_name) if snippet else None if section: # push and pop snippet's files into kwargs if '__files__' in kwargs: kwargs['__files__'].append(snippet.get_files_section()) else: kwargs['__files__'] = [snippet.get_files_section()] # use copy of kwargs, so that local kwargs don't get modified self._run_one_section(section, copy.deepcopy(kwargs)) kwargs['__files__'].pop() else: logger.warning('Couldn\'t find run section "{0}", in snippet {1} skipping.'.format(section_name, comm.split('(')[0])) elif comm_type.startswith('$'): # intentionally pass kwargs as dict, not as keywords self._assign_variable(comm_type, comm, kwargs) elif comm_type.startswith('if'): if self._evaluate(comm_type[2:].strip(), **kwargs): # run with original kwargs, so that they might be changed for code after this if self._run_one_section(comm, kwargs) elif len(section) > i + 1: next_section_dict = section[i + 1] next_section_comm_type, next_section_comm = list(next_section_dict.items())[0] if next_section_comm_type == 'else': execute_else = True elif comm_type == 'else': # else on its own means error, otherwise execute it if not list(section[i - 1].items())[0][0].startswith('if'): logger.warning('Yaml error: encountered "else" with no associated "if", skipping.') elif execute_else: execute_else = False # run with original kwargs, so that they might be changed for code after this if self._run_one_section(comm, kwargs) else: files = kwargs['__files__'][-1] if kwargs.get('__files__', None) else self._files run_command(comm_type, CommandFormatter.format(comm, self.template_dir, files, **kwargs), **kwargs)
def _github_add_remote_origin(cls, **kwargs): """Note: the kwargs are not the global context here, but what cls.format_args returns.""" reponame = kwargs['reponame'] login = kwargs['login'] # if system username != GH login, we need to use [email protected]{login}:... # else just [email protected]:... dash_login = '' if getpass.getuser() != login: dash_login = '******' + login try: logger.info('Adding Github repo as git remote ...') ret = ClHelper.run_command("git remote add origin [email protected]{dl}:{l}/{r}.git".\ format(dl=dash_login, l=login, r=reponame)) logger.info('Successfully added Github repo as git remote.') return (True, ret) except exceptions.ClException as e: logger.warning('Problem adding Github repo as git remote: {0}.'.format(e.output)) return (False, e.output)
def _construct_args(self, struct): args = [] # Construct properly the iterable args from either a dict, or a list if isinstance(struct, dict): temp_args = struct.items() elif isinstance(struct, list): temp_args = [list(argdict.items())[0] for argdict in struct] else: raise TypeError('Args struct should be dict or list, is {0}'.format(type(struct))) for (arg_name, arg_params) in temp_args: try: args.append(argument.Argument.construct_arg(arg_name, arg_params)) except exceptions.ExecutionException as e: msg = 'Problem when constructing argument {arg} in assistant {a}: {e}'.\ format(arg=arg_name, a=self.name, e=six.text_type(e)) logger.warning(msg) return args
def inner(func_cls, *args, **kwargs): if not cls._gh_module: logger.warning('PyGithub not installed, skipping Github auth procedures.') elif not func_cls._user: # authenticate user, possibly also creating authentication for future use login = kwargs['login'].encode(utils.defenc) if not six.PY3 else kwargs['login'] func_cls._user = cls._get_github_user(login, kwargs['ui']) if func_cls._user is None: msg = 'Github authentication failed, skipping Github command.' logger.warning(msg) return (False, msg) # create an ssh key for pushing if we don't have one if not cls._github_ssh_key_exists(): cls._github_create_ssh_key() # next, create ~/.ssh/config entry for the key, if system username != GH login if cls._ssh_key_needs_config_entry(): cls._create_ssh_config_entry() return func(func_cls, *args, **kwargs)
def logging(self, **kwargs): kwargs = self.proper_kwargs(**kwargs) for l in self._logging: handler_type, l_list = l.popitem() if handler_type == 'file': level, lfile = l_list expanded_lfile = self._format(lfile, **kwargs) # make dirs, create logger if not os.path.exists(os.path.dirname(expanded_lfile)): os.makedirs(os.path.dirname(expanded_lfile)) # add handler and formatter handler = logging.FileHandler(expanded_lfile, 'a+') formatter = logging.Formatter('%(asctime)-15s %(levelname)s - %(message)s') handler.setFormatter(formatter) handler.setLevel(getattr(logging, level.upper())) # register handler with the global logger logger.addHandler(handler) else: logger.warning('Unknown logger type {0}, ignoring.'.format(handler_type))
def logging(self, kwargs): # TODO: this doesn't seem to work, fix it... self.proper_kwargs('logging', kwargs) for l in self._logging: handler_type, l_list = l.popitem() if handler_type == 'file': level, lfile = l_list expanded_lfile = self._format(lfile, kwargs) # make dirs, create logger if not os.path.exists(os.path.dirname(expanded_lfile)): os.makedirs(os.path.dirname(expanded_lfile)) # add handler and formatter handler = logging.FileHandler(expanded_lfile, 'a+') formatter = logging.Formatter('%(asctime)-15s [%(event_type)] %(levelname)s - %(message)s') handler.setFormatter(formatter) handler.setLevel(getattr(logging, level.upper())) # register handler with the global logger logger.addHandler(handler) else: logger.warning('Unknown logger type {0}, ignoring.'.format(handler_type))
def inner(func_cls, *args, **kwargs): if not cls._gh_module: logger.warning( 'PyGithub not installed, skipping Github auth procedures.') elif not func_cls._user: # authenticate user, possibly also creating authentication for future use login = kwargs['login'].encode( utils.defenc) if not six.PY3 else kwargs['login'] func_cls._user = cls._get_github_user(login, kwargs['ui']) if func_cls._user is None: msg = 'Github authentication failed, skipping Github command.' logger.warning(msg) return (False, msg) # create an ssh key for pushing if we don't have one if not cls._github_ssh_key_exists(): cls._github_create_ssh_key() # next, create ~/.ssh/config entry for the key, if system username != GH login if cls._ssh_key_needs_config_entry(): cls._create_ssh_config_entry() return func(func_cls, *args, **kwargs)
def _dependencies_section(self, section, **kwargs): for dep in section: dep_type, dep_list = dep.popitem() # rpm dependencies (can't handle anything else yet) if dep_type == 'rpm': self._install_dependencies(*dep_list, **kwargs) elif dep_type == 'snippet': snippet, section_name = self._get_snippet_and_section_name( dep_list, **kwargs) section = snippet.get_dependencies_section( section_name) if snippet else None if section: self._dependencies_section(section, **kwargs) else: logger.warning( 'Couldn\'t find dependencies section "{0}", in snippet {1}, skipping.' .format(section_name, dep_list.split('(')[0])) else: logger.warning( 'Unknown dependency type {0}, skipping.'.format(dep_type))
def _dependencies_section(self, section, **kwargs): # "deps" is the same structure as gets returned by "dependencies" method skip_else = False deps = [] for i, dep in enumerate(section): for dep_type, dep_list in dep.items(): # rpm dependencies (can't handle anything else yet) if dep_type == 'call': section = self._get_section_from_call(dep_list, 'dependencies', **kwargs) if section is not None: deps.extend(self._dependencies_section(section, **kwargs)) else: logger.warning('Couldn\'t find dependencies section "{0}", in snippet {1}, skipping.'.format(dep_list.split('.'))) elif dep_type in package_managers.managers.keys(): # handle known types of deps the same, just by appending to "deps" list deps.append({dep_type: dep_list}) elif dep_type.startswith('if'): possible_else = None if len(section) > i + 1: # do we have "else" clause? possible_else = list(section[i + 1].items())[0] _, skip_else, to_run = self._get_section_from_condition((dep_type, dep_list), possible_else, **kwargs) if to_run: deps.extend(self._dependencies_section(to_run, **kwargs)) elif dep_type == 'else': # else on its own means error, otherwise execute it if not skip_else: logger.warning('Yaml error: encountered "else" with no associated "if", skipping.') skip_else = False else: logger.warning('Unknown dependency type {0}, skipping.'.format(dep_type)) return deps
def _github_create_repo(cls, **kwargs): """Create repo on GitHub. Note: the kwargs are not the global context here, but what cls.format_args returns. If repository already exists then CommandException will be raised. Raises: devassistant.exceptions.CommandException on error """ reponame = kwargs['reponame'] if reponame in map(lambda x: x.name, cls._user.get_repos()): msg = 'Failed to create Github repo: {0}/{1} alread exists.'.\ format(cls._user.login, reponame) logger.warning(msg) return (False, msg) else: msg = '' success = False try: new_repo = cls._user.create_repo(reponame, private=kwargs['private']) msg = new_repo.clone_url success = True except cls._gh_module.GithubException as e: gh_errs = e.data.get('errors', []) gh_errs = '; '.join(map(lambda err: err.get('message', ''), gh_errs)) msg = 'Failed to create GitHub repo. This sometime happens when you delete ' msg += 'a repo and then you want to create the same one immediately. If that\'s ' msg += 'the case, wait for few minutes and then try again.\n' msg += 'Github errors: ' + gh_errs except BaseException as e: msg = 'Failed to create Github repo: {0}'.\ format(getattr(e, 'message', 'Unknown error')) if success: logger.info('Your new repository: {0}'.format(new_repo.html_url)) else: logger.warning(msg) return (success, msg)
def _dot_devassistant_dependencies(cls, comm, **kwargs): struct = [] dda_content = cls._dot_devassistant_read(comm, **kwargs) original_assistant_path = dda_content.get('subassistant_path', []) if original_assistant_path: # if we have an original path, try to get original assistant original_path_as_dict = {} for i, subas in enumerate(original_assistant_path): original_path_as_dict[settings.SUBASSISTANT_N_STRING.format(i)] = subas from devassistant.bin import CreatorAssistant from devassistant import yaml_assistant try: path = CreatorAssistant().get_selected_subassistant_path(**original_path_as_dict) except exceptions.AssistantNotFoundException as e: path = [] logger.warning(str(e)) for a in path: #TODO: maybe figure out more DRY code (similar is in path_runner, too) if 'dependencies' in vars(a.__class__) or isinstance(a, yaml_assistant.YamlAssistant): struct.extend(a.dependencies(**dda_content.get('original_kwargs', {}))) struct.extend(kwargs['__assistant__']._dependencies_section(dda_content.get('dependencies', []), **kwargs)) run_command('dependencies', struct, **kwargs)
def _github_fork(cls, **kwargs): """Create a fork of repo from kwargs['fork_repo']. Note: the kwargs are not the global context here, but what cls.format_args returns. Raises: devassistant.exceptions.CommandException on error """ timeout = 300 # 5 minutes fork_login, fork_reponame = kwargs['repo_url'].split('/') logger.info('Forking {repo} for user {login} on Github ...'.\ format(login=kwargs['login'], repo=kwargs['repo_url'])) success = False msg = '' try: repo = cls._gh_module.Github().get_user(fork_login).get_repo(fork_reponame) fork = cls._user.create_fork(repo) while timeout > 0: time.sleep(5) timeout -= 5 try: fork.get_contents('/') # This function doesn't throw an exception when clonable success = True break except cls._gh_module.GithubException as e: if 'is empty' not in str(e): raise e msg = fork.ssh_url except cls._gh_module.GithubException as e: msg = 'Failed to create Github fork with error: {err}'.format(err=e) except BaseException as e: msg = 'Exception while forking GH repo: {0}'.\ format(getattr(e, 'message', 'Unknown error')) if success: logger.info('Fork is ready at {url}.'.format(url=fork.html_url)) else: logger.warning(msg) return (success, msg)
def dependencies_section(section, kwargs, runner=None): # "deps" is the same structure as gets returned by "dependencies" method skip_else = False deps = [] for i, dep in enumerate(section): if getattr(runner, 'stop_flag', False): break for dep_type, dep_list in dep.items(): # rpm dependencies (can't handle anything else yet) # we don't allow general commands, only "call"/"use" command here if dep_type in ['call', 'use']: deps.extend(Command(dep_type, dep_list, kwargs).run()) # handle known types of deps the same, just by appending to "deps" list elif dep_type in package_managers.managers.keys(): fmtd = list(map(lambda dep: format_str(dep, kwargs), dep_list)) deps.append({dep_type: fmtd}) elif dep_type.startswith('if'): possible_else = None if len(section) > i + 1: # do we have "else" clause? possible_else = list(section[i + 1].items())[0] _, skip_else, to_run = get_section_from_condition( (dep_type, dep_list), possible_else, kwargs) if to_run: deps.extend( dependencies_section(to_run, kwargs, runner=runner)) elif dep_type == 'else': # else on its own means error, otherwise execute it if not skip_else: msg = 'Yaml error: encountered "else" with no associated "if", skipping.' logger.error(msg) raise exceptions.YamlSyntaxError(msg) skip_else = False else: logger.warning( 'Unknown dependency type {0}, skipping.'.format(dep_type)) return deps
def get_assistants_from_file_hierarchy( cls, file_hierarchy, superassistant, role=settings.DEFAULT_ASSISTANT_ROLE): """Accepts file_hierarch as returned by cls.get_assistant_file_hierarchy and returns instances of YamlAssistant for loaded files Args: file_hierarchy: structure as described in cls.get_assistants_file_hierarchy role: role of all assistants in this hierarchy (we could find this out dynamically but it's not worth the pain) Returns: list of top level assistants from given hierarchy; these assistants contain references to instances of their subassistants (and their subassistants, ...) """ result = [] warn_msg = 'Failed to load assistant {source}, skipping subassistants.' for name, attrs in file_hierarchy.items(): loaded_yaml = yaml_loader.YamlLoader.load_yaml_by_path( attrs['source']) if loaded_yaml is None: # there was an error parsing yaml logger.warning(warn_msg.format(source=attrs['source'])) continue try: ass = cls.assistant_from_yaml(attrs['source'], loaded_yaml, superassistant, role=role) except exceptions.YamlError as e: logger.warning(e) continue ass._subassistants = cls.get_assistants_from_file_hierarchy( attrs['subhierarchy'], ass, role=role) result.append(ass) return result
def _github_create_auth(cls): """ Store token into ~/.gitconfig. Note: this uses cls._user.get_authorizations(), which only works if cls._user was authorized by login/password, doesn't work for token auth (TODO: why?). If token is not defined then store it into ~/.gitconfig file """ if not cls._token: try: auth = None for a in cls._user.get_authorizations(): if a.note == 'DevAssistant': auth = a if not auth: auth = cls._user.create_authorization( scopes=['repo', 'user', 'admin:public_key'], note="DevAssistant") ClHelper.run_command("git config --global github.token.{login} {token}".format( login=cls._user.login, token=auth.token)) ClHelper.run_command("git config --global github.user.{login} {login}".format( login=cls._user.login)) except cls._gh_module.GithubException as e: logger.warning('Creating authorization failed: {0}'.format(e))
def _dependencies_section(self, section, **kwargs): # "deps" is the same structure as gets returned by "dependencies" method skip_else = False deps = [] for i, dep in enumerate(section): for dep_type, dep_list in dep.items(): # rpm dependencies (can't handle anything else yet) if dep_type == 'call': section = self._get_section_from_call( dep_list, 'dependencies', **kwargs) if section is not None: deps.extend( self._dependencies_section(section, **kwargs)) else: logger.warning( 'Couldn\'t find dependencies section "{0}", in snippet {1}, skipping.' .format(dep_list.split('.'))) elif dep_type in package_managers.managers.keys( ): # handle known types of deps the same, just by appending to "deps" list deps.append({dep_type: dep_list}) elif dep_type.startswith('if'): possible_else = None if len(section) > i + 1: # do we have "else" clause? possible_else = list(section[i + 1].items())[0] _, skip_else, to_run = self._get_section_from_condition( (dep_type, dep_list), possible_else, **kwargs) if to_run: deps.extend( self._dependencies_section(to_run, **kwargs)) elif dep_type == 'else': # else on its own means error, otherwise execute it if not skip_else: logger.warning( 'Yaml error: encountered "else" with no associated "if", skipping.' ) skip_else = False else: logger.warning( 'Unknown dependency type {0}, skipping.'.format( dep_type)) return deps
def _run_one_section(self, section, kwargs): skip_else = False for i, command_dict in enumerate(section): if self.stop_flag: break for comm_type, comm in command_dict.items(): if comm_type.startswith('call'): # calling workflow: # 1) get proper run section (either from self or from snippet) # 2) if running snippet, add its files to kwargs['__files__'] # 3) actually run # 4) if running snippet, pop its files from kwargs['__files__'] sect = self._get_section_from_call(comm, 'run') if sect is None: logger.warning( 'Couldn\'t find section to run: {0}.'.format(comm)) continue if self._is_snippet_call(comm, **kwargs): # we're calling a snippet => add files and template_dir to kwargs snippet = yaml_snippet_loader.YamlSnippetLoader.get_snippet_by_name( comm.split('.')[0]) if '__files__' not in kwargs: kwargs['__files__'] = [] kwargs['__template_dir__'] = [] kwargs['__files__'].append(snippet.get_files_section()) kwargs['__template_dir__'].append( snippet.get_template_dir()) self._run_one_section(sect, copy.deepcopy(kwargs)) if self._is_snippet_call(comm, **kwargs): kwargs['__files__'].pop() kwargs['__template_dir__'].pop() elif comm_type.startswith('$'): # intentionally pass kwargs as dict, not as keywords try: self._assign_variable(comm_type, comm, kwargs) except exceptions.YamlSyntaxError as e: logger.error(e) raise e elif comm_type.startswith('if'): possible_else = None if len(section) > i + 1: # do we have "else" clause? possible_else = list(section[i + 1].items())[0] _, skip_else, to_run = self._get_section_from_condition( (comm_type, comm), possible_else, **kwargs) if to_run: # run with original kwargs, so that they might be changed for code after this if self._run_one_section(to_run, kwargs) elif comm_type == 'else': if not skip_else: logger.warning( 'Yaml error: encountered "else" with no associated "if", skipping.' ) skip_else = False elif comm_type.startswith('for'): # syntax: "for $i in $x: <section> or "for $i in cl_command: <section>" try: control_var, expression = self._parse_for(comm_type) except exceptions.YamlSyntaxError as e: logger.error(e) raise e try: eval_expression = evaluate_expression() except exceptions.YamlSyntaxError as e: logger.log(e) raise e for i in eval_expression: kwargs[control_var] = i self._run_one_section(comm, kwargs) elif comm_type.startswith('scl'): if '__scls__' not in kwargs: kwargs['__scls__'] = [] # list of lists of scl names kwargs['__scls__'].append(comm_type.split()[1:]) self._run_one_section(comm, kwargs) kwargs['__scls__'].pop() else: files = kwargs['__files__'][-1] if kwargs.get( '__files__', None) else self._files template_dir = kwargs['__template_dir__'][-1] if kwargs.get( '__template_dir__', None) else self.template_dir run_command( comm_type, CommandFormatter.format(comm_type, comm, template_dir, files, **kwargs), **kwargs)