def __init__(self, *args, **kwargs): """ Base class to wrap ctags program. - B{Keyword Arguments:} - B{tag_program:} (str) path to ctags executable, or name of a ctags program in path - B{files:} (sequence) files to process with ctags """ valid_kwargs = ['tag_program', 'files'] validator.validate(kwargs.keys(), valid_kwargs) self._file_list = list() """ A list of file names to process.""" self._executable_path = None """ The ctags executable.""" self.command_line = None """ The command line generated and used.""" self.warnings = list() """ A place to store warnings from ctags.""" if 'tag_program' in kwargs: if (self.ctags_executable(kwargs['tag_program'])): self._executable_path = kwargs['tag_program'] if 'files' in kwargs: self._file_list = list(kwargs['files'])
def generate_tagfile(self, output_file, **kwargs): """ Generates tag file from list of files. - B{Keyword Arguments:} - B{tag_program:} (str) path to ctags executable, or name of a ctags program in path - B{files:} (sequence) files to process with ctags - B{generator_options:} (dict) options to pass to ctags program @param output_file: File name and location to write tagfile. @type output_file: str @returns: file written @rtype: boolean @raise ValueError: ctags executable path not set or output file isn't valid """ valid_kwargs = ['tag_program', 'files', 'generator_options'] validator.validate(kwargs.keys(), valid_kwargs) # exuberant ctags 5.7 chops 'def' off the beginning of variables, if it starts with def _default_output_file = 'tags' if 'generator_options' in kwargs: if '-e' in kwargs['generator_options']: _default_output_file.upper() if output_file: if output_file != "-": if os.path.isdir(output_file): output_file = os.path.join(output_file, _default_output_file) else: (head, tail) = os.path.split(output_file) if len(head) == 0 and len(tail) == 0: raise ValueError("No output file set") if len(head) != 0: if not os.path.isdir(head): raise ValueError("Output directory " + head + " does not exist.") else: raise ValueError("No output file set") (gen_opts, file_list) = self._prepare_to_generate(kwargs) gen_opts['-f'] = '"' + output_file + '"' tag_args = self._dict_to_args(gen_opts) self.command_line = self._executable_path + ' ' + tag_args p = subprocess.Popen(self.command_line, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (out, err) = p.communicate(input=file_list.encode()) if sys.platform == 'win32': self.warnings = out.decode("utf-8").splitlines() else: self.warnings = err.decode("utf-8").splitlines() if (p.returncode == 0): return True return False
def generate_tags(self, **kwargs): """ Parses source files into list of tags. - B{Keyword Arguments:} - B{tag_program:} (str) path to ctags executable, or name of a ctags program in path - B{files:} (sequence) files to process with ctags - B{generator_options:} (dict) command-line options to pass to ctags program @returns: strings output by exuberant ctags @rtype: list @raise ValueError: ctags executable path not set, fails execution """ valid_kwargs = ['tag_program', 'files', 'generator_options'] validator.validate(kwargs.keys(), valid_kwargs) (gen_opts, file_list) = self._prepare_to_generate(kwargs) tag_args = self._dict_to_args(gen_opts) self.command_line = self._executable_path + ' ' + tag_args p = subprocess.Popen(self.command_line, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (out, err) = p.communicate(input=file_list.encode()) if p.returncode != 0: raise ValueError( "Ctags execution did not complete, return value: " + p.returncode + ".\nCommand line: " + self.command_line) results = out.decode("utf-8").splitlines() if sys.platform == 'win32': # check for warning strings in output if self._executable_path.rfind("/") >= 0: shortname = self._executable_path[self._executable_path. rfind("/"):] elif self._executable_path.rfind("\\") >= 0: shortname = self._executable_path[self._executable_path. rfind("\\"):] else: shortname = self._executable_path idxs = [] i = 0 for r in results: if r.find(shortname + self.__warning_str) == 0: idxs.append(i) i += 1 # reverse the list so we don't mess up index numbers as we're removing them idxs.sort(reverse=True) for i in idxs: self.warnings.append(results.pop(i)) else: self.warnings = err.decode("utf-8").splitlines() return results
def generate_tags(self, **kwargs): """ Parses source files into list of tags. - B{Keyword Arguments:} - B{tag_program:} (str) path to ctags executable, or name of a ctags program in path - B{files:} (sequence) files to process with ctags - B{generator_options:} (dict) command-line options to pass to ctags program @returns: strings output by exuberant ctags @rtype: list @raise ValueError: ctags executable path not set, fails execution """ valid_kwargs = ["tag_program", "files", "generator_options"] validator.validate(kwargs.keys(), valid_kwargs) (gen_opts, file_list) = self._prepare_to_generate(kwargs) tag_args = self._dict_to_args(gen_opts) self.command_line = self._executable_path + " " + tag_args p = subprocess.Popen( self.command_line, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True ) (out, err) = p.communicate(input=file_list.encode()) if p.returncode != 0: raise ValueError( "Ctags execution did not complete, return value: " + p.returncode + ".\nCommand line: " + self.command_line ) results = out.decode("utf-8").splitlines() if sys.platform == "win32": # check for warning strings in output if self._executable_path.rfind("/") >= 0: shortname = self._executable_path[self._executable_path.rfind("/") :] elif self._executable_path.rfind("\\") >= 0: shortname = self._executable_path[self._executable_path.rfind("\\") :] else: shortname = self._executable_path idxs = [] i = 0 for r in results: if r.find(shortname + self.__warning_str) == 0: idxs.append(i) i += 1 # reverse the list so we don't mess up index numbers as we're removing them idxs.sort(reverse=True) for i in idxs: self.warnings.append(results.pop(i)) else: self.warnings = err.decode("utf-8").splitlines() return results
def generate_tagfile(self, output_file, **kwargs): """ Generates tag file from list of files. - B{Keyword Arguments:} - B{tag_program:} (str) path to ctags executable, or name of a ctags program in path - B{files:} (sequence) files to process with ctags - B{generator_options:} (dict) options to pass to ctags program @param output_file: File name and location to write tagfile. @type output_file: str @returns: file written @rtype: boolean @raise ValueError: ctags executable path not set or output file isn't valid """ valid_kwargs = ["tag_program", "files", "generator_options"] validator.validate(kwargs.keys(), valid_kwargs) # exuberant ctags 5.7 chops 'def' off the beginning of variables, if it starts with def _default_output_file = "tags" if "generator_options" in kwargs: if "-e" in kwargs["generator_options"]: _default_output_file.upper() if output_file: if output_file != "-": if os.path.isdir(output_file): output_file = os.path.join(output_file, _default_output_file) else: (head, tail) = os.path.split(output_file) if len(head) == 0 and len(tail) == 0: raise ValueError("No output file set") if len(head) != 0: if not os.path.isdir(head): raise ValueError("Output directory " + head + " does not exist.") else: raise ValueError("No output file set") (gen_opts, file_list) = self._prepare_to_generate(kwargs) gen_opts["-f"] = '"' + output_file + '"' tag_args = self._dict_to_args(gen_opts) self.command_line = self._executable_path + " " + tag_args p = subprocess.Popen( self.command_line, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True ) (out, err) = p.communicate(input=file_list.encode()) if sys.platform == "win32": self.warnings = out.decode("utf-8").splitlines() else: self.warnings = err.decode("utf-8").splitlines() if p.returncode == 0: return True return False
def test_validator(self): fail = True try: validator.validate(['abc', 'ghi'], ['abc', 'def', 'ghi']) try: validator.validate(['abc', 'def', 'jkl'], ['abc', 'def', 'ghi']) except ParameterError: fail = False except: pass self.failIf(fail)
def __init__(self, tags=None, **kwargs): """ Initializes instances of ctags_file. - B{Keyword Arguments:} - B{harvesters:} (list) list of harvester classes @param tags: If I{tags} is a sequence, it will automatically be parsed. If it is a filename or path, it will be opened and parsed. @type tags: sequence or str """ valid_kwargs = ['harvesters'] validator.validate(kwargs.keys(), valid_kwargs) self._clear_variables() if tags: self.parse(tags, **kwargs)
def __init__(self, *args, **kwargs): """ Wraps the Exuberant Ctags program. - B{Keyword Arguments:} - B{tag_program:} (str) path to ctags executable, or name of a ctags program in path - B{files:} (sequence) files to process with ctags """ valid_kwargs = ['tag_program', 'files'] validator.validate(kwargs.keys(), valid_kwargs) self.version = None """ Exuberant ctags version number.""" self.language_info = None """ Exuberant ctags supported language parsing features.""" ctags_base.__init__(self, *args, **kwargs)
def feed_init(self, **kwargs): """ Initializes ctags_file data members and possible data harvesters. - B{Keyword Arguments:} - B{harvesters:} (list) list of harvester classes @raises ValueError: parsing error """ valid_kwargs = ['harvesters'] validator.validate(kwargs.keys(), valid_kwargs) self._clear_variables() self.__feed_harvesters = list() if 'harvesters' in kwargs: self.__feed_harvesters = kwargs['harvesters'] for h in self.__feed_harvesters: h.do_before()
def __init__(self, *args, **kwargs): """ Wraps the Exuberant Ctags program. - B{Keyword Arguments:} - B{tag_program:} (str) path to ctags executable, or name of a ctags program in path - B{files:} (sequence) files to process with ctags """ valid_kwargs = ["tag_program", "files"] validator.validate(kwargs.keys(), valid_kwargs) self.version = None """ Exuberant ctags version number.""" self.language_info = None """ Exuberant ctags supported language parsing features. None or dict. Language name as key, value is a list of language features.""" self.language_maps = None """ Exuberant ctags supported language file extensions to map to a language parser. None or dict. Language name is key, value is list of file extensions.""" self.all_extensions = None """ List of files and extensions that Exuberant Ctags knows how to map to a parser.""" ctags_base.__init__(self, *args, **kwargs)
def __init__(self, *args, **kwargs): """ Wraps the Exuberant Ctags program. - B{Keyword Arguments:} - B{tag_program:} (str) path to ctags executable, or name of a ctags program in path - B{files:} (sequence) files to process with ctags """ valid_kwargs = ['tag_program', 'files'] validator.validate(kwargs.keys(), valid_kwargs) self.version = None """ Exuberant ctags version number.""" self.language_info = None """ Exuberant ctags supported language parsing features. None or dict. Language name as key, value is a list of language features.""" self.language_maps = None """ Exuberant ctags supported language file extensions to map to a language parser. None or dict. Language name is key, value is list of file extensions.""" self.all_extensions = None """ List of files and extensions that Exuberant Ctags knows how to map to a parser.""" ctags_base.__init__(self, *args, **kwargs)
def starts_with(self, matchstr, **kwargs): """ Fetches an alphabetical list of unique tag names that begin with matchstr. - B{Parameters:} - B{matchstr:} (str) string to search for in tags db - B{Keyword Arguments:} - B{num_results:} (int) maximum number of results to return, 0 for all, default - B{case_sensitive:} (bool) whether to match case, default False @returns: matching tag names @rtype: list """ valid_kwargs = ['num_results', 'case_sensitive'] validator.validate(kwargs.keys(), valid_kwargs) final_list = [] case_sensitive = False num_results = 0 if 'num_results' in kwargs: num_results = int(kwargs['num_results']) if len(matchstr) == 0: if num_results: return self.__sorted_names[0:num_results] return self.__sorted_names[:] if 'case_sensitive' in kwargs: if kwargs['case_sensitive']: case_sensitive = True tag_names_that_start_with_char = [] if case_sensitive: if matchstr[0] not in self.__name_index: return [] else: if matchstr[0].lower() not in self.__name_index and matchstr[0].upper() not in self.__name_index: return [] if case_sensitive: idxs = self.__name_index[matchstr[0]] if idxs['first'] == idxs['last'] + 1: tag_names_that_start_with_char = self.__sorted_names[idxs['first']] else: tag_names_that_start_with_char = self.__sorted_names[idxs['first']:idxs['last'] + 1] else: if matchstr[0].lower() in self.__name_index: idxs = self.__name_index[matchstr[0].lower()] if idxs['first'] == idxs['last'] + 1: tag_names_that_start_with_char = self.__sorted_names[idxs['first']] else: tag_names_that_start_with_char = self.__sorted_names[idxs['first']:idxs['last'] + 1] if matchstr[0].upper() in self.__name_index: idxs = self.__name_index[matchstr[0].upper()] if idxs['first'] == idxs['last'] + 1: tag_names_that_start_with_char += [self.__sorted_names[idxs['first']]] else: tag_names_that_start_with_char += self.__sorted_names[idxs['first']:idxs['last'] + 1] if len(matchstr) == 1: if num_results == 0: return tag_names_that_start_with_char[:] else: return tag_names_that_start_with_char[0:num_results] if case_sensitive: for t in tag_names_that_start_with_char: if (t.find(matchstr) == 0): final_list.append(copy(t)) if num_results > 0 and len(final_list) == num_results: return final_list else: for t in tag_names_that_start_with_char: if (t.lower().find(matchstr.lower()) == 0): final_list.append(copy(t)) if num_results > 0 and len(final_list) == num_results: return final_list return final_list
def generate_object(self, **kwargs): """ Parses source files into a ctags_file instance. This method exists to avoid storing ctags generated data in an intermediate form before parsing. According to python documentation, this mechanism could deadlock due to other OS pipe buffers filling and blocking the child process. U{http://docs.python.org/library/subprocess.html} - B{Keyword Arguments:} - B{tag_program:} (str) path to ctags executable, or name of a ctags program in path - B{files:} (sequence) files to process with ctags - B{generator_options:} (dict) options to pass to ctags program - B{harvesters:} (list) list of harvester data classes for ctags_file to use while parsing @returns: generated instance of ctags_file on success, None on failure @rtype: (ctags_file or None) @raise ValueError: ctags executable path not set """ valid_kwargs = [ 'tag_program', 'files', 'generator_options', 'harvesters' ] validator.validate(kwargs.keys(), valid_kwargs) (gen_opts, file_list) = self._prepare_to_generate(kwargs) tag_args = self._dict_to_args(gen_opts) tagfile = ctags_file() harvesters = list() if 'harvesters' in kwargs: harvesters = kwargs['harvesters'] tagfile.feed_init(harvesters=harvesters) self.command_line = self._executable_path + ' ' + tag_args p = subprocess.Popen(self.command_line, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) p.stdin.write(file_list.encode()) # is this the cleanest way to do this? it makes the program execute, but I haven't found another way p.stdin.close() if sys.platform == "win32": if self._executable_path.rfind("/") >= 0: shortname = self._executable_path[self._executable_path. rfind("/"):] elif self._executable_path.rfind("\\") >= 0: shortname = self._executable_path[self._executable_path. rfind("\\"):] else: shortname = self._executable_path while p.poll() is None: line = p.stdout.readline().decode("utf-8") if not len(line): continue if sys.platform == 'win32' and line.find(shortname + self.__warning_str) == 0: self.warnings.append(line) else: tagfile.feed_line(line) # process the remaining buffer for line in p.stdout.read().decode("utf-8").splitlines(): if not len(line): continue if sys.platform == 'win32' and line.find(shortname + self.__warning_str) == 0: self.warnings.append(line) else: tagfile.feed_line(line) if sys.platform != 'win32': self.warnings = p.stderr.read().decode("utf-8").splitlines() tagfile.feed_finish() if p.returncode == 0: return tagfile else: return None
def generate_object(self, **kwargs): """ Parses source files into a ctags_file instance. This method exists to avoid storing ctags generated data in an intermediate form before parsing. According to python documentation, this mechanism could deadlock due to other OS pipe buffers filling and blocking the child process. U{http://docs.python.org/library/subprocess.html} - B{Keyword Arguments:} - B{tag_program:} (str) path to ctags executable, or name of a ctags program in path - B{files:} (sequence) files to process with ctags - B{generator_options:} (dict) options to pass to ctags program - B{harvesters:} (list) list of harvester data classes for ctags_file to use while parsing @returns: generated instance of ctags_file on success, None on failure @rtype: (ctags_file or None) @raise ValueError: ctags executable path not set """ valid_kwargs = ["tag_program", "files", "generator_options", "harvesters"] validator.validate(kwargs.keys(), valid_kwargs) (gen_opts, file_list) = self._prepare_to_generate(kwargs) tag_args = self._dict_to_args(gen_opts) tagfile = ctags_file() harvesters = list() if "harvesters" in kwargs: harvesters = kwargs["harvesters"] tagfile.feed_init(harvesters=harvesters) self.command_line = self._executable_path + " " + tag_args p = subprocess.Popen( self.command_line, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True ) p.stdin.write(file_list.encode()) # is this the cleanest way to do this? it makes the program execute, but I haven't found another way p.stdin.close() if sys.platform == "win32": if self._executable_path.rfind("/") >= 0: shortname = self._executable_path[self._executable_path.rfind("/") :] elif self._executable_path.rfind("\\") >= 0: shortname = self._executable_path[self._executable_path.rfind("\\") :] else: shortname = self._executable_path while p.poll() is None: line = p.stdout.readline().decode("utf-8") if not len(line): continue if sys.platform == "win32" and line.find(shortname + self.__warning_str) == 0: self.warnings.append(line) else: tagfile.feed_line(line) # process the remaining buffer for line in p.stdout.read().decode("utf-8").splitlines(): if not len(line): continue if sys.platform == "win32" and line.find(shortname + self.__warning_str) == 0: self.warnings.append(line) else: tagfile.feed_line(line) if sys.platform != "win32": self.warnings = p.stderr.read().decode("utf-8").splitlines() tagfile.feed_finish() if p.returncode == 0: return tagfile else: return None
def __init__(self, *args, **kwargs): """ A tag entry from ctags file. Initializes from str or keyword args. - B{Optional Parameters:} - B{args[0]}: (str) a ctags_entry repr or string from a tag file - B{Keyword Arguments:} - B{name}: (str) tag name - B{file}: (str) source file name - B{pattern}: (str) locator pattern for tag - B{line_number}: (int) locator line number - B{extensions}: (dict) extension fields - B{Raises:} - B{ValueError}: line_number or pattern isn't set, or a parameter type can't be transformed. """ valid_kwargs = ['name', 'file', 'pattern', 'line_number', 'extensions'] validator.validate(kwargs.keys(), valid_kwargs) self.name = None """ Tag name.""" self.file = None """ Source file of tag.""" self.pattern = None """ If not None, regular expression to locate this tag in self.file.""" self.line_number = None """ If not None, line number to locate this tag in self.file.""" self.extensions = None """ If not none, dict of extension fields embedded in comments in the tag entry, from exuberant ctags.""" self.__rep = None entry = dict() if len(args) == 1: if len(kwargs): raise ValueError("multiple tag data found in init") if type(args[0]) == dict: entry = args[0] elif (type(args[0]) == str or type(args[0]) == unicode) and len(args[0]): if args[0][0] == '{' and args[0][-1] == '}': # expect this to be a repr string # security anyone? entry = eval(args[0]) else: argstr = args[0].strip() # bah! uglies. if not _PYTHON_3000_ and type(argstr) is not unicode: argstr = unicode(argstr, "utf-8") # this should be a tag line, could use some safety checking here though (entry['name'], entry['file'], the_rest) = argstr.split('\t', 2) extension_fields = None if the_rest.find(_COMMENT_BEGIN_) > 0: (locator, junk, extension_fields) = the_rest.rpartition(_COMMENT_BEGIN_) else: locator = the_rest if locator.isdigit(): try: entry['line_number'] = int(locator) except ValueError: raise ValueError("Line number locator found for tag, but can't be converted to integer") else: # should be a regex pattern entry['pattern'] = locator entry['extensions'] = {} kind_arg_found = False if extension_fields: if extension_fields[0] == '\t': # probably exuberant ctags format extension_list = extension_fields[1:].split('\t') for ext in extension_list: if ':' in ext: (k, v) = ext.split(':', 1) entry['extensions'][k] = v if k == 'line' and 'line_number' not in entry: try: entry['line_number'] = int(v) except ValueError: raise ValueError("Extended tag 'line' found but can't be converted to integer.") else: if kind_arg_found: raise ValueError("Unknown extended tag found.") else: entry['extensions']['kind'] = ext kind_arg_found = True elif len(kwargs): entry = kwargs if 'file' in entry: self.file = entry['file'] else: raise ValueError("'file' parameter is required") if 'name' in entry: self.name = entry['name'] else: raise ValueError("'name' parameter is required") if 'pattern' in entry: self.pattern = entry['pattern'] if 'line_number' in entry: self.line_number = int(entry['line_number']) if not self.line_number and not self.pattern: raise ValueError("No valid locator for this tag.") if 'extensions' in entry: self.extensions = entry['extensions'] self.__rep = entry
def starts_with(self, matchstr, **kwargs): """ Fetches an alphabetical list of unique tag names that begin with matchstr. - B{Parameters:} - B{matchstr:} (str) string to search for in tags db - B{Keyword Arguments:} - B{num_results:} (int) maximum number of results to return, 0 for all, default - B{case_sensitive:} (bool) whether to match case, default False @returns: matching tag names @rtype: list """ valid_kwargs = ['num_results', 'case_sensitive'] validator.validate(kwargs.keys(), valid_kwargs) final_list = [] case_sensitive = False num_results = 0 if 'num_results' in kwargs: num_results = int(kwargs['num_results']) if len(matchstr) == 0: if num_results: return self.__sorted_names[0:num_results] return self.__sorted_names[:] if 'case_sensitive' in kwargs: if kwargs['case_sensitive']: case_sensitive = True tag_names_that_start_with_char = [] if case_sensitive: if matchstr[0] not in self.__name_index: return [] else: if matchstr[0].lower() not in self.__name_index and matchstr[ 0].upper() not in self.__name_index: return [] if case_sensitive: idxs = self.__name_index[matchstr[0]] if idxs['first'] == idxs['last'] + 1: tag_names_that_start_with_char = self.__sorted_names[ idxs['first']] else: tag_names_that_start_with_char = self.__sorted_names[ idxs['first']:idxs['last'] + 1] else: if matchstr[0].lower() in self.__name_index: idxs = self.__name_index[matchstr[0].lower()] if idxs['first'] == idxs['last'] + 1: tag_names_that_start_with_char = self.__sorted_names[ idxs['first']] else: tag_names_that_start_with_char = self.__sorted_names[ idxs['first']:idxs['last'] + 1] if matchstr[0].upper() in self.__name_index: idxs = self.__name_index[matchstr[0].upper()] if idxs['first'] == idxs['last'] + 1: tag_names_that_start_with_char += [ self.__sorted_names[idxs['first']] ] else: tag_names_that_start_with_char += self.__sorted_names[ idxs['first']:idxs['last'] + 1] if len(matchstr) == 1: if num_results == 0: return tag_names_that_start_with_char[:] else: return tag_names_that_start_with_char[0:num_results] if case_sensitive: for t in tag_names_that_start_with_char: if (t.find(matchstr) == 0): final_list.append(copy(t)) if num_results > 0 and len(final_list) == num_results: return final_list else: for t in tag_names_that_start_with_char: if (t.lower().find(matchstr.lower()) == 0): final_list.append(copy(t)) if num_results > 0 and len(final_list) == num_results: return final_list return final_list