def process_text_to_dict(self, input_text): composer = Composer() builder = idiopidae.parser.parse('Document', input_text + "\n\0") ext = self.artifact.input_ext name = "input_text%s" % ext # List any file extensions which don't map neatly to lexers. if ext == '.pycon': lexer = PythonConsoleLexer() elif ext == '.rbcon': lexer = RubyConsoleLexer() elif ext in ('.json', '.dexy'): lexer = JavascriptLexer() else: lexer = get_lexer_for_filename(name) formatter = get_formatter_for_filename(self.artifact.filename(), lineanchors='l') output_dict = OrderedDict() for i, s in enumerate(builder.sections): lines = builder.statements[i]['lines'] formatted_lines = composer.format(lines, lexer, formatter) output_dict[s] = formatted_lines return output_dict
def process_text_to_dict(self, input_text): composer = Composer() builder = idiopidae.parser.parse('Document', input_text + "\n\0") args = self.args().copy() lexer = self.create_lexer_instance(args) formatter = self.create_formatter_instance(args) output_dict = OrderedDict() lineno = 1 for i, s in enumerate(builder.sections): self.log.debug("In section no. %s name %s" % (i, s)) lines = builder.statements[i]['lines'] if len(lines) == 0: next if not re.match("^\d+$", s): # Manually named section, the sectioning comment takes up a # line, so account for this to keep line nos in sync. lineno += 1 formatter.linenostart = lineno formatted_lines = composer.format(lines, lexer, formatter) output_dict[s] = formatted_lines lineno += len(lines) return output_dict
def process_text_to_dict(self, input_text): composer = Composer() builder = idiopidae.parser.parse('Document', input_text + "\n\0") ext = self.artifact.input_ext name = "input_text%s" % ext if self.artifact.args.has_key('pyg-lexer'): lexer = get_lexer_by_name(self.artifact.args['pyg-lexer']) elif ext == '.pycon': lexer = PythonConsoleLexer() elif ext == '.rbcon': lexer = RubyConsoleLexer() elif ext in ('.json', '.dexy'): lexer = JavascriptLexer() elif ext in ('.php'): # If we are using idio, then our code will be in sections so we # need to start inline with PHP. To avoid this, use pyg instead of # idio. (Eventually should be able to specify lexer + options in config.) lexer = PhpLexer(startinline=True) else: lexer = get_lexer_for_filename(name) if self.artifact.args.has_key('idio'): idio_args = self.artifact.args['idio'] elif self.artifact.args.has_key('pygments'): idio_args = self.artifact.args['pygments'] else: idio_args = {} formatter_args = {'lineanchors' : self.artifact.web_safe_document_key() } formatter_args.update(idio_args) formatter = get_formatter_for_filename(self.artifact.filename(), **formatter_args) output_dict = OrderedDict() lineno = 1 for i, s in enumerate(builder.sections): lines = builder.statements[i]['lines'] if len(lines) == 0: next if not re.match("^\d+$", s): # Manually named section, the sectioning comment takes up a # line, so account for this to keep line nos in sync. lineno += 1 formatter.linenostart = lineno formatted_lines = composer.format(lines, lexer, formatter) output_dict[s] = formatted_lines lineno += len(lines) return output_dict
class PythonDocumentationFilter(Filter): ALIASES = ["pydoc"] INPUT_EXTENSIONS = [".txt"] OUTPUT_EXTENSIONS = ['.json'] COMPOSER = Composer() OUTPUT_DATA_TYPE = 'keyvalue' LEXER = PythonLexer() LATEX_FORMATTER = LatexFormatter() HTML_FORMATTER = HtmlFormatter(lineanchors="pydoc") def fetch_item_content(self, key, item): is_method = inspect.ismethod(item) is_function = inspect.isfunction(item) if is_method or is_function: # Get source code try: source = inspect.getsource(item) except IOError as e: source = "" # Process any idiopidae tags builder = idiopidae.parser.parse('Document', source + "\n\0") sections = {} for i, s in enumerate(builder.sections): lines = builder.statements[i]['lines'] sections[s] = "\n".join(l[1] for l in builder.statements[i]['lines']) if isinstance(sections, dict): if len(sections.keys()) > 1 or sections.keys()[0] != '1': for section_name, section_content in sections.iteritems(): self.add_source_for_key("%s:%s" % (key, section_name), section_content) else: self.add_source_for_key(key, sections['1']) else: self.add_source_for_key(key, sections) self.artifact.output_data.append("%s:doc" % key, inspect.getdoc(item)) self.artifact.output_data.append("%s:comments" % key, inspect.getcomments(item)) else: # not a function or a method try: # If this can be JSON-serialized, leave it alone... json.dumps(item) self.add_source_for_key(key, item) except TypeError: # ... if it can't, convert it to a string to avoid problems. self.add_source_for_key(key, str(item)) def highlight_html(self, source): return highlight(source, self.LEXER, self.HTML_FORMATTER) def highlight_latex(self, source): return highlight(source, self.LEXER, self.LATEX_FORMATTER) def add_source_for_key(self, key, source): """ Appends source code + syntax highlighted source code to persistent store. """ self.artifact.output_data.append("%s:value" % key, source) if not (type(source) == str or type(source) == unicode): source = inspect.getsource(source) self.artifact.output_data.append("%s:source" % key, source) self.artifact.output_data.append("%s:html-source" % key, self.highlight_html(source)) self.artifact.output_data.append("%s:latex-source" % key, self.highlight_latex(source)) def process_members(self, package_name, mod): """ Process all members of the package or module passed. """ name = mod.__name__ for k, m in inspect.getmembers(mod): self.log.debug("in %s processing element %s" % (mod.__name__, k)) if not inspect.isclass(m) and hasattr(m, '__module__') and m.__module__ and m.__module__.startswith(package_name): key = "%s.%s" % (m.__module__, k) self.fetch_item_content(key, m) elif inspect.isclass(m) and m.__module__.startswith(package_name): key = "%s.%s" % (mod.__name__, k) try: item_content = inspect.getsource(m) self.artifact.output_data.append("%s:doc" % key, inspect.getdoc(m)) self.artifact.output_data.append("%s:comments" % key, inspect.getcomments(m)) self.add_source_for_key(key, item_content) except IOError: self.log.debug("can't get source for %s" % key) self.add_source_for_key(key, "") try: for ck, cm in inspect.getmembers(m): key = "%s.%s.%s" % (name, k, ck) self.fetch_item_content(key, cm) except AttributeError: pass else: key = "%s.%s" % (name, k) self.fetch_item_content(key, m) def process_module(self, package_name, name): try: self.log.debug("Trying to import %s" % name) __import__(name) mod = sys.modules[name] self.log.debug("Success importing %s" % name) try: module_source = inspect.getsource(mod) json.dumps(module_source) self.add_source_for_key(name, inspect.getsource(mod)) except (UnicodeDecodeError, IOError, TypeError): self.log.debug("Unable to load module source for %s" % name) self.process_members(package_name, mod) except (ImportError, TypeError) as e: self.log.debug(e) def process(self): """ input_text should be a list of installed python libraries to document. """ package_names = self.artifact.input_data.as_text().split() packages = [__import__(package_name) for package_name in package_names] for package in packages: self.log.debug("processing package %s" % package) package_name = package.__name__ prefix = package.__name__ + "." self.process_members(package_name, package) if hasattr(package, '__path__'): for module_loader, name, ispkg in pkgutil.walk_packages(package.__path__, prefix=prefix): self.log.debug("in package %s processing module %s" % (package_name, name)) if not name.endswith("__main__"): self.process_module(package_name, name) else: self.process_module(package.__name__, package.__name__) self.artifact.output_data.save()
try: import markdown except ImportError, error: print "You don't appear to have the markdown module installed: %s" % error sys.exit(1) return markdown.markdown(content) def load_code(content): "Convert text via idiopidae to insert code" try: from idiopidae.runtime import Composer except ImportError, error: print "You don't appear to have the idiopidae module installed: %s" % error sys.exit(1) composer = Composer() temporary_file = open("build/code.txt", "w") temporary_file.write(content) temporary_file.close() output = composer.process(temporary_file.name) os.unlink(temporary_file.name) return output def generate_html(content): "Create an html file from the current content buffer" try: from BeautifulSoup import BeautifulSoup except ImportError, error: print "You don't appear to have BeautifulSoup installed: %s" % error sys.exit(1)
from idiopidae.runtime import Composer from pygments import highlight from pygments.formatters.latex import LatexFormatter from pygments.lexers.agile import PythonLexer import dexy import idiopidae.parser import inspect import json import pkgutil import sys lexer = PythonLexer() formatter = LatexFormatter() composer = Composer() def fetch_item_content(cm): is_method = inspect.ismethod(cm) is_function = inspect.isfunction(cm) if is_method or is_function: try: source = inspect.getsource(cm) except IOError: source = "" builder = idiopidae.parser.parse('Document', source + "\n\0") sections = {} for i, s in enumerate(builder.sections): lines = builder.statements[i]['lines'] sections[s] = composer.format(lines, lexer, formatter)