def process_text_to_dict(self, input_text):
composer = Composer()
builder = idiopidae.parser.parse('Document', input_text + "\n\0")
ext = self.artifact.input_ext
name = "input_text%s" % ext
# List any file extensions which don't map neatly to lexers.
if ext == '.pycon':
lexer = PythonConsoleLexer()
elif ext == '.rbcon':
lexer = RubyConsoleLexer()
elif ext in ('.json', '.dexy'):
lexer = JavascriptLexer()
else:
lexer = get_lexer_for_filename(name)
formatter = get_formatter_for_filename(self.artifact.filename(),
lineanchors='l')
output_dict = OrderedDict()
for i, s in enumerate(builder.sections):
lines = builder.statements[i]['lines']
formatted_lines = composer.format(lines, lexer, formatter)
output_dict[s] = formatted_lines
return output_dict
def process_text_to_dict(self, input_text):
composer = Composer()
builder = idiopidae.parser.parse('Document', input_text + "\n\0")
args = self.args().copy()
lexer = self.create_lexer_instance(args)
formatter = self.create_formatter_instance(args)
output_dict = OrderedDict()
lineno = 1
for i, s in enumerate(builder.sections):
self.log.debug("In section no. %s name %s" % (i, s))
lines = builder.statements[i]['lines']
if len(lines) == 0:
next
if not re.match("^\d+$", s):
# Manually named section, the sectioning comment takes up a
# line, so account for this to keep line nos in sync.
lineno += 1
formatter.linenostart = lineno
formatted_lines = composer.format(lines, lexer, formatter)
output_dict[s] = formatted_lines
lineno += len(lines)
return output_dict
def process_text_to_dict(self, input_text):
composer = Composer()
builder = idiopidae.parser.parse('Document', input_text + "\n\0")
ext = self.artifact.input_ext
name = "input_text%s" % ext
if self.artifact.args.has_key('pyg-lexer'):
lexer = get_lexer_by_name(self.artifact.args['pyg-lexer'])
elif ext == '.pycon':
lexer = PythonConsoleLexer()
elif ext == '.rbcon':
lexer = RubyConsoleLexer()
elif ext in ('.json', '.dexy'):
lexer = JavascriptLexer()
elif ext in ('.php'):
# If we are using idio, then our code will be in sections so we
# need to start inline with PHP. To avoid this, use pyg instead of
# idio. (Eventually should be able to specify lexer + options in config.)
lexer = PhpLexer(startinline=True)
else:
lexer = get_lexer_for_filename(name)
if self.artifact.args.has_key('idio'):
idio_args = self.artifact.args['idio']
elif self.artifact.args.has_key('pygments'):
idio_args = self.artifact.args['pygments']
else:
idio_args = {}
formatter_args = {'lineanchors' : self.artifact.web_safe_document_key() }
formatter_args.update(idio_args)
formatter = get_formatter_for_filename(self.artifact.filename(),
**formatter_args)
output_dict = OrderedDict()
lineno = 1
for i, s in enumerate(builder.sections):
lines = builder.statements[i]['lines']
if len(lines) == 0:
next
if not re.match("^\d+$", s):
# Manually named section, the sectioning comment takes up a
# line, so account for this to keep line nos in sync.
lineno += 1
formatter.linenostart = lineno
formatted_lines = composer.format(lines, lexer, formatter)
output_dict[s] = formatted_lines
lineno += len(lines)
return output_dict
class PythonDocumentationFilter(Filter):
ALIASES = ["pydoc"]
INPUT_EXTENSIONS = [".txt"]
OUTPUT_EXTENSIONS = ['.json']
COMPOSER = Composer()
OUTPUT_DATA_TYPE = 'keyvalue'
LEXER = PythonLexer()
LATEX_FORMATTER = LatexFormatter()
HTML_FORMATTER = HtmlFormatter(lineanchors="pydoc")
def fetch_item_content(self, key, item):
is_method = inspect.ismethod(item)
is_function = inspect.isfunction(item)
if is_method or is_function:
# Get source code
try:
source = inspect.getsource(item)
except IOError as e:
source = ""
# Process any idiopidae tags
builder = idiopidae.parser.parse('Document', source + "\n\0")
sections = {}
for i, s in enumerate(builder.sections):
lines = builder.statements[i]['lines']
sections[s] = "\n".join(l[1] for l in builder.statements[i]['lines'])
if isinstance(sections, dict):
if len(sections.keys()) > 1 or sections.keys()[0] != '1':
for section_name, section_content in sections.iteritems():
self.add_source_for_key("%s:%s" % (key, section_name), section_content)
else:
self.add_source_for_key(key, sections['1'])
else:
self.add_source_for_key(key, sections)
self.artifact.output_data.append("%s:doc" % key, inspect.getdoc(item))
self.artifact.output_data.append("%s:comments" % key, inspect.getcomments(item))
else: # not a function or a method
try:
# If this can be JSON-serialized, leave it alone...
json.dumps(item)
self.add_source_for_key(key, item)
except TypeError:
# ... if it can't, convert it to a string to avoid problems.
self.add_source_for_key(key, str(item))
def highlight_html(self, source):
return highlight(source, self.LEXER, self.HTML_FORMATTER)
def highlight_latex(self, source):
return highlight(source, self.LEXER, self.LATEX_FORMATTER)
def add_source_for_key(self, key, source):
"""
Appends source code + syntax highlighted source code to persistent store.
"""
self.artifact.output_data.append("%s:value" % key, source)
if not (type(source) == str or type(source) == unicode):
source = inspect.getsource(source)
self.artifact.output_data.append("%s:source" % key, source)
self.artifact.output_data.append("%s:html-source" % key, self.highlight_html(source))
self.artifact.output_data.append("%s:latex-source" % key, self.highlight_latex(source))
def process_members(self, package_name, mod):
"""
Process all members of the package or module passed.
"""
name = mod.__name__
for k, m in inspect.getmembers(mod):
self.log.debug("in %s processing element %s" % (mod.__name__, k))
if not inspect.isclass(m) and hasattr(m, '__module__') and m.__module__ and m.__module__.startswith(package_name):
key = "%s.%s" % (m.__module__, k)
self.fetch_item_content(key, m)
elif inspect.isclass(m) and m.__module__.startswith(package_name):
key = "%s.%s" % (mod.__name__, k)
try:
item_content = inspect.getsource(m)
self.artifact.output_data.append("%s:doc" % key, inspect.getdoc(m))
self.artifact.output_data.append("%s:comments" % key, inspect.getcomments(m))
self.add_source_for_key(key, item_content)
except IOError:
self.log.debug("can't get source for %s" % key)
self.add_source_for_key(key, "")
try:
for ck, cm in inspect.getmembers(m):
key = "%s.%s.%s" % (name, k, ck)
self.fetch_item_content(key, cm)
except AttributeError:
pass
else:
key = "%s.%s" % (name, k)
self.fetch_item_content(key, m)
def process_module(self, package_name, name):
try:
self.log.debug("Trying to import %s" % name)
__import__(name)
mod = sys.modules[name]
self.log.debug("Success importing %s" % name)
try:
module_source = inspect.getsource(mod)
json.dumps(module_source)
self.add_source_for_key(name, inspect.getsource(mod))
except (UnicodeDecodeError, IOError, TypeError):
self.log.debug("Unable to load module source for %s" % name)
self.process_members(package_name, mod)
except (ImportError, TypeError) as e:
self.log.debug(e)
def process(self):
"""
input_text should be a list of installed python libraries to document.
"""
package_names = self.artifact.input_data.as_text().split()
packages = [__import__(package_name) for package_name in package_names]
for package in packages:
self.log.debug("processing package %s" % package)
package_name = package.__name__
prefix = package.__name__ + "."
self.process_members(package_name, package)
if hasattr(package, '__path__'):
for module_loader, name, ispkg in pkgutil.walk_packages(package.__path__, prefix=prefix):
self.log.debug("in package %s processing module %s" % (package_name, name))
if not name.endswith("__main__"):
self.process_module(package_name, name)
else:
self.process_module(package.__name__, package.__name__)
self.artifact.output_data.save()
try:
import markdown
except ImportError, error:
print "You don't appear to have the markdown module installed: %s" % error
sys.exit(1)
return markdown.markdown(content)
def load_code(content):
"Convert text via idiopidae to insert code"
try:
from idiopidae.runtime import Composer
except ImportError, error:
print "You don't appear to have the idiopidae module installed: %s" % error
sys.exit(1)
composer = Composer()
temporary_file = open("build/code.txt", "w")
temporary_file.write(content)
temporary_file.close()
output = composer.process(temporary_file.name)
os.unlink(temporary_file.name)
return output
def generate_html(content):
"Create an html file from the current content buffer"
try:
from BeautifulSoup import BeautifulSoup
except ImportError, error:
print "You don't appear to have BeautifulSoup installed: %s" % error
sys.exit(1)
from idiopidae.runtime import Composer
from pygments import highlight
from pygments.formatters.latex import LatexFormatter
from pygments.lexers.agile import PythonLexer
import dexy
import idiopidae.parser
import inspect
import json
import pkgutil
import sys
lexer = PythonLexer()
formatter = LatexFormatter()
composer = Composer()
def fetch_item_content(cm):
is_method = inspect.ismethod(cm)
is_function = inspect.isfunction(cm)
if is_method or is_function:
try:
source = inspect.getsource(cm)
except IOError:
source = ""
builder = idiopidae.parser.parse('Document', source + "\n\0")
sections = {}
for i, s in enumerate(builder.sections):
lines = builder.statements[i]['lines']
sections[s] = composer.format(lines, lexer, formatter)
