def format_unencoded(self, tokensource, outfile):
# This line will force the CustomLexer above to produce a stream of tokens,
# which will also store styles in the EmptyStyle class
tokenlist = list(tokensource)
# This extra style class is created to trigger the __new__ method in StyleMeta, to initialize the styles
# that have been set in the CustomLexer
@add_metaclass(StyleMeta)
class ExtraStyle(EmptyStyle):
pass
outfile.write("{\n")
# Make sure that the settings for writing custom style commands (see next block) has the right settings.
# The command prefix is `PYGdefault` when no style has been set using `\usemintedstyle`.
cp = self.commandprefix
cp = (cp + "default") if cp == "PYG" else cp
self.options["commandprefix"] = cp
self.options["style"] = ExtraStyle
# The style commands in LaTeX are written directly to the block of code, instead of to the pygstyle file
# because the style can be different for every block of code (yay for ESV).
outfile.write("\\makeatletter\n")
for name, definition in LatexFormatter(**self.options).cmd2def.items():
outfile.write(r'\expandafter\def\csname %s@tok@%s\endcsname{%s}' % (cp, name, definition))
outfile.write("\n")
outfile.write("\\makeatother\n")
# After writing the style commands, just print out the Verbatim environment as usual
LatexFormatter.format_unencoded(self, tokenlist, outfile)
outfile.write("}\n")
def __init__(self, **options):
LatexFormatter.__init__(self, **options)
self.escapeinside = options.get('escapeinside', '')
if len(self.escapeinside) == 2:
self.left = self.escapeinside[0]
self.right = self.escapeinside[1]
else:
self.escapeinside = ''
def __init__(self, **options):
LatexFormatter.__init__(self, **options)
self.escapeinside = options.get('escapeinside', '')
if len(self.escapeinside) == 2:
self.left = self.escapeinside[0]
self.right = self.escapeinside[1]
else:
self.escapeinside = ''
def process(self):
assert self.output_data.state == 'ready'
if self.setting('pygments'):
self.lexer = get_lexer_for_filename(self.input_data.storage.data_file())
self.html_formatter = HtmlFormatter(lineanchors=self.output_data.web_safe_document_key())
self.latex_formatter = LatexFormatter()
if self.input_data.ext in ('.xml', '.txt'):
parser = etree.XMLParser()
elif self.input_data.ext == '.html':
parser = etree.HTMLParser()
else:
raise Exception("Unsupported extension %s" % self.input_data.ext)
tree = etree.parse(self.input_data.storage.data_file(), parser)
for element in tree.iter("*"):
element_keys = []
for attribute_name in self.setting('unique-attributes'):
if element.attrib.has_key(attribute_name):
element_keys.append(element.attrib[attribute_name])
for attribute_name in self.setting('qualified-attributes'):
if element.attrib.has_key(attribute_name):
element_keys.append(element.attrib[attribute_name])
element_keys.append("%s:%s" % (element.tag, element.attrib[attribute_name]))
for element_key in element_keys:
self.append_element_attributes_with_key(element, element_key)
self.output_data.save()
def print_style_defs(style):
"""
Print \def's for a style
"""
latex = LatexFormatter(style=style).get_style_defs()
# Fix dollar sign and single quotation mark.
# https://tex.stackexchange.com/a/255728
# https://tex.stackexchange.com/a/238177
replacements = [
(r'\def\PYZdl{\char`\$}',
r'\def\PYZdl{\char36}'),
(r'\def\PYZsq{\char`\'}',
r'\def\PYZsq{\textquotesingle}'),
]
for wrong, right in replacements:
latex = latex.replace(wrong, right)
print(latex)
class PythonTestFilter(Filter):
"""
Runs the tests in the specified module(s) (which must be installed on the
system) and returns a key-value store with test results, source code and
html and latex highlighted source code.
"""
ALIASES = ['pytest']
INPUT_EXTENSIONS = [".txt"]
OUTPUT_EXTENSIONS = ['.json']
LEXER = PythonLexer()
LATEX_FORMATTER = LatexFormatter()
HTML_FORMATTER = HtmlFormatter(lineanchors="pytest")
# TODO some way to ensure tests logs get written elsewhere, like to the artifact output, they are going to main log for now - very confusing
def process(self):
self.artifact.setup_kv_storage()
loader = nose.loader.TestLoader()
for module_name in self.artifact.input_text().split():
self.log.debug("Starting to process module '%s'" % module_name)
tests = loader.loadTestsFromName(module_name)
self.log.debug("Loaded tests.")
for test in tests:
self.log.debug("Running test suite %s" % test)
test_passed = nose.core.run(suite=test, argv=['nosetests'])
self.log.debug("Passed: %s" % test_passed)
for x in dir(test.context):
xx = test.context.__dict__[x]
if inspect.ismethod(xx) or inspect.isfunction(xx):
test_context_name = test.context.__name__
qualified_test_name = "%s.%s" % (test_context_name, xx.__name__)
source = inspect.getsource(xx.__code__)
html_source = highlight(source, self.LEXER, self.HTML_FORMATTER)
latex_source = highlight(source, self.LEXER, self.LATEX_FORMATTER)
if test_passed:
html_result = """ <div class="test-passed"> %s PASSED </div> """ % qualified_test_name
else:
html_result = """ <div class="test-failed"> %s FAILED </div> """ % qualified_test_name
self.artifact.output_data.append("%s:source" % qualified_test_name, source)
self.artifact.output_data.append("%s:html-source" % qualified_test_name, html_source)
self.artifact.output_data.append("%s:latex-source" % qualified_test_name, latex_source)
self.artifact.output_data.append("%s:test-passed" % qualified_test_name, test_passed)
self.artifact.output_data.append("%s:html-result" % qualified_test_name, html_result)
self.artifact.output_data.append("%s:html-source+result" % qualified_test_name, "%s\n%s" % (html_source, html_result))
self.artifact._storage.save()
def __init__(self, query, highlightFormat: str = "html"):
'''
construct me for the given query and highlightFormat
Args:
query(Query): the query to do the syntax highlighting for
highlightFormat(str): the highlight format to be used
'''
self.query = query
self.highlightFormat = highlightFormat
self.lexer = get_lexer_by_name(self.query.lang)
if self.highlightFormat == "html":
self.formatter = HtmlFormatter()
elif self.highlightFormat == "latex":
self.formatter = LatexFormatter()
def highlight_code(self):
html_snippets = []
if self.matched_line_number():
snippet_cluster_lns = self.compute_lines_to_highlight(self.adjacent_line_numbers())
snippets = []
for snippet_cluster_ln in snippet_cluster_lns:
snippet = []
for n in snippet_cluster_ln:
snippet.append(self.file_content_lines[n])
start_line = min(snippet_cluster_ln)
highlight_lines = map(lambda x: x - start_line + 1, self.matching_line_numbers)
snippets.append(("\n".join(snippet), start_line, highlight_lines))
html_snippets = [highlight(snippet[0], JavaLexer(), LatexFormatter(linenos=True, linenostart=snippet[1])) for snippet in snippets]
self.code_snippets = [ GitSearchItemSnippet( self.hl_snippet( snippet[0], snippet[1]), snippet[1]) for snippet in snippets]
if not html_snippets:
html_snippets.append(highlight(self.file_content, JavaLexer(), HtmlFormatter(linenos=True, anchorlinenos=True)))
self.code_snippets.append( GitSearchItemSnippet( self.hl_snippet( self.file_content, 0), 0) )
return "".join(html_snippets)
class PythonDocumentationFilter(Filter):
ALIASES = ["pydoc"]
INPUT_EXTENSIONS = [".txt"]
OUTPUT_EXTENSIONS = ['.json']
COMPOSER = Composer()
OUTPUT_DATA_TYPE = 'keyvalue'
LEXER = PythonLexer()
LATEX_FORMATTER = LatexFormatter()
HTML_FORMATTER = HtmlFormatter(lineanchors="pydoc")
def fetch_item_content(self, key, item):
is_method = inspect.ismethod(item)
is_function = inspect.isfunction(item)
if is_method or is_function:
# Get source code
try:
source = inspect.getsource(item)
except IOError as e:
source = ""
# Process any idiopidae tags
builder = idiopidae.parser.parse('Document', source + "\n\0")
sections = {}
for i, s in enumerate(builder.sections):
lines = builder.statements[i]['lines']
sections[s] = "\n".join(l[1] for l in builder.statements[i]['lines'])
if isinstance(sections, dict):
if len(sections.keys()) > 1 or sections.keys()[0] != '1':
for section_name, section_content in sections.iteritems():
self.add_source_for_key("%s:%s" % (key, section_name), section_content)
else:
self.add_source_for_key(key, sections['1'])
else:
self.add_source_for_key(key, sections)
self.artifact.output_data.append("%s:doc" % key, inspect.getdoc(item))
self.artifact.output_data.append("%s:comments" % key, inspect.getcomments(item))
else: # not a function or a method
try:
# If this can be JSON-serialized, leave it alone...
json.dumps(item)
self.add_source_for_key(key, item)
except TypeError:
# ... if it can't, convert it to a string to avoid problems.
self.add_source_for_key(key, str(item))
def highlight_html(self, source):
return highlight(source, self.LEXER, self.HTML_FORMATTER)
def highlight_latex(self, source):
return highlight(source, self.LEXER, self.LATEX_FORMATTER)
def add_source_for_key(self, key, source):
"""
Appends source code + syntax highlighted source code to persistent store.
"""
self.artifact.output_data.append("%s:value" % key, source)
if not (type(source) == str or type(source) == unicode):
source = inspect.getsource(source)
self.artifact.output_data.append("%s:source" % key, source)
self.artifact.output_data.append("%s:html-source" % key, self.highlight_html(source))
self.artifact.output_data.append("%s:latex-source" % key, self.highlight_latex(source))
def process_members(self, package_name, mod):
"""
Process all members of the package or module passed.
"""
name = mod.__name__
for k, m in inspect.getmembers(mod):
self.log.debug("in %s processing element %s" % (mod.__name__, k))
if not inspect.isclass(m) and hasattr(m, '__module__') and m.__module__ and m.__module__.startswith(package_name):
key = "%s.%s" % (m.__module__, k)
self.fetch_item_content(key, m)
elif inspect.isclass(m) and m.__module__.startswith(package_name):
key = "%s.%s" % (mod.__name__, k)
try:
item_content = inspect.getsource(m)
self.artifact.output_data.append("%s:doc" % key, inspect.getdoc(m))
self.artifact.output_data.append("%s:comments" % key, inspect.getcomments(m))
self.add_source_for_key(key, item_content)
except IOError:
self.log.debug("can't get source for %s" % key)
self.add_source_for_key(key, "")
try:
for ck, cm in inspect.getmembers(m):
key = "%s.%s.%s" % (name, k, ck)
self.fetch_item_content(key, cm)
except AttributeError:
pass
else:
key = "%s.%s" % (name, k)
self.fetch_item_content(key, m)
def process_module(self, package_name, name):
try:
self.log.debug("Trying to import %s" % name)
__import__(name)
mod = sys.modules[name]
self.log.debug("Success importing %s" % name)
try:
module_source = inspect.getsource(mod)
json.dumps(module_source)
self.add_source_for_key(name, inspect.getsource(mod))
except (UnicodeDecodeError, IOError, TypeError):
self.log.debug("Unable to load module source for %s" % name)
self.process_members(package_name, mod)
except (ImportError, TypeError) as e:
self.log.debug(e)
def process(self):
"""
input_text should be a list of installed python libraries to document.
"""
package_names = self.artifact.input_data.as_text().split()
packages = [__import__(package_name) for package_name in package_names]
for package in packages:
self.log.debug("processing package %s" % package)
package_name = package.__name__
prefix = package.__name__ + "."
self.process_members(package_name, package)
if hasattr(package, '__path__'):
for module_loader, name, ispkg in pkgutil.walk_packages(package.__path__, prefix=prefix):
self.log.debug("in package %s processing module %s" % (package_name, name))
if not name.endswith("__main__"):
self.process_module(package_name, name)
else:
self.process_module(package.__name__, package.__name__)
self.artifact.output_data.save()
from idiopidae.runtime import Composer
from pygments import highlight
from pygments.formatters.latex import LatexFormatter
from pygments.lexers.agile import PythonLexer
import dexy
import idiopidae.parser
import inspect
import json
import pkgutil
import sys
lexer = PythonLexer()
formatter = LatexFormatter()
composer = Composer()
def fetch_item_content(cm):
is_method = inspect.ismethod(cm)
is_function = inspect.isfunction(cm)
if is_method or is_function:
try:
source = inspect.getsource(cm)
except IOError:
source = ""
builder = idiopidae.parser.parse('Document', source + "\n\0")
sections = {}
for i, s in enumerate(builder.sections):
lines = builder.statements[i]['lines']
sections[s] = composer.format(lines, lexer, formatter)
def highlight_code(self, search_count, rank, targetpath):
html_snippets = []
matched_line_numbers = self.matched_line_number()
if matched_line_numbers:
snippet_cluster_lns = self.compute_lines_to_highlight(
self.adjacent_line_numbers())
snippets = []
##################### Data provision ######################
pure_snippets_for_data_requirement = [] ##
#snippet_cluster_lns ##lines_to_be_highlighted
#self.file_content_lines ##entire code
for snippet_cluster_ln in snippet_cluster_lns:
snippet = []
for n in snippet_cluster_ln:
snippet.append(self.file_content_lines[n])
pure_snippets_for_data_requirement.append(
self.file_content_lines[n])
start_line = min(snippet_cluster_ln)
# end_line = max(snippet_cluster_ln)
highlight_lines = map(lambda x: x - start_line + 1,
self.matching_line_numbers)
snippets.append(
("\n".join(snippet), start_line, highlight_lines))
# #Data provision_ Defect4J (모든) 각 쿼리별 랭킹순으로 하이라이트 라인번호 + 전체코드 파일
final_str = "\n".join(self.file_content_lines)
original_project_list = [
'knutwalker_google-closure-compiler',
'google_closure-compiler', 'weitzj_closure-compiler',
'jfree_jfreechart-fse', 'jfree_jfreechart',
'apache_commons-math', 'apache_commons-lang',
'mockito_mockito', 'bryceguo_mockito'
]
if not self.file_path.split(
'/'
)[6] in original_project_list: #Check the duplicate projects.
purepath = targetpath[:-4]
if not os.path.exists(purepath):
os.makedirs(purepath)
testcode_path = purepath + '_result_testcode'
if not os.path.exists(testcode_path):
os.makedirs(testcode_path)
if self.file_path.split(
'/')[-1] == 'Test.java': # 결과 파일이 Test.java 인 것들 걸러내기
pass
else:
final_path = purepath + "/" + str(rank) + '_' + str(
"||".join(self.file_path.split('/')[6:]))
write_file(final_path, str(final_str))
write_file(final_path + "_", str(snippet_cluster_lns))
print "*****************************", final_path, "is Done.."
# 여기서 test code 또한 찾아서 셋트로 돌려줘보자.
# >> 각각 결과코드에 해당하는 프로젝트 경로를 새로 다 뒤져서 현재 결과파일 앞뒤로 test 키워드 들어있는 파일들을 찾아본다.
# >> 테스트 파일 찾았으면 복붙
result_file_name = self.file_path.split('/')[-1]
result_pure_file_name = (
(self.file_path.split('/')[-1]).split('.')
)[0] #pure name of the java file (e.g., ABC.java -> ABC)
stopwords = ['A', 'a', 'test']
javafiles = java_files_from_dir('/'.join(
self.file_path.split('/')[:7]))
for javafile in javafiles:
if result_pure_file_name in stopwords:
continue
name_of_javafile = javafile.split('/')[-1].split(
'.')[0]
if 'test' in name_of_javafile and result_pure_file_name in name_of_javafile: #해당 결과파일의 이름과 'test' 라는 키워드가 들어가있는 파일이면 (e.g., xxxtest.java or testxxx.java)
content = read_file(javafile)
testcode_path = testcode_path + '/' + str(
rank) + '_' + str('||'.join(
javafile.split('/')[6:]))
write_file(testcode_path, content)
write_file('/Users/Falcon/Desktop/count.txt',
testcode_path)
write_file('/Users/Falcon/Desktop/count.txt',
javafile)
write_file('/Users/Falcon/Desktop/count.txt',
'**********************')
html_snippets = [
highlight(snippet[0], JavaLexer(),
LatexFormatter(linenos=True, linenostart=snippet[1]))
for snippet in snippets
]
self.code_snippets = [
GitSearchItemSnippet(self.hl_snippet(snippet[0], snippet[1]),
snippet[1]) for snippet in snippets
]
if not html_snippets:
html_snippets.append(
highlight(self.file_content, JavaLexer(),
HtmlFormatter(linenos=True, anchorlinenos=True)))
self.code_snippets.append(
GitSearchItemSnippet(self.hl_snippet(self.file_content, 0), 0))
return "".join(html_snippets)
class TalonLexer(RegexLexer):
name = "Talon Keyword Lexer"
aliases = ['talon']
tokens = {
'root': [
(r'\bphrase\b', Name.Function),
(r'\d+', Number),
(regexFromWordList(talon.f_keys), Number),
(regexFromWordList(talon.arrows + talon.modifiers), Name),
(regexFromWordList(talon.alpha_alt + talon.simple_keys + talon.alternate_keys), Keyword),
(regexFromWordList(talon.symbols), Operator),
(r'\s+', Whitespace),
(r'[\'`",\.\:\+\-\=\*\\/]', Operator),
(r'[a-zA-Z]+', Generic),
],
}
if __name__ == "__main__":
print("started")
lexer = DomsagxoLexer()
print("created lexer")
code = """ŝambaluli en la nokton signifas dudek horoj estas egala al la kato en domsaĝo finu"""
formatter = LatexFormatter(style=DomsagxoStyle)
formatter.full = True
with open('domsa.tex', 'w', encoding='utf8') as outFile:
highlight(code, lexer, formatter,
outFile)
# proceed arguments
if (not len(sys.argv) >= 2):
sys.exit("1 or 2 arguments are required")
aFile = sys.argv[1]
# by default, the formatter is HTML
formatter = HtmlFormatter(linenos=False, cssclass="code", style="trac")
aFormat = 'html'
if (len(sys.argv) == 3):
aFormat = sys.argv[2]
if (aFormat == 'tex'):
print "Latex!"
formatter = LatexFormatter(linenos=True,
cssclass="highlight",
style="borland",
full=True)
if (aFormat == 'png'):
formatter = ImageFormatter(line_numbers=True,
cssclass="highlight",
style="borland")
if (aFormat == 'svg'):
formatter = SvgFormatter(line_numbers=False,
cssclass="highlight",
style="borland")
else:
print "No format specified. By default: " + aFormat
filename = PATH_FOLDER + aFile + ".fml" #".fmm"
outputf = open(PATH_OUPUT + os.path.basename(aFile) + "." + aFormat, "w")
filin = open(filename, 'r')
