class PythonCorpus(GenericCorpus):
"""
The default UnnaturalCode Python corpus.
"""
name = 'Python corpus_name [MITLM]'
description = __doc__
language = 'Python'
# Get the singleton instance of the underlying Python language (source)
# model.
# [sigh]... this API.
_user = ucUser.pyUser(ngram_order=GOOD_ENOUGH_NGRAM_ORDER)
_sourceModel = _user.sm
_lang = _sourceModel.lang()
_mitlm = _sourceModel.cm
order = _mitlm.order
# Hard-coded because "it's the best! the best a language model can get!"
smoothing = 'ModKN'
def tokenize(self, string, mid_line=True):
"""
Tokenizes the given string in the manner appropriate for this
corpus's language model.
"""
return self._lang.lex(string, mid_line)
def main():
ucpy = pyUser()
parser = argparse.ArgumentParser(description='Add known-good Python files to UnnaturalCode.')
parser.add_argument('files', metavar='file', type=str, nargs='+',
help='A file to be added.')
args = parser.parse_args()
ucpy.sm.trainFile(args.files)
ucpy.release()
def main():
ucpy = pyUser()
parser = argparse.ArgumentParser(
description='Add known-good Python files to UnnaturalCode.')
parser.add_argument('files',
metavar='file',
type=str,
nargs='+',
help='A file to be added.')
args = parser.parse_args()
ucpy.sm.trainFile(args.files)
ucpy.release()
def main(mode="wrap"):
import re
import runpy
import sys
import traceback
from copy import deepcopy
import logging
import os
import imputil
from logging import debug, info, warning, error
#logging.getLogger().setLevel(logging.DEBUG)
savedSysPath = deepcopy(sys.path)
program = sys.argv[1]
del sys.argv[1]
# TODO: run this fn in a seperate proc using os.fork
def runit():
sys.path.insert(0, os.getcwd())
if not mode == "check":
sys.path.insert(0, os.path.dirname(program))
virtualEnvActivate = os.getenv("VIRTUALENV_ACTIVATE", None)
if not virtualEnvActivate is None:
execfile(virtualEnvActivate, dict(__file__=virtualEnvActivate))
try:
if mode == "check":
r = runpy.run_module(program)
else:
r = runpy.run_path(program, run_name="__main__")
except SyntaxError as se:
ei = sys.exc_info();
traceback.print_exc();
eip = (ei[0], str(ei[1]), traceback.extract_tb(ei[2]))
try:
eip[2].append(ei[1][1])
except IndexError:
eip[2].append((se.filename, se.lineno, None, None))
return (eip)
except Exception as e:
ei = sys.exc_info();
traceback.print_exc();
eip = (ei[0], str(ei[1]), traceback.extract_tb(ei[2]))
return (eip)
return ((None, "None", [(program, None, None, None)]))
e = runit()
if e[0] == None:
return
source = open(e[2][-1][0]).read()
sys.path = savedSysPath;
from unnaturalcode.ucUser import pyUser
ucpy = pyUser()
worst = ucpy.sm.worstWindows(ucpy.lm(source))
print("Suggest checking around %s:%d:%d" % (program, worst[0][0][10][2][0], worst[0][0][10][2][1]), file=sys.stderr)
print("Near:\n" + ucpy.lm(worst[0][0]).settle().deLex())
ucpy.release()
def main(mode="wrap"):
import re
import runpy
import sys
import traceback
from copy import deepcopy
import logging
import os
import imputil
from logging import debug, info, warning, error
# logging.getLogger().setLevel(logging.DEBUG)
savedSysPath = deepcopy(sys.path)
program = sys.argv[1]
del sys.argv[1]
# TODO: run this fn in a seperate proc using os.fork
def runit():
sys.path.insert(0, os.getcwd())
if not mode == "check":
sys.path.insert(0, os.path.dirname(program))
virtualEnvActivate = os.getenv("VIRTUALENV_ACTIVATE", None)
if not virtualEnvActivate is None:
execfile(virtualEnvActivate, dict(__file__=virtualEnvActivate))
try:
if mode == "check":
r = runpy.run_module(program)
else:
r = runpy.run_path(program, run_name="__main__")
except SyntaxError as se:
ei = sys.exc_info()
traceback.print_exc()
eip = (ei[0], str(ei[1]), traceback.extract_tb(ei[2]))
try:
eip[2].append(ei[1][1])
except IndexError:
eip[2].append((se.filename, se.lineno, None, None))
return eip
except Exception as e:
ei = sys.exc_info()
traceback.print_exc()
eip = (ei[0], str(ei[1]), traceback.extract_tb(ei[2]))
return eip
return (None, "None", [(program, None, None, None)])
e = runit()
if e[0] == None:
return
source = open(e[2][-1][0]).read()
sys.path = savedSysPath
from unnaturalcode.ucUser import pyUser
ucpy = pyUser()
worst = ucpy.sm.worstWindows(ucpy.lm(source))
print("Suggest checking around %s:%d:%d" % (program, worst[0][0][10][2][0], worst[0][0][10][2][1]), file=sys.stderr)
print("Near:\n" + ucpy.lm(worst[0][0]).settle().deLex())
ucpy.release()