def testProblem(userId: str,
sourcePath: str,
cfTestsIds: List[int] = None,
uTestsIds: List[int] = None,
validatorPath: str = None) -> None:
prob = dbMan.Problem.getByUserId(userId)
exe = compiler.compile(abspath(sourcePath))
if validatorPath != None:
val = compiler.compile(abspath(validatorPath))
val += ['--seperator', _sep]
else:
val = None
ts = dbMan.TestSet(prob.id, cfTestsIds, uTestsIds)
if len(ts.tests) == 0:
print(f'{Fore.RED}No tests where found{Fore.RESET}')
return
failedTests = []
maxElapsed: TestResult = None
for t in ts.tests:
tr = TestResult.runTest(t.input, t.answer, exe, t.id, val, _sep, '')
if tr.passed == False:
failedTests.append(tr)
if maxElapsed == None or tr.elapsed > maxElapsed.elapsed:
maxElapsed = tr
print(
f'Ran {len(ts.tests)} tests, {len(ts.tests) - len(failedTests)} {Fore.GREEN}passed{Fore.RESET} and {len(failedTests)} {Fore.RED}failed{Fore.RESET}'
)
print(
f'Max elapsed test is {maxElapsed.testId} and it took {maxElapsed.elapsed}ms'
)
if len(failedTests) > 0:
print(f'{Fore.RED}Failed{Fore.RESET} tests are:')
for t in failedTests:
print(t)
def stressTest(sourcePath: str,
n: int,
generatorPath: str,
outputPath: str,
validatorPath: str = None,
solverPath: str = None):
"""generator must first print the test case, answer(can be empty) and additional info(can be empty) for the validator all seperated by argument --seperator
if a solver is provided then generator answer will be ignored and instead solver will be called and its answer will be provided to the validator"""
exe, gen = compiler.compile(abspath(sourcePath)), compiler.compile(
abspath(generatorPath))
gen.append('--seperator')
gen.append(_sep)
if validatorPath != None:
val = compiler.compile(abspath(validatorPath))
val.append('--seperator')
val.append(_sep)
else:
val = None
sol = None if solverPath == None else compiler.compile(abspath(solverPath))
allGood = True
maxElapsed: TestResult = None
for i in range(1, n + 1):
print(f'Test case #{i}: ', end='')
genProc = subprocess.run(gen,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
check=True)
gpo = compiler.splitOnLine(_sep, genProc.stdout, 3)
sln = gpo[1] if sol == None else subprocess.run(
sol,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
check=True,
input=gpo[0]).stdout.strip()
t = TestResult.runTest(gpo[0],
sln,
exe,
validator=val,
validatorSep=_sep,
validatorAdditionalData=gpo[2])
if t.passed == False:
with open(outputPath, 'a+') as opt:
opt.write(('#' * 150) + '\n')
opt.write(str(t))
opt.write('\n' + ('#' * 150))
print(
f'{Fore.RED}Failed{Fore.RESET}, see file {outputPath} for additional info'
)
allGood = False
break
if maxElapsed == None or t.elapsed > maxElapsed.elapsed: maxElapsed = t
print(f'{Fore.GREEN}Passed{Fore.RESET}')
if allGood == False: return
print(f'Ran {n} tests {Fore.GREEN}successfully{Fore.RESET}')
print(f'Max elapsed test took {maxElapsed.elapsed}ms')
def testDocstrings(self):
c = compiler.compile('"doc"', '<string>', 'exec')
self.assert_('__doc__' in c.co_names)
c = compiler.compile('def f():\n "doc"', '<string>', 'exec')
g = {}
exec c in g
self.assertEquals(g['f'].__doc__, "doc")
def testDocstrings(self):
c = compiler.compile('"doc"', '<string>', 'exec')
self.assert_('__doc__' in c.co_names)
c = compiler.compile('def f():\n "doc"', '<string>', 'exec')
g = {}
exec c in g
self.assertEquals(g['f'].__doc__, "doc")
def main():
tmp_path = ''
args = read_cli_args()
cli.verbose(args.verbose)
config_setup()
cli.printv("Preprocessing input files.")
try:
tmp_path = preprocessor.multifile_process(args.input)
except (IOError, OSError) as e:
sys.exit(e.errno)
cli.printv("Compiling tmp file: " + tmp_path)
try:
compiler.compile(tmp_path, args.output)
except (IOError, OSError) as e:
if args.preserve_tmp == False:
preprocessor.remove_tmp_data(tmp_path)
if e.errno == errno.ENOENT:
cli.printe(str(e))
sys.exit(e.errno)
if args.preserve_tmp == False:
preprocessor.remove_tmp_data(tmp_path)
def run_benchmark(package_name, gen_schedule, gen_mem_instr,
is_training_algorithm):
Component.reset_ids()
dfg_name = f"{package_name}.json"
package_path = f"{DFG_ROOT}/{package_name}"
optimizations = {
'reorder_instr': True,
'unused_ni_opt': True,
'apply_reuse': True
}
file_path = f"{dfg_name}"
cfg_path = f'config.json'
if Path(f"{DFG_ROOT}/../../../compilation_output/{package_name}").exists():
shutil.rmtree(f"{DFG_ROOT}/../../../compilation_output/{package_name}")
compile(Path(f"{DFG_ROOT}/{dfg_name}").resolve(),
cfg_path,
f"{package_name}_input_data.txt",
f"{package_name}_input_weights.txt",
"meta.txt",
sort_alg="custom",
gen_sched_file=gen_schedule,
gen_mem_instr=gen_mem_instr,
save_data=True,
debug=False,
optimizations=optimizations,
show_ns_utilization=["NI", "NW", "ND"],
is_training_algorithm=is_training_algorithm) #
def testDocstrings(self):
c = compiler.compile('"doc"', "<string>", "exec")
self.assert_("__doc__" in c.co_names)
c = compiler.compile('def f():\n "doc"', "<string>", "exec")
g = {}
exec c in g
self.assertEquals(g["f"].__doc__, "doc")
def testCompileLibrary(self):
# A simple but large test. Compile all the code in the
# standard library and its test suite. This doesn't verify
# that any of the code is correct, merely the compiler is able
# to generate some kind of code for it.
libdir = os.path.dirname(unittest.__file__)
testdir = os.path.dirname(test.test_support.__file__)
for dir in [libdir, testdir]:
for basename in os.listdir(dir):
if not basename.endswith(".py"):
continue
if not TEST_ALL and random() < 0.98:
continue
path = os.path.join(dir, basename)
if test.test_support.verbose:
print "compiling", path
f = open(path, "U")
buf = f.read()
f.close()
if "badsyntax" in basename:
self.assertRaises(SyntaxError, compiler.compile, buf,
basename, "exec")
else:
compiler.compile(buf, basename, "exec")
def test_embedded_weights():
Component.reset_ids()
optimizations = {
'reorder_instr': False,
'unused_ni_opt': True,
'apply_reuse': True
}
package_name = "backprop_20_20_5"
# package_name = "reco_138_130_10"
dfg_name = f"{package_name}.json"
package_path = f"{Path(f'{__file__}').parent}/../{package_name}"
file_path = f"tests/dfg_json_files/{dfg_name}"
cfg_path = f'config.json'
if Path(package_path).exists():
shutil.rmtree(package_path)
compile(file_path,
cfg_path,
f"{package_name}_input_data.txt",
f"{package_name}_input_weights.txt",
"meta.txt",
sort_alg="custom",
gen_sched_file=False,
save_data=True,
gen_mem_instr=True,
debug=False,
show_ns_utilization=["ND", "NI", "NW"],
optimizations=optimizations)
def testCompileLibrary(self):
# A simple but large test. Compile all the code in the
# standard library and its test suite. This doesn't verify
# that any of the code is correct, merely the compiler is able
# to generate some kind of code for it.
next_time = time.time() + _PRINT_WORKING_MSG_INTERVAL
libdir = os.path.dirname(unittest.__file__)
testdir = os.path.dirname(test.test_support.__file__)
for dir in [libdir, testdir]:
for basename in os.listdir(dir):
# Print still working message since this test can be really slow
if next_time <= time.time():
next_time = time.time() + _PRINT_WORKING_MSG_INTERVAL
print >>sys.__stdout__, \
' testCompileLibrary still working, be patient...'
sys.__stdout__.flush()
if not basename.endswith(".py"):
continue
if not TEST_ALL and random() < 0.98:
continue
path = os.path.join(dir, basename)
if test.test_support.verbose:
print "compiling", path
f = open(path, "U")
buf = f.read()
f.close()
if "badsyntax" in basename:
self.assertRaises(SyntaxError, compiler.compile,
buf, basename, "exec")
else:
compiler.compile(buf, basename, "exec")
def testCompileLibrary(self):
# A simple but large test. Compile all the code in the
# standard library and its test suite. This doesn't verify
# that any of the code is correct, merely the compiler is able
# to generate some kind of code for it.
libdir = os.path.dirname(unittest.__file__)
testdir = os.path.dirname(test.test_support.__file__)
for dir in [libdir, testdir]:
for basename in os.listdir(dir):
if not basename.endswith(".py"):
continue
if not TEST_ALL and random() < 0.98:
continue
path = os.path.join(dir, basename)
if test.test_support.verbose:
print "compiling", path
f = open(path, "U")
buf = f.read()
f.close()
if "badsyntax" in basename:
self.assertRaises(SyntaxError, compiler.compile,
buf, basename, "exec")
else:
compiler.compile(buf, basename, "exec")
def testCompileLibrary(self):
# A simple but large test. Compile all the code in the
# standard library and its test suite. This doesn't verify
# that any of the code is correct, merely the compiler is able
# to generate some kind of code for it.
next_time = time.time() + _PRINT_WORKING_MSG_INTERVAL
libdir = os.path.dirname(unittest.__file__)
testdir = os.path.dirname(test.test_support.__file__)
for dir in [libdir, testdir]:
for basename in os.listdir(dir):
# Print still working message since this test can be really slow
if next_time <= time.time():
next_time = time.time() + _PRINT_WORKING_MSG_INTERVAL
print >>sys.__stdout__, \
' testCompileLibrary still working, be patient...'
sys.__stdout__.flush()
if not basename.endswith(".py"):
continue
if not TEST_ALL and random() < 0.98:
continue
path = os.path.join(dir, basename)
if test.test_support.verbose:
print "compiling", path
f = open(path, "U")
buf = f.read()
f.close()
if "badsyntax" in basename:
self.assertRaises(SyntaxError, compiler.compile,
buf, basename, "exec")
else:
compiler.compile(buf, basename, "exec")
def code2bc(code):
try:
bc = compiler.compile(code, "<file>", "eval")
except SyntaxError:
try:
bc = compiler.compile(code, "<file>", "exec")
except SyntaxError, e:
raise
def wrapper(*args, **kwargs):
query_string = f(*args, **kwargs)
if query_string == "False" or query_string == {"any": "False"}:
return None
if not isinstance(query_string, dict):
return compile(query_string, "<string>", "eval")
query_dict = query_string
for k, stmt in query_dict.iteritems():
query_dict[k] = compile(stmt, "<string>", "eval")
return query_dict
def pythonCodeContainsErrorOnCompile(self, code):
flag = False
ls = []
try:
compiler.compile(code,"file.py", mode = "exec")
except Exception as e:
flag = True
s = traceback.format_exc()
ls = s.split("\n")
return flag,ls
def wrapper(*args, **kwargs):
query_string = f(*args, **kwargs)
if query_string == "False" or query_string == {"any": "False"}:
return None
if not isinstance(query_string, dict):
return compile(query_string, "<string>", "eval")
query_dict = query_string
for k, stmt in query_dict.iteritems():
query_dict[k] = compile(stmt, "<string>", "eval")
return query_dict
def main(args):
if args["help"]:
print __doc__.rstrip()
return None
if not args["run"]:
if args["--verbose"]:
COMMAND_DEFAULTS["mvn"] = "mvn"
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger("quark")
log.propagate = False
hnd = ProgressHandler(verbose=args["--verbose"])
log.addHandler(hnd)
hnd.setFormatter(logging.Formatter("%(message)s"))
helpers.Code.identifier = "Quark %s run at %s" % (_metadata.__version__, datetime.datetime.now())
java = args["--java"]
ruby = args["--ruby"]
python = args["--python"]
javascript = args["--javascript"]
all = args["--all"] or not (java or python or javascript or ruby)
output = args["--output"]
try:
backends = []
if java or all:
check("mvn", ".")
backends.append(backend.Java)
if ruby or all:
backends.append(backend.Ruby)
if python or all:
check("python", ".")
backends.append(backend.Python)
if javascript or all:
check("npm", ".")
backends.append(backend.JavaScript)
filenames = args["<file>"]
for url in filenames:
if args["install"]:
compiler.install(url, *backends)
elif args["compile"]:
compiler.compile(url, output, *backends)
elif args["run"]:
compiler.run(url, args["<args>"], *backends)
else:
assert False
except compiler.QuarkError as err:
command_log.warn("")
return err
command_log.warn("Done")
def genewise(db, gt_filters, gt_req_filters, filter=None, columns=None, min_filters=None,
min_variants=1,
grouper="gene"):
assert os.path.exists(db)
orig_columns = [x.strip() for x in (columns or "").split(",")]
added_cols = add_cols(orig_columns, "||".join(gt_filters))
req_cols = add_cols(added_cols, "||".join(gt_req_filters))
added_cols = added_cols + req_cols
if grouper not in orig_columns:
added_cols.append(grouper)
columns = orig_columns + added_cols
assert not any(';' in c for c in columns)
# NOTE: we could make the WHERE part customizable.
query = "SELECT {columns} FROM variants WHERE (is_exonic = 1 AND impact_severity != 'LOW')"
if filter:
query += " AND " + filter
query += (" ORDER BY CHROM, %s" % grouper)
gq = GeminiQuery(db, include_gt_cols=True)
# use the GeminiQuery machinery to transform to something that's eval'able
# then compile once for speed.
cleaned_filters = []
for gt_filter in gt_filters:
assert gq._is_gt_filter_safe(gt_filter)
gt_filter = gq._correct_genotype_filter(gt_filter)
cleaned_filters.append(compile(gt_filter, gt_filter, 'eval'))
cleaned_reqs = []
for gt_filter in gt_req_filters:
assert gq._is_gt_filter_safe(gt_filter)
gt_filter = gq._correct_genotype_filter(gt_filter)
cleaned_reqs.append(compile(gt_filter, gt_filter, 'eval'))
gq.run(query.format(columns=", ".join(columns)))
if isinstance(grouper, basestring):
grouper = operator.itemgetter(grouper)
user_dict = dict(sample_info=gq.sample_info)
header_printed = False
for groupkey, grp in it.groupby(gq, grouper):
grp = list(grp)
for x in gen_results(list(grp), cleaned_filters, cleaned_reqs, min_filters or 0,
min_variants, columns, user_dict=user_dict):
for c in added_cols:
if c != grouper:
del x.print_fields[c]
if not header_printed:
print "\t".join(x.print_fields.keys())
header_printed = True
print x
def test_compile():
cleanup = True
Component.reset_ids()
package_name = "pm_linear55"
base_path = f"tests/test_dfgs"
dfg_name = f"{package_name}.json"
file_path = f"{base_path}/{dfg_name}"
cfg_path = f'tests/config.json'
compile(file_path, cfg_path, "../input_data.txt", "../input_weights.txt")
if cleanup:
shutil.rmtree(f"{base_path}/{package_name}")
def get_info(lstline):
if len(
lstline
) > 5: ##return what to reweight, reweight dictionary, reweight condition
return compiler.compile(lstline[2], '<string>',
'eval'), get_reweight(
lstline[0], lstline[3],
lstline[4]), compiler.compile(
lstline[5], '<string>', 'eval')
else: ##return what to reweight, reweight dictionary
return compiler.compile(lstline[2], '<string>',
'eval'), get_reweight(
lstline[0], lstline[3],
lstline[4]), "True"
def test_compile_optimized_docstrings(self):
"""
Ensure we strip docstrings with optimize=2, and retain them for
optimize=1
"""
src_code = "def f(): '''hi'''\n"
code = compile(src_code, "foo", "single", optimize=1)
consts = {k: v for k, v in zip(code.co_names, code.co_consts)}
self.assertIn("hi", consts["f"].co_consts)
code = compile(src_code, "foo", "single", optimize=2)
consts = {k: v for k, v in zip(code.co_names, code.co_consts)}
self.assertNotIn("hi", consts["f"].co_consts)
def pyCompile (text, encodedText, encoding, fileName): assert type(text) is unicode assert type(encodedText) is str assert encoding == None try: compiler.compile( pyNormalizeNewlines(encodedText), '', 'exec') except SyntaxError, e: msg = e.msg.decode(locale.getpreferredencoding()) pos = ( e.lineno - 1, e.offset - 1 ) link = (0, pos) return (msg, ( link, ), None)
def _compile(arg):
'''compile an expression into a code object ready for eval or exec'''
if not isinstance(arg,basestring):
return arg
import compiler
desc = 'eval(%r)' % arg
try:
# eval mode works for expressions, not assignments
mode = 'eval'
code = compiler.compile(arg,desc,mode)
except SyntaxError:
# exec mode necessary for assignments
mode = 'exec'
code = compiler.compile(arg,desc,mode)
return code
def _compile(arg):
'''compile an expression into a code object ready for eval or exec'''
if not isinstance(arg, basestring):
return arg
import compiler
desc = 'eval(%r)' % arg
try:
# eval mode works for expressions, not assignments
mode = 'eval'
code = compiler.compile(arg, desc, mode)
except SyntaxError:
# exec mode necessary for assignments
mode = 'exec'
code = compiler.compile(arg, desc, mode)
return code
def select_and_remove_individuals(self, prob, compiled_objects=True):
"""
Select several individuals from the population, remove them from the
island and returns them.
Instead of returning the object, we return a compiled version in order
to decrease memory usage.
:param prob: probability of selection of individuals for migration
:param compiled_objects: if True, instead of returning the list of trees,
return a compiled version of the list of trees.
Return the tree objects and the number of trees to move
"""
migration_size = max(2, math.ceil(self._popsize * prob))
db_list = selection.GetDBKeysAndFitness(
self._con,
self._tablename[-1])
#Select several uniq individuals
migration = selection.TournamentSelectDBSeveral(
int(migration_size),
10, #tournament size
0.8, #selection probability
db_list,
unique=True)
trees = []
#self._oid_to_replace = [] # Reset
for elem in migration:
o_id = elem[0]
#self._oid_to_repace.append(elem)
#Read from table
myresult, my_tree, \
my_tree_mapping, my_treedepth, \
my_evaluated, my_fitness = self._popwriter.get_individual(self._tablename[-1], o_id, True)
#Add in the returning set
trees.append( ( my_tree,
my_tree_mapping,
my_treedepth,
my_evaluated,
my_fitness))
#todo ask to popwriter to do that
self._con.execute("DELETE FROM %s WHERE o_id=%d;" % (self._tablename[-1], o_id))
self._con.commit()
if compiled_objects:
return compiler.compile(str(trees), '<string>', 'eval'), len(trees)
else:
return trees, len(trees)
def create_cobb_douglas(self, output, multiplier, exponents):
""" creates a Cobb-Douglas production function
A production function is a production process that produces the
given input goods according to the formula to the output
good.
Production_functions are than used as an argument in produce,
predict_vector_produce and predict_output_produce.
Args:
'output': Name of the output good
multiplier: Cobb-Douglas multiplier
{'input1': exponent1, 'input2': exponent2 ...}: dictionary
containing good names 'input' and corresponding exponents
Returns:
A production_function that can be used in produce etc.
Example:
self.plastic_production_function = self.create_cobb_douglas('plastic', {'oil' : 10, 'labor' : 1}, 0.000001)
self.produce(self.plastic_production_function, {'oil' : 20, 'labor' : 1})
"""
ordered_input = [input_good for input_good in exponents]
formula = output + '=' + str(multiplier) + '*' + '*'.join('(%s)**%f' % (input_good, exponent) for input_good, exponent in exponents.iteritems())
optimization = '*'.join(['(%s)**%f' % ('%s', exponents[good]) for good in ordered_input])
production_function = {}
production_function['type'] = 'cobb-douglas'
production_function['parameters'] = exponents
production_function['formula'] = formula
production_function['multiplier'] = multiplier
production_function['code'] = compiler.compile(formula, '<string>', 'exec')
production_function['output'] = [output]
production_function['input'] = ordered_input
production_function['optimization'] = optimization
return production_function
def evaluate(self, cog, globals, fname='cog generator'):
# figure out the right whitespace prefix for the output
prefOut = whitePrefix(self.markers)
intext = self.getCode()
if not intext:
return ''
# In Python 2.2, the last line has to end in a newline.
intext = "import cog\n" + intext + "\n"
if PRINT_COG:
print intext
code = compiler.compile(intext, filename=str(fname), mode='exec')
# Make sure the "cog" module has our state.
cog.cogmodule.msg = self.msg
cog.cogmodule.out = self.out
cog.cogmodule.outl = self.outl
cog.cogmodule.error = self.error
self.outstring = ''
eval(code, globals)
# We need to make sure that the last line in the output
# ends with a newline, or it will be joined to the
# end-output line, ruining cog's idempotency.
if self.outstring and self.outstring[-1] != '\n':
self.outstring += '\n'
return reindentBlock(self.outstring, prefOut)
def compile_interpret(self, ast, context=None):
if not context:
context = self.context
byte_code = compiler.compile(ast, context)
self.last_bc = ''
return self.interpret(byte_code)
def compile_files(dir):
print "Compiling",
line_len = 10
for file in os.listdir(dir):
base, ext = os.path.splitext(file)
if ext == '.py' and base[:4] == 'test':
source = os.path.join(dir, file)
line_len = line_len + len(file) + 1
if line_len > 75:
print "\n\t",
line_len = len(source) + 9
print file,
compile(source)
# make sure the .pyc file is not over-written
os.chmod(source + "c", 444)
print
def compile(widget, data=file):
cserial(None, 0, sctw)
page = getCurrentPage()
startSpinner()
obj = compiler.compile(page.get_data("view"),page.get_data("file"), id, tw, sb) #page.get_data("buffer")
stopSpinner()
return obj
def test9UserTriggers(self):
# Dependency Injection! (the nasty way):
triggers = \
'class Trigger(object):\n def evaluate():\n return False\nclass WordTrigger(Trigger):\n def __init__(self, word):\n self.word = word.lower()\n print "Injected!"\n def hash(self):\n raise NotImplementedError\nclass SubjectTrigger(Trigger):\n def __init__(self, word):\n self.word = word.lower()\n print "Injected Subject!"\n def hash(self):\n return "SUBJECT-" + self.word\nclass TitleTrigger(WordTrigger):\n def hash(self):\n return "TITLE-" + self.word\nclass SummaryTrigger(WordTrigger):\n def hash(self):\n return "SUMMARY-" + self.word\nclass NotTrigger(Trigger):\n def __init__(self, trigger):\n self.trigger = trigger\n print "Injected!"\n def hash(self):\n return "NOT-" + self.trigger.hash()\nclass PhraseTrigger(Trigger):\n def __init__(self, phrase):\n self.phrase = phrase\n print "Injected!"\n def hash(self):\n return "PHRASE-" + self.phrase\nclass CompositeTrigger(Trigger):\n def __init__(self, trigger1, trigger2):\n self.trigger1 = trigger1\n self.trigger2 = trigger2\nclass AndTrigger(CompositeTrigger):\n def hash(self):\n t1_hash = self.trigger1.hash()\n t2_hash = self.trigger2.hash()\n if t1_hash > t2_hash:\n return "AND-" + t1_hash + "-" + t2_hash\n else:\n return "AND-" + t2_hash + "-" + t1_hash\nclass OrTrigger(CompositeTrigger):\n def hash(self):\n t1_hash = self.trigger1.hash()\n t2_hash = self.trigger2.hash()\n if t1_hash > t2_hash:\n return "OR-" + t1_hash + "-" + t2_hash\n else:\n return "OR-" + t2_hash + "-" + t1_hash\n\n\n'
import compiler
f = open('Problem_Set5_Combined.py')
add_potential_points(5)
src = ""
for line in f:
if line == "if __name__ == '__main__':\n":
print "found line!"
break
src += line
src += triggers
print src
compiled = compiler.compile(src, 'compile-errors', 'exec')
exec compiled in locals(), globals()
t1 = SubjectTrigger('sports')
t2 = SummaryTrigger('Obama')
t3 = PhraseTrigger('Hillary Clinton')
t4 = OrTrigger(t2, t3)
trigger_map = {t1.hash(): False, t4.hash(): False}
print trigger_map
trigger_list = readTriggerConfig("triggers.txt")
for trigger in trigger_list:
hash = trigger.hash()
self.assertTrue(hash in trigger_map)
self.assertFalse(trigger_map[hash])
trigger_map[hash] = True
for found in trigger_map.values():
self.assertTrue(found, "Missing trigger")
add_points(5)
def migrateFile(
filePath, compiledPatches, compiledInfos,
hasPatchModule=False, options=None, encoding="UTF-8"):
logging.info(" - File: %s" % filePath)
# Read in original content
fileContent = filetool.read(filePath, encoding)
fileId = loader.extractFileContentId(fileContent);
# Apply patches
patchedContent = fileContent
if hasPatchModule and fileId is not None:
import patch
tree = treegenerator.createSyntaxTree(tokenizer.parseStream(fileContent))
# If there were any changes, compile the result
if patch.patch(fileId, tree):
options.prettyPrint = True # make sure it's set
patchedContent = compiler.compile(tree, options)
# apply RE patches
patchedContent = regtool(patchedContent, compiledPatches, True, filePath)
patchedContent = regtool(patchedContent, compiledInfos, False, filePath)
# Write file
if patchedContent != fileContent:
logging.info(" - %s has been modified. Storing modifications ..." % filePath)
filetool.save(filePath, patchedContent, encoding)
def compile_interpret(self, ast, context=None):
if not context:
context = self.context
byte_code = compiler.compile(ast, context)
self.last_bc = ''
return self.interpret(byte_code)
def run_package():
with ZipFile(PACKAGE_ZIP) as zipFile:
zipFile.extractall("store/package")
os.chdir("store/package")
for top, dirs, files in os.walk("."):
for file in files:
if file.endswith(".sh"):
path = os.path.join(top, file)
os.chmod(path, os.stat(path).st_mode | stat.S_IEXEC)
problem_node = ElementTree.parse("problem.xml")
for element in problem_node.iter():
source, binary = element.find("source"), element.find("binary")
if source is not None and binary is not None:
source_file = source.attrib["path"]
binary_file = binary.attrib["path"]
language = source.attrib["type"]
logger.info("compiling %s -> %s (%s)" %
(source_file, binary_file, language))
assert compile(language, source_file, binary_file)
with open("../logs/stdout.log", "a") as stdout, \
open("../logs/stderr.log", "a") as stderr:
subprocess.run(["./doall.sh"],
stdin=subprocess.DEVNULL,
stdout=stdout,
stderr=stderr,
timeout=time_limit,
check=True)
def inject(self, linker, sym):
# TODO: alloc pos will shift between inject()
# could use a separate segment for linker.
addrs = {}
with linker.binary.collect() as pt:
if len(self.syms) > 1:
pt.info('[LINK] %s (includes [%s])' %
(sym, ', '.join(self.syms.keys())))
else:
pt.info('[LINK] %s' % sym)
asm = compiler.compile(self.source, linker, syms=self.syms.keys())
table = '\n'.join(
[pt.arch.jmp('_' + sym) for sym in self.syms.keys()])
sep = 'PATCHKITJMPTABLE'
asm += ('\n.ascii "%s"\n__JMPTABLE__:\n' % sep) + table
addr = pt.binary.next_alloc('link')
raw = pt.asm(asm, addr=addr, att_syntax=True)
raw, jmps = raw.rsplit(sep, 1)
for sym, ins in zip(
self.syms.keys(),
pt.arch.dis(jmps, addr=addr + len(sep) + len(raw))):
addrs[sym] = ins.operands[0].imm
pt.inject(raw=raw, is_asm=True, target='link')
return addrs
def compile2(self, source):
#ast1 = cpy_parser.suite(source)
#ast2 = parser.suite(source)
#tup1 = cpy_parser.ast2tuple(ast1, 1)
#tup2 = parser.ast2tuple(ast2, 1)
#self.assertEquals(tup1, tup2)
#print "cpython compile", source
compiler.transformer.parser = cpy_parser
co1 = compiler.compile(source, '', 'exec')
#print "python compile", source
compiler.transformer.parser = parser
co2 = compiler.compile(source, '', 'exec')
return co1, co2
def compilePythonCode(self):
if(self.currentFile):
self.saveFile()
original_stdout = sys.stdout
try:
a = compiler.compile(self.currentText, self.currentFile,
mode = "exec")
# (http://stackoverflow.com/questions/4904079/
# execute-a-block-of-python-code-with-exec-capturing-all-its-output)
# captures stdout in temp stringIO buffer to get output
# and then reverts changes
buffer = StringIO()
sys.stdout = buffer
eval(a)
sys.stdout = original_stdout
val = buffer.getvalue()
rt = Tk()
outputText = ScrolledText(rt, width = 50)
outputText.insert(END, val)
outputText.pack()
rt.mainloop()
except:
print "Error!"
self.displayMessageBox("Error","There is an error in the code.")
sys.stdout = original_stdout
else:
self.saveAs()
def get_ezjail_module(args):
import ezjail
encoded_args = repr(args.encode('utf-8'))
encoded_lang = repr(ansible.constants.DEFAULT_MODULE_LANG)
encoded_complex = repr(jsonify({}))
fn = ezjail.__file__.replace('.pyc', '.py')
with open(fn) as f:
module_data = f.read()
module_data = module_data.replace(module_common.REPLACER,
module_common.MODULE_COMMON)
module_data = module_data.replace(module_common.REPLACER_ARGS,
encoded_args)
module_data = module_data.replace(module_common.REPLACER_LANG,
encoded_lang)
module_data = module_data.replace(module_common.REPLACER_COMPLEX,
encoded_complex)
ofn = fn.replace('/ezjail.py', '/tmp_ezjail.py')
with open(ofn, 'w') as f:
f.write(module_data)
code = compiler.compile(module_data, ofn, 'exec')
ezjail = imp.new_module('ezjail')
exec code in ezjail.__dict__
ezjail.__file__ = ofn
class AnsibleModule(ezjail.AnsibleModule):
def get_bin_path(self, arg, required=False, opt_dirs=[]):
return '/usr/bin/%s' % arg
ezjail.AnsibleModule = AnsibleModule
module = AnsibleModule(**ezjail.MODULE_SPECS)
return ezjail, module
def _compile(self, addr, **kwargs):
asm, jmp, sym, c, hex, raw = map(
kwargs.get, ('asm', 'jmp', 'sym', 'c', 'hex', 'raw'))
if sym is not None:
jmp = self.resolve(sym)
if jmp is not None:
asm = self.arch.jmp(jmp)
if asm is not None:
raw = self.asm(asm, addr=addr)
typ = 'asm'
elif c is not None:
#raise NotImplementedError
asm = compiler.compile(c, self.binary.linker)
raw = self.asm(asm, addr=addr, att_syntax=True)
self.debug('_compile -> %s' % binascii.hexlify(raw))
typ = 'c'
elif hex is not None:
raw = binascii.unhexlify(hex)
typ = 'raw'
elif raw is not None:
typ = 'raw'
else:
raise Exception(
'inject/patch parameter missing: need one of (asm, c, hex, raw)'
)
return raw, typ
def load_comments(self, pkgfile):
""" Open the package and load comments if any.
Return the loaded comments """
# Note: This has to be called with a Python
# source file (.py) only!
if not os.path.exists(pkgfile):
return ""
comment = ""
try:
of = open(pkgfile,'rb')
data = of.read()
if data:
# Create code object
try:
c = compiler.compile(data,pkgfile,'exec')
# Get the position of first line of code
if c:
lno = c.co_firstlineno
lnum = 0
# Read file till this line number
of.seek(0)
for line in of:
comment = "".join((comment, line))
lnum += 1
if lnum==lno or line=="\n": break
except SyntaxError, e:
pass
except Exception, e:
pass
def test_create_experiment_with_resources_spec(self, spawner_mock):
spec = compiler.compile(kind=kinds.EXPERIMENT, values=exec_experiment_resources_content)
mock_instance = spawner_mock.return_value
mock_instance.start_experiment.return_value = start_experiment_value
mock_instance.job_uuids = {'master': ['fa6203c189a855dd977019854a7ffcc3'],
'worker': ['3a9c9b0bd56b5e9fbdbd1a3d43d57960'],
'ps': ['59e3601232b85a3d8be2511f23a62945']}
mock_instance.spec = spec
with patch('scheduler.dockerizer_scheduler.create_build_job') as mock_start:
mock_start.return_value = BuildJobFactory(), True, True
experiment = ExperimentFactory(content=spec.raw_data)
assert experiment.is_independent is True
assert ExperimentStatus.objects.filter(experiment=experiment).count() == 3
assert list(ExperimentStatus.objects.filter(experiment=experiment).values_list(
'status', flat=True)) == [ExperimentLifeCycle.CREATED,
ExperimentLifeCycle.SCHEDULED,
ExperimentLifeCycle.STARTING]
experiment.refresh_from_db()
assert experiment.last_status == ExperimentLifeCycle.STARTING
# Assert 3 jobs were created with resources
assert ExperimentJob.objects.filter(experiment=experiment).count() == 3
assert JobResources.objects.count() == 3
jobs_statuses = ExperimentJob.objects.values_list('statuses__status', flat=True)
assert set(jobs_statuses) == {JobLifeCycle.CREATED, }
jobs = ExperimentJob.objects.filter(experiment=experiment)
assert experiment.calculated_status == ExperimentLifeCycle.STARTING
for job in jobs:
# Assert the jobs status is created
assert job.last_status == JobLifeCycle.CREATED
def check_achievements(self, team=None):
for template in AchievementTemplate.objects.all():
pred = template.predicate.replace('\r','')
code = compile(pred, '<string>', 'exec')
ns = {}
exec code in ns
team1 = self.home_team
team2 = self.away_team
for team in team1,team2:
try:
achievement = Achievement.objects.get(template=template, team=team)
except:
achievement = Achievement(template=template, team=team, points=0)
achievement.save()
changed, new_points = ns['match'](self, team, achievement.points)
if template.is_average:
achievement.game.clear()
achievement.game.add(self)
achievement.points = new_points
achievement.save()
elif changed:
if float(new_points) > float(achievement.points):
achievement.game.clear()
achievement.points = new_points
achievement.game.add(self)
achievement.save()
def _init_string(self, string):
if isinstance(string, HasNamespace):
string = string.string
self.string = string
self.code = compiler.compile(string, repr(self), self.mode)
self._inspect(string)
assert not self.sym_assigned, 'expression cannot contain assignment: %r' % string
def evaluate(self, cog, globals, fname='cog generator'):
# figure out the right whitespace prefix for the output
prefOut = whitePrefix(self.markers)
intext = self.getCode()
if not intext:
return ''
# In Python 2.2, the last line has to end in a newline.
intext = "import cog\n" + intext + "\n"
code = compiler.compile(intext, filename=str(fname), mode='exec')
# Make sure the "cog" module has our state.
cog.cogmodule.msg = self.msg
cog.cogmodule.out = self.out
cog.cogmodule.outl = self.outl
cog.cogmodule.error = self.error
self.outstring = ''
eval(code, globals)
# We need to make sure that the last line in the output
# ends with a newline, or it will be joined to the
# end-output line, ruining cog's idempotency.
if self.outstring and self.outstring[-1] != '\n':
self.outstring += '\n'
return reindentBlock(self.outstring, prefOut)
def compile_files(dir):
print "Compiling",
line_len = 10
for file in os.listdir(dir):
base, ext = os.path.splitext(file)
if ext == '.py' and base[:4] == 'test':
source = os.path.join(dir, file)
line_len = line_len + len(file) + 1
if line_len > 75:
print "\n\t",
line_len = len(source) + 9
print file,
compile(source)
# make sure the .pyc file is not over-written
os.chmod(source + "c", 444)
print
def compile_file(filename):
contents = read_file(filename)
itpr = interpreter.Interpreter()
ast = parser.parse(contents)
bytecode = compiler.compile(ast)
return itpr.interpret(bytecode).to_string()
def load_comments(self, pkgfile):
""" Open the package and load comments if any.
Return the loaded comments """
# Note: This has to be called with a Python
# source file (.py) only!
if not os.path.exists(pkgfile):
return ""
comment = ""
try:
of = open(pkgfile, 'rb')
data = of.read()
if data:
# Create code object
try:
c = compiler.compile(data, pkgfile, 'exec')
# Get the position of first line of code
if c:
lno = c.co_firstlineno
lnum = 0
# Read file till this line number
of.seek(0)
for line in of:
comment = "".join((comment, line))
lnum += 1
if lnum == lno or line == "\n": break
except SyntaxError, e:
pass
except Exception, e:
pass
def testTryExceptFinally(self):
# Test that except and finally clauses in one try stmt are recognized
c = compiler.compile("try:\n 1/0\nexcept:\n e = 1\nfinally:\n f = 1", "<string>", "exec")
dct = {}
exec c in dct
self.assertEquals(dct.get("e"), 1)
self.assertEquals(dct.get("f"), 1)
def create_production_function_fast(self, formula, output_goods, input_goods, typ='from_formula'):
""" creates a production function from formula, with given outputs
A production function is a production process that produces the
given input goods according to the formula to the output
goods.
Production_functions are then used as an argument in produce,
predict_vector_produce and predict_output_produce.
Args:
"formula": equation or set of equations that describe the
production process. (string) Several equation are separated by a ;
[output]: list of all output goods (left hand sides of the equations)
Returns:
A production_function that can be used in produce etc.
Example:
formula = 'golf_ball = (ball) * (paint / 2); waste = 0.1 * paint'
self.production_function = self.create_production_function(formula, 'golf', ['waste', 'paint'])
self.produce(self.production_function, {'ball' : 1, 'paint' : 2}
//exponential is ** not ^
"""
production_function = {}
production_function['type'] = typ
production_function['formula'] = formula
production_function['code'] = compiler.compile(formula, '<string>', 'exec')
production_function['output'] = output_goods
production_function['input'] = input_goods
return production_function
def judge_submission(submission):
'''
Judge the target submission
'''
if submission.language is None:
raise RuntimeError('Unknown Language')
else:
upload_result(
Report(result=Judge_result.PR, submission=submission.submission))
st, info = pull(lock=gloal_problem_lock.get(submission.problem),
problem=submission.problem)
if not st:
raise RuntimeError("Pull error: " + str(info))
st, info = create_tempfile(sourcefile=submission.sourcefile,
work_dir=submission.work_dir,
lang=submission.language,
code=submission.code)
if st != 'Success':
raise RuntimeError("Judger Error during creating tempfile: " +
str(info))
if submission.language.value.compile is True:
result, information = compile(submission=submission)
if result is Judge_result.CE:
upload_result(
Report(result=result,
complete=True,
submission=submission.submission,
compileerror_msg=information))
return
elif result is Judge_result.JE:
raise RuntimeError("Judger Error during compiling: " +
str(information))
submission.case = get_test_case(submission.problem)
if len(submission.case) == 0:
raise RuntimeError("Judger Error, because there is no test-data")
def rank_population(population, target_output):
# convert target output to ASCII number
target_output = [ord(c) for c in target_output]
scores = []
for genome in population:
score = 0
try:
result = compiler.compile(genome) # final output
result = [ord(c) for c in result] # ASCII value of result
# calculate each character in output and subtract from desired output
# perfect score = 256 for each character
# currently does not punish too long output - just looks for the target output first
try:
for i in range(len(target_output)):
score += 256 - abs(result[i] - target_output[i])
scores.append(score)
except IndexError: # output is too short
scores.append(score)
except:
scores.append(0)
return population, scores
def _init_string(self,string):
if isinstance(string,HasNamespace):
string = string.string
self.string = string
self.code = compiler.compile(string,repr(self),self.mode)
self._inspect(string)
assert not self.sym_assigned, 'expression cannot contain assignment: %r' % string
def load_module(self, fullname):
#~ print "--- patchwork_loader.load_module(self,\n\t%s)" % (fullname)
source = ""
if os.path.exists(self.path):
#~ print "\tRC FILE: %s" % (self.path);
source += file(self.path, "rU").read()
else:
#~ print "\tZIP FILE: %s" % (self.path);
source += self.importer.get_data(self.path).replace("\r\n",
"\n").replace(
"\r", "\n")
source += '''
import __builtin__
if __builtin__.open == open:
from boost.patchwork import _open_ as open
if isinstance(file,type):
from boost.patchwork import _file_ as file
'''
code = compiler.compile(source, self.path, 'exec')
mod = sys.modules.setdefault(fullname, imp.new_module(fullname))
mod.__file__ = os.path.join(self.importer.archive, self.path)
mod.__loader__ = self
if self.path.endswith("__init__.py"):
mod.__path__ = [
os.path.join(self.importer.archive, os.path.dirname(self.path))
]
exec code in mod.__dict__
return mod
def test9UserTriggers(self):
# Dependency Injection! (the nasty way):
triggers = \
'class Trigger(object):\n def evaluate():\n return False\nclass WordTrigger(Trigger):\n def __init__(self, word):\n self.word = word.lower()\n print "Injected!"\n def hash(self):\n raise NotImplementedError\nclass SubjectTrigger(Trigger):\n def __init__(self, word):\n self.word = word.lower()\n print "Injected Subject!"\n def hash(self):\n return "SUBJECT-" + self.word\nclass TitleTrigger(WordTrigger):\n def hash(self):\n return "TITLE-" + self.word\nclass SummaryTrigger(WordTrigger):\n def hash(self):\n return "SUMMARY-" + self.word\nclass NotTrigger(Trigger):\n def __init__(self, trigger):\n self.trigger = trigger\n print "Injected!"\n def hash(self):\n return "NOT-" + self.trigger.hash()\nclass PhraseTrigger(Trigger):\n def __init__(self, phrase):\n self.phrase = phrase\n print "Injected!"\n def hash(self):\n return "PHRASE-" + self.phrase\nclass CompositeTrigger(Trigger):\n def __init__(self, trigger1, trigger2):\n self.trigger1 = trigger1\n self.trigger2 = trigger2\nclass AndTrigger(CompositeTrigger):\n def hash(self):\n t1_hash = self.trigger1.hash()\n t2_hash = self.trigger2.hash()\n if t1_hash > t2_hash:\n return "AND-" + t1_hash + "-" + t2_hash\n else:\n return "AND-" + t2_hash + "-" + t1_hash\nclass OrTrigger(CompositeTrigger):\n def hash(self):\n t1_hash = self.trigger1.hash()\n t2_hash = self.trigger2.hash()\n if t1_hash > t2_hash:\n return "OR-" + t1_hash + "-" + t2_hash\n else:\n return "OR-" + t2_hash + "-" + t1_hash\n\n\n'
import compiler
f = open('ps5.py')
add_potential_points(5)
src = ""
for line in f:
if line == "if __name__ == '__main__':\n":
print "found line!"
break
src += line
src += triggers
print src
compiled = compiler.compile(src, 'compile-errors', 'exec')
exec compiled in locals(), globals()
t1 = SubjectTrigger('sports')
t2 = SummaryTrigger('Obama')
t3 = PhraseTrigger('Hillary Clinton')
t4 = OrTrigger(t2, t3)
trigger_map = {t1.hash() : False,
t4.hash() : False}
print trigger_map
trigger_list = readTriggerConfig("triggers.txt")
for trigger in trigger_list:
hash = trigger.hash()
self.assertTrue(hash in trigger_map)
self.assertFalse(trigger_map[hash])
trigger_map[hash] = True
for found in trigger_map.values():
self.assertTrue(found, "Missing trigger")
add_points(5)
def set_utility_function(self, formula, typ='from_formula'):
""" creates a utility function from formula
Utility_functions are then used as an argument in consume_with_utility,
predict_utility and predict_utility_and_consumption.
create_utility_function_fast is faster but more complicated utility_function
Args:
"formula": equation or set of equations that describe the
utility function. (string) needs to start with 'utility = ...'
Returns:
A utility_function
Example:
formula = 'utility = ball + paint'
self._utility_function = self.create_utility_function(formula)
self.consume_with_utility(self._utility_function, {'ball' : 1, 'paint' : 2})
//exponential is ** not ^
"""
parse_single_input = pp.Suppress(pp.Word(pp.alphas + "_", pp.alphanums + "_")) + pp.Suppress('=') \
+ pp.OneOrMore(pp.Suppress(pp.Optional(pp.Word(pp.nums + '*/+-().[]{} ')))
+ pp.Word(pp.alphas + "_", pp.alphanums + "_"))
parse_input = pp.delimitedList(parse_single_input, ';')
self._utility_function = {}
self._utility_function['type'] = typ
self._utility_function['formula'] = formula
self._utility_function['code'] = compiler.compile(formula, '<string>', 'exec')
self._utility_function['input'] = list(parse_input.parseString(formula))
def run(self,
query,
gt_filter=None,
show_variant_samples=False,
variant_samples_delim=',',
predicates=None,
needs_genotypes=False,
needs_genes=False,
show_families=False,
subjects=None):
"""
Execute a query against a Gemini database. The user may
specify:
1. (reqd.) an SQL `query`.
2. (opt.) a genotype filter.
"""
self.query = self.formatter.format_query(query)
self.gt_filter = gt_filter
if self._is_gt_filter_safe() is False:
sys.exit("ERROR: invalid --gt-filter command.")
self.show_variant_samples = show_variant_samples
self.variant_samples_delim = variant_samples_delim
self.needs_genotypes = needs_genotypes
self.needs_vcf_columns = False
if self.formatter.name == 'vcf':
self.needs_vcf_columns = True
self.needs_genes = needs_genes
self.show_families = show_families
self.subjects = subjects
if predicates:
self.predicates += predicates
# make sure the SELECT columns are separated by a
# comma and a space. then tokenize by spaces.
self.query = self.query.replace(',', ', ')
self.query_pieces = self.query.split()
if not any(s.startswith(("gt", "(gt")) for s in self.query_pieces) and \
not any(".gt" in s for s in self.query_pieces):
if self.gt_filter is None:
self.query_type = "no-genotypes"
else:
self.gt_filter = self._correct_genotype_filter()
self.query_type = "filter-genotypes"
else:
if self.gt_filter is None:
self.query_type = "select-genotypes"
else:
self.gt_filter = self._correct_genotype_filter()
self.query_type = "filter-genotypes"
if self.gt_filter:
import compiler
self.gt_filter_compiled = compiler.compile(self.gt_filter,
"<string>", 'eval')
self._apply_query()
self.query_executed = True
def get_ezjail_module(args):
import ezjail
encoded_args = repr(args.encode('utf-8'))
encoded_lang = repr(ansible.constants.DEFAULT_MODULE_LANG)
encoded_complex = repr(jsonify({}))
fn = ezjail.__file__.replace('.pyc', '.py')
with open(fn) as f:
module_data = f.read()
module_data = module_data.replace(module_common.REPLACER, module_common.MODULE_COMMON)
module_data = module_data.replace(module_common.REPLACER_ARGS, encoded_args)
module_data = module_data.replace(module_common.REPLACER_LANG, encoded_lang)
module_data = module_data.replace(module_common.REPLACER_COMPLEX, encoded_complex)
ofn = fn.replace('/ezjail.py', '/tmp_ezjail.py')
with open(ofn, 'w') as f:
f.write(module_data)
code = compiler.compile(module_data, ofn, 'exec')
ezjail = imp.new_module('ezjail')
exec code in ezjail.__dict__
ezjail.__file__ = ofn
class AnsibleModule(ezjail.AnsibleModule):
def get_bin_path(self, arg, required=False, opt_dirs=[]):
return '/usr/bin/%s' % arg
ezjail.AnsibleModule = AnsibleModule
module = AnsibleModule(**ezjail.MODULE_SPECS)
return ezjail, module
def set_utility_function_fast(self, formula, input_goods, typ='from_formula'):
""" creates a utility function from formula
Utility_functions are then used as an argument in consume_with_utility,
predict_utility and predict_utility_and_consumption.
create_utility_function_fast is faster but more complicated
Args:
"formula": equation or set of equations that describe the
production process. (string) Several equation are separated by a ;
[output]: list of all output goods (left hand sides of the equations)
Returns:
A utility_function that can be used in produce etc.
Example:
formula = 'utility = ball + paint'
self._utility_function = self.create_utility_function(formula, ['ball', 'paint'])
self.consume_with_utility(self._utility_function, {'ball' : 1, 'paint' : 2}
//exponential is ** not ^
"""
self._utility_function = {}
self._utility_function['type'] = typ
self._utility_function['formula'] = formula
self._utility_function['code'] = compiler.compile(formula, '<string>', 'exec')
self._utility_function['input'] = input_goods
def _init_string(self, string):
if isinstance(string, HasNamespace):
string = string.string
self.string = string
self.code = compiler.compile(string, repr(self), self.mode)
self._inspect(string)
if not self.sym_assigned:
raise SyntaxError, 'rule must contain assignment: %r' % string
