def main():
inp = "(- (+ 5 4) 1)"
comp = Compiler()
expr = comp.compile(inp)
expr.genC()
print
def process(src,filename=None):
from parser import Parser
from compiler import Compiler
parser = Parser(src,filename=filename)
block = parser.parse()
compiler = Compiler(block)
return compiler.compile()
class Game():
def __init__(self, gru_file=None):
self.compiler = Compiler()
if gru_file:
self.stream = self.compiler.decompile(gru_file)
else:
self.stream = self.compiler.compile(None)
self.metadata = self.stream.metadata
self.flags = Flags(self.stream)
self.wheel = Wheel(config)
self.title = Title(config)
self.inventory = Inventory(self.stream, self.flags, config)
self.combiner = Combiner(self.stream, self.flags, self.inventory)
self.page = Page(config, self.flags, self.combiner, self.inventory)
self.state = State(self.stream, self.flags, self.inventory, self.wheel, self.combiner, self.title, self.page)
if self.metadata.has_key("start"):
start = self.metadata["start"]
self.state.update(start)
else:
self.state.update("start")
def draw(self, tick):
self.inventory.draw()
self.wheel.draw()
self.title.draw()
self.page.draw(tick)
def compile_js(manifest,config):
js_file = os.path.join(cwd,'assets','__MODULE_ID__.js')
if not os.path.exists(js_file): return
from compiler import Compiler
try:
import json
except:
import simplejson as json
path = os.path.basename(js_file)
compiler = Compiler(cwd, manifest['moduleid'], manifest['name'], 'commonjs')
method = compiler.compile_commonjs_file(path,js_file)
exports = open('metadata.json','w')
json.dump({'exports':compiler.exports }, exports)
exports.close()
method += '\treturn filterDataInRange([NSData dataWithBytesNoCopy:data length:sizeof(data) freeWhenDone:NO], ranges[0]);'
f = os.path.join(cwd,'Classes','___PROJECTNAMEASIDENTIFIER___ModuleAssets.m')
c = open(f).read()
idx = c.find('return ')
before = c[0:idx]
after = """
}
@end
"""
newc = before + method + after
if newc!=c:
x = open(f,'w')
x.write(newc)
x.close()
def main():
if len(sys.argv) != 2:
print "Usage:"
print " main.py input output"
print "input is the file to be compiled"
print "output is the file where the generated code will be stored"
sys.exit(1)
try:
fp = open(sys.argv[1], "r")
except IOError:
print "Either the file", sys.argv[1], "does not exist or it cannot be read"
sys.exit(1)
inp = fp.read() + '\0'
fp.close()
comp = Compiler()
try:
output = open(sys.argv[2], "w")
except IOError:
print "File", sys.args[2], "could not be opened"
sys.exit(1)
printWriter = PrintWriter(output)
try:
program = comp.compile(inp, printWriter)
except Exception, e:
print e
def build_modules_info(self, resources_dir, app_bin_dir):
self.app_modules = []
(modules, external_child_modules) = bindings.get_all_module_bindings()
compiler = Compiler(self.tiapp, resources_dir, self.java, app_bin_dir, os.path.dirname(app_bin_dir))
compiler.compile(compile_bytecode=False, info_message=None)
for module in compiler.modules:
module_bindings = []
# TODO: we should also detect module properties
for method in compiler.module_methods:
if method.lower().startswith(module + ".") and "." not in method:
module_bindings.append(method[len(module) + 1 :])
module_class = None
module_apiName = None
for m in modules.keys():
if modules[m]["fullAPIName"].lower() == module:
module_class = m
module_apiName = modules[m]["fullAPIName"]
break
if module_apiName == None:
continue # module wasn't found
if "." not in module:
ext_modules = []
if module_class in external_child_modules:
for child_module in external_child_modules[module_class]:
if child_module["fullAPIName"].lower() in compiler.modules:
ext_modules.append(child_module)
self.app_modules.append(
{
"api_name": module_apiName,
"class_name": module_class,
"bindings": module_bindings,
"external_child_modules": ext_modules,
}
)
# discover app modules
detector = ModuleDetector(self.project_dir)
missing, detected_modules = detector.find_app_modules(self.tiapp, "android")
for missing_module in missing:
print "[WARN] Couldn't find app module: %s" % missing_module["id"]
self.custom_modules = []
for module in detected_modules:
if module.jar == None:
continue
module_jar = zipfile.ZipFile(module.jar)
module_bindings = bindings.get_module_bindings(module_jar)
if module_bindings is None:
continue
for module_class in module_bindings["modules"].keys():
module_id = module_bindings["proxies"][module_class]["proxyAttrs"]["id"]
print "[DEBUG] module_id = %s" % module_id
if module_id == module.manifest.moduleid:
print "[DEBUG] appending module: %s" % module_class
self.custom_modules.append({"class_name": module_class, "manifest": module.manifest})
def main():
import sys
filename = sys.argv[1]
f = open(filename)
comp = Compiler()
program = comp.compile(f.read())
program.genC()
def prepare_commonjs(module_dir):
truthy = ("true", "True", 1, "1", "Yes", "yes")
manifest_file = os.path.join(module_dir, "manifest")
if not os.path.exists(manifest_file):
print >> sys.stderr, "Manifest %s does not exist"
sys.exit(1)
with open(manifest_file, "r") as f:
lines = f.readlines()
id_lines = [l for l in lines if l.strip().startswith("moduleid:")]
if not id_lines:
print >> sys.stderr, "[ERROR] Manifest %s does not contain moduleid key." % manifest_file
sys.exit(1)
moduleid = id_lines[0].split(":")[1].strip()
commonjs_lines = [l for l in lines if l.strip().startswith("commonjs:")]
curval = False
if commonjs_lines:
curval = commonjs_lines[0].split(":")[1].strip() in truthy
commonjs_filename = os.path.join(module_dir, "assets", "%s.js" % moduleid)
is_commonjs = os.path.exists(commonjs_filename)
if (is_commonjs and not curval) or (not is_commonjs and curval):
# Need to re-write the key-value
for l in commonjs_lines:
lines.remove(l)
lines.append("commonjs: %s\n" % ("true" if is_commonjs else "false")) # Trying to avoid locale-specific true/false
with open(manifest_file, "w") as f:
f.writelines(lines)
print "[DEBUG] manifest re-written to set commonjs value"
if is_commonjs:
with open(os.path.join(module_android_dir, "generated", "CommonJsSourceProvider.java"), "r") as f:
source_provider_template = Template(f.read())
source_provider_class = source_provider_template.substitute(moduleid=moduleid)
output_folder = os.path.join(module_dir, "build", "generated", "java")
if not os.path.exists(output_folder):
os.makedirs(output_folder)
with open(os.path.join(output_folder, "CommonJsSourceProvider.java"), "w") as f:
f.write(source_provider_class)
# Determine which Titanium modules are used within the CommonJS code.
# We piggy-back on the functionality in compiler.py for this.
from compiler import Compiler
c = Compiler(None, None, None, None, None, None)
with open(commonjs_filename, "r") as f:
c.extract_modules(f.read())
if c.modules:
import simplejson
output_folder = os.path.join(module_dir, "build", "generated", "json")
if not os.path.exists(output_folder):
os.makedirs(output_folder)
with open(os.path.join(output_folder, "metadata.json"), "w") as f:
simplejson.dump({"exports": list(c.modules)}, f)
def build_modules_info(self, resources_dir, app_bin_dir, include_all_ti_modules=False):
self.app_modules = []
(modules, external_child_modules) = bindings.get_all_module_bindings()
compiler = Compiler(self.tiapp, resources_dir, self.java, app_bin_dir, os.path.dirname(app_bin_dir),
include_all_modules=include_all_ti_modules)
compiler.compile(compile_bytecode=False, info_message=None)
for module in compiler.modules:
module_bindings = []
# TODO: we should also detect module properties
for method in compiler.module_methods:
if method.lower().startswith(module+'.') and '.' not in method:
module_bindings.append(method[len(module)+1:])
module_class = None
module_apiName = None
for m in modules.keys():
if modules[m]['fullAPIName'].lower() == module:
module_class = m
module_apiName = modules[m]['fullAPIName']
break
if module_apiName == None: continue # module wasn't found
if '.' not in module:
ext_modules = []
if module_class in external_child_modules:
for child_module in external_child_modules[module_class]:
if child_module['fullAPIName'].lower() in compiler.modules:
ext_modules.append(child_module)
self.app_modules.append({
'api_name': module_apiName,
'class_name': module_class,
'bindings': module_bindings,
'external_child_modules': ext_modules
})
# discover app modules
detector = ModuleDetector(self.project_dir)
missing, detected_modules = detector.find_app_modules(self.tiapp, 'android')
for missing_module in missing: print '[WARN] Couldn\'t find app module: %s' % missing_module['id']
self.custom_modules = []
for module in detected_modules:
if module.jar == None: continue
module_jar = zipfile.ZipFile(module.jar)
module_bindings = bindings.get_module_bindings(module_jar)
if module_bindings is None: continue
for module_class in module_bindings['modules'].keys():
module_id = module_bindings['proxies'][module_class]['proxyAttrs']['id']
print '[DEBUG] module_id = %s' % module_id
if module_id == module.manifest.moduleid:
print '[DEBUG] appending module: %s' % module_class
self.custom_modules.append({
'class_name': module_class,
'manifest': module.manifest
})
def main():
inp = "a = 1 b = 3 : (- (+ a 4) b)"
comp = Compiler()
program = comp.compile(inp)
program.genC()
print
print "Value = ", program.eval()
def compile_grammar(grammar_txt, grammar_fname, compiled_parser_class_name,
package, inline_base):
g_parser = HandRolledGrammarParser(grammar_txt, grammar_fname)
g_ast, err = g_parser.parse()
if err:
return None, err
g, _ = Analyzer(g_ast).analyze()
compiler = Compiler(g, compiled_parser_class_name, package, inline_base)
return compiler.walk()
def build_modules_info(self, resources_dir, app_bin_dir):
compiler = Compiler(self.tiapp, resources_dir, self.java, app_bin_dir, os.path.dirname(app_bin_dir))
compiler.compile(compile_bytecode=False)
self.app_modules = []
template_dir = os.path.dirname(sys._getframe(0).f_code.co_filename)
android_modules_dir = os.path.abspath(os.path.join(template_dir, 'modules'))
modules = {}
for jar in os.listdir(android_modules_dir):
if not jar.endswith('.jar'): continue
module_path = os.path.join(android_modules_dir, jar)
module_jar = zipfile.ZipFile(module_path)
bindings_path = None
for name in module_jar.namelist():
if name.endswith('.json') and name.startswith('org/appcelerator/titanium/bindings/'):
bindings_path = name
break
if bindings_path is None: continue
bindings_json = module_jar.read(bindings_path)
module_bindings = simplejson.loads(bindings_json)
for module_class in module_bindings['modules'].keys():
full_api_name = module_bindings['proxies'][module_class]['proxyAttrs']['fullAPIName']
print '[INFO] module_class = ' + module_class + ', api_name=' + module_bindings['modules'][module_class]['apiName'] + ', full_api_name='+full_api_name
modules[module_class] = module_bindings['modules'][module_class]
modules[module_class]['fullAPIName'] = full_api_name
for module in compiler.modules:
bindings = []
# TODO: we should also detect module properties
for method in compiler.module_methods:
if method.lower().startswith(module+'.') and '.' not in method:
bindings.append(method[len(module)+1:])
module_class = None
module_apiName = None
for m in modules.keys():
if modules[m]['fullAPIName'].lower() == module:
module_class = m
module_apiName = modules[m]['fullAPIName']
break
if module_apiName == None: continue # module wasn't found
self.app_modules.append({
'api_name': module_apiName,
'class_name': module_class,
'bindings': bindings
})
def compile(filename, options):
f = open(filename, 'rt')
data = f.read()
f.close()
filename = filename.replace('\\', '/')
comp = Compiler(filename, options)
output = comp.compile(data)
if options.output:
with open(options.output, "wt") as f:
f.write('\n'.join(output))
else:
print '\n'.join(output)
def deploy(slug, testing_url, production_url, theme_url, production_server, production_dir):
build_dir = os.path.join(SETTINGS.BUILD_DIR, slug)
archive_dir = os.path.join(SETTINGS.ARCHIVE_DIR, slug)
compiler = Compiler(build_dir, testing_url, production_url, theme_url)
compiler.compile()
archiver = Archiver(slug, build_dir, archive_dir)
archive = archiver.archive()
deployer = Deployer(production_server, SETTINGS.SSH_KEY, archive_dir, production_dir)
deployer.deploy(archive)
return True
def compile_js(manifest, config):
js_file = os.path.join(cwd, "assets", "ti.mediapicker.js")
if not os.path.exists(js_file):
js_file = os.path.join(cwd, "..", "assets", "ti.mediapicker.js")
if not os.path.exists(js_file):
return
from compiler import Compiler
try:
import json
except:
import simplejson as json
compiler = Compiler(cwd, manifest["moduleid"], manifest["name"], "commonjs")
root_asset, module_assets = compiler.compile_module()
root_asset_content = (
"""
%s
return filterDataInRange([NSData dataWithBytesNoCopy:data length:sizeof(data) freeWhenDone:NO], ranges[0]);
"""
% root_asset
)
module_asset_content = (
"""
%s
NSNumber *index = [map objectForKey:path];
if (index == nil) {
return nil;
}
return filterDataInRange([NSData dataWithBytesNoCopy:data length:sizeof(data) freeWhenDone:NO], ranges[index.integerValue]);
"""
% module_assets
)
from tools import splice_code
assets_router = os.path.join(cwd, "Classes", "TiMediapickerModuleAssets.m")
splice_code(assets_router, "asset", root_asset_content)
splice_code(assets_router, "resolve_asset", module_asset_content)
# Generate the exports after crawling all of the available JS source
exports = open("metadata.json", "w")
json.dump({"exports": compiler.exports}, exports)
exports.close()
def test_python_unsuccessful_compilation(self):
# Unsuccessful, returns the compilation message as an error string
path_source = "unit_tests/fixtures/HelloWorldPythonCE.py"
path_executable = "unit_tests/test_sandbox/HelloWorldPythonCE.py"
message = Compiler.compile("Python", path_source, path_executable)
self.assertNotEqual("", message, "The Python compilation expected error message, but passed successfully.")
def test_cpp_successful_compilation(self):
# Successful, returns an empty string as an error message
path_source = "unit_tests/fixtures/HelloWorldCppOK.cpp"
path_executable = "unit_tests/test_sandbox/HelloWorldCppOK.o"
message = Compiler.compile("C++", path_source, path_executable)
self.assertEqual("", message, "The C++ compilation expected to pass, but failed.")
def test_compilation_timeout_fail(self):
# Too slow compilation: over the 10 second limit (>20s on the grader machine)
path_source = "unit_tests/fixtures/TemplateFibo.cpp"
path_executable = "unit_tests/test_sandbox/TemplateFibo.o"
message = Compiler.compile("C++", path_source, path_executable)
self.assertNotEqual("", message, "The C++ compilation expected to fail, but passed.")
def test_compilation_timeout_okay(self):
# Slow compilation, but within the 10 second limit
path_source = "unit_tests/fixtures/LongCompilation.cpp"
path_executable = "unit_tests/test_sandbox/LongCompilation.o"
message = Compiler.compile("C++", path_source, path_executable)
self.assertEqual("", message, "The C++ compilation failed, but expected to pass.")
def compile_js(manifest, config):
js_file = os.path.join(cwd, "assets", "ti.tvout.js")
if not os.path.exists(js_file):
return
sdk = config["TITANIUM_SDK"]
iphone_dir = os.path.join(sdk, "iphone")
sys.path.insert(0, iphone_dir)
from compiler import Compiler
path = os.path.basename(js_file)
metadata = Compiler.make_function_from_file(path, js_file)
method = metadata["method"]
eq = path.replace(".", "_")
method = " return %s;" % method
f = os.path.join(cwd, "Classes", "TiTvoutModuleAssets.m")
c = open(f).read()
idx = c.find("return ")
before = c[0:idx]
after = """
}
@end
"""
newc = before + method + after
if newc != c:
x = open(f, "w")
x.write(newc)
x.close()
def compile_js(manifest,config):
js_file = os.path.join(cwd,'assets','sg.flurry.js')
if not os.path.exists(js_file): return
sdk = config['TITANIUM_SDK']
iphone_dir = os.path.join(sdk,'iphone')
sys.path.insert(0,iphone_dir)
from compiler import Compiler
path = os.path.basename(js_file)
metadata = Compiler.make_function_from_file(path,js_file)
method = metadata['method']
eq = path.replace('.','_')
method = ' return %s;' % method
f = os.path.join(cwd,'Classes','SgFlurryModuleAssets.m')
c = open(f).read()
idx = c.find('return ')
before = c[0:idx]
after = """
}
@end
"""
newc = before + method + after
if newc!=c:
x = open(f,'w')
x.write(newc)
x.close()
def test_java_successful_compilation(self):
# Successful, returns an empty string as an error message
path_source = "unit_tests/fixtures/HelloWorldJavaOK.java"
path_executable = "unit_tests/test_sandbox/HelloWorldJavaOK.jar"
message = Compiler.compile("Java", path_source, path_executable)
self.assertEqual("", message, "The Java compilation expected to pass, but failed.")
self.assertTrue(path.exists(path_executable))
def main():
compiler = Compiler()
try:
opts, args = getopt.getopt(sys.argv[1:], "d:i:o:h", ["debug=", "inputfile=", "outputfile=", "help"])
except getopt.GetoptError as error:
print(error)
exit()
#Go through the options, initializing and configuring the compiler.
for opt, arg in opts:
if opt in ("-h", "--help"):
exit()
elif opt in ('-d', '--debug'):
compiler.setDebug(arg)
elif opt in ("-i", "--inputfile"):
try:
compiler.input = open(arg, mode='rt', encoding='utf-8')
except IOError as ioe:
print(ioe)
sys.exit(2)
elif opt in ("-o", "--outputfile"):
try:
compiler.output = open(arg, mode='wb')
except IOError as ioe:
print(ioe)
sys.exit(2)
compiler.compile()
sys.exit(0)
def compile_js(manifest,config):
moduleid = manifest['moduleid']
js_file = os.path.join(cwd,'assets',moduleid + '.js')
if not os.path.exists(js_file): return
from compiler import Compiler
try:
import json
except:
import simplejson as json
compiler = Compiler(cwd, moduleid, manifest['name'], 'commonjs')
root_asset, module_assets = compiler.compile_module()
root_asset_content = """
%s
return filterDataInRange([NSData dataWithBytesNoCopy:data length:sizeof(data) freeWhenDone:NO], ranges[0]);
""" % root_asset
module_asset_content = """
%s
NSNumber *index = [map objectForKey:path];
if (index == nil) {
return nil;
}
return filterDataInRange([NSData dataWithBytesNoCopy:data length:sizeof(data) freeWhenDone:NO], ranges[index.integerValue]);
""" % module_assets
from tools import splice_code
paths = glob.glob(os.path.join(cwd,'Classes','*ModuleAssets.m' ));
if (len(paths) == 0):
return;
assets_router = paths[0]
print(assets_router)
splice_code(assets_router, 'asset', root_asset_content)
splice_code(assets_router, 'resolve_asset', module_asset_content)
# Generate the exports after crawling all of the available JS source
exports = open('metadata.json','w')
json.dump({'exports':compiler.exports }, exports)
exports.close()
def test_java_unsuccessful_compilation(self):
# Unsuccessful, returns the compilation message as an error string
path_source = "unit_tests/fixtures/HelloWorldJavaCE.java"
path_executable = "unit_tests/test_sandbox/HelloWorldJavaCE.jar"
message = Compiler.compile("Java", path_source, path_executable)
self.assertIn("error", message)
self.assertNotEqual("", message, "The Java compilation expected error message, but passed successfully.")
self.assertFalse(path.exists(path_executable))
def test_java_different_class_name(self):
# Unsuccessful, returns the compilation message as an error string
path_source = "unit_tests/fixtures/HelloWorldJavaDifferentClassName.java"
path_executable = "unit_tests/test_sandbox/HelloWorldJavaDifferentClassName.jar"
message = Compiler.compile("Java", path_source, path_executable)
self.assertNotIn("error", message)
self.assertEqual("", message, "The Java compilation expected to pass, but failed.")
self.assertTrue(path.exists(path_executable))
def compile_js(manifest, config):
js_file = os.path.join(cwd, 'assets', 'ro.toshi.ti.mod.tisocketrocket.js')
if not os.path.exists(js_file):
js_file = os.path.join(cwd, '..', 'assets',
'ro.toshi.ti.mod.tisocketrocket.js')
if not os.path.exists(js_file): return
from compiler import Compiler
try:
import json
except:
import simplejson as json
compiler = Compiler(cwd, manifest['moduleid'], manifest['name'],
'commonjs')
root_asset, module_assets = compiler.compile_module()
root_asset_content = """
%s
return filterDataInRange([NSData dataWithBytesNoCopy:data length:sizeof(data) freeWhenDone:NO], ranges[0]);
""" % root_asset
module_asset_content = """
%s
NSNumber *index = [map objectForKey:path];
if (index == nil) {
return nil;
}
return filterDataInRange([NSData dataWithBytesNoCopy:data length:sizeof(data) freeWhenDone:NO], ranges[index.integerValue]);
""" % module_assets
from tools import splice_code
assets_router = os.path.join(cwd, 'Classes',
'RoToshiTiModTisocketrocketModuleAssets.m')
splice_code(assets_router, 'asset', root_asset_content)
splice_code(assets_router, 'resolve_asset', module_asset_content)
# Generate the exports after crawling all of the available JS source
exports = open('metadata.json', 'w')
json.dump({'exports': compiler.exports}, exports)
exports.close()
class Client:
# Instantiating SocketIO Client
__sio = socketio.AsyncClient()
__compiler = Compiler()
def __init__(self, PORT):
self.PORT = PORT
asyncio.run(self.start())
async def start(self):
await self.__sio.connect(f'http://localhost:{self.PORT}')
await self.__sio.wait()
# SocketIO events
# --------------------------------------------------------------
@__sio.event
def connect():
logger.info(
f'Connection to NodeJS established UID: {Client.__sio.sid}')
@__sio.event
def disconnect():
logger.info(f'Backend disconnected from NodeJS')
@__sio.event
async def submission(data):
# Deconstructing data
id, lang, code = data.values()
# Send an acknowledge packet
await Client.__sio.emit('submissionAck', {'id': id})
p = multiprocessing.Process(
name=f'Submission-{id}', target=Client.handleSubmission, args=(id, lang, code))
p.daemon = True
p.start()
@__sio.event
async def getPoolSize():
await Client.__sio.emit('getPoolSize', len(multiprocessing.active_children()))
def handleSubmission(id, lang, code):
# Maximum time
try:
with tlim(5):
logger.info(f'[PROCESSING] {multiprocessing.current_process().name}')
# Get compiler
c = Client.__compiler.getCompiler(lang)
s = Submission(id, lang, code, c)
s.start()
except TimeoutException:
return
finally:
pass
def test_python_successful_compilation(self):
# Successful, returns an empty string as an error message
path_source = "unit_tests/fixtures/HelloWorldPythonOK.py"
path_executable = "unit_tests/test_sandbox/HelloWorldPythonOK.py"
message = Compiler.compile("Python", path_source, path_executable)
self.assertEqual("", message, "The Python compilation expected to pass, but failed.")
self.assertTrue(path.exists(path_executable))
# The "compiled" file should be the same as the source
self.assertTrue(filecmp.cmp(path_source, path_executable, shallow=False))
def test_run_program_io(self):
path_source = os.path.join(self.PATH_FIXTURES, "reverser.cpp")
path_executable = os.path.join(config.PATH_SANDBOX, "reverser.o")
status = Compiler.compile(config.LANGUAGE_CPP, path_source, path_executable)
self.assertEqual(status, "")
run_result = Runner.run_program(sandbox=Sandbox(), executable_path=path_executable,
memory_limit=32000000, timeout=1.0, input_bytes=b"espr1t")
self.assertEqual(run_result.exit_code, 0)
self.assertEqual(run_result.output.decode().strip(), "t1rpse")
def test_run_program_output_limit(self):
path_source = os.path.join(self.PATH_FIXTURES, "outputlimit.cpp")
path_executable = os.path.join(config.PATH_SANDBOX, "outputlimit.o")
status = Compiler.compile(config.LANGUAGE_CPP, path_source, path_executable)
self.assertEqual(status, "")
run_result = Runner.run_program(sandbox=Sandbox(), executable_path=path_executable,
memory_limit=64000000, timeout=1.0, input_bytes=None)
self.assertEqual(run_result.exit_code, 0)
self.assertEqual(len(run_result.output.decode()), config.MAX_EXECUTION_OUTPUT)
def parse(self):
#initialize object
self.tokenizer = Tokenizer(self.file_name)
#remove all comments
self.tokenizer.remove_comments()
#get the tokens
self.tokenizer.prepare_tokens()
#create the new file names
xml_name = self.file_name.replace('.jack', '.xml')
#create the parser
compiler = Compiler(self.tokenizer, xml_name)
#run the parser
compiler.Compile()
def test_cpp_unsuccessful_compilation(self):
# Unsuccessful, returns the compilation message as an error string
language = "C++"
path_source = "unit_tests/fixtures/HelloWorldCE.cpp"
path_executable = "unit_tests/test_sandbox/HelloWorldCE.o"
message = Compiler.compile(language, path_source, path_executable)
self.assertNotEqual(
"", message,
"The C++ compilation expected error message, but passed successfully."
)
def __init__(self,
path_to_domain_file,
path_to_rules_file,
shuffle_randomly=True,
random_seed=None):
"""Initialize a RuleEngine object."""
self.domain, initial_facts = Compiler.parse_domain_file(
path_to_domain_file=path_to_domain_file)
self.rules = Compiler.parse_rules_file(
path_to_rules_file=path_to_rules_file)
self.shuffle_randomly = shuffle_randomly
self.working_memory = WorkingMemory(initial_facts=initial_facts)
self.actions = [
] # A list of Action objects, representing all the actions that have occurred
# If we received a random seed, use it to seed the random module
if random_seed is not None:
random.seed(random_seed)
else:
random.seed(time.time(
)) # Seed with the current UNIX time to introduce randomness
def judge(self,
language_config,
src,
max_cpu_time,
max_memory,
test_case_id,
spj_version=None,
spj_config=None,
spj_compile_config=None,
spj_src=None,
output=False):
# init
compile_config = language_config.get("compile")
run_config = language_config["run"]
submission_id = str(uuid.uuid4())
if spj_version:
self.compile_spj(spj_version=spj_version,
src=spj_src,
spj_compile_config=spj_compile_config,
test_case_id=test_case_id)
with InitSubmissionEnv(
JUDGER_WORKSPACE_BASE,
submission_id=str(submission_id)) as submission_dir:
if compile_config:
src_path = os.path.join(submission_dir,
compile_config["src_name"])
# write source code into file
with open(src_path, "w") as f:
f.write(src.encode("utf-8"))
# compile source code, return exe file path
exe_path = Compiler().compile(compile_config=compile_config,
src_path=src_path,
output_dir=submission_dir)
else:
exe_path = os.path.join(submission_dir, run_config["exe_name"])
with open(exe_path, "w") as f:
f.write(src.encode("utf-8"))
judge_client = JudgeClient(run_config=language_config["run"],
exe_path=exe_path,
max_cpu_time=max_cpu_time,
max_memory=max_memory,
test_case_id=str(test_case_id),
submission_dir=submission_dir,
spj_version=spj_version,
spj_config=spj_config,
output=output)
run_result = judge_client.run()
return run_result
def compile_js(manifest,config):
js_file = os.path.join(cwd,'assets','__MODULE_ID__.js')
if not os.path.exists(js_file): return
sdk = find_sdk(config)
iphone_dir = os.path.join(sdk,'iphone')
sys.path.insert(0,iphone_dir)
from compiler import Compiler
sys.path.append(os.path.join(sdk, "common"))
try:
import json
except:
import simplejson as json
path = os.path.basename(js_file)
compiler = Compiler(cwd, manifest['moduleid'], manifest['name'], 'commonjs')
metadata = compiler.make_function_from_file(path,js_file)
exports = open('metadata.json','w')
json.dump({'exports':compiler.exports }, exports)
exports.close()
method = metadata['method']
eq = path.replace('.','_')
method = ' return filterData(%s, @"%s");' % (method, manifest['moduleid'])
f = os.path.join(cwd,'Classes','___PROJECTNAMEASIDENTIFIER___ModuleAssets.mm')
c = open(f).read()
idx = c.find('return ')
before = c[0:idx]
after = """
}
@end
"""
newc = before + method + after
if newc!=c:
x = open(f,'w')
x.write(newc)
x.close()
def compile_js(manifest,config):
js_file = os.path.join(cwd,'assets','rebel.Parse.js')
if not os.path.exists(js_file):
js_file = os.path.join(cwd,'..','assets','rebel.Parse.js')
if not os.path.exists(js_file): return
from compiler import Compiler
try:
import json
except:
import simplejson as json
compiler = Compiler(cwd, manifest['moduleid'], manifest['name'], 'commonjs')
root_asset, module_assets = compiler.compile_module()
root_asset_content = """
{0!s}
return filterDataInRange([NSData dataWithBytesNoCopy:data length:sizeof(data) freeWhenDone:NO], ranges[0]);
""".format(root_asset)
module_asset_content = """
{0!s}
NSNumber *index = [map objectForKey:path];
if (index == nil) {{
return nil;
}}
return filterDataInRange([NSData dataWithBytesNoCopy:data length:sizeof(data) freeWhenDone:NO], ranges[index.integerValue]);
""".format(module_assets)
from tools import splice_code
assets_router = os.path.join(cwd,'Classes','RebelParseModuleAssets.m')
splice_code(assets_router, 'asset', root_asset_content)
splice_code(assets_router, 'resolve_asset', module_asset_content)
# Generate the exports after crawling all of the available JS source
exports = open('metadata.json','w')
json.dump({'exports':compiler.exports }, exports)
exports.close()
def test_run_program_memory_py(self):
path_source = os.path.join(self.PATH_FIXTURES, "hello.py")
path_executable = os.path.join(config.PATH_SANDBOX, "hello.py")
status = Compiler.compile(config.LANGUAGE_PYTHON, path_source, path_executable)
self.assertEqual(status, "")
run_result = Runner.run_program(sandbox=Sandbox(), executable_path=path_executable,
memory_limit=64000000, timeout=1.0)
self.assertEqual(run_result.exit_code, 0)
self.assertEqual(run_result.output.decode().strip(), "Hello, World!")
self.assertGreater(run_result.exec_memory, 0)
self.assertLess(run_result.exec_memory, 2000000) # Max 2MB overhead using this function
def standard_problem_helper(self, add_info, evaluator, language, source,
time_lb, memory_lb, expected_errors):
evaluator.create_sandbox_dir()
compilation_status = Compiler.compile(
language=language,
path_source=source,
path_executable=evaluator.path_executable)
self.assertEqual(compilation_status, "")
updater_results = []
add_info.side_effect = lambda result: updater_results.append(result)
self.assertTrue(evaluator.run_solution())
self.assertEqual(add_info.call_count, len(evaluator.tests) * 2)
actual_non_ok = {}
max_time, max_memory = -1e100, -1e100
for res in updater_results:
if res["status"] != TestStatus.TESTING.name:
# print(res)
if res["status"] != TestStatus.ACCEPTED.name:
# This is only because we only test wrong answers from the task with checkers
# In real tasks this is usually empty
if res["status"] == TestStatus.WRONG_ANSWER.name:
self.assertNotEqual(res["info"], "")
if res["status"] not in actual_non_ok:
actual_non_ok[res["status"]] = 1
else:
actual_non_ok[res["status"]] += 1
max_time = max(max_time, res["exec_time"])
max_memory = max(max_memory, res["exec_memory"])
time_upper_bound = evaluator.time_limit * 3 + max(
0.2, evaluator.time_limit * 0.2)
self.assertGreaterEqual(max_time, time_lb)
self.assertLessEqual(max_time, time_upper_bound)
self.assertGreaterEqual(max_memory, memory_lb)
self.assertLessEqual(max_memory, evaluator.memory_limit)
for key in actual_non_ok.keys():
if key not in expected_errors:
self.fail("Got status {} which was not expected.".format(key))
if actual_non_ok[key] != expected_errors[key]:
self.fail(
"Expected {} results with status {} but got {}.".format(
expected_errors[key], key, actual_non_ok[key]))
for key in expected_errors.keys():
if key not in actual_non_ok:
self.fail(
"Expected status {} but didn't receive it.".format(key))
def __init__(self, filename=None):
self.ps = PS1
self.toker = Tokenizer()
self.parser = Parser()
self.compiler = Compiler()
self.vm = vm.VM()
self.env = init_compiletime_env()
if filename is None:
self.infile = None
else:
self.infile = open(filename, 'r')
def compile_js(manifest, config):
js_file = os.path.join(cwd, 'assets',
'net.iamyellow.tigrowingtextfield.js')
if not os.path.exists(js_file): return
from compiler import Compiler
try:
import json
except:
import simplejson as json
path = os.path.basename(js_file)
compiler = Compiler(cwd, manifest['moduleid'], manifest['name'],
'commonjs')
metadata = compiler.make_function_from_file(path, js_file)
exports = open('metadata.json', 'w')
json.dump({'exports': compiler.exports}, exports)
exports.close()
method = metadata['method']
eq = path.replace('.', '_')
method = ' return filterData(%s, @"%s");' % (method, manifest['moduleid'])
f = os.path.join(cwd, 'Classes',
'NetIamyellowTigrowingtextfieldModuleAssets.m')
c = open(f).read()
idx = c.find('return ')
before = c[0:idx]
after = """
}
@end
"""
newc = before + method + after
if newc != c:
x = open(f, 'w')
x.write(newc)
x.close()
def dispatch_job(self, request, responder):
# Mark the server as not idle.
self._taking_work.set()
# Sanity checks on the requested command line
cmd = request.get('cmd')
if not cmd:
raise ArgumentError('Command is either missing or empty')
if not isinstance(cmd, list):
raise TypeError('Expected list, got %s' % type(cmd))
executable, args = cmd[0], cmd[1:]
cwd = request.get('cwd')
# Get a Compiler instance corresponding to that executable.
# The cwd is necessary because sometimes the path to the executable is
# relative.
compiler = Compiler.from_path(executable, cwd)
if not compiler:
raise RuntimeError('%s is not a known compiler' % executable)
# Parse the command line arguments in the main thread, this is fast
# enough not to be a problem in practice, and avoids dispatching
# compilations that can't be cached.
parsed_args = None
try:
parsed_args = compiler.parse_arguments(args)
except CannotCacheError:
LOG.exception('could not cache')
self.stats['non-cachable'] += 1
except NotACompilationError:
LOG.info('not a compilation')
self.stats['non-compile'] += 1
if not parsed_args:
# Return status code -2 when the compiler result can't be cached
# or when the compiler is invoked for something else than a
# compilation.
responder.respond(-2)
return
# Prepare job for run_command and send it to the job pool.
self._last_id += 1
job = {
'id': self._last_id,
'compiler': compiler,
'args': args,
'parsed_args': parsed_args,
'cwd': cwd,
}
self._responders[self._last_id] = responder
self._pool.add_job(job)
def compile_js(manifest, config):
js_file = os.path.join(cwd, 'assets',
'net.iamyellow.tikeyboardlistener.js')
if not os.path.exists(js_file): return
from compiler import Compiler
try:
import json
except:
import simplejson as json
path = os.path.basename(js_file)
compiler = Compiler(cwd, manifest['moduleid'], manifest['name'],
'commonjs')
method = compiler.compile_commonjs_file(path, js_file)
exports = open('metadata.json', 'w')
json.dump({'exports': compiler.exports}, exports)
exports.close()
method += '\treturn filterDataInRange([NSData dataWithBytesNoCopy:data length:sizeof(data) freeWhenDone:NO], ranges[0]);'
f = os.path.join(cwd, 'Classes',
'NetIamyellowTikeyboardlistenerModuleAssets.m')
c = open(f).read()
templ_search = ' moduleAsset\n{\n'
idx = c.find(templ_search) + len(templ_search)
before = c[0:idx]
after = """
}
@end
"""
newc = before + method + after
if newc != c:
x = open(f, 'w')
x.write(newc)
x.close()
def test_run_program_memory_python_fifty(self):
path_source = os.path.join(self.PATH_FIXTURES, "fifty.py")
path_executable = os.path.join(config.PATH_SANDBOX, "fifty.py")
status = Compiler.compile(config.LANGUAGE_PYTHON, path_source, path_executable)
self.assertEqual(status, "")
memory_target, memory_limit = 50000000, 64000000
run_result = Runner.run_program(sandbox=Sandbox(), executable_path=path_executable,
memory_limit=memory_limit, timeout=1.0, input_bytes=str(memory_target).encode())
self.assertEqual(run_result.exit_code, 0)
self.assertEqual(run_result.output.decode().strip(), "57864746")
self.assertGreater(run_result.exec_memory, memory_target)
self.assertLess(run_result.exec_memory, memory_limit)
def __init__(self,arch):
"""Deja todo listo para que la VM pueda interpretar los cuadruplos
,toma las constantes, los cuadruplos y las variables con sus nombres
del compiler y los pone en variables para usarse en la VM
Arguments:
arch {string} -- El nombre del archivo a ejecutar, por default en la carpeta /Pruebas
"""
c = Compiler(arch)
self.quadruples, ctes, self.addrNames = c.compile()
ctesInt = {key: value for key, value in ctes.items() if key < 38000}
ctesFloat = {key: value for key, value in ctes.items() if key > 37999 and key < 41000}
ctesChar = {key: value for key, value in ctes.items() if key > 40999}
memGlobal = Memory()
memLocal = {'temps': Memory(), 'local': Memory()}
memCtes = Memory()
memCtes.int = ctesInt
memCtes.float = ctesFloat
memCtes.char = ctesChar
self.memories = {'global' : memGlobal,'local': memLocal, 'ctes':memCtes}
self.currentFunction = 'global'
def test_dump():
directory = DUMP_FOLDER
xml_parser = WikiXML(namespace='http://www.mediawiki.org/xml/export-0.10/')
compiler = Compiler()
miss = 0
for wiki in directory.iterdir():
if wiki.is_file() and wiki.stem.startswith('enwiki'):
for root in xml_parser.from_xml(str(wiki)):
listener = None
try:
id, title, text = xml_parser.get(root)
logger.info(f'{title.text} compiling')
# if title.text.lower() == "athens":
# logger.info(text.text)
article = compiler.compile(text.text)
logger.info(f'{title.text} compiled')
except (ParseError, MalformedTag, RedirectFound) as e:
miss += 1
logger.info(f'{e.type}')
if miss > 0:
logger.warning(f'{miss} articles ignored')
def __init__(self, container):
Frame.__init__(self, container)
self.parent = container.parent
self.packed = False
self.button_lex = Button(self,
text="LEX",
command=lambda: self.lexical_analysis())
self.button_syn = Button(
self,
text="SYN",
command=lambda: self.lexical_and_syntax_analysis())
self.button_pfix = Button(self,
text="PFIX",
command=lambda: self.postfix())
self.button_run = Button(self,
text="RUN",
command=lambda: self.run(True))
self.button_lex.pack(side="left")
self.button_syn.pack(side="left")
self.button_pfix.pack(side="left")
self.button_run.pack(side="left")
self.compiler = Compiler()
def compile(self, src, object):
if not self.rebuildAll:
if not self.checkRebuild(src, object):
return True
print("Compiling " + src)
try:
c = Compiler(src, object, CompilerDispatcher())
except CompilerException as e:
print("**ERROR** " + e.message)
sys.exit(1)
self.compiles += 1
return True
def debugRun(source_code, vm):
#why this runs, its because of the lox interpreter environment, which remains in existence even after this function ends
scanner = Scanner(source_code)
scanner.scanTokens()
print(scanner.toString())
parser = Parser(scanner.token_list)
parser.parse()
print("##################################AST###################################")
print(ASTPrinter().printAll(parser.AST))
print("########################################################################")
compiler = Compiler()
compiler.compileAll(parser.AST)
disassembler = Disassembler(compiler.chunk)
print("#############################ByteCode###################################")
print(disassembler.pretty_print())
print("########################################################################")
vm.run(compiler.chunk)
def compile_js(manifest, config):
js_file = os.path.join(cwd, 'assets', '__MODULE_ID__.js')
if not os.path.exists(js_file): return
sdk = find_sdk(config)
iphone_dir = os.path.join(sdk, 'iphone')
sys.path.insert(0, iphone_dir)
from compiler import Compiler
path = os.path.basename(js_file)
compiler = Compiler(cwd, manifest['moduleid'], manifest['name'],
'commonjs')
metadata = compiler.make_function_from_file(path, js_file)
exports = open('metadata.json', 'w')
json.dump({'exports': compiler.exports}, exports)
exports.close()
method = metadata['method']
eq = path.replace('.', '_')
method = ' return %s;' % method
f = os.path.join(cwd, 'Classes',
'___PROJECTNAMEASIDENTIFIER___ModuleAssets.m')
c = open(f).read()
idx = c.find('return ')
before = c[0:idx]
after = """
}
@end
"""
newc = before + method + after
if newc != c:
x = open(f, 'w')
x.write(newc)
x.close()
def test_run_program_memory_java_with_imports(self):
path_source = os.path.join(self.PATH_FIXTURES, "hello_imports.java")
path_executable = os.path.join(config.PATH_SANDBOX, "hello_imports.jar")
status = Compiler.compile(config.LANGUAGE_JAVA, path_source, path_executable)
self.assertEqual(status, "")
run_result = Runner.run_program(sandbox=Sandbox(), executable_path=path_executable,
memory_limit=64000000, timeout=1.0)
self.assertEqual(run_result.exit_code, 0)
self.assertEqual(run_result.output.decode().strip(), "Hello, World!")
# Note that Java's Garbage Collector makes the process quite volatile (and unpredictable)
# in terms of memory usage. Set more relaxed limitations for Java
self.assertGreaterEqual(run_result.exec_memory, 0)
self.assertLess(run_result.exec_memory, 4000000) # Max 4MB overhead using this function
def __init__(self, parts: List[Part], student_dir, compiler_dir):
assert_uids(parts)
self.part_to_pos = {p: ii for ii, p in enumerate(parts)}
self.pos_to_part = parts
self.part_count = len(parts)
self.parts = {p.uid: p for p in parts}
self.students = {}
# make uids comparable
uids = reduce(operator.add, ([(p.uid, s.uid) for s in p.steps.values()]
for p in parts))
self.start = uids[0]
self.uid_progress = {uid: ii for ii, uid in enumerate(uids)}
print(self.uid_progress)
self.app_html: Optional[Template] = None
# command list
self.cmds = {'next': self.next, 'answer': self.answer, 'run': self.run}
# student directory
assert os.path.isdir(student_dir)
self.student_dir = student_dir
# compiler
self.comp = Compiler(working_dir=compiler_dir)
# converter
self.conv = Ansi2HTMLConverter()
def test_run_program_concurrent_hdd_access(self):
# Cannot test concurrency if there is none
if config.MAX_PARALLEL_WORKERS <= 1:
return
path_source = os.path.join(self.PATH_FIXTURES, "summer_hdd.cpp")
path_executable = os.path.join(config.PATH_SANDBOX, "summer_hdd.o")
status = Compiler.compile(config.LANGUAGE_CPP, path_source,
path_executable)
self.assertEqual(status, "")
# Write many integers into a (big) file
input_file = NamedTemporaryFile(mode="w+b", delete=False)
num_runs = self.get_num_runs(20)
target_size = 100000000 # 100MB
number_list = [999999999 - i for i in range(target_size // 10)]
expected_output = sum(number_list)
input_file.write(
str(number_list).replace('[', '').replace(']', '').replace(
',', '').encode("ascii"))
input_file.flush()
# Actual file size should be +/- 1% of the target
self.assertAlmostEqual(
os.path.getsize(input_file.name) / target_size, 1.0, 2)
times = [1e100] * num_runs
# Run the same thing several times as quickly as possible to see if there is any significant increase
pool = ThreadPoolExecutor(max_workers=config.MAX_PARALLEL_WORKERS)
futures = [
pool.submit(self.concurrent_helper, input_file.name,
path_executable) for _ in range(num_runs)
]
for i in range(len(futures)):
run_result = futures[i].result()
self.assertEqual(run_result.exit_code, 0)
self.assertEqual(int(run_result.output.decode().strip()),
expected_output)
times[i] = round(run_result.exec_time, 2)
input_file.close()
print(times)
print("TIME: {:.3f}s vs {:.3f}s".format(min(times), max(times)))
# Best time should be almost identical to the worst time in order to consider the results consistent
# (at most 10% difference or 0.05s, whichever larger)
self.assertLessEqual(max(times),
max(min(times) + 0.05,
min(times) * 1.1))
def test_approve(self):
micheline = Compiler(source).compile_contract()
vm = VM()
vm.load_contract(micheline)
storage = vm.contract.storage.dummy()
storage['owner'] = vm.context.sender
investor = "KT1EwUrkbmGxjiRvmEAa8HLGhjJeRocqVTFi"
storage = vm.contract.approve({
"spender": investor,
"value": 4
}).interpret(storage=storage, sender=vm.context.sender).storage
self.assertEqual(storage['allowances'],
{(vm.context.sender, investor): 4})
def test_close(self):
micheline = Compiler(source).compile_contract()
vm = VM()
vm.load_contract(micheline)
init_storage = vm.contract.storage.dummy()
init_storage['admin'] = vm.context.sender
new_storage = vm.contract.close("foo").interpret(storage=init_storage, sender=vm.context.sender).storage
self.assertEqual(new_storage['_close'], "foo")
try:
new_storage = vm.contract.close("foo").interpret(storage=init_storage)
assert 0
except MichelsonRuntimeError as e:
self.assertEqual(e.format_stdout(), "FAILWITH: 'Only admin can call this entrypoint'")
def test_run_program_exec_time_cpu_intensive(self):
path_source = os.path.join(self.PATH_FIXTURES, "cpuintensive.cpp")
path_executable = os.path.join(config.PATH_SANDBOX, "cpuintensive.o")
status = Compiler.compile(config.LANGUAGE_CPP, path_source, path_executable)
self.assertEqual(status, "")
# CPU intensive programs have their execution time recorded properly
start_time = perf_counter()
run_result = Runner.run_program(sandbox=Sandbox(), executable_path=path_executable,
memory_limit=32000000, timeout=0.5, input_bytes=None)
self.assertEqual(run_result.exit_code, 0)
self.assertGreaterEqual(run_result.exec_time, 0.29)
self.assertGreaterEqual(perf_counter() - start_time, 0.3)
self.assertLess(run_result.exec_time, 0.4)
self.assertLess(perf_counter() - start_time, 0.5)
self.assertNotEqual(run_result.output.decode().strip(), "")
def main(file: str):
with open(file, 'r') as file_:
text = file_.read()
compiler = Compiler()
scanner = compiler.get_scanner(text)
for token in scanner:
print(
token,
compiler.get_name(token.code if hasattr(token, 'code') else None))
compiler.output_messages()
def test_run_program_args_are_appended(self):
path_source = os.path.join(self.PATH_FIXTURES, "printargs.cpp")
path_executable = os.path.join(config.PATH_SANDBOX, "printargs.o")
status = Compiler.compile(config.LANGUAGE_CPP, path_source, path_executable)
self.assertEqual(status, "")
# Empty string when no arguments are passed
run_result = Runner.run_program(sandbox=Sandbox(), executable_path=path_executable,
memory_limit=32000000, timeout=0.5)
self.assertEqual(run_result.exit_code, 0)
self.assertEqual(run_result.output.decode().strip(), "")
# If there are arguments, prints their concatenation
run_result = Runner.run_program(sandbox=Sandbox(), executable_path=path_executable,
memory_limit=32000000, timeout=0.5, args=["foo", "bar", "baz"])
self.assertEqual(run_result.exit_code, 0)
self.assertEqual(run_result.output.decode().strip(), "foobarbaz")
def release(input_file, output_dir):
"Compile a cablelang script in release mode"
try:
mkdir(output_dir)
except:
pass
copy_std_to_dir(output_dir)
c = Compiler(input_file, output_dir + '/main.cpp')
c.compile()
c.write()
build_release(output_dir)
def test_open(self):
micheline = Compiler(source).compile_contract()
vm = VM()
vm.load_contract(micheline)
init_storage = vm.contract.storage.dummy()
init_storage['admin'] = vm.context.sender
new_storage = vm.contract.open({"_open": "foo", "manifest_url": "bar", "manifest_hash": "baz"}).interpret(storage=init_storage, sender=vm.context.sender).storage
self.assertEqual(new_storage['_open'], "foo")
self.assertEqual(new_storage['manifest_url'], "bar")
self.assertEqual(new_storage['manifest_hash'], "baz")
try:
new_storage = vm.contract.open({"_open": "foo", "manifest_url": "bar", "manifest_hash": "baz"}).interpret(storage=init_storage)
assert 0
except MichelsonRuntimeError as e:
self.assertEqual(e.format_stdout(), "FAILWITH: 'Only admin can call this entrypoint'")
