def generate_minimal_runtime_html(target, options, js_target, target_basename):
logger.debug('generating HTML for minimal runtime')
shell = utils.read_file(options.shell_path)
if settings.SINGLE_FILE:
# No extra files needed to download in a SINGLE_FILE build.
shell = shell.replace('{{{ DOWNLOAD_JS_AND_WASM_FILES }}}', '')
else:
shell = shell.replace('{{{ DOWNLOAD_JS_AND_WASM_FILES }}}', generate_minimal_runtime_load_statement(target_basename))
temp_files = shared.configuration.get_temp_files()
with temp_files.get_file(suffix='.js') as shell_temp:
utils.write_file(shell_temp, shell)
shell = shared.read_and_preprocess(shell_temp)
if re.search(r'{{{\s*SCRIPT\s*}}}', shell):
shared.exit_with_error('--shell-file "' + options.shell_path + '": MINIMAL_RUNTIME uses a different kind of HTML page shell file than the traditional runtime! Please see $EMSCRIPTEN/src/shell_minimal_runtime.html for a template to use as a basis.')
shell = shell.replace('{{{ TARGET_BASENAME }}}', target_basename)
shell = shell.replace('{{{ EXPORT_NAME }}}', settings.EXPORT_NAME)
shell = shell.replace('{{{ PTHREAD_WORKER_FILE }}}', settings.PTHREAD_WORKER_FILE)
# In SINGLE_FILE build, embed the main .js file into the .html output
if settings.SINGLE_FILE:
js_contents = utils.read_file(js_target)
shared.try_delete(js_target)
else:
js_contents = ''
shell = shell.replace('{{{ JS_CONTENTS_IN_SINGLE_FILE_BUILD }}}', js_contents)
shell = line_endings.convert_line_endings(shell, '\n', options.output_eol)
# Force UTF-8 output for consistency across platforms and with the web.
with open(target, 'wb') as f:
f.write(shell.encode('utf-8'))
def main():
js = utils.read_file(js_file)
ctors_start, ctors_end = find_ctors(js)
if ctors_start < 0:
logger.debug('ctor_evaller: no ctors')
sys.exit(0)
ctors_text = js[ctors_start:ctors_end]
if ctors_text.count('(') == 1:
logger.debug('ctor_evaller: push, but no ctors')
sys.exit(0)
num_ctors = ctors_text.count('function()')
logger.debug('ctor_evaller: %d ctors, from |%s|' % (num_ctors, ctors_text))
wasm_file = binary_file
logger.debug('ctor_evaller (wasm): trying to eval %d global constructors' %
num_ctors)
num_successful, new_js = eval_ctors(js, wasm_file, num_ctors)
if num_successful == 0:
logger.debug('ctor_evaller: not successful')
sys.exit(0)
logger.debug('ctor_evaller: we managed to remove %d ctors' %
num_successful)
utils.write_file(js_file, new_js)
def getExtendedAttribute(self, name): # noqa: U100
return None
input_file = sys.argv[1]
output_base = sys.argv[2]
shared.try_delete(output_base + '.cpp')
shared.try_delete(output_base + '.js')
p = WebIDL.Parser()
p.parse(r'''
interface VoidPtr {
};
''' + utils.read_file(input_file))
data = p.finish()
interfaces = {}
implements = {}
enums = {}
for thing in data:
if isinstance(thing, WebIDL.IDLInterface):
interfaces[thing.identifier.name] = thing
elif isinstance(thing, WebIDL.IDLImplementsStatement):
implements.setdefault(thing.implementor.identifier.name,
[]).append(thing.implementee.identifier.name)
elif isinstance(thing, WebIDL.IDLEnum):
enums[thing.identifier.name] = thing
def run_on_js(filename,
passes,
extra_info=None,
just_split=False,
just_concat=False):
with ToolchainProfiler.profile_block('js_optimizer.split_markers'):
if not isinstance(passes, list):
passes = [passes]
js = utils.read_file(filename)
if os.linesep != '\n':
js = js.replace(os.linesep,
'\n') # we assume \n in the splitting code
# Find suffix
suffix_marker = '// EMSCRIPTEN_GENERATED_FUNCTIONS'
suffix_start = js.find(suffix_marker)
suffix = ''
if suffix_start >= 0:
suffix_end = js.find('\n', suffix_start)
suffix = js[suffix_start:suffix_end] + '\n'
# if there is metadata, we will run only on the generated functions. If there isn't, we will run on everything.
# Find markers
start_funcs = js.find(start_funcs_marker)
end_funcs = js.rfind(end_funcs_marker)
if start_funcs < 0 or end_funcs < start_funcs or not suffix:
shared.exit_with_error(
'Invalid input file. Did not contain appropriate markers. (start_funcs: %s, end_funcs: %s, suffix_start: %s'
% (start_funcs, end_funcs, suffix_start))
minify_globals = 'minifyNames' in passes
if minify_globals:
passes = [
p if p != 'minifyNames' else 'minifyLocals' for p in passes
]
start_asm = js.find(start_asm_marker)
end_asm = js.rfind(end_asm_marker)
assert (start_asm >= 0) == (end_asm >= 0)
closure = 'closure' in passes
if closure:
passes = [p for p in passes
if p != 'closure'] # we will do it manually
cleanup = 'cleanup' in passes
if cleanup:
passes = [p for p in passes
if p != 'cleanup'] # we will do it manually
if not minify_globals:
with ToolchainProfiler.profile_block('js_optimizer.no_minify_globals'):
pre = js[:start_funcs + len(start_funcs_marker)]
post = js[end_funcs + len(end_funcs_marker):]
js = js[start_funcs + len(start_funcs_marker):end_funcs]
if 'asm' not in passes:
# can have Module[..] and inlining prevention code, push those to post
class Finals:
buf = []
def process(line):
if len(line) and (line.startswith(
('Module[', 'if (globalScope)'))
or line.endswith('["X"]=1;')):
Finals.buf.append(line)
return False
return True
js = '\n'.join(filter(process, js.split('\n')))
post = '\n'.join(Finals.buf) + '\n' + post
post = end_funcs_marker + post
else:
with ToolchainProfiler.profile_block('js_optimizer.minify_globals'):
# We need to split out the asm shell as well, for minification
pre = js[:start_asm + len(start_asm_marker)]
post = js[end_asm:]
asm_shell = js[start_asm + len(start_asm_marker):start_funcs +
len(start_funcs_marker)] + '''
EMSCRIPTEN_FUNCS();
''' + js[end_funcs + len(end_funcs_marker):end_asm + len(end_asm_marker)]
js = js[start_funcs + len(start_funcs_marker):end_funcs]
# we assume there is a maximum of one new name per line
minifier = Minifier(js)
def check_symbol_mapping(p):
if p.startswith('symbolMap='):
minifier.symbols_file = p.split('=', 1)[1]
return False
if p == 'profilingFuncs':
minifier.profiling_funcs = True
return False
return True
passes = list(filter(check_symbol_mapping, passes))
asm_shell_pre, asm_shell_post = minifier.minify_shell(
asm_shell, 'minifyWhitespace'
in passes).split('EMSCRIPTEN_FUNCS();')
asm_shell_post = asm_shell_post.replace('});', '})')
pre += asm_shell_pre + '\n' + start_funcs_marker
post = end_funcs_marker + asm_shell_post + post
minify_info = minifier.serialize()
if extra_info:
for key, value in extra_info.items():
assert key not in minify_info or value == minify_info[
key], [key, value, minify_info[key]]
minify_info[key] = value
# if DEBUG:
# print >> sys.stderr, 'minify info:', minify_info
with ToolchainProfiler.profile_block(
'js_optimizer.remove_suffix_and_split'):
# remove suffix if no longer needed
if suffix and 'last' in passes:
suffix_start = post.find(suffix_marker)
suffix_end = post.find('\n', suffix_start)
post = post[:suffix_start] + post[suffix_end:]
total_size = len(js)
funcs = split_funcs(js, just_split)
js = None
with ToolchainProfiler.profile_block('js_optimizer.split_to_chunks'):
# if we are making source maps, we want our debug numbering to start from the
# top of the file, so avoid breaking the JS into chunks
cores = shared.get_num_cores()
if not just_split:
intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE))
chunk_size = min(
MAX_CHUNK_SIZE,
max(MIN_CHUNK_SIZE, total_size / intended_num_chunks))
chunks = chunkify(funcs, chunk_size)
else:
# keep same chunks as before
chunks = [f[1] for f in funcs]
chunks = [chunk for chunk in chunks if len(chunk)]
if DEBUG and len(chunks):
print('chunkification: num funcs:',
len(funcs),
'actual num chunks:',
len(chunks),
'chunk size range:',
max(map(len, chunks)),
'-',
min(map(len, chunks)),
file=sys.stderr)
funcs = None
if len(chunks):
serialized_extra_info = suffix_marker + '\n'
if minify_globals:
serialized_extra_info += '// EXTRA_INFO:' + json.dumps(
minify_info)
elif extra_info:
serialized_extra_info += '// EXTRA_INFO:' + json.dumps(
extra_info)
with ToolchainProfiler.profile_block('js_optimizer.write_chunks'):
def write_chunk(chunk, i):
temp_file = temp_files.get('.jsfunc_%d.js' % i).name
with open(temp_file, 'w') as f:
f.write(chunk)
f.write(serialized_extra_info)
return temp_file
filenames = [
write_chunk(chunks[i], i) for i in range(len(chunks))
]
else:
filenames = []
with ToolchainProfiler.profile_block('run_optimizer'):
if len(filenames):
commands = [
config.NODE_JS + [ACORN_OPTIMIZER, f] + passes
for f in filenames
]
if os.environ.get('EMCC_SAVE_OPT_TEMP') and os.environ.get(
'EMCC_SAVE_OPT_TEMP') != '0':
for filename in filenames:
saved = 'save_' + os.path.basename(filename)
while os.path.exists(saved):
saved = 'input' + str(
int(
saved.replace('input', '').replace('.txt', ''))
+ 1) + '.txt'
shutil.copyfile(
filename,
os.path.join(shared.get_emscripten_temp_dir(), saved))
filenames = shared.run_multiple_processes(
commands, route_stdout_to_temp_files_suffix='js_opt.jo.js')
for filename in filenames:
temp_files.note(filename)
with ToolchainProfiler.profile_block('split_closure_cleanup'):
if closure or cleanup:
# run on the shell code, everything but what we acorn-optimize
start_asm = '// EMSCRIPTEN_START_ASM\n'
end_asm = '// EMSCRIPTEN_END_ASM\n'
cl_sep = 'wakaUnknownBefore(); var asm=wakaUnknownAfter(wakaGlobal,wakaEnv,wakaBuffer)\n'
with temp_files.get_file('.cl.js') as cle:
pre_1, pre_2 = pre.split(start_asm)
post_1, post_2 = post.split(end_asm)
with open(cle, 'w') as f:
f.write(pre_1)
f.write(cl_sep)
f.write(post_2)
cld = cle
if closure:
if DEBUG:
print('running closure on shell code', file=sys.stderr)
cld = building.closure_compiler(cld,
pretty='minifyWhitespace'
not in passes)
temp_files.note(cld)
elif cleanup:
if DEBUG:
print('running cleanup on shell code', file=sys.stderr)
acorn_passes = ['JSDCE']
if 'minifyWhitespace' in passes:
acorn_passes.append('minifyWhitespace')
cld = building.acorn_optimizer(cld, acorn_passes)
temp_files.note(cld)
coutput = utils.read_file(cld)
coutput = coutput.replace('wakaUnknownBefore();', start_asm)
after = 'wakaUnknownAfter'
start = coutput.find(after)
end = coutput.find(')', start)
# If the closure comment to suppress useless code is present, we need to look one
# brace past it, as the first is in there. Otherwise, the first brace is the
# start of the function body (what we want).
USELESS_CODE_COMMENT = '/** @suppress {uselessCode} */ '
USELESS_CODE_COMMENT_BODY = 'uselessCode'
brace = pre_2.find('{') + 1
has_useless_code_comment = False
if pre_2[brace:brace + len(USELESS_CODE_COMMENT_BODY
)] == USELESS_CODE_COMMENT_BODY:
brace = pre_2.find('{', brace) + 1
has_useless_code_comment = True
pre = coutput[:start] + '(' + (
USELESS_CODE_COMMENT if has_useless_code_comment else
'') + 'function(global,env,buffer) {\n' + pre_2[brace:]
post = post_1 + end_asm + coutput[end + 1:]
with ToolchainProfiler.profile_block('write_pre'):
filename += '.jo.js'
temp_files.note(filename)
f = open(filename, 'w')
f.write(pre)
pre = None
with ToolchainProfiler.profile_block('sort_or_concat'):
if not just_concat:
# sort functions by size, to make diffing easier and to improve aot times
funcses = []
for out_file in filenames:
funcses.append(split_funcs(utils.read_file(out_file), False))
funcs = [item for sublist in funcses for item in sublist]
funcses = None
if not os.environ.get('EMCC_NO_OPT_SORT'):
funcs.sort(key=lambda x: (len(x[1]), x[0]), reverse=True)
if 'last' in passes and len(funcs):
count = funcs[0][1].count('\n')
if count > 3000:
print(
'warning: Output contains some very large functions (%s lines in %s), consider building source files with -Os or -Oz)'
% (count, funcs[0][0]),
file=sys.stderr)
for func in funcs:
f.write(func[1])
funcs = None
else:
# just concat the outputs
for out_file in filenames:
f.write(utils.read_file(out_file))
with ToolchainProfiler.profile_block('write_post'):
f.write('\n')
f.write(post)
# No need to write suffix: if there was one, it is inside post which exists when suffix is there
f.write('\n')
f.close()
return filename
# This file further processes an output JS file to optimize down to even smaller size that Closure is unable to do
import os
import re
import sys
__scriptdir__ = os.path.dirname(os.path.abspath(__file__))
__rootdir__ = os.path.dirname(__scriptdir__)
sys.path.append(__rootdir__)
from tools import building
from tools.utils import read_file, write_file
f = read_file(sys.argv[1])
orig_size = len(f)
f = f.strip()
# The following are introduced to the output after Closure has run, so need to manually optimize them out.
f = re.sub(r'\s*//\s*EMSCRIPTEN_START_ASM\s*', '', f)
f = re.sub(r'\s*//\s*EMSCRIPTEN_START_FUNCS\s*', '', f)
f = re.sub(r'\s*//\s*EMSCRIPTEN_END_ASM\s*', '', f)
f = re.sub(r'\s*//\s*EMSCRIPTEN_END_FUNCS\s*', '', f)
f = re.sub(r'\s*/\*\* @suppress {uselessCode} \*/\s*', '', f)
f = f.replace('Module["asm"] = (function(global,env,buffer) {',
'Module["asm"]=(function(global,env,buffer){')
# https://github.com/google/closure-compiler/issues/3185
f = re.sub(r';new Int8Array\(\w+\);', ';', f)
f = re.sub(r';new Int16Array\(\w+\);', ';', f)
f = re.sub(r';new Int32Array\(\w+\);', ';', f)
def build_write_file_head():
product_list = []
product_list.append('Customer_ID')
for i in range(12):
num = str(i + 1)
product_list.append('Product_' + num.zfill(2))
return utils.list_to_str(product_list)
def build_write_list(cust_prod_dic):
write_list = []
write_list.append(build_write_file_head())
customer_list = list(cust_prod_dic.keys())
customer_list.sort()
for customer in iter(customer_list):
line = []
line.append(customer)
line.extend(cust_prod_dic[customer])
write_list.append(utils.list_to_str(line))
return write_list
if __name__ == '__main__':
print('start')
data = utils.read_file(
os.path.join(constant.data_files_path, constant.products_dat_name))
cust_prod = build_cust_prod(data)
cust_prod_write_list = build_write_list(cust_prod)
utils.write_file('cust_prod_test.dat', cust_prod_write_list)
print("end")
def up_to_date():
if os.path.exists(marker):
if utils.read_file(marker).strip() == url:
return True
return False
import os
import re
import json
import tools.utils as utils
from datetime import datetime
from tools.config import Config as cfg
from tools.config import Const as const
for path in cfg.PATHS:
account_name = utils.name_from_path(path)
print(
f"Stats for {account_name} ========================================\n")
pricelist = utils.read_file(os.path.join(path, 'pricelist.json'))
polldata = utils.read_file(os.path.join(
path, 'polldata.json')) # accepted, non-admin trades only
if not polldata:
print(f"{account_name} haven't made any trades yet.")
continue
item_data = {}
first_trade_time = datetime.now()
for trade in polldata:
trade_time = datetime.fromtimestamp(trade['finishTimestamp'] // 1000)
if trade_time < first_trade_time:
first_trade_time = trade_time
# get pures on both sides (in scrap)
from tools import constant as constant, utils as utils
import os
Count = [[0] * 12 for i in range(12)]
def build_count_list(data):
del data[0]
for line in data:
line_list = line.split(',')
for i in range(1, 12):
for j in range(i + 1, 13):
if line_list[i] == line_list[j] and line_list[i] == "T":
Count[i - 1][j - 1] += 1
return 0
if __name__ == '__main__':
print('start')
data = utils.read_file(
os.path.join(constant.data_files_path, constant.cust_prod_dat_name))
build_count_list(data)
for i in range(12):
print(Count[i])
print('\n')
print("end")
def deal_with_file(full_path):
text = read_file(full_path)
datapoint = Example.fromlist((text, text),
fields=[("src", DOC), ("trg", DOC)])
return datapoint
def build_write_list(cust_prod_dic):
write_list = []
write_list.append(head)
write_list.append(build_write_file_head())
write_list.append('ValueBand')
customer_list = list(cust_prod_dic.keys())
customer_list.sort()
for customer in iter(customer_list):
line = []
line.append(customer)
line.extend(cust_prod_dic[customer])
write_list.append(utils.list_to_str(line))
return write_list
if __name__ == "__main__":
print('start')
# data1 = utils.read_file('D:\program\IBM\data_files\cust_call_plus.dat')
data1 = utils.read_file(
os.path.join(constant.data_files_path, constant.cust_name1))
head = []
head = data1[0]
data2 = utils.read_file(
os.path.join(constant.data_files_path, constant.cust_name2))
cust_call_prod = merge(data1, data2)
cust_prod_write_list = build_write_list(cust_call_prod)
utils.write_file('cust_call_prod_test.dat', cust_prod_write_list)
print('end')
