def generate_minimal_runtime_html(target, options, js_target, target_basename): logger.debug('generating HTML for minimal runtime') shell = utils.read_file(options.shell_path) if settings.SINGLE_FILE: # No extra files needed to download in a SINGLE_FILE build. shell = shell.replace('{{{ DOWNLOAD_JS_AND_WASM_FILES }}}', '') else: shell = shell.replace('{{{ DOWNLOAD_JS_AND_WASM_FILES }}}', generate_minimal_runtime_load_statement(target_basename)) temp_files = shared.configuration.get_temp_files() with temp_files.get_file(suffix='.js') as shell_temp: utils.write_file(shell_temp, shell) shell = shared.read_and_preprocess(shell_temp) if re.search(r'{{{\s*SCRIPT\s*}}}', shell): shared.exit_with_error('--shell-file "' + options.shell_path + '": MINIMAL_RUNTIME uses a different kind of HTML page shell file than the traditional runtime! Please see $EMSCRIPTEN/src/shell_minimal_runtime.html for a template to use as a basis.') shell = shell.replace('{{{ TARGET_BASENAME }}}', target_basename) shell = shell.replace('{{{ EXPORT_NAME }}}', settings.EXPORT_NAME) shell = shell.replace('{{{ PTHREAD_WORKER_FILE }}}', settings.PTHREAD_WORKER_FILE) # In SINGLE_FILE build, embed the main .js file into the .html output if settings.SINGLE_FILE: js_contents = utils.read_file(js_target) shared.try_delete(js_target) else: js_contents = '' shell = shell.replace('{{{ JS_CONTENTS_IN_SINGLE_FILE_BUILD }}}', js_contents) shell = line_endings.convert_line_endings(shell, '\n', options.output_eol) # Force UTF-8 output for consistency across platforms and with the web. with open(target, 'wb') as f: f.write(shell.encode('utf-8'))
def main(): js = utils.read_file(js_file) ctors_start, ctors_end = find_ctors(js) if ctors_start < 0: logger.debug('ctor_evaller: no ctors') sys.exit(0) ctors_text = js[ctors_start:ctors_end] if ctors_text.count('(') == 1: logger.debug('ctor_evaller: push, but no ctors') sys.exit(0) num_ctors = ctors_text.count('function()') logger.debug('ctor_evaller: %d ctors, from |%s|' % (num_ctors, ctors_text)) wasm_file = binary_file logger.debug('ctor_evaller (wasm): trying to eval %d global constructors' % num_ctors) num_successful, new_js = eval_ctors(js, wasm_file, num_ctors) if num_successful == 0: logger.debug('ctor_evaller: not successful') sys.exit(0) logger.debug('ctor_evaller: we managed to remove %d ctors' % num_successful) utils.write_file(js_file, new_js)
def getExtendedAttribute(self, name): # noqa: U100 return None input_file = sys.argv[1] output_base = sys.argv[2] shared.try_delete(output_base + '.cpp') shared.try_delete(output_base + '.js') p = WebIDL.Parser() p.parse(r''' interface VoidPtr { }; ''' + utils.read_file(input_file)) data = p.finish() interfaces = {} implements = {} enums = {} for thing in data: if isinstance(thing, WebIDL.IDLInterface): interfaces[thing.identifier.name] = thing elif isinstance(thing, WebIDL.IDLImplementsStatement): implements.setdefault(thing.implementor.identifier.name, []).append(thing.implementee.identifier.name) elif isinstance(thing, WebIDL.IDLEnum): enums[thing.identifier.name] = thing
def run_on_js(filename, passes, extra_info=None, just_split=False, just_concat=False): with ToolchainProfiler.profile_block('js_optimizer.split_markers'): if not isinstance(passes, list): passes = [passes] js = utils.read_file(filename) if os.linesep != '\n': js = js.replace(os.linesep, '\n') # we assume \n in the splitting code # Find suffix suffix_marker = '// EMSCRIPTEN_GENERATED_FUNCTIONS' suffix_start = js.find(suffix_marker) suffix = '' if suffix_start >= 0: suffix_end = js.find('\n', suffix_start) suffix = js[suffix_start:suffix_end] + '\n' # if there is metadata, we will run only on the generated functions. If there isn't, we will run on everything. # Find markers start_funcs = js.find(start_funcs_marker) end_funcs = js.rfind(end_funcs_marker) if start_funcs < 0 or end_funcs < start_funcs or not suffix: shared.exit_with_error( 'Invalid input file. Did not contain appropriate markers. (start_funcs: %s, end_funcs: %s, suffix_start: %s' % (start_funcs, end_funcs, suffix_start)) minify_globals = 'minifyNames' in passes if minify_globals: passes = [ p if p != 'minifyNames' else 'minifyLocals' for p in passes ] start_asm = js.find(start_asm_marker) end_asm = js.rfind(end_asm_marker) assert (start_asm >= 0) == (end_asm >= 0) closure = 'closure' in passes if closure: passes = [p for p in passes if p != 'closure'] # we will do it manually cleanup = 'cleanup' in passes if cleanup: passes = [p for p in passes if p != 'cleanup'] # we will do it manually if not minify_globals: with ToolchainProfiler.profile_block('js_optimizer.no_minify_globals'): pre = js[:start_funcs + len(start_funcs_marker)] post = js[end_funcs + len(end_funcs_marker):] js = js[start_funcs + len(start_funcs_marker):end_funcs] if 'asm' not in passes: # can have Module[..] and inlining prevention code, push those to post class Finals: buf = [] def process(line): if len(line) and (line.startswith( ('Module[', 'if (globalScope)')) or line.endswith('["X"]=1;')): Finals.buf.append(line) return False return True js = '\n'.join(filter(process, js.split('\n'))) post = '\n'.join(Finals.buf) + '\n' + post post = end_funcs_marker + post else: with ToolchainProfiler.profile_block('js_optimizer.minify_globals'): # We need to split out the asm shell as well, for minification pre = js[:start_asm + len(start_asm_marker)] post = js[end_asm:] asm_shell = js[start_asm + len(start_asm_marker):start_funcs + len(start_funcs_marker)] + ''' EMSCRIPTEN_FUNCS(); ''' + js[end_funcs + len(end_funcs_marker):end_asm + len(end_asm_marker)] js = js[start_funcs + len(start_funcs_marker):end_funcs] # we assume there is a maximum of one new name per line minifier = Minifier(js) def check_symbol_mapping(p): if p.startswith('symbolMap='): minifier.symbols_file = p.split('=', 1)[1] return False if p == 'profilingFuncs': minifier.profiling_funcs = True return False return True passes = list(filter(check_symbol_mapping, passes)) asm_shell_pre, asm_shell_post = minifier.minify_shell( asm_shell, 'minifyWhitespace' in passes).split('EMSCRIPTEN_FUNCS();') asm_shell_post = asm_shell_post.replace('});', '})') pre += asm_shell_pre + '\n' + start_funcs_marker post = end_funcs_marker + asm_shell_post + post minify_info = minifier.serialize() if extra_info: for key, value in extra_info.items(): assert key not in minify_info or value == minify_info[ key], [key, value, minify_info[key]] minify_info[key] = value # if DEBUG: # print >> sys.stderr, 'minify info:', minify_info with ToolchainProfiler.profile_block( 'js_optimizer.remove_suffix_and_split'): # remove suffix if no longer needed if suffix and 'last' in passes: suffix_start = post.find(suffix_marker) suffix_end = post.find('\n', suffix_start) post = post[:suffix_start] + post[suffix_end:] total_size = len(js) funcs = split_funcs(js, just_split) js = None with ToolchainProfiler.profile_block('js_optimizer.split_to_chunks'): # if we are making source maps, we want our debug numbering to start from the # top of the file, so avoid breaking the JS into chunks cores = shared.get_num_cores() if not just_split: intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE)) chunk_size = min( MAX_CHUNK_SIZE, max(MIN_CHUNK_SIZE, total_size / intended_num_chunks)) chunks = chunkify(funcs, chunk_size) else: # keep same chunks as before chunks = [f[1] for f in funcs] chunks = [chunk for chunk in chunks if len(chunk)] if DEBUG and len(chunks): print('chunkification: num funcs:', len(funcs), 'actual num chunks:', len(chunks), 'chunk size range:', max(map(len, chunks)), '-', min(map(len, chunks)), file=sys.stderr) funcs = None if len(chunks): serialized_extra_info = suffix_marker + '\n' if minify_globals: serialized_extra_info += '// EXTRA_INFO:' + json.dumps( minify_info) elif extra_info: serialized_extra_info += '// EXTRA_INFO:' + json.dumps( extra_info) with ToolchainProfiler.profile_block('js_optimizer.write_chunks'): def write_chunk(chunk, i): temp_file = temp_files.get('.jsfunc_%d.js' % i).name with open(temp_file, 'w') as f: f.write(chunk) f.write(serialized_extra_info) return temp_file filenames = [ write_chunk(chunks[i], i) for i in range(len(chunks)) ] else: filenames = [] with ToolchainProfiler.profile_block('run_optimizer'): if len(filenames): commands = [ config.NODE_JS + [ACORN_OPTIMIZER, f] + passes for f in filenames ] if os.environ.get('EMCC_SAVE_OPT_TEMP') and os.environ.get( 'EMCC_SAVE_OPT_TEMP') != '0': for filename in filenames: saved = 'save_' + os.path.basename(filename) while os.path.exists(saved): saved = 'input' + str( int( saved.replace('input', '').replace('.txt', '')) + 1) + '.txt' shutil.copyfile( filename, os.path.join(shared.get_emscripten_temp_dir(), saved)) filenames = shared.run_multiple_processes( commands, route_stdout_to_temp_files_suffix='js_opt.jo.js') for filename in filenames: temp_files.note(filename) with ToolchainProfiler.profile_block('split_closure_cleanup'): if closure or cleanup: # run on the shell code, everything but what we acorn-optimize start_asm = '// EMSCRIPTEN_START_ASM\n' end_asm = '// EMSCRIPTEN_END_ASM\n' cl_sep = 'wakaUnknownBefore(); var asm=wakaUnknownAfter(wakaGlobal,wakaEnv,wakaBuffer)\n' with temp_files.get_file('.cl.js') as cle: pre_1, pre_2 = pre.split(start_asm) post_1, post_2 = post.split(end_asm) with open(cle, 'w') as f: f.write(pre_1) f.write(cl_sep) f.write(post_2) cld = cle if closure: if DEBUG: print('running closure on shell code', file=sys.stderr) cld = building.closure_compiler(cld, pretty='minifyWhitespace' not in passes) temp_files.note(cld) elif cleanup: if DEBUG: print('running cleanup on shell code', file=sys.stderr) acorn_passes = ['JSDCE'] if 'minifyWhitespace' in passes: acorn_passes.append('minifyWhitespace') cld = building.acorn_optimizer(cld, acorn_passes) temp_files.note(cld) coutput = utils.read_file(cld) coutput = coutput.replace('wakaUnknownBefore();', start_asm) after = 'wakaUnknownAfter' start = coutput.find(after) end = coutput.find(')', start) # If the closure comment to suppress useless code is present, we need to look one # brace past it, as the first is in there. Otherwise, the first brace is the # start of the function body (what we want). USELESS_CODE_COMMENT = '/** @suppress {uselessCode} */ ' USELESS_CODE_COMMENT_BODY = 'uselessCode' brace = pre_2.find('{') + 1 has_useless_code_comment = False if pre_2[brace:brace + len(USELESS_CODE_COMMENT_BODY )] == USELESS_CODE_COMMENT_BODY: brace = pre_2.find('{', brace) + 1 has_useless_code_comment = True pre = coutput[:start] + '(' + ( USELESS_CODE_COMMENT if has_useless_code_comment else '') + 'function(global,env,buffer) {\n' + pre_2[brace:] post = post_1 + end_asm + coutput[end + 1:] with ToolchainProfiler.profile_block('write_pre'): filename += '.jo.js' temp_files.note(filename) f = open(filename, 'w') f.write(pre) pre = None with ToolchainProfiler.profile_block('sort_or_concat'): if not just_concat: # sort functions by size, to make diffing easier and to improve aot times funcses = [] for out_file in filenames: funcses.append(split_funcs(utils.read_file(out_file), False)) funcs = [item for sublist in funcses for item in sublist] funcses = None if not os.environ.get('EMCC_NO_OPT_SORT'): funcs.sort(key=lambda x: (len(x[1]), x[0]), reverse=True) if 'last' in passes and len(funcs): count = funcs[0][1].count('\n') if count > 3000: print( 'warning: Output contains some very large functions (%s lines in %s), consider building source files with -Os or -Oz)' % (count, funcs[0][0]), file=sys.stderr) for func in funcs: f.write(func[1]) funcs = None else: # just concat the outputs for out_file in filenames: f.write(utils.read_file(out_file)) with ToolchainProfiler.profile_block('write_post'): f.write('\n') f.write(post) # No need to write suffix: if there was one, it is inside post which exists when suffix is there f.write('\n') f.close() return filename
# This file further processes an output JS file to optimize down to even smaller size that Closure is unable to do import os import re import sys __scriptdir__ = os.path.dirname(os.path.abspath(__file__)) __rootdir__ = os.path.dirname(__scriptdir__) sys.path.append(__rootdir__) from tools import building from tools.utils import read_file, write_file f = read_file(sys.argv[1]) orig_size = len(f) f = f.strip() # The following are introduced to the output after Closure has run, so need to manually optimize them out. f = re.sub(r'\s*//\s*EMSCRIPTEN_START_ASM\s*', '', f) f = re.sub(r'\s*//\s*EMSCRIPTEN_START_FUNCS\s*', '', f) f = re.sub(r'\s*//\s*EMSCRIPTEN_END_ASM\s*', '', f) f = re.sub(r'\s*//\s*EMSCRIPTEN_END_FUNCS\s*', '', f) f = re.sub(r'\s*/\*\* @suppress {uselessCode} \*/\s*', '', f) f = f.replace('Module["asm"] = (function(global,env,buffer) {', 'Module["asm"]=(function(global,env,buffer){') # https://github.com/google/closure-compiler/issues/3185 f = re.sub(r';new Int8Array\(\w+\);', ';', f) f = re.sub(r';new Int16Array\(\w+\);', ';', f) f = re.sub(r';new Int32Array\(\w+\);', ';', f)
def build_write_file_head(): product_list = [] product_list.append('Customer_ID') for i in range(12): num = str(i + 1) product_list.append('Product_' + num.zfill(2)) return utils.list_to_str(product_list) def build_write_list(cust_prod_dic): write_list = [] write_list.append(build_write_file_head()) customer_list = list(cust_prod_dic.keys()) customer_list.sort() for customer in iter(customer_list): line = [] line.append(customer) line.extend(cust_prod_dic[customer]) write_list.append(utils.list_to_str(line)) return write_list if __name__ == '__main__': print('start') data = utils.read_file( os.path.join(constant.data_files_path, constant.products_dat_name)) cust_prod = build_cust_prod(data) cust_prod_write_list = build_write_list(cust_prod) utils.write_file('cust_prod_test.dat', cust_prod_write_list) print("end")
def up_to_date(): if os.path.exists(marker): if utils.read_file(marker).strip() == url: return True return False
import os import re import json import tools.utils as utils from datetime import datetime from tools.config import Config as cfg from tools.config import Const as const for path in cfg.PATHS: account_name = utils.name_from_path(path) print( f"Stats for {account_name} ========================================\n") pricelist = utils.read_file(os.path.join(path, 'pricelist.json')) polldata = utils.read_file(os.path.join( path, 'polldata.json')) # accepted, non-admin trades only if not polldata: print(f"{account_name} haven't made any trades yet.") continue item_data = {} first_trade_time = datetime.now() for trade in polldata: trade_time = datetime.fromtimestamp(trade['finishTimestamp'] // 1000) if trade_time < first_trade_time: first_trade_time = trade_time # get pures on both sides (in scrap)
from tools import constant as constant, utils as utils import os Count = [[0] * 12 for i in range(12)] def build_count_list(data): del data[0] for line in data: line_list = line.split(',') for i in range(1, 12): for j in range(i + 1, 13): if line_list[i] == line_list[j] and line_list[i] == "T": Count[i - 1][j - 1] += 1 return 0 if __name__ == '__main__': print('start') data = utils.read_file( os.path.join(constant.data_files_path, constant.cust_prod_dat_name)) build_count_list(data) for i in range(12): print(Count[i]) print('\n') print("end")
def deal_with_file(full_path): text = read_file(full_path) datapoint = Example.fromlist((text, text), fields=[("src", DOC), ("trg", DOC)]) return datapoint
def build_write_list(cust_prod_dic): write_list = [] write_list.append(head) write_list.append(build_write_file_head()) write_list.append('ValueBand') customer_list = list(cust_prod_dic.keys()) customer_list.sort() for customer in iter(customer_list): line = [] line.append(customer) line.extend(cust_prod_dic[customer]) write_list.append(utils.list_to_str(line)) return write_list if __name__ == "__main__": print('start') # data1 = utils.read_file('D:\program\IBM\data_files\cust_call_plus.dat') data1 = utils.read_file( os.path.join(constant.data_files_path, constant.cust_name1)) head = [] head = data1[0] data2 = utils.read_file( os.path.join(constant.data_files_path, constant.cust_name2)) cust_call_prod = merge(data1, data2) cust_prod_write_list = build_write_list(cust_call_prod) utils.write_file('cust_call_prod_test.dat', cust_prod_write_list) print('end')