def __init__(self, name, cap=16, create=False, size=0): #buffer self.buffer = Buffer(name, 602116, create, size) # start loader service self.id_queue = multiprocessing.Manager().Queue(maxsize=cap) self.data_queue = multiprocessing.Manager().Queue(maxsize=cap) self.loader = multiprocessing.Process( target=Loader.loading, args=(self.id_queue, self.data_queue, name, int(cap/2))) self.loader.start() assert(self.loader.is_alive() == True) # table self.data_lock = threading.Lock() self.id_table = {} #id -> datanode self.data_refs = {} #id -> refs self.task_tails = {} self.task_heads = {} self.pending_id_lock = threading.Lock() self.pending_id = {} #id->namelist #replacer self.replacer = Replacer() # start a thread to listen data queue executor = ThreadPoolExecutor(max_workers=1) executor.submit(self.listener)
def main(): # http://stackoverflow.com/questions/15206010/how-to-pass-on-argparse-argument-to-function-as-kwargs args = command_line() assert (args.src_train_file is None) == (args.tgt_train_file is None) assert (args.src_dev_file is None) == (args.tgt_dev_file is None) # above 2 assertion lines are needed to guarantee that src_train_file being None implies tgt_train_file being None and so on... only_test = (args.src_train_file is None) and (args.src_dev_file is None) replacer = Replacer(args, mode=args.mode, src_embedding=args.src_embedding, src_embedding_topn=args.src_embedding_topn, tgt_embedding=args.tgt_embedding, src_voc_size=args.src_voc_size, tgt_voc_size=args.tgt_voc_size, only_test=only_test, dictionary=args.dictionary, dictionary_topn=args.dictionary_topn, src_sim_threshold=args.src_sim_threshold, tgt_sim_threshold=args.tgt_sim_threshold, lex_prob_threshold=args.lex_prob_threshold, backoff_to_unk=args.backoff_to_unk, replace_both=args.replace_both, guarantee_in_vocab_replace=args.guarantee_in_vocab_replace)
from parser import Parser from scrambler import Scrambler from replacer import Replacer import argparse if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "--file", help="The file that contains the function names to scramble") parser.add_argument("--password", help="The password used to encrypt the function names") args = parser.parse_args() parser = Parser(args.file, def_excludes=['main'], call_excludes=['printf']) scrambler = Scrambler(parser, args.password) func_names = scrambler.scramble_func_names() with open(args.file, 'r') as file: source_code = file.read() replacer = Replacer(source_code, func_names) print(replacer.replace())
def main(): file_name = args.file with open(file_name, 'r') as f: content = f.read() replacer = Replacer() has_full_preamble_declaration = replacer.check_for_full_pream( content ) # If contains '..begin full preamble' and '..end full preamble' content = replacer.set_aside_preamble(content) content = replacer.set_aside_ignore_markers(content) # Deal with comments while '%%%' in content: ind1 = content.index('%%%') ind2 = ind1 + 3 + content[ind1 + 3:].index('%%%') content = content[:ind1] + content[ind2 + 3:] lines = content.split('\n') for i in range(len(lines)): if ' %' in lines[i]: lines[i] = lines[i][:lines[i].index(' %')] elif '\t%' in lines[i]: lines[i] = lines[i][:lines[i].index('\t%')] elif len(lines[i]) > 1 and '%' == lines[i][0]: lines[i] = '' lines[i] = lines[i].strip() if not has_full_preamble_declaration: # Ignore all other stuff in the preamble if "..begin full preamble" defined # Set font font = find_content(lines, '..font', '12') # Set packages packages = [ 'amsmath', 'amssymb', 'amsthm', 'geometry', 'enumitem', 'fancyhdr' ] packages.extend(find_content(lines, '..usepackage', '').split()) # See if name is specified. Overrides --name flag if given. my_name = find_content(lines, '..name', None) # Set page size paper = find_content(lines, '..paper', 'letter') # Set page orientation orientation = find_content(lines, '..orient', 'portrait') # Set margin margin = find_content(lines, '..margin', '1') # Set indent indent = find_content(lines, '..indent', '0') # Set spacing spacing = find_content(lines, '..spacing', '1.5') line_spread = 1.3 if spacing == '1': line_spread = 1.0 elif spacing == '2': line_spread = 1.6 # Obvious qed_symbol = find_content(lines, '..qed', None) # Puts whatever follows "..assignment" as the left footer assignment = find_content(lines, '..assignment', None) # Asset path asset_path = find_content(lines, '..assetpath', None) assets_path = find_content(lines, '..assetspath', None) graphics_path = find_content(lines, '..graphicspath', None) if asset_path and assets_path or asset_path and graphics_path or assets_path and graphics_path: exit('Specify only 1 of asset(s)path or graphicspath, not both!') if assets_path: asset_path = assets_path if graphics_path: asset_path = graphics_path # Replacements occur here replacer.replace(lines, find_content, find_index) _start = find_index(lines, '..begin main') _end = find_index(lines, '..end main') if _end != -1: main_content = lines[_start + 1:_end] else: main_content = lines[_start + 1:] _content = "\n".join(main_content) """ Ignore and not transcribe commands surrounded by \verb||. Currently only ignores commands if they immediately follow "\verb|" Better detection still needed, use "..begin ignore" as an alternative. """ used_macros = [] pairs = load_pairs() for pair in pairs: if _content.find(pair[0]) != -1: used_macros.append(pair) used_renewed_commands = [] renewed = load_renew_commands() for pair in renewed: if _content.find(pair[0]) != -1: used_renewed_commands.append(pair) replacement_pairs = load_replacements() for pair in replacement_pairs: _content = _content.replace( '\\verb|{0}'.format(pair[0]), '\\verb|{0}'.format(pair[0][0] + 'afoswj' + pair[0][1:])) _content = _content.replace(pair[0], pair[1]) _content = _content.replace( '\\verb|{0}'.format(pair[0][0] + 'afoswj' + pair[0][1:]), '\\verb|{0}'.format(pair[0])) output = "\n% Created by Roger Hu's .pytex --> .tex latex transcriber\n" time_rn = datetime.now(pytz.timezone('America/Los_Angeles')) output += "% Compiled on {0} PDT\n\n".format(str(time_rn)[:-13]) compile_attempts = 1 if not has_full_preamble_declaration: output += '\n\\documentclass[{0}pt]{{article}}\n'.format(font) output += '\\usepackage{' if 'graphicx' not in packages and asset_path: packages.append('graphicx') if 'pdfpages' not in packages and '\\includepdf' in _content: packages.append('pdfpages') output += ', '.join(packages) output += '}\n' output += '\n' output += '\\geometry{{{0}paper, {1}, margin={2}in}}\n'.format( paper, orientation, margin) output += '\\setlength{{\\parindent}}{{{0}em}}\n'.format(indent) output += '\\linespread{{{0}}}\n'.format(line_spread) output += '\\pagestyle{fancy}\n' output += '\\fancyhf{}\n\n' # Commands compile_attempts = 1 if '\\label{' in _content: compile_attempts = 2 for pair in used_macros: output += '\\newcommand{' + pair[0] + '}{' + pair[1] + '}\n' if '\\begin{Parts}' in _content: output += "\n\\newcounter{resumer}\n\\setcounter{resumer}{0}\n" output += "\\newenvironment{Parts}{\n\\setcounter{resumer}{0}\n\\begin{enumerate}[label=(\\alph*)]\n" output += "\\newcommand\\Part{\\item}}{\\setcounter{resumer}{\\value{enumi}}\\end{enumerate}}\n" if '\\begin{ResumeParts}' in _content: output += "\n\\newenvironment{ResumeParts}{\n" + "\\begin{enumerate}[label=(\\alph*)]\n" output += "\\setcounter{enumi}{\\value{resumer}}\\newcommand\\Part{\\item}}{\\setcounter{resumer}{\\value{enumi}}\\end{enumerate}}\n" if '\\norm' in _content: output += '\\newcommand{\\norm}[1]{\\|#1\\|}\n' if '\\ddef' in _content: output += '\\newcommand{\\ddef}{\\mathrel{\\stackrel{\\makebox[0pt]{\\mbox{\\normalfont\\tiny def}}}{=}}}\n' if '\\sset' in _content: output += '\\newcommand{\\sset}{\\mathrel{\\stackrel{\\makebox[0pt]{\\mbox{\\normalfont\\tiny set}}}{=}}}\n' if "\\floor" in _content: output += '\\newcommand{\\floor}[1]{\\left\\lfloor #1 \\right\\rfloor}\n' if "\\ceil" in _content: output += '\\newcommand{\\ceil}[1]{\\left\\lceil #1 \\right\\rceil}\n' for pair in used_renewed_commands: output += '\\renewcommand' + pair[0] + '{' + pair[1] + '}\n' if qed_symbol: output += '\\renewcommand\\qedsymbol{{{0}}}\n'.format(qed_symbol) output += '\n' output += '\\renewcommand{\\headrulewidth}{0pt}\n' output += '\\renewcommand{\\footrulewidth}{1pt}\n' # Headers and Footers if my_name: output += '\\rhead{{{0}}}\n'.format(my_name) elif args.name: output += '\\rhead{{{0}}}\n'.format(args.name) output += '\\rfoot{\\fontsize{8}{8} \\selectfont \\thepage}\n' if assignment: output += '\\lfoot{\\fontsize{8}{8} \\selectfont ' + assignment + '}\n' # Theorem definition output += '\n' output += replacer.theorem_def output += '\n' if asset_path: output += '\\graphicspath{' paths = asset_path.split(' ') for path in paths: output += '{{{0}}}'.format(path) output += '}\n' # Between "..begin preamble" and "..end preamble" if replacer.preamble: output += '% This part is unaffected by transcription\n\n' output += replacer.preamble output += '\n% End of unaffected portion\n\n' else: # Between "..begin full preamble" and "..end full preamble" output += replacer.full_preamble + '\n' for pair in used_macros: output += '\\newcommand{' + pair[0] + '}{' + pair[1] + '}\n' for pair in used_renewed_commands: output += '\\renewcommand' + pair[0] + '{' + pair[1] + '}\n' output += "\\begin{document}\n\n" # Put back the content between "..begin ignore" and "..end ignore" for i in range(len(replacer.ignored_contents)): _content = _content.replace( '\\\\marker{0}//'.format(i), '% Ignored by transcriber\n' + replacer.ignored_contents[i] + '% End ignored region') # Main content output += _content # Final alignment if not "..align justify" end_align = replacer.end_align if end_align: output += '\n\\end{{{0}}}\n'.format(end_align) output += "\n\\end{document}\n" # Write to .tex file if not args.output: tex_file_name = file_name[:file_name.index('.')] + '.tex' else: tex_file_name = args.output + '.tex' with open(tex_file_name, 'w') as f: f.write(output) _print("Successfully transcribed to {0}!".format(tex_file_name)) time_out = 5 try: time_out = int(args.time) except Exception: pass try: compile_attempts = int(args.attempts) except Exception: pass if args.compile: try: for i in range(compile_attempts): exe = subprocess.run(['pdflatex', '{0}'.format(tex_file_name)], timeout=time_out, capture_output=True) _print("Successfully compiled to {0}.pdf!".format( tex_file_name[:-4])) except Exception: _print("An error has occured. Please manually compile the file.") elif args.force_compile: try: for i in range(compile_attempts): exe = subprocess.run([ 'pdflatex', '-interaction=nonstopmode', '{0}'.format(tex_file_name) ], timeout=time_out, capture_output=True) _print("Forceably compiled to {0}.pdf!".format( tex_file_name[:-4])) except Exception: _print("An error has occured. Please manually compile the file.") return tex_file_name[:-4]