def new_sources(branch=None, fedpkg=FEDPKG, no_new_sources=False): _ensure_branch(branch) if no_new_sources or not _has_valid_sources(): return sources = specfile.Spec().get_source_fns() cmd = fedpkg + ["new-sources"] + sources run(*cmd, direct=True)
def gene_random_data(input_info: dict) -> str: data = {} current_dir = pathlib.Path(__file__).parent.parent data_dir = os.path.join(current_dir, "temp_data") command = "mkdir -p " + data_dir logging.debug(command) cmd.run(command) checker.check_file_exist(data_dir) data_path = os.path.join(data_dir, "input.txt") data_file = open(data_path, "w") data_file.write(str(len(input_info)) + '\n') for name, info in input_info.items(): shape = info['shape'] data_type = info['data_type'] data_file.write(name + ' ' + str(len(shape)) + ' ' + ' '.join([str(dim) for dim in shape]) + ' ' + str(data_type) + '\n') if data_type == 0: data[name] = np.random.rand(*shape) np.savetxt(data_file, data[name].reshape(-1), fmt="%0.6f") elif data_type == 3: data[name] = np.random.randint(low=0, high=1, size=shape) np.savetxt(data_file, data[name].reshape(-1), fmt="%i") data_file.close() return data_path
def findR(): """ Find path to R scripting front-end (Rscript) Try Rscript, $R_HOME, whereis Rscript and HKLM\SOFTWARE\R-core\R*\InstallPath """ R = None if cmd.can_run("Rscript --version"): R = "Rscript" else: if "R_HOME" in os.environ: R = os.path.join(os.environ['R_HOME'], "bin", "Rscript") else: if cmd.on_windows(): reg_cmd = 'cmd /c %SystemRoot%\\system32\\reg.exe query HKLM\SOFTWARE\R-core /s /f InstallPath' reg_info, status = cmd.run(reg_cmd) # HKEY_LOCAL_MACHINE\SOFTWARE\R-core\R64\3.2.5 # InstallPath REG_SZ C:\Program Files\R\R-3.2.5 for line in reg_info.split('\n'): if 'InstallPath' in line: # ['', '', 'InstallPath', '', 'REG_SZ', '', 'C:\\Program Files\\R\\R-3.2.5'] R = os.path.join(line.split( ' ')[-1], "bin", "Rscript") else: if cmd.can_run("whereis ls"): whereis_info, status = cmd.run("whereis Rscript") # Rscript: /usr/bin/Rscript /usr/bin/X11/Rscript wrscript = whereis_info.split() if len(wrscript) >= 2: R = wrscript[1] return R
def evaluate_m2(out_file, m2_file, log_file=None): num_of_lines = cmd.wc(out_file) with open(m2_file) as m2_io: num_of_sents = sum(1 for line in m2_io if line.startswith("S ")) if num_of_lines != num_of_sents: log.error("Different number of sentences between text and M2 file:" " {} != {}".format(num_of_lines, num_of_sents)) if config['m2scorer'] == M2SCORER.MOSES: # C++ implementation of m2scorer from Moses. result = cmd.run("{moses}/bin/m2scorer --candidate {txt} --reference {m2}" \ .format(moses=config['mosesdecoder'], txt=out_file, m2=m2_file)) # Scorer is run by system shell because of the bug inside the scorer # script which cause propagation of forked threads to this script. else: result = cmd.run( "python {scripts}/m2scorer_fork --forks {threads} {txt} {m2}" \ .format(scripts=SCRIPTS_DIR, threads=config['threads'] or 4, txt=out_file, m2=m2_file)) if log_file: with open(log_file, 'w') as log_io: log_io.write(result) return tuple( float(line.rsplit(' ', 1)[-1]) for line in result.strip().split("\n"))
def run_tnn_model_check(proto_path, model_path, input_path, reference_output_path, is_tflite=False, align_batch=False): cmd.run("pwd") relative_path = "bin/model_check" model_check_path = parse_path.parse_path(relative_path) checker.check_file_exist(model_check_path) command = model_check_path + " -e -p " + proto_path + " -m " + \ model_path + " -i " + input_path + " -f " + reference_output_path + " -d NAIVE" if align_batch: command += " -b " logging.debug(command) ret = cmd.run(command) if ret == 0: print_align_message(is_tflite) else: print_not_align_message(None, is_tflite) return
def __save_ngrams(self, freq_file, ngram_file): log.debug( "Write n-gram frequencies into file {}...".format(ngram_file)) line_num = cmd.run("cat {0} | grep -Pn '^ +{1} .*' | tr ':' '\\t'" \ " | cut -f1 | tail -1".format(freq_file, self.min_count)) cmd.run("head -n {} {} | sed -r 's/^ *([0-9]+) (.*)/\\2\\t\\1/' >> {}" \ .format(line_num.strip(), freq_file, ngram_file))
def __calculate_freqs(self, feat_file, freq_file): command = r"cat {feats}" \ " | grep '^shared' | cut -c11-" \ " | tr ' ' '\\n' | sort -S 5G --parallel {threads}" \ " | uniq -c | sort -rn -S 5G --parallel {threads}" \ " | sed -r 's/ +([0-9]+)/\\1\\t/'" \ " > {freqs}" cmd.run( command.format( feats=feat_file, freqs=freq_file, threads=config['threads']))
def train(self, model, data): if os.path.exists(model): log.warn("model already exists: {}".format(model)) if not os.path.exists(data): log.error("data file does not exist: {}".format(data)) options = (config['vw-options'] or "") + VWTrainer.DEFAULT_OPTIONS log.info("train VW: {}".format(model)) cmd.run("{vw}/vowpalwabbit/vw -f {model} -d {data} -c {options}" \ .format(vw=self.vw, model=model, data=data, options=options))
def __count_frequencies(self, list_file): log.info( "Calculate n-gram frequencies in file {}...".format(list_file)) command = "cat {0}" \ " | sort -S 10G --parallel 8" \ " | uniq -c" \ " | sort -S 10G --parallel 8 -nr" \ " > {0}.freq".format(list_file) cmd.run(command) return list_file + '.freq'
def get_dump_dir_path(self) -> str: convert2tnn_path = pathlib.Path(__file__).parent.parent data_dir = os.path.join(convert2tnn_path, "temp_data/") if os.path.exists(data_dir): command = "rm -rf {}".format(data_dir) cmd.run(command) command = "mkdir {}".format(data_dir) cmd.run(command) return data_dir
def new_sources(branch=None, fedpkg=FEDPKG, no_new_sources=False): _ensure_branch(branch) if no_new_sources: log.info("skipping `%s new-sources` as requested." % fedpkg[0]) return if not os.path.isfile('sources'): log.info("'sources' file not found, skipping `%s new-sources`" % fedpkg[0]) return sources = specfile.Spec().get_source_fns() cmd = fedpkg + ['new-sources'] + sources run(*cmd, direct=True)
def run_tnn_model_check(proto_path, model_path, input_path, reference_output_path): cmd.run("pwd") relative_path = "bin/model_check" model_check_path = parse_path.parse_path(relative_path) checker.check_file_exist(model_check_path) command = model_check_path + " -p " + proto_path + " -m " + \ model_path + " -i " + input_path + " -f " + reference_output_path + " -d NAIVE" print(command) cmd.run(command) return
def gene_random_data(input_info: dict) -> str: data = {} current_dir = pathlib.Path(__file__).parent.parent data_dir = os.path.join(current_dir, "temp_data") command = "mkdir -p " + data_dir print(command) cmd.run("pwd") cmd.run(command) checker.check_file_exist(data_dir) data_path = os.path.join(data_dir, "input.txt") data_file = open(data_path, "w") for name, shape in input_info.items(): data[name] = np.random.rand(*shape) np.savetxt(data_file, data[name].reshape(-1), fmt="%0.18f") data_file.close() return data_path
def caffe2onnx(proto_path, model_path, output_path): work_dir = "../caffe2onnx/" command = "python3 caffe2onnx.py " + proto_path + " " + model_path + " -o " + output_path result = cmd.run(command, work_dir=work_dir) if result == 0: return True else: return False
def run_tnn_model_check(proto_path, model_path, input_path, reference_output_path): cmd.run("pwd") relative_path = "bin/model_check" model_check_path = parse_path.parse_path(relative_path) checker.check_file_exist(model_check_path) command = model_check_path + " -p " + proto_path + " -m " + \ model_path + " -i " + input_path + " -f " + reference_output_path + " -d NAIVE" logging.debug(command) ret = cmd.run(command) if ret == 0: print_align_message() else: print_not_align_message() return
def run_model_check(self) -> bool: model_check_path = os.path.join(self.dump_dir_path[:-10], "bin/model_check") tnn_model_path = self.tnn_proto_path[:-9] + ".tnnmodel" input_path = os.path.join(self.dump_dir_path, "input.txt") command = "{} -p {} -m {} -i {} -a {} -d NAIVE".format( model_check_path, self.tnn_proto_path, tnn_model_path, input_path, self.dump_dir_path) logging.debug(command) return cmd.run(command, log_level="error")
def test_example_bad(self): """ test example with R error""" sample_file = os.path.join("test", "bad.samples.txt") analysis_file = os.path.join("test", "bad.analysis.txt") shutil.copyfile(sample_file, "bad.samples.txt") out, status = cmd.run("python {} -g {}".format(script_path, analysis_file)) os.unlink("bad.samples.txt") self.assertEqual(status, -1)
def tflite2tnn(tf_path, tnn_path, not_fold_const=False): cmd.run("pwd") relative_path = "bin/TnnConverter" TnnConverter_path = parse_path.parse_path(relative_path) checker.check_file_exist(TnnConverter_path) command = TnnConverter_path + " -mt TFLITE -mp " + tf_path checker.check_file_exist(TnnConverter_path) checker.check_file_exist(tf_path) if tnn_path is None: tnn_path = os.path.dirname(tf_path) checker.check_file_exist(tnn_path) command = command + " -od " + tnn_path + "/" logging.debug(command) result = cmd.run(command) if result == 0: return True else: return False
def convert_tok(file_in, file_out, mode='moses-nltk'): # TODO: clean this! if not mode: converter = '' elif mode == 'moses-nltk': converter = \ ' | {moses}/scripts/tokenizer/detokenizer.perl -l en' \ ' | python {scripts}/nltk-tok.py' \ .format(moses=config['mosesdecoder'], scripts=SCRIPTS_DIR) elif mode == 'nltk-moses': converter = \ ' | python {scripts}/nltk-detok.py' \ ' | {moses}/scripts/tokenizer/tokenizer.perl -l en -threads {threads}' \ .format(moses=config['mosesdecoder'], scripts=SCRIPTS_DIR, threads=config['threads']) else: log.error("Convertion mode '{}' not supported".format(mode)) exit(1) cmd.run("cat {} {} > {}".format(file_in, converter, file_out))
def run( self, model, data, predictions, options=" -q st -b 26 --noconstant --loss_function logistic --hash all" ): if not os.path.exists(model): log.error("model does not exist: {}".format(model)) if not os.path.exists(data): log.error("data file does not exist: {}".format(data)) log.info("running VW: {}".format(model)) cmd.run( "{vw}/vowpalwabbit/vw -t -i {model} -d {data} -c {options} -r {pred}" .format(vw=self.vw, model=model, data=data, options=options, pred=predictions))
def edit(path): editor = os.environ.get('EDITOR') if not editor: editor = 'vim' log.info("$EDITOR not set. Falling back to %s." % editor) try: r = run(editor, path, direct=True) except exception.CommandNotFound as ex: raise exception.CommandNotFound( msg='Failed to find suitable text editor. Please set $EDITOR ' 'environment variable.') return r.success
def tag_file(self, tok_file, pos_file=None, lazy=True): if pos_file is None: pos_file = tok_file + '.pos' ann_file = pos_file + '.ann' if lazy and os.path.exists(pos_file): log.info("Tagging skipped because file {} exists".format(pos_file)) return pos_file log.info("Tagging file {}".format(tok_file)) command = "java -mx1025m -cp {0}/stanford-postagger.jar: " \ "edu.stanford.nlp.tagger.maxent.MaxentTagger " \ "-model {0}/models/english-left3words-distsim.tagger " \ "-sentenceDelimiter newline -tokenize false -tagSeparator \"{3}\" " \ "-textFile {1} -nthreads {4} 2> /dev/null > {2}" \ .format(self.tagger_dir, tok_file, ann_file, self.separator, self.threads) cmd.run(command) self.__extract_pos_tags(ann_file, pos_file) os.remove(ann_file) return pos_file
def tf2onnx(tf_path, input_names, output_name, onnx_path): work_dir = "./" command = "python3 -m tf2onnx.convert --graphdef " + tf_path command = command + " --inputs " + hack_name(input_names) command = command + " --inputs-as-nchw " + hack_name(input_names) command = command + " --outputs " + hack_name(output_name) command = command + " --output " + onnx_path command = command + " --opset 11" print(command) result = cmd.run(command, work_dir=work_dir) if result == 0: return True else: return False
def runR(r=findR(), script="", args=""): """ Run R script parameters - r: path to R scripting front-end (Rscript) - script: R script filename - args: script parameters return - output,status (0=ok -1=error) """ r_cmd = " ".join((r, script, args)) print("\nrunning R cmd: {}\n".format(r_cmd)) return cmd.run(r_cmd)
def make_srpm(package, dist=None, fedpkg=FEDPKG): cmd = list(fedpkg) if dist: dname, _, drls = dist.partition("-") if dname == "epel" and drls: cmd += ["--dist", "el" + drls] cmd.append("srpm") out = run(*cmd) m = re.search(r"/([^/\\]+\.src.rpm)\b", out) if not m: raise exception.CommandOutputParseError(tool=cmd[0], output=out) srpm = m.group(1) if not os.path.isfile(srpm): raise exception.FileNotFound(path=srpm) return {"srpm": srpm}
def make_srpm(package, dist=None, fedpkg=FEDPKG): cmd = list(fedpkg) if dist: dname, _, drls = dist.partition('-') if dname == 'epel' and drls: cmd += ['--dist', 'el' + drls] cmd.append('srpm') out = run(*cmd) m = re.search(r'/([^/\\]+\.src.rpm)\b', out) if not m: raise exception.CommandOutputParseError(tool=cmd[0], output=out) srpm = m.group(1) if not os.path.isfile(srpm): raise exception.FileNotFound(path=srpm) return {'srpm': srpm}
def test_example_GSE8597(self): """ test GSE8597 example """ sample_file = os.path.join("test", "MCF7_E2_CHX.GSE8597.samples.txt") analysis_file = os.path.join("test", "MCF7_E2_CHX.GSE8597.analysis.txt") shutil.copyfile(sample_file, "MCF7_E2_CHX.GSE8597.samples.txt") out, status = cmd.run("python {} -g {}".format(script_path, analysis_file)) os.remove("MCF7_E2_CHX.GSE8597.samples.txt") self.assertEqual(status, 0) # excel file created excel_file = "DiffExpression.GSE8597.xlsx" statinfo = os.stat(excel_file) self.assertTrue(statinfo.st_size != 0) os.remove(excel_file)
def tf2onnx(tf_path, input_names, output_name, onnx_path, not_fold_const=False): work_dir = "./" inputs, inputs_as_nchw = process_input_names(input_names) command = "python3 -m tf2onnx.convert --graphdef " + tf_path command = command + " --inputs " + inputs command = command + " --inputs-as-nchw " + inputs_as_nchw command = command + " --outputs " + hack_name(output_name) command = command + " --output " + onnx_path command = command + " --opset 11" if not_fold_const is False: command = command + " --fold_const" logging.debug(command) result = cmd.run(command, work_dir=work_dir) if result == 0: return True else: return False
def nvr(pkg=None, branch=None, default=exception.CantGuess): if not pkg: try: pkg = package() except exception.CantGuess: if default is exception.CantGuess: raise else: return default if not branch: try: branch = current_branch() except Exception: if default is exception.CantGuess: raise else: return default tag = branch if tag.startswith('el6-'): tag = "dist-6E-epel-testing-candidate" kojiout = run("koji", "latest-pkg", "--quiet", tag, pkg, fatal=False, print_output=True) if not kojiout.success: if default is exception.CantGuess: raise exception.CantGuess(what="nvr", why="koji query failed") else: return default m = re.match('^(\S+)\s+\S+\s+\S+$', kojiout) if not m: if default is exception.CantGuess: raise exception.CantGuess(what="nvr", why="can't parse koji output") else: return default return m.group(1)
def convert(onnx_path, output_dir=None, version="v1.0", optimize=True, half=False): """ 执行 onnx 转换为 tnn 的转换指令 :parameter: onnx_path: 需要转换的 onnx 文件的路径 output_path: 生成的 tnn 文件的路径 version: 转换模型的版本号 optimize: 是否需要对模型进行优化,默认是需要进行优化 halt: 是否需要转为 FP16 的模型,减小模型的大小 :return return_code :exception 执行超时 """ command = "python3 onnx2tnn.py " + onnx_path command = command + " -version=v1.0" checker.check_file_exist(onnx_path) if optimize is True: command = command + " -optimize=1" else: command = command + " -optimize=0" if half is True: command = command + " -half=1" else: command = command + " -half=0" if output_dir is None: output_dir = os.path.dirname(onnx_path) checker.check_file_exist(output_dir) command = command + " -o " + output_dir print("the onnx2tnn command:" + command) work_dir = "../onnx2tnn/onnx-converter/" result = cmd.run(command, work_dir=work_dir) if result == 0: print("onnx2tnn succeed!") else: print("onnx2tnn failed!")
def tf2onnx(tf_path, input_names, output_name, onnx_path, not_fold_const=False): work_dir = "./" input_info: dict = format_input(input_names) input_info_str: str = "" input_nchw_names: str = "" for item in input_info.items(): input_info_str += item[0] + item[1] + "," input_nchw_names += item[0] + "," command = "python3 -m tf2onnx.convert --graphdef " + tf_path command = command + " --inputs " + input_info_str command = command + " --inputs-as-nchw " + input_nchw_names command = command + " --outputs " + hack_name(output_name) command = command + " --output " + onnx_path command = command + " --opset 11" if not_fold_const is False: command = command + " --fold_const" logging.debug(command) result = cmd.run(command, work_dir=work_dir) if result == 0: return True else: return False
def download_file(url): run('curl', '-L', '-f', '-O', url, direct=True)
def convert(onnx_path, output_dir=None, version="v1.0", optimize=True, half=False, align=False, input_path=None, refer_path=None, input_names: str = None): """ 执行 onnx 转换为 tnn 的转换指令 :parameter: onnx_path: 需要转换的 onnx 文件的路径 output_path: 生成的 tnn 文件的路径 version: 转换模型的版本号 optimize: 是否需要对模型进行优化,默认是需要进行优化 half: 是否需要转为 FP16 的模型,减小模型的大小 :return return_code :exception 执行超时 """ logging.info("Converter ONNX to TNN Model\n") checker.check_file_exist(onnx_path) ret, current_shape = checker.check_onnx_dim(onnx_path) if ret is False and current_shape is not None: if input_names is None: throw_exception(current_shape) if input_names is not None and not ("[" in input_names and "]" in input_names): throw_exception(current_shape) proto_suffix = '.tnnproto' model_suffix = '.tnnmodel' command = "python3 onnx2tnn.py " + onnx_path command = command + " -version=v1.0" checker.check_file_exist(onnx_path) if optimize is True: command = command + " -optimize=1" else: command = command + " -optimize=0" if half is True: command = command + " -half=1" else: command = command + " -half=0" if output_dir is None: output_dir = os.path.dirname(onnx_path) checker.check_file_exist(output_dir) command = command + " -o " + output_dir logging.debug("The onnx2tnn command:" + command + "\n") if input_names is not None: new_input_names = "" for char in input_names: if char == "[": char = ":" if char == "]": continue new_input_names += char command = command + " -input_shape " + new_input_names work_dir = "../onnx2tnn/onnx-converter/" result = cmd.run(command, work_dir=work_dir) if result == 0: logging.info("Converter ONNX to TNN model succeed!\n") else: logging.error("Converter ONNX to TNN model failed!\n") sys.exit(return_code.CONVERT_FAILED) onnx_base_name = os.path.basename(onnx_path) if align is True: if optimize is True: tnn_proto_name = onnx_base_name[:-len('.onnx' )] + '.opt' + proto_suffix tnn_model_name = onnx_base_name[:-len('.onnx' )] + '.opt' + model_suffix else: tnn_proto_name = onnx_base_name[:-len('.onnx')] + proto_suffix tnn_model_name = onnx_base_name[:-len('.onnx')] + model_suffix tnn_proto_path = os.path.join(output_dir, tnn_proto_name) tnn_model_path = os.path.join(output_dir, tnn_model_name) if input_names is None: align_model.align_model(onnx_path, tnn_proto_path, tnn_model_path, input_path, refer_path) else: align_model.align_model(onnx_path, tnn_proto_path, tnn_model_path, input_path, refer_path, new_input_names)
def fedpkg_mockbuild(fedpkg=FEDPKG): cmd = list(fedpkg) + ['mockbuild'] run(*cmd, direct=True)
def remove_dump_file(self) -> bool: command = "rm -rf {}".format(self.dump_dir_path) cmd.run(command)
def edit(path): editor = os.environ.get('EDITOR', 'vim') r = run(editor, path, direct=True) return r.success
def parallelize_m2(m2_file, txt_file): cmd.run("perl {scripts}/make_parallel.perl < {m2} > {txt}" \ .format(scripts=SCRIPTS_DIR, m2=m2_file, txt=txt_file))