def new_sources(branch=None, fedpkg=FEDPKG, no_new_sources=False):
_ensure_branch(branch)
if no_new_sources or not _has_valid_sources():
return
sources = specfile.Spec().get_source_fns()
cmd = fedpkg + ["new-sources"] + sources
run(*cmd, direct=True)
def gene_random_data(input_info: dict) -> str:
data = {}
current_dir = pathlib.Path(__file__).parent.parent
data_dir = os.path.join(current_dir, "temp_data")
command = "mkdir -p " + data_dir
logging.debug(command)
cmd.run(command)
checker.check_file_exist(data_dir)
data_path = os.path.join(data_dir, "input.txt")
data_file = open(data_path, "w")
data_file.write(str(len(input_info)) + '\n')
for name, info in input_info.items():
shape = info['shape']
data_type = info['data_type']
data_file.write(name + ' ' + str(len(shape)) + ' ' +
' '.join([str(dim) for dim in shape]) + ' ' +
str(data_type) + '\n')
if data_type == 0:
data[name] = np.random.rand(*shape)
np.savetxt(data_file, data[name].reshape(-1), fmt="%0.6f")
elif data_type == 3:
data[name] = np.random.randint(low=0, high=1, size=shape)
np.savetxt(data_file, data[name].reshape(-1), fmt="%i")
data_file.close()
return data_path
def findR():
"""
Find path to R scripting front-end (Rscript)
Try Rscript, $R_HOME, whereis Rscript and
HKLM\SOFTWARE\R-core\R*\InstallPath
"""
R = None
if cmd.can_run("Rscript --version"):
R = "Rscript"
else:
if "R_HOME" in os.environ:
R = os.path.join(os.environ['R_HOME'], "bin", "Rscript")
else:
if cmd.on_windows():
reg_cmd = 'cmd /c %SystemRoot%\\system32\\reg.exe query HKLM\SOFTWARE\R-core /s /f InstallPath'
reg_info, status = cmd.run(reg_cmd)
# HKEY_LOCAL_MACHINE\SOFTWARE\R-core\R64\3.2.5
# InstallPath REG_SZ C:\Program Files\R\R-3.2.5
for line in reg_info.split('\n'):
if 'InstallPath' in line:
# ['', '', 'InstallPath', '', 'REG_SZ', '', 'C:\\Program Files\\R\\R-3.2.5']
R = os.path.join(line.split(
' ')[-1], "bin", "Rscript")
else:
if cmd.can_run("whereis ls"):
whereis_info, status = cmd.run("whereis Rscript")
# Rscript: /usr/bin/Rscript /usr/bin/X11/Rscript
wrscript = whereis_info.split()
if len(wrscript) >= 2:
R = wrscript[1]
return R
def evaluate_m2(out_file, m2_file, log_file=None):
num_of_lines = cmd.wc(out_file)
with open(m2_file) as m2_io:
num_of_sents = sum(1 for line in m2_io if line.startswith("S "))
if num_of_lines != num_of_sents:
log.error("Different number of sentences between text and M2 file:"
" {} != {}".format(num_of_lines, num_of_sents))
if config['m2scorer'] == M2SCORER.MOSES:
# C++ implementation of m2scorer from Moses.
result = cmd.run("{moses}/bin/m2scorer --candidate {txt} --reference {m2}" \
.format(moses=config['mosesdecoder'], txt=out_file, m2=m2_file))
# Scorer is run by system shell because of the bug inside the scorer
# script which cause propagation of forked threads to this script.
else:
result = cmd.run(
"python {scripts}/m2scorer_fork --forks {threads} {txt} {m2}" \
.format(scripts=SCRIPTS_DIR, threads=config['threads'] or 4,
txt=out_file, m2=m2_file))
if log_file:
with open(log_file, 'w') as log_io:
log_io.write(result)
return tuple(
float(line.rsplit(' ', 1)[-1]) for line in result.strip().split("\n"))
def run_tnn_model_check(proto_path,
model_path,
input_path,
reference_output_path,
is_tflite=False,
align_batch=False):
cmd.run("pwd")
relative_path = "bin/model_check"
model_check_path = parse_path.parse_path(relative_path)
checker.check_file_exist(model_check_path)
command = model_check_path + " -e -p " + proto_path + " -m " + \
model_path + " -i " + input_path + " -f " + reference_output_path + " -d NAIVE"
if align_batch:
command += " -b "
logging.debug(command)
ret = cmd.run(command)
if ret == 0:
print_align_message(is_tflite)
else:
print_not_align_message(None, is_tflite)
return
def __save_ngrams(self, freq_file, ngram_file):
log.debug(
"Write n-gram frequencies into file {}...".format(ngram_file))
line_num = cmd.run("cat {0} | grep -Pn '^ +{1} .*' | tr ':' '\\t'" \
" | cut -f1 | tail -1".format(freq_file, self.min_count))
cmd.run("head -n {} {} | sed -r 's/^ *([0-9]+) (.*)/\\2\\t\\1/' >> {}" \
.format(line_num.strip(), freq_file, ngram_file))
def __calculate_freqs(self, feat_file, freq_file):
command = r"cat {feats}" \
" | grep '^shared' | cut -c11-" \
" | tr ' ' '\\n' | sort -S 5G --parallel {threads}" \
" | uniq -c | sort -rn -S 5G --parallel {threads}" \
" | sed -r 's/ +([0-9]+)/\\1\\t/'" \
" > {freqs}"
cmd.run(
command.format(
feats=feat_file, freqs=freq_file, threads=config['threads']))
def train(self, model, data):
if os.path.exists(model):
log.warn("model already exists: {}".format(model))
if not os.path.exists(data):
log.error("data file does not exist: {}".format(data))
options = (config['vw-options'] or "") + VWTrainer.DEFAULT_OPTIONS
log.info("train VW: {}".format(model))
cmd.run("{vw}/vowpalwabbit/vw -f {model} -d {data} -c {options}" \
.format(vw=self.vw, model=model, data=data, options=options))
def __count_frequencies(self, list_file):
log.info(
"Calculate n-gram frequencies in file {}...".format(list_file))
command = "cat {0}" \
" | sort -S 10G --parallel 8" \
" | uniq -c" \
" | sort -S 10G --parallel 8 -nr" \
" > {0}.freq".format(list_file)
cmd.run(command)
return list_file + '.freq'
def get_dump_dir_path(self) -> str:
convert2tnn_path = pathlib.Path(__file__).parent.parent
data_dir = os.path.join(convert2tnn_path, "temp_data/")
if os.path.exists(data_dir):
command = "rm -rf {}".format(data_dir)
cmd.run(command)
command = "mkdir {}".format(data_dir)
cmd.run(command)
return data_dir
def new_sources(branch=None, fedpkg=FEDPKG, no_new_sources=False):
_ensure_branch(branch)
if no_new_sources:
log.info("skipping `%s new-sources` as requested." % fedpkg[0])
return
if not os.path.isfile('sources'):
log.info("'sources' file not found, skipping `%s new-sources`" %
fedpkg[0])
return
sources = specfile.Spec().get_source_fns()
cmd = fedpkg + ['new-sources'] + sources
run(*cmd, direct=True)
def run_tnn_model_check(proto_path, model_path, input_path,
reference_output_path):
cmd.run("pwd")
relative_path = "bin/model_check"
model_check_path = parse_path.parse_path(relative_path)
checker.check_file_exist(model_check_path)
command = model_check_path + " -p " + proto_path + " -m " + \
model_path + " -i " + input_path + " -f " + reference_output_path + " -d NAIVE"
print(command)
cmd.run(command)
return
def gene_random_data(input_info: dict) -> str:
data = {}
current_dir = pathlib.Path(__file__).parent.parent
data_dir = os.path.join(current_dir, "temp_data")
command = "mkdir -p " + data_dir
print(command)
cmd.run("pwd")
cmd.run(command)
checker.check_file_exist(data_dir)
data_path = os.path.join(data_dir, "input.txt")
data_file = open(data_path, "w")
for name, shape in input_info.items():
data[name] = np.random.rand(*shape)
np.savetxt(data_file, data[name].reshape(-1), fmt="%0.18f")
data_file.close()
return data_path
def caffe2onnx(proto_path, model_path, output_path):
work_dir = "../caffe2onnx/"
command = "python3 caffe2onnx.py " + proto_path + " " + model_path + " -o " + output_path
result = cmd.run(command, work_dir=work_dir)
if result == 0:
return True
else:
return False
def run_tnn_model_check(proto_path, model_path, input_path, reference_output_path):
cmd.run("pwd")
relative_path = "bin/model_check"
model_check_path = parse_path.parse_path(relative_path)
checker.check_file_exist(model_check_path)
command = model_check_path + " -p " + proto_path + " -m " + \
model_path + " -i " + input_path + " -f " + reference_output_path + " -d NAIVE"
logging.debug(command)
ret = cmd.run(command)
if ret == 0:
print_align_message()
else:
print_not_align_message()
return
def run_model_check(self) -> bool:
model_check_path = os.path.join(self.dump_dir_path[:-10], "bin/model_check")
tnn_model_path = self.tnn_proto_path[:-9] + ".tnnmodel"
input_path = os.path.join(self.dump_dir_path, "input.txt")
command = "{} -p {} -m {} -i {} -a {} -d NAIVE".format(
model_check_path, self.tnn_proto_path, tnn_model_path, input_path, self.dump_dir_path)
logging.debug(command)
return cmd.run(command, log_level="error")
def test_example_bad(self):
""" test example with R error"""
sample_file = os.path.join("test", "bad.samples.txt")
analysis_file = os.path.join("test", "bad.analysis.txt")
shutil.copyfile(sample_file, "bad.samples.txt")
out, status = cmd.run("python {} -g {}".format(script_path,
analysis_file))
os.unlink("bad.samples.txt")
self.assertEqual(status, -1)
def tflite2tnn(tf_path, tnn_path, not_fold_const=False):
cmd.run("pwd")
relative_path = "bin/TnnConverter"
TnnConverter_path = parse_path.parse_path(relative_path)
checker.check_file_exist(TnnConverter_path)
command = TnnConverter_path + " -mt TFLITE -mp " + tf_path
checker.check_file_exist(TnnConverter_path)
checker.check_file_exist(tf_path)
if tnn_path is None:
tnn_path = os.path.dirname(tf_path)
checker.check_file_exist(tnn_path)
command = command + " -od " + tnn_path + "/"
logging.debug(command)
result = cmd.run(command)
if result == 0:
return True
else:
return False
def convert_tok(file_in, file_out, mode='moses-nltk'):
# TODO: clean this!
if not mode:
converter = ''
elif mode == 'moses-nltk':
converter = \
' | {moses}/scripts/tokenizer/detokenizer.perl -l en' \
' | python {scripts}/nltk-tok.py' \
.format(moses=config['mosesdecoder'], scripts=SCRIPTS_DIR)
elif mode == 'nltk-moses':
converter = \
' | python {scripts}/nltk-detok.py' \
' | {moses}/scripts/tokenizer/tokenizer.perl -l en -threads {threads}' \
.format(moses=config['mosesdecoder'], scripts=SCRIPTS_DIR,
threads=config['threads'])
else:
log.error("Convertion mode '{}' not supported".format(mode))
exit(1)
cmd.run("cat {} {} > {}".format(file_in, converter, file_out))
def run(
self,
model,
data,
predictions,
options=" -q st -b 26 --noconstant --loss_function logistic --hash all"
):
if not os.path.exists(model):
log.error("model does not exist: {}".format(model))
if not os.path.exists(data):
log.error("data file does not exist: {}".format(data))
log.info("running VW: {}".format(model))
cmd.run(
"{vw}/vowpalwabbit/vw -t -i {model} -d {data} -c {options} -r {pred}"
.format(vw=self.vw,
model=model,
data=data,
options=options,
pred=predictions))
def edit(path):
editor = os.environ.get('EDITOR')
if not editor:
editor = 'vim'
log.info("$EDITOR not set. Falling back to %s." % editor)
try:
r = run(editor, path, direct=True)
except exception.CommandNotFound as ex:
raise exception.CommandNotFound(
msg='Failed to find suitable text editor. Please set $EDITOR '
'environment variable.')
return r.success
def edit(path):
editor = os.environ.get('EDITOR')
if not editor:
editor = 'vim'
log.info("$EDITOR not set. Falling back to %s." % editor)
try:
r = run(editor, path, direct=True)
except exception.CommandNotFound as ex:
raise exception.CommandNotFound(
msg='Failed to find suitable text editor. Please set $EDITOR '
'environment variable.')
return r.success
def tag_file(self, tok_file, pos_file=None, lazy=True):
if pos_file is None:
pos_file = tok_file + '.pos'
ann_file = pos_file + '.ann'
if lazy and os.path.exists(pos_file):
log.info("Tagging skipped because file {} exists".format(pos_file))
return pos_file
log.info("Tagging file {}".format(tok_file))
command = "java -mx1025m -cp {0}/stanford-postagger.jar: " \
"edu.stanford.nlp.tagger.maxent.MaxentTagger " \
"-model {0}/models/english-left3words-distsim.tagger " \
"-sentenceDelimiter newline -tokenize false -tagSeparator \"{3}\" " \
"-textFile {1} -nthreads {4} 2> /dev/null > {2}" \
.format(self.tagger_dir, tok_file, ann_file, self.separator,
self.threads)
cmd.run(command)
self.__extract_pos_tags(ann_file, pos_file)
os.remove(ann_file)
return pos_file
def tf2onnx(tf_path, input_names, output_name, onnx_path):
work_dir = "./"
command = "python3 -m tf2onnx.convert --graphdef " + tf_path
command = command + " --inputs " + hack_name(input_names)
command = command + " --inputs-as-nchw " + hack_name(input_names)
command = command + " --outputs " + hack_name(output_name)
command = command + " --output " + onnx_path
command = command + " --opset 11"
print(command)
result = cmd.run(command, work_dir=work_dir)
if result == 0:
return True
else:
return False
def runR(r=findR(), script="", args=""):
"""
Run R script
parameters
- r: path to R scripting front-end (Rscript)
- script: R script filename
- args: script parameters
return
- output,status (0=ok -1=error)
"""
r_cmd = " ".join((r, script, args))
print("\nrunning R cmd: {}\n".format(r_cmd))
return cmd.run(r_cmd)
def make_srpm(package, dist=None, fedpkg=FEDPKG):
cmd = list(fedpkg)
if dist:
dname, _, drls = dist.partition("-")
if dname == "epel" and drls:
cmd += ["--dist", "el" + drls]
cmd.append("srpm")
out = run(*cmd)
m = re.search(r"/([^/\\]+\.src.rpm)\b", out)
if not m:
raise exception.CommandOutputParseError(tool=cmd[0], output=out)
srpm = m.group(1)
if not os.path.isfile(srpm):
raise exception.FileNotFound(path=srpm)
return {"srpm": srpm}
def make_srpm(package, dist=None, fedpkg=FEDPKG):
cmd = list(fedpkg)
if dist:
dname, _, drls = dist.partition('-')
if dname == 'epel' and drls:
cmd += ['--dist', 'el' + drls]
cmd.append('srpm')
out = run(*cmd)
m = re.search(r'/([^/\\]+\.src.rpm)\b', out)
if not m:
raise exception.CommandOutputParseError(tool=cmd[0], output=out)
srpm = m.group(1)
if not os.path.isfile(srpm):
raise exception.FileNotFound(path=srpm)
return {'srpm': srpm}
def test_example_GSE8597(self):
""" test GSE8597 example """
sample_file = os.path.join("test", "MCF7_E2_CHX.GSE8597.samples.txt")
analysis_file = os.path.join("test",
"MCF7_E2_CHX.GSE8597.analysis.txt")
shutil.copyfile(sample_file, "MCF7_E2_CHX.GSE8597.samples.txt")
out, status = cmd.run("python {} -g {}".format(script_path,
analysis_file))
os.remove("MCF7_E2_CHX.GSE8597.samples.txt")
self.assertEqual(status, 0)
# excel file created
excel_file = "DiffExpression.GSE8597.xlsx"
statinfo = os.stat(excel_file)
self.assertTrue(statinfo.st_size != 0)
os.remove(excel_file)
def tf2onnx(tf_path, input_names, output_name, onnx_path, not_fold_const=False):
work_dir = "./"
inputs, inputs_as_nchw = process_input_names(input_names)
command = "python3 -m tf2onnx.convert --graphdef " + tf_path
command = command + " --inputs " + inputs
command = command + " --inputs-as-nchw " + inputs_as_nchw
command = command + " --outputs " + hack_name(output_name)
command = command + " --output " + onnx_path
command = command + " --opset 11"
if not_fold_const is False:
command = command + " --fold_const"
logging.debug(command)
result = cmd.run(command, work_dir=work_dir)
if result == 0:
return True
else:
return False
def nvr(pkg=None, branch=None, default=exception.CantGuess):
if not pkg:
try:
pkg = package()
except exception.CantGuess:
if default is exception.CantGuess:
raise
else:
return default
if not branch:
try:
branch = current_branch()
except Exception:
if default is exception.CantGuess:
raise
else:
return default
tag = branch
if tag.startswith('el6-'):
tag = "dist-6E-epel-testing-candidate"
kojiout = run("koji",
"latest-pkg",
"--quiet",
tag,
pkg,
fatal=False,
print_output=True)
if not kojiout.success:
if default is exception.CantGuess:
raise exception.CantGuess(what="nvr", why="koji query failed")
else:
return default
m = re.match('^(\S+)\s+\S+\s+\S+$', kojiout)
if not m:
if default is exception.CantGuess:
raise exception.CantGuess(what="nvr",
why="can't parse koji output")
else:
return default
return m.group(1)
def convert(onnx_path,
output_dir=None,
version="v1.0",
optimize=True,
half=False):
"""
执行 onnx 转换为 tnn 的转换指令
:parameter:
onnx_path: 需要转换的 onnx 文件的路径
output_path: 生成的 tnn 文件的路径
version: 转换模型的版本号
optimize: 是否需要对模型进行优化,默认是需要进行优化
halt: 是否需要转为 FP16 的模型,减小模型的大小
:return return_code
:exception 执行超时
"""
command = "python3 onnx2tnn.py " + onnx_path
command = command + " -version=v1.0"
checker.check_file_exist(onnx_path)
if optimize is True:
command = command + " -optimize=1"
else:
command = command + " -optimize=0"
if half is True:
command = command + " -half=1"
else:
command = command + " -half=0"
if output_dir is None:
output_dir = os.path.dirname(onnx_path)
checker.check_file_exist(output_dir)
command = command + " -o " + output_dir
print("the onnx2tnn command:" + command)
work_dir = "../onnx2tnn/onnx-converter/"
result = cmd.run(command, work_dir=work_dir)
if result == 0:
print("onnx2tnn succeed!")
else:
print("onnx2tnn failed!")
def nvr(pkg=None, branch=None, default=exception.CantGuess):
if not pkg:
try:
pkg = package()
except exception.CantGuess:
if default is exception.CantGuess:
raise
else:
return default
if not branch:
try:
branch = current_branch()
except Exception:
if default is exception.CantGuess:
raise
else:
return default
tag = branch
if tag.startswith('el6-'):
tag = "dist-6E-epel-testing-candidate"
kojiout = run("koji", "latest-pkg", "--quiet", tag, pkg,
fatal=False, print_output=True)
if not kojiout.success:
if default is exception.CantGuess:
raise exception.CantGuess(what="nvr",
why="koji query failed")
else:
return default
m = re.match('^(\S+)\s+\S+\s+\S+$', kojiout)
if not m:
if default is exception.CantGuess:
raise exception.CantGuess(what="nvr",
why="can't parse koji output")
else:
return default
return m.group(1)
def tf2onnx(tf_path, input_names, output_name, onnx_path, not_fold_const=False):
work_dir = "./"
input_info: dict = format_input(input_names)
input_info_str: str = ""
input_nchw_names: str = ""
for item in input_info.items():
input_info_str += item[0] + item[1] + ","
input_nchw_names += item[0] + ","
command = "python3 -m tf2onnx.convert --graphdef " + tf_path
command = command + " --inputs " + input_info_str
command = command + " --inputs-as-nchw " + input_nchw_names
command = command + " --outputs " + hack_name(output_name)
command = command + " --output " + onnx_path
command = command + " --opset 11"
if not_fold_const is False:
command = command + " --fold_const"
logging.debug(command)
result = cmd.run(command, work_dir=work_dir)
if result == 0:
return True
else:
return False
def download_file(url):
run('curl', '-L', '-f', '-O', url, direct=True)
def convert(onnx_path,
output_dir=None,
version="v1.0",
optimize=True,
half=False,
align=False,
input_path=None,
refer_path=None,
input_names: str = None):
"""
执行 onnx 转换为 tnn 的转换指令
:parameter:
onnx_path: 需要转换的 onnx 文件的路径
output_path: 生成的 tnn 文件的路径
version: 转换模型的版本号
optimize: 是否需要对模型进行优化,默认是需要进行优化
half: 是否需要转为 FP16 的模型,减小模型的大小
:return return_code
:exception 执行超时
"""
logging.info("Converter ONNX to TNN Model\n")
checker.check_file_exist(onnx_path)
ret, current_shape = checker.check_onnx_dim(onnx_path)
if ret is False and current_shape is not None:
if input_names is None:
throw_exception(current_shape)
if input_names is not None and not ("[" in input_names
and "]" in input_names):
throw_exception(current_shape)
proto_suffix = '.tnnproto'
model_suffix = '.tnnmodel'
command = "python3 onnx2tnn.py " + onnx_path
command = command + " -version=v1.0"
checker.check_file_exist(onnx_path)
if optimize is True:
command = command + " -optimize=1"
else:
command = command + " -optimize=0"
if half is True:
command = command + " -half=1"
else:
command = command + " -half=0"
if output_dir is None:
output_dir = os.path.dirname(onnx_path)
checker.check_file_exist(output_dir)
command = command + " -o " + output_dir
logging.debug("The onnx2tnn command:" + command + "\n")
if input_names is not None:
new_input_names = ""
for char in input_names:
if char == "[":
char = ":"
if char == "]":
continue
new_input_names += char
command = command + " -input_shape " + new_input_names
work_dir = "../onnx2tnn/onnx-converter/"
result = cmd.run(command, work_dir=work_dir)
if result == 0:
logging.info("Converter ONNX to TNN model succeed!\n")
else:
logging.error("Converter ONNX to TNN model failed!\n")
sys.exit(return_code.CONVERT_FAILED)
onnx_base_name = os.path.basename(onnx_path)
if align is True:
if optimize is True:
tnn_proto_name = onnx_base_name[:-len('.onnx'
)] + '.opt' + proto_suffix
tnn_model_name = onnx_base_name[:-len('.onnx'
)] + '.opt' + model_suffix
else:
tnn_proto_name = onnx_base_name[:-len('.onnx')] + proto_suffix
tnn_model_name = onnx_base_name[:-len('.onnx')] + model_suffix
tnn_proto_path = os.path.join(output_dir, tnn_proto_name)
tnn_model_path = os.path.join(output_dir, tnn_model_name)
if input_names is None:
align_model.align_model(onnx_path, tnn_proto_path, tnn_model_path,
input_path, refer_path)
else:
align_model.align_model(onnx_path, tnn_proto_path, tnn_model_path,
input_path, refer_path, new_input_names)
def fedpkg_mockbuild(fedpkg=FEDPKG):
cmd = list(fedpkg) + ['mockbuild']
run(*cmd, direct=True)
def remove_dump_file(self) -> bool:
command = "rm -rf {}".format(self.dump_dir_path)
cmd.run(command)
def edit(path):
editor = os.environ.get('EDITOR', 'vim')
r = run(editor, path, direct=True)
return r.success
def parallelize_m2(m2_file, txt_file):
cmd.run("perl {scripts}/make_parallel.perl < {m2} > {txt}" \
.format(scripts=SCRIPTS_DIR, m2=m2_file, txt=txt_file))
