def rename_files(self, prefix='My_mp3_'):
for f in self.files:
dir = dirname(f)
fname = basename(f)
new_fname = prefix + fname
new_f = path.join(dir, new_fname)
cmd = 'mv "{f}" "{new_f}"'.format(f=f, new_f=new_f)
show_var(['cmd'])
shell(cmd)
def evaluation_battery_translate(sample_eval_list, encoder, decoder, input_lang, output_lang, max_length, verbose=False):
for idx,sample in enumerate(sample_eval_list):
yq_predict, in_support = evaluate_translate(sample, encoder, decoder, input_lang, output_lang, max_length)
input_patterns = extract(np.logical_not(in_support),sample['xq'])
output_patterns = extract(np.logical_not(in_support),yq_predict)
target_patterns = extract(np.logical_not(in_support),sample['yq'])
inp = []
out = []
tgt = []
for i in range(len(input_patterns)):
#inp.append(input_patterns[i])
#out.append(output_patterns[i])
#tgt.append(target_patterns[i])
inp.append(' '.join(input_patterns[i]))
out.append(' '.join(output_patterns[i]))
tgt.append(' '.join(target_patterns[i]))
print('Input: ' + ' '.join(input_patterns[i]))
print('Output: ' + ' '.join(output_patterns[i]))
print('Target: ' + ' '.join(target_patterns[i]))
hyp = open('hyp_temp.txt','w')
ref = open('ref_temp.txt','w')
for line in out:
hyp.write(line + '\n')
for line in tgt:
ref.write(line + '\n')
hyp.close()
ref.close()
cmd = 'perl multi-bleu.perl ref_temp.txt < hyp_temp.txt'
stdout, stderr = shell(cmd)
if 'BLEU = ' in stdout:
num = stdout.split('BLEU = ', 1)[-1].split(',')[0]
print('BLEU: ' + num)
def fix_corrupted_file(file):
from efficiency.function import shell
cmd ='grep -nriF "</article>" {file} | tail -n1 |cut -d ":" -f 1 '\
.format(file=file)
outp, err = shell(cmd)
n_line = outp.strip()
cmd = 'head -{n_line} {file} > temp; ' \
'tail -n2 temp; ' \
'echo "</pmc-articleset>" >> temp; ' \
'mv temp {file}'.format(n_line=n_line, file=file)
os.system(cmd)
def get_total_num_lines_in_large_files(file_list, verbose=True):
from efficiency.function import shell
num_lines = []
for f in sorted(file_list):
cmd = "wc -l {} | cut -d ' ' -f 1".format(f)
stdout, stderr = shell(cmd)
num_line = int(stdout)
num_lines.append(num_line)
if verbose:
from efficiency.log import show_var
show_var(['cmd', 'num_line'], joiner='\t')
return sum(num_lines)
def download():
import os
path_chromdriver = '/usr/local/bin/chromedriver'
if not os.path.isfile(path_chromdriver):
from efficiency.function import shell
url = 'https://chromedriver.storage.googleapis.com/78.0.3904.11/chromedriver_linux64.zip'
cmd = 'curl -O {} \n' \
'unzip chromedriver_linux64.zip \n ' \
'rm chromedriver_linux64.zip'.format(url)
shell(cmd)
import pdb
pdb.set_trace()
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
options = Options()
options.add_argument("--headless")
options.add_argument("--no-sandbox")
options.add_argument("start-maximized")
options.add_argument("disable-infobars")
options.add_argument("--disable-extensions")
driver = webdriver.Chrome(chrome_options=options,
executable_path=path_chromdriver)
driver.get('http://google.com/')
import pdb
pdb.set_trace()
from selenium import webdriver
driver = webdriver.Chrome()
driver.get('https://www.ncbi.nlm.nih.gov/pmc/?term=paper')
driver.find_element_by_id('sendto').find_element_by_xpath('.//a').click()
driver.find_element_by_id('dest_File').click()
driver.find_element_by_xpath(
'.//button[@name="EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.SendToSubmit"@sid="1"@class="button_apply file ncbipopper-close-button"@type="submit"@cmd="File"]'
)
import pdb
pdb.set_trace()
driver.quit()
def gpu_mem(gpu_id=0):
from efficiency.function import shell
line = 9 + gpu_id * 3
cmd = "nvidia-smi | head -n {} | tail -n 1 | awk '{{print $9}}' | sed 's/MiB//' ".format(
line)
stdout, stderr = shell(cmd)
stdout = stdout.strip()
mem = int(stdout) if stdout != b'' else None
return mem
def multi_file_bleu(ref_files, hyp_files, detok=True, verbose=False):
'''
This is to get the average BLEU for hyp among ref0, ref1, ref2, ...
:param hyp_files: a list of filenames for hypothesis
:param ref_files: a list of filenames for references
:return: print a bleu score
'''
from efficiency.function import shell
# check for wrong input of ref_list, and correct it
if isinstance(ref_files, str):
ref_files = [ref_files]
ref_files, hyp_files = \
preprocess_files(ref_files, hyp_files, verbose=verbose)
outputs = []
script = BLEU_DETOK_FILE if detok else BLEU_FILE
for hyp in hyp_files:
cmd = 'perl {script} {refs} < {hyp} '.format(
script=script,refs=' '.join(ref_files), hyp=hyp)
if verbose: print('[cmd]', cmd)
stdout, stderr = shell(cmd)
bleu_prefix = 'BLEU = '
if verbose and not stdout.startswith(bleu_prefix):
print(stdout)
if bleu_prefix in stdout:
num = stdout.split(bleu_prefix, 1)[-1].split(',')[0]
output = float(num)
else:
# if stdout.startswith('Illegal division by zero'):
output = -1
outputs += [output]
if verbose:
print('{}-ref bleu for {}: {}'.format(len(ref_files), hyp, output))
return outputs