def run_sim_commands(command_list, test, bin_dir, use_lsf):
'''Run the given list of commands
command_list should be a list of tuples (desc, cmd, dirname) where desc is
a human-readable description of the test, cmd is a command to run and
dirname is the directory in which to run it (which will be created if
necessary).
test is a dictonary for given TEST matched from testlist.yaml
bin_dir is the general path to binary directory
If use_lsf is true, the commands in command_list begin with something like
'bsub -Is'. It seems that we always use interactive bsub, so we'll have a
local process per job, which we track with run_parallel_cmd.
'''
# If we're in LSF mode, we submit all the commands 'at once', which means
# we have to create the output directories in advance.
if use_lsf:
cmds = []
for desc, cmd, dirname in command_list:
os.makedirs(dirname, exist_ok=True)
cmds.append(cmd)
run_parallel_cmd(cmds, 600, check_return_code=True)
return
# We're not in LSF mode, so we'll create the output directories as we go.
# That should make it a bit easier to see how far we got if there was an
# error.
for desc, cmd, dirname in command_list:
os.makedirs(dirname, exist_ok=True)
cp_compiled_test(test, bin_dir)
logging.info("Running " + desc)
run_cmd(cmd, 300, check_return_code=True)
def gen_cov(base_dir, simulator, lsf_cmd):
"""Generate a merged coverage directory.
Args:
base_dir: the base simulation output directory (default: out/)
simulator: the chosen RTL simulator
lsf_cmd: command to run on LSF
"""
# Compile a list of all output seed-###/rtl_sim/test.vdb directories
dir_list = []
for entry in os.scandir(base_dir):
vdb_path = "%s/%s/rtl_sim/test.vdb" % (base_dir, entry.name)
if 'seed' in entry.name:
logging.info("Searching %s/%s for coverage database" %
(base_dir, entry.name))
if os.path.exists(vdb_path):
dir_list.append(vdb_path)
if dir_list == []:
logging.info("No coverage data available, exiting...")
sys.exit(RET_SUCCESS)
if simulator == 'vcs':
cov_cmd = "urg -full64 -format both -dbname test.vdb " \
"-report %s/rtl_sim/urgReport -dir" % base_dir
for cov_dir in dir_list:
cov_cmd += " %s" % cov_dir
logging.info("Generating merged coverage directory")
if lsf_cmd is not None:
cov_cmd = lsf_cmd + ' ' + cov_cmd
run_cmd(cov_cmd)
else:
logging.error("%s is an unsuported simulator! Exiting..." % simulator)
sys.exit(RET_FAIL)
def rtl_compile(compile_cmds, output_dir, lsf_cmd, opts):
"""Compile the testbench RTL
compile_cmds is a list of commands (each a string), which will have <out>
and <cmp_opts> substituted. Running them in sequence should compile the
testbench.
output_dir is the directory in which to generate the testbench (usually
something like 'out/rtl_sim'). This will be substituted for <out> in the
commands.
If lsf_cmd is not None, it should be a string to prefix onto commands to
run them through LSF. Here, this is not used for parallelism, but might
still be needed for licence servers.
opts is a string giving extra compilation options. This is substituted for
<cmp_opts> in the commands.
"""
logging.info("Compiling TB")
for cmd in compile_cmds:
cmd = subst_vars(cmd,
{
'out': output_dir,
'cmp_opts': opts
})
if lsf_cmd is not None:
cmd = lsf_cmd + ' ' + cmd
logging.debug("Compile command: %s" % cmd)
# Note that we don't use run_parallel_cmd here: the commands in
# compile_cmds need to be run serially.
run_cmd(cmd)
def _delete_from_registry(self):
"""
Deletes marked images from registry by invoking inbuild docker
distribution gc.
"""
cmd = [
"registry", "garbage-collect",
"/etc/docker-distribution/registry/config.yml"
]
lib.run_cmd(cmd, no_shell=not self._verbose)
def _delete_from_registry(self):
"""
Deletes marked images from registry by invoking inbuild docker
distribution gc.
"""
cmd = [
"registry",
"garbage-collect",
"/etc/docker-distribution/registry/config.yml"
]
lib.run_cmd(cmd, no_shell=not self._verbose)
def rtl_sim(sim_cmd, test_list, seed_gen, opts,
output_dir, bin_dir, lsf_cmd):
"""Run the testbench in the simulator
sim_cmd is the base command (as returned by get_simulator_cmd). This will
still have placeholders for test-specific arguments. test_list is a list of
test objects read from the testlist YAML file which gives the tests to run.
seed the seed generator to use to supply seeds for the simulations
(controls things like random delays on the bus). opts is a string of
plusargs to give to the simulator.
output_dir is the output directory for simulation files (and the directory
in which the simulator gets run). bin_dir is the directory containing
binaries to be run.
If lsf_cmd is not None, it should be prefixed on each command, which will
be run in parallel.
"""
logging.info("Running RTL simulation...")
sim_cmd = subst_vars(sim_cmd,
{
'out': output_dir,
'sim_opts': opts,
'cwd': _CORE,
'seed': str(seed_gen)
})
# Compute a list of pairs (cmd, dirname) where cmd is the command to run
# and dirname is the directory in which the command should be run.
# cmd_list = []
for test in test_list:
for i in range(test['iterations']):
cmd = get_test_sim_cmd(sim_cmd, test, i, output_dir, bin_dir, lsf_cmd)
# Creating directory for test simulation log
os.makedirs(cmd[2], exist_ok=True)
cp_compiled_test(test, bin_dir, i)
#logging.info("Running " + cmd[0])
#print("Command: " + cmd[1] + "\n")
run_cmd(cmd[1], 300, check_return_code=True)
'''it_cmd = subst_vars(sim_cmd, {'seed': str(seed_gen.gen(i))})
def make_diamond_database(protein_file,dbfile=False,threads=False):
if not dbfile:
name_clean = protein_file[:-6]
dbfile = '%s.dmnd' %name_clean
commands = ['diamond','makedb','--in',protein_file,'-d',dbfile]
if threads:
commands += ['-p',str(threads)]
logger.debug(' '.join(commands))
_ = run_cmd(commands)
return(dbfile)
def __restore_metadata(self):
"""
Restore the metadata and global objects is its a full restore
:return:
"""
# Source the hawq executable path
self.logger.info("Source the hawq executable path")
run_cmd("source $GPHOME/greenplum_path.sh")
# Backup file name
ddl_file = self.metadata_backup_dir + '/hdb_dump_' + self.backup_id + '_ddl.dmp'
global_file = self.metadata_backup_dir + '/hdb_dump_' + self.backup_id + '_global.dmp'
# If request to create database then create it with default encoding if not provided.
if self.create_target_db:
create_db_cmd = 'createdb ' + self.to_dbname + ' -E ' + self.target_db_encoding
run_cmd(create_db_cmd)
# Metadata restore command creator
pg_restore_cmd = self.__get_args("pg_restore", "--schema-only")
pg_restore_cmd = ' '.join(pg_restore_cmd)
metadata_file = 'hdfs dfs -cat ' + ddl_file + ' | '
# If generate list is requested.
if self.generate_list:
pg_restore_cmd += ' > ' + self.generate_list_location
pg_restore_cmd = metadata_file + pg_restore_cmd + ' ; exit $PIPESTATUS;'
run_cmd(pg_restore_cmd, self.ignore)
self.logger.info(
"Backup List for the backup ID \"{0}\" is generated at location: \"{1}\""
.format(self.backup_id, self.generate_list_location))
sys.exit(0)
# Else then this a full restore or user list restore
else:
pg_restore_cmd = metadata_file + pg_restore_cmd + ' ; exit $PIPESTATUS;'
run_cmd(pg_restore_cmd, self.ignore)
# If full restore or if requested to restore the global dump then
if self.global_restore or not (self.generate_list or self.user_list):
psql_cmd = 'psql -d ' + self.to_dbname + ' -f ' + global_file + ' -U ' + self.username
run_cmd(psql_cmd)
def __backup_metadata(self):
"""
Backup DDLs and metadata before dumping the data
:return
"""
# Source the hawq executable path
self.logger.info("Source the hawq executable path")
run_cmd("source $GPHOME/greenplum_path.sh")
# Backup file name
ddl_file = self.metadata_backup_dir + '/hdb_dump_' + self.backup_id + '_ddl.dmp'
global_file = self.metadata_backup_dir + '/hdb_dump_' + self.backup_id + '_global.dmp'
# Metadata backup command creator
pg_dump_cmd, pg_dumpall_cmd = self.__get_args("pg_dump",
"--schema-only",
"--format=c")
pg_dump_cmd = ' '.join(pg_dump_cmd)
pg_dump_cmd += ' | hdfs dfs -put - {0}'.format(ddl_file)
pg_dump_cmd += ' ; exit $PIPESTATUS;'
self.logger.info(
"Executing DDL backup, metadata backup file: \"{0}\"".format(
ddl_file))
run_cmd(pg_dump_cmd)
if pg_dumpall_cmd:
pg_dumpall_cmd += ' | hdfs dfs -put - {0}'.format(global_file)
pg_dumpall_cmd += ' ; exit $PIPESTATUS;'
self.logger.info(
"Executing global object backup, global backup file: \"{0}\"".
format(global_file))
run_cmd(pg_dumpall_cmd)
def makeblastdb(infile,dbtype='prot',title = False,outfile=False):
# Makes NCBI BLAST databases
current_path = os.getcwd()
# Changing directory to prevent some errors
os.chdir(infile.rpartition(os.sep)[0])
if not outfile:
outfile = infile.rpartition('.')[0]
commands = ["makeblastdb","-in",'"%s"' %infile,"-dbtype",dbtype,"-out",outfile]
if title:
commands += ["-title",title]
_ = run_cmd(commands)
os.chdir(current_path)
return(outfile)
def gen_cov(base_dir, simulator, lsf_cmd):
"""Generate a merged coverage directory.
Args:
base_dir: the base simulation output directory (default: out/)
simulator: the chosen RTL simulator
lsf_cmd: command to run on LSF
"""
# Compile a list of all output seed-###/rtl_sim/test.vdb directories
dir_list = []
# All generated coverage databases will be named "test.vdb"
vdb_dir_name = "test.vdb"
for path, dirs, files in os.walk(base_dir):
if vdb_dir_name in dirs:
vdb_path = os.path.join(path, vdb_dir_name)
logging.info("Found coverage database at %s" % vdb_path)
dir_list.append(vdb_path)
if dir_list == []:
logging.info("No coverage data available, exiting...")
sys.exit(RET_SUCCESS)
if simulator == 'vcs':
cov_cmd = "urg -full64 -format both -dbname test.vdb " \
"-report %s/rtl_sim/urgReport -dir" % base_dir
for cov_dir in dir_list:
cov_cmd += " %s" % cov_dir
logging.info("Generating merged coverage directory")
if lsf_cmd is not None:
cov_cmd = lsf_cmd + ' ' + cov_cmd
run_cmd(cov_cmd)
else:
logging.error("%s is an unsuported simulator! Exiting..." % simulator)
sys.exit(RET_FAIL)
def run():
nodes = get_nodes(count=1)
controller = nodes[0]
setup_controller(controller)
kompose_setup(controller)
run_cmd('iptables -F', host=controller)
run_cmd('git clone ' + str(repo_url) + ' /tmp/kompose-tests/',
host=controller)
run_cmd('cd /tmp/kompose-tests/ && ./run.sh', host=controller, stream=True)
def setup_controller(controller):
# provision controller
run_cmd(
"yum install -y git &&"
"yum install -y gcc libffi-devel python-devel openssl-devel && "
"yum install -y docker && "
"yum install -y golang && "
"yum install -y python-requests",
host=controller,
stream=True)
run_cmd("scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "
"docker_settings.sh root@%s:/tmp" % controller,
stream=True)
run_cmd("/tmp/docker_settings.sh", host=controller, stream=True)
def __get_data_location(self):
"""
Get all the schema and table names that this directory holds the backup for
:return: list of all relation that it has the backup
"""
cmd = "hdfs dfs -ls " + self.data_backup_dir + '/*'
output = run_cmd(cmd)
backup_object_list = []
for directory in output.split('\n'):
# Ignore blanks space and other unwanted output
if (directory.startswith('Found')
and directory.endswith('items')) or not directory.strip():
pass
# For the rest get the table and schema names
else:
schema = '"' + directory.split('/')[-2] + '"'
table = '"' + directory.split('/')[-1] + '"'
backup_object_list.append(schema + '.' + table)
return backup_object_list
def blast_worker(query,out,db_file,nr,queue,name,parse_function,parse_function_args,evalue,outfmt,kwargs):
# Blast this process' part of the query
logger.debug('Blasting process %s' %nr)
commands = ['blastp','-query',query,'-db',db_file,'-out',out,'-evalue', str(evalue), '-outfmt',outfmt]
if kwargs != {}:
for key,value in kwargs.items():
commands += ['-'+str(key),str(value)]
if not os.path.isfile(out):
_ = run_cmd(commands)
else:
logger.debug('%s already found. Not running BLAST' %out)
if parse_function:
logger.debug('Parsing function process: %s' %nr)
if parse_function_args:
parse_function_args = [out] + list(parse_function_args)
res = parse_function(*parse_function_args)
else:
res = parse_function(out)
else:
res = out
logger.debug('Process %s: blasting done!' %nr)
queue.put(res)
def run_diamond(protein_file,database,outname,replacetabs=False,tmpdir='/dev/shm/',maxhits=100,sens=False,moresens=False,threads=False,evalue=False,quiet=False):
commands = ['diamond','blastp','--query',protein_file,'--db',database,\
'--max-target-seqs',str(maxhits),'--out',outname,'--tmpdir',tmpdir,'--outfmt',\
'6', 'qseqid', 'sseqid', 'pident', 'length', 'mismatch', 'gapopen', 'qlen', 'slen', 'evalue', 'bitscore']
if threads:
commands += ['-p',str(threads)]
if evalue:
commands += ['--evalue',str(evalue)]
if moresens:
commands.append('--more-sensitive')
elif sens:
commands.append('--sensitive')
_ = run_cmd(commands)
if replacetabs:
f = open(outname)
fout = open(out_clean + '.tab','w')
for l in f:
l_new = l.replace('|','\t')
fout.write(l_new)
f.close()
fout.close()
outname = out_clean+'.tab'
elif opt in ('--qps',):
qps_flag = 1
qps_output = arg
elif opt in ('-i',):
input_file = arg
elif opt in ('-e',):
cdc.set_codec_id(arg.strip())
elif opt[1] in help.get_opt():
help.parse_opt(opt)
else:
assert False, "unknown option"
if bits_flag == 1:
cmd = cdc.get_bits_fcmd(input_file)
#print cmd
lib.run_cmd(cmd, bits_output,help.get_do_execute(),0)
if qp_flag == 1:
cmd = cdc.get_qp_fcmd(input_file)
#print cmd
lib.run_cmd(cmd, qp_output,help.get_do_execute(),0)
#if help.get_do_execute()==1:
# os.system(cmd)
#obtain_data()
#parse_cl()
#cdc=Codec_analysis("ashevcd")
#input_file="F:\\tmp\\2015.04.13\\cons.log"
#output_str=cdc.get_qp_vals(lib.format_file_path("F:\\tmp\\2015.04.13\\cons.log"))
qp_output = ''
qps_output = ''
for opt, arg in opts:
if opt in ('--bits', '-b'):
bits_flag = 1
bits_output = arg
elif opt in ('--qp', '-q'):
qp_flag = 1
qp_output = arg
elif opt in ('--qps', ):
qps_flag = 1
qps_output = arg
elif opt in ('-i', ):
input_file = arg
elif opt in ('-e', ):
cdc.set_id(arg.strip())
#elif opt[1] in help.get_opt():
# help.parse_opt(opt)
else:
help.parse_opt(opt)
if bits_flag == 1:
cmd = cdc.get_bits_cmd(input_file)
lib.run_cmd(cmd, help.get_do_execute(), bits_output)
if qp_flag == 1:
cmd = cdc.get_qp_cmd(input_file)
lib.run_cmd(cmd, help.get_do_execute(), qp_output)
#if help.get_do_execute()==1:
# os.system(cmd)
def run_hmmsearch(in_path, out_path, thr, hmm_db_path, outfile_base='domains'):
cmds = ['hmmsearch', '-o', out_path+outfile_base + '.out', '--tblout', out_path+outfile_base+'.tbl', '--domtblout', out_path + outfile_base + '.domtbl', \
'--acc', '--cut_ga', '--cpu', str(thr),hmm_db_path, in_path]
_ = run_cmd(cmds)
# param_list['encder_exe']=lib.get_encoder_exe()
#param_list['output_path']="F:/encoder_test_output/as265_output/"
#param_list['input_path']="E:/sequences/"
#param_list['i_frame_num_to_encode']=100
#param_list['input_filename']="BlowingBubbles_416x240_50.yuv"
#seq_name="BlowingBubbles_416x240_50"
#param_list['e_rctype']=1
cons = "tmp_cons.log"
#if len(sys.argv)> 1:
cons, extra_cls,do_execute = lib.configure_enc_param(enc, param_list)
cmd_line = lib.get_full_cdec_cmd(enc, param_list)
cmd_line += " " + extra_cls
#reg_file = None
#if lib.determin_sys() == "cygwin":
# cmd_line += (" 2>&1 |tee -a " + cons)
# pf = open(cons, "w")
# print >> pf, "%s" % cmd_line
# pf.close()
#else:
# reg_file = open(cons, 'w')
#
##os.system(cmd_line)
#print cmd_line
#subprocess.call(cmd_line, shell=True, stdout=reg_file, stderr=reg_file)
lib.run_cmd(cmd_line,do_execute,cons,1)
param_list['i_aq_mode'] = 0
param_list['i_lookahead'] = 10
param_list['b_cutree'] = 0
param_list['i_lowres'] = 1
encoder_list = ("as265", "x265")
# encoder_list = ("as265", )
# encoder_list = ("x265", )
#lib.Encoder_prop.SET_PATH("as265","")
#lib.Encoder_prop.SET_PATH("x265","")
lib.remove_some_tmp_files(param_list['output_path'])
do_execute = 0
for encoder_id in encoder_list:
enc.set_id(encoder_id)
for seq_name in seq_list:
for bitrate in br_list:
tag_str = "_" + encoder_id + "_bitrate" + str(bitrate)
lib.configure_seq_param(param_list, seq_name, tags=tag_str)
lib.check_params(param_list)
lib.set_rc_full_param(param_list, bitrate)
cmd = lib.get_full_cdec_cmd(enc, param_list)
print cmd
#os.system(cmd)
reg_file_name = param_list[
'output_path'] + seq_name + tag_str + "_cons.log"
#regression_file = open(reg_file_name, 'w')
#subprocess.call(cmd, stdout=regression_file, shell=True)
lib.run_cmd(cmd, do_execute, reg_file_name, 1)
def host_clean_up(controller):
run_cmd("rm -rf /usr/bin/kompose && rm -rf /tmp/kompose-tests",
host=controller)
def buildAndTest(submissionpath, sourceTestPath, no_remove):
points = 0
script_path = os.path.dirname(os.path.realpath(__file__))
# create temporary directory so that previous students' results will not affect subsequent tests
testCasePath = sourceTestPath
testCases = glob.glob(os.path.join(testCasePath, "*.simplec"))
#print(f"testCases {testCases}")
if not no_remove:
for i in glob.glob(os.path.join(submissionpath, "*.o")):
if os.path.exists(i):
os.remove(i)
progname = os.path.join(submissionpath, "simplec")
if os.path.exists(progname):
os.remove(progname)
if len(testCases) == 0:
print("# no tests found. double-check your path: " + testCasePath)
sys.exit()
if os.path.exists(submissionpath + "/simplec"):
os.remove(submissionpath + "/simplec")
out = subprocess.run(['make'],
cwd=submissionpath,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
output = ""
err = ""
if out.returncode != 0:
output += "Make failed."
print(output +
" Do you have a Makefile?") # can't even compile the compiler
return 0, output
else:
print("Build succeeded!")
points += build_points # points to build. tentative
# simpleC compilers lives so lets go through every test case now
for case in testCases:
base_name = os.path.basename(case)
ground_truth = case.replace(".simplec", ".ast")
output_file = base_name.replace(".simplec", ".out")
diff_file = base_name.replace(".simplec", ".diff")
print(f"Testing {base_name}:", end=" ")
with cd(submissionpath):
cmd = f"cat \"{case}\" | ./simplec > {output_file}"
#print(f"Running command: {cmd}")
return_code, stdout_, stderr_ = run_cmd(cmd)
cmd = f"diff -w -B \"{ground_truth}\" {output_file}"
#print(f"Running command: {cmd}")
return_code, stdout_, stderr_ = run_cmd(cmd, False)
if return_code == 0 and len(stdout_) == 0:
print("Success!")
points += test_case_points
if return_code == 1 and len(stdout_) > 0:
print(
f"Failure. See {diff_file} for diff and {output_file} for output."
)
diff_out = open(diff_file, "w")
diff_out.write(stdout_)
diff_out.close()
return_code, stdout_, stderr_ = run_cmd(cmd, False)
if return_code > 1:
print(
f"diff exited with an unknown return code. This shouldn't happen. Here is the stderr: {stderr_}"
)
print(
f"{int((points - build_points) / test_case_points)} / {len(testCases)} test casing passing. "
)
return points, output
param_list = lib.get_default_enc_param_list()
# param_list['encder_exe']=lib.get_encoder_exe()
#param_list['output_path']="F:/encoder_test_output/as265_output/"
#param_list['input_path']="E:/sequences/"
#param_list['i_frame_num_to_encode']=100
#param_list['input_filename']="BlowingBubbles_416x240_50.yuv"
#seq_name="BlowingBubbles_416x240_50"
#param_list['e_rctype']=1
cons = "tmp_cons.log"
#if len(sys.argv)> 1:
cons, extra_cls, do_execute = lib.configure_enc_param(enc, param_list)
cmd_line = lib.get_full_cdec_cmd(enc, param_list)
cmd_line += " " + extra_cls
#reg_file = None
#if lib.determin_sys() == "cygwin":
# cmd_line += (" 2>&1 |tee -a " + cons)
# pf = open(cons, "w")
# print >> pf, "%s" % cmd_line
# pf.close()
#else:
# reg_file = open(cons, 'w')
#
##os.system(cmd_line)
#print cmd_line
#subprocess.call(cmd_line, shell=True, stdout=reg_file, stderr=reg_file)
lib.run_cmd(cmd_line, do_execute, cons, 1)
extra_cmd += ' -vf scale=%s:%s' % (width, height)
if len(output_tag) == 0: # =='':
output_tag = "out"
cmd_list = []
output_list = []
for input_file in input_list:
print input_file
if not os.path.isfile(input_file):
logging.error('"%s" is not a file, but is in the input_list' %
input_file)
cmd_line, output_file = get_cmd_line(input_file, extension,
output_path, output_tag,
prepared_cmd, extra_cmd)
print cmd_line
cmd_list.append(cmd_line)
output_list.append(output_file)
if len(merged_file) > 0:
concat_str = "|".join(output_list)
print concat_str
cmd_line = FFMPEG_BIN
cmd_line += ' -i "concat:%s" -c copy -bsf:a aac_adtstoasc %s' % (
concat_str, merged_file)
print cmd_line
cmd_list.append(cmd_line)
logging.info('FINAL COMMANDS:')
for cmd in cmd_list:
lib.run_cmd(cmd, help.get_do_execute())
def kompose_setup(controller):
_print("Installing the kompose binary on the controller")
run_cmd(
"export GOPATH=/usr && go get github.com/kubernetes-incubator/kompose",
host=controller)
_print("Installing the oc binaries on the controller")
oc_download_url = requests.get(
"https://api.github.com/repos/openshift/origin/releases/latest").json(
)['assets'][2]['browser_download_url']
run_cmd("yum install wget -y", host=controller)
run_cmd("wget " + oc_download_url + " -O /tmp/oc.tar.gz", host=controller)
run_cmd("mkdir /tmp/ocdir && cd /tmp/ocdir && tar -xvvf /tmp/oc.tar.gz",
host=controller)
run_cmd("cp /tmp/ocdir/*/oc /usr/bin/", host=controller)
run_cmd("service docker restart", host=controller)
