def generate_tests(args, classpath, class_list_file, test_src_dir, junit_after_path, rc_classpath, time_limit=200, output_limit=4000):
# Methods to be omitted due to non-determinism.
omitted_methods = "\"(org\\.la4j\\.operation\\.ooplace\\.OoPlaceKroneckerProduct\\.applyCommon)|(PseudoOracle\\.verifyFace)|(org\\.znerd\\.math\\.NumberCentral\\.createRandomInteger)|(org\\.jbox2d\\.common\\.MathUtils\\.randomFloat.*)|(org\\.jbox2d\\.utests\\.MathTest\\.testFastMath)|(org\\.jbox2d\\.testbed\\.tests\\.DynamicTreeTest.*)|(org\\.la4j\\.Matrix.*)\""
selection_log_file = "dljc-out/selection-log.txt"
operation_log_file = "dljc-out/operation-history-log.txt"
randoop_log_file = "dljc-out/randoop-log.txt"
randoop_command = ["java", "-ea",
"-classpath", classpath,
"-Xbootclasspath/a:{}".format(rc_classpath),
"-javaagent:{}".format(rc_classpath),
"randoop.main.Main", "gentests",
"--classlist={}".format(class_list_file),
"--time-limit={}".format(time_limit),
"--omitmethods={}".format(omitted_methods),
"--junit-reflection-allowed=false",
"--flaky-test-behavior=DISCARD",
"--usethreads=true",
"--call-timeout=5",
"--silently-ignore-bad-class-names=true",
"--junit-output-dir={}".format(test_src_dir),
# Uncomment these lines to produce Randoop debugging logs
#"--log={}".format(randoop_log_file),
"--selection-log={}".format(selection_log_file),
"--operation-history-log={}".format(operation_log_file)]
if junit_after_path:
randoop_command.append("--junit-after-all={}".format(junit_after_path))
if output_limit and output_limit > 0:
randoop_command.append('--output-limit={}'.format(output_limit))
common.run_cmd(randoop_command, args, 'randoop')
def compile_test_cases(args, classpath, test_class_directory, files_to_compile):
compile_command = ["javac", "-g",
"-classpath", classpath,
"-d", test_class_directory]
compile_command.extend(files_to_compile)
common.run_cmd(compile_command, args, 'randoop')
def run(args, javac_commands, jars):
if not args.graph_jar:
print "Could not run graph tool: missing arg --graph-jar"
return
tool_command = ["java", "-jar", args.graph_jar]
for jc in javac_commands:
java_files = jc['java_files']
java_files_file = os.path.join(os.getcwd(), '__java_file_names.txt')
class_dir = common.class_directory(jc)
with open(java_files_file, 'w') as f:
for s in java_files:
f.write(s)
f.write("\n")
current_outdir = os.path.join(args.output_directory,
class_dir.replace(os.getcwd(),'').replace(os.sep,"_"))
cmd = tool_command + ["-o", current_outdir,
"-j", class_dir,
"-all",
"-source", java_files_file]
common.run_cmd(cmd)
def run(args, javac_commands, jars):
if not args.time_limit:
args.time_limit = 2
if not args.test_limit:
args.test_limit = 2
if not args.out_dir:
args.out_dir = "./dyntrace_output"
current_dir = os.path.dirname(os.path.realpath(__file__))
current_dir = os.path.join(current_dir, os.pardir)
current_dir = os.path.join(current_dir, os.pardir)
current_dir += "/"
dyntrace_command = ["java", "-jar", os.path.join(current_dir, "build/libs/DynSlicer.jar")]
i = 1
for jc in javac_commands:
javac_switches = jc['javac_switches']
cmd = dyntrace_command + [common.classpath(jc), common.class_directory(jc), args.out_dir, current_dir, args.time_limit, args.test_limit]
common.run_cmd(cmd)
i = i + 1
def run(args, javac_commands, jars):
bixie_jar = os.path.join(args.lib_dir, "bixie.jar")
base_command = ["java",
"-jar", bixie_jar,
"-html", os.path.join(args.output_directory, 'bixie_report')]
i = 1
for jc in javac_commands:
cmd = copy.copy(base_command)
if common.classpath(jc):
cmd.extend(["-cp", common.classpath(jc)])
if common.class_directory(jc):
cmd.extend(["-j", common.class_directory(jc)])
if common.source_path(jc):
cmd.extend(['-src', common.source_path(jc)])
out_filename = 'bixie-report-{}.log'.format(i)
cmd.extend(['-o', os.path.join(args.output_directory, out_filename)])
common.run_cmd(cmd, args, 'bixie')
i = i + 1
def add_project_to_corpus(project):
""" Assumes that the project_dir contains a
text file named build_command.txt that contains the build command(s) for the
project in this directory, and a clean_command.txt that will clean the project.
"""
common.clean_project(project)
"""Run dljc
Run Randoop to generate test sources
Compile test sources
Run daikon.Chicory on tests to create dtrace file
Precompute graph kernels that are independent of ontology stuff
"""
common.run_dljc(project,
['dyntrace', 'graphtool'],
['--graph-jar', common.get_jar('prog2dfg.jar'),
'--dyntrace-libs', common.LIBS_DIR])
""" run petablox """
#run_petablox(project_dir)
""" run graph kernel computation """
project_dir = common.get_project_dir(project)
kernel_file_path = common.get_kernel_path(project)
graph_kernel_cmd = ['python',
common.get_simprog('precompute_kernel.py'),
project_dir,
kernel_file_path
]
common.run_cmd(graph_kernel_cmd)
print 'Generated kernel file for {0}.'.format(project)
return kernel_file_path
def run(args, javac_commands, jars):
# the dist directory if CFI.
CFI_dist = os.path.join(os.environ['JSR308'], 'checker-framework-inference', 'dist')
CFI_command = ['java']
print os.environ
for jc in javac_commands:
target_cp = jc['javac_switches']['classpath'] + \
':' + os.path.join(args.lib_dir, 'ontology.jar')
cp = target_cp + \
':' + os.path.join(CFI_dist, 'checker.jar') + \
':' + os.path.join(CFI_dist, 'plume.jar') + \
':' + os.path.join(CFI_dist, 'com.microsoft.z3.jar') + \
':' + os.path.join(CFI_dist, 'checker-framework-inference.jar')
if 'CLASSPATH' in os.environ:
cp += ':' + os.environ['CLASSPATH']
cmd = CFI_command + ['-classpath', cp,
'checkers.inference.InferenceLauncher',
'--solverArgs', args.solverArgs,
'--cfArgs', args.cfArgs,
'--checker', args.checker,
'--solver', args.solver,
'--mode', args.mode,
'--hacks=true',
'--targetclasspath', target_cp,
'--logLevel=WARNING',
'-afud', args.afuOutputDir]
cmd.extend(jc['java_files'])
common.run_cmd(cmd, args, 'infer')
def main(options):
server_version = options.server_version
scanner_version = options.scanner_version
src_of_the_plugins = options.plugins_folder
src_of_the_project = options.projects_folder
noa = options.number_of_attempts
wait = options.wait
system = platform.system()
dst = options.client_folder
print_log_files = options.print_log
common.mkdir(dst)
# 0, a) Try to build the plugins with 'build.py'
if system == 'Windows':
common.run_cmd('py', ['-3', 'build.py', '--all'])
elif system == 'Linux':
common.run_cmd('python3', ['tools/build.py', '--all'])
if options.init == True:
# 0, b) download sonar-server'
download_sq_server(server_version, dst)
# 1) download sonar-scanner
download_sq_scanner(scanner_version, system, dst)
# 2) unzip both server and scanner
src = os.path.join(dst, 'sonarqube-%s.zip' % server_version)
unzip(src, dst)
if 'Windows' == system:
src = os.path.join(dst, 'sonar-scanner-cli-%s-windows.zip' % scanner_version)
elif 'Linux' == system:
src = os.path.join(dst, 'sonar-scanner-cli-%s-linux.zip' % scanner_version)
unzip(src, dst)
# 3) copy the plugins into the server dir
path = [dst, 'sonarqube-%s' % server_version, 'extensions', 'plugins']
path = os.path.join(*path)
copy_all_files_from_folder(src_of_the_plugins, path)
# 4) start the server with the defult config
start_sq_server(server_version, system, dst)
# 5) Validate the server is started succesfully
# 6) Analyze the given project
sleep(60)
if validate_running_of_sq_server(server_version, noa, wait):
print('SonarQube started properly!')
else:
print(('SonarQube did not start in time (-noa=%s (number of attempts))' % (noa)))
if print_log_files:
print_log(server_version, dst)
exit(1)
def run_petablox(project):
with common.cd(common.get_project_dir(project)):
petablox_cmd = ['java',
'-cp', common.get_jar('petablox.jar'),
'-Dpetablox.reflect.kind=none',
'-Dpetablox.run.analyses=cipa-0cfa-dlog',
'petablox.project.Boot']
common.run_cmd(petablox_cmd)
def run_chicory(chicory_classpath, classes_to_include, main_class, out_dir):
chicory_command = ["java",
"-classpath", chicory_classpath,
"daikon.Chicory",
"--output_dir={}".format(out_dir),
main_class]
common.run_cmd(chicory_command)
def recompile_checker_framework():
if not os.environ.get('JAVA_HOME'):
print "ERROR in pa2checker.recompile_checker_framework(): Gradle will fail if your JAVA_HOME environment variable is unset. Please set it and try again."
sys.exit(0)
type_infer_tool_dir = os.path.join(common.TOOLS_DIR, "checker-framework-inference")
with common.cd(type_infer_tool_dir):
common.setup_checker_framework_env()
common.run_cmd(["gradle", "dist", "-i"], print_output=True)
def run(args, javac_commands, jars):
for jc in javac_commands:
pprint.pformat(jc)
class_path = jc['javac_switches']['classpath']
cmd = get_tool_command(args, class_path, jc['java_files'])
common.run_cmd(cmd, args, 'check')
def compile_test_cases(compile_classpath, test_class_directory, files_to_compile):
if not os.path.exists(test_class_directory):
os.mkdir(test_class_directory)
compile_command = ["javac", "-g",
"-classpath", compile_classpath,
"-d", test_class_directory]
compile_command.extend(files_to_compile)
common.run_cmd(compile_command)
def run():
dbcfgs = json.loads(dbcfgs_json)
TRAF_DIR = '%s-%s' % (dbcfgs['traf_basename'], dbcfgs['traf_version'])
# untar traf package
TRAF_PACKAGE_FILE = '/tmp/' + dbcfgs['traf_package'].split('/')[-1]
run_cmd('mkdir -p %s' % TRAF_DIR)
run_cmd('tar xf %s -C %s' % (TRAF_PACKAGE_FILE, TRAF_DIR))
print 'Trafodion package extracted successfully!'
def run():
dbcfgs = json.loads(dbcfgs_json)
traf_dirname = dbcfgs['traf_dirname']
# untar traf package, package comes from copy_files.py
traf_package_file = '/tmp/' + dbcfgs['traf_package'].split('/')[-1]
run_cmd('mkdir -p ~/%s' % traf_dirname)
run_cmd('tar xf %s -C ~/%s' % (traf_package_file, traf_dirname))
print 'Trafodion package extracted successfully!'
def run(args, javac_commands, jars):
# checker-framework javac.
javacheck = os.environ['JSR308']+"/checker-framework/checker/bin/javac"
checker_command = [javacheck, "-processor", args.checker]
for jc in javac_commands:
pprint.pformat(jc)
javac_switches = jc['javac_switches']
cp = javac_switches['classpath']
java_files = ' '.join(jc['java_files'])
cmd = checker_command + ["-classpath", cp, java_files]
common.run_cmd(cmd, args, 'check')
def usersguide():
if platform.system() == 'Windows':
common.run_cmd('py', ['-3', 'generatedoc.py', '-css', 'style\\SourceMeter.css', '-html'],
False, 'doc/usersguide')
else:
common.run_cmd('python3', ['generatedoc.py', '-css', 'style/SourceMeter.css', '-html'],
False, 'doc/usersguide')
try:
shutil.copy('doc/usersguide/results/UG.html', 'doc/UG.html')
shutil.copy('doc/UG.html', 'src/sonarqube-gui-plugin/src/main/resources/static/help/usersguide.html')
except OSError:
print('Cannot copy usersguide. Please check if it was generated successfully.')
def daikon_print_xml(args, classpath, out_dir):
daikon_command = ["java", "-Xmx4G",
"-classpath", classpath,
"daikon.PrintInvariants",
"--wrap_xml",
"--output", os.path.join(out_dir, "invariants.xml"),
os.path.join(out_dir, "invariants.gz")]
common.run_cmd(daikon_command, args, 'daikon')
js = jsoninv.generate_json_invariants(args, out_dir)
with open(os.path.join(out_dir, 'invariants.json'), 'w') as f:
json.dump(js, f)
def run_dyncomp(args, classpath, main_class, out_dir, selects=[], omits=[]):
dyncomp_command = [
"java", "-Xmx3G", "-classpath", classpath, "daikon.DynComp",
"--approximate-omitted-ppts", "--output-dir={}".format(out_dir)
]
if no_jdk:
dyncomp_command.append("--rt-file=none")
dyncomp_command.extend(selects)
dyncomp_command.extend(omits)
dyncomp_command.append(main_class)
common.run_cmd(dyncomp_command, args, 'dyncomp')
def analyze(scanner_version, project_folder, system, dst):
cmd = ''
cwd = os.getcwd()
scanner_location = ''
if system == 'Windows':
scanner_location = cwd + '\\' + dst + '\\sonar-scanner-%s-windows\\bin\\sonar-scanner.bat' % scanner_version
os.chdir(project_folder)
common.run_cmd('start', [scanner_location])
elif system == 'Linux':
cmd = cwd + '/' + dst + '/sonar-scanner-%s-linux/bin/sonar-scanner&' % scanner_version
os.chdir(project_folder)
common.run_cmd(cmd, [])
os.chdir('..')
def analyze(scanner_version, project_folder, system, dst):
cmd = ''
cwd = os.getcwd()
scanner_location = ''
if system == 'Windows':
scanner_location = cwd + '\\' + dst + '\\sonar-scanner-%s-windows\\bin\\sonar-scanner.bat' % scanner_version
os.chdir(project_folder)
common.run_cmd('start', [scanner_location])
elif system == 'Linux':
cmd = cwd + '/' + dst + '/sonar-scanner-%s-linux/bin/sonar-scanner&' % scanner_version
os.chdir(project_folder)
common.run_cmd(cmd, [])
os.chdir('..')
def process_run(run_number):
processing_time = time.time()
verbose_msg("> starting run", run_number)
run_cmd(f"bash runner{run_number}.sh")
aod_name = f"AODRun5.{run_number}.root"
if not os.path.isfile(aod_name):
msg(f"++ something went wrong for run {run_number}, no output AOD file {aod_name} found.",
f"Please check: 'AODRun5.{run_number}.log'",
color=bcolors.FAIL)
verbose_msg("< complete run", run_number)
processing_time = time.time() - processing_time
verbose_msg(f"-- took {processing_time} seconds --",
color=bcolors.BOKGREEN)
def run():
dbcfgs = json.loads(dbcfgs_json)
if 'APACHE' in dbcfgs['distro']:
modcfgs = ParseJson(MODCFG_FILE).load()
MOD_CFGS = modcfgs['MOD_CFGS']
hdfs_xml_file = dbcfgs['hdfs_xml_file']
hbase_xml_file = dbcfgs['hbase_xml_file']
hbasexml = ParseXML(hbase_xml_file)
for key, value in MOD_CFGS['hbase-site'].items():
hbasexml.add_property(key, value)
hbasexml.write_xml()
hdfsxml = ParseXML(hdfs_xml_file)
for key, value in MOD_CFGS['hdfs-site'].items():
hdfsxml.add_property(key, value)
hdfsxml.write_xml()
print 'Apache Hadoop modification completed'
first_node = dbcfgs['first_rsnode']
local_host = socket.gethostname()
if first_node in local_host:
hadoop_home = dbcfgs['hadoop_home']
hbase_home = dbcfgs['hbase_home']
# stop
run_cmd(hbase_home + '/bin/stop-hbase.sh')
run_cmd(hadoop_home + '/sbin/stop-dfs.sh')
# start
run_cmd(hadoop_home + '/sbin/start-dfs.sh')
run_cmd(hbase_home + '/bin/start-hbase.sh')
print 'Apache Hadoop restart completed'
else:
print 'no apache distribution found, skipping'
def run():
dbcfgs = json.loads(dbcfgs_json)
if 'APACHE' in dbcfgs['distro']:
modcfgs = ParseJson(MODCFG_FILE).load()
MOD_CFGS = modcfgs['MOD_CFGS']
hdfs_xml_file = dbcfgs['hdfs_xml_file']
hbase_xml_file = dbcfgs['hbase_xml_file']
hbasexml = ParseXML(hbase_xml_file)
for key, value in MOD_CFGS['hbase-site'].items():
hbasexml.add_property(key, value)
hbasexml.write_xml()
hdfsxml = ParseXML(hdfs_xml_file)
for key, value in MOD_CFGS['hdfs-site'].items():
hdfsxml.add_property(key, value)
hdfsxml.write_xml()
print 'Apache Hadoop modification completed'
first_node = dbcfgs['first_rsnode']
local_host = socket.gethostname()
if first_node in local_host:
hadoop_home = dbcfgs['hadoop_home']
hbase_home = dbcfgs['hbase_home']
# stop
run_cmd(hbase_home + '/bin/stop-hbase.sh')
run_cmd(hadoop_home + '/sbin/stop-dfs.sh')
# start
run_cmd(hadoop_home + '/sbin/start-dfs.sh')
run_cmd(hbase_home + '/bin/start-hbase.sh')
print 'Apache Hadoop restart completed'
else:
print 'no apache distribution found, skipping'
def run_chicory(args, classpath, main_class, out_dir, selects=[], omits=[]):
chicory_command = ["java", "-Xmx3G",
"-classpath", classpath,
"daikon.Chicory",
"--output_dir={}".format(out_dir)]
dc_out_path = os.path.join(out_dir, "RegressionTestDriver.decls-DynComp")
chicory_command.append("--comparability-file={}".format(dc_out_path))
chicory_command.extend(selects)
chicory_command.extend(omits)
chicory_command.append(main_class)
common.run_cmd(chicory_command, args, 'chicory')
def run_chicory(args, classpath, main_class, out_dir, selects=[], omits=[]):
chicory_command = ["java", "-Xmx3G",
"-classpath", classpath,
"daikon.Chicory",
"--output_dir={}".format(out_dir)]
dc_out_path = os.path.join(out_dir, "RegressionTestDriver.decls-DynComp")
chicory_command.append("--comparability-file={}".format(dc_out_path))
chicory_command.extend(selects)
chicory_command.extend(omits)
chicory_command.append(main_class)
common.run_cmd(chicory_command, args, 'chicory')
def generate_tests(randoop_classpath, class_list_file, test_src_dir, time_limit, output_limit):
randoop_command = ["java", "-ea",
"-classpath", randoop_classpath,
"randoop.main.Main", "gentests",
'--classlist={}'.format(class_list_file),
"--timelimit={}".format(time_limit),
"--junit-reflection-allowed=false",
"--silently-ignore-bad-class-names=true",
'--junit-output-dir={}'.format(test_src_dir)]
if output_limit and output_limit > 0:
randoop_command.append('--outputlimit={}'.format(output_limit))
common.run_cmd(randoop_command)
def run_dyncomp(args, classpath, main_class, out_dir, selects=[], omits=[]):
dyncomp_command = ["java", "-Xmx3G",
"-classpath", classpath,
"daikon.DynComp",
"--approximate-omitted-ppts",
"--output-dir={}".format(out_dir)]
if no_jdk:
dyncomp_command.append("--rt-file=none")
dyncomp_command.extend(selects)
dyncomp_command.extend(omits)
dyncomp_command.append(main_class)
common.run_cmd(dyncomp_command, args, 'dyncomp')
def build_image(repo_path, repo_name):
for dockerfile, image in yield_dockerfiles(repo_path, repo_name):
try:
build_cmd = [
'docker', 'build', '-t', image, '-f', dockerfile, repo_path
]
if os.environ.get('PULL_BASEIMAGES', '0') == '1':
build_cmd.append('--pull')
run_cmd(build_cmd)
except subprocess.CalledProcessError:
logging.error('Failed to build %s!' % dockerfile)
sys.exit(1)
time.sleep(1)
def run():
hdfs_bin = '/usr/bin/hdfs'
dbcfgs = json.loads(dbcfgs_json)
DISTRO = dbcfgs['distro']
if 'CDH' in DISTRO:
parcel_lib = '/opt/cloudera/parcels/CDH/lib/hbase/lib'
if os.path.exists(parcel_lib):
hdfs_bin = '/opt/cloudera/parcels/CDH/bin/hdfs'
elif 'APACHE' in DISTRO:
hdfs_bin = dbcfgs['hadoop_home'] + '/bin/hdfs'
traf_loc = '/user/trafodion'
traf_user = dbcfgs['traf_user']
hdfs_user = dbcfgs['hdfs_user']
hbase_user = dbcfgs['hbase_user']
run_cmd_as_user(hdfs_user, '%s dfsadmin -safemode wait' % hdfs_bin)
run_cmd_as_user(
hdfs_user,
'%s dfs -mkdir -p %s/{trafodion_backups,bulkload,lobs} /bulkload /lobs /hbase/archive /hbase-staging'
% (hdfs_bin, traf_loc))
run_cmd_as_user(
hdfs_user, '%s dfs -chown -R %s:%s /hbase/archive /hbase-staging' %
(hdfs_bin, hbase_user, hbase_user))
run_cmd_as_user(
hdfs_user,
'%s dfs -chown -R %s:%s %s/{trafodion_backups,bulkload,lobs} /bulkload /lobs'
% (hdfs_bin, traf_user, traf_user, traf_loc))
run_cmd_as_user(
hdfs_user, '%s dfs -setfacl -R -m user:%s:rwx /hbase/archive' %
(hdfs_bin, traf_user))
run_cmd_as_user(
hdfs_user, '%s dfs -setfacl -R -m default:user:%s:rwx /hbase/archive' %
(hdfs_bin, traf_user))
run_cmd_as_user(
hdfs_user, '%s dfs -setfacl -R -m mask::rwx /hbase/archive' % hdfs_bin)
# Grant all privileges to the Trafodion principal in HBase
if dbcfgs['secure_hadoop'] == 'Y':
run_cmd(
'grant "%s", "RWXC" | sudo -u %s hbase shell > /tmp/hbase_shell.out'
% (traf_user, hbase_user))
has_err = cmd_output('grep -c ERROR /tmp/hbase_shell.out')
if int(has_err):
err('Failed to grant HBase privileges to %s' % traf_user)
run_cmd('rm /tmp/hbase_shell.out')
def __download_program(program_url):
filename = os.path.basename(program_url)
save_path = '%s/%s' % (ypc_bin, filename)
logger.info('Downloading analysis program to path: %s' % save_path)
ret = common.run_cmd('''wget -O %s %s''' % (save_path, program_url))
logger.info(ret)
return save_path
def run_minimap2(reference):
log.info('run minimap2')
fastq = '%s.hq.fastq' % prefix
sam_sort = "%s.hq_isoforms.fastq.sorted.sam" % prefix
cmd = 'shifter --image=robegan21/minimap2:2.10 minimap2 -t %s -ax splice -uf --secondary=no -C5 %s %s | sort -k 3,3 -k 4,4n > %s ' % (n_threads, reference, fastq, sam_sort)
std_out, std_err, exit_code = run_cmd(cmd, log)
log.info("minimap2 complete")
def run_daikon(args, classpath, out_dir, invcounts):
daikon_command = ["java", "-Xmx4G",
"-classpath", classpath,
"daikon.Daikon",
"-o", os.path.join(out_dir, "invariants.gz")]
if invcounts:
daikon_command.append("--config_option")
daikon_command.append("daikon.Daikon.calc_possible_invs=true")
if no_ternary:
daikon_command.append("--config_option")
daikon_command.append("daikon.inv.ternary.threeScalar.LinearTernary.enabled=false")
daikon_command.append("--config_option")
daikon_command.append("daikon.inv.ternary.threeScalar.LinearTernaryFloat.enabled=false")
daikon_command.append(os.path.join(out_dir, "RegressionTestDriver.dtrace.gz"))
common.run_cmd(daikon_command, args, 'daikon')
def proceed(handle_exit=True):
msg(f"Downloading '{toget}'", color=bcolors.OKGREEN)
print_now()
if Version == 0:
cpycmd = "alien_cp -v {} file:{}".format(toget, todir)
else:
cpycmd = "alien_cp -v {} file://{}".format(toget, todir)
verbose_msg("Running command", cpycmd)
if handle_exit:
try:
run_cmd(cpycmd)
except KeyboardInterrupt:
return False
else:
run_cmd(cpycmd)
return True
def run_daikon(args, classpath, out_dir, invcounts):
daikon_command = ["java", "-Xmx4G",
"-classpath", classpath,
"daikon.Daikon",
"-o", os.path.join(out_dir, "invariants.gz")]
if invcounts:
daikon_command.append("--config_option")
daikon_command.append("daikon.Daikon.calc_possible_invs=true")
if no_ternary:
daikon_command.append("--config_option")
daikon_command.append("daikon.inv.ternary.threeScalar.LinearTernary.enabled=false")
daikon_command.append("--config_option")
daikon_command.append("daikon.inv.ternary.threeScalar.LinearTernaryFloat.enabled=false")
daikon_command.append(os.path.join(out_dir, "RegressionTestDriver.dtrace.gz"))
common.run_cmd(daikon_command, args, 'daikon')
def run():
hdfs_bin = DEF_HDFS_BIN
dbcfgs = json.loads(dbcfgs_json)
distro = dbcfgs['distro']
if 'CDH' in distro:
parcel_lib = PARCEL_HBASE_LIB
if os.path.exists(parcel_lib): hdfs_bin = PARCEL_HDFS_BIN
elif 'APACHE' in distro:
hdfs_bin = dbcfgs['hadoop_home'] + '/bin/hdfs'
traf_loc = '/user/trafodion'
traf_user = dbcfgs['traf_user']
hdfs_user = dbcfgs['hdfs_user']
hbase_user = dbcfgs['hbase_user']
run_cmd_as_user(hdfs_user, '%s dfsadmin -safemode wait' % hdfs_bin)
run_cmd_as_user(
hdfs_user,
'%s dfs -mkdir -p %s/{trafodion_backups,bulkload,lobs} /bulkload /lobs /hbase/archive /hbase-staging'
% (hdfs_bin, traf_loc))
run_cmd_as_user(
hdfs_user, '%s dfs -chown -R %s:%s /hbase/archive /hbase-staging' %
(hdfs_bin, hbase_user, hbase_user))
run_cmd_as_user(
hdfs_user,
'%s dfs -chown -R %s:%s %s %s/{trafodion_backups,bulkload,lobs} /bulkload /lobs'
% (hdfs_bin, traf_user, traf_user, traf_loc, traf_loc))
run_cmd_as_user(
hdfs_user, '%s dfs -setfacl -R -m user:%s:rwx /hbase/archive' %
(hdfs_bin, traf_user))
run_cmd_as_user(
hdfs_user, '%s dfs -setfacl -R -m default:user:%s:rwx /hbase/archive' %
(hdfs_bin, traf_user))
run_cmd_as_user(
hdfs_user, '%s dfs -setfacl -R -m mask::rwx /hbase/archive' % hdfs_bin)
# Grant all privileges to the Trafodion principal in HBase
if dbcfgs['secure_hadoop'] == 'Y':
run_cmd(
'echo "grant \'%s\', \'RWXC\'" | %s su - %s -s /bin/bash -c "hbase shell" > /tmp/hbase_shell.out'
% (traf_user, get_sudo_prefix(), hbase_user))
has_err = cmd_output('grep -c ERROR /tmp/hbase_shell.out')
if int(has_err):
err('Failed to grant HBase privileges to %s' % traf_user)
run_cmd('rm /tmp/hbase_shell.out')
def run():
dbcfgs = json.loads(dbcfgs_json)
nodes = dbcfgs['node_list'].split(',')
scratch_locs = dbcfgs['scratch_locs'].split(',')
# this script is running by trafodion user, so get sqroot from env
traf_conf = os.environ['TRAF_CONF']
if traf_conf == '': err('TRAF_CONF var is empty')
sqconfig_file = traf_conf + '/sqconfig'
traf_var = os.environ['TRAF_VAR']
if traf_var == '': err('TRAF_VAR var is empty')
sqconfig_db_file = traf_var + '/sqconfig.db'
# If the configuration database file is not yet created,
# build the 'sqconfig' file with the nodes specified and compile it.
if not os.path.exists(sqconfig_db_file):
core, processor = run_cmd("lscpu|grep -E '(^CPU\(s\)|^Socket\(s\))'|awk '{print $2}'").split('\n')[:2]
core = int(core)-1 if int(core) <= 256 else 255
lines = ['begin node\n']
for node_id, node in enumerate(nodes):
line = 'node-id=%s;node-name=%s;cores=0-%d;processors=%s;roles=connection,aggregation,storage\n' % (node_id, node, core, processor)
lines.append(line)
lines.append('end node\n')
lines.append('\n')
lines.append('begin overflow\n')
for scratch_loc in scratch_locs:
line = 'hdd %s\n' % scratch_loc
lines.append(line)
lines.append('end overflow\n')
# write out the node section
with open(sqconfig_file, 'w') as f:
f.writelines(lines)
print 'sqconfig generated successfully!'
run_cmd('sqgen')
print 'sqgen ran successfully!'
else:
print 'Using existing configuration (%s)' % sqconfig_file
def run_o2_analysis(tmp_script_name,
remove_tmp_script=False,
explore_bad_files=False,
time_it=True):
global number_of_runs
verbose_msg("> starting run with", tmp_script_name)
cmd = f"bash {tmp_script_name}"
if do_bash_script:
with open("parallelbash.sh", "a") as fout:
with open("parallelbash.sh", "r") as fin:
lastline = fin.readlines()[-1]
if lastline.startswith("#"):
lastline = int(lastline.strip("#"))
else:
lastline = 0
fout.write(f"echo Running {lastline}\n")
fout.write(f"{cmd} &\n")
lastline += 1
if lastline % (bash_parallel_jobs + 1) == 0:
fout.write(f"wait\n")
fout.write(f"\n#{lastline}\n")
return
if explore_bad_files:
if run_cmd(cmd, check_status=True, throw_fatal=False,
time_it=time_it) == False:
list_name = os.listdir(os.path.dirname(tmp_script_name))
for i in list_name:
if "ListForRun5Analysis" in i:
list_name = i
break
if type(list_name) != list:
with open(
os.path.join(os.path.dirname(tmp_script_name),
list_name)) as f:
list_name = []
for i in f:
list_name.append(i)
warning_msg("Issue when running", tmp_script_name, "with",
list_name)
else:
run_cmd(cmd, log_file=f"{tmp_script_name}.log", time_it=time_it)
if remove_tmp_script:
os.remove(tmp_script_name)
verbose_msg("< end run with", tmp_script_name)
return tmp_script_name
def compilable(args, java_file, classpath):
"""
Test if a given java_file is compilable.
Return True if compilable, False otherwise.
"""
compile_cmd = ['javac', '-classpath', classpath, java_file]
compile_status = common.run_cmd(compile_cmd, args, None)
return compile_status['return_code'] == 0
def _wait_ping_response(device, timeout):
start_time = int(time.time())
while (int(time.time()) < start_time + timeout):
rv = common.run_cmd(["ping", "-c", "1", device.ip], raise_on_err=False)
if (rv.retval == 0):
return
raise NoConnectionError
def get_redis_info(host, port):
# pdb.set_trace()
cmd = "redis-cli -h %s -p %s info" % (host, port)
k, v = run_cmd(cmd)
if k:
return change_dict(v)
else:
print "get info form redis error host %s ,prot %s" % (host, port)
def running_tool(args, file_list, target_classpath, log_file_path=None):
"""
Runnning the debugged tool on given file_list.
Return the tool execution return code.
"""
tool_cmd = SUPPORT_TOOLS[args.debuggedTool].get_tool_command(args, target_classpath, file_list)
execute_status = common.run_cmd(tool_cmd, args, log_file_path)
return execute_status
def run_sqanti(reference,ann):
log.info("sqanti ...")
if not os.path.isdir("sqanti_out"):
os.makedirs("sqanti_out")
#cmd = "shifter --image=mjblow/sqanti:v1 sqanti_qc.py -d sqanti_out -g {0}.isoform.collapsed.gtf {1} {2} > sqanti.log 2>&1 ".format(prefix, ann, reference)
cmd = "shifter --image=mjblow/sqanti:v1 sqanti_qc.py -d sqanti_out -g {0}.isoform.collapsed.filtered.gtf {1} {2} > sqanti.log 2>&1 ".format(prefix, ann, reference)
std_out, std_err, exit_code = run_cmd(cmd, log)
log.info("sqanti complete")
def run():
dbcfgs = json.loads(dbcfgs_json)
nodes = dbcfgs['node_list'].split(',')
scratch_locs = dbcfgs['scratch_locs'].split(',')
# this script is running by trafodion user, so get sqroot from env
traf_home = os.environ['TRAF_HOME']
if traf_home == '': err('TRAF_HOME var is empty')
sqconfig_file = traf_home + '/sql/scripts/sqconfig'
sqconfig_persist_file = traf_home + '/sql/scripts/sqconfig.persist'
core, processor = run_cmd(
"lscpu|grep -E '(^CPU\(s\)|^Socket\(s\))'|awk '{print $2}'").split(
'\n')[:2]
core = int(core) - 1 if int(core) <= 256 else 255
lines = ['begin node\n']
if len(nodes) == 1:
lines.append('_virtualnodes 2\n')
else:
for node_id, node in enumerate(nodes):
line = 'node-id=%s;node-name=%s;cores=0-%d;processors=%s;roles=connection,aggregation,storage\n' % (
node_id, node, core, processor)
lines.append(line)
lines.append('end node\n')
lines.append('\n')
lines.append('begin overflow\n')
for scratch_loc in scratch_locs:
line = 'hdd %s\n' % scratch_loc
lines.append(line)
lines.append('end overflow\n')
# write out the node section
with open(sqconfig_file, 'w') as f:
f.writelines(lines)
print 'sqconfig generated successfully!'
run_cmd('sqgen')
print 'sqgen ran successfully!'
def _validate_plugins():
app_name = app_metadata['APP_NAME']
build_path = os.path.abspath(build_dir)
pluginval_bin_path = None
plugin_paths = []
if platform_name == 'Darwin':
pluginval_bin_path = os.path.join(pluginval_path, 'bin', 'mac',
'pluginval.app', 'Contents', 'MacOS',
'pluginval')
for c in build_configs:
plugin_paths += [
os.path.join(build_path, c,
'{app_name}.component'.format(app_name=app_name)),
os.path.join(build_path, c,
'{app_name}.vst'.format(app_name=app_name)),
os.path.join(build_path, c,
'{app_name}.vst3'.format(app_name=app_name)),
]
elif platform_name == 'Windows':
pluginval_bin_path = os.path.join(pluginval_path, 'bin', 'windows',
'pluginval.exe')
for c in build_configs:
arch_x64 = build_archs_win[1]
plugin_paths += [
os.path.join(build_path, arch_x64, c, 'VST',
'{app_name}.dll'.format(app_name=app_name)),
os.path.join(build_path, arch_x64, c, 'VST3',
'{app_name}.vst3'.format(app_name=app_name)),
]
elif platform_name == 'Linux':
pluginval_bin_path = os.path.join(pluginval_path, 'bin', 'linux',
'pluginval')
plugin_paths += [
os.path.join(build_path,
'lib{app_name}.so'.format(app_name=app_name))
]
for p in plugin_paths:
run_cmd(
'{pluginval_path} --strictness-level 5 --validate "{plugin_path}"'.
format(pluginval_path=pluginval_bin_path, plugin_path=p))
def run():
""" install Trafodion dependencies """
dbcfgs = json.loads(dbcfgs_json)
node_list = dbcfgs['node_list'].split(',')
offline = True if dbcfgs['offline_mode'] == 'Y' else False
if offline:
repo_content = LOCAL_REPO_PTR % (dbcfgs['repo_ip'],
dbcfgs['repo_http_port'])
with open(REPO_FILE, 'w') as f:
f.write(repo_content)
if not offline and not os.path.exists(EPEL_FILE):
run_cmd('yum install -y epel-release')
package_list = [
'apr',
'apr-util',
'expect',
'gzip',
'libiodbc-devel',
'lzo',
'lzop',
'pdsh', # epel
'perl-DBD-SQLite',
'perl-Params-Validate',
'perl-Time-HiRes',
'protobuf', # epel
'sqlite',
'snappy',
'unixODBC-devel',
'unzip'
]
if dbcfgs['ldap_security'].upper() == 'Y':
package_list += ['openldap-clients']
all_pkg_list = run_cmd('rpm -qa')
for pkg in package_list:
if pkg in all_pkg_list:
print 'Package %s had already been installed' % pkg
else:
print 'Installing %s ...' % pkg
if offline:
run_cmd(
'yum install -y --disablerepo=\* --enablerepo=traflocal %s'
% pkg)
else:
run_cmd('yum install -y %s' % pkg)
# pdsh should not exist on single node
if len(node_list) == 1 and not dbcfgs.has_key(
'traf_shadow'): # means we are running in adding node mode
cmd_output('yum remove -y pdsh')
# remove temp repo file
if offline: os.remove(REPO_FILE)
def get_hdfs_list(path):
return_dict = dict()
cmd = "hadoop fs -du -s %s" % path
output = run_cmd(cmd)
if output:
return_dict[output.split()[1]] = output.split()[0]
else:
return_dict[path] = 0
return return_dict
def start_sq_server(version, system, dst):
cmd = ''
cwd = os.getcwd()
if system == 'Windows':
sonar_location = cwd + '\\' + dst + '\\' + 'sonarqube-%s\\bin\\windows-x86-64\\StartSonar.bat' % version
common.run_cmd('start', [sonar_location])
elif system == 'Linux':
sonar_location = cwd + '/' + dst + '/sonarqube-%s' % version
temp = sonar_location
for root, dirs, files in os.walk(temp):
for file in files:
file_path = os.path.join(root, file)
os.chmod(file_path, 0o744)
cmd = os.path.join(sonar_location, 'bin/linux-x86-64/sonar.sh')
common.run_cmd(cmd, ['start', '&'])
print('Starting SQ server...')
def start_sq_server(version, system, dst):
cmd = ''
cwd = os.getcwd()
if system == 'Windows':
sonar_location = cwd + '\\' + dst + '\\' + 'sonarqube-%s\\bin\\windows-x86-64\\StartSonar.bat' % version
common.run_cmd('start', [sonar_location])
elif system == 'Linux':
sonar_location = cwd + '/' + dst + '/sonarqube-%s' % version
temp = sonar_location
for root, dirs, files in os.walk(temp):
for file in files:
file_path = os.path.join(root, file)
os.chmod(file_path, 0o744)
cmd = os.path.join(sonar_location, 'bin/linux-x86-64/sonar.sh')
common.run_cmd(cmd, ['start', '&'])
print('Starting SQ server...')
def generate_tests(args, classpath, class_list_file, test_src_dir, junit_after_path, time_limit=200, output_limit=4000):
randoop_command = ["java", "-ea",
"-classpath", classpath,
"randoop.main.Main", "gentests",
'--classlist={}'.format(class_list_file),
"--timelimit={}".format(time_limit),
"--junit-reflection-allowed=false",
"--ignore-flaky-tests=true",
"--timeout=5",
"--silently-ignore-bad-class-names=true",
'--junit-output-dir={}'.format(test_src_dir)]
if junit_after_path:
randoop_command.append("--junit-after-all={}".format(junit_after_path))
if output_limit and output_limit > 0:
randoop_command.append('--outputlimit={}'.format(output_limit))
common.run_cmd(randoop_command, args, 'randoop')
def get_hdfs_list(path):
return_dict = dict()
cmd = "hadoop fs -ls -R %s" % path
output = run_cmd(cmd)
for line in output.split("\n"):
if len(line.split()) == 8:
filename = line.split()[7]
size = line.split()[4]
return_dict[filename] = size
return return_dict
def main():
with common.cd(WORKING_DIR):
test_dtrace = "test.dtrace.gz"
test_inv_name = "TestInvariant"
ontology_to_daikon.create_daikon_invariant("README.md", test_inv_name)
cmd = ["javac", "-classpath", daikon_jar + ":.", test_inv_name + ".java"]
common.run_cmd(cmd, print_output=True)
print ("Finding program points")
ppts = find_ppts_that_establish_inv(test_dtrace, WORKING_DIR, test_inv_name)
print ("deleting temp files")
os.remove(test_inv_name + ".class")
os.remove(test_inv_name + ".java")
os.remove("test.inv.gz")
# output = run_daikon_on_dtrace_file(test_dtrace, checked_invariant="daikon.inv.unary.sequence.EltwiseIntLessThan")
# print output
# ppts = find_ppts_that_establish_inv_in_daikon_output(output, " sorted by ")
print ("Methods that establish FirstMuseInvariant:")
for ppt in ppts:
print ppt
def merge_aod(in_path="",
out_path="./",
input_file="AO2D.root",
must_have="ctf",
bunch_size=50,
skip_already_existing=True):
in_path = os.path.normpath(in_path)
out_path = os.path.normpath(out_path)
file_list = []
for root, dirs, files in os.walk(in_path):
for file in files:
if file == input_file:
to_merge = os.path.abspath(os.path.join(root, file))
print(to_merge)
if must_have is not None and must_have in to_merge:
file_list.append(to_merge)
verbose_msg("Found", len(file_list), "files called", input_file)
# Divide it in bunches
file_list = [
file_list[i:i + bunch_size]
for i in range(0, len(file_list), bunch_size)
]
for i in enumerate(file_list):
bunch_size = 0
with open("inputfile.txt", "w") as f:
for j in i[1]:
f.write(f"{j}\n")
bunch_size += os.path.getsize(j)
out_aod = os.path.join(out_path, f"AO2D_{i[0]}.root")
verbose_msg("Merging bunch of", len(i[1]), "files. I.e.",
bunch_size * 1e-6, "MB")
if skip_already_existing and os.path.isfile(out_aod):
verbose_msg(out_aod, "already existing, skipping")
continue
tmp_aod = os.path.join(out_path, "MergedAOD.root")
run_cmd(
f"o2-aod-merger --input inputfile.txt --output {tmp_aod} --skip-non-existing-files",
comment=f"Merging AODs into {out_aod}")
os.rename(tmp_aod, out_aod)
merged_size = os.path.getsize(out_aod)
msg("Produced a merged file of", merged_size * 1e-6, "MB from",
bunch_size * 1e-6, "MB, compression:", merged_size / bunch_size)
def run():
dbcfgs = json.loads(dbcfgs_json)
nodes = dbcfgs['node_list'].split(',')
scratch_locs = dbcfgs['scratch_locs'].split(',')
# this script is running by trafodion user, so get sqroot from env
traf_home = os.environ['TRAF_HOME']
if traf_home == '': err('TRAF_HOME var is empty')
sqconfig_file = traf_home + '/sql/scripts/sqconfig'
sqconfig_persist_file = traf_home + '/sql/scripts/sqconfig.persist'
core, processor = run_cmd("lscpu|grep -E '(^CPU\(s\)|^Socket\(s\))'|awk '{print $2}'").split('\n')[:2]
core = int(core)-1 if int(core) <= 256 else 255
lines = ['begin node\n']
if len(nodes) == 1:
lines.append('_virtualnodes 2\n')
else:
for node_id, node in enumerate(nodes):
line = 'node-id=%s;node-name=%s;cores=0-%d;processors=%s;roles=connection,aggregation,storage\n' % (node_id, node, core, processor)
lines.append(line)
lines.append('end node\n')
lines.append('\n')
lines.append('begin overflow\n')
for scratch_loc in scratch_locs:
line = 'hdd %s\n' % scratch_loc
lines.append(line)
lines.append('end overflow\n')
# write out the node section
with open(sqconfig_file, 'w') as f:
f.writelines(lines)
print 'sqconfig generated successfully!'
run_cmd('sqgen')
print 'sqgen ran successfully!'
def _build_pluginval():
root_path = os.getcwd()
pluginval_build_path = os.path.join(pluginval_path, 'install')
os.chdir(pluginval_build_path)
root_path_rel = os.path.relpath(root_path)
if platform_name == 'Darwin':
plugin_build_bin_name = 'mac_build'
elif platform_name == 'Windows':
plugin_build_bin_name = 'windows_build.bat'
elif platform_name == 'Linux':
plugin_build_bin_name = 'linux_build'
pluginval_build_bin_path = os.path.join(pluginval_build_path,
plugin_build_bin_name)
run_cmd(pluginval_build_bin_path)
os.chdir(root_path_rel)
def set_config(config_file, config, value):
config = config.strip()
value = value.strip()
config_string = f"{config} {value}"
run_cmd("sed -i -e \""
f"s/{config} .*$/{config_string}"
"\" " + config_file)
# Checking that the file has the correct configuration
with open(config_file) as f:
has_it = False
config_string = config_string.replace("\\", "").strip("/")
for lineno, line in enumerate(f):
if line.strip() == config_string:
verbose_msg(f"Found config string '{config_string}'",
f"at line #{lineno} '{line.strip()}'")
has_it = True
break
if not has_it:
fatal_msg("Configuration file", config_file,
f"does not have config string '{config_string}'")
def run_daikon_on_dtrace_file(dtrace_file,
classpath=daikon_jar,
checked_invariant=None):
cmd = ["java", "-classpath", classpath, "daikon.DaikonSimple", dtrace_file]
if checked_invariant:
cmd += [
"--disable-all-invariants", "--user-defined-invariant",
checked_invariant
]
cmd += ["--config_option", "daikon.Daikon.undo_opts=true"]
return common.run_cmd(cmd, print_output=True)['output']
def run(args, javac_commands, jars):
print os.environ
idx = 0
for jc in javac_commands:
jaif_file = "logs/infer_result_{}.jaif".format(idx)
cmd = get_tool_command(args, jc['javac_switches']['classpath'], jc['java_files'], jaif_file)
status = common.run_cmd(cmd, args, 'infer')
if args.crashExit and not status['return_code'] == 0:
print "----- CF Inference/Typecheck crashed! Terminates DLJC. -----"
sys.exit(1)
idx += 1
