def create_tar():
# tar the CMSSW framework
if "CMSSW946.tgz" in os.listdir(lpcHomeDir):
print("[ ] Deleting existing CMSSW946.tgz...")
sys_call("rm {}{}".format(lpcHomeDir, "CMSSW946.tgz"), shell=True)
print "[ ] Creating TAR file"
if sys_call(
"tar -C ~/nobackup/ -zcvf CMSSW946.tgz --exclude=\'{}\' --exclude=\'{}\' --exclude=\'{}\' --exclude=\'{}\' --exclude=\'{}\' --exclude=\'{}\' {}"
.format(
"CMSSW_9_4_6_patch1/src/TTTT_TMVA_DNN/" +
varsList.step2Sample2017,
"CMSSW_9_4_6_patch1/src/TTTT_TMVA_DNN/" +
varsList.step2Sample2018,
"CMSSW_9_4_6_patch1/src/TTTT_TMVA_DNN/condor_log*",
"CMSSW_9_4_6_patch1/src/TTTT_TMVA_DNN/dataset_*",
"CMSSW_9_4_6_patch1/src/TTTT_TMVA_DNN/CMSSW946.tgz",
"CMSSW_9_4_6_patch1/src/TTTT_TMVA_DNN/.git/*",
"CMSSW_9_4_6_patch1/"),
shell=True) == 1:
print "[ERR] Creating TAR file failed!"
sys.exit(1)
print("[ ] Transferring CMSSW946.tgz to EOS...")
if sys_call("xrdcp -f CMSSW946.tgz root://cmseos.fnal.gov//store/user/{}".
format(varsList.eosUserName),
shell=True) == 1:
print "[ERR] EOS trandfer of TAR file failed!"
sys.exit(1)
print("[OK ] EOS transfer complete")
def convert_to_obj(top_folder, out_top_dir, do_nothing_mlx, regex='.off$'):
input_files = files_in_subdirs(top_folder, regex)
copy_folder_structure(top_folder, out_top_dir)
for in_f in input_files:
out_f = osp.join(out_top_dir, in_f.replace(top_folder, ''))
out_f = out_f[:-len('off')] # TODO -> works only for .off
out_f = out_f + 'obj'
sys_call([mesh_lab_binary, '-i', in_f, '-o', out_f, '-s', do_nothing_mlx])
def compile_splitter():
# compile the sample splitting c++ script
if "splitROOT.out" in os.listdir(os.getcwd() + "/setup/"):
sys_call("rm {}/setup/splitROOT.out".format(os.getcwd()), shell=True)
print "[ ] Compiling splitROOT.cpp..."
if sys_call(
"g++ `root-config --cflags` `root-config --libs` -o ./setup/splitROOT.out ./setup/splitROOT.cpp",
shell=True) == 0:
print "[OK ] Compiled splitROOT.CPP"
else:
print "[ERR] Compiling splitROOT.cpp failed!"
sys.exit(1)
def eos_transfer(years=["2017", "2018"]):
#Transfer data to EOS
for year in years:
d = (varsList.step2Sample2017
if year == "2017" else varsList.step2Sample2018)
#Make EOS dir
sys_call("eosmkdir root://cmseos.fnal.gov//store/user/{}".format(
varsList.eosUserName + "/" + d + "/"),
shell=True)
# transfer one of the split samples to EOS
print "[ ] Transferring to EOS..."
if sys_call(
"xrdcp {}*split0.root root://cmseos.fnal.gov//store/user/{}".
format("./" + d + "/", varsList.eosUserName + "/" + d + "/"),
shell=True) == 1:
print "[ERR] EOS transfer of split ROOT file failed!"
sys.exit(1)
print("[OK ] EOS transfer complete")
def server_main():
# Prepare
log = get_logger("Main", [stdout])
os_type = running_os()
if os_type is None:
show_message(log, logging.ERROR, "OS type is unknown")
return
error_count = 0
error_flag = Event()
exit_flag = Event()
index = CAM_INDEX[os_type]
# Main loop
detection = Detection(index, external_function)
while not detection.stop_flag:
safe_call(detection.process, log, error_flag, exit_flag, True)
if exit_flag.is_set():
break
if error_flag.is_set():
if error_count < ERROR_COUNT_MAX:
error_flag.clear()
error_count += 1
else:
break
safe_call(detection.close, log, error_flag, exit_flag, False)
if error_flag.is_set():
show_message(log, logging.ERROR, "Error occurred %s times in loop" % error_count)
else:
log.info("Finished correctly")
if os_type is SYSTEMS.WINDOWS:
# This fixes some hang when closing with exit(), instead we call OS to kill ourself, how nice and clean /irony off
sys_call("taskkill /F /PID %i" % getpid(), shell=True)
def clone_repos(repos_list=None):
"""
Clones all the repos in the provided list
"""
if repos_list:
num_repos = len(repos_list)
confirm = input("Found {0} repo(s). Clone? [y/n] ".format(num_repos))
if re.search('(yes|Yes|YES|Y|y)', confirm):
returns = [sys_call(["git", "clone", repo]) for repo in repos_list]
return sum(returns)
else:
print("Cloning cancelled.")
return 0
else:
return 1
def split_root(years=["2017", "2018"]):
# run the root file splitting script
for year in years:
print("[ ] Splitting {} ROOT files...".format(year))
d = lpcHomeDir + (varsList.step2Sample2017
if year == "2017" else varsList.step2Sample2018)
for sample in (varsList.sig2017 + varsList.bkg2017 if year == "2017"
else varsList.sig2018 + varsList.bkg2018):
if sys_call(
"./setup/splitROOT.out {} {} {} {}".format(
d, # location of sample to split
d, # destination of split sample(s)
sample, # sample to split
3 # number of files to split into
),
shell=True) == 1:
print "[ERR] Splitting " + sample + " failed!"
print "[OK ] Finished splitting ROOT files"
def apply_script_to_files(top_folder, out_top_dir, script_file, regex='.off$'):
input_files = files_in_subdirs(top_folder, regex)
copy_folder_structure(top_folder, out_top_dir)
for in_f in input_files:
out_f = osp.join(out_top_dir, in_f.replace(top_folder, ''))
sys_call([mesh_lab_binary, '-i', in_f, '-o', out_f, '-s', script_file])
def download_samples(years=["2017", "2018"]):
#Transfer the samples from the BRUX server
#Returns True if all were downloaded successfully
brux_auth()
print "[ ] Transferring samples from BRUX for years: " + str(years)
# setup parameters that will differ between 2017 and 2018 samples
step2Sample = None
inputDirBRUX = None
inpu9tDirLPC = None
inputDirEOS = None
samples = None
samples_0 = None
for year in years:
if year == "2017":
step2Sample = varsList.step2Sample2017
inputDirBRUX = varsList.inputDirBRUX2017
inputDirLPC = varsList.inputDirLPC2017
inputDirEOS = varsList.inputDirEOS2017
samples = varsList.sig2017 + varsList.bkg2017
samples_0 = varsList.sig2017_0 + varsList.bkg2017_0
elif year == "2018":
step2Sample = varsList.step2Sample2018
inputDirBRUX = varsList.inputDirBRUX2018
inputDirLPC = varsList.inputDirLPC2018
inputDirEOS = varsList.inputDirEOS2018
samples = varsList.sig2018 + varsList.bkg2018
samples_0 = varsList.sig2018_0 + varsList.bkg2018_0
# output sample origin and destination when running setup
if step2Sample not in os.listdir(lpcHomeDir):
sys_call("mkdir {}{}".format(lpcHomeDir, step2Sample), shell=True)
print("[ ] Transferring files from {} to {}...".format(
varsList.bruxUserName + "@brux.hep.brown.edu:" + inputDirBRUX,
lpcHomeDir + step2Sample))
for sample in samples:
if sample not in os.listdir("{}{}".format(lpcHomeDir,
step2Sample)):
print("[ ] Transferring {}...".format(sample))
child = pexpect.spawn(
"scp -r {}@brux.hep.brown.edu:{}{} {}{}".format(
varsList.bruxUserName, inputDirBRUX, sample,
lpcHomeDir, step2Sample))
opt = 1
while opt == 1:
# Handle both the password prompt and the connection prompt.
opt = child.expect([
varsList.bruxUserName +
"@brux.hep.brown.edu's password: "******"Are you sure you want to continue connecting (yes/no)? "
])
if opt == 1:
# Confirm connection
child.sendline("yes")
# Send password
child.sendline(brux_pwd)
child.interact()
print "[OK ] Done."
else:
print(
"[OK ] {} has been transferred. Proceeding to next sample..."
.format(sample))
if check_samples(samples, lpcHomeDir + step2Sample) == False:
return False
print "[OK ] All samples downloaded"
return True
def render_views_with_fythumb(mesh_file, output_dir):
sys_call([fythumb_bin, '-i', mesh_file, '-o', output_dir, '-r'])
def pixels_to_triangles(pixel_file, off_file, camera_vertex, camera_twist, output_dir, out_file_name):
sys_call([Back_Tracer.fythumb_bin, '-i', off_file, '-s', pixel_file, '-o', output_dir,
'-v', camera_vertex, '-t', camera_twist, '-f', out_file_name])
def fythumb_compute_views_of_shape(mesh_file, output_dir):
sys_call([Back_Tracer.fythumb_bin, '-i', mesh_file, '-o', output_dir, '-r'])
