def runStoreAndTrain(self, data_dir, model_dir, lr, num_epochs, sum_dir):
runMaybeEnvInstallTF()
currentPath = os.path.dirname(os.path.abspath(__file__))
env_dir = os.path.join(currentPath, "..", "miniconda2")
pipeline_train_py = os.path.join(TRAFIC_LIB_DIR, "PipelineTrain.py")
cmd_py = str(pipeline_train_py) + ' --data_dir ' + str(data_dir) + ' --multiclass --summary_dir ' + str(sum_dir)+ ' --checkpoint_dir ' + str(model_dir) + ' --lr ' + str(lr) + ' --num_epochs ' + str(num_epochs)
cmd_virtenv = 'ENV_DIR="'+str(env_dir)+'";'
cmd_virtenv = cmd_virtenv + 'export PYTHONPATH=$ENV_DIR/envs/env_trafic/lib/python2.7/site-packages:$ENV_DIR/lib/:$ENV_DIR/lib/python2.7/lib-dynload/:$ENV_DIR/lib/python2.7/:$ENV_DIR/lib/python2.7/site-packages/:$PYTHONPATH;'
# cmd_virtenv = cmd_virtenv + 'export PYTHONHOME=$ENV_DIR/bin/:$PYTHONHOME;'
cmd_virtenv = cmd_virtenv + 'export PATH=$ENV_DIR/bin/:$PATH;'
cmd_virtenv = cmd_virtenv + 'source activate env_trafic;'
cmd_virtenv = cmd_virtenv + ' `which python` '
cmd_pipeline_train = cmd_virtenv + str(cmd_py) + ';'
if not os.path.exists("tmp_dir_lm_class"):
os.makedirs("tmp_dir_lm_class")
print(cmd_pipeline_train)
cmd = ["bash", "-c", str(cmd_pipeline_train)]
log_dir = os.path.join(sum_dir,"Logs")
if not os.path.isdir(log_dir):
os.makedirs(log_dir)
out = open(os.path.join(log_dir,"training_out.txt"), "wb")
err = open(os.path.join(log_dir,"training_err.txt"), "wb")
subprocess.Popen(cmd, stdout=out, stderr=err)
# print("\nout : " + str(out) + "\nerr : " + str(err))
return
def runClassification(self, data_file, model_dir, sum_dir, output_dir, dF_Path, name_fiber):
runMaybeEnvInstallTF()
currentPath = os.path.dirname(os.path.abspath(__file__))
env_dir = os.path.join(currentPath,"..", "..", "miniconda2")
cli_dir = os.path.join(currentPath,"..", "..","cli-modules")
polydatatransform = os.path.join(cli_dir, "polydatatransform")
lm_ped = os.path.join(currentPath, "Resources", "Landmarks", name_fiber + "_bundle_clean_landmarks.fcsv")
tmp_dir = os.path.join(currentPath, "tmp_dir_lm_class")
if not os.path.isdir(tmp_dir):
os.makedirs(tmp_dir)
new_lm_path = os.path.join(tmp_dir, "lm_class.fcsv")
cmd_polydatatransform = [polydatatransform, "--invertx", "--inverty", "--fiber_file", lm_ped, "-D", dF_Path, "-o", new_lm_path]
out, err = subprocess.Popen(cmd_polydatatransform, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
print("\nout : " + str(out))
pipeline_eval_py = os.path.join(TRAFIC_LIB_DIR, "PipelineEval.py")
cmd_py = str(pipeline_eval_py) + ' --data_file ' + str(data_file) + ' --biclass --summary_dir ' + str(sum_dir)+ ' --checkpoint_dir ' + str(model_dir) + ' --output_dir ' + str(output_dir) + ' --landmark_file ' + str(new_lm_path) + " --fiber_name "+ name_fiber
cmd_virtenv = 'ENV_DIR="'+str(env_dir)+'";'
cmd_virtenv = cmd_virtenv + 'export PYTHONPATH=$ENV_DIR/envs/env_trafic/lib/python2.7/site-packages:$ENV_DIR/lib/:$ENV_DIR/lib/python2.7/lib-dynload/:$ENV_DIR/lib/python2.7/:$ENV_DIR/lib/python2.7/site-packages/:$PYTHONPATH;'
cmd_virtenv = cmd_virtenv + 'export PATH=$ENV_DIR/bin/:$PATH;'
cmd_virtenv = cmd_virtenv + 'source activate env_trafic;'
cmd_virtenv = cmd_virtenv + 'LD_LIBRARY_PATH=$ENV_DIR/envs/env_trafic/lib/libc6_2.17/lib/:$ENV_DIR/envs/env_trafic/lib/libc6_2.17/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH $ENV_DIR/envs/env_trafic/lib/libc6_2.17/lib/x86_64-linux-gnu/ld-2.17.so `which python` '
cmd_pipeline_class = cmd_virtenv + str(cmd_py) + ';'
print(cmd_pipeline_class)
cmd = ["bash", "-c", str(cmd_pipeline_class)]
out = open(os.path.join(TRAFIC_LIB_DIR,"Logs","classification_out.txt"), "wb")
err = open(os.path.join(TRAFIC_LIB_DIR,"Logs","classification_err.txt"), "wb")
_, _ = subprocess.Popen(cmd, stdout=out, stderr=err).communicate()
# print("\nout : " + str(out) + "\nerr : " + str(err))
rmtree(tmp_dir)
# print("\nout : " + str(out) + "\nerr : " + str(err))
return
def runClassification(self, data_file, model_dir, sum_dir, output_dir, dF_Path):
runMaybeEnvInstallTF()
currentPath = os.path.dirname(os.path.abspath(__file__))
env_dir = os.path.join(currentPath, "..", "miniconda2")
cli_dir = os.path.join(currentPath, "..","cli-modules")
# polydatatransform = os.path.join(cli_dir, "polydatatransform")
polydatatransform = "/work/boucaud/builds/niral_utilities/bin/polydatatransform"
lm_ped = os.path.join(currentPath,"Resources", "Landmarks", "landmarks_32pts_afprop.fcsv") ###CHANGED HERE
tmp_dir = os.path.join(currentPath, "tmp_dir_lm_class")
if not os.path.isdir(tmp_dir):
os.makedirs(tmp_dir)
new_lm_path = os.path.join(tmp_dir, "lm_class.fcsv")
cmd_polydatatransform = [polydatatransform, "--invertx", "--inverty", "--fiber_file", lm_ped, "-D", dF_Path, "-o", new_lm_path]
print(cmd_polydatatransform)
out, err = subprocess.Popen(cmd_polydatatransform, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
print("\nout : " + str(out))
pipeline_eval_py = os.path.join(TRAFIC_LIB_DIR, "PipelineEval.py")
cmd_py = str(pipeline_eval_py) + ' --data_file ' + str(data_file) + ' --multiclass --summary_dir ' + str(sum_dir)+ ' --checkpoint_dir ' + str(model_dir) + ' --output_dir ' + str(output_dir) + ' --landmark_file ' + str(new_lm_path)
cmd_virtenv = 'ENV_DIR="'+str(env_dir)+'";'
cmd_virtenv = cmd_virtenv + 'export PYTHONPATH=$ENV_DIR/envs/env_trafic/lib/python2.7/site-packages:$ENV_DIR/lib/:$ENV_DIR/lib/python2.7/lib-dynload/:$ENV_DIR/lib/python2.7/:$ENV_DIR/lib/python2.7/site-packages/:$PYTHONPATH;'
cmd_virtenv = cmd_virtenv + 'export PATH=$ENV_DIR/bin/:$PATH;'
cmd_virtenv = cmd_virtenv + 'source activate env_trafic;'
cmd_virtenv = cmd_virtenv + ' `which python` '
cmd_pipeline_class = cmd_virtenv + str(cmd_py) + ';'
print(str(cmd_pipeline_class))
cmd = ["bash", "-c", str(cmd_pipeline_class)]
log_dir = os.path.join(sum_dir,"Logs")
if not os.path.isdir(log_dir):
os.makedirs(log_dir)
out = open(os.path.join(log_dir,"training_out.txt"), "wb")
err = open(os.path.join(log_dir,"training_err.txt"), "wb")
_, _ = subprocess.Popen(cmd, stdout=out, stderr=err).communicate()
# print("\nout : " + str(out) + "\nerr : " + str(err))
rmtree(tmp_dir)
# print("\nout : " + str(out) + "\nerr : " + str(err))
return