def __get_latest_outdir(self, out_dir):
from ntp.utils import find_latest_iteration
import glob
existing_dirs = glob.glob(out_dir + '_*')
latest = 1
if existing_dirs:
latest = find_latest_iteration(existing_dirs)
latest += 1
out_dir += '_{0}'.format(latest)
return out_dir
def __get_latest_outdir(self, out_dir):
from ntp.utils import find_latest_iteration
import glob
existing_dirs = glob.glob(out_dir + '_*')
latest = 1
if existing_dirs:
latest = find_latest_iteration(existing_dirs)
latest += 1
out_dir += '_{0}'.format(latest)
return out_dir
def __set_job_dirs(self, campaign, dataset):
import glob
out_dir = os.path.join(CONDOR_ROOT, campaign, dataset)
existing_dirs = glob.glob(out_dir + '_*')
latest = 1
if existing_dirs:
latest = find_latest_iteration(existing_dirs)
latest += 1
out_dir += '_{0}'.format(latest)
self.__job_dir = out_dir
self.__outdirs.append(out_dir)
self.__job_log_dir = os.path.join(self.__job_dir, 'log')
self.__setup_script = os.path.join(self.__job_dir, 'setup.sh')
self.__run_script = os.path.join(self.__job_dir, 'run.sh')
self.__run_config = os.path.join(self.__job_dir, 'config.json')
def return_datasets_to_check(self):
'''
Returns the newest version of datasets to check.
'''
list_of_all_files, datasets = self.get_folder()
list_of_datasets = []
for dataset in datasets:
list_of_files = []
for l in list_of_all_files:
if dataset in l:
list_of_files.append(l)
i = find_latest_iteration(list_of_files)
f_to_use = list_of_files[0].replace('_1',
'_' + str(i)) + '*.status'
list_of_datasets.append(f_to_use)
return list_of_datasets
def __set_job_dirs(self, campaign, dataset):
import glob
from time import strftime
date = strftime("%d_%b_%y")
out_dir = os.path.join(CONDOR_ROOT, campaign, date, dataset)
existing_dirs = glob.glob(out_dir + '_*')
latest = 1
if existing_dirs:
latest = find_latest_iteration(existing_dirs)
latest += 1
out_dir += '_{0}'.format(latest)
self.__job_dir = out_dir
self.__outdirs.append(out_dir)
self.__job_log_dir = os.path.join(self.__job_dir, 'log')
self.__setup_script = os.path.join(self.__job_dir, 'setup.sh')
self.__run_script = os.path.join(self.__job_dir, 'run.sh')
self.__run_config = os.path.join(self.__job_dir, 'config.json')
def __set_job_dirs(self, campaign, dataset):
out_dir = os.path.join(CONDOR_ROOT, campaign, dataset)
existing_dirs = glob.glob(out_dir + '_*')
latest = 1
if existing_dirs:
latest = find_latest_iteration(existing_dirs)
latest += 1
out_dir += '_{0}'.format(latest)
self.__job_dir = out_dir
self.__outdirs.append(out_dir)
self.__job_log_dir = os.path.join(self.__job_dir, 'log')
self.__setup_script = os.path.join(self.__job_dir, 'setup.sh')
self.__run_script = os.path.join(self.__job_dir, 'run.sh')
self.__merge_setup_script = os.path.join(self.__job_dir,
'merge_setup.sh')
self.__merge_script = os.path.join(self.__job_dir, 'merge.sh')
self.__analysis_setup_script = os.path.join(self.__job_dir,
'analysis_setup.sh')
self.__analysis_script = os.path.join(self.__job_dir, 'analysis.sh')
self.__run_config = os.path.join(self.__job_dir, 'config.json')