def check_for_existing_output(self, r, rname): """ Check whether given runcard already has output in the grid needs testing as it needs to be able to remove (many) things for production run It relies on the base seed from the src.header file to remove the output """ from pyHepGrid.src.header import grid_output_dir, logger logger.debug(F"Checking whether runcard {rname} has output for seeds " "that you are trying to submit...") checkname = r + "-" + rname files = self.gridw.get_dir_contents(grid_output_dir) first = True if checkname in files: from pyHepGrid.src.header import baseSeed, producRun for seed in range(baseSeed, baseSeed + producRun): filename = self.output_name(r, rname, seed) if filename in files: if first: self._press_yes_to_continue( "This runcard already has at least one output file " "at gfal:output with a seed you are trying to " F"submit (looked for '{checkname}').\n" "If you continue, it will be removed now.") logger.warning( F"Runcard {r} has at least one file at output") first = False self.gridw.delete(filename, grid_output_dir) logger.info("Output check complete")
def check_for_existing_output(self, r, rname): """ Check whether given runcard already has output in the grid needs testing as it needs to be able to remove (many) things for production run It relies on the base seed from the src.header file to remove the output """ from pyHepGrid.src.header import grid_output_dir, logger logger.debug( "Checking whether runcard {0} has output for seeds that you are trying to submit..." .format(rname)) checkname = r + "-" + rname files = self.gridw.get_dir_contents(grid_output_dir) first = True if checkname in files: from pyHepGrid.src.header import baseSeed, producRun for seed in range(baseSeed, baseSeed + producRun): filename = self.output_name(r, rname, seed) if filename in files: if first: self._press_yes_to_continue( "It seems this runcard already has at least one file at lfn:output with a seed you are trying to submit (looked for {}). Do you want to remove it/them?" .format(checkname)) logger.warning( "Runcard {0} has at least one file at output". format(r)) first = False self.gridw.delete(filename, grid_output_dir) logger.info("Output check complete")
def init_single_local_warmup(self, runcard, tag, continue_warmup=False, provided_warmup=False): import shutil from pyHepGrid.src.header import executable_src_dir, executable_exe, runcardDir, slurm_kill_exe run_dir = self.get_local_dir_name(runcard, tag) os.makedirs(run_dir, exist_ok=True) stdoutdir = self.get_stdout_dir_name(run_dir) os.makedirs(stdoutdir, exist_ok=True) # directory for slurm stdout files path_to_exe_full = self._exe_fullpath(executable_src_dir, executable_exe) shutil.copy(path_to_exe_full, run_dir) runcard_file = os.path.join(runcardDir, runcard) runcard_obj = PROGRAMruncard(runcard_file, logger=logger, grid_run=False, use_cvmfs=header.use_cvmfs_lhapdf, cvmfs_loc=header.cvmfs_lhapdf_location) self._check_warmup(runcard_obj, continue_warmup) logger.debug("Copying runcard {0} to {1}".format(runcard_file, run_dir)) shutil.copy(runcard_file, run_dir) shutil.copy(slurm_kill_exe, run_dir) if provided_warmup: # Copy warmup to rundir match, local = self._get_local_warmup_name(runcard_obj.warmup_filename(), provided_warmup) shutil.copy(match, run_dir) if runcard_obj.is_continuation(): # Assert warmup is present in dir. Will error out if not if continue_warmup: match, local = self._get_local_warmup_name(runcard_obj.warmup_filename(), run_dir)
def init_single_local_production(self, runcard, tag, provided_warmup=False): """ Initialise single production run for the local environment. Can probably be more tightly integrated with the warmup equivalent in future - lots of shared code that can be refactored.""" import shutil from pyHepGrid.src.header import executable_src_dir, executable_exe, runcardDir run_dir = self.get_local_dir_name(runcard, tag) os.makedirs(run_dir, exist_ok=True) stdoutdir = self.get_stdout_dir_name(run_dir) os.makedirs(stdoutdir, exist_ok=True) # directory for slurm stdout files path_to_exe_full = self._exe_fullpath(executable_src_dir, executable_exe) shutil.copy(path_to_exe_full, run_dir) runcard_file = os.path.join(runcardDir, runcard) runcard_obj = PROGRAMruncard(runcard_file, logger=logger, use_cvmfs=header.use_cvmfs_lhapdf, cvmfs_loc=header.cvmfs_lhapdf_location) self._check_production(runcard_obj) logger.debug("Copying runcard {0} to {1}".format(runcard_file, run_dir)) shutil.copy(runcard_file, run_dir) if provided_warmup: # Copy warmup to rundir match, local = self._get_local_warmup_name(runcard_obj.warmup_filename(), provided_warmup) shutil.copy(match, run_dir) else: # check warmup is in dir - check is case insensitive - be careful! rundirfiles = [i.lower() for i in os.listdir(run_dir)] if runcard_obj.warmup_filename() not in rundirfiles: logger.critical("No warmup found in run folder and no warmup provided manually")
def check_for_existing_output_local(self, r, rname, baseSeed, producRun): """ Check whether given runcard already has output in the local run dir (looks for log files) """ import re logger.debug( "Checking whether runcard {0} has output for seeds that you are " "trying to submit...".format(rname)) local_dir_name = self.get_local_dir_name(r, rname) files = os.listdir(local_dir_name) runcard = PROGRAMruncard(runcard_file=os.path.join(local_dir_name, r), logger=logger, grid_run=False, use_cvmfs=header.use_cvmfs_lhapdf, cvmfs_loc=header.cvmfs_lhapdf_location) runcard_id = runcard.runcard_dict_case_preserving["id"] logs = [f for f in files if f.endswith(".log") and runcard_id in f] logseed_regex = re.compile(r".s([0-9]+)\.[^\.]+$") existing_seeds = set( [int(logseed_regex.search(i).group(1)) for i in logs]) submission_seeds = set(range(baseSeed, baseSeed + producRun)) overlap = existing_seeds.intersection(submission_seeds) if overlap: logger.warning( "Log files for seed(s) {0} already exist in run folders. " "Running will overwrite the logfiles already present.".format( " ".join(str(i) for i in overlap))) self._press_yes_to_continue(None) return
def check_for_existing_warmup(self, r, rname): logger.debug("Checking whether this runcard is already at lfn:warmup") checkname = self.warmup_name(r, rname) if self.gridw.checkForThis(checkname, grid_warmup_dir): self._press_yes_to_continue( "File {1} already exists at lfn:{0}, do you want to remove it?" .format(grid_warmup_dir, checkname)) self.gridw.delete(checkname, grid_warmup_dir)
def _check_warmup(self, runcard, continue_warmup=False): logger.debug("Checking warmup in runcard {0}".format(runcard.name)) if not runcard.is_warmup(): self._press_yes_to_continue("Warmup is not active in runcard") if continue_warmup and not runcard.is_continuation(): self._press_yes_to_continue("Continue warmup is not active in runcard") if runcard.is_production(): self._press_yes_to_continue("Production is active in runcard")
def check_warmup_files(self, db_id, rcard, resubmit=False): """ Provides stats on whether a warmup file exists for a given run and optionally resubmit if absent """ import tempfile import tarfile origdir = os.path.abspath(os.getcwd()) tmpdir = tempfile.mkdtemp() os.chdir(tmpdir) logger.debug("Temporary directory: {0}".format(tmpdir)) rncards, dCards = util.expandCard() tags = ["runcard", "runfolder"] runcard_info = self.dbase.list_data(self.table, tags, db_id)[0] runcard = runcard_info["runcard"] rname = runcard_info["runfolder"] try: warmup_files = self._bring_warmup_files(runcard, rname, check_only=True) if warmup_files == []: status = "\033[93mMissing\033[0m" else: status = "\033[92mPresent\033[0m" except tarfile.ReadError: status = "\033[91mCorrupted\033[0m" run_id = "{0}-{1}:".format(runcard, rname) logger.info("[{0}] {2:55} {1:>20}".format(db_id, status, run_id)) os.chdir(origdir) if resubmit and "Present" not in status: done, wait, run, fail, unk = self.stats_job(db_id, do_print=False) if run + wait > 0: # Be more careful in case of unknown status logger.warning("Job still running. Not resubmitting for now") else: # Need to override dictCard for single job submission expandedCard = ([runcard], {runcard: rname}) logger.info( "Warmup not found and job ended. Resubmitting to ARC") from pyHepGrid.src.runArcjob import runWrapper runWrapper(rcard, expandedCard=expandedCard)
def init_production(self, provided_warmup=None, continue_warmup=False, local=False): """ Initialises a production run. If a warmup file is provided retrieval step is skipped/ Steps are: 0 - Retrieve warmup from the grid/local 1 - tar up executable, runcard and necessary files 2 - sent it to the grid storage """ import tempfile from pyHepGrid.src.header import runcardDir as runFol from pyHepGrid.src.header import executable_exe, executable_src_dir, \ grid_input_dir if local: self.init_local_production(provided_warmup=provided_warmup) return runFolders, dCards = util.expandCard() path_to_exe_full = self._exe_fullpath(executable_src_dir, executable_exe) origdir = os.path.abspath(os.getcwd()) tmpdir = tempfile.mkdtemp() # if provided warmup is a relative path, ensure we have the full path # before we change to the tmp directory if provided_warmup: if provided_warmup[0] != "/": provided_warmup = "{0}/{1}".format(origdir, provided_warmup) if provided_warmup: warmup_base = provided_warmup elif header.provided_warmup_dir: warmup_base = header.provided_warmup_dir else: logger.critical( "Retrieving warmup file from grid: Not implemented") os.chdir(tmpdir) logger.debug("Temporary directory: {0}".format(tmpdir)) if not os.path.isfile(path_to_exe_full): logger.critical( "Could not find executable at {0}".format(path_to_exe_full)) else: tar_name = os.path.basename(header.grid_executable) grid_exe_dir = os.path.dirname(header.grid_executable) exe_name = header.executable_exe os.system("cp -r " + path_to_exe_full + " " + exe_name) upload_exe = True if self.gridw.checkForThis(tar_name, grid_exe_dir): if not self._press_yes_to_continue( "Old executable found. Do you want to remove it?", fallback=1): logger.info( F"Removing old version of {tar_name} from Grid Storage" ) self.gridw.delete(tar_name, grid_exe_dir) else: upload_exe = False if upload_exe: self.tarw.tarFiles([exe_name], tar_name) self.gridw.send(tar_name, grid_exe_dir) for idx, runName in enumerate(runFolders): local = False tarfile = runName + ".tar.gz" # base_folder = runName.split("-")[0] # logger.info( # "Initialising {0} to {1} [{2}/{3}]".format( # runName, tarfile, idx + 1, len(runFolders))) # # runcards # run_dir = os.path.join(runFol, base_folder) # runFiles = dCards[runName].split("+") # for f in runFiles: # f = os.path.join(run_dir, f) # self._file_exists(f, logger) # os.system("cp -r " + f + " " + tmpdir) # # warmup files # for f in self._WARMUP_FILES: # f = os.path.join(warmup_base, base_folder, f) # self._file_exists(f, logger) # os.system("cp -r " + f + " " + tmpdir) logger.info("Sending JUNE code to Grid storage.") upload_data = True if self.gridw.checkForThis(tarfile, grid_input_dir): if not self._press_yes_to_continue( "Old data found. Do you want to remove it?", fallback=1): logger.info( F"Removing old version of {tarfile} from Grid Storage") self.gridw.delete(tarfile, grid_input_dir) else: upload_data = False if upload_data: # tar up & send to grid storage # self.tarw.tarFiles(self._WARMUP_FILES + runFiles, tarfile) if self.gridw.checkForThis(tarfile, grid_input_dir): logger.info( "Removing old version of {0} from Grid Storage".format( tarfile)) self.gridw.delete(tarfile, grid_input_dir) logger.info("Sending {0} to {1}".format( tarfile, grid_input_dir)) os.system("gfal-copy {0} {1}".format( origdir + "/" + tarfile, header.gfaldir + grid_input_dir)) # self.gridw.send(tarfile, grid_input_dir) if hasattr(header, 'world_list'): world_list = header.world_list for world in world_list: upload_world = True world_file = "{0}.hdf5".format(world) world_path = grid_input_dir + "/worlds" if self.gridw.checkForThis(world_file, world_path): if not self._press_yes_to_continue( "Old {0} found. Do you want to remove it?".format( world_file), fallback=1): logger.info( F"Removing old version of {world_file} from Grid Storage" ) self.gridw.delete(world_file, world_path) else: upload_world = False if upload_world: print("Sending {0} to Grid Storage.".format(world_file)) if self.gridw.checkForThis(world_file, world_path): logger.info( "Removing old version of {0} from Grid Storage". format(world_file)) self.gridw.delete(world_file, world_path) logger.info("Sending {0} to {1}".format( world_file, world_path)) os.system("gfal-copy {0} {1}".format( origdir + "/" + world_file, header.gfaldir + world_path)) if hasattr(header, 'latin_hypercube'): lhs_file = header.latin_hypercube upload_lhs = True lhs_file = "{0}.npy".format(lhs_file) if self.gridw.checkForThis(lhs_file, grid_input_dir): if not self._press_yes_to_continue( "Old {0} found. Do you want to remove it?".format( lhs_file), fallback=1): logger.info( F"Removing old version of {lhs_file} from Grid Storage" ) self.gridw.delete(lhs_file, grid_input_dir) else: upload_lhs = False if upload_lhs: print("Sending {0} to Grid Storage.".format(lhs_file)) if self.gridw.checkForThis(lhs_file, grid_input_dir): logger.info( "Removing old version of {0} from Grid Storage".format( lhs_file)) self.gridw.delete(lhs_file, grid_input_dir) logger.info("Sending {0} to {1}".format( lhs_file, grid_input_dir)) os.system("gfal-copy {0} {1}".format( origdir + "/" + lhs_file, header.gfaldir + grid_input_dir)) # clean up afterwards os.chdir(origdir) os.system("rm -r " + tmpdir)
def _bring_warmup_files(self, runcard, rname, shell=False, check_only=False, multichannel=False): """ Download the warmup file for a run to local directory extracts Vegas grid and log file and returns a list with their names check_only flag doesn't error out if the warmup doesn't exist, instead just returns and empty list for later use [intended for checkwarmup mode so multiple warmups can be checked consecutively. """ from pyHepGrid.src.header import grid_warmup_dir, logger gridFiles = [] suppress_errors = False if check_only: suppress_errors = True ## First bring the warmup .tar.gz outnm = self.warmup_name(runcard, rname) logger.debug("Warmup GFAL name: {0}".format(outnm)) tmpnm = "tmp.tar.gz" logger.debug("local tmp tar name: {0}".format(tmpnm)) success = self.gridw.bring(outnm, grid_warmup_dir, tmpnm, shell = shell, suppress_errors=suppress_errors) success == self.__check_pulled_warmup(success, tmpnm, warmup_extensions) if not success and not check_only: if self._press_yes_to_continue("Grid files failed to copy. Try backups from individual sockets?") == 0: backup_dir = os.path.join(grid_warmup_dir,outnm.replace(".tar.gz","")) backups = self.gridw.get_dir_contents(backup_dir) if len(backups) == 0: logger.critical("No backups found. Did the warmup complete successfully?") else: backup_files = backups.split() for idx, backup in enumerate(backup_files): logger.info("Attempting backup {1} [{0}]".format(idx+1, backup)) success = self.gridw.bring(backup, backup_dir, tmpnm, shell = shell, force=True) success == self.__check_pulled_warmup(success, tmpnm, warmup_extensions) if success: break if not success: logger.critical("Grid files failed to copy. Did the warmup complete successfully?") else: logger.info("Grid files copied ok.") ## Now extract only the Vegas grid files and log file gridp = warmup_extensions gridp += [i+"_channel" for i in gridp] extractFiles = self.tarw.extract_extensions(tmpnm, gridp+[".log",".txt","channels"]) try: gridFiles = [i for i in extractFiles if ".log" not in i] logfile = [i for i in extractFiles if ".log" in i][0] except IndexError as e: if not check_only: logger.critical("Logfile not found. Did the warmup complete successfully?") else: return [] if multichannel and len([i for i in gridFiles if "channels" in i]) ==0: logger.critical("No multichannel warmup found, but multichannel is set in the runcard.") elif multichannel: logger.info("Multichannel warmup files found.") if gridFiles == [] and not check_only: # No grid files found logger.critical("Grid files not found in warmup tarfile. Did the warmup complete successfully?") elif gridFiles == []: return [] ## Tag log file as -warmup newlog = logfile + "-warmup" os.rename(logfile, newlog) # Remove temporary tar files os.remove(tmpnm) gridFiles.append(newlog) # Make sure access to the file is correct! for i in gridFiles: util.spCall(["chmod", "a+wrx", i]) return gridFiles
def init_production(self, provided_warmup=None, continue_warmup=False, local=False): """ Initialises a production run. If a warmup file is provided retrieval step is skipped Steps are: 0 - Retrieve warmup from the grid/local 1 - tar up executable, runcard and necessary files 2 - sent it to the grid storage """ import tempfile from pyHepGrid.src.header import runcardDir as runFol from pyHepGrid.src.header import executable_exe, executable_src_dir, grid_input_dir if local: self.init_local_production(provided_warmup=provided_warmup) return rncards, dCards = util.expandCard() path_to_exe_full = self._exe_fullpath(executable_src_dir, executable_exe) origdir = os.path.abspath(os.getcwd()) tmpdir = tempfile.mkdtemp() # if provided warmup is a relative path, ensure we have the full path # before we change to the tmp directory if provided_warmup: if provided_warmup[0] != "/": provided_warmup = "{0}/{1}".format(origdir, provided_warmup) if provided_warmup: warmup_base = provided_warmup elif header.provided_warmup_dir: warmup_base = header.provided_warmup_dir else: # print("Retrieving warmup file from grid") # warmupFiles = self._bring_warmup_files(i, dCards[i], shell=True) logger.critical("Retrieving warmup file from grid: Not implemented") # setup LHAPDF if header.use_cvmfs_lhapdf: os.environ['LHAPDF_DATA_PATH'] = header.cvmfs_lhapdf_location # create Process dir in Sherpa self._init_Sherpa(warmup_base,rncards) os.chdir(tmpdir) logger.debug("Temporary directory: {0}".format(tmpdir)) # if not os.path.isfile(path_to_exe_full): # logger.critical("Could not find executable at {0}".format(path_to_exe_full)) # copy(path_to_exe_full, os.getcwd()) # files = [executable_exe] for idx, i in enumerate(rncards): local = False tarfile = i +"+"+ dCards[i] + ".tar.gz" base_folder = i.split("-")[0] + "/" logger.info("Initialising {0} to {1} [{2}/{3}]".format(i, tarfile, idx+1, len(rncards))) # runcards run_dir = runFol + base_folder runFiles = [dCards[i]+".yml"] for f in runFiles: os.system("cp -r "+run_dir+f+" "+tmpdir) # warmup files warmupFiles = ["Process", "Run.dat", "Results.db"] for f in warmupFiles: os.system("cp -r "+warmup_base+base_folder+f+" "+tmpdir) # tar up & send to grid storage self.tarw.tarFiles(warmupFiles+runFiles, tarfile) if self.gridw.checkForThis(tarfile, grid_input_dir): logger.info("Removing old version of {0} from Grid Storage".format(tarfile)) self.gridw.delete(tarfile, grid_input_dir) logger.info("Sending {0} to {1}".format(tarfile, grid_input_dir)) self.gridw.send(tarfile, grid_input_dir, shell=True) # clean up afterwards os.chdir(origdir) os.system("rm -r "+tmpdir)
def init_production(self, provided_warmup=None, continue_warmup=False, local=False): """ Initialises a production run. If a warmup file is provided retrieval step is skipped Steps are: 0 - Retrieve warmup from the grid/local 1 - tar up executable, runcard and necessary files 2 - sent it to the grid storage """ from shutil import copy import tempfile from pyHepGrid.src.header import runcardDir as runFol from pyHepGrid.src.header import executable_exe, executable_src_dir, logger if local: self.init_local_production(provided_warmup=provided_warmup) return rncards, dCards = util.expandCard() path_to_exe_full = self._exe_fullpath(executable_src_dir, executable_exe) origdir = os.path.abspath(os.getcwd()) tmpdir = tempfile.mkdtemp() # if provided warmup is a relative path, ensure we have the full path # before we change to the tmp directory if provided_warmup: if provided_warmup[0] != "/": provided_warmup = "{0}/{1}".format(origdir, provided_warmup) os.chdir(tmpdir) logger.debug("Temporary directory: {0}".format(tmpdir)) if not os.path.isfile(path_to_exe_full): logger.critical("Could not find executable at {0}".format(path_to_exe_full)) copy(path_to_exe_full, os.getcwd()) files = [executable_exe] for idx, i in enumerate(rncards): logger.info("Initialising {0} [{1}/{2}]".format(i, idx+1, len(rncards))) local = False # Check whether warmup/production is active in the runcard runcard_file = os.path.join(runFol, i) runcard_obj = PROGRAMruncard(runcard_file, logger=logger, use_cvmfs=header.use_cvmfs_lhapdf, cvmfs_loc=header.cvmfs_lhapdf_location) multichannel = self.check_runcard_multichannel(runcard_obj) self._check_production(runcard_obj) rname = dCards[i] tarfile = i + rname + ".tar.gz" copy(os.path.join(runFol, i), os.getcwd()) if provided_warmup: match, local = self._get_local_warmup_name(runcard_obj.warmup_filename(), provided_warmup) warmupFiles = [match] elif header.provided_warmup_dir: match, local = self._get_local_warmup_name(runcard_obj.warmup_filename(), header.provided_warmup_dir) warmupFiles = [match] else: logger.info("Retrieving warmup file from grid") warmupFiles = self._bring_warmup_files(i, rname, shell=True, multichannel=multichannel) self.tarw.tarFiles(files + [i] + warmupFiles, tarfile) if self.gridw.checkForThis(tarfile, header.grid_input_dir): logger.info("Removing old version of {0} from Grid Storage".format(tarfile)) self.gridw.delete(tarfile, header.grid_input_dir) logger.info("Sending {0} to GFAL {1}/".format(tarfile, header.grid_input_dir)) self.gridw.send(tarfile, header.grid_input_dir, shell=True) if local: util.spCall(["rm", i, tarfile]) else: util.spCall(["rm", i, tarfile] + warmupFiles) os.remove(executable_exe) os.chdir(origdir)
def init_warmup(self, provided_warmup=None, continue_warmup=False, local=False): """ Initialises a warmup run. An warmup file can be provided and it will be added to the .tar file sent to the grid storage. Steps are: 1 - tar up executable, runcard and necessary files 2 - sent it to the grid storage """ from shutil import copy import tempfile from pyHepGrid.src.header import executable_src_dir, executable_exe, logger from pyHepGrid.src.header import runcardDir as runFol if local: self.init_local_warmups(provided_warmup=provided_warmup, continue_warmup=continue_warmup) return origdir = os.path.abspath(os.getcwd()) tmpdir = tempfile.mkdtemp() # if provided warmup is a relative path, ensure we have the full path # before we change to the tmp directory if provided_warmup: if provided_warmup[0] != "/": provided_warmup = "{0}/{1}".format(origdir, provided_warmup) os.chdir(tmpdir) logger.debug("Temporary directory: {0}".format(tmpdir)) rncards, dCards = util.expandCard() path_to_exe_full = self._exe_fullpath(executable_src_dir, executable_exe) if not os.path.isfile(path_to_exe_full): logger.critical("Could not find executable at {0}".format(path_to_exe_full)) copy(path_to_exe_full, os.getcwd()) files = [executable_exe] for idx, i in enumerate(rncards): logger.info("Initialising {0} [{1}/{2}]".format(i, idx+1, len(rncards))) local = False warmupFiles = [] # Check whether warmup/production is active in the runcard runcard_file = os.path.join(runFol, i) if not os.path.isfile(runcard_file): self._press_yes_to_continue("Could not find runcard {0}".format(i), error="Could not find runcard") runcard_obj = PROGRAMruncard(runcard_file, logger=logger, use_cvmfs=header.use_cvmfs_lhapdf, cvmfs_loc=header.cvmfs_lhapdf_location) self._check_warmup(runcard_obj, continue_warmup) multichannel = self.check_runcard_multichannel(runcard_obj) if provided_warmup: # Copy warmup to current dir if not already there match, local = self._get_local_warmup_name(runcard_obj.warmup_filename(), provided_warmup) files += [match] rname = dCards[i] tarfile = i + rname + ".tar.gz" copy(os.path.join(runFol, i), os.getcwd()) if self.overwrite_warmup: checkname = self.warmup_name(i, rname) if self.gridw.checkForThis(checkname, header.grid_warmup_dir): logger.info("Warmup found in GFAL:{0}!".format(header.grid_warmup_dir)) warmup_files = self._bring_warmup_files(i, rname, shell=True, multichannel=multichannel) files += warmup_files logger.info("Warmup files found: {0}".format(" ".join(i for i in warmup_files))) self.tarw.tarFiles(files + [i], tarfile) if self.gridw.checkForThis(tarfile, header.grid_input_dir): # Could we cache this? Just to speed up ini logger.info("Removing old version of {0} from Grid Storage".format(tarfile)) self.gridw.delete(tarfile, header.grid_input_dir) logger.info("Sending {0} to gfal {1}/".format(tarfile, header.grid_input_dir)) self.gridw.send(tarfile, header.grid_input_dir, shell=True) if not local: for j in warmupFiles: os.remove(j) os.remove(i) os.remove(tarfile) os.remove(executable_exe) os.chdir(origdir)
def _check_production(self, runcard): logger.debug("Checking production in runcard {0}".format(runcard.name)) if runcard.is_warmup(): self._press_yes_to_continue("Warmup is active in runcard") if not runcard.is_production(): self._press_yes_to_continue("Production is not active in runcard")