def GetScopes(select_user=False, select_group=False, select_official=False): logging.info("Reading in the scopes:") Scopes = getGmdOutput("rucio list-scopes") ScopeList = ClearFromDuplicates([ Entry for Entry in Scopes if (select_user == True and Entry.find("user") != -1) or (select_group == True and Entry.find("group") != -1) or ( select_official == True and Entry.find("user") == -1 and Entry.find("group") == -1) ]) logging.info("Done found %d scopes" % (len(ScopeList))) return ScopeList
def GetDataSetReplicas(DS): Cmd = "rucio list-rules %s --csv" % (DS) logging.debug("Executing " + Cmd) Replicas = [] all_rses = getRSEs() for line in getGmdOutput(Cmd): try: ds_rse = line.split(",")[4].strip() except: continue if ds_rse in all_rses: Replicas.append(ds_rse) return Replicas
def ListDisk(RSE): if len(RSE) == 0: logging.error("No disk is given") return [] logging.info("Read content of " + RSE) OnDisk = getGmdOutput("rucio list-datasets-rse " + RSE) MyDataSets = [] for Candidates in OnDisk: if Candidates.startswith('-'): continue elif Candidates.startswith('SCOPE'): continue MyDataSets.append(Candidates) return MyDataSets
def GetUserRules(user): Cmd = "rucio list-rules --account %s --csv" % (user) logging.debug("Executing " + Cmd) OnDisk = getGmdOutput(Cmd) MyRules = [] for Rule in OnDisk: try: ID = Rule.split(",")[0].strip() DataSet = Rule.split(",")[2].strip() Rule_RSE = Rule.split(",")[4].strip() except: continue MyRules.append((ID, DataSet, Rule_RSE)) return MyRules
def GetDataSetInfo(DS, RSE, Subscriber=None): Cmd = "rucio list-rules %s --csv" % (DS) logging.debug("Executing " + Cmd) Rules = getGmdOutput(Cmd) for i in range(len(Rules)): Rule = Rules[i] try: ID = Rule.split(",")[0].strip() Owner = Rule.split(",")[1].strip() RuleRSE = Rule.split(",")[4].strip() if RuleRSE == RSE and (Subscriber == None or Subscriber == Owner): return ID, Owner except: continue return None, None
def get_num_scheduled(user_name): cmd = "squeue --format=\"%%i\" -u %s" % (user_name) num_jobs = 0 for i, line in enumerate(getGmdOutput(cmd)): if i == 0: continue if line.find("_") == -1 or (line.find("[") == -1 and line.find("]") == -1): num_jobs += 1 else: array_size = line[line.find("[") + 1:line.rfind("%") if line. rfind("%") != -1 else line.rfind("]")] try: num_jobs += int(array_size.split("-")[-1]) except: pass return num_jobs
def ListDiskWithSize(RSE): if len(RSE) == 0: logging.error("No disk is given") return [] logging.info("Read content of %s and also save the size of each dataset" % (RSE)) OnDisk = getGmdOutput("rucio list-datasets-rse %s --long" % (RSE)) MyDS = [] for Candidates in OnDisk: try: DS = Candidates.split("|")[1].strip() Size = Candidates.split("|")[3].strip() Stored = float(Size[:Size.find("/")]) / 1024 / 1024 / 1024 TotalSize = float(Size[Size.find("/") + 1:]) / 1024 / 1024 / 1024 except: continue logging.info("%s %s %.2f GB" % (DS, Stored, TotalSize)) MyDS.append((DS, TotalSize)) return sorted(MyDS, key=lambda size: size[1], reverse=True)
def __partition(self, RunTime): partition = "" OldTime = 1.e25 part_cmd = "sinfo --format='%%P %%l %%a'" for L in getGmdOutput(part_cmd): if len(L.split()) < 3: continue name = L.split()[0].replace("*", "") Time = L.split()[1] Mode = L.split()[2] t0 = TimeToSeconds(Time) if t0 > 0 and t0 > TimeToSeconds( RunTime) and t0 < OldTime and Mode == "up": partition = name OldTime = TimeToSeconds(Time) if len(partition) == 0: logging.error("Invalid run-time given %s" % (RunTime)) exit(1) return " --partition %s --time='%s' " % (partition, RunTime)
def ListDataFilesWithSize(DS): file_list = [] Cmd = "rucio list-files %s --csv" % (DS) for file_in_ds in getGmdOutput(Cmd): # group.perf-muons:group.perf-muons.17916903.EXT0._000027.NTUP_MCPTP.root,6DD081BE-7CAE-4AAE-8F9C-E1E3612AA09C,8b139e60,3.893 GB,None # file_name = file_in_ds.split(",")[0] ## Skim away the scope file_name = file_name[file_name.find(":") + 1:] file_size_str = file_in_ds.split(",")[3] unit = file_size_str.split(" ")[1].replace("i", "").upper() file_size = float(file_size_str.split(" ")[0]) if unit == "GB": file_size *= 1024 * 1024 * 1024 elif unit == "MB": file_size *= 1024 * 1024 elif unit == "KB": file_size *= 1024 else: logging.error("Unkown file-size %s" % (file_size_str)) exit(1) file_list += [(file_name, file_size)] return file_list
def GetDataSetFiles(dsname, RSE, protocols): logging.info("Get the files of the dataset %s at %s" % (dsname, RSE)) logging.info("Issuing command: rucio list-file-replicas --protocols %s --rse %s %s " % (protocols, RSE, dsname)) DSReplicas = getGmdOutput("rucio list-file-replicas --protocols %s --rse %s %s " % (protocols, RSE, dsname)) DS = [] for line in DSReplicas: Entry = None LineInfo = line.split() for i, column in enumerate(LineInfo): if RSE in column: try: Entry = LineInfo[i + 1] break except: logging.warning("There was some strange noise here ", column) pass if Entry: logging.info("Entry: " + Entry) ReplacePath = os.getenv("CLSUB_RUCIOREPLACEPATH") LocalPath = os.getenv("CLSUB_RUCIOLOCALPATH") if ReplacePath and LocalPath: Entry = Entry.replace(ReplacePath, LocalPath) DS.append(Entry) return DS
def getRSEs(): Cmd = "rucio list-rses" logging.debug("Executing " + Cmd) return sorted(getGmdOutput(Cmd))
def __slurm_id(self, job): Ids = [] jobName = '' ### Bare subids as requested by the user sub_ids = [] if isinstance(job, str): jobName = job #### feature to get only sub id's in a certain range elif isinstance(job, tuple): jobName = job[0] #### Users have the possibility to pipe either the string #### of job names [ "FirstJobToHold", "SecondJobToHold", "TheCake"] #### or to pipe a tuple which can be either of the form #### [ ("MyJobsArray" , [1,2,3,4,5,6,7,8,9,10,11]), "The cake"] #### meaning that the job ID's are constructed following the #### sequence 1-11. It's important to emphazise that the array #### *must* start with a 1. 0's are ignored by the system. There #### is also an third option, where the user parses #### ["MyJobArray", -1] #### This option is used to indicate a one-by-one dependency of #### tasks in 2 consecutive arrays. if isinstance(job[1], list): sub_ids = sorted([int(i) for i in job[1]]) if -1 in sub_ids: logging.warning( "<__slurm_id>: -1 found in sub ids. If you want to pipe a 1 by 1 dependence of subjobs in two arrays please add [ (%s, -1) ]" % (jobName)) else: logging.error("Invalid object:") logging.error(job) exit(1) ### Find all associated job-ids for J in getGmdOutput("squeue --format=\"%%j %%i\""): if len(J.strip()) == 0: continue fragments = J.split() if len(fragments) == 0: continue if fragments[0].strip() == jobName: cand_id = fragments[1].strip() ### The pending job is an array if cand_id.find("_") != -1: main_job = cand_id[:cand_id.find("_")] job_range = cand_id[cand_id.find("_") + 1:] ### We simply do not care about particular subjobs if len(sub_ids) == 0: if main_job not in Ids: Ids += [main_job] elif len(sub_ids) > 0: Ids += [ "%s_%d" % (main_job, i) for i in sub_ids if i > 0 ] elif cand_id not in Ids: Ids += [cand_id] Ids.sort() ### The allowed array size on the cluster is smaller than the ### requested size by the user --> jobs got split into multiple arrays if len(sub_ids) > 0 and max(sub_ids) > self.max_array_size(): split_Ids = [] for sub in sub_ids: ## How many times do we exceed the array size n = (sub - sub % self.max_array_size()) / self.max_array_size() rel_jobs = [ int(j.split("_")[0]) for j in Ids if int(j.split("_")[1]) == sub ] if len(rel_jobs) <= n: logging.warning( "<__slurm_id>: Failed to establish dependency on %s." % (jobName)) continue split_Ids += [ "%d_%d" % (rel_jobs[n], sub % self.max_array_size() if sub % self.max_array_size() != 0 else self.max_array_size()) ] return split_Ids return Ids