def run(self, input): if self.user == 'CMS': return {'Topdir':None,'Directory':None} topdir = castortools.lfnToCastor(castorBaseDir(user=self.user)) directory = '%s/%s' % (topdir,self.dataset) # directory = directory.replace('//','/') if not castortools.fileExists(directory): if hasattr(self,'create') and self.create: castortools.createCastorDir(directory) #castortools.chmod(directory,'775') if not castortools.isDirectory(directory): raise Exception("Dataset directory '%s' does not exist or could not be created" % directory) return {'Topdir':topdir,'Directory':directory}
def run(self, input): if self.user == 'CMS': return {'Topdir':None,'Directory':None} topdir = castortools.lfnToCastor(castorBaseDir(user=self.user)) directory = '%s/%s' % (topdir,self.dataset) directory = directory.replace('//','/') if not castortools.fileExists(directory): if hasattr(self,'create') and self.create: castortools.createCastorDir(directory) #castortools.chmod(directory,'775') if not castortools.isDirectory(directory): raise Exception("Dataset directory '%s' does not exist or could not be created" % directory) return {'Topdir':topdir,'Directory':directory}
def __init__(self, dataset, options): if not dataset.startswith(os.sep): dataset = os.sep + dataset self.dataset = dataset self.options = options self.topdir = castortools.lfnToCastor( castorBaseDir(user=options.user) ) self.directory = os.path.join(self.topdir, *self.dataset.split(os.sep)) #event counters self.eventsTotal = -1 self.eventsSeen = 0 self.test_result = None
def __init__(self, dataset, options): if not dataset.startswith(os.sep): dataset = os.sep + dataset self.dataset = dataset self.options = options self.topdir = castortools.lfnToCastor(castorBaseDir(user=options.user)) self.directory = os.path.join(self.topdir, *self.dataset.split(os.sep)) #event counters self.eventsTotal = -1 self.eventsSeen = 0 self.test_result = None
group.add_option("-t", "--tier", dest="tier", default='',help="The data tier to use, e.g. 'PAT_CMG_5_6_0'") group.add_option("-u", "--user", dest="user", default=None,help="The user space to write into") parser.add_option_group(group) (opts, datasets) = parser.parse_args() import ConfigParser config = ConfigParser.SafeConfigParser() #set options to be case sensitive config.optionxform = str config.add_section('MULTICRAB') config.set('MULTICRAB','cfg',opts.cfg) from CMGTools.Production.castorBaseDir import castorBaseDir import CMGTools.Production.eostools as castortools topdir = castortools.lfnToCastor(castorBaseDir(user=opts.user)) output_dirs = [] for d in datasets: #accept the user%dataset syntax, but ignore the user name for grid tokens = d.split('%') if len(tokens) == 2: d = tokens[1] safe_name = d.replace('/','_') if safe_name.startswith('_'): safe_name = safe_name[1:] if safe_name.endswith('_'): safe_name = safe_name[:-1]
def __init__(self, name, user, pattern='.*root'): self.lfnDir = castorBaseDir(user) + name self.castorDir = castortools.lfnToCastor(self.lfnDir) self.maskExists = False self.report = None super(Dataset, self).__init__(name, user, pattern)
if len(args) != 1: parser.print_help() sys.exit(1) # For multiple file input file = open(args[0], 'r') lines = file.readlines() for line in lines: line = re.sub("\s+", " ", line) fileown = line.split("%")[0].lstrip().rstrip() dataset = line.split("%")[1].lstrip().rstrip() files = [] if re.search('group', fileown): castor = eostools.lfnToEOS( castorBaseDir.castorBaseDir(fileown)) + dataset castor2 = eostools.lfnToEOS( castorBaseDir.castorBaseDir(fileown.strip("_group"))) + dataset else: castor = eostools.lfnToEOS( castorBaseDir.castorBaseDir(fileown)) + dataset castor2 = eostools.lfnToEOS( castorBaseDir.castorBaseDir(fileown + "_group")) + dataset LFN = eostools.eosToLFN(castor) LFN2 = eostools.eosToLFN(castor2) if eostools.isDirectory(castor): files = burrow(LFN) elif eostools.isDirectory(castor2): files = burrow(LFN2)
savpass = getpass.getpass("Enter Savannah Password: "******"" if options.commented: comment = raw_input("Please enter your comment here:") # Store full dataset name dirOrFile = args[0] # Castor base directory for specified user baseDir = castortools.lfnToCastor(castorBaseDir.castorBaseDir( options.user)) #get the user, after taking account of the user_area hack user, _ = castorBaseDir.getUserAndArea(options.user) # Locations of Logger(.tgz) and containing folder on Castor targetDir = baseDir + "/" + args[0].lstrip("/").rstrip("/") targetTgz = targetDir + "/Logger.tgz" # Get array containing 1) Primary dataset name; 2) Dataset Name; 3+) Tiers details = dirOrFile.lstrip("/").split("/") # If element is not Primary dataset of dataset name, add to the Tiers array count = 0 tiers = [] for i in details: if count > 1:
def __init__(self, name, user, pattern='.*root'): self.lfnDir = castorBaseDir(user) + name self.castorDir = castortools.lfnToCastor( self.lfnDir ) self.maskExists = False self.report = None super(Dataset, self).__init__(name, user, pattern)
castorDir = "" localDir = "" # opions.user could be of the form user_area user,area = cBaseDir.getUserAndArea(options.user) dataSets = '/afs/cern.ch/user/{first}/{user}/public/DataSets.txt'.format( first = user[0], # first letter of the username user = user ) ifile=open(dataSets,'r') pattern = re.compile( args[0] ) castorBaseDir = castortools.lfnToCastor(cBaseDir.castorBaseDir( options.user )) for line in ifile.readlines(): line = line.rstrip() if len(line)==0 or line[0]!='/': continue if pattern.search( line ): sampleName = line try: castorDir = castorBaseDir + sampleName except: sys.exit(1) localDir = None try: localDir = os.environ['CMGLOCALBASEDIR'] localDir += sampleName except:
default="empty_datasets.txt") parser.add_option_group(group) (options, args) = parser.parse_args() cmgdbApi = CmgdbApi(options.development) cmgdbApi.connect() columns, rows = cmgdbApi.sql( "select file_owner, path_name from dataset_details where dataset_is_open='Y' and (file_owner='cmgtools' or file_owner='cmgtools_group')" ) f = open(options.filename, 'w') for row in rows: fileown = row[0] dsName = row[1] if fileown == "--cmgtools": fileown = "cmgtools" lfnDir = castorBaseDir(fileown) + dsName castorDir = castortools.lfnToCastor(lfnDir) if castortools.datasetNotEmpty(castorDir, ".*root"): continue else: f.write(fileown + "%" + dsName + '\n') f.close()
#!/usr/bin/env python import CMGTools.Production.eostools as castortools from CMGTools.Production.castorBaseDir import castorBaseDir if __name__ == '__main__': import sys from optparse import OptionParser parser = OptionParser() parser.usage = "%prog <user>\nPrints the castor base directory of a given user." parser.add_option("-c", "--castorpath", action="store_true", dest="castorpath", help="Print full castor path. Otherwise print LFN, starting by /store", default=False) (options,args) = parser.parse_args() if len(args)!=1: parser.print_help() sys.exit(1) user = args[0] dir = castorBaseDir(user) if options.castorpath: dir = castortools.lfnToCastor( dir ) print dir
default=None, help="The user space to write into") parser.add_option_group(group) (opts, datasets) = parser.parse_args() import ConfigParser config = ConfigParser.SafeConfigParser() #set options to be case sensitive config.optionxform = str config.add_section('MULTICRAB') config.set('MULTICRAB', 'cfg', opts.cfg) from CMGTools.Production.castorBaseDir import castorBaseDir import CMGTools.Production.eostools as castortools topdir = castortools.lfnToCastor(castorBaseDir(user=opts.user)) output_dirs = [] for d in datasets: #accept the user%dataset syntax, but ignore the user name for grid tokens = d.split('%') if len(tokens) == 2: d = tokens[1] safe_name = d.replace('/', '_') if safe_name.startswith('_'): safe_name = safe_name[1:] if safe_name.endswith('_'): safe_name = safe_name[:-1]
parser.print_help() sys.exit(1) # For multiple file input file = open(args[0], 'r') lines = file.readlines() for line in lines: line = re.sub("\s+", " ", line) fileown = line.split("%")[0].lstrip().rstrip() dataset = line.split("%")[1].lstrip().rstrip() files = [] if re.search('group',fileown): castor = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown))+dataset castor2 = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown.strip("_group")))+dataset else: castor = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown))+dataset castor2 = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown+"_group"))+dataset LFN = eostools.eosToLFN(castor) LFN2 = eostools.eosToLFN(castor2) if eostools.isDirectory(castor): files = burrow(LFN) elif eostools.isDirectory(castor2): files = burrow(LFN2) print dataset for i in files: print "\t"+i
if len(args)!=1: parser.print_help() sys.exit(1) sampleName = args[0] pattern = fnmatch.translate( options.wildcard ) # preparing castor dir ----------------- import CMGTools.Production.castorBaseDir as castorBaseDir try: cdir = castorBaseDir.castorBaseDir( options.user ) except: print 'user does not have a castor base dir' sys.exit(1) cdir += sampleName if not castortools.fileExists( cdir ): print 'Directory ', cdir, 'does not exist' print 'Please check the sample name, and the user. You can specify a different user using the -u option' sys.exit(2) # making local source directory --------- ldir = "./"+sampleName
) parser.add_option_group(group) (options, args) = parser.parse_args() cmgdbApi = CmgdbApi(options.development) cmgdbApi.connect() columns, rows = cmgdbApi.sql( "select file_owner, path_name from dataset_details where dataset_is_open='Y' and (file_owner='cmgtools' or file_owner='cmgtools_group')" ) f = open(options.filename, "w") for row in rows: fileown = row[0] dsName = row[1] if fileown == "--cmgtools": fileown = "cmgtools" lfnDir = castorBaseDir(fileown) + dsName castorDir = castortools.lfnToCastor(lfnDir) if castortools.datasetNotEmpty(castorDir, ".*root"): continue else: f.write(fileown + "%" + dsName + "\n") f.close()