def run(self, input):
if self.user == 'CMS':
return {'Topdir':None,'Directory':None}
topdir = castortools.lfnToCastor(castorBaseDir(user=self.user))
directory = '%s/%s' % (topdir,self.dataset)
# directory = directory.replace('//','/')
if not castortools.fileExists(directory):
if hasattr(self,'create') and self.create:
castortools.createCastorDir(directory)
#castortools.chmod(directory,'775')
if not castortools.isDirectory(directory):
raise Exception("Dataset directory '%s' does not exist or could not be created" % directory)
return {'Topdir':topdir,'Directory':directory}
def run(self, input):
if self.user == 'CMS':
return {'Topdir':None,'Directory':None}
topdir = castortools.lfnToCastor(castorBaseDir(user=self.user))
directory = '%s/%s' % (topdir,self.dataset)
directory = directory.replace('//','/')
if not castortools.fileExists(directory):
if hasattr(self,'create') and self.create:
castortools.createCastorDir(directory)
#castortools.chmod(directory,'775')
if not castortools.isDirectory(directory):
raise Exception("Dataset directory '%s' does not exist or could not be created" % directory)
return {'Topdir':topdir,'Directory':directory}
def __init__(self, dataset, options):
if not dataset.startswith(os.sep):
dataset = os.sep + dataset
self.dataset = dataset
self.options = options
self.topdir = castortools.lfnToCastor( castorBaseDir(user=options.user) )
self.directory = os.path.join(self.topdir, *self.dataset.split(os.sep))
#event counters
self.eventsTotal = -1
self.eventsSeen = 0
self.test_result = None
def __init__(self, dataset, options):
if not dataset.startswith(os.sep):
dataset = os.sep + dataset
self.dataset = dataset
self.options = options
self.topdir = castortools.lfnToCastor(castorBaseDir(user=options.user))
self.directory = os.path.join(self.topdir, *self.dataset.split(os.sep))
#event counters
self.eventsTotal = -1
self.eventsSeen = 0
self.test_result = None
group.add_option("-t", "--tier", dest="tier", default='',help="The data tier to use, e.g. 'PAT_CMG_5_6_0'")
group.add_option("-u", "--user", dest="user", default=None,help="The user space to write into")
parser.add_option_group(group)
(opts, datasets) = parser.parse_args()
import ConfigParser
config = ConfigParser.SafeConfigParser()
#set options to be case sensitive
config.optionxform = str
config.add_section('MULTICRAB')
config.set('MULTICRAB','cfg',opts.cfg)
from CMGTools.Production.castorBaseDir import castorBaseDir
import CMGTools.Production.eostools as castortools
topdir = castortools.lfnToCastor(castorBaseDir(user=opts.user))
output_dirs = []
for d in datasets:
#accept the user%dataset syntax, but ignore the user name for grid
tokens = d.split('%')
if len(tokens) == 2:
d = tokens[1]
safe_name = d.replace('/','_')
if safe_name.startswith('_'):
safe_name = safe_name[1:]
if safe_name.endswith('_'):
safe_name = safe_name[:-1]
def __init__(self, name, user, pattern='.*root'):
self.lfnDir = castorBaseDir(user) + name
self.castorDir = castortools.lfnToCastor(self.lfnDir)
self.maskExists = False
self.report = None
super(Dataset, self).__init__(name, user, pattern)
if len(args) != 1:
parser.print_help()
sys.exit(1)
# For multiple file input
file = open(args[0], 'r')
lines = file.readlines()
for line in lines:
line = re.sub("\s+", " ", line)
fileown = line.split("%")[0].lstrip().rstrip()
dataset = line.split("%")[1].lstrip().rstrip()
files = []
if re.search('group', fileown):
castor = eostools.lfnToEOS(
castorBaseDir.castorBaseDir(fileown)) + dataset
castor2 = eostools.lfnToEOS(
castorBaseDir.castorBaseDir(fileown.strip("_group"))) + dataset
else:
castor = eostools.lfnToEOS(
castorBaseDir.castorBaseDir(fileown)) + dataset
castor2 = eostools.lfnToEOS(
castorBaseDir.castorBaseDir(fileown + "_group")) + dataset
LFN = eostools.eosToLFN(castor)
LFN2 = eostools.eosToLFN(castor2)
if eostools.isDirectory(castor):
files = burrow(LFN)
elif eostools.isDirectory(castor2):
files = burrow(LFN2)
savpass = getpass.getpass("Enter Savannah Password: "******""
if options.commented:
comment = raw_input("Please enter your comment here:")
# Store full dataset name
dirOrFile = args[0]
# Castor base directory for specified user
baseDir = castortools.lfnToCastor(castorBaseDir.castorBaseDir(
options.user))
#get the user, after taking account of the user_area hack
user, _ = castorBaseDir.getUserAndArea(options.user)
# Locations of Logger(.tgz) and containing folder on Castor
targetDir = baseDir + "/" + args[0].lstrip("/").rstrip("/")
targetTgz = targetDir + "/Logger.tgz"
# Get array containing 1) Primary dataset name; 2) Dataset Name; 3+) Tiers
details = dirOrFile.lstrip("/").split("/")
# If element is not Primary dataset of dataset name, add to the Tiers array
count = 0
tiers = []
for i in details:
if count > 1:
def __init__(self, name, user, pattern='.*root'):
self.lfnDir = castorBaseDir(user) + name
self.castorDir = castortools.lfnToCastor( self.lfnDir )
self.maskExists = False
self.report = None
super(Dataset, self).__init__(name, user, pattern)
castorDir = ""
localDir = ""
# opions.user could be of the form user_area
user,area = cBaseDir.getUserAndArea(options.user)
dataSets = '/afs/cern.ch/user/{first}/{user}/public/DataSets.txt'.format(
first = user[0], # first letter of the username
user = user
)
ifile=open(dataSets,'r')
pattern = re.compile( args[0] )
castorBaseDir = castortools.lfnToCastor(cBaseDir.castorBaseDir( options.user ))
for line in ifile.readlines():
line = line.rstrip()
if len(line)==0 or line[0]!='/': continue
if pattern.search( line ):
sampleName = line
try:
castorDir = castorBaseDir + sampleName
except:
sys.exit(1)
localDir = None
try:
localDir = os.environ['CMGLOCALBASEDIR']
localDir += sampleName
except:
default="empty_datasets.txt")
parser.add_option_group(group)
(options, args) = parser.parse_args()
cmgdbApi = CmgdbApi(options.development)
cmgdbApi.connect()
columns, rows = cmgdbApi.sql(
"select file_owner, path_name from dataset_details where dataset_is_open='Y' and (file_owner='cmgtools' or file_owner='cmgtools_group')"
)
f = open(options.filename, 'w')
for row in rows:
fileown = row[0]
dsName = row[1]
if fileown == "--cmgtools":
fileown = "cmgtools"
lfnDir = castorBaseDir(fileown) + dsName
castorDir = castortools.lfnToCastor(lfnDir)
if castortools.datasetNotEmpty(castorDir, ".*root"):
continue
else:
f.write(fileown + "%" + dsName + '\n')
f.close()
#!/usr/bin/env python
import CMGTools.Production.eostools as castortools
from CMGTools.Production.castorBaseDir import castorBaseDir
if __name__ == '__main__':
import sys
from optparse import OptionParser
parser = OptionParser()
parser.usage = "%prog <user>\nPrints the castor base directory of a given user."
parser.add_option("-c", "--castorpath", action="store_true",
dest="castorpath",
help="Print full castor path. Otherwise print LFN, starting by /store",
default=False)
(options,args) = parser.parse_args()
if len(args)!=1:
parser.print_help()
sys.exit(1)
user = args[0]
dir = castorBaseDir(user)
if options.castorpath:
dir = castortools.lfnToCastor( dir )
print dir
default=None,
help="The user space to write into")
parser.add_option_group(group)
(opts, datasets) = parser.parse_args()
import ConfigParser
config = ConfigParser.SafeConfigParser()
#set options to be case sensitive
config.optionxform = str
config.add_section('MULTICRAB')
config.set('MULTICRAB', 'cfg', opts.cfg)
from CMGTools.Production.castorBaseDir import castorBaseDir
import CMGTools.Production.eostools as castortools
topdir = castortools.lfnToCastor(castorBaseDir(user=opts.user))
output_dirs = []
for d in datasets:
#accept the user%dataset syntax, but ignore the user name for grid
tokens = d.split('%')
if len(tokens) == 2:
d = tokens[1]
safe_name = d.replace('/', '_')
if safe_name.startswith('_'):
safe_name = safe_name[1:]
if safe_name.endswith('_'):
safe_name = safe_name[:-1]
parser.print_help()
sys.exit(1)
# For multiple file input
file = open(args[0], 'r')
lines = file.readlines()
for line in lines:
line = re.sub("\s+", " ", line)
fileown = line.split("%")[0].lstrip().rstrip()
dataset = line.split("%")[1].lstrip().rstrip()
files = []
if re.search('group',fileown):
castor = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown))+dataset
castor2 = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown.strip("_group")))+dataset
else:
castor = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown))+dataset
castor2 = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown+"_group"))+dataset
LFN = eostools.eosToLFN(castor)
LFN2 = eostools.eosToLFN(castor2)
if eostools.isDirectory(castor):
files = burrow(LFN)
elif eostools.isDirectory(castor2):
files = burrow(LFN2)
print dataset
for i in files: print "\t"+i
if len(args)!=1:
parser.print_help()
sys.exit(1)
sampleName = args[0]
pattern = fnmatch.translate( options.wildcard )
# preparing castor dir -----------------
import CMGTools.Production.castorBaseDir as castorBaseDir
try:
cdir = castorBaseDir.castorBaseDir( options.user )
except:
print 'user does not have a castor base dir'
sys.exit(1)
cdir += sampleName
if not castortools.fileExists( cdir ):
print 'Directory ', cdir, 'does not exist'
print 'Please check the sample name, and the user. You can specify a different user using the -u option'
sys.exit(2)
# making local source directory ---------
ldir = "./"+sampleName
)
parser.add_option_group(group)
(options, args) = parser.parse_args()
cmgdbApi = CmgdbApi(options.development)
cmgdbApi.connect()
columns, rows = cmgdbApi.sql(
"select file_owner, path_name from dataset_details where dataset_is_open='Y' and (file_owner='cmgtools' or file_owner='cmgtools_group')"
)
f = open(options.filename, "w")
for row in rows:
fileown = row[0]
dsName = row[1]
if fileown == "--cmgtools":
fileown = "cmgtools"
lfnDir = castorBaseDir(fileown) + dsName
castorDir = castortools.lfnToCastor(lfnDir)
if castortools.datasetNotEmpty(castorDir, ".*root"):
continue
else:
f.write(fileown + "%" + dsName + "\n")
f.close()
