def read(self, lfn, local=False):
"""Reads a report from storage"""
if local:
cat = file(lfn).read()
else:
cat = castortools.cat(castortools.lfnToCastor(lfn))
return json.loads(cat)
def publish(self, report):
"""Publish a file"""
for path in report['PathList']:
_, name = tempfile.mkstemp('.txt', text=True)
json.dump(report, file(name,'w'), sort_keys=True, indent=4)
fname = '%s_%s.txt' % (self.parent, report['DateCreated'])
#rename the file locally - TODO: This is a potential problem
nname = os.path.join(os.path.dirname(name),fname)
os.rename(name, nname)
castor_path = castortools.lfnToCastor(path)
new_name = '%s/%s' % (castor_path, fname)
castortools.xrdcp(nname,path)
time.sleep(1)
if castortools.fileExists(new_name):
#castortools.move(old_name, new_name)
#castortools.chmod(new_name, '644')
print "File published: '%s'" % castortools.castorToLFN(new_name)
os.remove(nname)
else:
pathhash = path.replace('/','.')
hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname)
shutil.move(nname, hashed_name)
print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % (castor_path, hashed_name)
def publish(self, report):
"""Publish a file"""
for path in report['PathList']:
_, name = tempfile.mkstemp('.txt', text=True)
json.dump(report, file(name, 'w'), sort_keys=True, indent=4)
fname = '%s_%s.txt' % (self.parent, report['DateCreated'])
#rename the file locally - TODO: This is a potential problem
nname = os.path.join(os.path.dirname(name), fname)
os.rename(name, nname)
castor_path = castortools.lfnToCastor(path)
new_name = '%s/%s' % (castor_path, fname)
castortools.xrdcp(nname, path)
time.sleep(1)
if castortools.fileExists(new_name):
#castortools.move(old_name, new_name)
#castortools.chmod(new_name, '644')
print "File published: '%s'" % castortools.castorToLFN(
new_name)
os.remove(nname)
else:
pathhash = path.replace('/', '.')
hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname)
shutil.move(nname, hashed_name)
print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % (
castor_path, hashed_name)
def read(self, lfn, local = False):
"""Reads a report from storage"""
if local:
cat = file(lfn).read()
else:
cat = castortools.cat(castortools.lfnToCastor(lfn))
return json.loads(cat)
def run(self, input):
if self.user == 'CMS':
return {'Topdir':None,'Directory':None}
topdir = castortools.lfnToCastor(castorBaseDir(user=self.user))
directory = '%s/%s' % (topdir,self.dataset)
# directory = directory.replace('//','/')
if not castortools.fileExists(directory):
if hasattr(self,'create') and self.create:
castortools.createCastorDir(directory)
#castortools.chmod(directory,'775')
if not castortools.isDirectory(directory):
raise Exception("Dataset directory '%s' does not exist or could not be created" % directory)
return {'Topdir':topdir,'Directory':directory}
def run(self, input):
if self.user == 'CMS':
return {'Topdir':None,'Directory':None}
topdir = castortools.lfnToCastor(castorBaseDir(user=self.user))
directory = '%s/%s' % (topdir,self.dataset)
directory = directory.replace('//','/')
if not castortools.fileExists(directory):
if hasattr(self,'create') and self.create:
castortools.createCastorDir(directory)
#castortools.chmod(directory,'775')
if not castortools.isDirectory(directory):
raise Exception("Dataset directory '%s' does not exist or could not be created" % directory)
return {'Topdir':topdir,'Directory':directory}
def __init__(self, dataset, options):
if not dataset.startswith(os.sep):
dataset = os.sep + dataset
self.dataset = dataset
self.options = options
self.topdir = castortools.lfnToCastor(castorBaseDir(user=options.user))
self.directory = os.path.join(self.topdir, *self.dataset.split(os.sep))
#event counters
self.eventsTotal = -1
self.eventsSeen = 0
self.test_result = None
def __init__(self, dataset, options):
if not dataset.startswith(os.sep):
dataset = os.sep + dataset
self.dataset = dataset
self.options = options
self.topdir = castortools.lfnToCastor( castorBaseDir(user=options.user) )
self.directory = os.path.join(self.topdir, *self.dataset.split(os.sep))
#event counters
self.eventsTotal = -1
self.eventsSeen = 0
self.test_result = None
def castorBaseDir( user=os.environ['USER'], area = None):
"""Gets the top level directory to use for writing for 'user'"""
if area is None:
user, area = getUserAndArea(user)
d = '/store/cmst3/%s/%s/CMG' % (area,user)
exists = castortools.isDirectory( castortools.lfnToCastor(d) )
if exists:
return d
else:
msg = "The directory '%s' does not exist. Please check the username and area (user/group). You may need to create the directory yourself." % d
print >> sys.stderr, msg
raise NameError(msg)
def castorBaseDir(user=os.environ['USER'], area=None):
"""Gets the top level directory to use for writing for 'user'"""
if area is None:
user, area = getUserAndArea(user)
d = 'root://eoscms.cern.ch//eos/cms/store/cmst3/%s/%s/CMG' % (area, user)
exists = castortools.isDirectory(castortools.lfnToCastor(d))
if exists:
return d
else:
msg = "The directory '%s' does not exist. Please check the username and area (user/group). You may need to create the directory yourself." % d
print >> sys.stderr, msg
raise NameError(msg)
default="empty_datasets.txt")
parser.add_option_group(group)
(options, args) = parser.parse_args()
cmgdbApi = CmgdbApi(options.development)
cmgdbApi.connect()
columns, rows = cmgdbApi.sql(
"select file_owner, path_name from dataset_details where dataset_is_open='Y' and (file_owner='cmgtools' or file_owner='cmgtools_group')"
)
f = open(options.filename, 'w')
for row in rows:
fileown = row[0]
dsName = row[1]
if fileown == "--cmgtools":
fileown = "cmgtools"
lfnDir = castorBaseDir(fileown) + dsName
castorDir = castortools.lfnToCastor(lfnDir)
if castortools.datasetNotEmpty(castorDir, ".*root"):
continue
else:
f.write(fileown + "%" + dsName + '\n')
f.close()
default=None,
help="The user space to write into")
parser.add_option_group(group)
(opts, datasets) = parser.parse_args()
import ConfigParser
config = ConfigParser.SafeConfigParser()
#set options to be case sensitive
config.optionxform = str
config.add_section('MULTICRAB')
config.set('MULTICRAB', 'cfg', opts.cfg)
from CMGTools.Production.castorBaseDir import castorBaseDir
import CMGTools.Production.eostools as castortools
topdir = castortools.lfnToCastor(castorBaseDir(user=opts.user))
output_dirs = []
for d in datasets:
#accept the user%dataset syntax, but ignore the user name for grid
tokens = d.split('%')
if len(tokens) == 2:
d = tokens[1]
safe_name = d.replace('/', '_')
if safe_name.startswith('_'):
safe_name = safe_name[1:]
if safe_name.endswith('_'):
safe_name = safe_name[:-1]
savpass = getpass.getpass("Enter Savannah Password: "******""
if options.commented:
comment = raw_input("Please enter your comment here:")
# Store full dataset name
dirOrFile = args[0]
# Castor base directory for specified user
baseDir = castortools.lfnToCastor(castorBaseDir.castorBaseDir(
options.user))
#get the user, after taking account of the user_area hack
user, _ = castorBaseDir.getUserAndArea(options.user)
# Locations of Logger(.tgz) and containing folder on Castor
targetDir = baseDir + "/" + args[0].lstrip("/").rstrip("/")
targetTgz = targetDir + "/Logger.tgz"
# Get array containing 1) Primary dataset name; 2) Dataset Name; 3+) Tiers
details = dirOrFile.lstrip("/").split("/")
# If element is not Primary dataset of dataset name, add to the Tiers array
count = 0
tiers = []
for i in details:
if count > 1:
def __init__(self, name, user, pattern='.*root'):
self.lfnDir = castorBaseDir(user) + name
self.castorDir = castortools.lfnToCastor( self.lfnDir )
self.maskExists = False
self.report = None
super(Dataset, self).__init__(name, user, pattern)
castorDir = ""
localDir = ""
# opions.user could be of the form user_area
user,area = cBaseDir.getUserAndArea(options.user)
dataSets = '/afs/cern.ch/user/{first}/{user}/public/DataSets.txt'.format(
first = user[0], # first letter of the username
user = user
)
ifile=open(dataSets,'r')
pattern = re.compile( args[0] )
castorBaseDir = castortools.lfnToCastor(cBaseDir.castorBaseDir( options.user ))
for line in ifile.readlines():
line = line.rstrip()
if len(line)==0 or line[0]!='/': continue
if pattern.search( line ):
sampleName = line
try:
castorDir = castorBaseDir + sampleName
except:
sys.exit(1)
localDir = None
try:
localDir = os.environ['CMGLOCALBASEDIR']
localDir += sampleName
except:
if options.tier != "":
sampleNameDir += "/" + options.tier
# testing that the crab file exists
try:
oldCrab = open('crab.cfg','r')
except Exception, e:
print "Cannot find crab.cfg file in current directory. Error was '%s'." % str(e)
sys.exit(1)
# preparing castor dir -----------------
import castorBaseDir
cdir = castortools.lfnToCastor( castorBaseDir.castorBaseDir(user=options.user) )
cdir += sampleNameDir
if castortools.isCastorFile( cdir ) and not options.force:
print 'The destination castor directory already exists:'
print cdir
print 'Please check. If everything is fine, run again with the -f option.'
sys.exit(1)
rfmkdir = 'rfmkdir -m 775 -p ' + cdir
print rfmkdir
castortools.createCastorDir(cdir)
castortools.chmod(cdir, '775')
# making local crab directory ---------
ldir = '.' + sampleNameDir
def __init__(self, name, user, pattern='.*root'):
self.lfnDir = castorBaseDir(user) + name
self.castorDir = castortools.lfnToCastor(self.lfnDir)
self.maskExists = False
self.report = None
super(Dataset, self).__init__(name, user, pattern)
#!/usr/bin/env python
import CMGTools.Production.eostools as castortools
from CMGTools.Production.castorBaseDir import castorBaseDir
if __name__ == '__main__':
import sys
from optparse import OptionParser
parser = OptionParser()
parser.usage = "%prog <user>\nPrints the castor base directory of a given user."
parser.add_option("-c", "--castorpath", action="store_true",
dest="castorpath",
help="Print full castor path. Otherwise print LFN, starting by /store",
default=False)
(options,args) = parser.parse_args()
if len(args)!=1:
parser.print_help()
sys.exit(1)
user = args[0]
dir = castorBaseDir(user)
if options.castorpath:
dir = castortools.lfnToCastor( dir )
print dir
group.add_option("-t", "--tier", dest="tier", default='',help="The data tier to use, e.g. 'PAT_CMG_5_6_0'")
group.add_option("-u", "--user", dest="user", default=None,help="The user space to write into")
parser.add_option_group(group)
(opts, datasets) = parser.parse_args()
import ConfigParser
config = ConfigParser.SafeConfigParser()
#set options to be case sensitive
config.optionxform = str
config.add_section('MULTICRAB')
config.set('MULTICRAB','cfg',opts.cfg)
from CMGTools.Production.castorBaseDir import castorBaseDir
import CMGTools.Production.eostools as castortools
topdir = castortools.lfnToCastor(castorBaseDir(user=opts.user))
output_dirs = []
for d in datasets:
#accept the user%dataset syntax, but ignore the user name for grid
tokens = d.split('%')
if len(tokens) == 2:
d = tokens[1]
safe_name = d.replace('/','_')
if safe_name.startswith('_'):
safe_name = safe_name[1:]
if safe_name.endswith('_'):
safe_name = safe_name[:-1]
)
parser.add_option_group(group)
(options, args) = parser.parse_args()
cmgdbApi = CmgdbApi(options.development)
cmgdbApi.connect()
columns, rows = cmgdbApi.sql(
"select file_owner, path_name from dataset_details where dataset_is_open='Y' and (file_owner='cmgtools' or file_owner='cmgtools_group')"
)
f = open(options.filename, "w")
for row in rows:
fileown = row[0]
dsName = row[1]
if fileown == "--cmgtools":
fileown = "cmgtools"
lfnDir = castorBaseDir(fileown) + dsName
castorDir = castortools.lfnToCastor(lfnDir)
if castortools.datasetNotEmpty(castorDir, ".*root"):
continue
else:
f.write(fileown + "%" + dsName + "\n")
f.close()
