def ParseOptions(self):
(self.options_,self.args_) = self.parser_.parse_args()
if self.options_.remoteCopy == None:
self.remoteOutputDir_ = ""
else:
# removing possible trailing slash
self.remoteOutputDir_ = self.options_.remoteCopy.rstrip('/')
if not castortools.isLFN( self.remoteOutputDir_ ):
print 'When providing an output directory, you must give its LFN, starting by /store. You gave:'
print self.remoteOutputDir_
sys.exit(1)
self.remoteOutputDir_ = castortools.lfnToEOS( self.remoteOutputDir_ )
dirExist = castortools.isDirectory( self.remoteOutputDir_ )
# nsls = 'nsls %s > /dev/null' % self.remoteOutputDir_
# dirExist = os.system( nsls )
if dirExist is False:
print 'creating ', self.remoteOutputDir_
if castortools.isEOSFile( self.remoteOutputDir_ ):
# the output directory is currently a file..
# need to remove it.
castortools.rm( self.remoteOutputDir_ )
castortools.createEOSDir( self.remoteOutputDir_ )
else:
# directory exists.
if self.options_.negate is False and self.options_.force is False:
#COLIN need to reimplement protectedRemove in eostools
raise ValueError( ' '.join(['directory ', self.remoteOutputDir_, ' already exists.']))
# if not castortools.protectedRemove( self.remoteOutputDir_, '.*root'):
# the user does not want to delete the root files
self.remoteOutputFile_ = ""
self.ManageOutputDir()
return (self.options_, self.args_)
def ParseOptions(self):
(self.options_, self.args_) = self.parser_.parse_args()
if self.options_.remoteCopy == None:
self.remoteOutputDir_ = ""
else:
# removing possible trailing slash
import CMGTools.Production.eostools as castortools
self.remoteOutputDir_ = self.options_.remoteCopy.rstrip('/')
if "psi.ch" in self.remoteOutputDir_: # T3 @ PSI:
# overwriting protection to be improved
if self.remoteOutputDir_.startswith("/pnfs/psi.ch"):
os.system("gfal-mkdir srm://t3se01.psi.ch/" +
self.remoteOutputDir_)
outputDir = self.options_.outputDir
if outputDir == None:
today = datetime.today()
outputDir = 'OutCmsBatch_%s' % today.strftime(
"%d%h%y_%H%M")
self.remoteOutputDir_ += "/" + outputDir
os.system("gfal-mkdir srm://t3se01.psi.ch/" +
self.remoteOutputDir_)
else:
print "remote directory must start with /pnfs/psi.ch to send to the tier3 at PSI"
print self.remoteOutputDir_, "not valid"
sys.exit(1)
else: # assume EOS
if not castortools.isLFN(self.remoteOutputDir_):
print 'When providing an output directory, you must give its LFN, starting by /store. You gave:'
print self.remoteOutputDir_
sys.exit(1)
self.remoteOutputDir_ = castortools.lfnToEOS(
self.remoteOutputDir_)
dirExist = castortools.isDirectory(self.remoteOutputDir_)
# nsls = 'nsls %s > /dev/null' % self.remoteOutputDir_
# dirExist = os.system( nsls )
if dirExist is False:
print 'creating ', self.remoteOutputDir_
if castortools.isEOSFile(self.remoteOutputDir_):
# the output directory is currently a file..
# need to remove it.
castortools.rm(self.remoteOutputDir_)
castortools.createEOSDir(self.remoteOutputDir_)
else:
# directory exists.
if self.options_.negate is False and self.options_.force is False:
#COLIN need to reimplement protectedRemove in eostools
raise ValueError(' '.join([
'directory ', self.remoteOutputDir_,
' already exists.'
]))
# if not castortools.protectedRemove( self.remoteOutputDir_, '.*root'):
# the user does not want to delete the root files
self.remoteOutputFile_ = ""
self.ManageOutputDir()
return (self.options_, self.args_)
def ParseOptions(self):
(self.options_, self.args_) = self.parser_.parse_args()
if self.options_.remoteCopy == None:
self.remoteOutputDir_ = ""
else:
# removing possible trailing slash
self.remoteOutputDir_ = self.options_.remoteCopy.rstrip("/")
if "psi.ch" in self.remoteOutputDir_: # T3 @ PSI:
# overwriting protection to be improved
if self.remoteOutputDir_.startswith("/pnfs/psi.ch"):
ld_lib_path = os.environ.get("LD_LIBRARY_PATH")
if ld_lib_path != "None":
os.environ["LD_LIBRARY_PATH"] = (
"/usr/lib64/:" + ld_lib_path
) # to solve gfal conflict with CMSSW
os.system("gfal-mkdir srm://t3se01.psi.ch/" + self.remoteOutputDir_)
outputDir = self.options_.outputDir
if outputDir == None:
today = datetime.today()
outputDir = "OutCmsBatch_%s" % today.strftime("%d%h%y_%H%M")
self.remoteOutputDir_ += "/" + outputDir
os.system("gfal-mkdir srm://t3se01.psi.ch/" + self.remoteOutputDir_)
if ld_lib_path != "None":
os.environ["LD_LIBRARY_PATH"] = ld_lib_path # back to original to avoid conflicts
else:
print "remote directory must start with /pnfs/psi.ch to send to the tier3 at PSI"
print self.remoteOutputDir_, "not valid"
sys.exit(1)
else: # assume EOS
if not castortools.isLFN(self.remoteOutputDir_):
print "When providing an output directory, you must give its LFN, starting by /store. You gave:"
print self.remoteOutputDir_
sys.exit(1)
self.remoteOutputDir_ = castortools.lfnToEOS(self.remoteOutputDir_)
dirExist = castortools.isDirectory(self.remoteOutputDir_)
# nsls = 'nsls %s > /dev/null' % self.remoteOutputDir_
# dirExist = os.system( nsls )
if dirExist is False:
print "creating ", self.remoteOutputDir_
if castortools.isEOSFile(self.remoteOutputDir_):
# the output directory is currently a file..
# need to remove it.
castortools.rm(self.remoteOutputDir_)
castortools.createEOSDir(self.remoteOutputDir_)
else:
# directory exists.
if self.options_.negate is False and self.options_.force is False:
# COLIN need to reimplement protectedRemove in eostools
raise ValueError(" ".join(["directory ", self.remoteOutputDir_, " already exists."]))
# if not castortools.protectedRemove( self.remoteOutputDir_, '.*root'):
# the user does not want to delete the root files
self.remoteOutputFile_ = ""
self.ManageOutputDir()
return (self.options_, self.args_)
def run(self, input):
if self.user == 'CMS':
return {'Topdir':None,'Directory':None}
topdir = castortools.lfnToCastor(castorBaseDir(user=self.user))
directory = '%s/%s' % (topdir,self.dataset)
directory = directory.replace('//','/')
if not castortools.fileExists(directory):
if hasattr(self,'create') and self.create:
castortools.createCastorDir(directory)
#castortools.chmod(directory,'775')
if not castortools.isDirectory(directory):
raise Exception("Dataset directory '%s' does not exist or could not be created" % directory)
return {'Topdir':topdir,'Directory':directory}
def run(self, input):
if self.user == 'CMS':
return {'Topdir':None,'Directory':None}
topdir = castortools.lfnToCastor(castorBaseDir(user=self.user))
directory = '%s/%s' % (topdir,self.dataset)
# directory = directory.replace('//','/')
if not castortools.fileExists(directory):
if hasattr(self,'create') and self.create:
castortools.createCastorDir(directory)
#castortools.chmod(directory,'775')
if not castortools.isDirectory(directory):
raise Exception("Dataset directory '%s' does not exist or could not be created" % directory)
return {'Topdir':topdir,'Directory':directory}
def castorBaseDir( user=os.environ['USER'], area = None):
"""Gets the top level directory to use for writing for 'user'"""
if area is None:
user, area = getUserAndArea(user)
d = '/store/cmst3/%s/%s/CMG' % (area,user)
exists = castortools.isDirectory( castortools.lfnToCastor(d) )
if exists:
return d
else:
msg = "The directory '%s' does not exist. Please check the username and area (user/group). You may need to create the directory yourself." % d
print >> sys.stderr, msg
raise NameError(msg)
def castorBaseDir(user=os.environ['USER'], area=None):
"""Gets the top level directory to use for writing for 'user'"""
if area is None:
user, area = getUserAndArea(user)
d = 'root://eoscms.cern.ch//eos/cms/store/cmst3/%s/%s/CMG' % (area, user)
exists = castortools.isDirectory(castortools.lfnToCastor(d))
if exists:
return d
else:
msg = "The directory '%s' does not exist. Please check the username and area (user/group). You may need to create the directory yourself." % d
print >> sys.stderr, msg
raise NameError(msg)
def ParseOptions(self):
(self.options_,self.args_) = self.parser_.parse_args()
if self.options_.remoteCopy == None:
self.remoteOutputDir_ = ""
else:
# removing possible trailing slash
import CMGTools.Production.eostools as castortools
self.remoteOutputDir_ = self.options_.remoteCopy.rstrip('/')
if "psi.ch" in self.remoteOutputDir_: # T3 @ PSI:
# overwriting protection to be improved
if self.remoteOutputDir_.startswith("/pnfs/psi.ch"):
os.system("gfal-mkdir srm://t3se01.psi.ch/"+self.remoteOutputDir_)
outputDir = self.options_.outputDir
if outputDir==None:
today = datetime.today()
outputDir = 'OutCmsBatch_%s' % today.strftime("%d%h%y_%H%M")
self.remoteOutputDir_+="/"+outputDir
os.system("gfal-mkdir srm://t3se01.psi.ch/"+self.remoteOutputDir_)
else:
print "remote directory must start with /pnfs/psi.ch to send to the tier3 at PSI"
print self.remoteOutputDir_, "not valid"
sys.exit(1)
else: # assume EOS
if not castortools.isLFN( self.remoteOutputDir_ ):
print 'When providing an output directory, you must give its LFN, starting by /store. You gave:'
print self.remoteOutputDir_
sys.exit(1)
self.remoteOutputDir_ = castortools.lfnToEOS( self.remoteOutputDir_ )
dirExist = castortools.isDirectory( self.remoteOutputDir_ )
# nsls = 'nsls %s > /dev/null' % self.remoteOutputDir_
# dirExist = os.system( nsls )
if dirExist is False:
print 'creating ', self.remoteOutputDir_
if castortools.isEOSFile( self.remoteOutputDir_ ):
# the output directory is currently a file..
# need to remove it.
castortools.rm( self.remoteOutputDir_ )
castortools.createEOSDir( self.remoteOutputDir_ )
else:
# directory exists.
if self.options_.negate is False and self.options_.force is False:
#COLIN need to reimplement protectedRemove in eostools
raise ValueError( ' '.join(['directory ', self.remoteOutputDir_, ' already exists.']))
# if not castortools.protectedRemove( self.remoteOutputDir_, '.*root'):
# the user does not want to delete the root files
self.remoteOutputFile_ = ""
self.ManageOutputDir()
return (self.options_, self.args_)
directory = '%s/%s' % (topdir,d)
if opts.tier:
directory = os.path.join(directory,opts.tier)
directory = directory.replace('//','/')
config.set(safe_name,'CMSSW.datasetpath',d)
lfn = castortools.castorToLFN(directory)
config.set(safe_name,'USER.user_remote_dir',lfn)
output_dirs.append(lfn)
#create the directory on EOS
if not castortools.fileExists(directory):
castortools.createCastorDir(directory)
castortools.chmod(directory,'775')
if not castortools.isDirectory(directory):
raise Exception("Dataset directory '%s' does not exist or could not be created" % directory)
config.write(file(opts.output,'wb'))
from logger import logger
logDir = 'Logger'
os.mkdir(logDir)
log = logger( logDir )
log.logCMSSW()
log.addFile( os.path.join( os.getcwd(), opts.cfg) )
log.addFile( os.path.join( os.getcwd(), opts.output) )
for d in output_dirs:
log.stageOut(d)
lines = file.readlines()
for line in lines:
line = re.sub("\s+", " ", line)
fileown = line.split("%")[0].lstrip().rstrip()
dataset = line.split("%")[1].lstrip().rstrip()
files = []
if re.search('group', fileown):
castor = eostools.lfnToEOS(
castorBaseDir.castorBaseDir(fileown)) + dataset
castor2 = eostools.lfnToEOS(
castorBaseDir.castorBaseDir(fileown.strip("_group"))) + dataset
else:
castor = eostools.lfnToEOS(
castorBaseDir.castorBaseDir(fileown)) + dataset
castor2 = eostools.lfnToEOS(
castorBaseDir.castorBaseDir(fileown + "_group")) + dataset
LFN = eostools.eosToLFN(castor)
LFN2 = eostools.eosToLFN(castor2)
if eostools.isDirectory(castor):
files = burrow(LFN)
elif eostools.isDirectory(castor2):
files = burrow(LFN2)
print dataset
for i in files:
print "\t" + i
#print "Dataset: "+dataset+" not found"
for p in self.problem_files:
print >> sys.stderr, p[0],p[1]
if __name__ == '__main__':
parser = OptionParser()
parser.usage = """
%s --src source_dir --dest dest_dir
Both the source and destination must exist
"""
parser.add_option("-s", "--src", dest="source",
help="The source directory", default=None)
parser.add_option("-d", "--dest", dest="dest",
help="The destination directory", default=None)
(options,args) = parser.parse_args()
if options.source is None or options.dest is None:
print >> sys.stderr, 'Both the source and destination must be set'
sys.exit(-1)
if not castortools.isDirectory(options.source) or not castortools.isDirectory(options.dest):
print >> sys.stderr, 'Both the source and destination directories must exist'
sys.exit(-1)
m = Migrate(options.source,options.dest)
m.migrate()
import sys, time
#this taken from the main of cmsStage
argv = sys.argv[1:]
(args, debug, force) = parseOpts(argv)
if not os.path.isfile(args[0]):
print args[0], 'does not exist.'
sys.exit(1)
source = cmsFile(args[0], "rfio")
destination = cmsFile(args[1], "stageout")
checkArgs(source, destination, force)
#find the destination LFN
dest = args[1]
if eostools.isDirectory(dest):
dest = os.path.join(dest, os.path.basename(args[0]))
sleep_lengths = [1, 10, 60, 600, 1800]
return_code = 0
for i in xrange(5):
#sleep for a while before running
time.sleep(sleep_lengths[i])
try:
#run cmsStage
print 'cmsStage %s [%d/5]' % (' '.join(argv), i + 1)
main(argv)
except SystemExit, e:
directory = '%s/%s' % (topdir, d)
if opts.tier:
directory = os.path.join(directory, opts.tier)
directory = directory.replace('//', '/')
config.set(safe_name, 'CMSSW.datasetpath', d)
lfn = castortools.castorToLFN(directory)
config.set(safe_name, 'USER.user_remote_dir', lfn)
output_dirs.append(lfn)
#create the directory on EOS
if not castortools.fileExists(directory):
castortools.createCastorDir(directory)
castortools.chmod(directory, '775')
if not castortools.isDirectory(directory):
raise Exception(
"Dataset directory '%s' does not exist or could not be created"
% directory)
config.write(file(opts.output, 'wb'))
from logger import logger
logDir = 'Logger'
os.mkdir(logDir)
log = logger(logDir)
log.logCMSSW()
log.addFile(os.path.join(os.getcwd(), opts.cfg))
log.addFile(os.path.join(os.getcwd(), opts.output))
for d in output_dirs:
file = open(args[0], 'r')
lines = file.readlines()
for line in lines:
line = re.sub("\s+", " ", line)
fileown = line.split("%")[0].lstrip().rstrip()
dataset = line.split("%")[1].lstrip().rstrip()
files = []
if re.search('group',fileown):
castor = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown))+dataset
castor2 = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown.strip("_group")))+dataset
else:
castor = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown))+dataset
castor2 = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown+"_group"))+dataset
LFN = eostools.eosToLFN(castor)
LFN2 = eostools.eosToLFN(castor2)
if eostools.isDirectory(castor):
files = burrow(LFN)
elif eostools.isDirectory(castor2):
files = burrow(LFN2)
print dataset
for i in files: print "\t"+i
#print "Dataset: "+dataset+" not found"
import sys, time
#this taken from the main of cmsStage
argv = sys.argv[1:]
(args, debug, force ) = parseOpts( argv )
if not os.path.isfile(args[0]):
print args[0], 'does not exist.'
sys.exit(1)
source = cmsFile( args[0], "rfio" )
destination = cmsFile( args[1], "stageout" )
checkArgs( source, destination, force )
#find the destination LFN
dest = args[1]
if eostools.isDirectory(dest):
dest = os.path.join(dest,os.path.basename(args[0]))
sleep_lengths = [1,10,60,600,1800]
return_code = 0
for i in xrange(5):
#sleep for a while before running
time.sleep(sleep_lengths[i])
try:
#run cmsStage
print 'cmsStage %s [%d/5]' % (' '.join(argv) , i+1)
main(argv)
except SystemExit, e:
