def createFilePath(filePath):
try:
eos.lsl(filePath)
except IOError:
print "filePath = %s does not yet exist, creating it." % filePath
eos.mkdir(filePath)
time.sleep(3)
eos.chmod(filePath, 777)
def createFilePath(filePath):
try:
eos.lsl(filePath)
except IOError:
print "filePath = %s does not yet exist, creating it." % filePath
eos.mkdir(filePath)
time.sleep(3)
eos.chmod(filePath, 777)
def local_version_current(castor_file, local_directory = LOCAL_DIRECTORY):
''' Check if the local copy of [castor_file] exists and is up to date '''
local_file = local_version(castor_file, local_directory)
if not os.path.exists(local_file):
return False
local_stat = os.stat(local_file)
# Get last mod time of local file
#local_mtime = time.ctime(local_stat.st_mtime)
local_mtime = time.localtime(local_stat.st_mtime)
local_size = local_stat.st_size
# This call is memorized
castor_stat = None
if is_on_castor(castor_file):
castor_stat = list(castor.nslsl(castor_file))[0]
elif is_on_eos(castor_file):
castor_stat = list(eos.lsl(castor_file))[0]
else:
raise ValueError("Invalid fileName = %s !!" % castor_file)
castor_size = castor_stat["size"]
#castor_mtime = time.mktime(
# unixtime_from_timestamp(castor_stat["Last modify"]))
castor_mtime = castor_stat['time']
#print local_mtime, castor_mtime
# Check sizes are same
if local_size != castor_size:
print "Local copy of", castor_file, " is the wrong size: %i != %i"% (local_size, castor_size)
return False
# Check local file is newer
if local_mtime < castor_mtime:
print "Local copy of", castor_file, " is outdated!"
print "local:", time.asctime(local_mtime), \
"castor:", time.asctime(castor_mtime)
return False
return True
def expand_file_list(fileEntries):
for fileEntry in fileEntries:
if fileEntry.find("*") != -1:
if is_on_castor(fileEntry):
for file in castor.nslsl(clean_name(fileEntry)):
yield "rfio:" + file['path']
elif is_on_eos(fileEntry):
for file in eos.lsl(clean_name(fileEntry)):
yield file['path']
else:
yield fileEntry
def eos_source(directory):
" Build a generator that lists file in a EOS directory, sorted by time "
print "<eos_source>", directory
# First sort by time
files = list(eos.lsl(directory))
# Sort by time
files.sort(key = lambda x: x['time'])
for file_info in files:
if not file_info['size']:
print "Warning <eos_source>: file %s has size 0" % \
file_info['path']
yield file_info
def eos_source(directory):
" Build a generator that lists file in a EOS directory, sorted by time "
print "<eos_source>", directory
# First sort by time
files = list(eos.lsl(directory))
# Sort by time
files.sort(key=lambda x: x['time'])
for file_info in files:
if not file_info['size']:
print "Warning <eos_source>: file %s has size 0" % \
file_info['path']
yield file_info
if inputFilePath.find('/store/') == 0:
mode = 'eos'
else:
mode = 'local'
if jobId is None:
reg.overrideJobId(
channel, '2011Oct30'
) # CV: need to overwrite this in order to match Mauro's filenames
jobId = reg.getJobId(channel)
print(" jobId = %s" % jobId)
if mode == 'castor':
files = [file_info for file_info in castor.nslsl(inputFilePath)]
elif mode == 'eos':
files = [file_info for file_info in eos.lsl(inputFilePath)]
else:
commandLine = '%s %s' % (options['executable_ls'][mode], inputFilePath)
args = shlex.split(commandLine)
retval = subprocess.Popen(args, stdout=subprocess.PIPE)
#retval.wait()
files = retval.stdout.read().split('\n')
#print(" files = %s" % files)
fileName_regex = r"(?P<fileName_base>[a-zA-Z0-9_]+)_(?P<gridJob>\d*)(_(?P<gridTry>\d*))*_(?P<hash>[a-zA-Z0-9]*).root"
fileName_matcher = re.compile(fileName_regex)
fileNamesAndProperties_dict = {}
print("--> found %i file" % len(files))
currentFilePath = os.path.join(currentFilePath, filePath_item)
if len(currentFilePath) <= 1:
continue
if not os.path.exists(currentFilePath):
os.mkdir(currentFilePath)
if outputFilePath.find("/castor") != -1:
try:
castor.rfstat(outputFilePath)
except RuntimeError:
print "outputFilePath does not yet exist, creating it."
os.system("rfmkdir %s" % outputFilePath)
os.system("rfchmod 777 %s" % outputFilePath)
elif outputFilePath.find("/store") != -1:
try:
eos.lsl(outputFilePath)
except IOError:
print "outputFilePath does not yet exist, creating it."
eos.mkdir(outputFilePath)
else:
hostname = socket.gethostname()
numChars = min(len(hostname), len(outputFileMachine))
if outputFileMachine[:numChars] != hostname[:numChars]:
raise ValueError("Cannot create output file path = %s on machine %s !!" % (outputFilePath, outputFileMachine))
if not os.path.isdir(outputFilePath):
print "outputFilePath does not yet exist, creating it."
createFilePath_recursively(outputFilePath)
if not os.path.isdir("lxbatch"):
os.mkdir('lxbatch')
#--------------------------------------------------------------------------------
# set input files
inputFilePath = '/store/user/veelken/CMSSW_5_3_x/skims/GoldenZmumu/2012Oct09/'
inputFile_regex = r"[a-zA-Z0-9_/:.]*goldenZmumuEvents_ZplusJets_madgraph_2012Oct09_AOD_(?P<gridJob>\d*)(_(?P<gridTry>\d*))*_(?P<hash>[a-zA-Z0-9]*).root"
# check if name of inputFile matches regular expression
inputFileNames = []
files = None
if inputFilePath.startswith('/castor/'):
files = [
"".join(["rfio:", file_info['path']])
for file_info in castor.nslsl(inputFilePath)
]
elif inputFilePath.startswith('/store/'):
files = [file_info['path'] for file_info in eos.lsl(inputFilePath)]
else:
files = [
"".join(["file:", inputFilePath, file])
for file in os.listdir(inputFilePath)
]
for file in files:
#print "file = %s" % file
inputFile_matcher = re.compile(inputFile_regex)
if inputFile_matcher.match(file):
inputFileNames.append(file)
#print "inputFileNames = %s" % inputFileNames
process.source.fileNames = cms.untracked.vstring(inputFileNames)
#--------------------------------------------------------------------------------
raise ValueError("Invalid mass-point = %i !!" % massPoint)
else:
raise ValueError("Invalid channel = %s !!" % channel)
inputFile_regex = \
r"[a-zA-Z0-9_/:.]*genTauLeptonPairSkim_(ggHiggs|ggPhi|vbfHiggs)%i_%s_(?P<gridJob>\d*)(_(?P<gridTry>\d*))*_(?P<hash>[a-zA-Z0-9]*).root" % (massPoint, channel)
else:
raise ValueError("Invalid sample type = %s !!" % sample_type)
# check if name of inputFile matches regular expression
inputFileNames = []
for inputFilePath in inputFilePaths:
files = None
if inputFilePath.startswith('/castor/'):
files = [ "".join([ "rfio:", file_info['path'] ]) for file_info in castor.nslsl(inputFilePath) ]
elif inputFilePath.startswith('/store/'):
files = [ file_info['path'] for file_info in eos.lsl(inputFilePath) ]
else:
files = [ "".join([ "file:", inputFilePath, file ]) for file in os.listdir(inputFilePath) ]
for file in files:
#print "file = %s" % file
inputFile_matcher = re.compile(inputFile_regex)
if inputFile_matcher.match(file):
inputFileNames.append(file)
print "inputFileNames = %s" % inputFileNames
process.source.fileNames = cms.untracked.vstring(inputFileNames)
#--------------------------------------------------------------------------------
process.testSVfitTrackLikelihoodProductionSequence = cms.Sequence()
#--------------------------------------------------------------------------------
def createFilePath(filePath):
try:
eos.lsl(filePath)
except IOError:
print "filePath = %s does not yet exist, creating it." % filePath
eos.mkdir(filePath)
def createFilePath(filePath):
try:
eos.lsl(filePath)
except IOError:
print "filePath = %s does not yet exist, creating it." % filePath
eos.mkdir(filePath)
if len(currentFilePath) <= 1:
continue
if not os.path.exists(currentFilePath):
os.mkdir(currentFilePath)
if outputFilePath.find("/castor") != -1:
try:
castor.rfstat(outputFilePath)
except RuntimeError:
print "outputFilePath does not yet exist, creating it."
os.system("rfmkdir %s" % outputFilePath)
os.system("rfchmod 777 %s" % outputFilePath)
elif outputFilePath.find("/store") != -1:
try:
eos.lsl(outputFilePath)
except IOError:
print "outputFilePath does not yet exist, creating it."
eos.mkdir(outputFilePath)
else:
hostname = socket.gethostname()
numChars = min(len(hostname), len(outputFileMachine))
if outputFileMachine[:numChars] != hostname[:numChars]:
raise ValueError(
"Cannot create output file path = %s on machine %s !!" %
(outputFilePath, outputFileMachine))
if not os.path.isdir(outputFilePath):
print "outputFilePath does not yet exist, creating it."
createFilePath_recursively(outputFilePath)
if not os.path.isdir("lxbatch"):
if inputFilePath.find('/castor/') == 0:
mode = 'castor'
if inputFilePath.find('/store/') == 0:
mode = 'eos'
else:
mode = 'local'
if jobId is None:
reg.overrideJobId(channel, '2011Oct30') # CV: need to overwrite this in order to match Mauro's filenames
jobId = reg.getJobId(channel)
print(" jobId = %s" % jobId)
if mode == 'castor':
files = [ file_info for file_info in castor.nslsl(inputFilePath) ]
elif mode == 'eos':
files = [ file_info for file_info in eos.lsl(inputFilePath) ]
else:
commandLine = '%s %s' % (options['executable_ls'][mode], inputFilePath)
args = shlex.split(commandLine)
retval = subprocess.Popen(args, stdout = subprocess.PIPE)
#retval.wait()
files = retval.stdout.read().split('\n')
#print(" files = %s" % files)
fileName_regex = r"(?P<fileName_base>[a-zA-Z0-9_]+)_(?P<gridJob>\d*)(_(?P<gridTry>\d*))*_(?P<hash>[a-zA-Z0-9]*).root"
fileName_matcher = re.compile(fileName_regex)
fileNamesAndProperties_dict = {}
print("--> found %i file" % len(files))
print("checking DPM files in outputFilePath = %s" % outputFilePath)
outputFileInfos = [ outputFileInfo for outputFileInfo in dpm.lsl(outputFilePath) ]
print outputFileInfos
for outputFileInfo in outputFileInfos:
if outputFileInfo['size'] == 0 and publishDirHash_matcher.match(outputFileInfo['file']):
subDirectory = outputFileInfo['path']
if not subDirectory.endswith('/'):
subDirectory += '/'
print("checking DPM files in subDirectory = %s" % subDirectory)
outputFileInfos.extend(dpm.lsl(subDirectory))
else:
print("checking DPM files in outputFilePath = %s" % outputFilePath)
outputFileInfos = [ outputFileInfo for outputFileInfo in dpm.lsl(outputFilePath) ]
elif outputFilePath.find("/store/") != -1:
print("checking EOS files in outputFilePath = %s" % outputFilePath)
outputFileInfos = [ outputFileInfo for outputFileInfo in eos.lsl(outputFilePath) ]
else:
raise ValueError("Failed to identify file-system for path = %s !!" % castor_output_directory)
#print "outputFileInfos:"
#print [ outputFileInfo['file'] for outputFileInfo in outputFileInfos ]
# check if job got aborted or stuck in state 'Submitted', 'Ready' or 'Scheduled' for more than one day
isCrabFailure = False
foundStart = False
foundEnd = False
for crabStatus_line in crabStatus_lines:
# CV: check if job status cannot be determined,
# due to crab internal error
if isCrabFailure:
continue
if crabStatus_line.find("Traceback (most recent call last):") != -1:
print("checking DPM files in subDirectory = %s" %
subDirectory)
outputFileInfos.extend(dpm.lsl(subDirectory))
else:
print("checking DPM files in outputFilePath = %s" %
outputFilePath)
outputFileInfos = [
outputFileInfo
for outputFileInfo in dpm.lsl(outputFilePath)
]
elif outputFilePath.find("/store/") != -1:
print("checking EOS files in outputFilePath = %s" %
outputFilePath)
outputFileInfos = [
outputFileInfo
for outputFileInfo in eos.lsl(outputFilePath)
]
else:
raise ValueError(
"Failed to identify file-system for path = %s !!" %
castor_output_directory)
#print "outputFileInfos:"
#print [ outputFileInfo['file'] for outputFileInfo in outputFileInfos ]
# check if job got aborted or stuck in state 'Submitted', 'Ready' or 'Scheduled' for more than one day
isCrabFailure = False
foundStart = False
foundEnd = False
for crabStatus_line in crabStatus_lines:
# CV: check if job status cannot be determined,
# due to crab internal error
