def registerInputData(filepath, size, prefix='/cepc/lustre-ro'):
infoDict = {}
infoDict['PFN'] = ''
infoDict['Size'] = size
infoDict['SE'] = 'IHEP-STORM'
infoDict['GUID'] = commands.getoutput('uuidgen')
infoDict['Checksum'] = ''
fileDict = {}
lfn = prefix + filepath
fileDict[lfn] = infoDict
fcc = FileCatalogClient('DataManagement/FileCatalog')
rm = ReplicaManager()
result = {}
result['lfn'] = lfn
result['is_registered'] = False
#查询
for repeatTimes in range(10):
is_registered = fcc.isFile(lfn)
if (is_registered['OK']
and is_registered['Value']['Successful'].has_key(lfn)):
break
# else:
# continue
if not is_registered['OK']: #查询失败
result['is_registered'] = 'querry error. unkown'
print 'Failed to query %s in DFC. Error message is %s' % (
lfn, is_registered['Message'])
if is_registered['Value']['Successful'][lfn]: #已注册
result['is_registered'] = True
for repeatTimes in range(10):
is_removed = rm.removeCatalogFile(lfn) #删除
if (is_removed['OK']
and is_removed['Value']['Successful'][lfn]['FileCatalog']):
result['is_removed'] = True
break
# else:
# continue
if not is_removed['OK']: #删除失败
result['is_removed'] = 'remove error'
print 'Failed to remove %s from DFC.' % lfn
#add
for repeatTimes in range(10):
is_added = fcc.addFile(fileDict) #add/register
if (is_added['OK'] and is_added['Value']['Successful'][lfn]):
result['OK'] = True
return result
# else:
# continue
if not is_added['OK']: #add unsuccessfully
result['OK'] = False
result['Message'] = is_added['Message']
elif is_added['Value']['Failed']:
result['OK'] = False
result['Message'] = 'Failed to add file' + lfn
return result
def registerInputData(filepath, size, prefix = '/cepc/lustre-ro'):
infoDict = {}
infoDict['PFN'] = ''
infoDict['Size'] = size
infoDict['SE'] = 'IHEP-STORM'
infoDict['GUID'] = commands.getoutput('uuidgen')
infoDict['Checksum'] = ''
fileDict = {}
lfn = prefix + filepath
fileDict[lfn] = infoDict
fcc = FileCatalogClient('DataManagement/FileCatalog')
rm = ReplicaManager()
result = {}
result['lfn'] = lfn
result['is_registered'] = False
#查询
for repeatTimes in range(10):
is_registered = fcc.isFile(lfn)
if (is_registered['OK'] and is_registered['Value']['Successful'].has_key(lfn)):
break
# else:
# continue
if not is_registered['OK']:#查询失败
result['is_registered'] = 'querry error. unkown'
print 'Failed to query %s in DFC. Error message is %s' %(lfn, is_registered['Message'])
if is_registered['Value']['Successful'][lfn]:#已注册
result['is_registered'] = True
for repeatTimes in range(10):
is_removed = rm.removeCatalogFile(lfn)#删除
if (is_removed['OK'] and is_removed['Value']['Successful'][lfn]['FileCatalog']):
result['is_removed'] = True
break
# else:
# continue
if not is_removed['OK']:#删除失败
result['is_removed'] = 'remove error'
print 'Failed to remove %s from DFC.' %lfn
#add
for repeatTimes in range(10):
is_added = fcc.addFile(fileDict)#add/register
if (is_added['OK'] and is_added['Value']['Successful'][lfn]):
result['OK'] = True
return result
# else:
# continue
if not is_added['OK']:#add unsuccessfully
result['OK'] = False
result['Message'] = is_added['Message']
elif is_added['Value']['Failed']:
result['OK'] = False
result['Message'] = 'Failed to add file' + lfn
return result
def insert(attributes):
dir = "/BES3/File/"+attributes["resonance"]+"/"+attributes["bossVer"]
#get real eventType and expNum in catalog in amga
eventType = _get_event_type(attributes["eventType"])
expNum = _get_exp_num(attributes["expNum"])
if attributes["streamId"]=="stream0":
dir=dir+"/data"
else:
dir=dir+"/mc"
client = FileCatalogClient()
# make sure insertion directory exists in DFC
insertDir = createCatalog(client,dir,eventType,expNum,attributes)
#do we need to explicitly add LFN attr in DFC?
# we do need to get the PFN, size, se for addFile (below)
#TODO pfn = ?
# need to add PFN attribute to the list, probably in control.py
pfn = attributes["PFN"]
lfn=insertDir+"/"+attributes["LFN"]
attributes["LFN"] = lfn
#TODO se = ? temporarily use a made-up SE name
se = 'BES-TEST'
# register files in DFC
infoDict = {}
#infoDict['PFN'] = attributes['PFN']
infoDict['PFN'] = pfn
infoDict['Size'] = attributes['fileSize']
infoDict['SE'] = se
# infoDict['guid'] = attributes['guid']
infoDict['Checksum'] = ''
result = client.addFile({lfn:infoDict})
print result
def processDir(initPath,recursive=False,host=None,fcInit=None,dfcInit=None):
""" Process one directory, possibly recursively
"""
global dirCount, fileCount, globalStart, dnCache, roleCache, outputFile
fc = fcInit
if not fc:
fc = LcgFileCatalogClient.LcgFileCatalogClient( host=host )
#fc = FileCatalogClient()
dfc = dfcInit
if not dfc:
#dfc = LcgFileCatalogClient.LcgFileCatalogClient( host=host )
dfc = FileCatalogClient()
start = time.time()
initPath = initPath.rstrip("/")
resultList = fc.listDirectory(initPath,True)
#print resultList
#return S_OK()
lfc_time = (time.time() - start)
s = time.time()
print resultList
if resultList['OK']:
# Add directories
if resultList['Value']['Failed']:
return S_ERROR("Path %s failed: %s" % (initPath,resultList['Value']['Failed'][initPath]))
dirDict = resultList['Value']['Successful'][initPath]['SubDirs']
paths = {}
for path,info in dirDict.items():
print info
paths[path] = {}
paths[path]['Mode'] = info['Mode']
owner = getUserNameAndGroup( info )
#owner = getDNandRole( info )
if owner:
paths[path]['Owner'] = owner
#return S_OK()
p_dirs = time.time() - s
s = time.time()
nDir = len(paths)
if nDir:
print "Adding %d directories in %s" % (nDir,initPath)
result = dfc.createDirectory(paths)
if not result['OK']:
print "Error adding directories:",result['Message']
dirCount += nDir
print "Total directories added", dirCount
e_dirs = time.time() - s
# Add files
s = time.time()
fileDict = resultList['Value']['Successful'][initPath]['Files']
lfns = {}
for lfn,info in fileDict.items():
lfns[lfn] = {}
lfns[lfn]['Size'] = info['MetaData']['Size']
lfns[lfn]['Checksum'] = info['MetaData']['Checksum']
if 'GUID' in info['MetaData']:
lfns[lfn]['GUID'] = info['MetaData']['GUID']
else:
lfns[lfn]['GUID'] = makeGuid()
lfns[lfn]['Mode'] = info['MetaData']['Mode']
lfns[lfn]['PFN'] = ''
owner = getUserNameAndGroup( info['MetaData'] )
if owner:
lfns[lfn]['Owner'] = owner
if info['Replicas']:
seList = info['Replicas'].keys()
lfns[lfn]['SE'] = seList
p_files = time.time() - s
s = time.time()
nFile = len(lfns)
nRep = 0
if nFile:
for lfn in lfns:
if 'SE' in lfns[lfn]:
nRep += len(lfns[lfn]['SE'])
print "Adding %d files in %s" % (nFile,initPath)
#print lfns
done = False
count = 0
error = False
while not done:
count += 1
result = dfc.addFile(lfns)
#print result
if not result['OK']:
print "Error adding files %d:" % count,result['Message']
if count > 10:
print "Completely failed path", initPath
break
error = True
time.sleep(2)
elif error:
print "Successfully added files on retry %d" % count
done = True
else:
done = True
fileCount += nFile
print "Total files added", fileCount
e_files = time.time() - s
dfc_time = time.time() - start - lfc_time
total_time = time.time() - globalStart
format = "== %s: time lfc/dfc %.2f/%.2f, files %d/%d, dirs %d/%d, time: %.2f/%.2f/%.2f/%.2f %.2f \n"
outputFile = open('lfc_dfc.out','a')
outputFile.write( format % (initPath,lfc_time,dfc_time,nFile,fileCount,nDir,dirCount,p_dirs,e_dirs,p_files,e_files,total_time) )
outputFile.close()
# print format % (initPath,lfc_time,dfc_time,nFile,fileCount,nDir,dirCount,p_dirs,e_dirs,p_files,e_files,total_time)
# Go into directories
if recursive:
for path in paths:
result = processDir(path,True,host=host,fcInit=fc,dfcInit=dfc)
if result['OK']:
nFile += result['Value'].get('NumberOfFiles',0)
nDir += result['Value'].get('NumberOfDirectories',0)
nRep += result['Value'].get('NumberOfReplicas',0)
resultDict = {}
resultDict['NumberOfFiles'] = nFile
resultDict['NumberOfDirectories'] = nDir
resultDict['NumberOfReplicas'] = nRep
resultDict['Path'] = initPath
return S_OK(resultDict)
lfn = "%s/%s" % (dirPath, filename)
size = random.randint(1, 1000)
se = random.choice(storageElements)
guid = "%s%s" % (''.join(map(str, rndTab)), filename)
checksum = guid
lfnDict[lfn] = {
'PFN': lfn,
'SE': se,
'Size': size,
'GUID': guid,
'Checksum': checksum
}
beforeI = time.time()
beforeCI = time.clock()
res = fc.addFile(lfnDict)
afterCI = time.clock()
afterI = time.time()
queryInsertTime = afterI - beforeI
queryInsertTimeC = afterCI - beforeCI
extra = "insert "
if not res['OK']:
extra += res['Message']
else:
extra += "%s %s %s" % (len(lfnDict),
len(res['Value'].get('Successful', [])),
len(res['Value'].get('Failed', [])))
pass
fl.write("%s\t%s\t%s\t%s\n" % (beforeI, afterI, queryInsertTime, extra))
fl.flush()
dirPath = '/' + '/'.join(map(str, rndTab))
nbOfFiles = random.randint( 1, maxInsert )
lfnDict = {}
for f in xrange(nbOfFiles):
filename = "%s.txt"%(f)
lfn = "%s/%s"%(dirPath, filename)
size = random.randint(1,1000)
se = random.choice(storageElements)
guid = "%s%s"%(''.join(map(str,rndTab)),filename)
checksum = guid
lfnDict[ lfn ] = { 'PFN' : lfn, 'SE' : se, 'Size' : size, 'GUID' : guid, 'Checksum' : checksum}
beforeI = time.time()
beforeCI = time.clock()
res = fc.addFile(lfnDict)
afterCI = time.clock()
afterI = time.time()
queryInsertTime = afterI - beforeI
queryInsertTimeC = afterCI - beforeCI
extra = "insert "
if not res['OK']:
extra += res['Message']
else:
extra += "%s %s %s"%(len(lfnDict), len(res['Value'].get('Successful', [])), len(res['Value'].get('Failed', [])))
pass
fl.write("%s\t%s\t%s\t%s\n"%(beforeI, afterI, queryInsertTime, extra))
fl.flush()
os.fsync(fl)
fl2.write("%s\t%s\t%s\t%s\n"%(beforeCI, afterCI, queryInsertTimeC, extra))
def processDir(initPath,
writerQueue,
recursive=False,
host=None,
fcInit=None,
dfcInit=None):
""" Process one directory, possibly recursively
"""
global globalStart, dnCache, roleCache
# print "AT >>> processDir initPath", initPath
fc = fcInit
if not fc:
fc = LcgFileCatalogClient.LcgFileCatalogClient(host=host)
dfc = dfcInit
if not dfc:
dfc = FileCatalogClient()
start = time.time()
resultList = fc.listDirectory(initPath, True)
if not resultList['OK']:
result = S_ERROR("Failed LFC lookup for %s" % initPath)
result['Path'] = initPath
return result
lfc_time = (time.time() - start)
s = time.time()
if resultList['OK']:
# Add directories
if resultList['Value']['Failed']:
result = S_ERROR(
"Path %s failed: %s" %
(initPath, resultList['Value']['Failed'][initPath]))
return result
dirDict = resultList['Value']['Successful'][initPath]['SubDirs']
paths = {}
for path, info in dirDict.items():
paths[path] = {}
paths[path]['Mode'] = info['Mode']
owner = getUserNameAndGroup(info)
if owner:
paths[path]['Owner'] = owner
p_dirs = time.time() - s
s = time.time()
nDir = len(paths)
if nDir:
# print "Adding %d directories in %s" % ( nDir, initPath )
result = dfc.createDirectory(paths)
if not result['OK']:
print("Error adding directories:%s" % result['Message'])
e_dirs = time.time() - s
# Add files
s = time.time()
fileDict = resultList['Value']['Successful'][initPath]['Files']
lfns = {}
for lfn, info in fileDict.items():
# print info['MetaData']
lfns[lfn] = {}
lfns[lfn]['Size'] = info['MetaData']['Size']
lfns[lfn]['Checksum'] = info['MetaData'].get('Checksum', '')
lfns[lfn]['GUID'] = info['MetaData']['GUID']
lfns[lfn]['Mode'] = info['MetaData']['Mode']
lfns[lfn]['PFN'] = ''
owner = getUserNameAndGroup(info['MetaData'])
if owner:
lfns[lfn]['Owner'] = owner
if info['Replicas']:
seList = list(info['Replicas'])
lfns[lfn]['SE'] = seList
p_files = time.time() - s
s = time.time()
nFile = len(lfns)
nRep = 0
if nFile:
for lfn in lfns:
if 'SE' in lfns[lfn]:
nRep += len(lfns[lfn]['SE'])
# print "Adding %d files in %s" % ( nFile, initPath )
done = False
count = 0
error = False
while not done:
count += 1
result = dfc.addFile(lfns)
if not result['OK']:
print("Error adding files %d:" % count, result['Message'])
if count > 10:
print("Completely failed path %s" % initPath)
break
error = True
time.sleep(2)
elif error:
print("Successfully added files on retry %d" % count)
done = True
else:
done = True
e_files = time.time() - s
dfc_time = time.time() - start - lfc_time
total_time = time.time() - globalStart
format = "== %s: time lfc/dfc %.2f/%.2f, files %d, dirs %d, reps %d, time: %.2f/%.2f/%.2f/%.2f %.2f \n"
writerQueue.put(
format % (initPath, lfc_time, dfc_time, nFile, nDir, nRep, p_dirs,
e_dirs, p_files, e_files, total_time))
# outputFile = open('lfc_dfc.out','a')
# outputFile.write( format % (initPath,lfc_time,dfc_time,nFile,nDir,nRep,p_dirs,e_dirs,p_files,e_files,total_time) )
# outputFile.close()
# print format % (initPath,lfc_time,dfc_time,nFile,fileCount,nDir,dirCount,p_dirs,e_dirs,p_files,e_files,total_time)
# Go into directories
if recursive:
for path in paths:
result = processDir(path,
writerQueue,
recursive=True,
host=host,
fcInit=fc,
dfcInit=dfc)
if result['OK']:
nFile += result['Value'].get('NumberOfFiles', 0)
nDir += result['Value'].get('NumberOfDirectories', 0)
nRep += result['Value'].get('NumberOfReplicas', 0)
resultDict = {}
resultDict['NumberOfFiles'] = nFile
resultDict['NumberOfDirectories'] = nDir
resultDict['NumberOfReplicas'] = nRep
resultDict['Path'] = initPath
resultDict['Directories'] = list(dirDict)
# print "AT >>> processDir",initPath,"done %.2f" % (time.time()-start)
toRet = S_OK(resultDict)
toRet['writerQueue'] = writerQueue
return toRet