def Do(self, item):
badger = Badger()
result = badger.downloadFilesByFilelist([item])#,destDir)
if not result['OK']:
#print result['Message'],type(result['Message'])
errorDict.update(result['Message'])
print errorDict
class DownloadWorker(IWorker):
"""
"""
#if file failed download,then append in errorDict
errorDict = {}
def __init__(self):
self.badger = Badger()
if queryFlag:
self.m_list = self.badger.getFilesByMetadataQuery(setQuery)
elif setNameFlag:
self.m_list = self.badger.getFilesByDatasetName(setName)
elif dfcDirFlag:
self.m_list = self.badger.listDir(dfcDir)
elif filenameFlag:
with open(filename) as f:
for line in f:
self.m_list.append(line.strip())
def get_file_list(self):
return self.m_list
def Do(self, item):
badger = Badger()
result = badger.downloadFilesByFilelist([item])#,destDir)
if not result['OK']:
#print result['Message'],type(result['Message'])
errorDict.update(result['Message'])
print errorDict
def Do(self, item):
badger = Badger()
#print "world"
result = badger.downloadFilesByFilelist([item])#,destDir)
#print "result",result
if result['OK']:
self.db[item] = '2'
self.db.sync()
def getFileList(self):
badger = Badger()
result = badger.getFilesByDatasetName(setName)
fileList = []
if result['OK']:
fileList = result['Value']
if fileList:
self.dirName = os.path.dirname(fileList[0])
for file in fileList:
print >>self.listFile, os.path.basename(file)
self.readyNum = len(fileList)
self.listFile.close()
time_print('There are %s files ready for download' % self.readyNum)
def getDataset(self):
'''Gets the dataset from the bookkeeping for current dict.'''
if not self.name: return None
badger = Badger()
files = []
files = badger.getFilesByDatasetName(self.name)
ds = BesDataset()
for f in files:
logicalFile = "LFN:"+f
logger.debug("zhangxm log: data files LFN: %s", f)
ds.files.append(logicalFile)
return GPIProxyObjectFactory(ds)
def datasetGet(): badger = Badger() badger.updateDataset(setName) datasetTotalSize = badger.getDatasetMetadata(setName)['Value']['TotalSize'] originLocalfileSize = getCurrentDirTotalSize(destDir) #check file size of the destDir before download print "start download..." start = time.time() dw = DownloadWorker() mw = MultiWorker(dw,5) mw.main() dw.Clear() total=time.time()-start print "Finished,total time is %s"%total
def __init__(self):
self.badger = Badger()
if queryFlag:
self.m_list = self.badger.getFilesByMetadataQuery(setQuery)
elif setNameFlag:
self.m_list = self.badger.getFilesByDatasetName(setName)
elif dfcDirFlag:
self.m_list = self.badger.listDir(dfcDir)
elif filenameFlag:
with open(filename) as f:
for line in f:
self.m_list.append(line.strip())
"""
upload a set of file to SE and register them in DFC.
"""
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Base import Script
Script.registerSwitch("r","dir","the directory that dataset files located")
Script.setUsageMessage('\n'.join([__doc__,
'Usage:',
'%s dir'% Script.scriptName,
'Arguments:'
' dir: dir is a logical directory in DFC']))
Script.parseCommandLine(ignoreErrors=True)
dir = Script.getPositionalArgs()
#print dir
if len(dir)!=1:
Script.showHelp()
from BESDIRAC.Badger.API.Badger import Badger
badger = Badger()
localdir = dir[0]
exitCode = 0
result = badger.uploadAndRegisterFiles(localdir)
if not result:
print 'ERROR %s'%(result['Message'])
exitCode = 2
DIRAC.exit(exitCode)
#!/usr/bin/env python
#mtime:2013/12/09
"""
besdirac-dms-dataset-filelist
get the filelist of the given dataset
Usage:
besdirac-dms-dataset-filelist<datasetname>
"""
__RCSID__ = "$Id$"
import pprint
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
if len(args)!=1:
Script.showHelp()
datasetName = args[0]
from BESDIRAC.Badger.API.Badger import Badger
badger = Badger()
result = badger.getFilesByDatasetName(datasetName)
if result['OK']:
fileList = result['Value']
pprint.pprint(fileList)
exit(0)
"""
upload a set of file to SE and register them in DFC.
"""
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Base import Script
Script.registerSwitch("r", "dir", "the directory that dataset files located")
Script.setUsageMessage('\n'.join([
__doc__, 'Usage:',
'%s dir' % Script.scriptName, 'Arguments:'
' dir: dir is a logical directory in DFC'
]))
Script.parseCommandLine(ignoreErrors=True)
dir = Script.getPositionalArgs()
#print dir
if len(dir) != 1:
Script.showHelp()
from BESDIRAC.Badger.API.Badger import Badger
badger = Badger()
localdir = dir[0]
exitCode = 0
result = badger.uploadAndRegisterFiles(localdir)
if not result:
print 'ERROR %s' % (result['Message'])
exitCode = 2
DIRAC.exit(exitCode)
#!/usr/bin/env python
import time
import pprint
import DIRAC
import os
from DIRAC.Core.Base import Script
Script.parseCommandLine(ignoreErrors=True)
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
#from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
args = Script.getPositionalArgs()
strArgs = ' '.join(args)
from BESDIRAC.Badger.API.Badger import Badger
badger = Badger()
def query():
result = badger.getFilesByMetadataQuery('eventType=all bossVer=6.6.3 resonance=4260 runL>=29755 runH<29760 runL!=29756')
pprint.pprint(result)
def add(dn,path,con):
result = badger.registerDataset(dn,path,con)
pprint.pprint(result)
def get(dn):
result = badger.getDatasetDescription(dn)
pprint.pprint(result)
def list():
result = badger.listDatasets()
pprint.pprint(result)
'''checksum,compare the size of LFN files and Local files
Usage :
besdirac-dms-check-files <dfcDir> <localDir>
Example: besdirac-dms-check-files /dir1 /dir2
'''
import os.path
import pprint
import DIRAC
from DIRAC.Core.Base import Script
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
Script.setUsageMessage(__doc__)
Script.parseCommandLine(ignoreErrors=True)
from BESDIRAC.Badger.API.Badger import Badger
badger = Badger()
dirs = Script.getPositionalArgs()
dfcdir = dirs[0]
localDir = dirs[1]
#get DFC file dict
lfns = badger.listDir(dfcdir)
lfnDict = badger.getSize(lfns)
base_lfnDict = {}
for k,v in lfnDict.items():
k = os.path.basename(k)
base_lfnDict[k] = v
#get local files dict
localFiles = badger.getFilenamesByLocaldir(localDir)
base_localDict = {}
for file in localFiles:
besdirac-dms-dataset-delete <datasetname>
"""
__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
if len(args)!=1:
Script.showHelp()
datasetName = args[0]
from BESDIRAC.Badger.API.Badger import Badger
badger = Badger()
from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager
rm = ReplicaManager()
result = badger.getFilesByDatasetName(datasetName)
if result['OK']:
fileList = result['Value']
fileCountDict = badger.reCalcCount(fileList,False)
badger.removeDataset(datasetName)
for file in fileCountDict:
if fileCountDict[file] == 0:
result = rm.removeFile(file)
if not result['OK']:
print "Failed remove file %s"%file
#!/usr/bin/env python
# mtime:2013/12/09
"""
besdirac-dms-dataset-check
check if the dataset changed
Usage:
besdirac-dms-dataset-check <datasetname>
"""
__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
if len(args) != 1:
Script.showHelp()
datasetName = args[0]
from BESDIRAC.Badger.API.Badger import Badger
badger = Badger()
badger.checkDataset(datasetName)
exit(0)
def Do(self, item):
badger = Badger()
result = badger.uploadAndRegisterFiles([item],ePoint=energyPoint)
if result['OK']:
self.db[item] = '2'
self.db.sync()
#!/usr/bin/env python
#mtime:2013/12/09
"""
besdirac-dms-dataset-release
release a dataset from freeze to dynamic
Usage:
besdirac-dms-dataset-release <datasetname>
"""
__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
if len(args)!=1:
Script.showHelp()
datasetName = args[0]
from BESDIRAC.Badger.API.Badger import Badger
badger = Badger()
badger.releaseDataset(datasetName)
exit(0)
#!/usr/bin/env python
#mtime:2013/12/09
"""
besdirac-dms-dataset-describe
describe a dataset from DB
Usage:
besdirac-dms-dataset-describe <datasetname>
"""
__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
if len(args)!=1:
Script.showHelp()
datasetName = args[0]
from BESDIRAC.Badger.API.Badger import Badger
badger = Badger()
badger.getDatasetDescription(datasetName)
exit(0)
#!/usr/bin/env python
#mtime:2013/12/09
"""
besdirac-dms-dataset-freeze
freeze a dataset
Usage:
besdirac-dms-dataset-freeze <datasetname>
"""
__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
if len(args)!=1:
Script.showHelp()
datasetName = args[0]
from BESDIRAC.Badger.API.Badger import Badger
badger = Badger()
badger.freezeDataset(datasetName)
exit(0)
#!/usr/bin/env python
#mtime:2013/12/09
"""
besdirac-dms-dataset-update
if a dataset changed,should update
Usage:
besdirac-dms-dataset-update <datasetname>
"""
__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
if len(args)!=1:
Script.showHelp()
datasetName = args[0]
from BESDIRAC.Badger.API.Badger import Badger
badger = Badger()
badger.updateDataset(datasetName)
exit(0)
#!/usr/bin/env python #mtime:2013/12/09 """ list dataste names and their metadata """ __RCSID__ = "$Id$" import time import pprint import DIRAC from DIRAC.Core.Base import Script Script.parseCommandLine(ignoreErrors=True) Script.setUsageMessage(__doc__) from BESDIRAC.Badger.API.Badger import Badger badger = Badger() badger.listDatasets() exitCode = 0 DIRAC.exit(exitCode)
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
# print len(args)
if len(args) < 3:
Script.showHelp()
exit(-1)
datasetName = args[0]
path = args[1]
strArg = args[2]
from BESDIRAC.Badger.API.Badger import Badger
badger = Badger()
prefix = badger.getDatasetNamePrefix()
datasetName = prefix + datasetName
# print datasetName
result = badger.registerDataset(datasetName, path, strArg)
if result["OK"]:
resVal = badger.getFilesByDatasetName(datasetName)
if resVal["OK"]:
fileList = resVal["Value"]
badger.reCalcCount(fileList)
# print result
exit(0)
download a set of files as a dataset from SE to the current directory
"""
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Base import Script
Script.registerSwitch("s","datasetName","the dataset you want to download")
Script.setUsageMessage('\n'.join([__doc__,
'Usage:',
'%s dir'% Script.scriptName,
'Arguments:'
' datasetName: the dataset you want to download']))
Script.parseCommandLine(ignoreErrors=True)
datasetName = Script.getPositionalArgs()
#print dir
if len(datasetName)!=1:
Script.showHelp()
from BESDIRAC.Badger.API.Badger import Badger
badger = Badger()
exitCode = 0
datasetName = datasetName[0]
result = badger.dowloadFileByDatasetName(datasetName)
if not result:
print 'ERROR %s'%(result['Message'])
exitCode = 1
DIRAC.exit(exitCode)
