def Do(self, item):
badger = Badger()
result = badger.downloadFilesByFilelist([item])#,destDir)
if not result['OK']:
#print result['Message'],type(result['Message'])
errorDict.update(result['Message'])
print errorDict
class DownloadWorker(IWorker):
"""
"""
#if file failed download,then append in errorDict
errorDict = {}
def __init__(self):
self.badger = Badger()
if queryFlag:
self.m_list = self.badger.getFilesByMetadataQuery(setQuery)
elif setNameFlag:
self.m_list = self.badger.getFilesByDatasetName(setName)
elif dfcDirFlag:
self.m_list = self.badger.listDir(dfcDir)
elif filenameFlag:
with open(filename) as f:
for line in f:
self.m_list.append(line.strip())
def get_file_list(self):
return self.m_list
def Do(self, item):
badger = Badger()
result = badger.downloadFilesByFilelist([item])#,destDir)
if not result['OK']:
#print result['Message'],type(result['Message'])
errorDict.update(result['Message'])
print errorDict
def Do(self, item):
badger = Badger()
#print "world"
result = badger.downloadFilesByFilelist([item]) #,destDir)
#print "result",result
if result['OK']:
self.db[item] = '2'
self.db.sync()
def __init__(self):
self.badger = Badger()
if queryFlag:
self.m_list = self.badger.getFilesByMetadataQuery(setQuery)
elif setNameFlag:
self.m_list = self.badger.getFilesByDatasetName(setName)
elif dfcDirFlag:
self.m_list = self.badger.listDir(dfcDir)
elif filenameFlag:
with open(filename) as f:
for line in f:
self.m_list.append(line.strip())
def getFileList(self):
badger = Badger()
result = badger.getFilesByDatasetName(setName)
fileList = []
if result['OK']:
fileList = result['Value']
if fileList:
self.dirName = os.path.dirname(fileList[0])
for file in fileList:
print >> self.listFile, os.path.basename(file)
self.readyNum = len(fileList)
self.listFile.close()
time_print('There are %s files ready for download' % self.readyNum)
def datasetGet():
badger = Badger()
badger.updateDataset(setName)
datasetTotalSize = badger.getDatasetMetadata(setName)['Value']['TotalSize']
originLocalfileSize = getCurrentDirTotalSize(
destDir) #check file size of the destDir before download
print "start download..."
start = time.time()
dw = DownloadWorker()
mw = MultiWorker(dw, 5)
mw.main()
dw.Clear()
total = time.time() - start
print "Finished,total time is %s" % total
def __init__(self):
self.badger = Badger()
if queryFlag:
self.db, self.dbName = getDB(setQuery,
self.badger.getFilesByMetadataQuery)
#print self.db,self.dbName
elif setNameFlag:
self.db, self.dbName = getDB(setName,
self.badger.getFilesByDatasetName)
#print self.db,self.dbName
elif dfcDirFlag:
self.db, self.dbName = getDB(dfcDir, self.badger.listDir)
#!/usr/bin/env python
#mtime:2013/12/09
"""
besdirac-dms-dataset-freeze
freeze a dataset
Usage:
besdirac-dms-dataset-freeze <datasetname>
"""
__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
if len(args) != 1:
Script.showHelp()
datasetName = args[0]
from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
badger.freezeDataset(datasetName)
exit(0)
#!/usr/bin/env python
#mtime:2013/12/09
"""
besdirac-dms-dataset-filelist
get the filelist of the given dataset
Usage:
besdirac-dms-dataset-filelist<datasetname>
"""
__RCSID__ = "$Id$"
import pprint
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
if len(args) != 1:
Script.showHelp()
datasetName = args[0]
from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
result = badger.getFilesByDatasetName(datasetName)
if result['OK']:
fileList = result['Value']
pprint.pprint(fileList)
exit(0)
def __init__(self):
self.badger = Badger()
self.db, self.dbName = getDB(setName,
self.badger.getFilesByDatasetName)
#!/usr/bin/env python
#mtime:2013/12/09
"""
besdirac-dms-dataset-check
check if the dataset changed
Usage:
besdirac-dms-dataset-check <datasetname>
"""
__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
if len(args)!=1:
Script.showHelp()
datasetName = args[0]
from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
badger.checkDataset(datasetName)
exit(0)
#!/usr/bin/env python
import time
import pprint
import DIRAC
import os
from DIRAC.Core.Base import Script
Script.parseCommandLine(ignoreErrors=True)
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
#from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
args = Script.getPositionalArgs()
strArgs = ' '.join(args)
from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
def query():
result = badger.getFilesByMetadataQuery(
'eventType=all bossVer=6.6.3 resonance=4260 runL>=29755 runH<29760 runL!=29756'
)
pprint.pprint(result)
def add(dn, path, con):
result = badger.registerDataset(dn, path, con)
pprint.pprint(result)
def get(dn):
result = badger.getDatasetDescription(dn)
pprint.pprint(result)
def __init__(self, localdir):
self.badger = Badger()
#self.m_list = badger.getFilenamesByLocaldir(localdir)
self.db, self.dbName = getDB(localdir,
self.badger.getFilenamesByLocaldir)
def Do(self, item):
badger = Badger()
result = badger.uploadAndRegisterFiles([item], ePoint=energyPoint)
if result['OK']:
self.db[item] = '2'
self.db.sync()
#!/usr/bin/env python
#mtime:2013/12/09
"""
besdirac-dms-dataset-release
release a dataset from freeze to dynamic
Usage:
besdirac-dms-dataset-release <datasetname>
"""
__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
if len(args) != 1:
Script.showHelp()
datasetName = args[0]
from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
badger.releaseDataset(datasetName)
exit(0)
#!/usr/bin/env python
#mtime:2013/12/09
"""
besdirac-dms-dataset-describe
describe a dataset from DB
Usage:
besdirac-dms-dataset-describe <datasetname>
"""
__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
if len(args) != 1:
Script.showHelp()
datasetName = args[0]
from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
badger.getDatasetDescription(datasetName)
exit(0)
besdirac-dms-dataset-delete <datasetname>
"""
__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
if len(args) != 1:
Script.showHelp()
datasetName = args[0]
from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager
rm = ReplicaManager()
result = badger.getFilesByDatasetName(datasetName)
if result['OK']:
fileList = result['Value']
fileCountDict = badger.reCalcCount(fileList, False)
badger.removeDataset(datasetName)
for file in fileCountDict:
if fileCountDict[file] == 0:
result = rm.removeFile(file)
if not result['OK']:
print "Failed remove file %s" % file
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
#print len(args)
if len(args) < 3:
Script.showHelp()
exit(-1)
datasetName = args[0]
path = args[1]
strArg = args[2]
from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
prefix = badger.getDatasetNamePrefix()
datasetName = prefix + datasetName
#print datasetName
result = badger.registerDataset(datasetName, path, strArg)
if result['OK']:
resVal = badger.getFilesByDatasetName(datasetName)
if resVal['OK']:
fileList = resVal['Value']
badger.reCalcCount(fileList)
#print result
exit(0)
Usage :
besdirac-dms-check-files <dfcDir> <localDir>
Example: besdirac-dms-check-files /dir1 /dir2
'''
import os.path
import pprint
import DIRAC
from DIRAC.Core.Base import Script
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
Script.setUsageMessage(__doc__)
Script.parseCommandLine(ignoreErrors=True)
from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
dirs = Script.getPositionalArgs()
dfcdir = dirs[0]
localDir = dirs[1]
#get DFC file dict
lfns = badger.listDir(dfcdir)
lfnDict = badger.getSize(lfns)
base_lfnDict = {}
for k, v in lfnDict.items():
k = os.path.basename(k)
base_lfnDict[k] = v
#get local files dict
localFiles = badger.getFilenamesByLocaldir(localDir)
base_localDict = {}
for file in localFiles:
#!/usr/bin/env python #mtime:2013/12/09 """ list dataste names and their metadata """ __RCSID__ = "$Id$" import time import pprint import DIRAC from DIRAC.Core.Base import Script Script.parseCommandLine(ignoreErrors=True) Script.setUsageMessage(__doc__) from IHEPDIRAC.Badger.API.Badger import Badger badger = Badger() badger.listDatasets() exitCode = 0 DIRAC.exit(exitCode)
#!/usr/bin/env python
#mtime:2013/12/09
"""
besdirac-dms-dataset-update
if a dataset changed,should update
Usage:
besdirac-dms-dataset-update <datasetname>
"""
__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
if len(args) != 1:
Script.showHelp()
datasetName = args[0]
from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
badger.updateDataset(datasetName)
exit(0)