Ejemplos de Badger en Python

Ejemplo n.º 1

 def Do(self, item):
   badger = Badger()
   result = badger.downloadFilesByFilelist([item])#,destDir)
   if not result['OK']:
     #print result['Message'],type(result['Message'])
     errorDict.update(result['Message'])
     print errorDict

Ejemplo n.º 2

class DownloadWorker(IWorker):
  """
  """
  #if file failed download,then append in errorDict
  errorDict = {}

  def __init__(self):
    self.badger = Badger()
    if queryFlag:
      self.m_list = self.badger.getFilesByMetadataQuery(setQuery)
    elif setNameFlag:
      self.m_list = self.badger.getFilesByDatasetName(setName)
    elif dfcDirFlag:
      self.m_list = self.badger.listDir(dfcDir)
    elif filenameFlag:
      with open(filename) as f:
        for line in f:
          self.m_list.append(line.strip())

  def get_file_list(self):
    return self.m_list

  def Do(self, item):
    badger = Badger()
    result = badger.downloadFilesByFilelist([item])#,destDir)
    if not result['OK']:
      #print result['Message'],type(result['Message'])
      errorDict.update(result['Message'])
      print errorDict

Ejemplo n.º 3

 def Do(self, item):
     badger = Badger()
     #print "world"
     result = badger.downloadFilesByFilelist([item])  #,destDir)
     #print "result",result
     if result['OK']:
         self.db[item] = '2'
         self.db.sync()

Ejemplo n.º 4

 def __init__(self):
   self.badger = Badger()
   if queryFlag:
     self.m_list = self.badger.getFilesByMetadataQuery(setQuery)
   elif setNameFlag:
     self.m_list = self.badger.getFilesByDatasetName(setName)
   elif dfcDirFlag:
     self.m_list = self.badger.listDir(dfcDir)
   elif filenameFlag:
     with open(filename) as f:
       for line in f:
         self.m_list.append(line.strip())

Ejemplo n.º 5

 def getFileList(self):
     badger = Badger()
     result = badger.getFilesByDatasetName(setName)
     fileList = []
     if result['OK']:
         fileList = result['Value']
         if fileList:
             self.dirName = os.path.dirname(fileList[0])
     for file in fileList:
         print >> self.listFile, os.path.basename(file)
     self.readyNum = len(fileList)
     self.listFile.close()
     time_print('There are %s files ready for download' % self.readyNum)

Ejemplo n.º 6

def datasetGet():
    badger = Badger()
    badger.updateDataset(setName)
    datasetTotalSize = badger.getDatasetMetadata(setName)['Value']['TotalSize']
    originLocalfileSize = getCurrentDirTotalSize(
        destDir)  #check file size of the destDir before download
    print "start download..."
    start = time.time()

    dw = DownloadWorker()
    mw = MultiWorker(dw, 5)
    mw.main()
    dw.Clear()

    total = time.time() - start
    print "Finished,total time is %s" % total

Ejemplo n.º 7

 def __init__(self):
     self.badger = Badger()
     if queryFlag:
         self.db, self.dbName = getDB(setQuery,
                                      self.badger.getFilesByMetadataQuery)
         #print self.db,self.dbName
     elif setNameFlag:
         self.db, self.dbName = getDB(setName,
                                      self.badger.getFilesByDatasetName)
         #print self.db,self.dbName
     elif dfcDirFlag:
         self.db, self.dbName = getDB(dfcDir, self.badger.listDir)

Ejemplo n.º 8

Archivo: besdirac-dms-dataset-freeze.py Proyecto: zhangxt-ihep/IHEPDIRAC

#!/usr/bin/env python
#mtime:2013/12/09
"""
besdirac-dms-dataset-freeze
  freeze a dataset
  Usage:
    besdirac-dms-dataset-freeze <datasetname>
"""

__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script

Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()

if len(args) != 1:
    Script.showHelp()
datasetName = args[0]

from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
badger.freezeDataset(datasetName)
exit(0)

Ejemplo n.º 9

Archivo: besdirac-dms-dataset-filelist.py Proyecto: zhangxt-ihep/IHEPDIRAC

#!/usr/bin/env python
#mtime:2013/12/09
"""
besdirac-dms-dataset-filelist
  get the filelist of the given dataset
  Usage:
    besdirac-dms-dataset-filelist<datasetname>
"""

__RCSID__ = "$Id$"
import pprint
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script

Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()

if len(args) != 1:
    Script.showHelp()
datasetName = args[0]

from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
result = badger.getFilesByDatasetName(datasetName)
if result['OK']:
    fileList = result['Value']
pprint.pprint(fileList)
exit(0)

Ejemplo n.º 10

 def __init__(self):
     self.badger = Badger()
     self.db, self.dbName = getDB(setName,
                                  self.badger.getFilesByDatasetName)

Ejemplo n.º 11

Archivo: besdirac-dms-dataset-check.py Proyecto: zhangxt-ihep/IHEPDIRAC

#!/usr/bin/env python
#mtime:2013/12/09
"""
besdirac-dms-dataset-check
  check if the dataset changed 
  Usage:
    besdirac-dms-dataset-check <datasetname>
"""

__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script

Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()

if len(args)!=1:
  Script.showHelp()
datasetName = args[0]

from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
badger.checkDataset(datasetName)
exit(0)

Ejemplo n.º 12

#!/usr/bin/env python
import time
import pprint
import DIRAC
import os
from DIRAC.Core.Base import Script
Script.parseCommandLine(ignoreErrors=True)
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
#from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
args = Script.getPositionalArgs()
strArgs = ' '.join(args)
from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()


def query():
    result = badger.getFilesByMetadataQuery(
        'eventType=all bossVer=6.6.3 resonance=4260 runL>=29755 runH<29760 runL!=29756'
    )
    pprint.pprint(result)


def add(dn, path, con):
    result = badger.registerDataset(dn, path, con)
    pprint.pprint(result)


def get(dn):
    result = badger.getDatasetDescription(dn)
    pprint.pprint(result)

Ejemplo n.º 13

Archivo: besdirac-dms-add-files.py Proyecto: zhangxt-ihep/IHEPDIRAC

 def __init__(self, localdir):
     self.badger = Badger()
     #self.m_list = badger.getFilenamesByLocaldir(localdir)
     self.db, self.dbName = getDB(localdir,
                                  self.badger.getFilenamesByLocaldir)

Ejemplo n.º 14

Archivo: besdirac-dms-add-files.py Proyecto: zhangxt-ihep/IHEPDIRAC

 def Do(self, item):
     badger = Badger()
     result = badger.uploadAndRegisterFiles([item], ePoint=energyPoint)
     if result['OK']:
         self.db[item] = '2'
         self.db.sync()

Ejemplo n.º 15

#!/usr/bin/env python
#mtime:2013/12/09
"""
besdirac-dms-dataset-release
  release a dataset from freeze to dynamic
  Usage:
    besdirac-dms-dataset-release <datasetname>
"""

__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script

Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()

if len(args) != 1:
    Script.showHelp()
datasetName = args[0]

from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
badger.releaseDataset(datasetName)
exit(0)

Ejemplo n.º 16

Archivo: besdirac-dms-dataset-status.py Proyecto: zhangxt-ihep/IHEPDIRAC

#!/usr/bin/env python
#mtime:2013/12/09
"""
besdirac-dms-dataset-describe
  describe a dataset from DB
  Usage:
    besdirac-dms-dataset-describe <datasetname>
"""

__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script

Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()

if len(args) != 1:
    Script.showHelp()
datasetName = args[0]

from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
badger.getDatasetDescription(datasetName)
exit(0)

Ejemplo n.º 17

    besdirac-dms-dataset-delete <datasetname>
"""

__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script

Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()

if len(args) != 1:
    Script.showHelp()
datasetName = args[0]

from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager
rm = ReplicaManager()

result = badger.getFilesByDatasetName(datasetName)

if result['OK']:
    fileList = result['Value']
    fileCountDict = badger.reCalcCount(fileList, False)
    badger.removeDataset(datasetName)

    for file in fileCountDict:
        if fileCountDict[file] == 0:
            result = rm.removeFile(file)
            if not result['OK']:
                print "Failed remove file %s" % file

Ejemplo n.º 18

from DIRAC.Core.Base import Script

Script.setUsageMessage(__doc__)

args = Script.getPositionalArgs()
#print len(args)
if len(args) < 3:
    Script.showHelp()
    exit(-1)

datasetName = args[0]
path = args[1]
strArg = args[2]

from IHEPDIRAC.Badger.API.Badger import Badger

badger = Badger()
prefix = badger.getDatasetNamePrefix()
datasetName = prefix + datasetName

#print datasetName
result = badger.registerDataset(datasetName, path, strArg)
if result['OK']:
    resVal = badger.getFilesByDatasetName(datasetName)
    if resVal['OK']:
        fileList = resVal['Value']
        badger.reCalcCount(fileList)
#print result

exit(0)

Ejemplo n.º 19

Archivo: besdirac-dms-check-files.py Proyecto: zhangxt-ihep/IHEPDIRAC

   Usage :
    besdirac-dms-check-files <dfcDir> <localDir>
    Example: besdirac-dms-check-files /dir1  /dir2
'''
import os.path
import pprint
import DIRAC
from DIRAC.Core.Base import Script

from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient

Script.setUsageMessage(__doc__)
Script.parseCommandLine(ignoreErrors=True)
from IHEPDIRAC.Badger.API.Badger import Badger

badger = Badger()
dirs = Script.getPositionalArgs()
dfcdir = dirs[0]
localDir = dirs[1]

#get DFC file dict
lfns = badger.listDir(dfcdir)
lfnDict = badger.getSize(lfns)
base_lfnDict = {}
for k, v in lfnDict.items():
    k = os.path.basename(k)
    base_lfnDict[k] = v
#get local files dict
localFiles = badger.getFilenamesByLocaldir(localDir)
base_localDict = {}
for file in localFiles:

Ejemplo n.º 20

Archivo: besdirac-dms-dataset-list.py Proyecto: zhangxt-ihep/IHEPDIRAC

#!/usr/bin/env python
#mtime:2013/12/09
"""
list dataste names and their metadata
"""
__RCSID__ = "$Id$"

import time
import pprint
import DIRAC
from DIRAC.Core.Base import Script
Script.parseCommandLine(ignoreErrors=True)
Script.setUsageMessage(__doc__)

from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
badger.listDatasets()
exitCode = 0
DIRAC.exit(exitCode)

Ejemplo n.º 21

Archivo: besdirac-dms-dataset-update.py Proyecto: zhangxt-ihep/IHEPDIRAC

#!/usr/bin/env python
#mtime:2013/12/09
"""
besdirac-dms-dataset-update
  if a dataset changed,should update 
  Usage:
    besdirac-dms-dataset-update <datasetname>
"""

__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script

Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()

if len(args) != 1:
    Script.showHelp()
datasetName = args[0]

from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
badger.updateDataset(datasetName)
exit(0)