コード例 #1
0
ファイル: diracfs.py プロジェクト: xmimi/diracfs
 def unlink(self, path):
     print '*** unlink', path
     from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
     from COMDIRAC.Interfaces import DCatalog
     fcc = FileCatalogClientCLI(DCatalog().catalog)
     fcc.do_rm(path)
     return 0
コード例 #2
0
ファイル: Badger.py プロジェクト: jemtchou/BESDIRAC
    def getFilesByMetadataQuery(self, query):
        """Return a list of LFNs satisfying given query conditions.

           Example usage:
           >>> badger.getFilesByMetadataQuery('resonance=jpsi bossVer=6.5.5 expNum=exp1')
           ['/bes/File/jpsi/6.5.5/data/all/exp1/file1', .....]

        """
        #TODO: checking of output, error catching


        fc = self.client
        #TODO: calling the FileCatalog CLI object and its private method
        # is not a good way of doing this! but use it to allow construction of
        # the query meantime, until createQuery is made a public method
        cli = FileCatalogClientCLI(fc)
        metadataDict = cli._FileCatalogClientCLI__createQuery(query)
        result = fc.findFilesByMetadata(metadataDict,'/')
        if result['OK']:
            lfns = fc.findFilesByMetadata(metadataDict,'/')['Value']
            lfns.sort()
            return lfns
        else:
            print "ERROR: No files found which match query conditions."
            return None
コード例 #3
0
ファイル: DiracFS.py プロジェクト: DIRACGrid/FSDIRAC
 def unlink ( self, path ):
     print '*** unlink', path
     from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
     from COMDIRAC.Interfaces import DCatalog
     fcc = FileCatalogClientCLI( DCatalog().catalog )
     fcc.do_rm(path)
     return 0
コード例 #4
0
ファイル: DiracFS.py プロジェクト: DIRACGrid/FSDIRAC
 def create (self, path, flags, mode):
     from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
     from COMDIRAC.Interfaces import DCatalog
     fcc = FileCatalogClientCLI( DCatalog().catalog )
     tmp = str(time.time())+str(random.random())
     f = open(self.tmpdir+'/'+tmp, 'w+b')
     f.write("\0")
     f.close()
     fcc.do_add( path+" "+self.tmpdir+"/"+tmp+" "+self.SE )
     os.remove(self.tmpdir+'/'+tmp)
     self.file[path] = {"handler":os.tmpfile(),"modified":False}#,"mode":os.stat(path)[stat.ST_MODE]}
     self.file[path]["handler"].write("\0")
     return 0
コード例 #5
0
ファイル: diracfs.py プロジェクト: xmimi/diracfs
 def create(self, path, flags, mode):
     from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
     from COMDIRAC.Interfaces import DCatalog
     fcc = FileCatalogClientCLI(DCatalog().catalog)
     tmp = str(time.time()) + str(random.random())
     f = open(self.tmpdir + '/' + tmp, 'w+b')
     f.write("\0")
     f.close()
     fcc.do_add(path + " " + self.tmpdir + "/" + tmp + " " + self.SE)
     os.remove(self.tmpdir + '/' + tmp)
     self.file[path] = {
         "handler": os.tmpfile(),
         "modified": False
     }  #,"mode":os.stat(path)[stat.ST_MODE]}
     self.file[path]["handler"].write("\0")
     return 0
コード例 #6
0
ファイル: diracfs.py プロジェクト: xmimi/diracfs
 def chmod(self, path, mode):
     print '*** chmod', path, oct(mode)
     from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
     from COMDIRAC.Interfaces import DCatalog
     FileCatalogClientCLI(DCatalog().catalog).do_chmod(
         str(oct(mode & 0777))[1:] + " " + path)
     #return -errno.ENOSYS
     return 0
コード例 #7
0
def main():
    fcType = 'FileCatalog'
    Script.registerSwitch(
        "f:", "file-catalog=",
        "   Catalog client type to use (default %s)" % fcType)
    Script.parseCommandLine(ignoreErrors=False)

    from DIRAC import gConfig, exit as dexit
    from DIRAC.Resources.Catalog.FileCatalogFactory import FileCatalogFactory

    __RCSID__ = "$Id$"

    fcType = gConfig.getValue("/LocalSite/FileCatalog", "")

    res = gConfig.getSections("/Resources/FileCatalogs", listOrdered=True)
    if not res['OK']:
        dexit(1)
    fcList = res['Value']
    if not fcType:
        if res['OK']:
            fcType = res['Value'][0]

    for switch in Script.getUnprocessedSwitches():
        if switch[0].lower() == "f" or switch[0].lower() == "file-catalog":
            fcType = switch[1]

    if not fcType:
        print("No file catalog given and defaults could not be obtained")
        sys.exit(1)

    from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI

    result = FileCatalogFactory().createCatalog(fcType)
    if not result['OK']:
        print(result['Message'])
        if fcList:
            print("Possible choices are:")
            for fc in fcList:
                print(' ' * 5, fc)
        sys.exit(1)
    print("Starting %s client" % fcType)
    catalog = result['Value']
    cli = FileCatalogClientCLI(catalog)
    cli.cmdloop()
コード例 #8
0
ファイル: Badger.py プロジェクト: besdiracgrid/BESDIRAC
 def registerDataset(self, datasetName, path, conditions):
     """Register a new dataset in DFC. Takes dataset name and string with
        conditions for new dataset as arguments.
        datasetname format:  
        resonance_BossVer_eventtype_round_runL_runH_stream0_datatype
        resonance_BossVer_eventtype_round_runL_runH_streamID_datatype
        type(conditions) is str,like "resonance=jpsi bossVer=655 round=round1"
     """
     fc = self.client
     cli = FileCatalogClientCLI(fc)
     metadataDict = cli._FileCatalogClientCLI__createQuery(conditions)
     metadataDict["Path"] = path
     result = fc.addDataset(datasetName, metadataDict)
     if not result["OK"]:
         print ("Error: %s" % result["Message"])
         return S_ERROR()
     else:
         print "Added dataset %s with conditions %s" % (datasetName, conditions)
         return S_OK()
コード例 #9
0
ファイル: Badger.py プロジェクト: zhangxt-ihep/IHEPDIRAC
 def registerDataset(self, datasetName, path, conditions):
     """Register a new dataset in DFC. Takes dataset name and string with
        conditions for new dataset as arguments.
        datasetname format:  
        resonance_BossVer_eventtype_round_runL_runH_stream0_datatype
        resonance_BossVer_eventtype_round_runL_runH_streamID_datatype
        type(conditions) is str,like "resonance=jpsi bossVer=655 round=round1"
     """
     fc = self.client
     cli = FileCatalogClientCLI(fc)
     metadataDict = cli._FileCatalogClientCLI__createQuery(conditions)
     metadataDict['Path'] = path
     result = fc.addDataset(datasetName, metadataDict)
     if not result['OK']:
         print("Error: %s" % result['Message'])
         return S_ERROR()
     else:
         print "Added dataset %s with conditions %s" % (datasetName,
                                                        conditions)
         return S_OK()
コード例 #10
0
ファイル: diracfs.py プロジェクト: xmimi/diracfs
 def rmdir(self, path):
     print '*** rmdir', path
     from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
     result = FileCatalogClient().listDirectory(path)
     flist = result['Value']['Successful'][path]['Files'].keys()
     if not flist:
         from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
         from COMDIRAC.Interfaces import DCatalog
         FileCatalogClientCLI(DCatalog().catalog).do_rmdir(path)
         return 0
     else:
         return -errno.ENOTEMPTY
コード例 #11
0
ファイル: DiracFS.py プロジェクト: DIRACGrid/FSDIRAC
    def release ( self, path, flags ):
        print '*** release', path, flags
        #if self.file[path]["mode"]&(stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH):
        #    return -errno.EACCES
        #print self.file[path]
        if self.file[path]["modified"]:
            self.file[path]["handler"].seek(0)
            off = 1 if self.file[path]["handler"].read(1) == '\0' else 0

            tmp = str(time.time())+str(random.random())
            f = open(self.tmpdir+'/'+tmp,"w+b")
            self.file[path]["handler"].seek(off)
            f.write(self.file[path]["handler"].read())
            f.close()
    
            from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
            from COMDIRAC.Interfaces import DCatalog
            fcc = FileCatalogClientCLI( DCatalog().catalog )
            fcc.do_rm(path)
            fcc.do_add( path+" "+self.tmpdir+"/"+tmp+" "+self.SE )
            os.remove(self.tmpdir+'/'+tmp)
        self.file[path]["handler"].close()
        del self.file[path]
        return 0
コード例 #12
0
ファイル: diracfs.py プロジェクト: xmimi/diracfs
    def release(self, path, flags):
        print '*** release', path, flags
        #if self.file[path]["mode"]&(stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH):
        #    return -errno.EACCES
        #print self.file[path]
        if self.file[path]["modified"]:
            self.file[path]["handler"].seek(0)
            off = 1 if self.file[path]["handler"].read(1) == '\0' else 0

            tmp = str(time.time()) + str(random.random())
            f = open(self.tmpdir + '/' + tmp, "w+b")
            self.file[path]["handler"].seek(off)
            f.write(self.file[path]["handler"].read())
            f.close()

            from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
            from COMDIRAC.Interfaces import DCatalog
            fcc = FileCatalogClientCLI(DCatalog().catalog)
            fcc.do_rm(path)
            fcc.do_add(path + " " + self.tmpdir + "/" + tmp + " " + self.SE)
            os.remove(self.tmpdir + '/' + tmp)
        self.file[path]["handler"].close()
        del self.file[path]
        return 0
コード例 #13
0
ファイル: dreplicas.py プロジェクト: DIRACGrid/COMDIRAC
                                       'Arguments:',
                                       '  lfn:     logical file name',] )
                          )

  Script.parseCommandLine( ignoreErrors = True )
  args = Script.getPositionalArgs()

  from optparse import OptionParser

  parser = OptionParser( )

  ( options, args ) = parser.parse_args( )

  session = DSession( )
  catalog = DCatalog( )

  if len( args ) < 1:
    print "No argument provided\n%s:" % Script.scriptName
    Script.showHelp( )
    DIRAC.exit( -1 )

  Script.enableCS( )

  from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
  fccli = FileCatalogClientCLI( catalog.catalog )

  for arg in args:
    # lfn
    lfn = pathFromArgument( session, args[ 0 ] )
    fccli.do_replicas( lfn )
コード例 #14
0
ファイル: drmdir.py プロジェクト: atsareg/COMDIRAC
if __name__ == "__main__":
  import sys

  from DIRAC.Core.Base import Script

  Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
                                       'Usage:',
                                       '  %s [Path]...' % Script.scriptName,
                                       'Arguments:',
                                       '  Path:     directory path',] )
                          )

  Script.parseCommandLine( ignoreErrors = True )
  args = Script.getPositionalArgs()

  session = DSession( )

  if len( args ) < 1:
    print "Error: No argument provided\n%s:" % Script.scriptName
    Script.showHelp( )
    DIRAC.exit( -1 )

  Script.enableCS( )

  from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
  fccli = FileCatalogClientCLI( createCatalog( ) )

  for p in pathFromArguments( session, args ):
    fccli.do_rmdir( p )

コード例 #15
0
ファイル: dmkdir.py プロジェクト: atsareg/COMDIRAC
  import sys

  from DIRAC.Core.Base import Script

  Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
                                       'Usage:',
                                       '  %s Path...' % Script.scriptName,
                                       'Arguments:',
                                       '  Path:     path to new directory',] )
                          )

  Script.parseCommandLine( ignoreErrors = True )
  args = Script.getPositionalArgs()

  session = DSession( )

  if len( args ) < 1:
    print "Error: No argument provided\n%s:" % Script.scriptName
    Script.showHelp( )
    DIRAC.exit( -1 )

  Script.enableCS( )

  from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI

  fccli = FileCatalogClientCLI( createCatalog( ) )

  for p in pathFromArguments( session, args ):
    fccli.do_mkdir( p )

コード例 #16
0
ファイル: dchmod.py プロジェクト: atsareg/COMDIRAC
Script.enableCS( )

Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()

session = DSession( )
catalog = DCatalog( )

if len( args ) < 2:
  print "Error: not enough arguments provided\n%s:" % Script.scriptName
  Script.showHelp( )
  DIRAC.exit( -1 )

mode = args[ 0 ]

lfns = [ ]
for path in args[ 1: ]:
  lfns.append( pathFromArgument( session, path ))

optstr = ""
if params.recursive:
  optstr = "-R "

from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI

fccli = FileCatalogClientCLI( catalog.catalog )

for lfn in lfns:
  fccli.do_chmod( optstr + mode + " " + lfn )
コード例 #17
0
res = gConfig.getSections("/Resources/FileCatalogs",listOrdered = True)
if not res['OK']:
  dexit(1)
fcList = res['Value']
if not fcType:
  if res['OK']:
    fcType = res['Value'][0]

for switch in Script.getUnprocessedSwitches():
  if switch[0].lower() == "f" or switch[0].lower() == "file-catalog":
    fcType = switch[1]

if not fcType:
  print "No file catalog given and defaults could not be obtained"
  sys.exit(1)

from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI

result = FileCatalogFactory().createCatalog(fcType)
if not result['OK']:
  print result['Message']
  if fcList:
    print "Possible choices are:"
    for fc in fcList:
      print ' '*5,fc
  sys.exit(1)
print "Starting %s client" % fcType
catalog = result['Value']
cli = FileCatalogClientCLI( catalog )
cli.cmdloop()
コード例 #18
0
ファイル: diracfs.py プロジェクト: xmimi/diracfs
 def mkdir(self, path, mode):
     print '*** mkdir', path, oct(mode)
     from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
     from COMDIRAC.Interfaces import DCatalog
     FileCatalogClientCLI(DCatalog().catalog).do_mkdir(path)
     return 0
コード例 #19
0
ファイル: dfind.py プロジェクト: DIRACGrid/COMDIRAC
  from DIRAC.Core.Base import Script

  Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
                                       'Usage:',
                                       '  %s [options] lfn metaspec...' % Script.scriptName,
                                       'Arguments:',
                                       ' lfn:         directory entry in the FileCatalog',
                                       ' metaspec:    metadata index specifcation (of the form: "meta=value" or "meta<value", "meta!=value", etc.)',
                                       '', 'Examples:',
                                       '  $ dfind . "some_integer_metadata>1"',
                                       ] )
                          )

  Script.parseCommandLine( ignoreErrors = True )
  args = Script.getPositionalArgs()

  session = DSession( )
  catalog = DCatalog( )

  if len( args ) < 2:
    print "Error: No argument provided\n%s:" % Script.scriptName
    Script.showHelp( )
    DIRAC.exit( -1 )

  lfn = pathFromArgument( session, args[0] )

  from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
  fccli = FileCatalogClientCLI( catalog.catalog )

  fccli.do_find( lfn + " " + " ".join( args[1:] ) )
コード例 #20
0
ファイル: dget.py プロジェクト: atsareg/COMDIRAC
  lfns = args[ :-1 ]
  pairs = [ ]

  # STRANGE: dirac only accepts directories for download destination
  if not os.path.isdir( local_path ):
    print "Error: Destination local path must be a directory"
    DIRAC.exit( -1 )

  if os.path.isdir( local_path ):
    # we can accept one ore more lfns
    for lfn in lfns:
      # STRANGE: dirac only accepts directories for download destination
      #pairs.append( (pathFromArgument( session, lfn ), os.path.join( local_path, os.path.basename( lfn )) ))
      pairs.append( (pathFromArgument( session, lfn ), local_path ))
  else:
    if len( lfns ) > 1:
      print "Error: Destination path must be a directory when downloading multiple local files"
      DIRAC.exit( -1 )

    # local filename replace lfn filename
    pairs.append( (pathFromArgument( session, lfn ), local_path ))

Script.enableCS( )

from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI

fccli = FileCatalogClientCLI( catalog.catalog )

for lfn, local_path in pairs:
  fccli.do_get( lfn + " " + local_path )
コード例 #21
0
res = gConfig.getSections("/Resources/FileCatalogs", listOrdered=True)
if not res['OK']:
    dexit(1)
fcList = res['Value']
if not fcType:
    if res['OK']:
        fcType = res['Value'][0]

for switch in Script.getUnprocessedSwitches():
    if switch[0].lower() == "f" or switch[0].lower() == "file-catalog":
        fcType = switch[1]

if not fcType:
    print "No file catalog given and defaults could not be obtained"
    sys.exit(1)

from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI

result = FileCatalogFactory().createCatalog(fcType)
if not result['OK']:
    print result['Message']
    if fcList:
        print "Possible choices are:"
        for fc in fcList:
            print ' ' * 5, fc
    sys.exit(1)
print "Starting %s client" % fcType
catalog = result['Value']
cli = FileCatalogClientCLI(catalog)
cli.cmdloop()
コード例 #22
0
ファイル: drepl.py プロジェクト: DIRACGrid/COMDIRAC
    DIRAC.exit( -1 )

  # default lfn: same file name as local_path
  lfns = pathFromArguments( session, args )

  # destination SE
  dsts = [ ]

  if params.destinationSE:
    dsts = params.destinationSE.split( "," )
  else:
    dsts = session.getReplicationSEs()
    if not dsts:
      dsts = [ session.getEnv( "default_se", "DIRAC-USER" )[ "Value" ] ]

  srcopt = ""
  if params.sourceSE:
    srcopt = " " + params.sourceSE


  Script.enableCS()

  from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
  fccli = FileCatalogClientCLI( catalog.catalog )

  for lfn in lfns:
    for dst in dsts:
      # print "replicating", lfn, "to SE", dst
      fccli.do_replicate( lfn + " " + dst + srcopt )

コード例 #23
0
ファイル: dmeta.py プロジェクト: DIRACGrid/COMDIRAC
                                       '  $ dmeta rm ./some_lfn_file some_meta',
                                       '  $ dmeta ls ./some_lfn_file',
                                       ] )
                          )
  Script.registerSwitch( "i:", "index=", "set or remove metadata indices", params.setIndex )
  Script.registerSwitch( "I", "list-index", "list defined metadata indices", params.setListIndex )

  Script.parseCommandLine( ignoreErrors = True )
  args = Script.getPositionalArgs()

  session = DSession( )
  catalog = DCatalog( )

  from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI

  fccli = FileCatalogClientCLI( catalog.catalog )

  if params.getIndex( ):
    if params.getIndex( ) == "r":
      for meta in args:
        cmdline = "index -r %s" % meta
        #print cmdline
        fccli.do_meta( cmdline )
    else:
      for arg in args:
        meta, type = arg.split( "=" )
        cmdline = "index -%s %s %s" % ( params.getIndex( ), meta, type )
        #print cmdline
        fccli.do_meta( cmdline )
    DIRAC.exit( 0 )
コード例 #24
0
Script.registerSwitch("f:", "file-catalog=",
                      "   Catalog client type to use (default %s)" % _fcType)

Script.parseCommandLine(ignoreErrors=False)

# By a Switch ?
for switch in Script.getUnprocessedSwitches():
    if switch[0].lower() == "f" or switch[0].lower() == "file-catalog":
        _fcType = switch[1]

# Create File Catalog Client
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
_client = FileCatalogClient("DataManagement/" + _fcType)

from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
_helper_fc_cli = FileCatalogClientCLI(_client)
_helper_create_query = _helper_fc_cli._FileCatalogClientCLI__createQuery


def getLFNsByQueryString(query):
    metadataDict = _helper_create_query(query)

    lfns = _client.findFilesByMetadata(metadataDict, '/')

    if not lfns["OK"]:
        import sys
        sys.stderr.write(repr(lfns))
        return list()

    return lfns["Value"]
コード例 #25
0
    fcType = res['Value']

if not fcType:
    res = gConfig.getSections("/Resources/FileCatalogs", listOrdered=True)
    if res['OK']:
        fcType = res['Value'][0]

for switch in Script.getUnprocessedSwitches():
    if switch[0].lower() == "f" or switch[0].lower() == "file-catalog":
        fcType = switch[1]

from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI

if fcType == "LcgFileCatalog" or fcType == "LFC":
    from DIRAC.Resources.Catalog.LcgFileCatalogClient import LcgFileCatalogClient
    cli = FileCatalogClientCLI(LcgFileCatalogClient())
    try:
        host = os.environ['LFC_HOST']
    except Exception, x:
        print "LFC_HOST environment variable not defined"
        sys.exit(1)
    print "Starting LFC FileCatalog client"
    cli.cmdloop()
elif fcType == "LcgFileCatalogProxy" or fcType == "LFCproxy":
    from DIRAC.Resources.Catalog.LcgFileCatalogProxyClient import LcgFileCatalogProxyClient
    cli = FileCatalogClientCLI(LcgFileCatalogProxyClient())
    print "Starting LFC Proxy FileCatalog client"
    cli.cmdloop()
elif fcType == "LcgFileCatalogCombined" or fcType == "LFCCombined":
    from DIRAC.Resources.Catalog.LcgFileCatalogCombinedClient import LcgFileCatalogCombinedClient
    cli = FileCatalogClientCLI(LcgFileCatalogCombinedClient())
コード例 #26
0
ファイル: dput.py プロジェクト: DIRACGrid/COMDIRAC
    if catalog.isDir( lfn ):
      # we can accept one ore more local files
      for lp in local_paths:
        pairs.append( (lp, os.path.join( lfn, os.path.basename( lp )) ))
    else:
      if len( local_paths ) > 1:
        print "Error: Destination LFN must be a directory when registering multiple local files"
        Script.showHelp( )
        DIRAC.exit( -1 )

      # lfn filename replace local filename
      pairs.append( (local_path, lfn ))

  # destination SE
  se = params.destinationSE
  if not se:
    retVal = session.getEnv( "default_se", "DIRAC-USER" )
    if not retVal[ "OK" ]:
      critical( retVal[ "Message" ] )
    se = retVal[ "Value" ]

  Script.enableCS( )

  from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
  fccli = FileCatalogClientCLI( catalog.catalog )

  for local_path, lfn in pairs:
    fccli.do_add( lfn + " " + local_path + " " + se )