def unlink(self, path):
print '*** unlink', path
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
from COMDIRAC.Interfaces import DCatalog
fcc = FileCatalogClientCLI(DCatalog().catalog)
fcc.do_rm(path)
return 0
def getFilesByMetadataQuery(self, query):
"""Return a list of LFNs satisfying given query conditions.
Example usage:
>>> badger.getFilesByMetadataQuery('resonance=jpsi bossVer=6.5.5 expNum=exp1')
['/bes/File/jpsi/6.5.5/data/all/exp1/file1', .....]
"""
#TODO: checking of output, error catching
fc = self.client
#TODO: calling the FileCatalog CLI object and its private method
# is not a good way of doing this! but use it to allow construction of
# the query meantime, until createQuery is made a public method
cli = FileCatalogClientCLI(fc)
metadataDict = cli._FileCatalogClientCLI__createQuery(query)
result = fc.findFilesByMetadata(metadataDict,'/')
if result['OK']:
lfns = fc.findFilesByMetadata(metadataDict,'/')['Value']
lfns.sort()
return lfns
else:
print "ERROR: No files found which match query conditions."
return None
def unlink ( self, path ):
print '*** unlink', path
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
from COMDIRAC.Interfaces import DCatalog
fcc = FileCatalogClientCLI( DCatalog().catalog )
fcc.do_rm(path)
return 0
def create (self, path, flags, mode):
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
from COMDIRAC.Interfaces import DCatalog
fcc = FileCatalogClientCLI( DCatalog().catalog )
tmp = str(time.time())+str(random.random())
f = open(self.tmpdir+'/'+tmp, 'w+b')
f.write("\0")
f.close()
fcc.do_add( path+" "+self.tmpdir+"/"+tmp+" "+self.SE )
os.remove(self.tmpdir+'/'+tmp)
self.file[path] = {"handler":os.tmpfile(),"modified":False}#,"mode":os.stat(path)[stat.ST_MODE]}
self.file[path]["handler"].write("\0")
return 0
def create(self, path, flags, mode):
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
from COMDIRAC.Interfaces import DCatalog
fcc = FileCatalogClientCLI(DCatalog().catalog)
tmp = str(time.time()) + str(random.random())
f = open(self.tmpdir + '/' + tmp, 'w+b')
f.write("\0")
f.close()
fcc.do_add(path + " " + self.tmpdir + "/" + tmp + " " + self.SE)
os.remove(self.tmpdir + '/' + tmp)
self.file[path] = {
"handler": os.tmpfile(),
"modified": False
} #,"mode":os.stat(path)[stat.ST_MODE]}
self.file[path]["handler"].write("\0")
return 0
def chmod(self, path, mode):
print '*** chmod', path, oct(mode)
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
from COMDIRAC.Interfaces import DCatalog
FileCatalogClientCLI(DCatalog().catalog).do_chmod(
str(oct(mode & 0777))[1:] + " " + path)
#return -errno.ENOSYS
return 0
def main():
fcType = 'FileCatalog'
Script.registerSwitch(
"f:", "file-catalog=",
" Catalog client type to use (default %s)" % fcType)
Script.parseCommandLine(ignoreErrors=False)
from DIRAC import gConfig, exit as dexit
from DIRAC.Resources.Catalog.FileCatalogFactory import FileCatalogFactory
__RCSID__ = "$Id$"
fcType = gConfig.getValue("/LocalSite/FileCatalog", "")
res = gConfig.getSections("/Resources/FileCatalogs", listOrdered=True)
if not res['OK']:
dexit(1)
fcList = res['Value']
if not fcType:
if res['OK']:
fcType = res['Value'][0]
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "f" or switch[0].lower() == "file-catalog":
fcType = switch[1]
if not fcType:
print("No file catalog given and defaults could not be obtained")
sys.exit(1)
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
result = FileCatalogFactory().createCatalog(fcType)
if not result['OK']:
print(result['Message'])
if fcList:
print("Possible choices are:")
for fc in fcList:
print(' ' * 5, fc)
sys.exit(1)
print("Starting %s client" % fcType)
catalog = result['Value']
cli = FileCatalogClientCLI(catalog)
cli.cmdloop()
def registerDataset(self, datasetName, path, conditions):
"""Register a new dataset in DFC. Takes dataset name and string with
conditions for new dataset as arguments.
datasetname format:
resonance_BossVer_eventtype_round_runL_runH_stream0_datatype
resonance_BossVer_eventtype_round_runL_runH_streamID_datatype
type(conditions) is str,like "resonance=jpsi bossVer=655 round=round1"
"""
fc = self.client
cli = FileCatalogClientCLI(fc)
metadataDict = cli._FileCatalogClientCLI__createQuery(conditions)
metadataDict["Path"] = path
result = fc.addDataset(datasetName, metadataDict)
if not result["OK"]:
print ("Error: %s" % result["Message"])
return S_ERROR()
else:
print "Added dataset %s with conditions %s" % (datasetName, conditions)
return S_OK()
def registerDataset(self, datasetName, path, conditions):
"""Register a new dataset in DFC. Takes dataset name and string with
conditions for new dataset as arguments.
datasetname format:
resonance_BossVer_eventtype_round_runL_runH_stream0_datatype
resonance_BossVer_eventtype_round_runL_runH_streamID_datatype
type(conditions) is str,like "resonance=jpsi bossVer=655 round=round1"
"""
fc = self.client
cli = FileCatalogClientCLI(fc)
metadataDict = cli._FileCatalogClientCLI__createQuery(conditions)
metadataDict['Path'] = path
result = fc.addDataset(datasetName, metadataDict)
if not result['OK']:
print("Error: %s" % result['Message'])
return S_ERROR()
else:
print "Added dataset %s with conditions %s" % (datasetName,
conditions)
return S_OK()
def rmdir(self, path):
print '*** rmdir', path
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
result = FileCatalogClient().listDirectory(path)
flist = result['Value']['Successful'][path]['Files'].keys()
if not flist:
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
from COMDIRAC.Interfaces import DCatalog
FileCatalogClientCLI(DCatalog().catalog).do_rmdir(path)
return 0
else:
return -errno.ENOTEMPTY
def release ( self, path, flags ):
print '*** release', path, flags
#if self.file[path]["mode"]&(stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH):
# return -errno.EACCES
#print self.file[path]
if self.file[path]["modified"]:
self.file[path]["handler"].seek(0)
off = 1 if self.file[path]["handler"].read(1) == '\0' else 0
tmp = str(time.time())+str(random.random())
f = open(self.tmpdir+'/'+tmp,"w+b")
self.file[path]["handler"].seek(off)
f.write(self.file[path]["handler"].read())
f.close()
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
from COMDIRAC.Interfaces import DCatalog
fcc = FileCatalogClientCLI( DCatalog().catalog )
fcc.do_rm(path)
fcc.do_add( path+" "+self.tmpdir+"/"+tmp+" "+self.SE )
os.remove(self.tmpdir+'/'+tmp)
self.file[path]["handler"].close()
del self.file[path]
return 0
def release(self, path, flags):
print '*** release', path, flags
#if self.file[path]["mode"]&(stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH):
# return -errno.EACCES
#print self.file[path]
if self.file[path]["modified"]:
self.file[path]["handler"].seek(0)
off = 1 if self.file[path]["handler"].read(1) == '\0' else 0
tmp = str(time.time()) + str(random.random())
f = open(self.tmpdir + '/' + tmp, "w+b")
self.file[path]["handler"].seek(off)
f.write(self.file[path]["handler"].read())
f.close()
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
from COMDIRAC.Interfaces import DCatalog
fcc = FileCatalogClientCLI(DCatalog().catalog)
fcc.do_rm(path)
fcc.do_add(path + " " + self.tmpdir + "/" + tmp + " " + self.SE)
os.remove(self.tmpdir + '/' + tmp)
self.file[path]["handler"].close()
del self.file[path]
return 0
'Arguments:',
' lfn: logical file name',] )
)
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
from optparse import OptionParser
parser = OptionParser( )
( options, args ) = parser.parse_args( )
session = DSession( )
catalog = DCatalog( )
if len( args ) < 1:
print "No argument provided\n%s:" % Script.scriptName
Script.showHelp( )
DIRAC.exit( -1 )
Script.enableCS( )
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
fccli = FileCatalogClientCLI( catalog.catalog )
for arg in args:
# lfn
lfn = pathFromArgument( session, args[ 0 ] )
fccli.do_replicas( lfn )
if __name__ == "__main__":
import sys
from DIRAC.Core.Base import Script
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [Path]...' % Script.scriptName,
'Arguments:',
' Path: directory path',] )
)
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
session = DSession( )
if len( args ) < 1:
print "Error: No argument provided\n%s:" % Script.scriptName
Script.showHelp( )
DIRAC.exit( -1 )
Script.enableCS( )
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
fccli = FileCatalogClientCLI( createCatalog( ) )
for p in pathFromArguments( session, args ):
fccli.do_rmdir( p )
import sys
from DIRAC.Core.Base import Script
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s Path...' % Script.scriptName,
'Arguments:',
' Path: path to new directory',] )
)
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
session = DSession( )
if len( args ) < 1:
print "Error: No argument provided\n%s:" % Script.scriptName
Script.showHelp( )
DIRAC.exit( -1 )
Script.enableCS( )
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
fccli = FileCatalogClientCLI( createCatalog( ) )
for p in pathFromArguments( session, args ):
fccli.do_mkdir( p )
Script.enableCS( ) Script.parseCommandLine( ignoreErrors = True ) args = Script.getPositionalArgs() session = DSession( ) catalog = DCatalog( ) if len( args ) < 2: print "Error: not enough arguments provided\n%s:" % Script.scriptName Script.showHelp( ) DIRAC.exit( -1 ) mode = args[ 0 ] lfns = [ ] for path in args[ 1: ]: lfns.append( pathFromArgument( session, path )) optstr = "" if params.recursive: optstr = "-R " from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI fccli = FileCatalogClientCLI( catalog.catalog ) for lfn in lfns: fccli.do_chmod( optstr + mode + " " + lfn )
res = gConfig.getSections("/Resources/FileCatalogs",listOrdered = True)
if not res['OK']:
dexit(1)
fcList = res['Value']
if not fcType:
if res['OK']:
fcType = res['Value'][0]
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "f" or switch[0].lower() == "file-catalog":
fcType = switch[1]
if not fcType:
print "No file catalog given and defaults could not be obtained"
sys.exit(1)
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
result = FileCatalogFactory().createCatalog(fcType)
if not result['OK']:
print result['Message']
if fcList:
print "Possible choices are:"
for fc in fcList:
print ' '*5,fc
sys.exit(1)
print "Starting %s client" % fcType
catalog = result['Value']
cli = FileCatalogClientCLI( catalog )
cli.cmdloop()
def mkdir(self, path, mode):
print '*** mkdir', path, oct(mode)
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
from COMDIRAC.Interfaces import DCatalog
FileCatalogClientCLI(DCatalog().catalog).do_mkdir(path)
return 0
from DIRAC.Core.Base import Script
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [options] lfn metaspec...' % Script.scriptName,
'Arguments:',
' lfn: directory entry in the FileCatalog',
' metaspec: metadata index specifcation (of the form: "meta=value" or "meta<value", "meta!=value", etc.)',
'', 'Examples:',
' $ dfind . "some_integer_metadata>1"',
] )
)
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
session = DSession( )
catalog = DCatalog( )
if len( args ) < 2:
print "Error: No argument provided\n%s:" % Script.scriptName
Script.showHelp( )
DIRAC.exit( -1 )
lfn = pathFromArgument( session, args[0] )
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
fccli = FileCatalogClientCLI( catalog.catalog )
fccli.do_find( lfn + " " + " ".join( args[1:] ) )
lfns = args[ :-1 ]
pairs = [ ]
# STRANGE: dirac only accepts directories for download destination
if not os.path.isdir( local_path ):
print "Error: Destination local path must be a directory"
DIRAC.exit( -1 )
if os.path.isdir( local_path ):
# we can accept one ore more lfns
for lfn in lfns:
# STRANGE: dirac only accepts directories for download destination
#pairs.append( (pathFromArgument( session, lfn ), os.path.join( local_path, os.path.basename( lfn )) ))
pairs.append( (pathFromArgument( session, lfn ), local_path ))
else:
if len( lfns ) > 1:
print "Error: Destination path must be a directory when downloading multiple local files"
DIRAC.exit( -1 )
# local filename replace lfn filename
pairs.append( (pathFromArgument( session, lfn ), local_path ))
Script.enableCS( )
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
fccli = FileCatalogClientCLI( catalog.catalog )
for lfn, local_path in pairs:
fccli.do_get( lfn + " " + local_path )
res = gConfig.getSections("/Resources/FileCatalogs", listOrdered=True)
if not res['OK']:
dexit(1)
fcList = res['Value']
if not fcType:
if res['OK']:
fcType = res['Value'][0]
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "f" or switch[0].lower() == "file-catalog":
fcType = switch[1]
if not fcType:
print "No file catalog given and defaults could not be obtained"
sys.exit(1)
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
result = FileCatalogFactory().createCatalog(fcType)
if not result['OK']:
print result['Message']
if fcList:
print "Possible choices are:"
for fc in fcList:
print ' ' * 5, fc
sys.exit(1)
print "Starting %s client" % fcType
catalog = result['Value']
cli = FileCatalogClientCLI(catalog)
cli.cmdloop()
DIRAC.exit( -1 )
# default lfn: same file name as local_path
lfns = pathFromArguments( session, args )
# destination SE
dsts = [ ]
if params.destinationSE:
dsts = params.destinationSE.split( "," )
else:
dsts = session.getReplicationSEs()
if not dsts:
dsts = [ session.getEnv( "default_se", "DIRAC-USER" )[ "Value" ] ]
srcopt = ""
if params.sourceSE:
srcopt = " " + params.sourceSE
Script.enableCS()
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
fccli = FileCatalogClientCLI( catalog.catalog )
for lfn in lfns:
for dst in dsts:
# print "replicating", lfn, "to SE", dst
fccli.do_replicate( lfn + " " + dst + srcopt )
' $ dmeta rm ./some_lfn_file some_meta',
' $ dmeta ls ./some_lfn_file',
] )
)
Script.registerSwitch( "i:", "index=", "set or remove metadata indices", params.setIndex )
Script.registerSwitch( "I", "list-index", "list defined metadata indices", params.setListIndex )
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
session = DSession( )
catalog = DCatalog( )
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
fccli = FileCatalogClientCLI( catalog.catalog )
if params.getIndex( ):
if params.getIndex( ) == "r":
for meta in args:
cmdline = "index -r %s" % meta
#print cmdline
fccli.do_meta( cmdline )
else:
for arg in args:
meta, type = arg.split( "=" )
cmdline = "index -%s %s %s" % ( params.getIndex( ), meta, type )
#print cmdline
fccli.do_meta( cmdline )
DIRAC.exit( 0 )
Script.registerSwitch("f:", "file-catalog=",
" Catalog client type to use (default %s)" % _fcType)
Script.parseCommandLine(ignoreErrors=False)
# By a Switch ?
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "f" or switch[0].lower() == "file-catalog":
_fcType = switch[1]
# Create File Catalog Client
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
_client = FileCatalogClient("DataManagement/" + _fcType)
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
_helper_fc_cli = FileCatalogClientCLI(_client)
_helper_create_query = _helper_fc_cli._FileCatalogClientCLI__createQuery
def getLFNsByQueryString(query):
metadataDict = _helper_create_query(query)
lfns = _client.findFilesByMetadata(metadataDict, '/')
if not lfns["OK"]:
import sys
sys.stderr.write(repr(lfns))
return list()
return lfns["Value"]
fcType = res['Value']
if not fcType:
res = gConfig.getSections("/Resources/FileCatalogs", listOrdered=True)
if res['OK']:
fcType = res['Value'][0]
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "f" or switch[0].lower() == "file-catalog":
fcType = switch[1]
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
if fcType == "LcgFileCatalog" or fcType == "LFC":
from DIRAC.Resources.Catalog.LcgFileCatalogClient import LcgFileCatalogClient
cli = FileCatalogClientCLI(LcgFileCatalogClient())
try:
host = os.environ['LFC_HOST']
except Exception, x:
print "LFC_HOST environment variable not defined"
sys.exit(1)
print "Starting LFC FileCatalog client"
cli.cmdloop()
elif fcType == "LcgFileCatalogProxy" or fcType == "LFCproxy":
from DIRAC.Resources.Catalog.LcgFileCatalogProxyClient import LcgFileCatalogProxyClient
cli = FileCatalogClientCLI(LcgFileCatalogProxyClient())
print "Starting LFC Proxy FileCatalog client"
cli.cmdloop()
elif fcType == "LcgFileCatalogCombined" or fcType == "LFCCombined":
from DIRAC.Resources.Catalog.LcgFileCatalogCombinedClient import LcgFileCatalogCombinedClient
cli = FileCatalogClientCLI(LcgFileCatalogCombinedClient())
if catalog.isDir( lfn ):
# we can accept one ore more local files
for lp in local_paths:
pairs.append( (lp, os.path.join( lfn, os.path.basename( lp )) ))
else:
if len( local_paths ) > 1:
print "Error: Destination LFN must be a directory when registering multiple local files"
Script.showHelp( )
DIRAC.exit( -1 )
# lfn filename replace local filename
pairs.append( (local_path, lfn ))
# destination SE
se = params.destinationSE
if not se:
retVal = session.getEnv( "default_se", "DIRAC-USER" )
if not retVal[ "OK" ]:
critical( retVal[ "Message" ] )
se = retVal[ "Value" ]
Script.enableCS( )
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
fccli = FileCatalogClientCLI( catalog.catalog )
for local_path, lfn in pairs:
fccli.do_add( lfn + " " + local_path + " " + se )