def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(["LFN: Logical File Name or file containing LFNs"])
Script.parseCommandLine(ignoreErrors=True)
lfns = Script.getPositionalArgs()
if len(lfns) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
if len(lfns) == 1:
try:
with open(lfns[0], "r") as f:
lfns = f.read().splitlines()
except Exception:
pass
result = dirac.getFile(lfns, printOutput=True)
if not result["OK"]:
print("ERROR %s" % (result["Message"]))
exitCode = 2
DIRAC.exit(exitCode)
def getfile(lfn):
dirac = Dirac()
res = dirac.getFile( lfn )
if not res['OK']:
print 'Error downloading lfn: ' + lfn
return res['Message']
def do_get(self,args):
""" Download file from grid and store in a local directory
usage:
get <lfn> [<local_directory>]
"""
argss = args.split()
lfn = argss[0]
lfn = self.getPath(lfn)
dir = ''
if len(argss)>1:
dir = argss[1]
dirac = Dirac()
localCWD = ''
if dir:
localCWD = os.getcwd()
os.chdir(dir)
result = dirac.getFile(lfn)
if localCWD:
os.chdir(localCWD)
if not result['OK']:
print 'Error: %s' %(result['Message'])
else:
print "File %s successfully downloaded" % lfn
def main():
lfn = '/cepc/lustre-ro/' + filepath
dirac = Dirac()
result = dirac.getFile(lfn)
if not result['OK']:
gLogger.error('Download file error: %s' % result['Message'])
return 2
return 0
def getfile(lfn):
dirac = Dirac()
print('Start downloading ' + lfn + '\n', end='')
res = dirac.getFile(lfn, destDir=TEMPDIR)
if not res['OK']:
print('Error downloading lfn:' + lfn + '\n', end='')
return res['Message']
name = os.path.basename(lfn)
os.rename(os.path.join('.incomplete', name), name)
print('Successfully downloaded file:' + lfn + '\n', end='')
def main():
# Read command line options
parser = argparse.ArgumentParser(description="Download collection files from Dirac")
parser.add_argument("--indir", default=None, help="Dirac repository")
parser.add_argument("--outdir", default="", help="Output file directory")
args = parser.parse_args()
print("Download file from: {}".format(args.indir))
print("Output directory: {}".format(args.outdir))
# Create output directory
if not os.path.exists(args.outdir):
os.makedirs(args.outdir)
# Get list of files
batcmd = "dirac-dms-user-lfns --BaseDir {}".format(args.indir)
result = subprocess.check_output(batcmd, shell=True)
file_list = result.split()[-1]
# try reading the lfns file
try:
grid_file_list = open(file_list)
except IOError:
raise IOError("cannot read lfns file list...")
dirac = Dirac()
file_collection = []
for line in grid_file_list.read().splitlines():
if len(file_collection) < 100:
file_collection.append(line) # files will be downloaded later
else:
dirac.getFile(file_collection, destDir=args.outdir)
file_collection = [] # there won't be any loop at the end
if file_collection:
dirac.getFile(file_collection, destDir=args.outdir)
def downloadFilesByDatasetName(self, dataset_name):
"""downLoad a set of files form SE.
use getFilesByDatasetName() get a list of lfns and download these files.
Example usage:
>>>badger.downloadFilesByDatasetName('psipp_661_data_all_exp2')i
"""
dirac = Dirac()
fileList = self.getFilesByDatasetName(dataset_name)
result = dirac.getFile(fileList, printOutput=True)
if not result['OK']:
print 'ERROR %s' % (result['Message'])
return S_ERROR(result['Message'])
else:
return S_OK()
def downloadFilesByFilelist(self, fileList, destDir=""):
"""downLoad a set of files form SE.
use getFilesByFilelist() get a list of lfns and download these files.
fileList get from function getFilesByDatesetName()
Example usage:
>>>badger.downloadFilesByFilelist(fileList)
"""
errorDict = {}
dirac = Dirac()
# fileList = self.getFilesByDatasetName(dataset_name)
for lfn in fileList:
result = dirac.getFile(lfn, destDir, printOutput=False)
if not result["OK"]:
errorDict[lfn] = result["Message"]
if errorDict:
serr = S_ERROR()
serr["errorDict"] = errorDict
return serr
else:
return S_OK("File download successfully.")
def downloadFilesByFilelist(self, fileList, destDir=''):
"""downLoad a set of files form SE.
use getFilesByFilelist() get a list of lfns and download these files.
fileList get from function getFilesByDatesetName()
Example usage:
>>>badger.downloadFilesByFilelist(fileList)
"""
errorDict = {}
dirac = Dirac()
#fileList = self.getFilesByDatasetName(dataset_name)
for lfn in fileList:
result = dirac.getFile(lfn, destDir, printOutput=False)
if not result['OK']:
errorDict[lfn] = result['Message']
if errorDict:
serr = S_ERROR()
serr["errorDict"] = errorDict
return serr
else:
return S_OK("File download successfully.")
def main():
input_file_jobvar_name = 'JSUB_' + os.environ.get(
'JSUB_input_file_jobvar_name', 'input_file'
) # if input file is local, and put to DFC through jsub register.
input_lfn_jobvar_name = 'JSUB_' + os.environ.get(
'JSUB_input_lfn_jobvar_name',
'input_lfn') # if input file is directly from DFC
source_lfn_prefix = os.environ.get('JSUB_source_lfn_prefix')
input_path = os.environ.get(input_file_jobvar_name)
input_lfn = os.environ.get(input_lfn_jobvar_name)
destination = os.environ.get('JSUB_destination', './')
if input_lfn is None:
if source_lfn_prefix:
lfn = os.path.join(source_lfn_prefix, input_path)
else:
lfn = input_path_to_lfn(input_path)
else:
lfn = input_lfn
fname = os.path.basename(lfn)
# gfal_prefix = 'srm://storm.ihep.ac.cn:8444'
# gfal_path = gfal_prefix + input_path
# dest_dir = os.environ.get('destination_dir','./')
# dest_filename = dest_dir + os.path.basename(input_path)
# os.system('gfal-copy {} {}'.format(gfal_path,dest_filename)) #replace the input file
# download the file to the action folder
dirac = Dirac()
result = dirac.getFile(lfn)
if not result['OK']:
gLogger.error('Download file error: %s' % result['Message'])
return 2
# mv to destination
os.system('mv %s %s' % (fname, destination))
return 0
def main():
Script.parseCommandLine(ignoreErrors=True)
lfns = Script.getPositionalArgs()
if len(lfns) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
if len(lfns) == 1:
try:
with open(lfns[0], 'r') as f:
lfns = f.read().splitlines()
except Exception:
pass
result = dirac.getFile(lfns, printOutput=True)
if not result['OK']:
print('ERROR %s' % (result['Message']))
exitCode = 2
DIRAC.exit(exitCode)
if k.lower() in ["save"]:
dir_save = v
gLogger.info("Dataset Name: ", dataset)
gLogger.info("Save in: ", dir_save)
# Get the list of LFNs in one dataset
from DIRAC.Core.DISET.RPCClient import RPCClient
transferRequest = RPCClient("Transfer/Dataset")
res = transferRequest.list(dataset)
if not res["OK"]:
gLogger.error(res)
DIRAC.exit(-1)
file_list = [v[1] for v in res["Value"]]
gLogger.debug("File List", file_list)
# Begin to save file
# Refer to dirac-dms-get-file.py in DIRAC/Interfaces/scripts
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
res = dirac.getFile( file_list, destDir = dir_save, printOutput = True )
if not res["OK"]:
gLogger.error(res)
DIRAC.exit(-1)
DIRAC.exit(0)
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... LFN ...' % Script.scriptName, 'Arguments:',
' LFN: Logical File Name or file containing LFNs'
]))
Script.parseCommandLine(ignoreErrors=True)
lfns = Script.getPositionalArgs()
if len(lfns) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
if len(lfns) == 1:
try:
f = open(lfns[0], 'r')
lfns = f.read().splitlines()
f.close()
except:
pass
result = dirac.getFile(lfns, printOutput=True)
if not result['OK']:
print('ERROR %s' % (result['Message']))
exitCode = 2
DIRAC.exit(exitCode)
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ... LFN ...' % Script.scriptName,
'Arguments:',
' LFN: Logical File Name or file containing LFNs' ] ) )
Script.parseCommandLine( ignoreErrors = True )
lfns = Script.getPositionalArgs()
if len( lfns ) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
if len( lfns ) == 1:
try:
f = open( lfns[0], 'r' )
lfns = f.read().splitlines()
f.close()
except:
pass
result = dirac.getFile( lfns, printOutput = True )
if not result['OK']:
print 'ERROR %s' % ( result['Message'] )
exitCode = 2
DIRAC.exit( exitCode )
leave('Could not determine SEs for replicas of %s' % oldLFN, exitCode=2)
gLogger.info('Existing LFN has replicas at: %s' % ', '.join(storageElements))
oldGUID = dirac.getLfnMetadata(oldLFN)
if not oldGUID['OK'] or oldGUID['Value']['Failed']:
leave('Could not obtain GUID from LFC for %s - %s' % oldLFN,
oldGUID,
exitCode=2)
oldGUID = oldGUID['Value']['Successful'][oldLFN]['GUID']
gLogger.verbose('Existing GUID is %s' % oldGUID)
# retrieve original file
localFile = os.path.join(directory, os.path.basename(oldLFN))
if not os.path.exists(localFile):
download = dirac.getFile(oldLFN, directory)
if not download['OK'] or download['Value']['Failed']:
leave('Could not download file with message - %s' %
download['Message'],
download,
exitCode=2)
else:
gLogger.always('Found file %s in local directory, will not re-download' %
os.path.basename(oldLFN))
newGUID = makeGuid(localFile)[localFile]
if newGUID == oldGUID:
leave(
'Old and new GUIDs have the same value (%s), exiting without changes' %
oldGUID)
def _downloadSingleFile(self, remotePath, localPath):
gLogger.debug('getfile from %s to %s' % (remotePath, localPath))
dirac = Dirac()
result = dirac.getFile(remotePath, os.path.dirname(localPath))
return result['OK']
result = subprocess.check_output(batcmd, shell=True)
file_list = result.split()[-1]
else:
file_list = args.file_list
# try reading the lfns file
try:
GRID_file_list = open(file_list).read()
except IOError:
raise IOError("cannot read lfns file list...")
file_collection = []
for line in GRID_file_list:
line = line.strip()
if args.match not in line:
continue
# don't download if already in current directory
if glob(basename(line)):
continue
if len(file_collection) < 100:
file_collection.append(line)
else:
dirac.getFile(file_collection, destDir=args.outdir)
file_collection = []
if file_collection:
dirac.getFile(file_collection, destDir=args.outdir)
#!/usr/bin/env python2
from os.path import basename, expandvars
from glob import glob
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
file_collection = []
for line in open(expandvars('$CTA_SOFT/tino_cta/vo.cta.in2p3.fr-user-t-tmichael.lfns')):
line = line.strip()
if "prod3b/paranal_LND/classified_events_proton_tail" not in line:
continue
# don't download if already in current directory
if glob(basename(line)):
continue
if len(file_collection) < 100:
file_collection.append(line)
else:
dirac.getFile(file_collection)
file_collection = []
if file_collection:
dirac.getFile(file_collection)
def _downloadSingleFile(self, remotePath, localPath):
gLogger.debug('getfile from %s to %s' % (remotePath, localPath))
dirac = Dirac()
result = dirac.getFile(remotePath, os.path.dirname(localPath))
return result['OK']
import os
## NB parseCommandLine first then import Dirac!!
from DIRAC.Core.Base.Script import parseCommandLine
from DIRAC.Interfaces.API.Dirac import Dirac
parseCommandLine()
dirac=Dirac()
dirac.getFile('###LFN###', os.getcwd())
file_list = result.split()[-1]
else:
file_list = args.file_list
# try reading the lfns file
try:
GRID_file_list = open(file_list).read()
except IOError:
raise IOError("cannot read lfns file list...")
file_collection = []
for line in GRID_file_list:
line = line.strip()
if args.match not in line:
continue
# don't download if already in current directory
if glob(basename(line)):
continue
if len(file_collection) < 100:
file_collection.append(line)
else:
dirac.getFile(file_collection, destDir=args.outdir)
file_collection = []
if file_collection:
dirac.getFile(file_collection, destDir=args.outdir)