def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(["LFN: Logical File Name or file containing LFNs"]) Script.parseCommandLine(ignoreErrors=True) lfns = Script.getPositionalArgs() if len(lfns) < 1: Script.showHelp() from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() exitCode = 0 if len(lfns) == 1: try: with open(lfns[0], "r") as f: lfns = f.read().splitlines() except Exception: pass result = dirac.getFile(lfns, printOutput=True) if not result["OK"]: print("ERROR %s" % (result["Message"])) exitCode = 2 DIRAC.exit(exitCode)
def getfile(lfn): dirac = Dirac() res = dirac.getFile( lfn ) if not res['OK']: print 'Error downloading lfn: ' + lfn return res['Message']
def do_get(self,args): """ Download file from grid and store in a local directory usage: get <lfn> [<local_directory>] """ argss = args.split() lfn = argss[0] lfn = self.getPath(lfn) dir = '' if len(argss)>1: dir = argss[1] dirac = Dirac() localCWD = '' if dir: localCWD = os.getcwd() os.chdir(dir) result = dirac.getFile(lfn) if localCWD: os.chdir(localCWD) if not result['OK']: print 'Error: %s' %(result['Message']) else: print "File %s successfully downloaded" % lfn
def main(): lfn = '/cepc/lustre-ro/' + filepath dirac = Dirac() result = dirac.getFile(lfn) if not result['OK']: gLogger.error('Download file error: %s' % result['Message']) return 2 return 0
def getfile(lfn): dirac = Dirac() print('Start downloading ' + lfn + '\n', end='') res = dirac.getFile(lfn, destDir=TEMPDIR) if not res['OK']: print('Error downloading lfn:' + lfn + '\n', end='') return res['Message'] name = os.path.basename(lfn) os.rename(os.path.join('.incomplete', name), name) print('Successfully downloaded file:' + lfn + '\n', end='')
def main(): # Read command line options parser = argparse.ArgumentParser(description="Download collection files from Dirac") parser.add_argument("--indir", default=None, help="Dirac repository") parser.add_argument("--outdir", default="", help="Output file directory") args = parser.parse_args() print("Download file from: {}".format(args.indir)) print("Output directory: {}".format(args.outdir)) # Create output directory if not os.path.exists(args.outdir): os.makedirs(args.outdir) # Get list of files batcmd = "dirac-dms-user-lfns --BaseDir {}".format(args.indir) result = subprocess.check_output(batcmd, shell=True) file_list = result.split()[-1] # try reading the lfns file try: grid_file_list = open(file_list) except IOError: raise IOError("cannot read lfns file list...") dirac = Dirac() file_collection = [] for line in grid_file_list.read().splitlines(): if len(file_collection) < 100: file_collection.append(line) # files will be downloaded later else: dirac.getFile(file_collection, destDir=args.outdir) file_collection = [] # there won't be any loop at the end if file_collection: dirac.getFile(file_collection, destDir=args.outdir)
def downloadFilesByDatasetName(self, dataset_name): """downLoad a set of files form SE. use getFilesByDatasetName() get a list of lfns and download these files. Example usage: >>>badger.downloadFilesByDatasetName('psipp_661_data_all_exp2')i """ dirac = Dirac() fileList = self.getFilesByDatasetName(dataset_name) result = dirac.getFile(fileList, printOutput=True) if not result['OK']: print 'ERROR %s' % (result['Message']) return S_ERROR(result['Message']) else: return S_OK()
def downloadFilesByFilelist(self, fileList, destDir=""): """downLoad a set of files form SE. use getFilesByFilelist() get a list of lfns and download these files. fileList get from function getFilesByDatesetName() Example usage: >>>badger.downloadFilesByFilelist(fileList) """ errorDict = {} dirac = Dirac() # fileList = self.getFilesByDatasetName(dataset_name) for lfn in fileList: result = dirac.getFile(lfn, destDir, printOutput=False) if not result["OK"]: errorDict[lfn] = result["Message"] if errorDict: serr = S_ERROR() serr["errorDict"] = errorDict return serr else: return S_OK("File download successfully.")
def downloadFilesByFilelist(self, fileList, destDir=''): """downLoad a set of files form SE. use getFilesByFilelist() get a list of lfns and download these files. fileList get from function getFilesByDatesetName() Example usage: >>>badger.downloadFilesByFilelist(fileList) """ errorDict = {} dirac = Dirac() #fileList = self.getFilesByDatasetName(dataset_name) for lfn in fileList: result = dirac.getFile(lfn, destDir, printOutput=False) if not result['OK']: errorDict[lfn] = result['Message'] if errorDict: serr = S_ERROR() serr["errorDict"] = errorDict return serr else: return S_OK("File download successfully.")
def main(): input_file_jobvar_name = 'JSUB_' + os.environ.get( 'JSUB_input_file_jobvar_name', 'input_file' ) # if input file is local, and put to DFC through jsub register. input_lfn_jobvar_name = 'JSUB_' + os.environ.get( 'JSUB_input_lfn_jobvar_name', 'input_lfn') # if input file is directly from DFC source_lfn_prefix = os.environ.get('JSUB_source_lfn_prefix') input_path = os.environ.get(input_file_jobvar_name) input_lfn = os.environ.get(input_lfn_jobvar_name) destination = os.environ.get('JSUB_destination', './') if input_lfn is None: if source_lfn_prefix: lfn = os.path.join(source_lfn_prefix, input_path) else: lfn = input_path_to_lfn(input_path) else: lfn = input_lfn fname = os.path.basename(lfn) # gfal_prefix = 'srm://storm.ihep.ac.cn:8444' # gfal_path = gfal_prefix + input_path # dest_dir = os.environ.get('destination_dir','./') # dest_filename = dest_dir + os.path.basename(input_path) # os.system('gfal-copy {} {}'.format(gfal_path,dest_filename)) #replace the input file # download the file to the action folder dirac = Dirac() result = dirac.getFile(lfn) if not result['OK']: gLogger.error('Download file error: %s' % result['Message']) return 2 # mv to destination os.system('mv %s %s' % (fname, destination)) return 0
def main(): Script.parseCommandLine(ignoreErrors=True) lfns = Script.getPositionalArgs() if len(lfns) < 1: Script.showHelp() from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() exitCode = 0 if len(lfns) == 1: try: with open(lfns[0], 'r') as f: lfns = f.read().splitlines() except Exception: pass result = dirac.getFile(lfns, printOutput=True) if not result['OK']: print('ERROR %s' % (result['Message'])) exitCode = 2 DIRAC.exit(exitCode)
if k.lower() in ["save"]: dir_save = v gLogger.info("Dataset Name: ", dataset) gLogger.info("Save in: ", dir_save) # Get the list of LFNs in one dataset from DIRAC.Core.DISET.RPCClient import RPCClient transferRequest = RPCClient("Transfer/Dataset") res = transferRequest.list(dataset) if not res["OK"]: gLogger.error(res) DIRAC.exit(-1) file_list = [v[1] for v in res["Value"]] gLogger.debug("File List", file_list) # Begin to save file # Refer to dirac-dms-get-file.py in DIRAC/Interfaces/scripts from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() res = dirac.getFile( file_list, destDir = dir_save, printOutput = True ) if not res["OK"]: gLogger.error(res) DIRAC.exit(-1) DIRAC.exit(0)
Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... LFN ...' % Script.scriptName, 'Arguments:', ' LFN: Logical File Name or file containing LFNs' ])) Script.parseCommandLine(ignoreErrors=True) lfns = Script.getPositionalArgs() if len(lfns) < 1: Script.showHelp() from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() exitCode = 0 if len(lfns) == 1: try: f = open(lfns[0], 'r') lfns = f.read().splitlines() f.close() except: pass result = dirac.getFile(lfns, printOutput=True) if not result['OK']: print('ERROR %s' % (result['Message'])) exitCode = 2 DIRAC.exit(exitCode)
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1], 'Usage:', ' %s [option|cfgfile] ... LFN ...' % Script.scriptName, 'Arguments:', ' LFN: Logical File Name or file containing LFNs' ] ) ) Script.parseCommandLine( ignoreErrors = True ) lfns = Script.getPositionalArgs() if len( lfns ) < 1: Script.showHelp() from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() exitCode = 0 if len( lfns ) == 1: try: f = open( lfns[0], 'r' ) lfns = f.read().splitlines() f.close() except: pass result = dirac.getFile( lfns, printOutput = True ) if not result['OK']: print 'ERROR %s' % ( result['Message'] ) exitCode = 2 DIRAC.exit( exitCode )
leave('Could not determine SEs for replicas of %s' % oldLFN, exitCode=2) gLogger.info('Existing LFN has replicas at: %s' % ', '.join(storageElements)) oldGUID = dirac.getLfnMetadata(oldLFN) if not oldGUID['OK'] or oldGUID['Value']['Failed']: leave('Could not obtain GUID from LFC for %s - %s' % oldLFN, oldGUID, exitCode=2) oldGUID = oldGUID['Value']['Successful'][oldLFN]['GUID'] gLogger.verbose('Existing GUID is %s' % oldGUID) # retrieve original file localFile = os.path.join(directory, os.path.basename(oldLFN)) if not os.path.exists(localFile): download = dirac.getFile(oldLFN, directory) if not download['OK'] or download['Value']['Failed']: leave('Could not download file with message - %s' % download['Message'], download, exitCode=2) else: gLogger.always('Found file %s in local directory, will not re-download' % os.path.basename(oldLFN)) newGUID = makeGuid(localFile)[localFile] if newGUID == oldGUID: leave( 'Old and new GUIDs have the same value (%s), exiting without changes' % oldGUID)
def _downloadSingleFile(self, remotePath, localPath): gLogger.debug('getfile from %s to %s' % (remotePath, localPath)) dirac = Dirac() result = dirac.getFile(remotePath, os.path.dirname(localPath)) return result['OK']
result = subprocess.check_output(batcmd, shell=True) file_list = result.split()[-1] else: file_list = args.file_list # try reading the lfns file try: GRID_file_list = open(file_list).read() except IOError: raise IOError("cannot read lfns file list...") file_collection = [] for line in GRID_file_list: line = line.strip() if args.match not in line: continue # don't download if already in current directory if glob(basename(line)): continue if len(file_collection) < 100: file_collection.append(line) else: dirac.getFile(file_collection, destDir=args.outdir) file_collection = [] if file_collection: dirac.getFile(file_collection, destDir=args.outdir)
#!/usr/bin/env python2 from os.path import basename, expandvars from glob import glob from DIRAC.Core.Base import Script Script.parseCommandLine() from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() file_collection = [] for line in open(expandvars('$CTA_SOFT/tino_cta/vo.cta.in2p3.fr-user-t-tmichael.lfns')): line = line.strip() if "prod3b/paranal_LND/classified_events_proton_tail" not in line: continue # don't download if already in current directory if glob(basename(line)): continue if len(file_collection) < 100: file_collection.append(line) else: dirac.getFile(file_collection) file_collection = [] if file_collection: dirac.getFile(file_collection)
import os ## NB parseCommandLine first then import Dirac!! from DIRAC.Core.Base.Script import parseCommandLine from DIRAC.Interfaces.API.Dirac import Dirac parseCommandLine() dirac=Dirac() dirac.getFile('###LFN###', os.getcwd())
file_list = result.split()[-1] else: file_list = args.file_list # try reading the lfns file try: GRID_file_list = open(file_list).read() except IOError: raise IOError("cannot read lfns file list...") file_collection = [] for line in GRID_file_list: line = line.strip() if args.match not in line: continue # don't download if already in current directory if glob(basename(line)): continue if len(file_collection) < 100: file_collection.append(line) else: dirac.getFile(file_collection, destDir=args.outdir) file_collection = [] if file_collection: dirac.getFile(file_collection, destDir=args.outdir)