def getOutputData(baseDir, logLevel="INFO"): gLogger.setLevel(logLevel) exitCode = 0 res = getProxyInfo(False, False) if not res["OK"]: gLogger.error("Failed to get client proxy information.", res["Message"]) DIRAC.exit(71) print "Will search for files in %s" % baseDir activeDirs = [baseDir] # ######################################################################################################## # # before is from dirac-dms-user-lfns rm = DataManager() allFiles = [] while len(activeDirs) > 0: currentDir = activeDirs[0] res = rm.getFilesFromDirectory(currentDir) activeDirs.remove(currentDir) if not res["OK"]: gLogger.error("Error retrieving directory contents", "%s %s" % (currentDir, res["Message"])) else: allFiles = res["Value"] # ######################################################################################################## # # get file ntries = 5 # getFile supports bulk requests files_to_transfer = sortList(allFiles) successful_files = [] failed_files = [] while ntries > 0: if len(failed_files): files_to_transfer = failed_files gLogger.info("getting the following *list* of files %s" % str(files_to_transfer)) result = rm.getFile(files_to_transfer) if not result["OK"]: gLogger.error("Could not complete DataManager request") gLogger.error(str(result["Message"])) gLogger.info("sleep for 10s and re-try") time.sleep(10) break # next is to check what files we got successful_files = result["Value"]["Successful"].keys() failed_files = result["Value"]["Failed"].keys() if len(failed_files): gLogger.info("Could not retrieve one or more files") for key in failed_files: gLogger.error("%s:%s" % (key, result["Value"]["Failed"][key])) for s in successful_files: files_to_transfer.remove(s) for f in failed_files: gLogger.verbose("could not retrieve: %s" % f) else: break ntries -= 1 if len(failed_files): gLogger.error("ERROR could not get all files after %i trials. Giving up :(" % ntries) exitCode = 23 if exitCode: return {"OK": False, "Message": "Failed to finish operations.", "RC": exitCode} return S_OK(successful_files)
class SoftwareManager(object): """ Manage software setup """ def __init__(self, soft_category): """ Constructor """ self.CVMFS_DIR = '/cvmfs/sw.cta-observatory.org/software' self.LFN_ROOT = '/vo.cta.in2p3.fr/software' self.SOFT_CATEGORY_DICT = soft_category self.dm = DataManager() def _search_software(self, package, version, compiler, use_cvmfs): ''' Look for sotfware package ''' # software package category category = self.SOFT_CATEGORY_DICT[package] # look for software on cvmfs if use_cvmfs: package_dir = os.path.join(self.CVMFS_DIR, 'centos7', compiler, category, package, version) if os.path.isdir(package_dir): DIRAC.gLogger.notice('Found package %s version %s at:\n%s' % (package, version, package_dir)) return DIRAC.S_OK({'Source':'cvmfs', 'Path':package_dir}) else: DIRAC.gLogger.warn('%s\n not found on cvmfs'%package_dir) # look for tarball in the Dirac file catalog else: package_dir = os.path.join(self.LFN_ROOT, 'centos7', compiler, category, package, version) DIRAC.gLogger.notice('Looking for tarball in %s'%package_dir) results = self.dm.getFilesFromDirectory(package_dir) try: first_file_path = results['Value'][0] if first_file_path[-7:] == '.tar.gz': results = self.dm.getActiveReplicas(first_file_path) if results['OK']: return DIRAC.S_OK({'Source':'tarball', 'Path':package_dir}) except: DIRAC.gLogger.warn('No usual tarball found in the directory') return DIRAC.S_ERROR('Could not find package %s / %s / %s in any location' % (package, version, compiler)) def find_software(self, package, version, compiler='gcc48_default'): """ check if the software package is installed in any software area Keyword arguments: package -- package name as the directory name version -- software version as the directory name compiler -- compiler version and configuration """ # first check if cvmfs is available ops_helper = Operations() use_cvmfs = ops_helper.getValue('SoftwarePolicy/UseCvmfs', bool) DIRAC.gLogger.notice('SoftwarePolicy for UseCvmfs is:', use_cvmfs) # get platform and cpu information try: os_name, cpu_name, inst = get_os_and_cpu_info() DIRAC.gLogger.notice('Running %s on a %s ' %(os_name, cpu_name)) except: inst = 'sse4' DIRAC.gLogger.warn('Could not determine platform and cpu information') if compiler == 'gcc48_default': results = self._search_software(package, version, compiler, use_cvmfs) return results elif compiler == 'gcc48_sse4': # assume all processors have at least sse4 results = self._search_software(package, version, compiler, use_cvmfs) return results elif compiler == 'gcc48_avx': if inst in ['avx', 'avx2', 'avx512']: results = self._search_software(package, version, compiler, use_cvmfs) return results else: DIRAC.gLogger.warn('CPU has no avx instructions, running sse4 version') compiler = 'gcc48_sse4' results = self._search_software(package, version, compiler, use_cvmfs) return results elif compiler == 'gcc48_avx2': if inst in ['avx2', 'avx512']: results = self._search_software(package, version, compiler, use_cvmfs) return results else: DIRAC.gLogger.warn('CPU has no avx2 instructions, running sse4 version') compiler = 'gcc48_sse4' results = self._search_software(package, version, compiler, use_cvmfs) return results elif compiler == 'gcc48_avx512': if inst is 'avx512': results = self._search_software(package, version, compiler, use_cvmfs) return results else: DIRAC.gLogger.warn('CPU has no avx512 instructions, running sse4 version') compiler = 'gcc48_sse4' results = self._search_software(package, version, compiler, use_cvmfs) return results elif compiler == 'gcc48_matchcpu': compiler = 'gcc48_%s'%inst results = self._search_software(package, version, compiler, use_cvmfs) return results else: DIRAC.S_ERROR('Unknown compiler specified: %s'%compiler) return DIRAC.S_ERROR('Could not find package %s version %s / %s in any location' % (package, version, compiler)) def install_dirac_scripts(self, package_dir): """ copy DIRAC scripts in the current directory """ dirac_scripts = glob.glob(os.path.join(package_dir, 'dirac_*')) try: for one_file in dirac_scripts: shutil.copy2(one_file, '.') return DIRAC.S_OK() except shutil.Error as error: return DIRAC.S_ERROR('Failed to install DIRAC scripts:\n%s'%error) def dump_setup_script_path(self, package_dir, textfilename = 'setup_script_path.txt'): """ dump the path to setupPackage.sh in a one line ascii file to be read and source by the following script """ script_path = os.path.join(package_dir, 'setupPackage.sh') open(textfilename, 'w').writelines(script_path + '\n') return DIRAC.S_OK() def install_software(self, tar_lfn, target_dir='.'): """ install software package in the current directory """ DIRAC.gLogger.notice('Installing package at %s'%tar_lfn) # Download the tar file DIRAC.gLogger.notice('Trying to download package:', tar_lfn) res = self.dm.getFile(tar_lfn) if not res['OK']: return res if tar_lfn in res['Value']['Successful']: DIRAC.gLogger.notice(' Package downloaded successfully:', tar_lfn) else: error = 'Failed to download package:', tar_lfn return DIRAC.S_ERROR(error) # Extract the tar file to the target directory tar_mode = "r|*" tar = tarfile.open(tar_lfn, tar_mode) for tarInfo in tar: tar.extract(tarInfo, target_dir) tar.close() os.unlink(tar_lfn) # Done DIRAC.gLogger.notice('Package %s installed successfully at:\n%s' %(tar_lfn, target_dir)) return DIRAC.S_OK(target_dir)
#log gLogger.notice('Checking with DFC folder: {0}\nFor SE(s): {1}'.format( dfcDir, SEs)) if fixDiscrepancy: gLogger.notice( 'Would fix the discrepancy by replicating files from other SEs.') gLogger.notice("\n\n") from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient fcc = FileCatalogClient('DataManagement/FileCatalog') from DIRAC.DataManagementSystem.Client.DataManager import DataManager dm = DataManager() #get all files in dfcDir gf_result = dm.getFilesFromDirectory(dfcDir) if not gf_result['OK']: gLogger.error('Failed to list directory %s: %s' % (dfcDir, gf_result['Message'])) exit() files = gf_result['Value'] fileNumber = len(files) gLogger.notice('There are {0} files in dir {1}'.format(fileNumber, dfcDir)) #get replicas in the SE gr_result = dm.getReplicasFromDirectory(dfcDir) if not gr_result['OK']: gLogger.error('Failed to get replicas from directory %s: %s' % (dfcDir, gr_result['Message'])) exit()