def __set_log_level(level): level = max(0, level) level = min(3, level) log_level_value = logging.ERROR - (level * 10) if level < 3: gfal2.set_verbose(gfal2.verbose_level.verbose) else: gfal2.set_verbose(gfal2.verbose_level.debug) root_logger = logging.getLogger() root_logger.setLevel(log_level_value) handler = logging.StreamHandler(sys.stderr) handler.setLevel(log_level_value) handler.setFormatter(logging.Formatter('%(levelname)s %(message)s')) if sys.stderr.isatty(): logging.addLevelName( logging.DEBUG, "\033[1;2m%-8s\033[1;m" % logging.getLevelName(logging.DEBUG)) logging.addLevelName( logging.INFO, "\033[1;34m%-8s\033[1;m" % logging.getLevelName(logging.INFO)) logging.addLevelName( logging.ERROR, "\033[1;31m%-8s\033[1;m" % logging.getLevelName(logging.ERROR)) logging.addLevelName( logging.WARNING, "\033[1;33m%-8s\033[1;m" % logging.getLevelName(logging.WARNING)) root_logger.addHandler(handler)
def connect(self): """ Establishes the actual connection to the referred RSE. If we decide to use gfal, init should be done here. :raises RSEAccessDenied """ self.logger(logging.DEBUG, 'connecting to storage') gfal2.set_verbose(gfal2.verbose_level.verbose) self.__ctx = gfal2.creat_context() # pylint: disable=no-member self.__ctx.set_opt_string_list( "SRM PLUGIN", "TURL_PROTOCOLS", ["gsiftp", "rfio", "gsidcap", "dcap", "kdcap"]) self.__ctx.set_opt_string("XROOTD PLUGIN", "XRD.WANTPROT", "gsi,unix") self.__ctx.set_opt_boolean("XROOTD PLUGIN", "NORMALIZE_PATH", False) auth_configured = False if self.auth_token: self.__ctx.set_opt_string("BEARER", "TOKEN", self.auth_token) auth_configured = True # Configure gfal authentication to use the rucio client proxy if and only if gfal didn't initialize its credentials already # (https://gitlab.cern.ch/dmc/gfal2/-/blob/48cfe3476392c884b53d00799198b1238603a406/src/core/common/gfal_common.c#L79) if not auth_configured: try: self.__ctx.get_opt_string("X509", "CERT") self.__ctx.get_opt_string("X509", "KEY") auth_configured = True except gfal2.GError: # pylint: disable=no-member pass if not auth_configured: try: self.__ctx.get_opt_string("BEARER", "TOKEN") auth_configured = True except gfal2.GError: # pylint: disable=no-member pass if not auth_configured: proxy = config.config_get('client', 'client_x509_proxy', default=None, raise_exception=False) if proxy: self.logger( logging.INFO, 'Configuring authentication to use {}'.format(proxy)) self.__ctx.set_opt_string("X509", "CERT", proxy) self.__ctx.set_opt_string("X509", "KEY", proxy) if TIMEOUT: try: timeout = int(TIMEOUT) self.__ctx.set_opt_integer("HTTP PLUGIN", "OPERATION_TIMEOUT", timeout) self.__ctx.set_opt_integer("SRM PLUGIN", "OPERATION_TIMEOUT", timeout) self.__ctx.set_opt_integer("GRIDFTP PLUGIN", "OPERATION_TIMEOUT", timeout) except ValueError: self.logger(logging.ERROR, 'wrong timeout value %s', TIMEOUT)
def test_creat_and copy(self): gfal2.set_verbose(gfal2.verbose_level.debug) c = gfal2.creat_context() c.timeout=200 self.assertTrue(t.timeout == 200 ) d = c.copy() self.assertTrue(d.timeout == 200 ) c.timeout = 20 del c self.assertTrue(d.timeout == 200 )
def createGfal2Context(logLevel="normal", emulate=False): """ Create a gfal2 context object :param logLevel: string with the gfal2 log level :param emulate: boolean to be used by unit tests :return: the gfal2 context object """ if emulate: return None ctx = gfal2.creat_context() gfal2.set_verbose(gfal2.verbose_level.names[logLevel]) return ctx
def test_creat_and_delete_params(self): gfal2.set_verbose(gfal2.verbose_level.debug) context = gfal2.creat_context() t = context.transfer_params() self.assertTrue(t.timeout > 0 ) t.timeout = 10 self.assertTrue(t.timeout == 10 ) self.assertTrue(self.check_checksum == False) t.check_checksum = True self.assertTrue(self.check_checksum == True) self.assertTrue(t.src_spacetoken == "") t.src_spacetoken = "TOKENDTEAM" self.assertTrue(t.src_spacetoken == "TOKENDTEAM")
def get_space_usage(self): """ Get RSE space usage information. :returns: a list with dict containing 'totalsize' and 'unusedsize' :raises ServiceUnavailable: if some generic error occured in the library. """ rse_name = self.rse['rse'] dest = '/tmp/rucio-gsiftp-site-size_' + rse_name space_usage_url = '' # url of space usage json, woud be nicer to have it in rse_settings agis = requests.get( 'http://atlas-agis-api.cern.ch/request/ddmendpoint/query/list/?json' ).json() agis_token = '' for res in agis: if rse_name == res['name']: agis_token = res['token'] space_usage_url = res['space_usage_url'] import gfal2 # pylint: disable=import-error gfal2.set_verbose(gfal2.verbose_level.normal) # pylint: disable=no-member try: if os.path.exists(dest): os.remove(dest) ctx = gfal2.creat_context() # pylint: disable=no-member ctx.set_opt_string_list( "SRM PLUGIN", "TURL_PROTOCOLS", ["gsiftp", "rfio", "gsidcap", "dcap", "kdcap"]) params = ctx.transfer_parameters() params.timeout = 3600 ret = ctx.filecopy(params, str(space_usage_url), str('file://' + dest)) if ret == 0: data_file = open(dest) data = json.load(data_file) data_file.close() if agis_token not in list(data.keys()): print('ERROR: space usage json has different token as key') else: totalsize = int(data[agis_token]['total_space']) used = int(data[agis_token]['used_space']) unusedsize = totalsize - used return totalsize, unusedsize except Exception as error: print(error) raise exception.ServiceUnavailable(error)
def connect(self): """ Establishes the actual connection to the referred RSE. If we decide to use gfal, init should be done here. :raises RSEAccessDenied """ self.logger.debug('connecting') gfal2.set_verbose(gfal2.verbose_level.verbose) self.__ctx = gfal2.creat_context() # pylint: disable=no-member self.__ctx.set_opt_string_list("SRM PLUGIN", "TURL_PROTOCOLS", ["gsiftp", "rfio", "gsidcap", "dcap", "kdcap"]) self.__ctx.set_opt_string("XROOTD PLUGIN", "XRD.WANTPROT", "gsi,unix") self.__ctx.set_opt_boolean("XROOTD PLUGIN", "NORMALIZE_PATH", False) if self.auth_token: self.__ctx.set_opt_string("BEARER", "TOKEN", self.auth_token)
def __setup_logger(level, log_file): # Handle logging level level = max(0, level) level = min(3, level) log_level_value = logging.ERROR - (level * 10) if level < 3: gfal2.set_verbose(gfal2.verbose_level.verbose) else: gfal2.set_verbose(gfal2.verbose_level.debug) # Handle log file log_stream = sys.stdout if log_file: log_stream = open(log_file, 'w+') root_logger = logging.getLogger() root_logger.setLevel(log_level_value) handler = logging.StreamHandler(log_stream) handler.setLevel(log_level_value) handler.setFormatter(logging.Formatter('%(levelname)s %(message)s')) if log_stream.isatty(): logging.addLevelName( logging.DEBUG, "\033[1;2m%-8s\033[1;m" % logging.getLevelName(logging.DEBUG)) logging.addLevelName( logging.INFO, "\033[1;34m%-8s\033[1;m" % logging.getLevelName(logging.INFO)) logging.addLevelName( logging.ERROR, "\033[1;31m%-8s\033[1;m" % logging.getLevelName(logging.ERROR)) logging.addLevelName( logging.WARNING, "\033[1;33m%-8s\033[1;m" % logging.getLevelName(logging.WARNING)) root_logger.addHandler(handler)
rses_lists = [] for single_rse in default_rses: rses_lists.append(single_rse['rse']) print(rses_lists) # Gfal settings gfal = Gfal2Context() print(json.dumps(client.whoami(), indent=4, sort_keys=True)) print(json.dumps(client.ping(), indent=4, sort_keys=True)) sys.path.append("/usr/lib64/python3.6/site-packages/") import gfal2 from gfal2 import Gfal2Context, GError gfal2.set_verbose(gfal2.verbose_level.debug) gfal = Gfal2Context() # In[3]: # Specific Functions for Rucio ## Get files from base mouting point of RSE defined on DEFAULT_PATH def check_directory(path): try: full_path = gfal.listdir(str(path)) is_dir_or_not = True except:
def gfal_exec(method, args, nonerrors = {}, return_value = False): """ GFAL2 execution function @param method Name of the Gfal2Context method to execute. @param args Tuple of arguments to pass to the method @param nonerrors Dictionary of error code translation for non-errors. @param return_value If True, simply return the return value of the function. @return (exit code, start time, finish time, error message, log string) """ start_time = None finish_time = None log = '' for attempt in xrange(5): # gfal2 knows to write to the logger. Redirect to StringIO and dump the full log at the end. stream = cStringIO.StringIO() LOG.handlers.pop() handler = logging.StreamHandler(stream) handler.setFormatter(logging.Formatter(fmt = '%(asctime)s: %(message)s')) LOG.addHandler(handler) start_time = int(time.time()) try: gfal2.set_verbose(gfal2.verbose_level.verbose) context = gfal2.creat_context() result = getattr(gfal2.Gfal2Context, method)(context, *args) finish_time = int(time.time()) except gfal2.GError as err: if return_value: raise exitcode, msg = err.code, str(err) c = find_msg_code(msg) if c is not None: exitcode = c if exitcode in nonerrors: return 0, start_time, int(time.time()), nonerrors[exitcode], '' elif exitcode in irrecoverable_errors: break except Exception as exc: if return_value: raise exitcode, msg = -1, str(exc) else: exitcode, msg = 0, None finally: handler.flush() log_tmp = stream.getvalue().strip() # give a nice indent to each line log = ''.join(' %s\n' % line for line in log_tmp.split('\n')) stream.close() break if return_value: return result else: # all variables would be defined even when all attempts are exhausted return exitcode, start_time, finish_time, msg, log
#!/usr/bin/env python import optparse import sys import gfal2 gfal2.set_verbose(gfal2.verbose_level.debug) def event_callback(event): print("[%s] %s %s %s" % (event.timestamp, event.domain, event.stage, event.description)) def monitor_callback(src, dst, average, instant, transferred, elapsed): print("[%4d] %.2fMB (%.2fKB/s)\r" % (elapsed, transferred / 1048576, average / 1024)) sys.stdout.flush() if __name__ == '__main__': # Parse arguments parser = optparse.OptionParser(usage=('usage: %prog [options] ' 'source destination')) parser.add_option('-s', '--source-space-token', dest='source_space_token',
def test_creat_and_delete_instance(self): gfal2.set_verbose(gfal2.verbose_level.debug) context = gfal2.creat_context() del context
import sys from pprint import pprint import gfal2 gfal2.set_verbose(gfal2.verbose_level.warning) # from dirent.h DT_DIR = 4 # gfal2.Gfal2Context.Dirent d_type directory DT_REG = 8 # gfal2.Gfal2Context.Dirent d_type file class ProgressMeter(object): def __init__(self, update_interval=10): self._update_interval = update_interval self._last_update = 0 def __call__(self, src, dst, average, instant, transferred, elapsed): if elapsed - self._last_update > self._update_interval: sys.stdout.write("[%4d] %.2fMB (%.2fKB/s)\n" % (elapsed, transferred / 1048576, average / 1024)) sys.stdout.flush() self._last_update = elapsed def ls(url, dt_filter=None): ctx = gfal2.creat_context() dirp = ctx.opendir(url) ret = [] while True: dirent,stat = dirp.readpp() if dirent is None: break if dirent.d_name in ('.', '..'): continue
default="ops") app.add_argument("--srmv", help="srm version to use", default='2') app.add_argument("--ldap-url", help="LDAP URL", dest="ldap_url", default="ldap://lcg-bdii.cern.ch:2170") app.add_argument("--se-timeout", dest="se_timeout", type=int, help="storage operations timeout", default=60) # Reasonable defaults for timeouts LCG_GFAL_BDII_TIMEOUT = 10 gfal2.set_verbose(gfal2.verbose_level.normal) # Service version(s) svcVers = ['1', '2'] svcVer = '2' workdir_metric = tempfile.mkdtemp() # files and patterns _fileTest = workdir_metric + '/testFile.txt' _fileTestIn = workdir_metric + '/testFileIn.txt' _fileSRMPattern = 'testfile-put-%s-%s.txt' # time, uuid _voInfoDictionary = {} # Instantiate gfal2 ctx = gfal2.creat_context()
# should have the '/pnfs/...' format. Script output: # # ONLINE: means that the file is only on disk # # NEARLINE: means that the file in only on tape # # ONLINE_AND_NEARLINE: means that the file is on disk # # and tape # # ===================================================================== # """ from __future__ import print_function import sys from collections import Counter try: import gfal2 as gfal # pylint: disable=import-error gfal.set_verbose(gfal.verbose_level.warning) except ImportError: print("GFAL CANNOT BE IMPORTED") from GRID_LRT.Staging.srmlist import srmlist from GRID_LRT.auth import grid_credentials def main(filename, verbose=True): """Main function that takes in a file name and returns a list of tuples of filenames and staging statuses. The input file can be both srm:// and gsiftp:// links. Args: :param filename: The filename holding the links whose have to be checked :type filename: str :param verbose: A toggle to turn off printing out the status of each file.
def set_verbose_mode(params): # print params modes = [ gfal2.verbose_level.normal, gfal2.verbose_level.verbose, gfal2.verbose_level.debug, gfal2.verbose_level.trace ] gfal2.set_verbose(modes[params.verbose])