def __set_log_level(level):
level = max(0, level)
level = min(3, level)
log_level_value = logging.ERROR - (level * 10)
if level < 3:
gfal2.set_verbose(gfal2.verbose_level.verbose)
else:
gfal2.set_verbose(gfal2.verbose_level.debug)
root_logger = logging.getLogger()
root_logger.setLevel(log_level_value)
handler = logging.StreamHandler(sys.stderr)
handler.setLevel(log_level_value)
handler.setFormatter(logging.Formatter('%(levelname)s %(message)s'))
if sys.stderr.isatty():
logging.addLevelName(
logging.DEBUG,
"\033[1;2m%-8s\033[1;m" % logging.getLevelName(logging.DEBUG))
logging.addLevelName(
logging.INFO,
"\033[1;34m%-8s\033[1;m" % logging.getLevelName(logging.INFO))
logging.addLevelName(
logging.ERROR,
"\033[1;31m%-8s\033[1;m" % logging.getLevelName(logging.ERROR))
logging.addLevelName(
logging.WARNING, "\033[1;33m%-8s\033[1;m" %
logging.getLevelName(logging.WARNING))
root_logger.addHandler(handler)
def connect(self):
"""
Establishes the actual connection to the referred RSE.
If we decide to use gfal, init should be done here.
:raises RSEAccessDenied
"""
self.logger(logging.DEBUG, 'connecting to storage')
gfal2.set_verbose(gfal2.verbose_level.verbose)
self.__ctx = gfal2.creat_context() # pylint: disable=no-member
self.__ctx.set_opt_string_list(
"SRM PLUGIN", "TURL_PROTOCOLS",
["gsiftp", "rfio", "gsidcap", "dcap", "kdcap"])
self.__ctx.set_opt_string("XROOTD PLUGIN", "XRD.WANTPROT", "gsi,unix")
self.__ctx.set_opt_boolean("XROOTD PLUGIN", "NORMALIZE_PATH", False)
auth_configured = False
if self.auth_token:
self.__ctx.set_opt_string("BEARER", "TOKEN", self.auth_token)
auth_configured = True
# Configure gfal authentication to use the rucio client proxy if and only if gfal didn't initialize its credentials already
# (https://gitlab.cern.ch/dmc/gfal2/-/blob/48cfe3476392c884b53d00799198b1238603a406/src/core/common/gfal_common.c#L79)
if not auth_configured:
try:
self.__ctx.get_opt_string("X509", "CERT")
self.__ctx.get_opt_string("X509", "KEY")
auth_configured = True
except gfal2.GError: # pylint: disable=no-member
pass
if not auth_configured:
try:
self.__ctx.get_opt_string("BEARER", "TOKEN")
auth_configured = True
except gfal2.GError: # pylint: disable=no-member
pass
if not auth_configured:
proxy = config.config_get('client',
'client_x509_proxy',
default=None,
raise_exception=False)
if proxy:
self.logger(
logging.INFO,
'Configuring authentication to use {}'.format(proxy))
self.__ctx.set_opt_string("X509", "CERT", proxy)
self.__ctx.set_opt_string("X509", "KEY", proxy)
if TIMEOUT:
try:
timeout = int(TIMEOUT)
self.__ctx.set_opt_integer("HTTP PLUGIN", "OPERATION_TIMEOUT",
timeout)
self.__ctx.set_opt_integer("SRM PLUGIN", "OPERATION_TIMEOUT",
timeout)
self.__ctx.set_opt_integer("GRIDFTP PLUGIN",
"OPERATION_TIMEOUT", timeout)
except ValueError:
self.logger(logging.ERROR, 'wrong timeout value %s', TIMEOUT)
def test_creat_and copy(self):
gfal2.set_verbose(gfal2.verbose_level.debug)
c = gfal2.creat_context()
c.timeout=200
self.assertTrue(t.timeout == 200 )
d = c.copy()
self.assertTrue(d.timeout == 200 )
c.timeout = 20
del c
self.assertTrue(d.timeout == 200 )
def createGfal2Context(logLevel="normal", emulate=False):
"""
Create a gfal2 context object
:param logLevel: string with the gfal2 log level
:param emulate: boolean to be used by unit tests
:return: the gfal2 context object
"""
if emulate:
return None
ctx = gfal2.creat_context()
gfal2.set_verbose(gfal2.verbose_level.names[logLevel])
return ctx
def test_creat_and_delete_params(self):
gfal2.set_verbose(gfal2.verbose_level.debug)
context = gfal2.creat_context()
t = context.transfer_params()
self.assertTrue(t.timeout > 0 )
t.timeout = 10
self.assertTrue(t.timeout == 10 )
self.assertTrue(self.check_checksum == False)
t.check_checksum = True
self.assertTrue(self.check_checksum == True)
self.assertTrue(t.src_spacetoken == "")
t.src_spacetoken = "TOKENDTEAM"
self.assertTrue(t.src_spacetoken == "TOKENDTEAM")
def get_space_usage(self):
"""
Get RSE space usage information.
:returns: a list with dict containing 'totalsize' and 'unusedsize'
:raises ServiceUnavailable: if some generic error occured in the library.
"""
rse_name = self.rse['rse']
dest = '/tmp/rucio-gsiftp-site-size_' + rse_name
space_usage_url = ''
# url of space usage json, woud be nicer to have it in rse_settings
agis = requests.get(
'http://atlas-agis-api.cern.ch/request/ddmendpoint/query/list/?json'
).json()
agis_token = ''
for res in agis:
if rse_name == res['name']:
agis_token = res['token']
space_usage_url = res['space_usage_url']
import gfal2 # pylint: disable=import-error
gfal2.set_verbose(gfal2.verbose_level.normal) # pylint: disable=no-member
try:
if os.path.exists(dest):
os.remove(dest)
ctx = gfal2.creat_context() # pylint: disable=no-member
ctx.set_opt_string_list(
"SRM PLUGIN", "TURL_PROTOCOLS",
["gsiftp", "rfio", "gsidcap", "dcap", "kdcap"])
params = ctx.transfer_parameters()
params.timeout = 3600
ret = ctx.filecopy(params, str(space_usage_url),
str('file://' + dest))
if ret == 0:
data_file = open(dest)
data = json.load(data_file)
data_file.close()
if agis_token not in list(data.keys()):
print('ERROR: space usage json has different token as key')
else:
totalsize = int(data[agis_token]['total_space'])
used = int(data[agis_token]['used_space'])
unusedsize = totalsize - used
return totalsize, unusedsize
except Exception as error:
print(error)
raise exception.ServiceUnavailable(error)
def connect(self):
"""
Establishes the actual connection to the referred RSE.
If we decide to use gfal, init should be done here.
:raises RSEAccessDenied
"""
self.logger.debug('connecting')
gfal2.set_verbose(gfal2.verbose_level.verbose)
self.__ctx = gfal2.creat_context() # pylint: disable=no-member
self.__ctx.set_opt_string_list("SRM PLUGIN", "TURL_PROTOCOLS", ["gsiftp", "rfio", "gsidcap", "dcap", "kdcap"])
self.__ctx.set_opt_string("XROOTD PLUGIN", "XRD.WANTPROT", "gsi,unix")
self.__ctx.set_opt_boolean("XROOTD PLUGIN", "NORMALIZE_PATH", False)
if self.auth_token:
self.__ctx.set_opt_string("BEARER", "TOKEN", self.auth_token)
def __setup_logger(level, log_file):
# Handle logging level
level = max(0, level)
level = min(3, level)
log_level_value = logging.ERROR - (level * 10)
if level < 3:
gfal2.set_verbose(gfal2.verbose_level.verbose)
else:
gfal2.set_verbose(gfal2.verbose_level.debug)
# Handle log file
log_stream = sys.stdout
if log_file:
log_stream = open(log_file, 'w+')
root_logger = logging.getLogger()
root_logger.setLevel(log_level_value)
handler = logging.StreamHandler(log_stream)
handler.setLevel(log_level_value)
handler.setFormatter(logging.Formatter('%(levelname)s %(message)s'))
if log_stream.isatty():
logging.addLevelName(
logging.DEBUG,
"\033[1;2m%-8s\033[1;m" % logging.getLevelName(logging.DEBUG))
logging.addLevelName(
logging.INFO,
"\033[1;34m%-8s\033[1;m" % logging.getLevelName(logging.INFO))
logging.addLevelName(
logging.ERROR,
"\033[1;31m%-8s\033[1;m" % logging.getLevelName(logging.ERROR))
logging.addLevelName(
logging.WARNING, "\033[1;33m%-8s\033[1;m" %
logging.getLevelName(logging.WARNING))
root_logger.addHandler(handler)
rses_lists = []
for single_rse in default_rses:
rses_lists.append(single_rse['rse'])
print(rses_lists)
# Gfal settings
gfal = Gfal2Context()
print(json.dumps(client.whoami(), indent=4, sort_keys=True))
print(json.dumps(client.ping(), indent=4, sort_keys=True))
sys.path.append("/usr/lib64/python3.6/site-packages/")
import gfal2
from gfal2 import Gfal2Context, GError
gfal2.set_verbose(gfal2.verbose_level.debug)
gfal = Gfal2Context()
# In[3]:
# Specific Functions for Rucio
## Get files from base mouting point of RSE defined on DEFAULT_PATH
def check_directory(path):
try:
full_path = gfal.listdir(str(path))
is_dir_or_not = True
except:
def gfal_exec(method, args, nonerrors = {}, return_value = False):
"""
GFAL2 execution function
@param method Name of the Gfal2Context method to execute.
@param args Tuple of arguments to pass to the method
@param nonerrors Dictionary of error code translation for non-errors.
@param return_value If True, simply return the return value of the function.
@return (exit code, start time, finish time, error message, log string)
"""
start_time = None
finish_time = None
log = ''
for attempt in xrange(5):
# gfal2 knows to write to the logger. Redirect to StringIO and dump the full log at the end.
stream = cStringIO.StringIO()
LOG.handlers.pop()
handler = logging.StreamHandler(stream)
handler.setFormatter(logging.Formatter(fmt = '%(asctime)s: %(message)s'))
LOG.addHandler(handler)
start_time = int(time.time())
try:
gfal2.set_verbose(gfal2.verbose_level.verbose)
context = gfal2.creat_context()
result = getattr(gfal2.Gfal2Context, method)(context, *args)
finish_time = int(time.time())
except gfal2.GError as err:
if return_value:
raise
exitcode, msg = err.code, str(err)
c = find_msg_code(msg)
if c is not None:
exitcode = c
if exitcode in nonerrors:
return 0, start_time, int(time.time()), nonerrors[exitcode], ''
elif exitcode in irrecoverable_errors:
break
except Exception as exc:
if return_value:
raise
exitcode, msg = -1, str(exc)
else:
exitcode, msg = 0, None
finally:
handler.flush()
log_tmp = stream.getvalue().strip()
# give a nice indent to each line
log = ''.join(' %s\n' % line for line in log_tmp.split('\n'))
stream.close()
break
if return_value:
return result
else:
# all variables would be defined even when all attempts are exhausted
return exitcode, start_time, finish_time, msg, log
#!/usr/bin/env python
import optparse
import sys
import gfal2
gfal2.set_verbose(gfal2.verbose_level.debug)
def event_callback(event):
print("[%s] %s %s %s" % (event.timestamp,
event.domain,
event.stage,
event.description))
def monitor_callback(src, dst, average, instant, transferred, elapsed):
print("[%4d] %.2fMB (%.2fKB/s)\r" % (elapsed,
transferred / 1048576,
average / 1024))
sys.stdout.flush()
if __name__ == '__main__':
# Parse arguments
parser = optparse.OptionParser(usage=('usage: %prog [options] '
'source destination'))
parser.add_option('-s',
'--source-space-token',
dest='source_space_token',
def test_creat_and_delete_instance(self):
gfal2.set_verbose(gfal2.verbose_level.debug)
context = gfal2.creat_context()
del context
import sys
from pprint import pprint
import gfal2
gfal2.set_verbose(gfal2.verbose_level.warning)
# from dirent.h
DT_DIR = 4 # gfal2.Gfal2Context.Dirent d_type directory
DT_REG = 8 # gfal2.Gfal2Context.Dirent d_type file
class ProgressMeter(object):
def __init__(self, update_interval=10):
self._update_interval = update_interval
self._last_update = 0
def __call__(self, src, dst, average, instant, transferred, elapsed):
if elapsed - self._last_update > self._update_interval:
sys.stdout.write("[%4d] %.2fMB (%.2fKB/s)\n" % (elapsed, transferred / 1048576, average / 1024))
sys.stdout.flush()
self._last_update = elapsed
def ls(url, dt_filter=None):
ctx = gfal2.creat_context()
dirp = ctx.opendir(url)
ret = []
while True:
dirent,stat = dirp.readpp()
if dirent is None:
break
if dirent.d_name in ('.', '..'):
continue
default="ops")
app.add_argument("--srmv", help="srm version to use", default='2')
app.add_argument("--ldap-url",
help="LDAP URL",
dest="ldap_url",
default="ldap://lcg-bdii.cern.ch:2170")
app.add_argument("--se-timeout",
dest="se_timeout",
type=int,
help="storage operations timeout",
default=60)
# Reasonable defaults for timeouts
LCG_GFAL_BDII_TIMEOUT = 10
gfal2.set_verbose(gfal2.verbose_level.normal)
# Service version(s)
svcVers = ['1', '2']
svcVer = '2'
workdir_metric = tempfile.mkdtemp()
# files and patterns
_fileTest = workdir_metric + '/testFile.txt'
_fileTestIn = workdir_metric + '/testFileIn.txt'
_fileSRMPattern = 'testfile-put-%s-%s.txt' # time, uuid
_voInfoDictionary = {}
# Instantiate gfal2
ctx = gfal2.creat_context()
# should have the '/pnfs/...' format. Script output: #
# ONLINE: means that the file is only on disk #
# NEARLINE: means that the file in only on tape #
# ONLINE_AND_NEARLINE: means that the file is on disk #
# and tape #
# ===================================================================== #
"""
from __future__ import print_function
import sys
from collections import Counter
try:
import gfal2 as gfal # pylint: disable=import-error
gfal.set_verbose(gfal.verbose_level.warning)
except ImportError:
print("GFAL CANNOT BE IMPORTED")
from GRID_LRT.Staging.srmlist import srmlist
from GRID_LRT.auth import grid_credentials
def main(filename, verbose=True):
"""Main function that takes in a file name and returns a list of tuples of
filenames and staging statuses. The input file can be both srm:// and gsiftp:// links.
Args:
:param filename: The filename holding the links whose have to be checked
:type filename: str
:param verbose: A toggle to turn off printing out the status of each file.
def set_verbose_mode(params): # print params modes = [ gfal2.verbose_level.normal, gfal2.verbose_level.verbose, gfal2.verbose_level.debug, gfal2.verbose_level.trace ] gfal2.set_verbose(modes[params.verbose])
