def get(self):
"""
Ping the server and retrieve the server version.
.. :quickref: Ping; Ping the server.
**Example request**:
.. sourcecode:: http
GET /ping HTTP/1.1
Host: rucio-server.com
Accept: application/json
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: application/json
{"version": "1.15.0"}
:status 200: OK.
:status 500: Internal Error.
:returns: JSON dictionary with the version.
"""
return Response(dumps({"version": version.version_string()}),
content_type="application/json")
def get(self):
"""
Ping the server and retrieve the server version.
.. :quickref: Ping; Ping the server.
**Example request**:
.. sourcecode:: http
GET /ping HTTP/1.1
Host: rucio-server.com
Accept: application/json
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: application/json
{"version": "1.15.0"}
:status 200: OK.
:status 406: Not Acceptable.
:returns: JSON dictionary with the version.
"""
headers = self.get_headers()
response = jsonify(version=version.version_string())
response.headers.extend(headers)
return response
def get(self):
"""
---
summary: Ping
description: Ping the server and get data about it.
tags:
- Ping
responses:
200:
description: OK
content:
application/json:
schema:
type: object
properties:
version:
descripption: The server version.
type: string
406:
description: Not acceptable
"""
headers = self.get_headers()
response = jsonify(version=version.version_string())
response.headers.extend(headers)
return response
def check_token(rendered_tpl):
token = None
js_token = ""
js_account = ""
def_account = None
accounts = None
cookie_accounts = None
rucio_ui_version = version.version_string()
render = template.render(join(dirname(__file__), '../templates'))
if ctx.env.get('SSL_CLIENT_VERIFY') != 'SUCCESS':
return render.problem("No certificate provided. Please authenticate with a cerficate registered in Rucio.")
dn = ctx.env.get('SSL_CLIENT_S_DN')
# try to get and check the rucio session token from cookie
session_token = cookies().get('x-rucio-auth-token')
validate_token = authentication.validate_auth_token(session_token)
# if there is no session token or if invalid: get a new one.
if validate_token is None:
# get all accounts for an identity. Needed for account switcher in UI.
accounts = identity.list_accounts_for_identity(dn, 'x509')
cookie_accounts = accounts
try:
try:
# try to get the default account for the identity.
def_account = identity.get_default_account(dn, 'x509')
except IdentityError:
# if there is no default account used the first from the list off accounts.
def_account = accounts[0]
token = authentication.get_auth_token_x509(def_account,
dn,
'webui',
ctx.env.get('REMOTE_ADDR'))
except:
return render.problem("Your certificate (%s) is not registered in Rucio. Please contact <a href=\"mailto:[email protected]\">Rucio Support</a>" % dn)
# write the token and account to javascript variables, that will be used in the HTML templates.
js_token = __to_js('token', token)
js_account = __to_js('account', def_account)
# if there was no valid session token write the new token to a cookie.
if token:
setcookie('x-rucio-auth-token', value=token, expires=3600, path='/')
if cookie_accounts:
values = ""
for acc in cookie_accounts:
values += acc + " "
setcookie('rucio-available-accounts', value=values[:-1], path='/')
return render.base(js_token, js_account, rucio_ui_version, rendered_tpl)
def access_granted(valid_token_dict, rendered_tpl=None):
"""
Assuming validated token dictionary is provided, renders required template page.
:param valid_token_dict: token validation dictionary
:param rendered_tpl: rendered template
:returns: rendered base temmplate with rendered_tpl content
"""
js_account = __to_js('account', valid_token_dict['account'])
js_token = __to_js('token', valid_token_dict['token'])
rucio_ui_version = version.version_string()
policy = config_get('policy', 'permission')
return RENDERER.base(js_token, js_account, rucio_ui_version, policy,
rendered_tpl)
def get(self):
"""
Ping the server and retrieve the server version.
.. :quickref: Ping; Ping the server.
**Example request**:
.. sourcecode:: http
GET /ping HTTP/1.1
Host: rucio-server.com
Accept: application/json
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: application/json
{"version": "1.15.0"}
:status 200: OK.
:status 406: Not Acceptable.
:status 500: Internal Error.
:returns: JSON dictionary with the version.
"""
headers = Headers()
headers.set('Access-Control-Allow-Origin',
request.environ.get('HTTP_ORIGIN'))
headers.set('Access-Control-Allow-Headers',
request.environ.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS'))
headers.set('Access-Control-Allow-Methods', '*')
headers.set('Access-Control-Allow-Credentials', 'true')
headers.set('Cache-Control',
'no-cache, no-store, max-age=0, must-revalidate')
headers.add('Cache-Control', 'post-check=0, pre-check=0')
headers.set('Pragma', 'no-cache')
response = jsonify(version=version.version_string())
response.headers.extend(headers)
return response
def GET(self):
"""
.. http:get:: /ping
Get server version information.
**Example request**:
.. sourcecode:: http
GET /ping HTTP/1.1
Host: rucio-server.com
Accept: application/json
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: application/json
{
"version": "0.2.9"
}
:statuscode 200: no error
:statuscode 406: Not Acceptable
:statuscode 500: InternalError
"""
header('Access-Control-Allow-Origin', ctx.env.get('HTTP_ORIGIN'))
header('Access-Control-Allow-Headers',
ctx.env.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS'))
header('Access-Control-Allow-Methods', '*')
header('Access-Control-Allow-Credentials', 'true')
header('Content-Type', 'application/json')
header('Cache-Control',
'no-cache, no-store, max-age=0, must-revalidate')
header('Cache-Control', 'post-check=0, pre-check=0', False)
header('Pragma', 'no-cache')
return dumps({"version": version.version_string()})
def GET(self):
"""
.. http:get:: /ping
Get server version information.
**Example request**:
.. sourcecode:: http
GET /ping HTTP/1.1
Host: rucio-server.com
Accept: application/json
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: application/json
{
"version": "0.2.9"
}
:statuscode 200: no error
:statuscode 500: InternalError
"""
header('Access-Control-Allow-Origin', ctx.env.get('HTTP_ORIGIN'))
header('Access-Control-Allow-Headers', ctx.env.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS'))
header('Access-Control-Allow-Methods', '*')
header('Access-Control-Allow-Credentials', 'true')
header('Content-Type', 'application/json')
header('Cache-Control', 'no-cache, no-store, max-age=0, must-revalidate')
header('Cache-Control', 'post-check=0, pre-check=0', False)
header('Pragma', 'no-cache')
return dumps({"version": version.version_string()})
def __init__(self,
rucio_host=None,
auth_host=None,
account=None,
ca_cert=None,
auth_type=None,
creds=None,
timeout=None,
user_agent='rucio-clients'):
"""
Constructor of the BaseClient.
:param rucio_host: the address of the rucio server, if None it is read from the config file.
:param rucio_port: the port of the rucio server, if None it is read from the config file.
:param auth_host: the address of the rucio authentication server, if None it is read from the config file.
:param auth_port: the port of the rucio authentication server, if None it is read from the config file.
:param account: the account to authenticate to rucio.
:param use_ssl: enable or disable ssl for commucation. Default is enabled.
:param ca_cert: the path to the rucio server certificate.
:param auth_type: the type of authentication (e.g.: 'userpass', 'kerberos' ...)
:param creds: a dictionary with credentials needed for authentication.
:param user_agent: indicates the client
"""
self.host = rucio_host
self.list_hosts = []
self.auth_host = auth_host
self.session = session()
self.user_agent = "%s/%s" % (user_agent, version.version_string()
) # e.g. "rucio-clients/0.2.13"
sys.argv[0] = sys.argv[0].split('/')[-1]
self.script_id = '::'.join(sys.argv[0:2])
if self.script_id == '': # Python interpreter used
self.script_id = 'python'
try:
if self.host is None:
self.host = config_get('client', 'rucio_host')
if self.auth_host is None:
self.auth_host = config_get('client', 'auth_host')
except (NoOptionError, NoSectionError) as error:
raise MissingClientParameter(
'Section client and Option \'%s\' cannot be found in config file'
% error.args[0])
self.account = account
self.ca_cert = ca_cert
self.auth_type = auth_type
self.creds = creds
self.auth_token = None
self.headers = {}
self.timeout = timeout
self.request_retries = self.REQUEST_RETRIES
if auth_type is None:
LOG.debug(
'no auth_type passed. Trying to get it from the environment variable RUCIO_AUTH_TYPE and config file.'
)
if 'RUCIO_AUTH_TYPE' in environ:
if environ['RUCIO_AUTH_TYPE'] not in ('userpass', 'x509',
'x509_proxy', 'gss'):
raise MissingClientParameter(
'Possible RUCIO_AUTH_TYPE values: userpass, x509, x509_proxy, gss vs. '
+ environ['RUCIO_AUTH_TYPE'])
self.auth_type = environ['RUCIO_AUTH_TYPE']
else:
try:
self.auth_type = config_get('client', 'auth_type')
except (NoOptionError, NoSectionError) as error:
raise MissingClientParameter(
'Option \'%s\' cannot be found in config file' %
error.args[0])
if creds is None:
LOG.debug(
'no creds passed. Trying to get it from the config file.')
self.creds = {}
try:
if self.auth_type == 'userpass':
self.creds['username'] = config_get('client', 'username')
self.creds['password'] = config_get('client', 'password')
elif self.auth_type == 'x509':
self.creds['client_cert'] = path.abspath(
path.expanduser(
path.expandvars(config_get('client',
'client_cert'))))
self.creds['client_key'] = path.abspath(
path.expanduser(
path.expandvars(config_get('client',
'client_key'))))
elif self.auth_type == 'x509_proxy':
self.creds['client_proxy'] = path.abspath(
path.expanduser(
path.expandvars(
config_get('client', 'client_x509_proxy'))))
except (NoOptionError, NoSectionError) as error:
if error.args[0] != 'client_key':
raise MissingClientParameter(
'Option \'%s\' cannot be found in config file' %
error.args[0])
rucio_scheme = urlparse(self.host).scheme
auth_scheme = urlparse(self.auth_host).scheme
if rucio_scheme != 'http' and rucio_scheme != 'https':
raise ClientProtocolNotSupported('\'%s\' not supported' %
rucio_scheme)
if auth_scheme != 'http' and auth_scheme != 'https':
raise ClientProtocolNotSupported('\'%s\' not supported' %
auth_scheme)
if (rucio_scheme == 'https'
or auth_scheme == 'https') and ca_cert is None:
LOG.debug(
'no ca_cert passed. Trying to get it from the config file.')
try:
self.ca_cert = path.expandvars(config_get('client', 'ca_cert'))
except (NoOptionError, NoSectionError) as error:
raise MissingClientParameter(
'Option \'%s\' cannot be found in config file' %
error.args[0])
self.list_hosts = [self.host]
if account is None:
LOG.debug(
'no account passed. Trying to get it from the config file.')
try:
self.account = config_get('client', 'account')
except (NoOptionError, NoSectionError):
try:
self.account = environ['RUCIO_ACCOUNT']
except KeyError:
raise MissingClientParameter(
'Option \'account\' cannot be found in config file and RUCIO_ACCOUNT is not set.'
)
token_path = self.TOKEN_PATH_PREFIX + self.account
self.token_file = token_path + '/' + self.TOKEN_PREFIX + self.account
self.__authenticate()
try:
self.request_retries = int(config_get('client', 'request_retries'))
except NoOptionError:
LOG.debug(
'request_retries not specified in config file. Taking default.'
)
except ValueError:
LOG.debug('request_retries must be an integer. Taking default.')
def check_token(rendered_tpl):
attribs = None
token = None
js_token = ""
js_account = ""
def_account = None
accounts = None
cookie_accounts = None
rucio_ui_version = version.version_string()
ui_account = None
if 'ui_account' in input():
ui_account = input()['ui_account']
render = template.render(join(dirname(__file__), '../templates'))
if ctx.env.get('SSL_CLIENT_VERIFY') != 'SUCCESS':
return render.problem(
"No certificate provided. Please authenticate with a certificate registered in Rucio."
)
dn = ctx.env.get('SSL_CLIENT_S_DN')
msg = "Your certificate (%s) is not mapped to any rucio account." % dn
msg += "<br><br><font color=\"red\">First, please make sure it is correctly registered in <a href=\"https://voms2.cern.ch:8443/voms/atlas\">VOMS</a> and be patient until it has been fully propagated through the system.</font>"
msg += "<br><br>Then, if it is still not working please contact <a href=\"mailto:[email protected]\">DDM Support</a>."
# try to get and check the rucio session token from cookie
session_token = cookies().get('x-rucio-auth-token')
validate_token = authentication.validate_auth_token(session_token)
# check if ui_account param is set and if yes, force new token
if ui_account:
accounts = identity.list_accounts_for_identity(dn, 'x509')
if len(accounts) == 0:
return render.problem(msg)
if ui_account not in accounts:
return render.problem(
"The rucio account (%s) you selected is not mapped to your certificate (%s). Please select another account or none at all to automatically use your default account."
% (ui_account, dn))
cookie_accounts = accounts
if (validate_token is None) or (validate_token['account'] !=
ui_account):
try:
token = authentication.get_auth_token_x509(
ui_account, dn, 'webui', ctx.env.get('REMOTE_ADDR'))
except:
return render.problem(msg)
attribs = list_account_attributes(ui_account)
js_token = __to_js('token', token)
js_account = __to_js('account', def_account)
else:
# if there is no session token or if invalid: get a new one.
if validate_token is None:
# get all accounts for an identity. Needed for account switcher in UI.
accounts = identity.list_accounts_for_identity(dn, 'x509')
if len(accounts) == 0:
return render.problem(msg)
cookie_accounts = accounts
# try to set the default account to the user account, if not available take the first account.
def_account = accounts[0]
for account in accounts:
account_info = get_account_info(account)
if account_info.account_type == AccountType.USER:
def_account = account
break
selected_account = cookies().get('rucio-selected-account')
if (selected_account):
def_account = selected_account
try:
token = authentication.get_auth_token_x509(
def_account, dn, 'webui', ctx.env.get('REMOTE_ADDR'))
except:
return render.problem(msg)
attribs = list_account_attributes(def_account)
# write the token and account to javascript variables, that will be used in the HTML templates.
js_token = __to_js('token', token)
js_account = __to_js('account', def_account)
# if there was no valid session token write the new token to a cookie.
if token:
setcookie('x-rucio-auth-token', value=token, path='/')
setcookie('rucio-auth-token-created-at', value=int(time()), path='/')
if cookie_accounts:
values = ""
for acc in cookie_accounts:
values += acc + " "
setcookie('rucio-available-accounts', value=values[:-1], path='/')
if attribs:
setcookie('rucio-account-attr', value=dumps(attribs), path='/')
if ui_account:
setcookie('rucio-selected-account', value=ui_account, path='/')
return render.base(js_token, js_account, rucio_ui_version, rendered_tpl)
def test_rucio_version(self):
"""CLIENT(USER): Rucio version"""
cmd = 'bin/rucio --version'
exitcode, out, err = execute(cmd)
nose.tools.assert_equal(err, 'rucio %s\n' % version.version_string())
revno = run_git_command(revno_cmd).decode()
version_file = open("lib/rucio/vcsversion.py", 'w')
version_file.write(
"""'''\n"""
"""This file is automatically generated by setup.py, So don't edit it. :)\n"""
"""'''\n"""
"""VERSION_INFO = {\n"""
""" 'final': False,\n"""
""" 'version': '%s',\n"""
""" 'branch_nick': '%s',\n"""
""" 'revision_id': '%s',\n"""
""" 'revno': %s\n"""
"""}""" % (pkg_version, branch_nick, revid, revno))
version_file.close()
else:
pkg_version = version.version_string()
cmdclass = {}
try:
from sphinx.setup_command import BuildDoc
class local_BuildDoc(BuildDoc):
def run(self):
for builder in ['html']: # 'man','latex'
self.builder = builder
self.finalize_options()
BuildDoc.run(self)
cmdclass['build_sphinx'] = local_BuildDoc
except:
def saml_authentication(method, rendered_tpl):
"""
Login with SAML
:param method: method type, GET or POST
:param rendered_tpl: page to be rendered
"""
attribs = None
token = None
js_token = ""
js_account = ""
def_account = None
accounts = None
cookie_accounts = None
rucio_ui_version = version.version_string()
policy = config_get('policy', 'permission')
# Initialize variables for sending SAML request
SAML_PATH = join(dirname(__file__), 'saml/')
request = ctx.env
data = dict(input())
req = prepare_webpy_request(request, data)
auth = OneLogin_Saml2_Auth(req, custom_base_path=SAML_PATH)
saml_user_data = cookies().get('saml-user-data')
render = template.render(join(dirname(__file__), '../templates'))
session_token = cookies().get('x-rucio-auth-token')
validate_token = authentication.validate_auth_token(session_token)
if method == "GET":
# If user data is not present, redirect to IdP for authentication
if not saml_user_data:
return seeother(auth.login())
# If user data is present and token is valid, render the required page
elif validate_token:
js_token = __to_js('token', session_token)
js_account = __to_js('account', def_account)
return render.base(js_token, js_account, rucio_ui_version, policy,
rendered_tpl)
# If user data is present but token is not valid, create a new one
saml_nameid = cookies().get('saml-nameid')
accounts = identity.list_accounts_for_identity(saml_nameid, 'saml')
cookie_accounts = accounts
try:
token = authentication.get_auth_token_saml(
def_account, saml_nameid, 'webui',
ctx.env.get('REMOTE_ADDR')).token
except:
return render.problem('Cannot get auth token')
attribs = list_account_attributes(def_account)
# write the token and account to javascript variables, that will be used in the HTML templates.
js_token = __to_js('token', token)
js_account = __to_js('account', def_account)
set_cookies(token, cookie_accounts, attribs)
return render.base(js_token, js_account, rucio_ui_version, policy,
rendered_tpl)
# If method is POST, check the received SAML response and redirect to home if valid
auth.process_response()
errors = auth.get_errors()
if not errors:
if auth.is_authenticated():
setcookie('saml-user-data', value=auth.get_attributes(), path='/')
setcookie('saml-session-index',
value=auth.get_session_index(),
path='/')
setcookie('saml-nameid', value=auth.get_nameid(), path='/')
saml_nameid = auth.get_nameid()
accounts = identity.list_accounts_for_identity(saml_nameid, 'saml')
cookie_accounts = accounts
# try to set the default account to the user account, if not available take the first account.
def_account = accounts[0]
for account in accounts:
account_info = get_account_info(account)
if account_info.account_type == AccountType.USER:
def_account = account
break
selected_account = cookies().get('rucio-selected-account')
if (selected_account):
def_account = selected_account
try:
token = authentication.get_auth_token_saml(
def_account, saml_nameid, 'webui',
ctx.env.get('REMOTE_ADDR')).token
except:
return render.problem('Cannot get auth token')
attribs = list_account_attributes(def_account)
# write the token and account to javascript variables, that will be used in the HTML templates.
js_token = __to_js('token', token)
js_account = __to_js('account', def_account)
set_cookies(token, cookie_accounts, attribs)
return seeother("/")
return render.problem("Not authenticated")
return render.problem("Error while processing SAML")
def log_in(data, rendered_tpl):
attribs = None
token = None
js_token = ""
js_account = ""
def_account = None
accounts = None
cookie_accounts = None
rucio_ui_version = version.version_string()
policy = config_get('policy', 'permission')
render = template.render(join(dirname(__file__), '../templates'))
# # try to get and check the rucio session token from cookie
session_token = cookies().get('x-rucio-auth-token')
validate_token = authentication.validate_auth_token(session_token)
# if token is valid, render the requested page.
if validate_token and not data:
token = session_token
js_token = __to_js('token', token)
js_account = __to_js('account', def_account)
return render.base(js_token, js_account, rucio_ui_version, policy,
rendered_tpl)
else:
# if there is no session token or if invalid: get a new one.
# if user tries to access a page through URL without logging in, then redirect to login page.
if rendered_tpl:
return render.login()
# get all accounts for an identity. Needed for account switcher in UI.
accounts = identity.list_accounts_for_identity(data.username,
'userpass')
if len(accounts) == 0:
return render.problem('No accounts for the given identity.')
cookie_accounts = accounts
# try to set the default account to the user account, if not available take the first account.
def_account = accounts[0]
for account in accounts:
account_info = get_account_info(account)
if account_info.account_type == AccountType.USER:
def_account = account
break
selected_account = cookies().get('rucio-selected-account')
if (selected_account):
def_account = selected_account
try:
token = authentication.get_auth_token_user_pass(
def_account, data.username, data.password.encode("ascii"),
'webui', ctx.env.get('REMOTE_ADDR')).token
except:
return render.problem('Cannot get auth token')
attribs = list_account_attributes(def_account)
# write the token and account to javascript variables, that will be used in the HTML templates.
js_token = __to_js('token', token)
js_account = __to_js('account', def_account)
set_cookies(token, cookie_accounts, attribs)
return seeother('/')
def _send_request(self, url, headers=None, type='GET', data=None, params=None):
"""
Helper method to send requests to the rucio server. Gets a new token and retries if an unauthorized error is returned.
:param url: the http url to use.
:param headers: additional http headers to send.
:param type: the http request type to use.
:param data: post data.
:param params: (optional) Dictionary or bytes to be sent in the url query string.
:return: the HTTP return body.
"""
r = None
retry = 0
hds = {'X-Rucio-Auth-Token': self.auth_token, 'X-Rucio-Account': self.account, 'X-Rucio-Appid': '', 'Connection': 'Keep-Alive', 'User-Agent': '%s/%s' % (self.user_agent, version.version_string())}
if headers is not None:
hds.update(headers)
while retry <= self.request_retries:
try:
if type == 'GET':
r = self.session.get(url, headers=hds, verify=self.ca_cert, timeout=self.timeout, params=params, stream=True)
elif type == 'PUT':
r = self.session.put(url, headers=hds, data=data, verify=self.ca_cert, timeout=self.timeout)
elif type == 'POST':
r = self.session.post(url, headers=hds, data=data, verify=self.ca_cert, timeout=self.timeout, stream=True)
elif type == 'DEL':
r = self.session.delete(url, headers=hds, data=data, verify=self.ca_cert, timeout=self.timeout)
else:
return
# except ConnectionError, e:
# LOG.warning('ConnectionError: ' + str(e))
# retry += 1
# if retry > self.request_retries:
# raise
# continue
except SSLError, e:
LOG.warning('SSLError: ' + str(e))
retry += 1
self.ca_cert = False
if retry > self.request_retries:
raise
continue
if r is not None and r.status_code == codes.unauthorized:
self.__get_token()
hds['X-Rucio-Auth-Token'] = self.auth_token
retry += 1
else:
break
def __init__(self,
rucio_host=None,
auth_host=None,
account=None,
ca_cert=None,
auth_type=None,
creds=None,
timeout=600,
user_agent='rucio-clients'):
"""
Constructor of the BaseClient.
:param rucio_host: the address of the rucio server, if None it is read from the config file.
:param rucio_port: the port of the rucio server, if None it is read from the config file.
:param auth_host: the address of the rucio authentication server, if None it is read from the config file.
:param auth_port: the port of the rucio authentication server, if None it is read from the config file.
:param account: the account to authenticate to rucio.
:param use_ssl: enable or disable ssl for commucation. Default is enabled.
:param ca_cert: the path to the rucio server certificate.
:param auth_type: the type of authentication (e.g.: 'userpass', 'kerberos' ...)
:param creds: a dictionary with credentials needed for authentication.
:param user_agent: indicates the client
"""
self.host = rucio_host
self.list_hosts = []
self.auth_host = auth_host
self.session = Session()
self.user_agent = "%s/%s" % (user_agent, version.version_string()
) # e.g. "rucio-clients/0.2.13"
sys.argv[0] = sys.argv[0].split('/')[-1]
self.script_id = '::'.join(sys.argv[0:2])
if self.script_id == '': # Python interpreter used
self.script_id = 'python'
try:
if self.host is None:
self.host = config_get('client', 'rucio_host')
if self.auth_host is None:
self.auth_host = config_get('client', 'auth_host')
except (NoOptionError, NoSectionError) as error:
raise MissingClientParameter(
'Section client and Option \'%s\' cannot be found in config file'
% error.args[0])
self.account = account
self.ca_cert = ca_cert
self.auth_type = auth_type
self.creds = creds
self.auth_token = None
self.headers = {}
self.timeout = timeout
self.request_retries = self.REQUEST_RETRIES
if auth_type is None:
LOG.debug(
'no auth_type passed. Trying to get it from the environment variable RUCIO_AUTH_TYPE and config file.'
)
if 'RUCIO_AUTH_TYPE' in environ:
if environ['RUCIO_AUTH_TYPE'] not in [
'userpass', 'x509', 'x509_proxy', 'gss', 'ssh', 'saml'
]:
raise MissingClientParameter(
'Possible RUCIO_AUTH_TYPE values: userpass, x509, x509_proxy, gss, ssh, saml, oidc, vs. '
+ environ['RUCIO_AUTH_TYPE'])
self.auth_type = environ['RUCIO_AUTH_TYPE']
else:
try:
self.auth_type = config_get('client', 'auth_type')
except (NoOptionError, NoSectionError) as error:
raise MissingClientParameter(
'Option \'%s\' cannot be found in config file' %
error.args[0])
if creds is None:
LOG.debug(
'no creds passed. Trying to get it from the config file.')
self.creds = {}
try:
if self.auth_type in ['userpass', 'saml']:
self.creds['username'] = config_get('client', 'username')
self.creds['password'] = config_get('client', 'password')
elif self.auth_type == 'oidc':
self.creds['oidc_auto'] = config_get('client', 'oidc_auto')
self.creds['oidc_scope'] = config_get(
'client', 'oidc_scope')
self.creds['oidc_audience'] = config_get(
'client', 'oidc_audience')
self.creds['oidc_polling'] = config_get(
'client', 'oidc_polling')
self.creds['oidc_refresh_lifetime'] = config_get(
'client', 'oidc_refresh_lifetime')
self.creds['oidc_issuer'] = config_get(
'client', 'oidc_issuer')
if self.creds['oidc_auto']:
self.creds['oidc_username'] = config_get(
'client', 'oidc_username')
self.creds['oidc_password'] = config_get(
'client', 'oidc_password')
elif self.auth_type == 'x509':
self.creds['client_cert'] = path.abspath(
path.expanduser(
path.expandvars(config_get('client',
'client_cert'))))
self.creds['client_key'] = path.abspath(
path.expanduser(
path.expandvars(config_get('client',
'client_key'))))
elif self.auth_type == 'x509_proxy':
try:
self.creds['client_proxy'] = path.abspath(
path.expanduser(
path.expandvars(
config_get('client',
'client_x509_proxy'))))
except NoOptionError as error:
# Recreate the classic GSI logic for locating the proxy:
# - $X509_USER_PROXY, if it is set.
# - /tmp/x509up_u`id -u` otherwise.
# If neither exists (at this point, we don't care if it exists but is invalid), then rethrow
if 'X509_USER_PROXY' in environ:
self.creds['client_proxy'] = environ[
'X509_USER_PROXY']
else:
fname = '/tmp/x509up_u%d' % geteuid()
if path.exists(fname):
self.creds['client_proxy'] = fname
else:
raise MissingClientParameter(
'Cannot find a valid X509 proxy; not in %s, $X509_USER_PROXY not set, and '
'\'x509_proxy\' not set in the configuration file.'
% fname)
elif self.auth_type == 'ssh':
self.creds['ssh_private_key'] = path.abspath(
path.expanduser(
path.expandvars(
config_get('client', 'ssh_private_key'))))
except (NoOptionError, NoSectionError) as error:
if error.args[0] != 'client_key':
raise MissingClientParameter(
'Option \'%s\' cannot be found in config file' %
error.args[0])
rucio_scheme = urlparse(self.host).scheme
auth_scheme = urlparse(self.auth_host).scheme
if rucio_scheme != 'http' and rucio_scheme != 'https':
raise ClientProtocolNotSupported('\'%s\' not supported' %
rucio_scheme)
if auth_scheme != 'http' and auth_scheme != 'https':
raise ClientProtocolNotSupported('\'%s\' not supported' %
auth_scheme)
if (rucio_scheme == 'https'
or auth_scheme == 'https') and ca_cert is None:
LOG.debug(
'no ca_cert passed. Trying to get it from the config file.')
try:
self.ca_cert = path.expandvars(config_get('client', 'ca_cert'))
except (NoOptionError, NoSectionError) as error:
raise MissingClientParameter(
'Option \'%s\' cannot be found in config file' %
error.args[0])
self.list_hosts = [self.host]
if account is None:
LOG.debug(
'no account passed. Trying to get it from the config file.')
try:
self.account = environ['RUCIO_ACCOUNT']
except KeyError:
try:
self.account = config_get('client', 'account')
except (NoOptionError, NoSectionError):
raise MissingClientParameter(
'Option \'account\' cannot be found in config file and RUCIO_ACCOUNT is not set.'
)
token_path = self.TOKEN_PATH_PREFIX + self.account
self.token_file = token_path + '/' + self.TOKEN_PREFIX + self.account
self.__authenticate()
try:
self.request_retries = int(config_get('client', 'request_retries'))
except NoOptionError:
LOG.debug(
'request_retries not specified in config file. Taking default.'
)
except ValueError:
LOG.debug('request_retries must be an integer. Taking default.')
revno = run_git_command(revno_cmd).decode()
version_file = open("lib/rucio/vcsversion.py", 'w')
version_file.write(
"""'''\n"""
"""This file is automatically generated by setup.py, So don't edit it. :)\n"""
"""'''\n"""
"""VERSION_INFO = {\n"""
""" 'final': False,\n"""
""" 'version': '%s',\n"""
""" 'branch_nick': '%s',\n"""
""" 'revision_id': '%s',\n"""
""" 'revno': %s\n"""
"""}""" % (pkg_version, branch_nick, revid, revno))
version_file.close()
else:
pkg_version = version.version_string().decode()
cmdclass = {}
try:
from sphinx.setup_command import BuildDoc
class local_BuildDoc(BuildDoc):
def run(self):
for builder in ['html']: # 'man','latex'
self.builder = builder
self.finalize_options()
BuildDoc.run(self)
cmdclass['build_sphinx'] = local_BuildDoc
except:
# General information about the project. project = u'Rucio' copyright = u'2015, CERN' # Show ToDos todo_include_todos = True # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # from rucio import version as rucio_version # The short X.Y version. version = rucio_version.canonical_version_string() # The full version, including alpha/beta/rc tags. release = rucio_version.version_string() # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: # today = '' # Else, today_fmt is used as the format for a strftime call. # today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = []
def __init__(self, *args, **kwargs):
_sdist.__init__(self, *args, **kwargs)
self.packaging = "default value for this option"
def get_file_list(self):
print "Chosen packaging option: " + name
self.distribution.data_files = data_files
_sdist.get_file_list(self)
cmdclass['sdist'] = CustomSdist
setup(
name=name,
version=version.version_string(),
packages=packages,
package_dir={'': 'lib'},
data_files=data_files,
script_args=copy_args,
cmdclass=cmdclass,
include_package_data=True,
scripts=scripts,
# doc=cmdclass,
author="Rucio",
author_email="*****@*****.**",
description=description,
license="Apache License, Version 2.0",
url="http://rucio.cern.ch/",
classifiers=[
'Development Status :: 5 - Production/Stable',
def __init__(self, rucio_host=None, auth_host=None, account=None, ca_cert=None, auth_type=None, creds=None, timeout=600, user_agent='rucio-clients', vo=None, logger=None):
"""
Constructor of the BaseClient.
:param rucio_host: The address of the rucio server, if None it is read from the config file.
:param rucio_port: The port of the rucio server, if None it is read from the config file.
:param auth_host: The address of the rucio authentication server, if None it is read from the config file.
:param auth_port: The port of the rucio authentication server, if None it is read from the config file.
:param account: The account to authenticate to rucio.
:param use_ssl: Enable or disable ssl for commucation. Default is enabled.
:param ca_cert: The path to the rucio server certificate.
:param auth_type: The type of authentication (e.g.: 'userpass', 'kerberos' ...)
:param creds: Dictionary with credentials needed for authentication.
:param user_agent: Indicates the client.
:param vo: The VO to authenticate into.
:param logger: Logger object to use. If None, use the default LOG created by the module
"""
self.host = rucio_host
self.list_hosts = []
self.auth_host = auth_host
self.logger = logger or LOG
self.session = Session()
self.user_agent = "%s/%s" % (user_agent, version.version_string()) # e.g. "rucio-clients/0.2.13"
sys.argv[0] = sys.argv[0].split('/')[-1]
self.script_id = '::'.join(sys.argv[0:2])
if self.script_id == '': # Python interpreter used
self.script_id = 'python'
try:
if self.host is None:
self.host = config_get('client', 'rucio_host')
if self.auth_host is None:
self.auth_host = config_get('client', 'auth_host')
except (NoOptionError, NoSectionError) as error:
raise MissingClientParameter('Section client and Option \'%s\' cannot be found in config file' % error.args[0])
try:
self.trace_host = config_get('trace', 'trace_host')
except (NoOptionError, NoSectionError):
self.trace_host = self.host
self.logger.debug('No trace_host passed. Using rucio_host instead')
self.account = account
self.vo = vo
self.ca_cert = ca_cert
self.auth_type = auth_type
self.creds = creds
self.auth_token = None
self.auth_token_file_path = config_get('client', 'auth_token_file_path', False, None)
self.headers = {}
self.timeout = timeout
self.request_retries = self.REQUEST_RETRIES
self.token_exp_epoch = None
self.token_exp_epoch_file = None
self.auth_oidc_refresh_active = config_get_bool('client', 'auth_oidc_refresh_active', False, False)
# defining how many minutes before token expires, oidc refresh (if active) should start
self.auth_oidc_refresh_before_exp = config_get_int('client', 'auth_oidc_refresh_before_exp', False, 20)
if auth_type is None:
self.logger.debug('No auth_type passed. Trying to get it from the environment variable RUCIO_AUTH_TYPE and config file.')
if 'RUCIO_AUTH_TYPE' in environ:
if environ['RUCIO_AUTH_TYPE'] not in ['userpass', 'x509', 'x509_proxy', 'gss', 'ssh', 'saml', 'oidc']:
raise MissingClientParameter('Possible RUCIO_AUTH_TYPE values: userpass, x509, x509_proxy, gss, ssh, saml, oidc, vs. ' + environ['RUCIO_AUTH_TYPE'])
self.auth_type = environ['RUCIO_AUTH_TYPE']
else:
try:
self.auth_type = config_get('client', 'auth_type')
except (NoOptionError, NoSectionError) as error:
raise MissingClientParameter('Option \'%s\' cannot be found in config file' % error.args[0])
if self.auth_type == 'oidc':
if not self.creds:
self.creds = {}
# if there are defautl values, check if rucio.cfg does not specify them, otherwise put default
if 'oidc_refresh_lifetime' not in self.creds or self.creds['oidc_refresh_lifetime'] is None:
self.creds['oidc_refresh_lifetime'] = config_get('client', 'oidc_refresh_lifetime', False, None)
if 'oidc_issuer' not in self.creds or self.creds['oidc_issuer'] is None:
self.creds['oidc_issuer'] = config_get('client', 'oidc_issuer', False, None)
if 'oidc_audience' not in self.creds or self.creds['oidc_audience'] is None:
self.creds['oidc_audience'] = config_get('client', 'oidc_audience', False, None)
if 'oidc_auto' not in self.creds or self.creds['oidc_auto'] is False:
self.creds['oidc_auto'] = config_get_bool('client', 'oidc_auto', False, False)
if self.creds['oidc_auto']:
if 'oidc_username' not in self.creds or self.creds['oidc_username'] is None:
self.creds['oidc_username'] = config_get('client', 'oidc_username', False, None)
if 'oidc_password' not in self.creds or self.creds['oidc_password'] is None:
self.creds['oidc_password'] = config_get('client', 'oidc_password', False, None)
if 'oidc_scope' not in self.creds or self.creds['oidc_scope'] == 'openid profile':
self.creds['oidc_scope'] = config_get('client', 'oidc_scope', False, 'openid profile')
if 'oidc_polling' not in self.creds or self.creds['oidc_polling'] is False:
self.creds['oidc_polling'] = config_get_bool('client', 'oidc_polling', False, False)
if not self.creds:
self.logger.debug('No creds passed. Trying to get it from the config file.')
self.creds = {}
try:
if self.auth_type in ['userpass', 'saml']:
self.creds['username'] = config_get('client', 'username')
self.creds['password'] = config_get('client', 'password')
elif self.auth_type == 'x509':
self.creds['client_cert'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'client_cert'))))
if not path.exists(self.creds['client_cert']):
raise MissingClientParameter('X.509 client certificate not found: %s' % self.creds['client_cert'])
self.creds['client_key'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'client_key'))))
if not path.exists(self.creds['client_key']):
raise MissingClientParameter('X.509 client key not found: %s' % self.creds['client_key'])
else:
perms = oct(os.stat(self.creds['client_key']).st_mode)[-3:]
if perms not in ['400', '600']:
raise CannotAuthenticate('X.509 authentication selected, but private key (%s) permissions are liberal (required: 400 or 600, found: %s)' % (self.creds['client_key'], perms))
elif self.auth_type == 'x509_proxy':
try:
self.creds['client_proxy'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'client_x509_proxy'))))
except NoOptionError:
# Recreate the classic GSI logic for locating the proxy:
# - $X509_USER_PROXY, if it is set.
# - /tmp/x509up_u`id -u` otherwise.
# If neither exists (at this point, we don't care if it exists but is invalid), then rethrow
if 'X509_USER_PROXY' in environ:
self.creds['client_proxy'] = environ['X509_USER_PROXY']
else:
fname = '/tmp/x509up_u%d' % geteuid()
if path.exists(fname):
self.creds['client_proxy'] = fname
else:
raise MissingClientParameter('Cannot find a valid X509 proxy; not in %s, $X509_USER_PROXY not set, and '
'\'x509_proxy\' not set in the configuration file.' % fname)
elif self.auth_type == 'ssh':
self.creds['ssh_private_key'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'ssh_private_key'))))
except (NoOptionError, NoSectionError) as error:
if error.args[0] != 'client_key':
raise MissingClientParameter('Option \'%s\' cannot be found in config file' % error.args[0])
rucio_scheme = urlparse(self.host).scheme
auth_scheme = urlparse(self.auth_host).scheme
if rucio_scheme != 'http' and rucio_scheme != 'https':
raise ClientProtocolNotSupported('\'%s\' not supported' % rucio_scheme)
if auth_scheme != 'http' and auth_scheme != 'https':
raise ClientProtocolNotSupported('\'%s\' not supported' % auth_scheme)
if (rucio_scheme == 'https' or auth_scheme == 'https') and ca_cert is None:
self.logger.debug('HTTPS is required, but no ca_cert was passed. Trying to get it from X509_CERT_DIR.')
self.ca_cert = os.environ.get('X509_CERT_DIR', None)
if self.ca_cert is None:
self.logger.debug('HTTPS is required, but no ca_cert was passed and X509_CERT_DIR is not defined. Trying to get it from the config file.')
try:
self.ca_cert = path.expandvars(config_get('client', 'ca_cert'))
except (NoOptionError, NoSectionError):
self.logger.debug('No ca_cert found in configuration. Falling back to Mozilla default CA bundle (certifi).')
self.ca_cert = True
self.list_hosts = [self.host]
if account is None:
self.logger.debug('No account passed. Trying to get it from the RUCIO_ACCOUNT environment variable or the config file.')
try:
self.account = environ['RUCIO_ACCOUNT']
except KeyError:
try:
self.account = config_get('client', 'account')
except (NoOptionError, NoSectionError):
pass
if vo is None:
self.logger.debug('No VO passed. Trying to get it from environment variable RUCIO_VO.')
try:
self.vo = environ['RUCIO_VO']
except KeyError:
self.logger.debug('No VO found. Trying to get it from the config file.')
try:
self.vo = config_get('client', 'vo')
except (NoOptionError, NoSectionError):
self.logger.debug('No VO found. Using default VO.')
self.vo = 'def'
token_filename_suffix = "for_default_account" if self.account is None else "for_account_" + self.account
# if token file path is defined in the rucio.cfg file, use that file. Currently this prevents authenticating as another user or VO.
if self.auth_token_file_path:
self.token_file = self.auth_token_file_path
self.token_path = '/'.join(self.token_file.split('/')[:-1])
else:
self.token_path = self.TOKEN_PATH_PREFIX + getpass.getuser()
if self.vo != 'def':
self.token_path += '@%s' % self.vo
self.token_file = self.token_path + '/' + self.TOKEN_PREFIX + token_filename_suffix
self.token_exp_epoch_file = self.token_path + '/' + self.TOKEN_EXP_PREFIX + token_filename_suffix
self.__authenticate()
try:
self.request_retries = int(config_get('client', 'request_retries'))
except (NoOptionError, RuntimeError):
LOG.debug('request_retries not specified in config file. Taking default.')
except ValueError:
self.logger.debug('request_retries must be an integer. Taking default.')
def test_rucio_version(self):
"""CLIENT(USER): Rucio version"""
cmd = 'bin/rucio --version'
exitcode, out, err = execute(cmd)
nose.tools.assert_equal(err, 'rucio %s\n' % version.version_string())
