default='tst')
}
try:
add_vo(new_vo=vo['vo'],
issuer='super_root',
description='A VO to test multi-vo features',
email='N/A',
vo='def')
except Duplicate:
print('VO {} already added'.format(vo['vo']) % locals())
else:
vo = {}
c = Client()
try:
c.add_account('jdoe', 'SERVICE', '*****@*****.**')
except Duplicate:
print('Account jdoe already added' % locals())
try:
add_account_attribute(
account=InternalAccount('root', **vo), key='admin', value=True
) # bypass client as schema validation fails at API level
except Exception as error:
print(error)
try:
c.add_account('panda', 'SERVICE', '*****@*****.**')
add_account_attribute(account=InternalAccount('panda', **vo),
key='admin',
value=True)
from rucio.client import Client
from rucio.common.exception import Duplicate
if __name__ == '__main__':
info = []
f = open ('tools/atlas_accounts.csv')
# f = open ('user_accounts.csv')
for line in f.readlines():
account, dn, email = line.rstrip().split('\t')
info.append((account, dn, email))
f.close()
c = Client()
for account, dn, email in info:
try:
c.add_account(account=account, type='USER')
except Duplicate:
print 'Account %(account)s already added' % locals()
try:
c.add_identity(account=account, identity=dn, authtype='X509', email=email , default=True)
except Duplicate:
print 'Identity %(account)s already added' % locals()
try:
scope = 'user.' + account
c.add_scope(account, scope)
except Duplicate:
print 'Scope %(scope)s already added' % locals()
except Duplicate:
pass
except Exception:
errno, errstr = sys.exc_info()[:2]
trcbck = traceback.format_exc()
print "Interrupted processing with %s %s %s." % (errno, errstr, trcbck)
sys.exit(CRITICAL)
rses_total -= 1
print "2. Adding %s artificial RSEs finished." % tmp
# 3. Create known users
tmp = users_total
for user in known_users:
print "Adding account %s" % user[0]
try:
c.add_account(user[0], user[1])
c.add_scope(user[0], user[0]) # Adding default scope
except Duplicate:
print "User %s already exists" % user[0]
users_total -= 1
print "3. Adding %s known users finished." % (tmp - users_total)
tmp = users_total
# 4. Fill up DB to total number of Users
while users_total:
try:
c.add_account("user%s" % users_total, "user") # Adding user
c.add_scope("user%s" % users_total, "user%s" % users_total) # Adding default scope
except Duplicate:
print "User user%s already exists" % users_total
users_total -= 1
print "4. Adding %s artificial users (and default scopes) finished." % tmp
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Vincent Garonne, <*****@*****.**>, 2013
from rucio.client import Client
from rucio.common.exception import Duplicate
if __name__ == '__main__':
info = []
f = open ('tools/atlas_groups.csv')
for line in f.readlines():
group = line.rstrip()
info.append(group)
f.close()
c = Client()
for group in info:
try:
account = group.split('.')[1]
c.add_account(account=account, type='GROUP')
except Duplicate:
print 'Account %(account)s already added' % locals()
try:
c.add_scope(account, scope=group)
except Duplicate:
print 'Scope %(group)s already added' % locals()
pass
except Exception:
errno, errstr = sys.exc_info()[:2]
trcbck = traceback.format_exc()
print 'Interrupted processing with %s %s %s.' % (errno, errstr,
trcbck)
sys.exit(CRITICAL)
rses_total -= 1
print '2. Adding %s artificial RSEs finished.' % tmp
# 3. Create known users
tmp = users_total
for user in known_users:
print 'Adding account %s' % user[0]
try:
c.add_account(user[0], user[1], None)
c.add_scope(user[0], user[0]) # Adding default scope
except Duplicate:
print 'User %s already exists' % user[0]
users_total -= 1
print '3. Adding %s known users finished.' % (tmp - users_total)
tmp = users_total
# 4. Fill up DB to total number of Users
while users_total:
try:
c.add_account('user%s' % users_total, 'user', None) # Adding user
c.add_scope('user%s' % users_total,
'user%s' % users_total) # Adding default scope
except Duplicate:
print 'User user%s already exists' % users_total
users_total -= 1
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Vincent Garonne, <*****@*****.**>, 2013
# - Mario Lassnig, <*****@*****.**>, 2014
from rucio.client import Client
from rucio.common.exception import Duplicate
if __name__ == "__main__":
c = Client()
try:
c.add_account("jdoe", "SERVICE")
except Duplicate:
print "Account jdoe already added" % locals()
try:
c.add_scope("jdoe", "mock")
except Duplicate:
print "Scope mock already added" % locals()
# add your accounts here, if you test against CERN authed nodes
additional_test_accounts = [
(
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=mlassnig/CN=663551/CN=Mario Lassnig",
"x509",
"*****@*****.**",
),
def executeForVO(self, vo):
"""
Execute one SE and user synchronisation cycle for a VO.
:param str vo: Virtual organisation name.
:return: S_OK or S_ERROR
:rtype: dict
"""
valid_protocols = ["srm", "gsiftp", "davs", "https", "root"]
default_email = None
try:
try:
client = Client(account="root", auth_type="userpass")
except Exception as err:
self.log.info(
"Login to Rucio as root with password failed. Will try host cert/key",
str(err))
certKeyTuple = Locations.getHostCertificateAndKeyLocation()
if not certKeyTuple:
self.log.error("Hostcert/key location not set")
return S_ERROR("Hostcert/key location not set")
hostcert, hostkey = certKeyTuple
self.log.info("Logging in with a host cert/key pair:")
self.log.debug("account: ",
self.clientConfig[vo]["privilegedAccount"])
self.log.debug("rucio host: ",
self.clientConfig[vo]["rucioHost"])
self.log.debug("auth host: ",
self.clientConfig[vo]["authHost"])
self.log.debug("CA cert path: ", self.caCertPath)
self.log.debug("Cert location: ", hostcert)
self.log.debug("Key location: ", hostkey)
self.log.debug("VO: ", vo)
client = Client(
account=self.clientConfig[vo]["privilegedAccount"],
rucio_host=self.clientConfig[vo]["rucioHost"],
auth_host=self.clientConfig[vo]["authHost"],
ca_cert=self.caCertPath,
auth_type="x509",
creds={
"client_cert": hostcert,
"client_key": hostkey
},
timeout=600,
user_agent="rucio-clients",
vo=vo,
)
self.log.info("Rucio client instantiated for VO:", vo)
# Get the storage elements from Dirac Configuration and create them in Rucio
newRSE = False
self.log.info("Synchronizing SEs for VO ", vo)
result = getStorageElements(vo)
if result["OK"]:
rses = [rse["rse"] for rse in client.list_rses()]
for se in result["Value"]:
if se not in rses:
# The SE doesn't exist. Will create it
newRSE = True
self.log.info(
"Rucio Storage Element does not exist and will be created:",
se)
try:
client.add_rse(rse=se,
deterministic=True,
volatile=False)
except Exception as err:
self.log.error(
"Cannot create RSE",
"[RSE: %s, Error: %s]" % (se, str(err)))
continue
# Add RSE attributes for the new RSE
ret = gConfig.getOptionsDict(
"Resources/FTSEndpoints/FTS3")
ftsList = ""
if ret["OK"]:
ftsList = ",".join(ret["Value"].values())
dictRSEAttributes = {
"naming_convention": "BelleII",
"ANY": True,
"fts": ftsList
}
for key in dictRSEAttributes:
self.log.info(
"Setting RSE attributes",
"[RSE: %s, Attr. name: %s, Value: %s]" %
(se, key, dictRSEAttributes[key]),
)
client.add_rse_attribute(
se, key, value=dictRSEAttributes[key])
client.set_local_account_limit("root", se,
100000000000000000)
# Create the protocols
try:
protocols = client.get_protocols(se)
except RSEProtocolNotSupported as err:
self.log.info("Cannot get protocols for",
"[RSE %s, Error: %s]" % (se, str(err)))
protocols = []
existing_protocols = []
for prot in protocols:
existing_protocols.append(
(str(prot["scheme"]), str(prot["hostname"]),
str(prot["port"]), str(prot["prefix"])))
protocols_to_create = []
for params in result["Value"][se]:
prot = (
str(params["scheme"]),
str(params["hostname"]),
str(params["port"]),
str(params["prefix"]),
)
protocols_to_create.append(prot)
if prot not in existing_protocols and prot[
0] in valid_protocols:
# The protocol defined in Dirac does not exist in Rucio. Will be created
self.log.info(
"Will create new protocol:",
"%s://%s:%s%s on %s" %
(params["scheme"], params["hostname"],
params["port"], params["prefix"], se),
)
try:
client.add_protocol(rse=se, params=params)
except Duplicate as err:
self.log.info(
"Protocol already exists on",
"[RSE: %s, schema:%s]" %
(se, params["scheme"]))
except Exception as err:
self.log.error(
"Cannot create protocol on RSE",
"[RSE: %s, Error: %s]" % (se, str(err)))
else:
update = False
for protocol in protocols:
if prot == (
str(protocol["scheme"]),
str(protocol["hostname"]),
str(protocol["port"]),
str(protocol["prefix"]),
):
# Check if the protocol defined in Dirac has the same priority as the one defined in Rucio
for domain in ["lan", "wan"]:
for activity in [
"read", "write", "delete"
]:
if (params["domains"][domain]
[activity] !=
protocol["domains"][domain]
[activity]):
update = True
break
if (params["domains"]["wan"]
["third_party_copy"] !=
protocol["domains"]["wan"]
["third_party_copy"]):
update = True
if update:
data = {
"prefix":
params["prefix"],
"read_lan":
params["domains"]["lan"]["read"],
"read_wan":
params["domains"]["wan"]["read"],
"write_lan":
params["domains"]["lan"]["write"],
"write_wan":
params["domains"]["wan"]["write"],
"delete_lan":
params["domains"]["lan"]["delete"],
"delete_wan":
params["domains"]["wan"]["delete"],
"third_party_copy":
params["domains"]["wan"]["write"],
}
self.log.info(
"Will update protocol:",
"%s://%s:%s%s on %s" % (
params["scheme"],
params["hostname"],
params["port"],
params["prefix"],
se,
),
)
client.update_protocols(
rse=se,
scheme=params["scheme"],
data=data,
hostname=params["hostname"],
port=params["port"],
)
for prot in existing_protocols:
if prot not in protocols_to_create:
self.log.info(
"Will delete protocol:", "%s://%s:%s%s on %s" %
(prot[0], prot[1], prot[2], prot[3], se))
client.delete_protocols(se,
scheme=prot[0],
hostname=prot[1],
port=prot[2])
else:
self.log.error("Cannot get SEs:", result["Message"])
# If new RSE added, add distances
rses = [rse["rse"] for rse in client.list_rses()]
if newRSE:
self.log.info("Adding distances")
for src_rse, dest_rse in permutations(rses, r=2):
try:
client.add_distance(src_rse, dest_rse, {
"ranking": 1,
"distance": 10
})
except Exception as err:
self.log.error(
"Cannot add distance for",
"Source RSE: %s, Dest RSE: %s, Error:%s" %
(src_rse, dest_rse, str(err)),
)
# Collect the shares from Dirac Configuration and create them in Rucio
self.log.info("Synchronizing shares")
result = Operations().getOptionsDict("Production/SEshares")
if result["OK"]:
rseDict = result["Value"]
for rse in rses:
try:
self.log.info("Setting productionSEshare for",
"[RSE: %s : Share: %s", rse,
rseDict.get(rse, 0))
client.add_rse_attribute(rse, "productionSEshare",
rseDict.get(rse, 0))
except Exception as err:
self.log.error(
"Cannot create productionSEshare for RSE:", rse)
else:
self.log.error("Cannot get SEs", result["Message"])
result = Operations().getSections("Shares")
if result["OK"]:
for dataLevel in result["Value"]:
result = Operations().getOptionsDict("Shares/%s" %
dataLevel)
if not result["OK"]:
self.log.error("Cannot get SEs:" % result["Message"])
continue
rseDict = result["Value"]
for rse in rses:
try:
self.log.info(
"Setting", "%sShare for %s : %s" %
(dataLevel, rse, rseDict.get(rse, 0)))
client.add_rse_attribute(rse,
"%sShare" % dataLevel,
rseDict.get(rse, 0))
except Exception as err:
self.log.error("Cannot create share:",
"%sShare for %s", dataLevel, rse)
else:
self.log.error("Cannot get shares:", result["Message"])
# Create the RSE attribute PrimaryDataSE and OccupancyLFN
result = gConfig.getValue(
"Resources/StorageElementGroups/PrimarySEs")
result = getStorageElements(vo)
if result["OK"]:
allSEs = result["Value"]
primarySEs = resolveSEGroup("PrimarySEs", allSEs)
self.log.info("Will set primarySEs flag to:", str(primarySEs))
for rse in rses:
if rse in allSEs:
storage = StorageElement(rse)
if not storage.valid:
self.log.warn(
"Storage element is not valid. Skipped RSE:",
rse)
continue
occupancyLFN = storage.options.get("OccupancyLFN")
try:
client.add_rse_attribute(rse, "OccupancyLFN",
occupancyLFN)
except Exception as err:
self.log.error(
"Cannot create RSE attribute OccupancyLFN for",
"[RSE: %s, Error: %s]" % (rse, str(err)))
if rse in primarySEs:
try:
client.add_rse_attribute(rse, "PrimaryDataSE",
True)
except Exception as err:
self.log.error(
"Cannot create RSE attribute PrimaryDataSE for",
"[RSE: %s, Error: %s]" % (rse, str(err)),
)
else:
try:
client.delete_rse_attribute(rse, "PrimaryDataSE")
except RSEAttributeNotFound:
pass
except Exception as err:
self.log.error(
"Cannot remove RSE attribute PrimaryDataSE for",
"[RSE: %s, Error: %s]" % (rse, str(err)),
)
self.log.info("RSEs synchronized for VO: ", vo)
# Collect the user accounts from Dirac Configuration and create user accounts in Rucio
self.log.info("Synchronizing accounts for VO", vo)
listAccounts = [
str(acc["account"]) for acc in client.list_accounts()
]
listScopes = [str(scope) for scope in client.list_scopes()]
dnMapping = {}
diracUsers = getUsersInVO(vo)
self.log.debug(" Will consider following Dirac users for",
"[VO: %s, Dirac users: %s]" % (vo, diracUsers))
for account in diracUsers:
dn = getUserOption(account, "DN")
email = getUserOption(account, "Email")
dnMapping[dn] = email
if account not in listAccounts:
self.log.info("Will create account with associated DN ",
"[account: %s, DN: %s]" % (account, dn))
try:
client.add_account(account, "USER", email)
listAccounts.append(account)
except Exception as err:
self.log.error(
"Cannot create account",
"[account: %s, Error: %s]" % (account, str(err)))
try:
client.add_identity(account=account,
identity=dn,
authtype="X509",
email=email,
default=True)
except Exception as err:
self.log.error(
"Cannot add identity for account",
"[Identity: dn=%s, account:%s, Error: %s]" %
(dn, account, str(err)),
)
self.log.error(
" Account/identity skipped (it will not be created in Rucio)",
"[%s/%s]" % (account, dn))
continue
for rse in rses:
client.set_local_account_limit(account, rse,
1000000000000000)
else:
try:
client.add_identity(account=account,
identity=dn,
authtype="X509",
email=email,
default=True)
except Duplicate:
pass
except Exception as err:
self.log.error(
"Cannot create identity for account",
"[DN: %s, account: %s, Error: %s]" %
(dn, account, str(err)),
)
scope = "user." + account
if scope not in listScopes:
try:
self.log.info("Will create a scope",
"[Scope: %s]" % scope)
client.add_scope(account, scope)
self.log.info("Scope successfully added",
"[Scope: %s]" % scope)
except Exception as err:
self.log.error(
"Cannot create a scope",
"[Scope: %s, Error: %s]" % (scope, str(err)))
# Collect the group accounts from Dirac Configuration and create service accounts in Rucio
result = getGroupsForVO(vo)
if result["OK"]:
groups = result["Value"]
self.log.debug(" Will consider following Dirac groups for",
"[%s VO: %s]" % (vo, groups))
else:
groups = []
self.log.debug("No Dirac groups for", "%s VO " % vo)
self.log.debug("No Rucio service accounts will be created")
for group in groups:
if group not in listAccounts:
self.log.info(
"Will create SERVICE account for Dirac group:",
str(group))
try:
client.add_account(group, "SERVICE", None)
listAccounts.append(group)
except Exception as err:
self.log.error(
"Cannot create SERVICE account for",
"[group: %s, Error: %s]" % (group, str(err)))
for rse in rses:
client.set_local_account_limit(account, rse,
1000000000000000)
for dn in getDNsInGroup(group):
try:
client.add_identity(account=group,
identity=dn,
authtype="X509",
email=dnMapping.get(
dn, default_email))
except Duplicate:
pass
except Exception as err:
self.log.error(
"Cannot create identity for account",
"[identity %s, account %s, Error: %s]" %
(dn, group, str(err)),
)
self.log.error(format_exc())
# Collect the group accounts from Dirac Configuration and create service accounts in Rucio
result = getHosts()
if not result["OK"]:
self.log.error("Cannot get host accounts:",
"%s" % result["Message"])
else:
hosts = result["Value"]
for host in hosts:
dn = getHostOption(host, "DN")
email = dnMapping.get(dn, default_email)
try:
client.add_identity(account="dirac_srv",
identity=dn,
authtype="X509",
email=email)
except Duplicate:
pass
except Exception as err:
self.log.error(
"Cannot create identity for account dirac_srv:",
"[DN: %s, Error: %s]" % (dn, str(err)))
self.log.error(format_exc())
return S_OK()
except Exception as exc:
self.log.exception("Synchronisation for VO failed. VO skipped ",
"VO=%s" % vo,
lException=exc)
return S_ERROR(str(format_exc()))
#!/usr/bin/env python
from rucio.client import Client
from rucio.common.exception import Duplicate
from rucio.core.account import add_account_attribute
from rucio.common.types import InternalAccount
if __name__ == '__main__':
c = Client()
try:
c.add_account('abruzzese', 'SERVICE', '*****@*****.**')
add_account_attribute(account=InternalAccount('abruzzese'),
key='admin',
value=True)
except Duplicate:
print('Account abruzzese already added' % locals())
try:
add_account_attribute(
account=InternalAccount('root'), key='admin', value=True
) # bypass client as schema validation fails at API level
except Exception as error:
print(error)
try:
c.add_identity(account='abruzzese',
identity='bruzzese_user',
authtype='userpass',
password='******',
email='*****@*****.**')
except Exception:
print('Already added: ')
try:
pass
except Exception:
errno, errstr = sys.exc_info()[:2]
trcbck = traceback.format_exc()
print 'Interrupted processing with %s %s %s.' % (errno, errstr,
trcbck)
sys.exit(CRITICAL)
rses_total -= 1
print '2. Adding %s artificial RSEs finished.' % tmp
# 3. Create known users
tmp = users_total
for user in known_users:
print 'Adding account %s' % user[0]
try:
c.add_account(user[0], user[1])
c.add_scope(user[0], user[0]) # Adding default scope
except Duplicate:
print 'User %s already exists' % user[0]
users_total -= 1
print '3. Adding %s known users finished.' % (tmp - users_total)
tmp = users_total
# 4. Fill up DB to total number of Users
while users_total:
try:
c.add_account('user%s' % users_total, 'user') # Adding user
c.add_scope('user%s' % users_total,
'user%s' % users_total) # Adding default scope
except Duplicate:
print 'User user%s already exists' % users_total
users_total -= 1
