def test_add_key():
""" META (CORE): Add a new key """
types = [{'type': DIDType.FILE, 'expected': KeyType.FILE},
{'type': DIDType.CONTAINER, 'expected': KeyType.CONTAINER},
{'type': DIDType.DATASET, 'expected': KeyType.DATASET},
{'type': KeyType.ALL, 'expected': KeyType.ALL},
{'type': KeyType.DERIVED, 'expected': KeyType.DERIVED},
{'type': KeyType.FILE, 'expected': KeyType.FILE},
{'type': KeyType.COLLECTION, 'expected': KeyType.COLLECTION},
{'type': KeyType.CONTAINER, 'expected': KeyType.CONTAINER},
{'type': KeyType.DATASET, 'expected': KeyType.DATASET},
{'type': 'FILE', 'expected': KeyType.FILE},
{'type': 'ALL', 'expected': KeyType.ALL},
{'type': 'COLLECTION', 'expected': KeyType.COLLECTION},
{'type': 'DATASET', 'expected': KeyType.DATASET},
{'type': 'D', 'expected': KeyType.DATASET},
{'type': 'FILE', 'expected': KeyType.FILE},
{'type': 'F', 'expected': KeyType.FILE},
{'type': 'DERIVED', 'expected': KeyType.DERIVED},
{'type': 'C', 'expected': KeyType.CONTAINER}]
for key_type in types:
key_name = 'datatype%s' % str(uuid())
add_key(key_name, key_type['type'])
stored_key_type = session.get_session().query(models.DIDKey).filter_by(key=key_name).one()['key_type']
assert stored_key_type, key_type['expected']
with pytest.raises(UnsupportedKeyType):
add_key('datatype', DIDType.ARCHIVE)
with pytest.raises(UnsupportedKeyType):
add_key('datatype', 'A')
def test_repair_a_rule_with_source_replica_expression(self):
""" JUDGE EVALUATOR: Test the judge when a with two rules with source_replica_expression"""
scope = InternalScope('mock', **self.vo)
files = create_files(3, scope, self.rse4_id)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.DATASET, self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
# Add a first rule to the DS
rule_id1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0]
rule_id2 = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=self.rse3, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, source_replica_expression=self.rse1)[0]
assert(RuleState.REPLICATING == get_rule(rule_id1)['state'])
assert(RuleState.STUCK == get_rule(rule_id2)['state'])
successful_transfer(scope=scope, name=files[0]['name'], rse_id=self.rse1_id, nowait=False)
successful_transfer(scope=scope, name=files[1]['name'], rse_id=self.rse1_id, nowait=False)
successful_transfer(scope=scope, name=files[2]['name'], rse_id=self.rse1_id, nowait=False)
# Also make replicas AVAILABLE
session = get_session()
replica = session.query(models.RSEFileAssociation).filter_by(scope=scope, name=files[0]['name'], rse_id=self.rse1_id).one()
replica.state = ReplicaState.AVAILABLE
replica = session.query(models.RSEFileAssociation).filter_by(scope=scope, name=files[1]['name'], rse_id=self.rse1_id).one()
replica.state = ReplicaState.AVAILABLE
replica = session.query(models.RSEFileAssociation).filter_by(scope=scope, name=files[2]['name'], rse_id=self.rse1_id).one()
replica.state = ReplicaState.AVAILABLE
session.commit()
rule_repairer(once=True)
assert(RuleState.OK == get_rule(rule_id1)['state'])
assert(RuleState.REPLICATING == get_rule(rule_id2)['state'])
def db_session():
from rucio.db.sqla import session
db_session = session.get_session()
yield db_session
db_session.commit()
db_session.close()
def get_traffic_from_db():
"""
Gets the size of the current requests
for each link.
"""
session = get_session()
collector = []
query = '''SELECT
SUM(bytes),
atlas_rucio.id2rse(source_rse_id),
atlas_rucio.id2rse(dest_rse_id)
FROM atlas_rucio.requests WHERE
(state='D' or
state='S' or
state='F' or
state='L')
group by source_rse_id, dest_rse_id'''
try:
result = session.execute(query)
for row in result:
link = {'bytes': row[0], 'src_rse': row[1], 'dst_rse': row[2]}
collector.append(link)
except Exception, exception:
print exception
sys.exit()
def count_expired_tokens(account):
session = get_session()
result = session.query(models.Token).filter(and_(models.Token.account == account, # pylint: disable=no-member
models.Token.expired_at <= datetime.datetime.utcnow()))\
.all()
count = len(result)
return count
def get_token_count(account):
session = get_session()
result = session.query(models.Token).filter_by(account=account).all() # pylint: disable=no-member
for token in result:
print(token.token, token.expired_at, token.refresh_token,
token.refresh_expired_at, token.oidc_scope)
return len(result)
def setUpClass(cls):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {
'vo':
config_get('client',
'vo',
raise_exception=False,
default='tst')
}
cls.multi_vo = True
else:
cls.vo = {}
cls.multi_vo = False
# Add test account
cls.account = InternalAccount(
''.join(random.choice(string.ascii_uppercase) for x in range(10)),
**cls.vo)
add_account(account=cls.account,
type=AccountType.USER,
email='*****@*****.**')
# Add test RSE
cls.rse1 = 'MOCK'
cls.rse2 = 'MOCK2'
cls.rse1_id = get_rse_id(rse=cls.rse1, **cls.vo)
cls.rse2_id = get_rse_id(rse=cls.rse2, **cls.vo)
cls.db_session = session.get_session()
def convert_to_svo(old_vo, delete_vos=False, commit_changes=False, skip_history=False, echo=True):
"""
Converts a multi-VO database to a single-VO one by renaming the given VO and (optionally) deleting entries for other VOs and the super_root.
Intended to be run on a copy of the original database that contains several VOs.
:param old_vo: The 3 character string for the old VO.
:param delete_vos: If True then all entries associated with a VO other than `old_vo` will be deleted.
:param commit_changes: If True then changes are made against the database directly and the old super_root account will be (soft) deleted.
If False, then nothing is commited and the commands needed are dumped to be run later.
:param skip_history: If True then tables without FKC containing historical data will not be converted to save time.
"""
if not config_get_bool('common', 'multi_vo', False, False):
print('Multi-VO mode is not enabled in the config file, aborting conversion.')
return
rename_vo(old_vo, 'def', commit_changes=commit_changes, skip_history=skip_history)
s = session.get_session()
if delete_vos:
success_all = True
for vo in list_vos(session=s):
if vo['vo'] != 'def':
success = remove_vo(vo['vo'], commit_changes=commit_changes, skip_history=skip_history)
success_all = success_all and success
if commit_changes and success_all:
del_account(InternalAccount('super_root', vo='def'), session=s)
s.close()
def save_oidc_token(account,
lifetime_access=0,
lifetime_refresh=0,
refresh_token=None,
refresh=False,
final_state=None):
session = get_session()
expired_at = datetime.datetime.utcnow() + datetime.timedelta(
seconds=lifetime_access)
refresh_expired_at = None
if lifetime_refresh > 0:
refresh_expired_at = datetime.datetime.utcnow() + datetime.timedelta(
seconds=lifetime_refresh)
if lifetime_refresh == 0 and refresh_token:
refresh_expired_at = datetime.datetime.utcnow()
new_token = models.Token(account=account,
token=rndstr(),
refresh_token=refresh_token,
refresh=refresh,
oidc_scope=json.dumps({'state': final_state}),
expired_at=expired_at,
refresh_expired_at=refresh_expired_at,
identity="SUB=myid, ISS=mockiss")
new_token.save(session=session)
session.commit() # pylint: disable=no-member
session.expunge(new_token) # pylint: disable=no-member
return None
def convert_to_mvo(new_vo, description, email, create_super_root=False, commit_changes=False, skip_history=False, echo=True):
"""
Converts a single-VO database to a multi-VO one with the specified VO details.
:param new_vo: The 3 character string for the new VO.
:param description: Full description of the new VO.
:param email: Admin email for the new VO.
:param create_super_root: If True and the renaming was successful, then create a super_root account at VO def.
:param commit_changes: If True then changes are made against the database directly.
If False, then nothing is commited and the commands needed are dumped to be run later.
:param skip_history: If True then tables without FKC containing historical data will not be converted to save time.
"""
if not config_get_bool('common', 'multi_vo', False, False):
print('Multi-VO mode is not enabled in the config file, aborting conversion.')
return
s = session.get_session()
vos = [vo['vo'] for vo in list_vos(session=s)]
if new_vo not in vos:
insert_new_vo = True
else:
insert_new_vo = False
success = rename_vo('def', new_vo, insert_new_vo=insert_new_vo, description=description, email=email,
commit_changes=commit_changes, skip_history=skip_history)
if create_super_root and success:
create_root_account(create_counters=False)
s.close()
def reset_config_table():
""" Clear the config table and install any default entires needed for the tests.
"""
db_session = session.get_session()
db_session.query(models.Config).delete()
db_session.commit()
config_db.set("vo-map", "testvo1", "tst")
config_db.set("vo-map", "testvo2", "ts2")
def test_db_connection():
""" DB (CORE): Test db connection """
session = get_session()
if session.bind.dialect.name == 'oracle':
session.execute('select 1 from dual')
else:
session.execute('select 1')
session.close()
def setUp(self):
self.session = get_session()
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
self.vo = {}
self.tmp_scope = InternalScope('mock', **self.vo)
self.root = InternalAccount('root', **self.vo)
def setUp(self):
self.account = 'root'
self.scope = 'mock'
self.upload_client = UploadClient()
self.file_sizes = 2
self.rse = 'MOCK4'
self.rse_id = get_rse(self.rse).id
self.session = get_session()
def setUp(self):
self.scope = InternalScope('mock')
self.rse = 'MOCK4'
self.rse2 = 'MOCK3'
self.account = InternalAccount('root')
self.rse_id = get_rse_id(self.rse)
self.rse2_id = get_rse_id(self.rse2)
self.db_session = session.get_session()
def setUp(self):
self.scope = 'mock'
self.rse = 'MOCK4'
self.rse2 = 'MOCK3'
self.account = 'root'
self.rse_id = get_rse_id(self.rse)
self.rse2_id = get_rse_id(self.rse2)
self.db_session = session.get_session()
def reset_rses():
db_session = session.get_session()
for rse in db_session.query(models.RSE).all():
rse.deleted = False
rse.deleted_at = None
rse.save(session=db_session)
add_rse_attribute(rse=rse['rse'], key=rse['rse'], value=True, session=db_session)
db_session.commit()
def count_refresh_tokens_expired_or_none(account):
session = get_session()
result = session.query(models.Token).filter(and_(models.Token.account == account))\
.filter(or_(models.Token.refresh_expired_at.__eq__(None), models.Token.refresh_expired_at <= datetime.datetime.utcnow()))\
.all() # pylint: disable=no-member
count = len(result)
return count
def setUpClass(cls):
cls.upload_client = UploadClient()
cls.session = get_session()
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
cls.rse_id = get_rse_id(cls.rse, session=cls.session, **cls.vo)
def create_root_account():
""" Inserts the default root account to an existing database. Make sure to change the default password later. """
up_id = 'ddmlab'
up_pwd = '2ccee6f6dd1bc2269cddd7cd5e47578e98e430539807c36df23fab7dd13e7583'
up_email = '*****@*****.**'
x509_id = '/C=CH/ST=Geneva/O=CERN/OU=PH-ADP-CO/CN=DDMLAB Client Certificate/[email protected]'
x509_email = '*****@*****.**'
gss_id = '*****@*****.**'
gss_email = '*****@*****.**'
ssh_id = 'ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq5LySllrQFpPL614sulXQ7wnIr1aGhGtl8b+HCB/'\
'0FhMSMTHwSjX78UbfqEorZV16rXrWPgUpvcbp2hqctw6eCbxwqcgu3uGWaeS5A0iWRw7oXUh6ydn'\
'Vy89zGzX1FJFFDZ+AgiZ3ytp55tg1bjqqhK1OSC0pJxdNe878TRVVo5MLI0S/rZY2UovCSGFaQG2'\
'iLj14wz/YqI7NFMUuJFR4e6xmNsOP7fCZ4bGMsmnhR0GmY0dWYTupNiP5WdYXAfKExlnvFLTlDI5'\
'Mgh4Z11NraQ8pv4YE1woolYpqOc/IMMBBXFniTT4tC7cgikxWb9ZmFe+r4t6yCDpX4IL8L5GOQ== ddmlab'
ssh_email = '*****@*****.**'
try:
up_id = config_get('bootstrap', 'userpass_identity')
up_pwd = config_get('bootstrap', 'userpass_pwd')
up_email = config_get('bootstrap', 'userpass_email')
x509_id = config_get('bootstrap', 'x509_identity')
x509_email = config_get('bootstrap', 'x509_email')
gss_id = config_get('bootstrap', 'gss_identity')
gss_email = config_get('bootstrap', 'gss_email')
ssh_id = config_get('bootstrap', 'ssh_identity')
ssh_email = config_get('bootstrap', 'ssh_email')
except:
pass
# print 'Config values are missing (check rucio.cfg{.template}). Using hardcoded defaults.'
s = session.get_session()
account = models.Account(account=InternalAccount('root'), account_type=AccountType.SERVICE, status=AccountStatus.ACTIVE)
identity1 = models.Identity(identity=up_id, identity_type=IdentityType.USERPASS, password=up_pwd, salt='0', email=up_email)
iaa1 = models.IdentityAccountAssociation(identity=identity1.identity, identity_type=identity1.identity_type, account=account.account, is_default=True)
# X509 authentication
identity2 = models.Identity(identity=x509_id, identity_type=IdentityType.X509, email=x509_email)
iaa2 = models.IdentityAccountAssociation(identity=identity2.identity, identity_type=identity2.identity_type, account=account.account, is_default=True)
# GSS authentication
identity3 = models.Identity(identity=gss_id, identity_type=IdentityType.GSS, email=gss_email)
iaa3 = models.IdentityAccountAssociation(identity=identity3.identity, identity_type=identity3.identity_type, account=account.account, is_default=True)
# SSH authentication
identity4 = models.Identity(identity=ssh_id, identity_type=IdentityType.SSH, email=ssh_email)
iaa4 = models.IdentityAccountAssociation(identity=identity4.identity, identity_type=identity4.identity_type, account=account.account, is_default=True)
# Account counters
create_counters_for_new_account(account=account.account, session=s)
# Apply
s.add_all([account, identity1, identity2, identity3, identity4])
s.commit()
s.add_all([iaa1, iaa2, iaa3, iaa4])
s.commit()
def create_base_vo():
""" Creates the base VO """
s = get_session()
vo = models.VO(vo='def', description='Default base VO', email='N/A')
s.add_all([vo])
s.commit()
def setUp(self):
self.account = InternalAccount('root')
self.scope = InternalScope('mock')
self.upload_client = UploadClient()
self.account_client = AccountClient()
self.file_sizes = 2
self.rse = 'MOCK4'
self.rse_id = get_rse_id(self.rse)
self.session = get_session()
def setUpClass(cls):
cls.account = InternalAccount('jdoe')
cls.rse_1_name = 'MOCK4'
cls.rse_2_name = 'MOCK5'
cls.mock1_id = get_rse_id(cls.rse_1_name)
cls.mock2_id = get_rse_id(cls.rse_2_name)
cls.db_session = session.get_session()
cls.rse_1 = {'id': cls.mock1_id, 'staging_area': False}
cls.rse_2 = {'id': cls.mock2_id, 'staging_area': False}
def core_config_mock(request):
"""
Fixture to allow having per-test core.config tables without affecting the other parallel tests.
This override works only in tests which use core function calls directly, not in the ones working
via the API, because the normal config table is not touched and the rucio instance answering API
calls is not aware of this mock.
This fixture acts by creating a new copy of the "config" sql table using the :memory: sqlite engine.
Accesses to the "models.Config" table are then redirected to this temporary table via mock.patch().
"""
from unittest import mock
from rucio.common.utils import generate_uuid
from sqlalchemy.pool import StaticPool
from rucio.db.sqla.models import ModelBase, BASE, Column, String, PrimaryKeyConstraint
from rucio.db.sqla.session import get_session, get_maker, get_engine, create_engine, declarative_base
# Get the fixture parameters
table_content = []
params = __get_fixture_param(request)
if params:
table_content = params.get("table_content", table_content)
# Create an in-memory dropdown replacement table for the "models.Config" table
engine = create_engine('sqlite://',
connect_args={'check_same_thread': False},
poolclass=StaticPool)
InMemoryBase = declarative_base(bind=engine)
class InMemoryConfig(InMemoryBase, ModelBase):
__tablename__ = 'configs_' + generate_uuid()
section = Column(String(128))
opt = Column(String(128))
value = Column(String(4000))
_table_args = (PrimaryKeyConstraint('section',
'opt',
name='CONFIGS_PK'), )
InMemoryBase.metadata.create_all()
# Register the new table with the associated engine into the sqlalchemy sessionmaker
# In theory, this code must be protected by rucio.db.scla.session._LOCK, but this code will be executed
# during test case initialization, so there is no risk here to have concurrent calls from within the
# same process
current_engine = get_engine()
get_maker().configure(binds={BASE: current_engine, InMemoryBase: engine})
# Fill the table with the requested mock data
session = get_session()()
for section, option, value in (table_content or []):
InMemoryConfig(section=section, opt=option,
value=value).save(flush=True, session=session)
session.commit()
with mock.patch('rucio.core.config.models.Config', new=InMemoryConfig):
yield
def check_deleted_tokens(account):
session = get_session()
result = session.query(models.Token).filter_by(account=account).all() # pylint: disable=no-member
all_deleted = True
for elem in result:
if elem.refresh_token is not None:
if elem.refresh_token not in str(elem.oidc_scope):
if 'deleted' in str(elem.oidc_scope):
all_deleted = False
return all_deleted
def test_fill_counter_history(self):
"""ACCOUNT COUNTER (CORE): Fill the usage history with the current value."""
db_session = session.get_session()
db_session.query(models.AccountUsageHistory).delete()
db_session.commit()
account_counter.fill_account_counter_history_table()
history_usage = [(usage['rse_id'], usage['files'], usage['account'], usage['bytes']) for usage in db_session.query(models.AccountUsageHistory)]
current_usage = [(usage['rse_id'], usage['files'], usage['account'], usage['bytes']) for usage in db_session.query(models.AccountUsage)]
for usage in history_usage:
assert usage in current_usage
def test_fill_counter_history(self):
"""RSE COUNTER (CORE): Fill the usage history with the current value."""
db_session = session.get_session()
db_session.query(models.RSEUsageHistory).delete()
db_session.commit()
rse_counter.fill_rse_counter_history_table()
history_usage = [(usage['rse_id'], usage['files'], usage['source'], usage['used']) for usage in db_session.query(models.RSEUsageHistory)]
current_usage = [(usage['rse_id'], usage['files'], usage['source'], usage['used']) for usage in db_session.query(models.RSEUsage)]
for usage in history_usage:
assert usage in current_usage
def setUpClass(cls):
cls.db_session = session.get_session()
cls.dialect = cls.db_session.bind.dialect.name
cls.dest_rse = 'MOCK'
cls.source_rse = 'MOCK4'
cls.dest_rse_id = get_rse_id(cls.dest_rse)
cls.source_rse_id = get_rse_id(cls.source_rse)
cls.scope = InternalScope('mock')
cls.account = InternalAccount('root')
cls.user_activity = 'User Subscription'
cls.all_activities = 'all_activities'
def purge_bin():
try:
session = get_session()
sql = "select table_name from user_tables"
for table in session.execute(sql):
query = "drop table %s cascade constraints purge" % table[0]
session.execute(query)
except:
pass
finally:
session.remove()
def setUpClass(cls):
cls.db_session = session.get_session()
cls.dest_rse = 'MOCK'
cls.source_rse = 'MOCK4'
cls.dest_rse_id = get_rse_id(cls.dest_rse)
cls.source_rse_id = get_rse_id(cls.source_rse)
cls.scope = InternalScope('mock')
cls.account = InternalAccount('root')
cls.user_activity = 'User Subscription'
cls.all_activities = 'all_activities'
set_rse_transfer_limits(cls.dest_rse_id, cls.user_activity, max_transfers=1, session=cls.db_session)
set('throttler_release_strategy', 'dest_%s' % cls.dest_rse_id, 'fifo', session=cls.db_session)
