def setup(self):
'''
setup
'''
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': 'tst'}
else:
self.vo = {}
self.cacert = config_get('test', 'cacert')
self.marker = '$> '
def __setup_new_vo():
multi_vo = config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False)
if not multi_vo:
pytest.skip(
'multi_vo mode is not enabled. Running multi_vo tests in single_vo mode would result in failures.'
)
new_vo = 'new'
if not vo_core.vo_exists(vo=new_vo):
vo_core.add_vo(vo=new_vo, description='Test', email='*****@*****.**')
return new_vo
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {
'vo':
config_get('client',
'vo',
raise_exception=False,
default='tst')
}
else:
self.vo = {}
def setUpClass(cls):
cls.upload_client = UploadClient()
cls.account_client = AccountClient()
cls.session = get_session()
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
cls.account = InternalAccount('root', **cls.vo)
cls.scope = InternalScope('mock', **cls.vo)
cls.rse_id = get_rse_id(cls.rse, session=cls.session, **cls.vo)
def setUpClass(cls):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
cls.vo = {}
cls.account = InternalAccount('jdoe', **cls.vo)
cls.rse_1_name = 'MOCK4'
cls.rse_2_name = 'MOCK5'
cls.mock1_id = get_rse_id(cls.rse_1_name, **cls.vo)
cls.mock2_id = get_rse_id(cls.rse_2_name, **cls.vo)
cls.db_session = session.get_session()
cls.rse_1 = {'id': cls.mock1_id, 'staging_area': False}
cls.rse_2 = {'id': cls.mock2_id, 'staging_area': False}
def query_policy_packages(self):
from rucio.core.vo import list_vos
try:
multivo = config.config_get_bool('common', 'multi_vo')
except (NoOptionError, NoSectionError):
multivo = False
if not multivo:
# single policy package
self.try_importing_policy()
else:
# policy package per VO
vos = list_vos()
for vo in vos:
self.try_importing_policy(vo)
def setup(self):
'''
setup
'''
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo_header = '-H "X-Rucio-VO: tst"'
else:
self.vo_header = ''
self.cacert = config_get('test', 'cacert')
self.usercert = config_get('test', 'usercert')
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.marker = '$> '
def test_core_temporary_dids():
""" TMP DATA IDENTIFIERS (CORE): """
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
vo = {
'vo': config_get('client',
'vo',
raise_exception=False,
default='tst')
}
else:
vo = {}
scope = InternalScope('mock', **vo)
root = InternalAccount('root', **vo)
temporary_dids = []
rse = 'MOCK'
rse_id = get_rse_id(rse=rse, **vo)
for _ in range(10):
temporary_dids.append({
'scope': scope,
'name': 'object_%s' % generate_uuid(),
'rse_id': rse_id,
'bytes': 1,
'path': None
})
add_temporary_dids(dids=temporary_dids, account=root)
compose(scope=scope,
name='file_%s' % generate_uuid(),
rse_id=rse_id,
bytes=10,
sources=temporary_dids,
account=root,
md5=None,
adler32=None,
pfn=None,
meta={},
rules=[],
parent_scope=None,
parent_name=None)
dids = list_expired_temporary_dids(rse_id=rse_id, limit=10)
rowcount = delete_temporary_dids(dids=dids)
assert rowcount == 10
def add_vo(vo, description, email, session=None):
"""
Add a VO and setup a new root user.
New root user will have account name 'root' and a userpass identity with username: '******' and password: '******'
:param vo: 3-letter unique tag for a VO.
:param descrition: Descriptive string for the VO (e.g. Full name).
:param email: Contact email for the VO.
:param session: The db session in use.
"""
if not config_get_bool(
'common', 'multi_vo', raise_exception=False, default=False):
raise exception.UnsupportedOperation(
'VO operations cannot be performed in single VO mode.')
if len(vo) != 3:
raise exception.RucioException('Invalid VO tag, must be 3 chars.')
new_vo = models.VO(vo=vo, description=description, email=email)
try:
new_vo.save(session=session)
except IntegrityError:
raise exception.Duplicate('VO {} already exists!'.format(vo))
except DatabaseError as error:
raise exception.RucioException(error.args)
from rucio.core.account import add_account, list_identities
from rucio.core.identity import add_account_identity
new_root = InternalAccount('root', vo=vo)
add_account(account=new_root,
type=AccountType['SERVICE'],
email=email,
session=session)
add_account_identity(identity='root@{}'.format(vo),
type=IdentityType['USERPASS'],
account=new_root,
email=email,
default=False,
password='******',
session=session)
for ident in list_identities(account=InternalAccount('super_root',
vo='def'),
session=session):
add_account_identity(identity=ident['identity'],
type=ident['type'],
account=new_root,
email='',
session=session)
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': get_vo()}
self.filter = {'filter_': self.vo}
else:
self.vo = {}
self.filter = {'filter_': {'vo': 'def'}}
self.rse1 = rse_name_generator()
self.rse2 = rse_name_generator()
self.rse3 = rse_name_generator()
self.rse4 = rse_name_generator()
self.rse5 = rse_name_generator()
self.rse1_id = rse.add_rse(self.rse1, **self.vo)
self.rse2_id = rse.add_rse(self.rse2, **self.vo)
self.rse3_id = rse.add_rse(self.rse3, **self.vo)
self.rse4_id = rse.add_rse(self.rse4, **self.vo)
self.rse5_id = rse.add_rse(self.rse5, **self.vo)
# Add Attributes
self.attribute = attribute_name_generator()
rse.add_rse_attribute(self.rse1_id, self.attribute, "at")
rse.add_rse_attribute(self.rse2_id, self.attribute, "de")
rse.add_rse_attribute(self.rse3_id, self.attribute, "fr")
rse.add_rse_attribute(self.rse4_id, self.attribute, "uk")
rse.add_rse_attribute(self.rse5_id, self.attribute, "us")
# Add numeric Attributes
self.attribute_numeric = attribute_name_generator()
rse.add_rse_attribute(self.rse1_id, self.attribute_numeric, 10)
rse.add_rse_attribute(self.rse2_id, self.attribute_numeric, 20)
rse.add_rse_attribute(self.rse3_id, self.attribute_numeric, 30)
rse.add_rse_attribute(self.rse4_id, self.attribute_numeric, 40)
rse.add_rse_attribute(self.rse5_id, self.attribute_numeric, 50)
# Add Tags
self.tag1 = tag_generator()
self.tag2 = tag_generator()
rse.add_rse_attribute(self.rse1_id, self.tag1, True)
rse.add_rse_attribute(self.rse2_id, self.tag1, True)
rse.add_rse_attribute(self.rse3_id, self.tag1, True)
rse.add_rse_attribute(self.rse4_id, self.tag2, True)
rse.add_rse_attribute(self.rse5_id, self.tag2, True)
def setUpClass(cls):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
cls.vo = {'vo': get_vo()}
else:
cls.vo = {}
@transactional_session
def __cleanup_updated_dids(session=None):
session.query(UpdatedDID).delete()
__cleanup_updated_dids()
# Add test RSE
cls.rse1 = 'MOCK'
cls.rse3 = 'MOCK3'
cls.rse4 = 'MOCK4'
cls.rse5 = 'MOCK5'
cls.rse1_id = get_rse_id(rse=cls.rse1, **cls.vo)
cls.rse3_id = get_rse_id(rse=cls.rse3, **cls.vo)
cls.rse4_id = get_rse_id(rse=cls.rse4, **cls.vo)
cls.rse5_id = get_rse_id(rse=cls.rse5, **cls.vo)
# Add Tags
cls.T1 = tag_generator()
cls.T2 = tag_generator()
add_rse_attribute(cls.rse1_id, cls.T1, True)
add_rse_attribute(cls.rse3_id, cls.T1, True)
add_rse_attribute(cls.rse4_id, cls.T2, True)
add_rse_attribute(cls.rse5_id, cls.T1, True)
# Add fake weights
add_rse_attribute(cls.rse1_id, "fakeweight", 10)
add_rse_attribute(cls.rse3_id, "fakeweight", 0)
add_rse_attribute(cls.rse4_id, "fakeweight", 0)
add_rse_attribute(cls.rse5_id, "fakeweight", 0)
# Add quota
cls.jdoe = InternalAccount('jdoe', **cls.vo)
cls.root = InternalAccount('root', **cls.vo)
set_local_account_limit(cls.jdoe, cls.rse1_id, -1)
set_local_account_limit(cls.jdoe, cls.rse3_id, -1)
set_local_account_limit(cls.jdoe, cls.rse4_id, -1)
set_local_account_limit(cls.jdoe, cls.rse5_id, -1)
set_local_account_limit(cls.root, cls.rse1_id, -1)
set_local_account_limit(cls.root, cls.rse3_id, -1)
set_local_account_limit(cls.root, cls.rse4_id, -1)
set_local_account_limit(cls.root, cls.rse5_id, -1)
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo_header = '-H "X-Rucio-VO: %s"' % get_long_vo()
else:
self.vo_header = ''
self.cacert = config_get('test', 'cacert')
self.usercert = config_get('test', 'usercert')
self.userkey = config_get('test', 'userkey')
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.marker = '$> '
def setUpClass(cls):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo_header = {'X-Rucio-VO': 'tst'}
cls.vo = {'vo': 'tst'}
else:
cls.vo_header = {}
cls.vo = {}
cls.projects = [
'data12_900GeV', 'data12_8TeV', 'data13_900GeV', 'data13_8TeV'
]
cls.pattern1 = r'(_tid|physics_(Muons|JetTauEtmiss|Egamma)\..*\.ESD|express_express(?!.*NTUP|.*\.ESD|.*RAW)|(physics|express)(?!.*NTUP).* \
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo_header = {
'X-Rucio-VO':
config_get('client',
'vo',
raise_exception=False,
default='tst')
}
else:
self.vo_header = {}
self.scopes = [scope_name_generator() for _ in range(5)]
def setUpClass(cls):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
else:
cls.vo = {}
cls.sub_client = SubscriptionClient()
cls.did_client = DIDClient()
cls.projects = [
'data12_900GeV', 'data12_8TeV', 'data13_900GeV', 'data13_8TeV'
]
cls.pattern1 = r'(_tid|physics_(Muons|JetTauEtmiss|Egamma)\..*\.ESD|express_express(?!.*NTUP|.*\.ESD|.*RAW)|(physics|express)(?!.*NTUP).* \
def setUp(self):
self.rse = 'MOCK4'
self.file_sizes = 2
self.upload_client = UploadClient()
self.account_client = AccountClient()
self.session = get_session()
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
self.vo = {}
self.account = InternalAccount('root', **self.vo)
self.scope = InternalScope('mock', **self.vo)
self.rse_id = get_rse_id(self.rse, session=self.session, **self.vo)
def setUpClass(cls):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
cls.vo = {'vo': get_vo()}
else:
cls.vo = {}
cls.db_session = session.get_session()
cls.dest_rse = 'MOCK'
cls.dest_rse2 = 'MOCK2'
cls.source_rse = 'MOCK4'
cls.source_rse2 = 'MOCK5'
cls.dest_rse_id = get_rse_id(cls.dest_rse, **cls.vo)
cls.dest_rse_id2 = get_rse_id(cls.dest_rse2, **cls.vo)
cls.source_rse_id = get_rse_id(cls.source_rse, **cls.vo)
cls.source_rse_id2 = get_rse_id(cls.source_rse2, **cls.vo)
def setUpClass(cls):
cls.dataset = 'dataset_%s' % generate_uuid()
cls.rule_client = RuleClient()
cls.did_client = DIDClient()
cls.replica_client = ReplicaClient()
cls.upload_client = UploadClient()
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
cls.rse_id = get_rse_id(rse=cls.rse, **cls.vo)
def get_long_vo():
""" Get the VO name from the config file for testing.
Don't map the name to a short version.
:returns: VO name string.
"""
vo_name = 'def'
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
vo_name = config_get('client',
'vo',
raise_exception=False,
default=None)
return vo_name
def test_reaper():
""" REAPER2 (DAEMON): Test the reaper2 daemon."""
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
vo = {}
rse_name = rse_name_generator()
rse_id = rse_core.add_rse(rse_name, **vo)
mock_protocol = {'scheme': 'MOCK',
'hostname': 'localhost',
'port': 123,
'prefix': '/test/reaper',
'impl': 'rucio.rse.protocols.mock.Default',
'domains': {
'lan': {'read': 1,
'write': 1,
'delete': 1},
'wan': {'read': 1,
'write': 1,
'delete': 1}}}
rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol)
nb_files = 30
file_size = 2147483648 # 2G
file_names = []
for i in range(nb_files):
file_name = 'lfn' + generate_uuid()
file_names.append(file_name)
replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip', **vo),
name=file_name, bytes=file_size,
tombstone=datetime.utcnow() - timedelta(days=1),
account=InternalAccount('root', **vo), adler32=None, md5=None)
rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=800)
rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=10737418240)
rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10)
if vo:
reaper(once=True, rses=[], include_rses='vo=%s&(%s)' % (vo['vo'], rse_name), exclude_rses=[])
reaper(once=True, rses=[], include_rses='vo=%s&(%s)' % (vo['vo'], rse_name), exclude_rses=[])
else:
reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=[])
reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=[])
assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in file_names], rse_expression=rse_name))) == nb_files - 5
def get_rses_to_process(rses, include_rses, exclude_rses):
"""
Return the list of RSEs to process based on rses, include_rses and exclude_rses
:param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs.
:param exclude_rses: RSE expression to exclude RSEs from the Reaper.
:param include_rses: RSE expression to include RSEs.
:returns: A list of RSEs to process
"""
result = REGION.get('rses_to_process')
if result is not NO_VALUE:
return result
all_rses = list_rses()
if rses:
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
logging.warning(
'Ignoring argument rses, this is only available in a single-VO setup. Please try an RSE Expression with include_rses if it is required.'
)
rses = all_rses
else:
invalid = set(rses) - set([rse['rse'] for rse in all_rses])
if invalid:
msg = 'RSE{} {} cannot be found'.format(
's' if len(invalid) > 1 else '',
', '.join([repr(rse) for rse in invalid]))
raise RSENotFound(msg)
rses = [rse for rse in all_rses if rse['rse'] in rses]
else:
rses = all_rses
if include_rses:
included_rses = parse_expression(include_rses)
rses = [rse for rse in rses if rse in included_rses]
if exclude_rses:
excluded_rses = parse_expression(exclude_rses)
rses = [rse for rse in rses if rse not in excluded_rses]
REGION.set('rses_to_process', rses)
logging.info('Reaper: This instance will work on RSEs: %s',
', '.join([rse['rse'] for rse in rses]))
return rses
def poller(once=False,
activities=None,
sleep_time=60,
fts_bulk=100,
db_bulk=1000,
older_than=60,
activity_shares=None,
partition_wait_time=10):
"""
Main loop to check the status of a transfer primitive with a transfertool.
"""
try:
timeout = config_get('conveyor', 'poll_timeout')
timeout = float(timeout)
except NoOptionError:
timeout = None
multi_vo = config_get_bool('common', 'multi_vo', False, None)
logger_prefix = executable = 'conveyor-poller'
if activities:
activities.sort()
executable += '--activities ' + str(activities)
if activity_shares:
activities.sort()
executable += '--activity_shares' + str(activity_shares)
if FILTER_TRANSFERTOOL:
executable += ' --filter-transfertool ' + FILTER_TRANSFERTOOL
run_conveyor_daemon(
once=once,
graceful_stop=graceful_stop,
executable=executable,
logger_prefix=logger_prefix,
partition_wait_time=partition_wait_time,
sleep_time=sleep_time,
run_once_fnc=functools.partial(
run_once,
fts_bulk=fts_bulk,
db_bulk=db_bulk,
older_than=older_than,
activity_shares=activity_shares,
multi_vo=multi_vo,
timeout=timeout,
),
activities=activities,
heart_beat_older_than=3600,
)
def setUp(self):
self.session = get_session()
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.tmp_scope = InternalScope('mock', **self.vo)
self.root = InternalAccount('root', **self.vo)
self.mongo_meta = MongoDidMeta(host='mongo',
port=27017,
db='test_db',
collection='test_collection')
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': 'tst'}
else:
self.vo = {}
self.account = 'root'
self.scope = 'mock'
self.upload_client = UploadClient()
self.file_sizes = 2
self.rse = 'MOCK4'
self.rse_id = get_rse_id(self.rse, **self.vo)
self.session = get_session()
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo_header = '-H "X-Rucio-VO: %s"' % get_long_vo()
else:
self.vo_header = ''
self.cacert = config_get('test', 'cacert')
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.base_client = BaseClient()
self.token = self.base_client.headers['X-Rucio-Auth-Token']
self.replica_client = ReplicaClient()
def __init__(self, external_host):
"""
Initializes the transfertool
:param external_host: The external host where the transfertool API is running
"""
usercert = config_get('conveyor', 'usercert', False, None)
self.deterministic_id = config_get_bool('conveyor', 'use_deterministic_id', False, False)
super(FTS3Transfertool, self).__init__(external_host)
if self.external_host.startswith('https://'):
self.cert = (usercert, usercert)
self.verify = False
else:
self.cert = None
self.verify = True # True is the default setting of a requests.* method
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.scope = InternalScope('mock', **self.vo)
self.rse = 'MOCK4'
self.rse2 = 'MOCK3'
self.account = InternalAccount('root', **self.vo)
self.rse_id = get_rse_id(self.rse, **self.vo)
self.rse2_id = get_rse_id(self.rse2, **self.vo)
self.db_session = session.get_session()
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {
'vo':
config_get('client',
'vo',
raise_exception=False,
default='tst')
}
else:
self.vo = {}
self.tmp_scope = InternalScope('mock', **self.vo)
self.root = InternalAccount('root', **self.vo)
def setup(self):
''' INTERNAL TYPES: Setup the tests '''
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': 'tst'}
else:
self.vo = {}
self.base = InternalType('test', **self.vo)
self.same = InternalType('test', **self.vo)
self.diff = InternalType('different', **self.vo)
self.base_account = InternalAccount('test', **self.vo)
self.base_scope = InternalScope('test', **self.vo)
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {
'vo':
config_get('client',
'vo',
raise_exception=False,
default='tst')
}
self.vo_header = {'X-Rucio-VO': self.vo['vo']}
else:
self.vo = {}
self.vo_header = {}
def kronos_file(once=False, process=0, total_processes=1, thread=0, total_threads=1, brokers_resolved=None, dataset_queue=None):
"""
Main loop to consume tracer reports.
"""
logging.info('tracer consumer starting')
chunksize = config_get_int('tracer-kronos', 'chunksize')
prefetch_size = config_get_int('tracer-kronos', 'prefetch_size')
subscription_id = config_get('tracer-kronos', 'subscription_id')
use_ssl = True
try:
use_ssl = config_get_bool('tracer-kronos', 'use_ssl')
except:
pass
if not use_ssl:
username = config_get('tracer-kronos', 'username')
password = config_get('tracer-kronos', 'password')
excluded_usrdns = set(config_get('tracer-kronos', 'excluded_usrdns').split(','))
conns = []
for broker in brokers_resolved:
if not use_ssl:
conns.append(Connection(host_and_ports=[(broker, config_get_int('tracer-kronos', 'port'))],
use_ssl=False,
reconnect_attempts_max=config_get_int('tracer-kronos', 'reconnect_attempts')))
else:
conns.append(Connection(host_and_ports=[(broker, config_get_int('tracer-kronos', 'port'))],
use_ssl=True,
ssl_key_file=config_get('tracer-kronos', 'ssl_key_file'),
ssl_cert_file=config_get('tracer-kronos', 'ssl_cert_file'),
ssl_version=PROTOCOL_TLSv1,
reconnect_attempts_max=config_get_int('tracer-kronos', 'reconnect_attempts')))
logging.info('(kronos_file) tracer consumer started')
while not graceful_stop.is_set():
for conn in conns:
if not conn.is_connected():
logging.info('(kronos_file) connecting to %s' % conn.transport._Transport__host_and_ports[0][0])
record_counter('daemons.tracer.kronos.reconnect.%s' % conn.transport._Transport__host_and_ports[0][0].split('.')[0])
conn.set_listener('rucio-tracer-kronos', AMQConsumer(broker=conn.transport._Transport__host_and_ports[0], conn=conn, chunksize=chunksize, subscription_id=subscription_id, excluded_usrdns=excluded_usrdns, dataset_queue=dataset_queue))
conn.start()
if not use_ssl:
conn.connect(username, password)
else:
conn.connect()
conn.subscribe(destination=config_get('tracer-kronos', 'queue'), ack='client-individual', id=subscription_id, headers={'activemq.prefetchSize': prefetch_size})
sleep(1)
logging.info('(kronos_file) graceful stop requested')
for conn in conns:
try:
conn.disconnect()
except:
pass
logging.info('(kronos_file) graceful stop done')
