def delete_quarantined_replicas(rse, replicas, session=None):
"""
Delete file replicas.
:param rse: the rse name.
:param files: the list of files to delete.
:param ignore_availability: Ignore the RSE blacklisting.
:param session: The database session in use.
"""
rse_id = get_rse_id(rse, session=session)
conditions = []
for replica in replicas:
conditions.append(models.QuarantinedReplica.path == replica['path'])
if conditions:
session.query(models.QuarantinedReplica).\
filter(models.QuarantinedReplica.rse_id == rse_id).\
filter(or_(*conditions)).\
delete(synchronize_session=False)
session.\
bulk_insert_mappings(models.QuarantinedReplica.__history_mapper__.class_,
[{'rse_id': rse_id, 'path': replica['path'],
'bytes': replica.get('bytes'),
'created_at': replica.get('created_at'),
'deleted_at': datetime.datetime.utcnow()}
for replica in replicas])
def get_local_account_usage(account, rse, issuer):
"""
Get the account usage and connect it with (if available) the account limits of the account.
:param account: The account to read.
:param rse: The rse to read (If none, get all).
:param issuer: The issuer account.
:returns: List of dicts {'rse_id', 'bytes_used', 'files_used', 'bytes_limit'}
"""
rse_id = None
if rse:
rse_id = get_rse_id(rse=rse)
kwargs = {'account': account, 'rse': rse, 'rse_id': rse_id}
if not rucio.api.permission.has_permission(issuer=issuer, action='get_local_account_usage', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not list account usage.' % (issuer))
account = InternalAccount(account)
if not account_exists(account=account):
raise rucio.common.exception.AccountNotFound('Account %s does not exist' % (account))
return [api_update_return_dict(d) for d in account_limit_core.get_local_account_usage(account=account, rse_id=rse_id)]
def touch_dataset_locks(dataset_locks, session=None):
"""
Update the accessed_at timestamp of the given dataset locks + eol_at.
:param replicas: the list of dataset locks.
:param session: The database session in use.
:returns: True, if successful, False otherwise.
"""
rse_ids, now = {}, datetime.utcnow()
for dataset_lock in dataset_locks:
if 'rse_id' not in dataset_lock:
if dataset_lock['rse'] not in rse_ids:
rse_ids[dataset_lock['rse']] = get_rse_id(
rse=dataset_lock['rse'], session=session)
dataset_lock['rse_id'] = rse_ids[dataset_lock['rse']]
eol_at = define_eol(dataset_lock['scope'],
dataset_lock['name'],
rses=[{
'id': dataset_lock['rse_id']
}],
session=session)
try:
session.query(models.DatasetLock).filter_by(scope=dataset_lock['scope'], name=dataset_lock['name'], rse_id=dataset_lock['rse_id']).\
update({'accessed_at': dataset_lock.get('accessed_at') or now}, synchronize_session=False)
session.query(models.ReplicationRule).filter_by(
scope=dataset_lock['scope'],
name=dataset_lock['name']).update({'eol_at': eol_at},
synchronize_session=False)
except DatabaseError:
return False
return True
def add_quarantined_replicas(rse, replicas, session=None):
"""
Bulk add quarantined file replicas.
:param rse: The rse name.
:param replicas: A list of dicts with the replica information.
:param session: The database session in use.
"""
rse_id = get_rse_id(rse, session=session)
for chunk in chunks(replicas, 100):
# Exlude files that have a registered replica. This is a
# safeguard against potential issues in the Auditor.
file_clause = []
for replica in chunk:
file_clause.append(
and_(
models.RSEFileAssociation.scope == replica.get(
'scope',
None), models.RSEFileAssociation.name == replica.get(
'name', None),
models.RSEFileAssociation.rse_id == rse_id))
file_query = session.query(models.RSEFileAssociation.scope,
models.RSEFileAssociation.name,
models.RSEFileAssociation.rse_id).\
with_hint(models.RSEFileAssociation, "index(REPLICAS REPLICAS_PK)", 'oracle').\
filter(or_(*file_clause))
existing_replicas = [(scope, name, rseid)
for scope, name, rseid in file_query]
chunk = [
replica for replica in chunk
if (replica.get('scope', None), replica.get('name', None),
rse_id) not in existing_replicas
]
# Exclude files that have already been added to the quarantined
# replica table.
quarantine_clause = []
for replica in chunk:
quarantine_clause.append(
and_(models.QuarantinedReplica.path == replica['path'],
models.QuarantinedReplica.rse_id == rse_id))
quarantine_query = session.query(models.QuarantinedReplica.path,
models.QuarantinedReplica.rse_id).\
filter(or_(*quarantine_clause))
quarantine_replicas = [(path, rseid)
for path, rseid in quarantine_query]
chunk = [
replica for replica in chunk
if (replica['path'], rse_id) not in quarantine_replicas
]
session.bulk_insert_mappings(models.QuarantinedReplica,
[{
'rse_id': rse_id,
'path': file['path'],
'scope': file.get('scope'),
'name': file.get('name'),
'bytes': file.get('bytes')
} for file in chunk])
def delete_replicas(rse, files, issuer, ignore_availability=False):
"""
Bulk delete file replicas.
:param rse: The RSE name.
:param files: The list of files.
:param issuer: The issuer account.
:param ignore_availability: Ignore the RSE blacklisting.
:returns: True is successful, False otherwise
"""
validate_schema(name='r_dids', obj=files)
rse_id = get_rse_id(rse=rse)
kwargs = {'rse': rse, 'rse_id': rse_id}
if not permission.has_permission(issuer=issuer, action='delete_replicas', kwargs=kwargs):
raise exception.AccessDenied('Account %s can not delete file replicas on %s' % (issuer, rse))
if not permission.has_permission(issuer=issuer, action='skip_availability_check', kwargs=kwargs):
ignore_availability = False
for f in files:
f['scope'] = InternalScope(f['scope'])
replica.delete_replicas(rse_id=rse_id, files=files, ignore_availability=ignore_availability)
def test_list_expired_dids_with_locked_rules(self):
""" UNDERTAKER (CORE): Test that the undertaker does not list expired dids with locked rules"""
tmp_scope = 'mock'
# Add quota
set_account_limit('jdoe', get_rse_id('MOCK'), -1)
dsn = {
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
'DATASET',
'lifetime':
-1,
'rules': [{
'account': 'jdoe',
'copies': 1,
'rse_expression': 'MOCK',
'locked': True,
'grouping': 'DATASET'
}]
}
add_dids(dids=[dsn], account='root')
for did in list_expired_dids(limit=1000):
assert (did['scope'] != dsn['scope']
and did['name'] != dsn['name'])
def add_dids(dids, issuer, vo='def'):
"""
Bulk Add did.
:param dids: A list of dids.
:param issuer: The issuer account.
:param vo: The VO to act on.
"""
for d in dids:
if 'rse' in d:
rse_id = None
if d['rse'] is not None:
rse_id = get_rse_id(rse=d['rse'], vo=vo)
d['rse_id'] = rse_id
kwargs = {'issuer': issuer, 'dids': dids}
if not rucio.api.permission.has_permission(
issuer=issuer, vo=vo, action='add_dids', kwargs=kwargs):
raise rucio.common.exception.AccessDenied(
'Account %s can not bulk add data identifier' % (issuer))
issuer = InternalAccount(issuer, vo=vo)
for d in dids:
d['scope'] = InternalScope(d['scope'], vo=vo)
if 'dids' in d.keys():
for child in d['dids']:
child['scope'] = InternalScope(child['scope'], vo=vo)
return did.add_dids(dids, account=issuer)
def attach_dids(scope, name, attachment, issuer):
"""
Append content to data did.
:param attachment: The attachment.
:param issuer: The issuer account.
"""
validate_schema(name='attachment', obj=attachment)
rse_id = None
if 'rse' in attachment:
if attachment['rse'] is not None:
rse_id = get_rse_id(rse=attachment['rse'])
attachment['rse_id'] = rse_id
kwargs = {'scope': scope, 'name': name, 'attachment': attachment}
if not rucio.api.permission.has_permission(issuer=issuer, action='attach_dids', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not add data identifiers to %s:%s' % (issuer, scope, name))
scope = InternalScope(scope)
issuer = InternalAccount(issuer)
if 'account' in attachment.keys():
attachment['account'] = InternalAccount(attachment['account'])
for d in attachment['dids']:
d['scope'] = InternalScope(d['scope'])
if rse_id is not None:
dids = did.attach_dids(scope=scope, name=name, dids=attachment['dids'],
account=attachment.get('account', issuer), rse_id=rse_id)
else:
dids = did.attach_dids(scope=scope, name=name, dids=attachment['dids'],
account=attachment.get('account', issuer))
return dids
def update_replicas_states(rse, files, issuer):
"""
Update File replica information and state.
:param rse: The RSE name.
:param files: The list of files.
:param issuer: The issuer account.
"""
validate_schema(name='dids', obj=files)
rse_id = get_rse_id(rse=rse)
kwargs = {'rse': rse, 'rse_id': rse_id}
if not permission.has_permission(
issuer=issuer, action='update_replicas_states', kwargs=kwargs):
raise exception.AccessDenied(
'Account %s can not update file replicas state on %s' %
(issuer, rse))
replicas = []
for file in files:
rep = file
rep['rse_id'] = rse_id
rep['scope'] = InternalScope(rep['scope'])
replicas.append(rep)
replica.update_replicas_states(replicas=replicas)
def list_bad_replicas_status(state=BadFilesStatus.BAD,
rse=None,
younger_than=None,
older_than=None,
limit=None,
list_pfns=False):
"""
List the bad file replicas history states. Method used by the rucio-ui.
:param state: The state of the file (SUSPICIOUS or BAD).
:param rse: The RSE name.
:param younger_than: datetime object to select bad replicas younger than this date.
:param older_than: datetime object to select bad replicas older than this date.
:param limit: The maximum number of replicas returned.
"""
rse_id = None
if rse is not None:
rse_id = get_rse_id(rse=rse)
replicas = replica.list_bad_replicas_status(state=state,
rse_id=rse_id,
younger_than=younger_than,
older_than=older_than,
limit=limit,
list_pfns=list_pfns)
return [api_update_return_dict(r) for r in replicas]
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.scope = InternalScope('mock', **self.vo)
self.rse = 'MOCK4'
self.rse2 = 'MOCK3'
self.account = InternalAccount('root', **self.vo)
self.rse_id = get_rse_id(self.rse, **self.vo)
self.rse2_id = get_rse_id(self.rse2, **self.vo)
self.db_session = session.get_session()
def test_list_datasets_per_rse(self):
""" REPLICA (CLIENT): List datasets in RSE."""
rule_client = RuleClient()
did_client = DIDClient()
scope = 'mock'
dataset = 'dataset_' + str(generate_uuid())
did_client.add_dataset(scope=scope, name=dataset)
rule_client.add_replication_rule(dids=[{
'scope': scope,
'name': dataset
}],
account='root',
copies=1,
rse_expression='MOCK',
grouping='DATASET')
replicas = [
r for r in list_datasets_per_rse(
rse_id=get_rse_id(rse='MOCK', **self.vo),
filters={
'scope': InternalScope(scope, **self.vo),
'name': 'data*'
})
]
assert replicas != []
def touch_replicas(replicas, session=None):
"""
Update the accessed_at timestamp of the given file replicas/dids.
:param replicas: the list of replicas.
:param session: The database session in use.
:returns: True, if successful, False otherwise.
"""
rse_ids, now = {}, datetime.utcnow()
for replica in replicas:
if 'rse_id' not in replica:
if replica['rse'] not in rse_ids:
rse_ids[replica['rse']] = get_rse_id(rse=replica['rse'], session=session)
replica['rse_id'] = rse_ids[replica['rse']]
try:
session.query(models.RSEFileAssociation).filter_by(rse_id=replica['rse_id'], scope=replica['scope'], name=replica['name']).\
update({'accessed_at': replica.get('accessed_at') or now}, synchronize_session=False)
session.query(models.DataIdentifier).filter_by(scope=replica['scope'], name=replica['name'], did_type=DIDType.FILE).\
update({'accessed_at': replica.get('accessed_at') or now}, synchronize_session=False)
except DatabaseError:
return False
# resolve_datasets = session.query(models.DataIdentifierAssociation).filter_by(child_scope=replica['scope'], child_name=replica['name'], did_type=DIDType.DATASET)
# for dataset in resolve_datasets:
# session.query(models.DataIdentifier).filter_by(scope=dataset['scope'], name=dataset['name'], did_type=DIDType.DATASET).\
# update({'accessed_at': replica.get('accessed_at') or now}, synchronize_session=False)
# session.query(models.DatasetLock).filter_by(scope=dataset['scope'], name=dataset['name'], rse_id=replica['rse_id']).\
# update({'accessed_at': replica.get('accessed_at') or now}, synchronize_session=False)
return True
def update_replicas_states(replicas, nowait=False, session=None):
"""
Update File replica information and state.
:param replicas: The list of replicas.
:param nowait: Nowait parameter for the for_update queries.
:param session: The database session in use.
"""
rse_ids = {}
for replica in replicas:
if 'rse_id' not in replica:
if replica['rse'] not in rse_ids:
rse_ids[replica['rse']] = get_rse_id(rse=replica['rse'], session=session)
replica['rse_id'] = rse_ids[replica['rse']]
query = session.query(models.RSEFileAssociation).filter_by(rse_id=replica['rse_id'], scope=replica['scope'], name=replica['name'])
if isinstance(replica['state'], str) or isinstance(replica['state'], unicode):
replica['state'] = ReplicaState.from_string(replica['state'])
if replica['state'] == ReplicaState.BEING_DELETED:
query = query.filter_by(lock_cnt=0)
if replica['state'] == ReplicaState.AVAILABLE:
rucio.core.lock.successful_transfer(scope=replica['scope'], name=replica['name'], rse_id=replica['rse_id'], nowait=nowait, session=session)
if 'path' in replica and replica['path']:
rowcount = query.update({'state': replica['state'], 'path': replica['path']}, synchronize_session=False)
else:
rowcount = query.update({'state': replica['state']}, synchronize_session=False)
if not rowcount:
raise exception.UnsupportedOperation('State %(state)s for replica %(scope)s:%(name)s cannot be updated' % replica)
return True
def attach_dids_to_dids(attachments, issuer, ignore_duplicate=False, vo='def'):
"""
Append content to dids.
:param attachments: The contents.
:param issuer: The issuer account.
:param ignore_duplicate: If True, ignore duplicate entries.
:param vo: The VO to act on.
"""
validate_schema(name='attachments', obj=attachments)
for a in attachments:
if 'rse' in a:
rse_id = None
if a['rse'] is not None:
rse_id = get_rse_id(rse=a['rse'], vo=vo)
a['rse_id'] = rse_id
if not rucio.api.permission.has_permission(
issuer=issuer,
vo=vo,
action='attach_dids_to_dids',
kwargs={'attachments': attachments}):
raise rucio.common.exception.AccessDenied(
'Account %s can not add data identifiers' % (issuer))
issuer = InternalAccount(issuer, vo=vo)
for attachment in attachments:
attachment['scope'] = InternalScope(attachment['scope'], vo=vo)
for d in attachment['dids']:
d['scope'] = InternalScope(d['scope'], vo=vo)
return did.attach_dids_to_dids(attachments=attachments,
account=issuer,
ignore_duplicate=ignore_duplicate)
def get_request_by_did(scope, name, rse, rse_id=None, request_type=None, session=None):
"""
Retrieve a request by its DID for a destination RSE.
:param scope: The scope of the data identifier.
:param name: The name of the data identifier.
:param rse: The destination RSE of the request.
:param rse_id: The destination RSE ID of the request. Overrides rse param!
:param request_type: The type of request as rucio.db.constants.RequestType.
:param session: Database session to use.
:returns: Request as a dictionary.
"""
record_counter('core.request.get_request_by_did')
try:
tmp = session.query(models.Request).filter_by(scope=scope,
name=name)
if rse_id:
tmp = tmp.filter_by(dest_rse_id=rse_id)
else:
tmp = tmp.filter_by(dest_rse_id=get_rse_id(rse))
if request_type:
tmp = tmp.filter_by(request_type=request_type)
tmp = tmp.first()
if not tmp:
return
else:
tmp = dict(tmp)
tmp.pop('_sa_instance_state')
return tmp
except IntegrityError, e:
raise RucioException(e.args)
def on_message(self, frame):
'''
on_message
'''
record_counter('daemons.cache.consumer2.message')
try:
msg = json.loads(frame.body)
if isinstance(msg, dict) and 'operation' in msg.keys():
for f in msg['files']:
f['scope'] = InternalScope(f['scope'])
if 'rse_id' in msg:
rse_id = msg['rse_id']
else:
rse_id = get_rse_id(rse=msg['rse'],
vo=msg.get('vo', 'def'))
rse_vo_str = msg['rse']
if 'vo' in msg and msg['vo'] != 'def':
rse_vo_str = '{} on {}'.format(rse_vo_str, msg['vo'])
if msg['operation'] == 'add_replicas':
logging.info('add_replicas to RSE %s: %s ' %
(rse_vo_str, str(msg['files'])))
add_volatile_replicas(rse_id=rse_id, replicas=msg['files'])
elif msg['operation'] == 'delete_replicas':
logging.info('delete_replicas to RSE %s: %s ' %
(rse_vo_str, str(msg['files'])))
delete_volatile_replicas(rse_id=rse_id,
replicas=msg['files'])
except:
logging.error(str(format_exc()))
def test_inc_dec_get_counter(self):
"""ACCOUNT COUNTER (CORE): Increase, decrease and get counter """
account_update(once=True)
rse_id = get_rse_id(rse='MOCK', **self.vo)
account = InternalAccount('jdoe', **self.vo)
account_counter.del_counter(rse_id=rse_id, account=account)
account_counter.add_counter(rse_id=rse_id, account=account)
cnt = get_usage(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': 0, 'bytes': 0})
count, sum = 0, 0
for i in range(10):
account_counter.increase(rse_id=rse_id,
account=account,
files=1,
bytes=2.147e+9)
account_update(once=True)
count += 1
sum += 2.147e+9
cnt = get_usage(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
for i in range(4):
account_counter.decrease(rse_id=rse_id,
account=account,
files=1,
bytes=2.147e+9)
account_update(once=True)
count -= 1
sum -= 2.147e+9
cnt = get_usage(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
for i in range(5):
account_counter.increase(rse_id=rse_id,
account=account,
files=1,
bytes=2.147e+9)
account_update(once=True)
count += 1
sum += 2.147e+9
cnt = get_usage(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
for i in range(8):
account_counter.decrease(rse_id=rse_id,
account=account,
files=1,
bytes=2.147e+9)
account_update(once=True)
count -= 1
sum -= 2.147e+9
cnt = get_usage(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
def list_rebalance_rule_candidates(rse, mode=None, session=None):
"""
List the rebalance rule candidates based on the agreed on specification
:param rse: RSE of the source.
:param mode: Rebalancing mode.
:param session: DB Session.
"""
rse_id = get_rse_id(rse)
# dumps can be applied only for decommission since the dumps doesn't contain info from dids
if mode == 'decommission':
return _list_rebalance_rule_candidates_dump(rse, mode)
# the rest is done with sql query
from_date = datetime.utcnow() + timedelta(days=60)
to_date = datetime.now() - timedelta(days=60)
allowed_accounts = [
InternalAccount(a) for a in ('panda', 'root', 'ddmadmin')
]
allowed_grouping = [RuleGrouping.DATASET, RuleGrouping.ALL]
external_dsl = aliased(models.DatasetLock)
count_locks = select([func.count()]).where(
and_(external_dsl.scope == models.DatasetLock.scope,
external_dsl.name == models.DatasetLock.name,
external_dsl.rse_id == models.DatasetLock.rse_id)).as_scalar()
query = session.query(models.DatasetLock.scope,
models.DatasetLock.name,
models.ReplicationRule.id,
models.ReplicationRule.rse_expression,
models.ReplicationRule.subscription_id,
models.DataIdentifier.bytes,
models.DataIdentifier.length,
case([(or_(models.DatasetLock.length < 1, models.DatasetLock.length.is_(None)), 0)],
else_=cast(models.DatasetLock.bytes / models.DatasetLock.length, Integer))).\
join(models.ReplicationRule, models.ReplicationRule.id == models.DatasetLock.rule_id).\
join(models.DataIdentifier, and_(models.DatasetLock.scope == models.DataIdentifier.scope, models.DatasetLock.name == models.DataIdentifier.name)).\
filter(models.DatasetLock.rse_id == rse_id).\
filter(or_(models.ReplicationRule.expires_at > from_date, models.ReplicationRule.expires_at.is_(None))).\
filter(models.ReplicationRule.created_at < to_date).\
filter(models.ReplicationRule.account.in_(allowed_accounts)).\
filter(models.ReplicationRule.state == RuleState.OK).\
filter(models.ReplicationRule.did_type == DIDType.DATASET).\
filter(models.ReplicationRule.copies == 1).\
filter(models.ReplicationRule.child_rule_id.is_(None)).\
filter(models.ReplicationRule.grouping.in_(allowed_grouping)).\
filter(models.DataIdentifier.bytes.isnot(None)).\
filter(models.DataIdentifier.is_open == 0).\
filter(models.DataIdentifier.did_type == DIDType.DATASET).\
filter(case([(or_(models.DatasetLock.length < 1, models.DatasetLock.length.is_(None)), 0)],
else_=cast(models.DatasetLock.bytes / models.DatasetLock.length, Integer)) > 1000000000).\
filter(count_locks == 1)
summary = query.order_by(
case([(or_(models.DatasetLock.length < 1,
models.DatasetLock.length.is_(None)), 0)],
else_=cast(models.DatasetLock.bytes / models.DatasetLock.length,
Integer)), models.DatasetLock.accessed_at).all()
return summary
def setUpClass(cls):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
cls.vo = {'vo': 'tst'}
else:
cls.vo = {}
cls.db_session = session.get_session()
cls.dest_rse = 'MOCK'
cls.source_rse = 'MOCK4'
cls.dest_rse_id = get_rse_id(cls.dest_rse, **cls.vo)
cls.source_rse_id = get_rse_id(cls.source_rse, **cls.vo)
cls.scope = InternalScope('mock', **cls.vo)
cls.account = InternalAccount('root', **cls.vo)
cls.user_activity = 'User Subscription'
cls.all_activities = 'all_activities'
set_rse_transfer_limits(cls.dest_rse_id, cls.user_activity, max_transfers=1, session=cls.db_session)
set('throttler_release_strategy', 'dest_%s' % cls.dest_rse_id, 'fifo', session=cls.db_session)
def setUp(self):
self.account = 'root'
self.scope = 'mock'
self.upload_client = UploadClient()
self.file_sizes = 2
self.rse = 'MOCK4'
self.rse_id = get_rse_id(self.rse)
self.session = get_session()
def setUpClass(cls):
cls.upload_client = UploadClient()
cls.session = get_session()
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
cls.rse_id = get_rse_id(cls.rse, session=cls.session, **cls.vo)
def setUpClass(cls):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
else:
cls.vo = {}
cls.account = InternalAccount('jdoe', **cls.vo)
cls.rse_1_name = 'MOCK4'
cls.rse_2_name = 'MOCK5'
cls.mock1_id = get_rse_id(cls.rse_1_name, **cls.vo)
cls.mock2_id = get_rse_id(cls.rse_2_name, **cls.vo)
cls.db_session = session.get_session()
cls.rse_1 = {'id': cls.mock1_id, 'staging_area': False}
cls.rse_2 = {'id': cls.mock2_id, 'staging_area': False}
def setUpClass(cls):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
else:
cls.vo = {}
# Add test RSE
cls.rse1 = 'MOCK'
cls.rse3 = 'MOCK3'
cls.rse4 = 'MOCK4'
cls.rse5 = 'MOCK5'
cls.rse1_id = get_rse_id(rse=cls.rse1, **cls.vo)
cls.rse3_id = get_rse_id(rse=cls.rse3, **cls.vo)
cls.rse4_id = get_rse_id(rse=cls.rse4, **cls.vo)
cls.rse5_id = get_rse_id(rse=cls.rse5, **cls.vo)
# Add Tags
cls.T1 = tag_generator()
cls.T2 = tag_generator()
add_rse_attribute(cls.rse1_id, cls.T1, True)
add_rse_attribute(cls.rse3_id, cls.T1, True)
add_rse_attribute(cls.rse4_id, cls.T2, True)
add_rse_attribute(cls.rse5_id, cls.T1, True)
# Add fake weights
add_rse_attribute(cls.rse1_id, "fakeweight", 10)
add_rse_attribute(cls.rse3_id, "fakeweight", 0)
add_rse_attribute(cls.rse4_id, "fakeweight", 0)
add_rse_attribute(cls.rse5_id, "fakeweight", 0)
# Add quota
cls.jdoe = InternalAccount('jdoe', **cls.vo)
cls.root = InternalAccount('root', **cls.vo)
set_local_account_limit(cls.jdoe, cls.rse1_id, -1)
set_local_account_limit(cls.jdoe, cls.rse3_id, -1)
set_local_account_limit(cls.jdoe, cls.rse4_id, -1)
set_local_account_limit(cls.jdoe, cls.rse5_id, -1)
set_local_account_limit(cls.root, cls.rse1_id, -1)
set_local_account_limit(cls.root, cls.rse3_id, -1)
set_local_account_limit(cls.root, cls.rse4_id, -1)
set_local_account_limit(cls.root, cls.rse5_id, -1)
def tmp_rse_info(rse=None, vo='def', rse_id=None, session=None):
if rse_id is None:
# This can be called directly by client tools if they're co-located on a server
# i.e. running rucio cli on a server and during the test suite.
# We have to map to VO name here for this situations, despite this nominally
# not being a client interface.
rse_id = get_rse_id(rse=rse, vo=map_vo(vo))
return get_rse_protocols(rse_id=rse_id, session=session)
def list_quarantined_replicas(rse,
limit,
worker_number=None,
total_workers=None,
session=None):
"""
List RSE Quarantined File replicas.
:param rse: the rse name.
:param limit: The maximum number of replicas returned.
:param worker_number: id of the executing worker.
:param total_workers: Number of total workers.
:param session: The database session in use.
:returns: a list of dictionary replica.
"""
rse_id = get_rse_id(rse, session=session)
query = session.query(models.QuarantinedReplica.path,
models.QuarantinedReplica.bytes,
models.QuarantinedReplica.scope,
models.QuarantinedReplica.name,
models.QuarantinedReplica.created_at).\
filter(models.QuarantinedReplica.rse_id == rse_id)
# do no delete valid replicas
stmt = exists(select([1]).prefix_with("/*+ index(REPLICAS REPLICAS_PK) */", dialect='oracle')).\
where(and_(models.RSEFileAssociation.scope == models.QuarantinedReplica.scope,
models.RSEFileAssociation.name == models.QuarantinedReplica.name,
models.RSEFileAssociation.rse_id == models.QuarantinedReplica.rse_id))
query = query.filter(not_(stmt))
if worker_number and total_workers and total_workers - 1 > 0:
if session.bind.dialect.name == 'oracle':
bindparams = [
bindparam('worker_number', worker_number - 1),
bindparam('total_workers', total_workers - 1)
]
query = query.filter(
text('ORA_HASH(path, :total_workers) = :worker_number',
bindparams=bindparams))
elif session.bind.dialect.name == 'mysql':
query = query.filter('mod(md5(path), %s) = %s' %
(total_workers - 1, worker_number - 1))
elif session.bind.dialect.name == 'postgresql':
query = query.filter(
'mod(abs((\'x\'||md5(path))::bit(32)::int), %s) = %s' %
(total_workers - 1, worker_number - 1))
return [{
'path': path,
'rse': rse,
'rse_id': rse_id,
'created_at': created_at,
'scope': scope,
'name': name,
'bytes': bytes
} for path, bytes, scope, name, created_at in query.limit(limit)]
def add_did(scope, name, did_type, issuer, account=None, statuses={}, meta={}, rules=[], lifetime=None, dids=[], rse=None, vo='def', session=None):
"""
Add data did.
:param scope: The scope name.
:param name: The data identifier name.
:param did_type: The data identifier type.
:param issuer: The issuer account.
:param account: The account owner. If None, then issuer is selected as owner.
:param statuses: Dictionary with statuses, e.g.g {'monotonic':True}.
:meta: Meta-data associated with the data identifier is represented using key/value pairs in a dictionary.
:rules: Replication rules associated with the data did. A list of dictionaries, e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ].
:param lifetime: DID's lifetime (in seconds).
:param dids: The content.
:param rse: The RSE name when registering replicas.
:param vo: The VO to act on.
:param session: The database session in use.
"""
v_did = {'name': name, 'type': did_type.upper(), 'scope': scope}
validate_schema(name='did', obj=v_did, vo=vo)
validate_schema(name='dids', obj=dids, vo=vo)
validate_schema(name='rse', obj=rse, vo=vo)
kwargs = {'scope': scope, 'name': name, 'type': did_type, 'issuer': issuer, 'account': account, 'statuses': statuses, 'meta': meta, 'rules': rules, 'lifetime': lifetime}
if not rucio.api.permission.has_permission(issuer=issuer, vo=vo, action='add_did', kwargs=kwargs, session=session):
raise rucio.common.exception.AccessDenied('Account %s can not add data identifier to scope %s' % (issuer, scope))
if account is not None:
account = InternalAccount(account, vo=vo)
issuer = InternalAccount(issuer, vo=vo)
scope = InternalScope(scope, vo=vo)
for d in dids:
d['scope'] = InternalScope(d['scope'], vo=vo)
for r in rules:
r['account'] = InternalAccount(r['account'], vo=vo)
rse_id = None
if rse is not None:
rse_id = get_rse_id(rse=rse, vo=vo, session=session)
if did_type == 'DATASET':
# naming_convention validation
extra_meta = naming_convention.validate_name(scope=scope, name=name, did_type='D', session=session)
# merge extra_meta with meta
for k in extra_meta or {}:
if k not in meta:
meta[k] = extra_meta[k]
elif meta[k] != extra_meta[k]:
print("Provided metadata %s doesn't match the naming convention: %s != %s" % (k, meta[k], extra_meta[k]))
raise rucio.common.exception.InvalidObject("Provided metadata %s doesn't match the naming convention: %s != %s" % (k, meta[k], extra_meta[k]))
# Validate metadata
meta_core.validate_meta(meta=meta, did_type=DIDType[did_type.upper()], session=session)
return did.add_did(scope=scope, name=name, did_type=DIDType[did_type.upper()], account=account or issuer,
statuses=statuses, meta=meta, rules=rules, lifetime=lifetime,
dids=dids, rse_id=rse_id, session=session)
def get_signed_url_server(rse, service, op, url, vo='def'):
'''
get_signed_url_server
'''
from rucio.core.rse import get_rse_id
from rucio.core.credential import get_signed_url
rse_id = get_rse_id(rse=rse, vo=vo)
return get_signed_url(rse_id, service, op, url)
def setUpClass(cls):
cls.db_session = session.get_session()
cls.dest_rse = 'MOCK'
cls.source_rse = 'MOCK4'
cls.dest_rse_id = get_rse_id(cls.dest_rse)
cls.source_rse_id = get_rse_id(cls.source_rse)
cls.scope = 'mock'
cls.account = 'root'
cls.user_activity = 'User Subscription'
cls.all_activities = 'all_activities'
set_rse_transfer_limits(cls.dest_rse,
cls.user_activity,
max_transfers=1,
session=cls.db_session)
set('throttler_release_strategy',
'dest_%s' % cls.dest_rse_id,
'fifo',
session=cls.db_session)
def list_requests(src_rses, dst_rses, states, issuer):
"""
List all requests in a specific state from a source RSE to a destination RSE.
:param src_rses: source RSEs.
:param dst_rses: destination RSEs.
:param states: list of request states.
:param issuer: Issuing account as a string.
"""
src_rse_ids = [get_rse_id(rse=rse) for rse in src_rses]
dst_rse_ids = [get_rse_id(rse=rse) for rse in dst_rses]
kwargs = {'src_rse_id': src_rse_ids, 'dst_rse_id': dst_rse_ids, 'issuer': issuer}
if not permission.has_permission(issuer=issuer, action='list_requests', kwargs=kwargs):
raise exception.AccessDenied('%(issuer)s cannot list requests from RSE %(src_rse)s to RSE %(dst_rse)s' % locals())
for req in request.list_requests(src_rse_ids, dst_rse_ids, states):
yield api_update_return_dict(req)
def get_distance(source, destination, issuer, vo='def', session=None):
"""
Get distances between rses.
:param source: The source RSE.
:param destination: The destination RSE.
:param issuer: The issuer account.
:param vo: The VO to act on.
:param session: The database session in use.
:returns distance: List of dictionaries.
"""
distances = distance_module.get_distances(
src_rse_id=rse_module.get_rse_id(source, vo=vo, session=session),
dest_rse_id=rse_module.get_rse_id(destination, vo=vo, session=session),
session=session)
return [api_update_return_dict(d, session=session) for d in distances]
def setUp(self):
self.account = InternalAccount('root')
self.scope = InternalScope('mock')
self.upload_client = UploadClient()
self.account_client = AccountClient()
self.file_sizes = 2
self.rse = 'MOCK4'
self.rse_id = get_rse_id(self.rse)
self.session = get_session()
def add_distance(source,
destination,
issuer,
vo='def',
ranking=None,
distance=None,
geoip_distance=None,
active=None,
submitted=None,
finished=None,
failed=None,
transfer_speed=None):
"""
Add a src-dest distance.
:param source: The source.
:param destination: The destination.
:param issuer: The issuer account.
:param vo: The VO to act on.
:param ranking: Ranking as an integer.
:param distance: Distance as an integer.
:param geoip_distance: GEOIP Distance as an integer.
:param active: Active FTS transfers as an integer.
:param submitted: Submitted FTS transfers as an integer.
:param finished: Finished FTS transfers as an integer.
:param failed: Failed FTS transfers as an integer.
:param transfer_speed: FTS transfer speed as an integer.
"""
kwargs = {'source': source, 'destination': destination}
if not permission.has_permission(
issuer=issuer, vo=vo, action='add_distance', kwargs=kwargs):
raise exception.AccessDenied('Account %s can not add RSE distances' %
(issuer))
return distance_module.add_distance(
src_rse_id=rse_module.get_rse_id(source, vo=vo),
dest_rse_id=rse_module.get_rse_id(destination, vo=vo),
ranking=ranking,
agis_distance=distance,
geoip_distance=geoip_distance,
active=active,
submitted=submitted,
finished=finished,
failed=failed,
transfer_speed=transfer_speed)
def get_dataset_locks_by_rse(rse):
"""
Get the dataset locks of an RSE.
:param rse: RSE name.
:return: List of dicts {'rse_id': ..., 'state': ...}
"""
rse_id = get_rse_id(rse=rse)
return lock.get_dataset_locks_by_rse_id(rse_id=rse_id)
def get_replica_atime(replica, session=None):
"""
Get the accessed_at timestamp for a replica. Just for testing.
:param replicas: List of dictionaries {scope, name, rse_id, path}
:param session: Database session to use.
:returns: A datetime timestamp with the last access time.
"""
if 'rse_id' not in replica:
replica['rse_id'] = get_rse_id(rse=replica['rse'], session=session)
return session.query(models.RSEFileAssociation.accessed_at).filter_by(scope=replica['scope'], name=replica['name'], rse_id=replica['rse_id']).\
with_hint(models.RSEFileAssociation, text="INDEX(REPLICAS REPLICAS_PK)", dialect_name='oracle').one()[0]
def get_account_limit(account, rse):
"""
Lists the limitation names/values for the specified account name and rse name.
REST API: http://<host>:<port>/rucio/account/<account>/limits
:param account: The account name.
:param rse: The rse name.
:returns: The account limit.
"""
rse_id = get_rse_id(rse=rse)
return {rse: account_limit_core.get_account_limit(account=account, rse_id=rse_id)}
def list_unlocked_replicas(rse, limit, bytes=None, rse_id=None, worker_number=None, total_workers=None, delay_seconds=0, session=None):
"""
List RSE File replicas with no locks.
:param rse: the rse name.
:param bytes: the amount of needed bytes.
:param session: The database session in use.
:returns: a list of dictionary replica.
"""
if not rse_id:
rse_id = get_rse_id(rse=rse, session=session)
# filter(models.RSEFileAssociation.state != ReplicaState.BEING_DELETED).\
none_value = None # Hack to get pep8 happy...
query = session.query(models.RSEFileAssociation.scope, models.RSEFileAssociation.name, models.RSEFileAssociation.bytes, models.RSEFileAssociation.tombstone).\
filter(models.RSEFileAssociation.tombstone < datetime.utcnow()).\
filter(models.RSEFileAssociation.lock_cnt == 0).\
filter(case([(models.RSEFileAssociation.tombstone != none_value, models.RSEFileAssociation.rse_id), ]) == rse_id).\
filter(or_(models.RSEFileAssociation.state.in_((ReplicaState.AVAILABLE, ReplicaState.UNAVAILABLE)),
and_(models.RSEFileAssociation.state == ReplicaState.BEING_DELETED, models.RSEFileAssociation.updated_at < datetime.utcnow() - timedelta(seconds=delay_seconds)))).\
order_by(models.RSEFileAssociation.tombstone).\
with_hint(models.RSEFileAssociation, "INDEX(replicas REPLICAS_TOMBSTONE_IDX)", 'oracle')
if worker_number and total_workers and total_workers - 1 > 0:
if session.bind.dialect.name == 'oracle':
bindparams = [bindparam('worker_number', worker_number - 1), bindparam('total_workers', total_workers - 1)]
query = query.filter(text('ORA_HASH(name, :total_workers) = :worker_number', bindparams=bindparams))
elif session.bind.dialect.name == 'mysql':
query = query.filter('mod(md5(name), %s) = %s' % (total_workers - 1, worker_number - 1))
elif session.bind.dialect.name == 'postgresql':
query = query.filter('mod(abs((\'x\'||md5(name))::bit(32)::int), %s) = %s' % (total_workers - 1, worker_number - 1))
query = query.limit(limit)
rows = list()
neededSpace = bytes
totalbytes = 0
for (scope, name, bytes, tombstone) in query.yield_per(1000):
if tombstone != OBSOLETE and neededSpace is not None and totalbytes >= neededSpace:
break
d = {'scope': scope, 'name': name, 'bytes': bytes}
rows.append(d)
if tombstone != OBSOLETE:
totalbytes += bytes
return rows
def set_account_limit(account, rse, bytes, issuer):
"""
Set an account limit..
:param account: The account name.
:param rse: The rse name.
:param bytes: The limit in bytes.
:param issuer: The issuer account_core.
"""
kwargs = {'account': account, 'rse': rse, 'bytes': bytes}
if not rucio.api.permission.has_permission(issuer=issuer, action='set_account_limit', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not set account limits.' % (issuer))
if not account_exists(account=account):
raise rucio.common.exception.AccountNotFound('Account %s does not exist' % (account))
rse_id = get_rse_id(rse=rse)
account_limit_core.set_account_limit(account=account, rse_id=rse_id, bytes=bytes)
def delete_account_limit(account, rse, issuer):
"""
Delete an account limit..
:param account: The account name.
:param rse: The rse name.
:param issuer: The issuer account_core.
:returns: True if successful; False otherwise.
"""
kwargs = {'account': account, 'rse': rse}
if not rucio.api.permission.has_permission(issuer=issuer, action='delete_account_limit', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not delete account limits.' % (issuer))
if not account_exists(account=account):
raise rucio.common.exception.AccountNotFound('Account %s does not exist' % (account))
rse_id = get_rse_id(rse=rse)
return account_limit_core.delete_account_limit(account=account, rse_id=rse_id)
def get_replica(rse, scope, name, rse_id=None, session=None):
"""
Get File replica.
:param rse: the rse name.
:param scope: the scope name.
:param name: The data identifier name.
:param rse_id: The RSE Id.
:param session: The database session in use.
:returns: A dictionary with the list of replica attributes.
"""
if not rse_id:
rse_id = get_rse_id(rse=rse, session=session)
row = session.query(models.RSEFileAssociation).filter_by(rse_id=rse_id, scope=scope, name=name).one()
d = {}
for column in row.__table__.columns:
d[column.name] = getattr(row, column.name)
return d
def get_account_usage(account, rse, issuer):
"""
Get the account usage and connect it with (if available) the account limits of the account.
:param account: The account to read.
:param rse: The rse to read (If none, get all).
:param issuer: The issuer account.
:returns: List of dicts {'rse_id', 'bytes_used', 'files_used', 'bytes_limit'}
"""
kwargs = {'account': account, 'rse': rse}
if not rucio.api.permission.has_permission(issuer=issuer, action='get_account_usage', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not list account usage.' % (issuer))
if not account_exists(account=account):
raise rucio.common.exception.AccountNotFound('Account %s does not exist' % (account))
rse_id = None
if rse:
rse_id = get_rse_id(rse=rse)
return account_limit_core.get_account_usage(account=account, rse_id=rse_id)
def update_replica_lock_counter(rse, scope, name, value, rse_id=None, session=None):
"""
Update File replica lock counters.
:param rse: the rse name.
:param scope: the tag name.
:param name: The data identifier name.
:param value: The number of created/deleted locks.
:param rse_id: The id of the RSE.
:param session: The database session in use.
:returns: True or False.
"""
if not rse_id:
rse_id = get_rse_id(rse=rse, session=session)
# WTF BUG in the mysql-driver: lock_cnt uses the already updated value! ACID? Never heard of it!
if session.bind.dialect.name == 'mysql':
rowcount = session.query(models.RSEFileAssociation).\
filter_by(rse_id=rse_id, scope=scope, name=name).\
update({'lock_cnt': models.RSEFileAssociation.lock_cnt + value,
'tombstone': case([(models.RSEFileAssociation.lock_cnt + value < 0,
datetime.utcnow()), ],
else_=None)},
synchronize_session=False)
else:
rowcount = session.query(models.RSEFileAssociation).\
filter_by(rse_id=rse_id, scope=scope, name=name).\
update({'lock_cnt': models.RSEFileAssociation.lock_cnt + value,
'tombstone': case([(models.RSEFileAssociation.lock_cnt + value == 0,
datetime.utcnow()), ],
else_=None)},
synchronize_session=False)
return bool(rowcount)
'scope': response['scope'],
'name': response['name'],
'state': ReplicaState.UNAVAILABLE}],
session=session)
except:
logging.critical("Could not update replica state for failed transfer %s:%s at %s (%s)" % (response['scope'],
response['name'],
rse_update_name,
traceback.format_exc()))
raise
tss = time.time()
try:
lock.failed_transfer(response['scope'],
response['name'],
rse_core.get_rse_id(rse=rse_update_name, session=session),
session=session)
except:
logging.warn('Could not update lock for failed transfer %s:%s at %s (%s)' % (response['scope'],
response['name'],
rse_update_name,
traceback.format_exc()))
raise
record_timer('daemons.conveyor.common.update_request_state.lock-failed_transfer', (time.time()-tss)*1000)
else:
logging.warn('REQUEUED DID %s:%s REQUEST %s AS %s TRY %s' % (response['scope'],
response['name'],
response['request_id'],
new_req['request_id'],
new_req['retry_count']))
