def delete_quarantined_replicas(rse, replicas, session=None): """ Delete file replicas. :param rse: the rse name. :param files: the list of files to delete. :param ignore_availability: Ignore the RSE blacklisting. :param session: The database session in use. """ rse_id = get_rse_id(rse, session=session) conditions = [] for replica in replicas: conditions.append(models.QuarantinedReplica.path == replica['path']) if conditions: session.query(models.QuarantinedReplica).\ filter(models.QuarantinedReplica.rse_id == rse_id).\ filter(or_(*conditions)).\ delete(synchronize_session=False) session.\ bulk_insert_mappings(models.QuarantinedReplica.__history_mapper__.class_, [{'rse_id': rse_id, 'path': replica['path'], 'bytes': replica.get('bytes'), 'created_at': replica.get('created_at'), 'deleted_at': datetime.datetime.utcnow()} for replica in replicas])
def get_local_account_usage(account, rse, issuer): """ Get the account usage and connect it with (if available) the account limits of the account. :param account: The account to read. :param rse: The rse to read (If none, get all). :param issuer: The issuer account. :returns: List of dicts {'rse_id', 'bytes_used', 'files_used', 'bytes_limit'} """ rse_id = None if rse: rse_id = get_rse_id(rse=rse) kwargs = {'account': account, 'rse': rse, 'rse_id': rse_id} if not rucio.api.permission.has_permission(issuer=issuer, action='get_local_account_usage', kwargs=kwargs): raise rucio.common.exception.AccessDenied('Account %s can not list account usage.' % (issuer)) account = InternalAccount(account) if not account_exists(account=account): raise rucio.common.exception.AccountNotFound('Account %s does not exist' % (account)) return [api_update_return_dict(d) for d in account_limit_core.get_local_account_usage(account=account, rse_id=rse_id)]
def touch_dataset_locks(dataset_locks, session=None): """ Update the accessed_at timestamp of the given dataset locks + eol_at. :param replicas: the list of dataset locks. :param session: The database session in use. :returns: True, if successful, False otherwise. """ rse_ids, now = {}, datetime.utcnow() for dataset_lock in dataset_locks: if 'rse_id' not in dataset_lock: if dataset_lock['rse'] not in rse_ids: rse_ids[dataset_lock['rse']] = get_rse_id( rse=dataset_lock['rse'], session=session) dataset_lock['rse_id'] = rse_ids[dataset_lock['rse']] eol_at = define_eol(dataset_lock['scope'], dataset_lock['name'], rses=[{ 'id': dataset_lock['rse_id'] }], session=session) try: session.query(models.DatasetLock).filter_by(scope=dataset_lock['scope'], name=dataset_lock['name'], rse_id=dataset_lock['rse_id']).\ update({'accessed_at': dataset_lock.get('accessed_at') or now}, synchronize_session=False) session.query(models.ReplicationRule).filter_by( scope=dataset_lock['scope'], name=dataset_lock['name']).update({'eol_at': eol_at}, synchronize_session=False) except DatabaseError: return False return True
def add_quarantined_replicas(rse, replicas, session=None): """ Bulk add quarantined file replicas. :param rse: The rse name. :param replicas: A list of dicts with the replica information. :param session: The database session in use. """ rse_id = get_rse_id(rse, session=session) for chunk in chunks(replicas, 100): # Exlude files that have a registered replica. This is a # safeguard against potential issues in the Auditor. file_clause = [] for replica in chunk: file_clause.append( and_( models.RSEFileAssociation.scope == replica.get( 'scope', None), models.RSEFileAssociation.name == replica.get( 'name', None), models.RSEFileAssociation.rse_id == rse_id)) file_query = session.query(models.RSEFileAssociation.scope, models.RSEFileAssociation.name, models.RSEFileAssociation.rse_id).\ with_hint(models.RSEFileAssociation, "index(REPLICAS REPLICAS_PK)", 'oracle').\ filter(or_(*file_clause)) existing_replicas = [(scope, name, rseid) for scope, name, rseid in file_query] chunk = [ replica for replica in chunk if (replica.get('scope', None), replica.get('name', None), rse_id) not in existing_replicas ] # Exclude files that have already been added to the quarantined # replica table. quarantine_clause = [] for replica in chunk: quarantine_clause.append( and_(models.QuarantinedReplica.path == replica['path'], models.QuarantinedReplica.rse_id == rse_id)) quarantine_query = session.query(models.QuarantinedReplica.path, models.QuarantinedReplica.rse_id).\ filter(or_(*quarantine_clause)) quarantine_replicas = [(path, rseid) for path, rseid in quarantine_query] chunk = [ replica for replica in chunk if (replica['path'], rse_id) not in quarantine_replicas ] session.bulk_insert_mappings(models.QuarantinedReplica, [{ 'rse_id': rse_id, 'path': file['path'], 'scope': file.get('scope'), 'name': file.get('name'), 'bytes': file.get('bytes') } for file in chunk])
def delete_replicas(rse, files, issuer, ignore_availability=False): """ Bulk delete file replicas. :param rse: The RSE name. :param files: The list of files. :param issuer: The issuer account. :param ignore_availability: Ignore the RSE blacklisting. :returns: True is successful, False otherwise """ validate_schema(name='r_dids', obj=files) rse_id = get_rse_id(rse=rse) kwargs = {'rse': rse, 'rse_id': rse_id} if not permission.has_permission(issuer=issuer, action='delete_replicas', kwargs=kwargs): raise exception.AccessDenied('Account %s can not delete file replicas on %s' % (issuer, rse)) if not permission.has_permission(issuer=issuer, action='skip_availability_check', kwargs=kwargs): ignore_availability = False for f in files: f['scope'] = InternalScope(f['scope']) replica.delete_replicas(rse_id=rse_id, files=files, ignore_availability=ignore_availability)
def test_list_expired_dids_with_locked_rules(self): """ UNDERTAKER (CORE): Test that the undertaker does not list expired dids with locked rules""" tmp_scope = 'mock' # Add quota set_account_limit('jdoe', get_rse_id('MOCK'), -1) dsn = { 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{ 'account': 'jdoe', 'copies': 1, 'rse_expression': 'MOCK', 'locked': True, 'grouping': 'DATASET' }] } add_dids(dids=[dsn], account='root') for did in list_expired_dids(limit=1000): assert (did['scope'] != dsn['scope'] and did['name'] != dsn['name'])
def add_dids(dids, issuer, vo='def'): """ Bulk Add did. :param dids: A list of dids. :param issuer: The issuer account. :param vo: The VO to act on. """ for d in dids: if 'rse' in d: rse_id = None if d['rse'] is not None: rse_id = get_rse_id(rse=d['rse'], vo=vo) d['rse_id'] = rse_id kwargs = {'issuer': issuer, 'dids': dids} if not rucio.api.permission.has_permission( issuer=issuer, vo=vo, action='add_dids', kwargs=kwargs): raise rucio.common.exception.AccessDenied( 'Account %s can not bulk add data identifier' % (issuer)) issuer = InternalAccount(issuer, vo=vo) for d in dids: d['scope'] = InternalScope(d['scope'], vo=vo) if 'dids' in d.keys(): for child in d['dids']: child['scope'] = InternalScope(child['scope'], vo=vo) return did.add_dids(dids, account=issuer)
def attach_dids(scope, name, attachment, issuer): """ Append content to data did. :param attachment: The attachment. :param issuer: The issuer account. """ validate_schema(name='attachment', obj=attachment) rse_id = None if 'rse' in attachment: if attachment['rse'] is not None: rse_id = get_rse_id(rse=attachment['rse']) attachment['rse_id'] = rse_id kwargs = {'scope': scope, 'name': name, 'attachment': attachment} if not rucio.api.permission.has_permission(issuer=issuer, action='attach_dids', kwargs=kwargs): raise rucio.common.exception.AccessDenied('Account %s can not add data identifiers to %s:%s' % (issuer, scope, name)) scope = InternalScope(scope) issuer = InternalAccount(issuer) if 'account' in attachment.keys(): attachment['account'] = InternalAccount(attachment['account']) for d in attachment['dids']: d['scope'] = InternalScope(d['scope']) if rse_id is not None: dids = did.attach_dids(scope=scope, name=name, dids=attachment['dids'], account=attachment.get('account', issuer), rse_id=rse_id) else: dids = did.attach_dids(scope=scope, name=name, dids=attachment['dids'], account=attachment.get('account', issuer)) return dids
def update_replicas_states(rse, files, issuer): """ Update File replica information and state. :param rse: The RSE name. :param files: The list of files. :param issuer: The issuer account. """ validate_schema(name='dids', obj=files) rse_id = get_rse_id(rse=rse) kwargs = {'rse': rse, 'rse_id': rse_id} if not permission.has_permission( issuer=issuer, action='update_replicas_states', kwargs=kwargs): raise exception.AccessDenied( 'Account %s can not update file replicas state on %s' % (issuer, rse)) replicas = [] for file in files: rep = file rep['rse_id'] = rse_id rep['scope'] = InternalScope(rep['scope']) replicas.append(rep) replica.update_replicas_states(replicas=replicas)
def list_bad_replicas_status(state=BadFilesStatus.BAD, rse=None, younger_than=None, older_than=None, limit=None, list_pfns=False): """ List the bad file replicas history states. Method used by the rucio-ui. :param state: The state of the file (SUSPICIOUS or BAD). :param rse: The RSE name. :param younger_than: datetime object to select bad replicas younger than this date. :param older_than: datetime object to select bad replicas older than this date. :param limit: The maximum number of replicas returned. """ rse_id = None if rse is not None: rse_id = get_rse_id(rse=rse) replicas = replica.list_bad_replicas_status(state=state, rse_id=rse_id, younger_than=younger_than, older_than=older_than, limit=limit, list_pfns=list_pfns) return [api_update_return_dict(r) for r in replicas]
def setUp(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} else: self.vo = {} self.scope = InternalScope('mock', **self.vo) self.rse = 'MOCK4' self.rse2 = 'MOCK3' self.account = InternalAccount('root', **self.vo) self.rse_id = get_rse_id(self.rse, **self.vo) self.rse2_id = get_rse_id(self.rse2, **self.vo) self.db_session = session.get_session()
def test_list_datasets_per_rse(self): """ REPLICA (CLIENT): List datasets in RSE.""" rule_client = RuleClient() did_client = DIDClient() scope = 'mock' dataset = 'dataset_' + str(generate_uuid()) did_client.add_dataset(scope=scope, name=dataset) rule_client.add_replication_rule(dids=[{ 'scope': scope, 'name': dataset }], account='root', copies=1, rse_expression='MOCK', grouping='DATASET') replicas = [ r for r in list_datasets_per_rse( rse_id=get_rse_id(rse='MOCK', **self.vo), filters={ 'scope': InternalScope(scope, **self.vo), 'name': 'data*' }) ] assert replicas != []
def touch_replicas(replicas, session=None): """ Update the accessed_at timestamp of the given file replicas/dids. :param replicas: the list of replicas. :param session: The database session in use. :returns: True, if successful, False otherwise. """ rse_ids, now = {}, datetime.utcnow() for replica in replicas: if 'rse_id' not in replica: if replica['rse'] not in rse_ids: rse_ids[replica['rse']] = get_rse_id(rse=replica['rse'], session=session) replica['rse_id'] = rse_ids[replica['rse']] try: session.query(models.RSEFileAssociation).filter_by(rse_id=replica['rse_id'], scope=replica['scope'], name=replica['name']).\ update({'accessed_at': replica.get('accessed_at') or now}, synchronize_session=False) session.query(models.DataIdentifier).filter_by(scope=replica['scope'], name=replica['name'], did_type=DIDType.FILE).\ update({'accessed_at': replica.get('accessed_at') or now}, synchronize_session=False) except DatabaseError: return False # resolve_datasets = session.query(models.DataIdentifierAssociation).filter_by(child_scope=replica['scope'], child_name=replica['name'], did_type=DIDType.DATASET) # for dataset in resolve_datasets: # session.query(models.DataIdentifier).filter_by(scope=dataset['scope'], name=dataset['name'], did_type=DIDType.DATASET).\ # update({'accessed_at': replica.get('accessed_at') or now}, synchronize_session=False) # session.query(models.DatasetLock).filter_by(scope=dataset['scope'], name=dataset['name'], rse_id=replica['rse_id']).\ # update({'accessed_at': replica.get('accessed_at') or now}, synchronize_session=False) return True
def update_replicas_states(replicas, nowait=False, session=None): """ Update File replica information and state. :param replicas: The list of replicas. :param nowait: Nowait parameter for the for_update queries. :param session: The database session in use. """ rse_ids = {} for replica in replicas: if 'rse_id' not in replica: if replica['rse'] not in rse_ids: rse_ids[replica['rse']] = get_rse_id(rse=replica['rse'], session=session) replica['rse_id'] = rse_ids[replica['rse']] query = session.query(models.RSEFileAssociation).filter_by(rse_id=replica['rse_id'], scope=replica['scope'], name=replica['name']) if isinstance(replica['state'], str) or isinstance(replica['state'], unicode): replica['state'] = ReplicaState.from_string(replica['state']) if replica['state'] == ReplicaState.BEING_DELETED: query = query.filter_by(lock_cnt=0) if replica['state'] == ReplicaState.AVAILABLE: rucio.core.lock.successful_transfer(scope=replica['scope'], name=replica['name'], rse_id=replica['rse_id'], nowait=nowait, session=session) if 'path' in replica and replica['path']: rowcount = query.update({'state': replica['state'], 'path': replica['path']}, synchronize_session=False) else: rowcount = query.update({'state': replica['state']}, synchronize_session=False) if not rowcount: raise exception.UnsupportedOperation('State %(state)s for replica %(scope)s:%(name)s cannot be updated' % replica) return True
def attach_dids_to_dids(attachments, issuer, ignore_duplicate=False, vo='def'): """ Append content to dids. :param attachments: The contents. :param issuer: The issuer account. :param ignore_duplicate: If True, ignore duplicate entries. :param vo: The VO to act on. """ validate_schema(name='attachments', obj=attachments) for a in attachments: if 'rse' in a: rse_id = None if a['rse'] is not None: rse_id = get_rse_id(rse=a['rse'], vo=vo) a['rse_id'] = rse_id if not rucio.api.permission.has_permission( issuer=issuer, vo=vo, action='attach_dids_to_dids', kwargs={'attachments': attachments}): raise rucio.common.exception.AccessDenied( 'Account %s can not add data identifiers' % (issuer)) issuer = InternalAccount(issuer, vo=vo) for attachment in attachments: attachment['scope'] = InternalScope(attachment['scope'], vo=vo) for d in attachment['dids']: d['scope'] = InternalScope(d['scope'], vo=vo) return did.attach_dids_to_dids(attachments=attachments, account=issuer, ignore_duplicate=ignore_duplicate)
def get_request_by_did(scope, name, rse, rse_id=None, request_type=None, session=None): """ Retrieve a request by its DID for a destination RSE. :param scope: The scope of the data identifier. :param name: The name of the data identifier. :param rse: The destination RSE of the request. :param rse_id: The destination RSE ID of the request. Overrides rse param! :param request_type: The type of request as rucio.db.constants.RequestType. :param session: Database session to use. :returns: Request as a dictionary. """ record_counter('core.request.get_request_by_did') try: tmp = session.query(models.Request).filter_by(scope=scope, name=name) if rse_id: tmp = tmp.filter_by(dest_rse_id=rse_id) else: tmp = tmp.filter_by(dest_rse_id=get_rse_id(rse)) if request_type: tmp = tmp.filter_by(request_type=request_type) tmp = tmp.first() if not tmp: return else: tmp = dict(tmp) tmp.pop('_sa_instance_state') return tmp except IntegrityError, e: raise RucioException(e.args)
def on_message(self, frame): ''' on_message ''' record_counter('daemons.cache.consumer2.message') try: msg = json.loads(frame.body) if isinstance(msg, dict) and 'operation' in msg.keys(): for f in msg['files']: f['scope'] = InternalScope(f['scope']) if 'rse_id' in msg: rse_id = msg['rse_id'] else: rse_id = get_rse_id(rse=msg['rse'], vo=msg.get('vo', 'def')) rse_vo_str = msg['rse'] if 'vo' in msg and msg['vo'] != 'def': rse_vo_str = '{} on {}'.format(rse_vo_str, msg['vo']) if msg['operation'] == 'add_replicas': logging.info('add_replicas to RSE %s: %s ' % (rse_vo_str, str(msg['files']))) add_volatile_replicas(rse_id=rse_id, replicas=msg['files']) elif msg['operation'] == 'delete_replicas': logging.info('delete_replicas to RSE %s: %s ' % (rse_vo_str, str(msg['files']))) delete_volatile_replicas(rse_id=rse_id, replicas=msg['files']) except: logging.error(str(format_exc()))
def test_inc_dec_get_counter(self): """ACCOUNT COUNTER (CORE): Increase, decrease and get counter """ account_update(once=True) rse_id = get_rse_id(rse='MOCK', **self.vo) account = InternalAccount('jdoe', **self.vo) account_counter.del_counter(rse_id=rse_id, account=account) account_counter.add_counter(rse_id=rse_id, account=account) cnt = get_usage(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': 0, 'bytes': 0}) count, sum = 0, 0 for i in range(10): account_counter.increase(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count += 1 sum += 2.147e+9 cnt = get_usage(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum}) for i in range(4): account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count -= 1 sum -= 2.147e+9 cnt = get_usage(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum}) for i in range(5): account_counter.increase(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count += 1 sum += 2.147e+9 cnt = get_usage(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum}) for i in range(8): account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count -= 1 sum -= 2.147e+9 cnt = get_usage(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum})
def list_rebalance_rule_candidates(rse, mode=None, session=None): """ List the rebalance rule candidates based on the agreed on specification :param rse: RSE of the source. :param mode: Rebalancing mode. :param session: DB Session. """ rse_id = get_rse_id(rse) # dumps can be applied only for decommission since the dumps doesn't contain info from dids if mode == 'decommission': return _list_rebalance_rule_candidates_dump(rse, mode) # the rest is done with sql query from_date = datetime.utcnow() + timedelta(days=60) to_date = datetime.now() - timedelta(days=60) allowed_accounts = [ InternalAccount(a) for a in ('panda', 'root', 'ddmadmin') ] allowed_grouping = [RuleGrouping.DATASET, RuleGrouping.ALL] external_dsl = aliased(models.DatasetLock) count_locks = select([func.count()]).where( and_(external_dsl.scope == models.DatasetLock.scope, external_dsl.name == models.DatasetLock.name, external_dsl.rse_id == models.DatasetLock.rse_id)).as_scalar() query = session.query(models.DatasetLock.scope, models.DatasetLock.name, models.ReplicationRule.id, models.ReplicationRule.rse_expression, models.ReplicationRule.subscription_id, models.DataIdentifier.bytes, models.DataIdentifier.length, case([(or_(models.DatasetLock.length < 1, models.DatasetLock.length.is_(None)), 0)], else_=cast(models.DatasetLock.bytes / models.DatasetLock.length, Integer))).\ join(models.ReplicationRule, models.ReplicationRule.id == models.DatasetLock.rule_id).\ join(models.DataIdentifier, and_(models.DatasetLock.scope == models.DataIdentifier.scope, models.DatasetLock.name == models.DataIdentifier.name)).\ filter(models.DatasetLock.rse_id == rse_id).\ filter(or_(models.ReplicationRule.expires_at > from_date, models.ReplicationRule.expires_at.is_(None))).\ filter(models.ReplicationRule.created_at < to_date).\ filter(models.ReplicationRule.account.in_(allowed_accounts)).\ filter(models.ReplicationRule.state == RuleState.OK).\ filter(models.ReplicationRule.did_type == DIDType.DATASET).\ filter(models.ReplicationRule.copies == 1).\ filter(models.ReplicationRule.child_rule_id.is_(None)).\ filter(models.ReplicationRule.grouping.in_(allowed_grouping)).\ filter(models.DataIdentifier.bytes.isnot(None)).\ filter(models.DataIdentifier.is_open == 0).\ filter(models.DataIdentifier.did_type == DIDType.DATASET).\ filter(case([(or_(models.DatasetLock.length < 1, models.DatasetLock.length.is_(None)), 0)], else_=cast(models.DatasetLock.bytes / models.DatasetLock.length, Integer)) > 1000000000).\ filter(count_locks == 1) summary = query.order_by( case([(or_(models.DatasetLock.length < 1, models.DatasetLock.length.is_(None)), 0)], else_=cast(models.DatasetLock.bytes / models.DatasetLock.length, Integer)), models.DatasetLock.accessed_at).all() return summary
def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': 'tst'} else: cls.vo = {} cls.db_session = session.get_session() cls.dest_rse = 'MOCK' cls.source_rse = 'MOCK4' cls.dest_rse_id = get_rse_id(cls.dest_rse, **cls.vo) cls.source_rse_id = get_rse_id(cls.source_rse, **cls.vo) cls.scope = InternalScope('mock', **cls.vo) cls.account = InternalAccount('root', **cls.vo) cls.user_activity = 'User Subscription' cls.all_activities = 'all_activities' set_rse_transfer_limits(cls.dest_rse_id, cls.user_activity, max_transfers=1, session=cls.db_session) set('throttler_release_strategy', 'dest_%s' % cls.dest_rse_id, 'fifo', session=cls.db_session)
def setUp(self): self.account = 'root' self.scope = 'mock' self.upload_client = UploadClient() self.file_sizes = 2 self.rse = 'MOCK4' self.rse_id = get_rse_id(self.rse) self.session = get_session()
def setUpClass(cls): cls.upload_client = UploadClient() cls.session = get_session() if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} cls.rse_id = get_rse_id(cls.rse, session=cls.session, **cls.vo)
def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': get_vo()} else: cls.vo = {} cls.account = InternalAccount('jdoe', **cls.vo) cls.rse_1_name = 'MOCK4' cls.rse_2_name = 'MOCK5' cls.mock1_id = get_rse_id(cls.rse_1_name, **cls.vo) cls.mock2_id = get_rse_id(cls.rse_2_name, **cls.vo) cls.db_session = session.get_session() cls.rse_1 = {'id': cls.mock1_id, 'staging_area': False} cls.rse_2 = {'id': cls.mock2_id, 'staging_area': False}
def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': get_vo()} else: cls.vo = {} # Add test RSE cls.rse1 = 'MOCK' cls.rse3 = 'MOCK3' cls.rse4 = 'MOCK4' cls.rse5 = 'MOCK5' cls.rse1_id = get_rse_id(rse=cls.rse1, **cls.vo) cls.rse3_id = get_rse_id(rse=cls.rse3, **cls.vo) cls.rse4_id = get_rse_id(rse=cls.rse4, **cls.vo) cls.rse5_id = get_rse_id(rse=cls.rse5, **cls.vo) # Add Tags cls.T1 = tag_generator() cls.T2 = tag_generator() add_rse_attribute(cls.rse1_id, cls.T1, True) add_rse_attribute(cls.rse3_id, cls.T1, True) add_rse_attribute(cls.rse4_id, cls.T2, True) add_rse_attribute(cls.rse5_id, cls.T1, True) # Add fake weights add_rse_attribute(cls.rse1_id, "fakeweight", 10) add_rse_attribute(cls.rse3_id, "fakeweight", 0) add_rse_attribute(cls.rse4_id, "fakeweight", 0) add_rse_attribute(cls.rse5_id, "fakeweight", 0) # Add quota cls.jdoe = InternalAccount('jdoe', **cls.vo) cls.root = InternalAccount('root', **cls.vo) set_local_account_limit(cls.jdoe, cls.rse1_id, -1) set_local_account_limit(cls.jdoe, cls.rse3_id, -1) set_local_account_limit(cls.jdoe, cls.rse4_id, -1) set_local_account_limit(cls.jdoe, cls.rse5_id, -1) set_local_account_limit(cls.root, cls.rse1_id, -1) set_local_account_limit(cls.root, cls.rse3_id, -1) set_local_account_limit(cls.root, cls.rse4_id, -1) set_local_account_limit(cls.root, cls.rse5_id, -1)
def tmp_rse_info(rse=None, vo='def', rse_id=None, session=None): if rse_id is None: # This can be called directly by client tools if they're co-located on a server # i.e. running rucio cli on a server and during the test suite. # We have to map to VO name here for this situations, despite this nominally # not being a client interface. rse_id = get_rse_id(rse=rse, vo=map_vo(vo)) return get_rse_protocols(rse_id=rse_id, session=session)
def list_quarantined_replicas(rse, limit, worker_number=None, total_workers=None, session=None): """ List RSE Quarantined File replicas. :param rse: the rse name. :param limit: The maximum number of replicas returned. :param worker_number: id of the executing worker. :param total_workers: Number of total workers. :param session: The database session in use. :returns: a list of dictionary replica. """ rse_id = get_rse_id(rse, session=session) query = session.query(models.QuarantinedReplica.path, models.QuarantinedReplica.bytes, models.QuarantinedReplica.scope, models.QuarantinedReplica.name, models.QuarantinedReplica.created_at).\ filter(models.QuarantinedReplica.rse_id == rse_id) # do no delete valid replicas stmt = exists(select([1]).prefix_with("/*+ index(REPLICAS REPLICAS_PK) */", dialect='oracle')).\ where(and_(models.RSEFileAssociation.scope == models.QuarantinedReplica.scope, models.RSEFileAssociation.name == models.QuarantinedReplica.name, models.RSEFileAssociation.rse_id == models.QuarantinedReplica.rse_id)) query = query.filter(not_(stmt)) if worker_number and total_workers and total_workers - 1 > 0: if session.bind.dialect.name == 'oracle': bindparams = [ bindparam('worker_number', worker_number - 1), bindparam('total_workers', total_workers - 1) ] query = query.filter( text('ORA_HASH(path, :total_workers) = :worker_number', bindparams=bindparams)) elif session.bind.dialect.name == 'mysql': query = query.filter('mod(md5(path), %s) = %s' % (total_workers - 1, worker_number - 1)) elif session.bind.dialect.name == 'postgresql': query = query.filter( 'mod(abs((\'x\'||md5(path))::bit(32)::int), %s) = %s' % (total_workers - 1, worker_number - 1)) return [{ 'path': path, 'rse': rse, 'rse_id': rse_id, 'created_at': created_at, 'scope': scope, 'name': name, 'bytes': bytes } for path, bytes, scope, name, created_at in query.limit(limit)]
def add_did(scope, name, did_type, issuer, account=None, statuses={}, meta={}, rules=[], lifetime=None, dids=[], rse=None, vo='def', session=None): """ Add data did. :param scope: The scope name. :param name: The data identifier name. :param did_type: The data identifier type. :param issuer: The issuer account. :param account: The account owner. If None, then issuer is selected as owner. :param statuses: Dictionary with statuses, e.g.g {'monotonic':True}. :meta: Meta-data associated with the data identifier is represented using key/value pairs in a dictionary. :rules: Replication rules associated with the data did. A list of dictionaries, e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ]. :param lifetime: DID's lifetime (in seconds). :param dids: The content. :param rse: The RSE name when registering replicas. :param vo: The VO to act on. :param session: The database session in use. """ v_did = {'name': name, 'type': did_type.upper(), 'scope': scope} validate_schema(name='did', obj=v_did, vo=vo) validate_schema(name='dids', obj=dids, vo=vo) validate_schema(name='rse', obj=rse, vo=vo) kwargs = {'scope': scope, 'name': name, 'type': did_type, 'issuer': issuer, 'account': account, 'statuses': statuses, 'meta': meta, 'rules': rules, 'lifetime': lifetime} if not rucio.api.permission.has_permission(issuer=issuer, vo=vo, action='add_did', kwargs=kwargs, session=session): raise rucio.common.exception.AccessDenied('Account %s can not add data identifier to scope %s' % (issuer, scope)) if account is not None: account = InternalAccount(account, vo=vo) issuer = InternalAccount(issuer, vo=vo) scope = InternalScope(scope, vo=vo) for d in dids: d['scope'] = InternalScope(d['scope'], vo=vo) for r in rules: r['account'] = InternalAccount(r['account'], vo=vo) rse_id = None if rse is not None: rse_id = get_rse_id(rse=rse, vo=vo, session=session) if did_type == 'DATASET': # naming_convention validation extra_meta = naming_convention.validate_name(scope=scope, name=name, did_type='D', session=session) # merge extra_meta with meta for k in extra_meta or {}: if k not in meta: meta[k] = extra_meta[k] elif meta[k] != extra_meta[k]: print("Provided metadata %s doesn't match the naming convention: %s != %s" % (k, meta[k], extra_meta[k])) raise rucio.common.exception.InvalidObject("Provided metadata %s doesn't match the naming convention: %s != %s" % (k, meta[k], extra_meta[k])) # Validate metadata meta_core.validate_meta(meta=meta, did_type=DIDType[did_type.upper()], session=session) return did.add_did(scope=scope, name=name, did_type=DIDType[did_type.upper()], account=account or issuer, statuses=statuses, meta=meta, rules=rules, lifetime=lifetime, dids=dids, rse_id=rse_id, session=session)
def get_signed_url_server(rse, service, op, url, vo='def'): ''' get_signed_url_server ''' from rucio.core.rse import get_rse_id from rucio.core.credential import get_signed_url rse_id = get_rse_id(rse=rse, vo=vo) return get_signed_url(rse_id, service, op, url)
def setUpClass(cls): cls.db_session = session.get_session() cls.dest_rse = 'MOCK' cls.source_rse = 'MOCK4' cls.dest_rse_id = get_rse_id(cls.dest_rse) cls.source_rse_id = get_rse_id(cls.source_rse) cls.scope = 'mock' cls.account = 'root' cls.user_activity = 'User Subscription' cls.all_activities = 'all_activities' set_rse_transfer_limits(cls.dest_rse, cls.user_activity, max_transfers=1, session=cls.db_session) set('throttler_release_strategy', 'dest_%s' % cls.dest_rse_id, 'fifo', session=cls.db_session)
def list_requests(src_rses, dst_rses, states, issuer): """ List all requests in a specific state from a source RSE to a destination RSE. :param src_rses: source RSEs. :param dst_rses: destination RSEs. :param states: list of request states. :param issuer: Issuing account as a string. """ src_rse_ids = [get_rse_id(rse=rse) for rse in src_rses] dst_rse_ids = [get_rse_id(rse=rse) for rse in dst_rses] kwargs = {'src_rse_id': src_rse_ids, 'dst_rse_id': dst_rse_ids, 'issuer': issuer} if not permission.has_permission(issuer=issuer, action='list_requests', kwargs=kwargs): raise exception.AccessDenied('%(issuer)s cannot list requests from RSE %(src_rse)s to RSE %(dst_rse)s' % locals()) for req in request.list_requests(src_rse_ids, dst_rse_ids, states): yield api_update_return_dict(req)
def get_distance(source, destination, issuer, vo='def', session=None): """ Get distances between rses. :param source: The source RSE. :param destination: The destination RSE. :param issuer: The issuer account. :param vo: The VO to act on. :param session: The database session in use. :returns distance: List of dictionaries. """ distances = distance_module.get_distances( src_rse_id=rse_module.get_rse_id(source, vo=vo, session=session), dest_rse_id=rse_module.get_rse_id(destination, vo=vo, session=session), session=session) return [api_update_return_dict(d, session=session) for d in distances]
def setUp(self): self.account = InternalAccount('root') self.scope = InternalScope('mock') self.upload_client = UploadClient() self.account_client = AccountClient() self.file_sizes = 2 self.rse = 'MOCK4' self.rse_id = get_rse_id(self.rse) self.session = get_session()
def add_distance(source, destination, issuer, vo='def', ranking=None, distance=None, geoip_distance=None, active=None, submitted=None, finished=None, failed=None, transfer_speed=None): """ Add a src-dest distance. :param source: The source. :param destination: The destination. :param issuer: The issuer account. :param vo: The VO to act on. :param ranking: Ranking as an integer. :param distance: Distance as an integer. :param geoip_distance: GEOIP Distance as an integer. :param active: Active FTS transfers as an integer. :param submitted: Submitted FTS transfers as an integer. :param finished: Finished FTS transfers as an integer. :param failed: Failed FTS transfers as an integer. :param transfer_speed: FTS transfer speed as an integer. """ kwargs = {'source': source, 'destination': destination} if not permission.has_permission( issuer=issuer, vo=vo, action='add_distance', kwargs=kwargs): raise exception.AccessDenied('Account %s can not add RSE distances' % (issuer)) return distance_module.add_distance( src_rse_id=rse_module.get_rse_id(source, vo=vo), dest_rse_id=rse_module.get_rse_id(destination, vo=vo), ranking=ranking, agis_distance=distance, geoip_distance=geoip_distance, active=active, submitted=submitted, finished=finished, failed=failed, transfer_speed=transfer_speed)
def get_dataset_locks_by_rse(rse): """ Get the dataset locks of an RSE. :param rse: RSE name. :return: List of dicts {'rse_id': ..., 'state': ...} """ rse_id = get_rse_id(rse=rse) return lock.get_dataset_locks_by_rse_id(rse_id=rse_id)
def get_replica_atime(replica, session=None): """ Get the accessed_at timestamp for a replica. Just for testing. :param replicas: List of dictionaries {scope, name, rse_id, path} :param session: Database session to use. :returns: A datetime timestamp with the last access time. """ if 'rse_id' not in replica: replica['rse_id'] = get_rse_id(rse=replica['rse'], session=session) return session.query(models.RSEFileAssociation.accessed_at).filter_by(scope=replica['scope'], name=replica['name'], rse_id=replica['rse_id']).\ with_hint(models.RSEFileAssociation, text="INDEX(REPLICAS REPLICAS_PK)", dialect_name='oracle').one()[0]
def get_account_limit(account, rse): """ Lists the limitation names/values for the specified account name and rse name. REST API: http://<host>:<port>/rucio/account/<account>/limits :param account: The account name. :param rse: The rse name. :returns: The account limit. """ rse_id = get_rse_id(rse=rse) return {rse: account_limit_core.get_account_limit(account=account, rse_id=rse_id)}
def list_unlocked_replicas(rse, limit, bytes=None, rse_id=None, worker_number=None, total_workers=None, delay_seconds=0, session=None): """ List RSE File replicas with no locks. :param rse: the rse name. :param bytes: the amount of needed bytes. :param session: The database session in use. :returns: a list of dictionary replica. """ if not rse_id: rse_id = get_rse_id(rse=rse, session=session) # filter(models.RSEFileAssociation.state != ReplicaState.BEING_DELETED).\ none_value = None # Hack to get pep8 happy... query = session.query(models.RSEFileAssociation.scope, models.RSEFileAssociation.name, models.RSEFileAssociation.bytes, models.RSEFileAssociation.tombstone).\ filter(models.RSEFileAssociation.tombstone < datetime.utcnow()).\ filter(models.RSEFileAssociation.lock_cnt == 0).\ filter(case([(models.RSEFileAssociation.tombstone != none_value, models.RSEFileAssociation.rse_id), ]) == rse_id).\ filter(or_(models.RSEFileAssociation.state.in_((ReplicaState.AVAILABLE, ReplicaState.UNAVAILABLE)), and_(models.RSEFileAssociation.state == ReplicaState.BEING_DELETED, models.RSEFileAssociation.updated_at < datetime.utcnow() - timedelta(seconds=delay_seconds)))).\ order_by(models.RSEFileAssociation.tombstone).\ with_hint(models.RSEFileAssociation, "INDEX(replicas REPLICAS_TOMBSTONE_IDX)", 'oracle') if worker_number and total_workers and total_workers - 1 > 0: if session.bind.dialect.name == 'oracle': bindparams = [bindparam('worker_number', worker_number - 1), bindparam('total_workers', total_workers - 1)] query = query.filter(text('ORA_HASH(name, :total_workers) = :worker_number', bindparams=bindparams)) elif session.bind.dialect.name == 'mysql': query = query.filter('mod(md5(name), %s) = %s' % (total_workers - 1, worker_number - 1)) elif session.bind.dialect.name == 'postgresql': query = query.filter('mod(abs((\'x\'||md5(name))::bit(32)::int), %s) = %s' % (total_workers - 1, worker_number - 1)) query = query.limit(limit) rows = list() neededSpace = bytes totalbytes = 0 for (scope, name, bytes, tombstone) in query.yield_per(1000): if tombstone != OBSOLETE and neededSpace is not None and totalbytes >= neededSpace: break d = {'scope': scope, 'name': name, 'bytes': bytes} rows.append(d) if tombstone != OBSOLETE: totalbytes += bytes return rows
def set_account_limit(account, rse, bytes, issuer): """ Set an account limit.. :param account: The account name. :param rse: The rse name. :param bytes: The limit in bytes. :param issuer: The issuer account_core. """ kwargs = {'account': account, 'rse': rse, 'bytes': bytes} if not rucio.api.permission.has_permission(issuer=issuer, action='set_account_limit', kwargs=kwargs): raise rucio.common.exception.AccessDenied('Account %s can not set account limits.' % (issuer)) if not account_exists(account=account): raise rucio.common.exception.AccountNotFound('Account %s does not exist' % (account)) rse_id = get_rse_id(rse=rse) account_limit_core.set_account_limit(account=account, rse_id=rse_id, bytes=bytes)
def delete_account_limit(account, rse, issuer): """ Delete an account limit.. :param account: The account name. :param rse: The rse name. :param issuer: The issuer account_core. :returns: True if successful; False otherwise. """ kwargs = {'account': account, 'rse': rse} if not rucio.api.permission.has_permission(issuer=issuer, action='delete_account_limit', kwargs=kwargs): raise rucio.common.exception.AccessDenied('Account %s can not delete account limits.' % (issuer)) if not account_exists(account=account): raise rucio.common.exception.AccountNotFound('Account %s does not exist' % (account)) rse_id = get_rse_id(rse=rse) return account_limit_core.delete_account_limit(account=account, rse_id=rse_id)
def get_replica(rse, scope, name, rse_id=None, session=None): """ Get File replica. :param rse: the rse name. :param scope: the scope name. :param name: The data identifier name. :param rse_id: The RSE Id. :param session: The database session in use. :returns: A dictionary with the list of replica attributes. """ if not rse_id: rse_id = get_rse_id(rse=rse, session=session) row = session.query(models.RSEFileAssociation).filter_by(rse_id=rse_id, scope=scope, name=name).one() d = {} for column in row.__table__.columns: d[column.name] = getattr(row, column.name) return d
def get_account_usage(account, rse, issuer): """ Get the account usage and connect it with (if available) the account limits of the account. :param account: The account to read. :param rse: The rse to read (If none, get all). :param issuer: The issuer account. :returns: List of dicts {'rse_id', 'bytes_used', 'files_used', 'bytes_limit'} """ kwargs = {'account': account, 'rse': rse} if not rucio.api.permission.has_permission(issuer=issuer, action='get_account_usage', kwargs=kwargs): raise rucio.common.exception.AccessDenied('Account %s can not list account usage.' % (issuer)) if not account_exists(account=account): raise rucio.common.exception.AccountNotFound('Account %s does not exist' % (account)) rse_id = None if rse: rse_id = get_rse_id(rse=rse) return account_limit_core.get_account_usage(account=account, rse_id=rse_id)
def update_replica_lock_counter(rse, scope, name, value, rse_id=None, session=None): """ Update File replica lock counters. :param rse: the rse name. :param scope: the tag name. :param name: The data identifier name. :param value: The number of created/deleted locks. :param rse_id: The id of the RSE. :param session: The database session in use. :returns: True or False. """ if not rse_id: rse_id = get_rse_id(rse=rse, session=session) # WTF BUG in the mysql-driver: lock_cnt uses the already updated value! ACID? Never heard of it! if session.bind.dialect.name == 'mysql': rowcount = session.query(models.RSEFileAssociation).\ filter_by(rse_id=rse_id, scope=scope, name=name).\ update({'lock_cnt': models.RSEFileAssociation.lock_cnt + value, 'tombstone': case([(models.RSEFileAssociation.lock_cnt + value < 0, datetime.utcnow()), ], else_=None)}, synchronize_session=False) else: rowcount = session.query(models.RSEFileAssociation).\ filter_by(rse_id=rse_id, scope=scope, name=name).\ update({'lock_cnt': models.RSEFileAssociation.lock_cnt + value, 'tombstone': case([(models.RSEFileAssociation.lock_cnt + value == 0, datetime.utcnow()), ], else_=None)}, synchronize_session=False) return bool(rowcount)
'scope': response['scope'], 'name': response['name'], 'state': ReplicaState.UNAVAILABLE}], session=session) except: logging.critical("Could not update replica state for failed transfer %s:%s at %s (%s)" % (response['scope'], response['name'], rse_update_name, traceback.format_exc())) raise tss = time.time() try: lock.failed_transfer(response['scope'], response['name'], rse_core.get_rse_id(rse=rse_update_name, session=session), session=session) except: logging.warn('Could not update lock for failed transfer %s:%s at %s (%s)' % (response['scope'], response['name'], rse_update_name, traceback.format_exc())) raise record_timer('daemons.conveyor.common.update_request_state.lock-failed_transfer', (time.time()-tss)*1000) else: logging.warn('REQUEUED DID %s:%s REQUEST %s AS %s TRY %s' % (response['scope'], response['name'], response['request_id'], new_req['request_id'], new_req['retry_count']))